def __init__(self, dbName, stockList, features, start, end, batch_size=50, volume_average_days=30, buffer_bias_ratio=0, window_size=50, test_portion=0.15, portion_reversed=False, is_permed=False): """ :param start: Unix time :param end: Unix time :param access_period: the data access period of the input matrix. :param trade_period: the trading period of the agent. :param global_period: the data access period of the global price matrix. if it is not equal to the access period, there will be inserted observations :param coin_filter: number of coins that would be selected :param window_size: periods of input data :param train_portion: portion of training set :param is_permed: if False, the sample inside a mini-batch is in order :param validation_portion: portion of cross-validation set :param test_portion: portion of test set :param portion_reversed: if False, the order to sets are [train, validation, test] else the order is [test, validation, train] """ self.dbName = dbName self.stockList = stockList self.__stock_number = len(stockList) self.__features = features self.feature_number = len(features) self.start = start self.end = end self.__history_manager = gdm.HistoryManager(dbName, stockList) self.__global_data, self.dateSet = self.__history_manager.get_global_data_matrix_from_mongodb(start, end, features) # portfolio vector memory, [time, assets] self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis, columns=self.__global_data.major_axis) #(2833, 11) portfolio vector memory #print ('=============================>',self.__PVM.shape) #print (self.__PVM.iloc[0,:]) self.__PVM = self.__PVM.fillna(1.0 / self.__stock_number) #初始化权重信息,权重都是相等的 self._window_size = window_size #窗口大小 31 self._num_periods = len(self.__global_data.minor_axis) #print (self._num_periods) #数据个数 self.__divide_data(test_portion, portion_reversed) #训练集和测试集的index self._portion_reversed = portion_reversed self.__is_permed = is_permed self.__batch_size = batch_size self.__delta = 0 # the count of global increased self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0], end_index=self._train_ind[-1], sample_bias=buffer_bias_ratio, batch_size=self.__batch_size, stock_number=self.__stock_number, is_permed=self.__is_permed) logging.info("the number of training examples is %s" ", of test examples is %s" % (self._num_train_samples, self._num_test_samples)) logging.debug("the training set is from %s to %s" % (min(self._train_ind), max(self._train_ind))) logging.debug("the test set is from %s to %s" % (min(self._test_ind), max(self._test_ind)))
def __init__(self, batch_size=60, buffer_bias_ratio=0, coin_filter=1, window_size=50, feature_number=3, portion_reversed=False, online=False, is_permed=False): """ :param coin_filter: number of coins that would be selected :param window_size: periods of input data :param portion_reversed: if False, the order to sets are [train, validation, test] else the order is [test, validation, train] """ self.__coin_no = coin_filter type_list = get_type_list(feature_number) self.__features = type_list self.feature_number = feature_number self.__history_manager = gdm.HistoryManager(coin_number=coin_filter, online=online) self.__global_data = self.__history_manager.get_global_panel( features=type_list) # self.__global_market_capticalization = self.__history_manager.get_market_capticalization() # self.__global_all_market_capticalization = self.__history_manager.get_all_market_capticalization() self.__stockIndex_data = self.__history_manager.get_stockIndex_panel() self.stock_code = self.__history_manager.stock_code # portfolio vector memory, [time, assets] self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis, columns=self.__global_data.major_axis) self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no) # print(self.__PVM) self._window_size = window_size self._num_periods = len(self.__global_data.minor_axis) self.__batch_size = batch_size self.__divide_data() self._portion_reversed = portion_reversed self.__is_permed = is_permed self.__delta = 0 # the count of global increased end_index = self._train_ind[-1] self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0], end_index=end_index, sample_bias=buffer_bias_ratio, batch_size=self.__batch_size, coin_number=self.__coin_no, is_permed=self.__is_permed) logging.info("the number of training examples is %s" ", of test examples is %s" % (self._num_train_samples, self._num_test_samples)) logging.debug("the training set is from %s to %s" % (min(self._train_ind), max(self._train_ind))) logging.debug("the test set is from %s to %s" % (min(self._test_ind), max(self._test_ind)))
def __init__(self, start, end, period, batch_size=50, volume_average_days=30, buffer_bias_ratio=0, market="poloniex", asset_filter=1, window_size=50, feature_number=3, test_portion=0.15, portion_reversed=False, online=False, is_permed=False): """ :param start: Unix time :param end: Unix time :param access_period: the data access period of the input matrix. :param trade_period: the trading period of the agent. :param global_period: the data access period of the global price matrix. if it is not equal to the access period, there will be inserted observations :param asset_filter: number of assets that would be selected :param window_size: periods of input data :param train_portion: portion of training set :param is_permed: if False, the sample inside a mini-batch is in order :param validation_portion: portion of cross-validation set :param test_portion: portion of test set :param portion_reversed: if False, the order to sets are [train, validation, test] else the order is [test, validation, train] """ start = int(start) self.__end = int(end) # assert window_size >= MIN_NUM_PERIOD self.__asset_no = asset_filter type_list = get_type_list(feature_number) self.__features = type_list self.feature_number = feature_number volume_forward = get_volume_forward(self.__end - start, test_portion, portion_reversed) self.__history_manager = gdm.HistoryManager( asset_number=asset_filter, end=self.__end, volume_average_days=volume_average_days, volume_forward=volume_forward) if market == "poloniex": # poloniex == bitcoin market. self.__global_data = self.__history_manager.get_global_panel( start, self.__end, period=period, features=type_list) else: raise ValueError("market {} is not valid".format(market)) self.__period_length = period # portfolio vector memory, [time, assets] self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis, columns=self.__global_data.major_axis) self.__PVM = self.__PVM.fillna(1.0 / self.__asset_no) self._window_size = window_size self._num_periods = len(self.__global_data.minor_axis) self.__divide_data(test_portion, portion_reversed) self._portion_reversed = portion_reversed self.__is_permed = is_permed self.__batch_size = batch_size self.__delta = 0 # the count of global increased end_index = self._train_ind[-1] self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0], end_index=end_index, sample_bias=buffer_bias_ratio, batch_size=self.__batch_size, asset_number=self.__asset_no, is_permed=self.__is_permed) logging.info("the number of training examples is %s" ", of test examples is %s" % (self._num_train_samples, self._num_test_samples)) logging.debug("the training set is from %s to %s" % (min(self._train_ind), max(self._train_ind))) logging.debug("the test set is from %s to %s" % (min(self._test_ind), max(self._test_ind)))
def __init__(self, start, end, period, batch_size=50, buffer_bias_ratio=0, coin_filter=1, window_size=50, feature_number=3, test_portion=0.15, portion_reversed=False, is_permed=False): """ :param start: Unix time :param end: Unix time :param coin_filter: number of coins that would be selected :param window_size: periods of input data :param train_portion: portion of training set :param is_permed: if False, the sample inside a mini-batch is in order :param validation_portion: portion of cross-validation set :param test_portion: portion of test set :param portion_reversed: if False, the order to sets are [train, validation, test] else the order is [test, validation, train] """ start = int(start) self.__end = int(end) self.__coin_no = coin_filter type_list = get_type_list(feature_number) self.__features = type_list self.feature_number = feature_number self.__history_manager = gdm.HistoryManager(coin_number=coin_filter, end=self.__end) self.__global_data = self.__history_manager.get_global_panel( start, self.__end, period=period, features=type_list) self.__period_length = period # portfolio vector memory, [time, assets] self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis, columns=self.__global_data.major_axis) self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no) self._window_size = window_size self._num_periods = len(self.__global_data.minor_axis) self.__divide_data(test_portion, portion_reversed) self._portion_reversed = portion_reversed self.__is_permed = is_permed self.__batch_size = batch_size self.__delta = 0 # the count of global increased end_index = self._train_ind[-1] self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0], end_index=end_index, sample_bias=buffer_bias_ratio, batch_size=self.__batch_size, coin_number=self.__coin_no, is_permed=self.__is_permed) print("the number of training examples is %s" ", of test examples is %s" % (self._num_train_samples, self._num_test_samples)) print("the training set is from %s to %s" % (min(self._train_ind), max(self._train_ind))) print("the test set is from %s to %s" % (min(self._test_ind), max(self._test_ind)))
def __init__(self, start, end, period=86400, batch_size=50, volume_average_days=30, buffer_bias_ratio=0, market="poloniex", coin_filter=1, window_size=50, feature_number=3, test_portion=0.15, portion_reversed=False, online=False, is_permed=False): """ :param start: Unix time :param end: Unix time :param access_period: the data access period of the input matrix. :param trade_period: the trading period of the agent. :param period、global_period: the data access period of the global price matrix. if it is not equal to the access period, there will be inserted observations :param coin_filter: number of coins that would be selected :param window_size: periods of input data :param train_portion: portion of training set :param is_permed: if False, the sample inside a mini-batch is in order :param validation_portion: portion of cross-validation set :param test_portion: portion of test set :param portion_reversed: if False, the order to sets are [train, validation, test] else the order is [test, validation, train] """ start = int(start) self.__end = int(end) self.__coin_no = coin_filter # the number of coins type_list = get_type_list(feature_number) self.__features = type_list self.feature_number = feature_number # the number of features volume_forward = get_volume_forward(self.__end - start, test_portion, portion_reversed) self.__history_manager = gdm.HistoryManager( coin_number=coin_filter, end=self.__end, volume_average_days=volume_average_days, volume_forward=volume_forward, online=online) self.__global_data = self.__history_manager.get_global_panel( start, self.__end, period=period, features=type_list) self.__period_length = period self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis, columns=self.__global_data.major_axis) # #print("self.__global_data.minor_axis",self.__global_data.minor_axis) """ for example self.__global_data.minor_axis DatetimeIndex(['2015-12-31 16:00:00', '2015-12-31 16:30:00', '2015-12-31 17:00:00', '2015-12-31 17:30:00', '2015-12-31 18:00:00', '2015-12-31 18:30:00', '2015-12-31 19:00:00', '2015-12-31 19:30:00', '2015-12-31 20:00:00', '2015-12-31 20:30:00', ... '2017-12-31 11:30:00', '2017-12-31 12:00:00', '2017-12-31 12:30:00', '2017-12-31 13:00:00', '2017-12-31 13:30:00', '2017-12-31 14:00:00', '2017-12-31 14:30:00', '2017-12-31 15:00:00', '2017-12-31 15:30:00', '2017-12-31 16:00:00'], dtype='datetime64[ns]', length=35089, freq=None) """ self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no) self._window_size = window_size self._num_periods = len(self.__global_data.minor_axis) self.__divide_data(test_portion, portion_reversed) self._portion_reversed = portion_reversed self.__is_permed = is_permed self.__batch_size = batch_size self.__delta = 0 # the count of global increased end_index = self._train_ind[-1] self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0], end_index=end_index, sample_bias=buffer_bias_ratio, batch_size=self.__batch_size, coin_number=self.__coin_no, is_permed=self.__is_permed) # experience replay buffer logging.info("the number of training examples is %s" ", of test examples is %s" % (self._num_train_samples, self._num_test_samples)) logging.debug("the training set is from %s to %s" % (min(self._train_ind), max(self._train_ind))) logging.debug("the test set is from %s to %s" % (min(self._test_ind), max(self._test_ind)))
def __init__(self, start, end, period, market, feature_number, features_list, batch_size=50, volume_average_days=30, buffer_bias_ratio=0, coin_filter=1, window_size=50, test_portion=0.15, portion_reversed=False, online=False, is_permed=False): """ :param start: Unix time :param end: Unix time :param access_period: the data access period of the input matrix. :param trade_period: the trading period of the agent. :param global_period: the data access period of the global price matrix. if it is not equal to the access period, there will be inserted observations :param coin_filter: number of coins that would be selected :param window_size: periods of input data :param train_portion: portion of training set :param is_permed: if False, the sample inside a mini-batch is in order :param validation_portion: portion of cross-validation set :param test_portion: portion of test set :param portion_reversed: if False, the order to sets are [train, validation, test] else the order is [test, validation, train] """ # access json file and retrieve any relevant information the user wants to include with open("./pgportfolio/net_config.json") as file: config = json.load(file) config = preprocess_config(config) market = config["input"]["market_type"] start = int(start) self.__end = int(end) # assert window_size >= MIN_NUM_PERIOD self.__coin_no = coin_filter self.__features = features_list self.feature_number = feature_number volume_forward = get_volume_forward(self.__end - start, test_portion, portion_reversed) if market == "poloniex": self.__history_manager = gdm.HistoryManager( coin_number=coin_filter, end=self.__end, volume_average_days=volume_average_days, volume_forward=volume_forward, online=online) self.__global_data = self.__history_manager.get_global_dataframe( start, self.__end, period=period, features=features_list) elif market == "yahoo": with open("./pgportfolio/net_config.json") as file: config = json.load(file) config = preprocess_config(config) stock_data = config["input"]["stocks"] self.__history_manager = sgdm.StockHistoryManager( coin_number=coin_filter, end=self.__end, stocks=stock_data, volume_average_days=volume_average_days, volume_forward=volume_forward, online=online) self.__global_data = self.__history_manager.get_global_dataframe( start, self.__end, features_list, stock_data) elif market == "alphaVantage": with open("./pgportfolio/net_config.json") as file: config = json.load(file) config = preprocess_config(config) stock_data = config["input"][ "stocks"] # contains user defined securities api_key = config["input"]["api_key"] # user Alpha Vantage API Key api_call_limit = config["input"][ "api_call_limit"] # user Alpha Vantage call limit api_interval = config["input"][ "api_interval"] # time interval for the data # initiatize stock history manager class self.__history_manager = avgdm.AlphaVantageHistoryManager( coin_number=coin_filter, end=self.__end, online=online, stocks=stock_data, api_key=api_key, api_call_limit=api_call_limit, api_interval=api_interval, volume_average_days=volume_average_days, volume_forward=volume_forward) # return a dataframe of all securities data and corresponding tech. ind. self.__global_data = self.__history_manager.get_global_dataframe( start, self.__end, online, features_list, stock_data, api_key, api_call_limit, api_interval) # fill_dates_alphaVantage() else: raise ValueError("market {} is not valid".format(market)) #Go from [coins*features, index] to [features, coins, index] self.raw = self.__global_data.values.reshape( len(self.__global_data.index), len(self.__global_data.columns.levels[0]), len(self.__global_data.columns.levels[1]), ) self.raw = self.raw.transpose(2, 1, 0) self.__period_length = period # portfolio vector memory, [time, assets] self.__PVM = pd.DataFrame( index=self.__global_data.index, #time index columns=self.__global_data.columns.levels[0] ) #first level is coin names self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no) print(self.__PVM) self._window_size = window_size self._num_periods = len(self.__global_data.index) self.__divide_data(test_portion, portion_reversed) self._portion_reversed = portion_reversed self.__is_permed = is_permed self.__batch_size = batch_size self.__delta = 0 # the count of global increased end_index = self._train_ind[-1] self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0], end_index=end_index, sample_bias=buffer_bias_ratio, batch_size=self.__batch_size, coin_number=self.__coin_no, is_permed=self.__is_permed) logging.info("the number of training examples is %s" ", of test examples is %s" % (self._num_train_samples, self._num_test_samples)) logging.debug("the training set is from %s to %s" % (min(self._train_ind), max(self._train_ind))) logging.debug("the test set is from %s to %s" % (min(self._test_ind), max(self._test_ind)))
def __init__(self, start, end, period, batch_size=50, volume_average_days=30, buffer_bias_ratio=0, market="poloniex", coin_filter=1, window_size=50, feature_number=3, test_portion=0.15, portion_reversed=False, online=False, is_permed=False): """ :param start: Unix time :param end: Unix time :param access_period: the data access period of the input matrix. :param trade_period: the trading period of the agent. :param global_period: the data access period of the global price matrix. if it is not equal to the access period, there will be inserted observations :param coin_filter: number of coins that would be selected :param window_size: periods of input data :param train_portion: portion of training set :param is_permed: if False, the sample inside a mini-batch is in order :param validation_portion: portion of cross-validation set :param test_portion: portion of test set :param portion_reversed: if False, the order to sets are [train, validation, test] else the order is [test, validation, train] """ start = int(start) self.__end = int(end) # assert window_size >= MIN_NUM_PERIOD self.__coin_no = coin_filter type_list = get_type_list(feature_number) self.__features = type_list self.feature_number = feature_number self.market = market volume_forward = get_volume_forward(self.__end - start, test_portion, portion_reversed) if market == "poloniex": self.__history_manager = gdm.HistoryManager( coin_number=coin_filter, end=self.__end, volume_average_days=volume_average_days, volume_forward=volume_forward, online=online) self.__global_data = self.__history_manager.get_global_panel( start, self.__end, period=period, features=type_list) # If the market is gdax, read directly from the pickle file at GDAX_DIR elif market == "gdax": global_data = pd.read_pickle(GDAX_DIR) # If 4 features are used (close, high, low, open): if feature_number == 4: self.__global_data = global_data # If 3 features are used (close, high, low): elif feature_number == 3: self.__global_data = global_data.loc[['close', 'high', 'low']] # If 2 features are used (close, volume): elif feature_number == 2: # gdax does not have volume data raise NotImplementedError( "the feature volume is not supported currently") # If 1 features is used (close): elif feature_number == 1: self.__global_data = global_data.loc[['close']] else: raise ValueError("feature number could not be %s" % feature_number) else: raise ValueError("market {} is not valid".format(market)) self.__period_length = period # portfolio vector memory, [time, assets] self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis, columns=self.__global_data.major_axis) self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no) self._window_size = window_size self._num_periods = len(self.__global_data.minor_axis) self.__divide_data(test_portion, portion_reversed) self._portion_reversed = portion_reversed self.__is_permed = is_permed self.__batch_size = batch_size self.__delta = 0 # the count of global increased end_index = self._train_ind[-1] self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0], end_index=end_index, sample_bias=buffer_bias_ratio, batch_size=self.__batch_size, coin_number=self.__coin_no, is_permed=self.__is_permed) logging.info("the number of training examples is %s" ", of test examples is %s" % (self._num_train_samples, self._num_test_samples)) logging.debug("the training set is from %s to %s" % (min(self._train_ind), max(self._train_ind))) logging.debug("the test set is from %s to %s" % (min(self._test_ind), max(self._test_ind)))