def __init__(self, batch_size=60, buffer_bias_ratio=0, coin_filter=1, window_size=50, feature_number=3, portion_reversed=False, online=False, is_permed=False): """ :param coin_filter: number of coins that would be selected :param window_size: periods of input data :param portion_reversed: if False, the order to sets are [train, validation, test] else the order is [test, validation, train] """ self.__coin_no = coin_filter type_list = get_type_list(feature_number) self.__features = type_list self.feature_number = feature_number self.__history_manager = gdm.HistoryManager(coin_number=coin_filter, online=online) self.__global_data = self.__history_manager.get_global_panel( features=type_list) # self.__global_market_capticalization = self.__history_manager.get_market_capticalization() # self.__global_all_market_capticalization = self.__history_manager.get_all_market_capticalization() self.__stockIndex_data = self.__history_manager.get_stockIndex_panel() self.stock_code = self.__history_manager.stock_code # portfolio vector memory, [time, assets] self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis, columns=self.__global_data.major_axis) self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no) # print(self.__PVM) self._window_size = window_size self._num_periods = len(self.__global_data.minor_axis) self.__batch_size = batch_size self.__divide_data() self._portion_reversed = portion_reversed self.__is_permed = is_permed self.__delta = 0 # the count of global increased end_index = self._train_ind[-1] self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0], end_index=end_index, sample_bias=buffer_bias_ratio, batch_size=self.__batch_size, coin_number=self.__coin_no, is_permed=self.__is_permed) logging.info("the number of training examples is %s" ", of test examples is %s" % (self._num_train_samples, self._num_test_samples)) logging.debug("the training set is from %s to %s" % (min(self._train_ind), max(self._train_ind))) logging.debug("the test set is from %s to %s" % (min(self._test_ind), max(self._test_ind)))
def __init__(self, start, end, period, batch_size=50, volume_average_days=30, buffer_bias_ratio=0, market="poloniex", asset_filter=1, window_size=50, feature_number=3, test_portion=0.15, portion_reversed=False, online=False, is_permed=False): """ :param start: Unix time :param end: Unix time :param access_period: the data access period of the input matrix. :param trade_period: the trading period of the agent. :param global_period: the data access period of the global price matrix. if it is not equal to the access period, there will be inserted observations :param asset_filter: number of assets that would be selected :param window_size: periods of input data :param train_portion: portion of training set :param is_permed: if False, the sample inside a mini-batch is in order :param validation_portion: portion of cross-validation set :param test_portion: portion of test set :param portion_reversed: if False, the order to sets are [train, validation, test] else the order is [test, validation, train] """ start = int(start) self.__end = int(end) # assert window_size >= MIN_NUM_PERIOD self.__asset_no = asset_filter type_list = get_type_list(feature_number) self.__features = type_list self.feature_number = feature_number volume_forward = get_volume_forward(self.__end - start, test_portion, portion_reversed) self.__history_manager = gdm.HistoryManager( asset_number=asset_filter, end=self.__end, volume_average_days=volume_average_days, volume_forward=volume_forward) if market == "poloniex": # poloniex == bitcoin market. self.__global_data = self.__history_manager.get_global_panel( start, self.__end, period=period, features=type_list) else: raise ValueError("market {} is not valid".format(market)) self.__period_length = period # portfolio vector memory, [time, assets] self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis, columns=self.__global_data.major_axis) self.__PVM = self.__PVM.fillna(1.0 / self.__asset_no) self._window_size = window_size self._num_periods = len(self.__global_data.minor_axis) self.__divide_data(test_portion, portion_reversed) self._portion_reversed = portion_reversed self.__is_permed = is_permed self.__batch_size = batch_size self.__delta = 0 # the count of global increased end_index = self._train_ind[-1] self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0], end_index=end_index, sample_bias=buffer_bias_ratio, batch_size=self.__batch_size, asset_number=self.__asset_no, is_permed=self.__is_permed) logging.info("the number of training examples is %s" ", of test examples is %s" % (self._num_train_samples, self._num_test_samples)) logging.debug("the training set is from %s to %s" % (min(self._train_ind), max(self._train_ind))) logging.debug("the test set is from %s to %s" % (min(self._test_ind), max(self._test_ind)))
def __init__(self, start, end, period, batch_size=50, buffer_bias_ratio=0, coin_filter=1, window_size=50, feature_number=3, test_portion=0.15, portion_reversed=False, is_permed=False): """ :param start: Unix time :param end: Unix time :param coin_filter: number of coins that would be selected :param window_size: periods of input data :param train_portion: portion of training set :param is_permed: if False, the sample inside a mini-batch is in order :param validation_portion: portion of cross-validation set :param test_portion: portion of test set :param portion_reversed: if False, the order to sets are [train, validation, test] else the order is [test, validation, train] """ start = int(start) self.__end = int(end) self.__coin_no = coin_filter type_list = get_type_list(feature_number) self.__features = type_list self.feature_number = feature_number self.__history_manager = gdm.HistoryManager(coin_number=coin_filter, end=self.__end) self.__global_data = self.__history_manager.get_global_panel( start, self.__end, period=period, features=type_list) self.__period_length = period # portfolio vector memory, [time, assets] self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis, columns=self.__global_data.major_axis) self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no) self._window_size = window_size self._num_periods = len(self.__global_data.minor_axis) self.__divide_data(test_portion, portion_reversed) self._portion_reversed = portion_reversed self.__is_permed = is_permed self.__batch_size = batch_size self.__delta = 0 # the count of global increased end_index = self._train_ind[-1] self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0], end_index=end_index, sample_bias=buffer_bias_ratio, batch_size=self.__batch_size, coin_number=self.__coin_no, is_permed=self.__is_permed) print("the number of training examples is %s" ", of test examples is %s" % (self._num_train_samples, self._num_test_samples)) print("the training set is from %s to %s" % (min(self._train_ind), max(self._train_ind))) print("the test set is from %s to %s" % (min(self._test_ind), max(self._test_ind)))
def predict_portfolio(algo, investment, chosen_coins, date, old_omega): # Set config if algo.isdigit(): config = load_config(algo) else: base_dir = os.path.abspath(os.path.dirname(__file__)) file_path = os.path.join(os.path.dirname(base_dir), 'PGPortfolio/pgportfolio/net_config.json') with open(file_path) as file: config = json.load(file) input_config = config["input"] feature_number = input_config["feature_number"] period = input_config["global_period"] volume = input_config["volume_average_days"] features = get_type_list(feature_number) # Set start and end times for retrieving data date = date - 300 * (date % period < 300) end = int( date - (date % period) ) # converts into closest round time, minus 5 min if too close from it if algo.isdigit(): # Use absolute path, to change to your configuration base_dir = os.path.abspath(os.path.dirname(__file__)) net_dir = os.path.join( os.path.dirname(base_dir), 'PGPortfolio/train_package/' + algo + '/netfile') #train_package_path = "/home/michael/cryptofolio/Django/projet/PGPortfolio/train_package/" #net_dir = train_package_path + algo + "/netfile" start = end - volume * period # setting start to 30 periods before end start = int(start - (start % period)) elif algo in ['bcrp', 'best']: start = end - 90 * volume * period # setting start to 2700 periods before end (special algos) start = int(start - (start % period)) else: start = end - volume * period # setting start to 30 periods before end start = int(start - (start % period)) # Create history matrix coin_list = create_coinlist( ) # a pandas dataframe with coins (index), pair, volume, price (in BTC) history = create_history(coin_list, chosen_coins, start, end, period, features) # history matrix # Generate new omega if algo.isdigit(): os.environ["CUDA_VISIBLE_DEVICES"] = "" with tf.device("/cpu:0"): myAgent = NNAgent(config=config, restore_dir=net_dir, device="cpu") omega = myAgent.decide_by_history(history, old_omega) elif algo in valid_algorithms: package = importlib.import_module("pgportfolio.tdagent.algorithms." + algo) myAgent = getattr(package, algo.upper())() y = (history[0, :, 1:] / history[0, :, :-1]) y = np.concatenate((np.ones((1, y.shape[1])), y), axis=0) omega = myAgent.decide_by_history(y.T, old_omega) else: print("Invalid algorithm") omega = None # Current BTC per Currency price (transformed into Currency per BTC) v = 1.0 / history[0, :, -1] v = np.concatenate((np.zeros(1), v), axis=0) # Round omega to percentage round_omega = np.around(omega, decimals=1) # Compute omega in term of currencies, not percentage portfolio = investment * round_omega * v return portfolio, round_omega
def __init__(self, start, end, period=86400, batch_size=50, volume_average_days=30, buffer_bias_ratio=0, market="poloniex", coin_filter=1, window_size=50, feature_number=3, test_portion=0.15, portion_reversed=False, online=False, is_permed=False): """ :param start: Unix time :param end: Unix time :param access_period: the data access period of the input matrix. :param trade_period: the trading period of the agent. :param period、global_period: the data access period of the global price matrix. if it is not equal to the access period, there will be inserted observations :param coin_filter: number of coins that would be selected :param window_size: periods of input data :param train_portion: portion of training set :param is_permed: if False, the sample inside a mini-batch is in order :param validation_portion: portion of cross-validation set :param test_portion: portion of test set :param portion_reversed: if False, the order to sets are [train, validation, test] else the order is [test, validation, train] """ start = int(start) self.__end = int(end) self.__coin_no = coin_filter # the number of coins type_list = get_type_list(feature_number) self.__features = type_list self.feature_number = feature_number # the number of features volume_forward = get_volume_forward(self.__end - start, test_portion, portion_reversed) self.__history_manager = gdm.HistoryManager( coin_number=coin_filter, end=self.__end, volume_average_days=volume_average_days, volume_forward=volume_forward, online=online) self.__global_data = self.__history_manager.get_global_panel( start, self.__end, period=period, features=type_list) self.__period_length = period self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis, columns=self.__global_data.major_axis) # #print("self.__global_data.minor_axis",self.__global_data.minor_axis) """ for example self.__global_data.minor_axis DatetimeIndex(['2015-12-31 16:00:00', '2015-12-31 16:30:00', '2015-12-31 17:00:00', '2015-12-31 17:30:00', '2015-12-31 18:00:00', '2015-12-31 18:30:00', '2015-12-31 19:00:00', '2015-12-31 19:30:00', '2015-12-31 20:00:00', '2015-12-31 20:30:00', ... '2017-12-31 11:30:00', '2017-12-31 12:00:00', '2017-12-31 12:30:00', '2017-12-31 13:00:00', '2017-12-31 13:30:00', '2017-12-31 14:00:00', '2017-12-31 14:30:00', '2017-12-31 15:00:00', '2017-12-31 15:30:00', '2017-12-31 16:00:00'], dtype='datetime64[ns]', length=35089, freq=None) """ self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no) self._window_size = window_size self._num_periods = len(self.__global_data.minor_axis) self.__divide_data(test_portion, portion_reversed) self._portion_reversed = portion_reversed self.__is_permed = is_permed self.__batch_size = batch_size self.__delta = 0 # the count of global increased end_index = self._train_ind[-1] self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0], end_index=end_index, sample_bias=buffer_bias_ratio, batch_size=self.__batch_size, coin_number=self.__coin_no, is_permed=self.__is_permed) # experience replay buffer logging.info("the number of training examples is %s" ", of test examples is %s" % (self._num_train_samples, self._num_test_samples)) logging.debug("the training set is from %s to %s" % (min(self._train_ind), max(self._train_ind))) logging.debug("the test set is from %s to %s" % (min(self._test_ind), max(self._test_ind)))
def __init__(self, start, end, period, batch_size=50, volume_average_days=30, buffer_bias_ratio=0, market="poloniex", coin_filter=1, window_size=50, feature_number=3, test_portion=0.15, portion_reversed=False, online=False, is_permed=False): """ :param start: Unix time :param end: Unix time :param access_period: the data access period of the input matrix. :param trade_period: the trading period of the agent. :param global_period: the data access period of the global price matrix. if it is not equal to the access period, there will be inserted observations :param coin_filter: number of coins that would be selected :param window_size: periods of input data :param train_portion: portion of training set :param is_permed: if False, the sample inside a mini-batch is in order :param validation_portion: portion of cross-validation set :param test_portion: portion of test set :param portion_reversed: if False, the order to sets are [train, validation, test] else the order is [test, validation, train] """ start = int(start) self.__end = int(end) # assert window_size >= MIN_NUM_PERIOD self.__coin_no = coin_filter type_list = get_type_list(feature_number) self.__features = type_list self.feature_number = feature_number volume_forward = get_volume_forward(self.__end-start, test_portion, portion_reversed) self.__history_manager = gdm.HistoryManager(coin_number=coin_filter, end=self.__end, volume_average_days=volume_average_days, volume_forward=volume_forward, online=online) if market == "poloniex": self.__global_data = self.__history_manager.get_global_panel(start, self.__end, period=period, features=type_list) else: raise ValueError("market {} is not valid".format(market)) self.__period_length = period # portfolio vector memory, [time, assets] self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis, columns=self.__global_data.major_axis) self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no) self._window_size = window_size self._num_periods = len(self.__global_data.minor_axis) self.__divide_data(test_portion, portion_reversed) self._portion_reversed = portion_reversed self.__is_permed = is_permed self.__batch_size = batch_size self.__delta = 0 # the count of global increased end_index = self._train_ind[-1] self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0], end_index=end_index, sample_bias=buffer_bias_ratio, batch_size=self.__batch_size, coin_number=self.__coin_no, is_permed=self.__is_permed) logging.info("the number of training examples is %s" ", of test examples is %s" % (self._num_train_samples, self._num_test_samples)) logging.debug("the training set is from %s to %s" % (min(self._train_ind), max(self._train_ind))) logging.debug("the test set is from %s to %s" % (min(self._test_ind), max(self._test_ind)))
def __init__(self, start, end, period, batch_size=50, volume_average_days=30, buffer_bias_ratio=0, market="poloniex", coin_filter=1, window_size=50, feature_number=3, test_portion=0.15, portion_reversed=False, online=False, is_permed=False): """ :param start: Unix time :param end: Unix time :param access_period: the data access period of the input matrix. :param trade_period: the trading period of the agent. :param global_period: the data access period of the global price matrix. if it is not equal to the access period, there will be inserted observations :param coin_filter: number of coins that would be selected :param window_size: periods of input data :param train_portion: portion of training set :param is_permed: if False, the sample inside a mini-batch is in order :param validation_portion: portion of cross-validation set :param test_portion: portion of test set :param portion_reversed: if False, the order to sets are [train, validation, test] else the order is [test, validation, train] """ start = int(start) self.__end = int(end) # assert window_size >= MIN_NUM_PERIOD self.__coin_no = coin_filter type_list = get_type_list(feature_number) self.__features = type_list self.feature_number = feature_number self.market = market volume_forward = get_volume_forward(self.__end - start, test_portion, portion_reversed) if market == "poloniex": self.__history_manager = gdm.HistoryManager( coin_number=coin_filter, end=self.__end, volume_average_days=volume_average_days, volume_forward=volume_forward, online=online) self.__global_data = self.__history_manager.get_global_panel( start, self.__end, period=period, features=type_list) # If the market is gdax, read directly from the pickle file at GDAX_DIR elif market == "gdax": global_data = pd.read_pickle(GDAX_DIR) # If 4 features are used (close, high, low, open): if feature_number == 4: self.__global_data = global_data # If 3 features are used (close, high, low): elif feature_number == 3: self.__global_data = global_data.loc[['close', 'high', 'low']] # If 2 features are used (close, volume): elif feature_number == 2: # gdax does not have volume data raise NotImplementedError( "the feature volume is not supported currently") # If 1 features is used (close): elif feature_number == 1: self.__global_data = global_data.loc[['close']] else: raise ValueError("feature number could not be %s" % feature_number) else: raise ValueError("market {} is not valid".format(market)) self.__period_length = period # portfolio vector memory, [time, assets] self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis, columns=self.__global_data.major_axis) self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no) self._window_size = window_size self._num_periods = len(self.__global_data.minor_axis) self.__divide_data(test_portion, portion_reversed) self._portion_reversed = portion_reversed self.__is_permed = is_permed self.__batch_size = batch_size self.__delta = 0 # the count of global increased end_index = self._train_ind[-1] self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0], end_index=end_index, sample_bias=buffer_bias_ratio, batch_size=self.__batch_size, coin_number=self.__coin_no, is_permed=self.__is_permed) logging.info("the number of training examples is %s" ", of test examples is %s" % (self._num_train_samples, self._num_test_samples)) logging.debug("the training set is from %s to %s" % (min(self._train_ind), max(self._train_ind))) logging.debug("the test set is from %s to %s" % (min(self._test_ind), max(self._test_ind)))