def __init__(self,
batch_size=60,
buffer_bias_ratio=0,
coin_filter=1,
window_size=50,
feature_number=3,
portion_reversed=False,
online=False,
is_permed=False):
"""
:param coin_filter: number of coins that would be selected
:param window_size: periods of input data
:param portion_reversed: if False, the order to sets are [train, validation, test]
else the order is [test, validation, train]
"""
self.__coin_no = coin_filter
type_list = get_type_list(feature_number)
self.__features = type_list
self.feature_number = feature_number
self.__history_manager = gdm.HistoryManager(coin_number=coin_filter,
online=online)
self.__global_data = self.__history_manager.get_global_panel(
features=type_list)
# self.__global_market_capticalization = self.__history_manager.get_market_capticalization()
# self.__global_all_market_capticalization = self.__history_manager.get_all_market_capticalization()
self.__stockIndex_data = self.__history_manager.get_stockIndex_panel()
self.stock_code = self.__history_manager.stock_code
# portfolio vector memory, [time, assets]
self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis,
columns=self.__global_data.major_axis)
self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no)
# print(self.__PVM)
self._window_size = window_size
self._num_periods = len(self.__global_data.minor_axis)
self.__batch_size = batch_size
self.__divide_data()
self._portion_reversed = portion_reversed
self.__is_permed = is_permed
self.__delta = 0 # the count of global increased
end_index = self._train_ind[-1]
self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0],
end_index=end_index,
sample_bias=buffer_bias_ratio,
batch_size=self.__batch_size,
coin_number=self.__coin_no,
is_permed=self.__is_permed)
logging.info("the number of training examples is %s"
", of test examples is %s" %
(self._num_train_samples, self._num_test_samples))
logging.debug("the training set is from %s to %s" %
(min(self._train_ind), max(self._train_ind)))
logging.debug("the test set is from %s to %s" %
(min(self._test_ind), max(self._test_ind)))
def __init__(self,
start,
end,
period,
batch_size=50,
volume_average_days=30,
buffer_bias_ratio=0,
market="poloniex",
asset_filter=1,
window_size=50,
feature_number=3,
test_portion=0.15,
portion_reversed=False,
online=False,
is_permed=False):
"""
:param start: Unix time
:param end: Unix time
:param access_period: the data access period of the input matrix.
:param trade_period: the trading period of the agent.
:param global_period: the data access period of the global price matrix.
if it is not equal to the access period, there will be inserted observations
:param asset_filter: number of assets that would be selected
:param window_size: periods of input data
:param train_portion: portion of training set
:param is_permed: if False, the sample inside a mini-batch is in order
:param validation_portion: portion of cross-validation set
:param test_portion: portion of test set
:param portion_reversed: if False, the order to sets are [train, validation, test]
else the order is [test, validation, train]
"""
start = int(start)
self.__end = int(end)
# assert window_size >= MIN_NUM_PERIOD
self.__asset_no = asset_filter
type_list = get_type_list(feature_number)
self.__features = type_list
self.feature_number = feature_number
volume_forward = get_volume_forward(self.__end - start, test_portion,
portion_reversed)
self.__history_manager = gdm.HistoryManager(
asset_number=asset_filter,
end=self.__end,
volume_average_days=volume_average_days,
volume_forward=volume_forward)
if market == "poloniex": # poloniex == bitcoin market.
self.__global_data = self.__history_manager.get_global_panel(
start, self.__end, period=period, features=type_list)
else:
raise ValueError("market {} is not valid".format(market))
self.__period_length = period
# portfolio vector memory, [time, assets]
self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis,
columns=self.__global_data.major_axis)
self.__PVM = self.__PVM.fillna(1.0 / self.__asset_no)
self._window_size = window_size
self._num_periods = len(self.__global_data.minor_axis)
self.__divide_data(test_portion, portion_reversed)
self._portion_reversed = portion_reversed
self.__is_permed = is_permed
self.__batch_size = batch_size
self.__delta = 0 # the count of global increased
end_index = self._train_ind[-1]
self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0],
end_index=end_index,
sample_bias=buffer_bias_ratio,
batch_size=self.__batch_size,
asset_number=self.__asset_no,
is_permed=self.__is_permed)
logging.info("the number of training examples is %s"
", of test examples is %s" %
(self._num_train_samples, self._num_test_samples))
logging.debug("the training set is from %s to %s" %
(min(self._train_ind), max(self._train_ind)))
logging.debug("the test set is from %s to %s" %
(min(self._test_ind), max(self._test_ind)))
def __init__(self,
start,
end,
period,
batch_size=50,
buffer_bias_ratio=0,
coin_filter=1,
window_size=50,
feature_number=3,
test_portion=0.15,
portion_reversed=False,
is_permed=False):
"""
:param start: Unix time
:param end: Unix time
:param coin_filter: number of coins that would be selected
:param window_size: periods of input data
:param train_portion: portion of training set
:param is_permed: if False, the sample inside a mini-batch is in order
:param validation_portion: portion of cross-validation set
:param test_portion: portion of test set
:param portion_reversed: if False, the order to sets are [train, validation, test]
else the order is [test, validation, train]
"""
start = int(start)
self.__end = int(end)
self.__coin_no = coin_filter
type_list = get_type_list(feature_number)
self.__features = type_list
self.feature_number = feature_number
self.__history_manager = gdm.HistoryManager(coin_number=coin_filter,
end=self.__end)
self.__global_data = self.__history_manager.get_global_panel(
start, self.__end, period=period, features=type_list)
self.__period_length = period
# portfolio vector memory, [time, assets]
self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis,
columns=self.__global_data.major_axis)
self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no)
self._window_size = window_size
self._num_periods = len(self.__global_data.minor_axis)
self.__divide_data(test_portion, portion_reversed)
self._portion_reversed = portion_reversed
self.__is_permed = is_permed
self.__batch_size = batch_size
self.__delta = 0 # the count of global increased
end_index = self._train_ind[-1]
self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0],
end_index=end_index,
sample_bias=buffer_bias_ratio,
batch_size=self.__batch_size,
coin_number=self.__coin_no,
is_permed=self.__is_permed)
print("the number of training examples is %s"
", of test examples is %s" %
(self._num_train_samples, self._num_test_samples))
print("the training set is from %s to %s" %
(min(self._train_ind), max(self._train_ind)))
print("the test set is from %s to %s" %
(min(self._test_ind), max(self._test_ind)))
def predict_portfolio(algo, investment, chosen_coins, date, old_omega):
# Set config
if algo.isdigit():
config = load_config(algo)
else:
base_dir = os.path.abspath(os.path.dirname(__file__))
file_path = os.path.join(os.path.dirname(base_dir),
'PGPortfolio/pgportfolio/net_config.json')
with open(file_path) as file:
config = json.load(file)
input_config = config["input"]
feature_number = input_config["feature_number"]
period = input_config["global_period"]
volume = input_config["volume_average_days"]
features = get_type_list(feature_number)
# Set start and end times for retrieving data
date = date - 300 * (date % period < 300)
end = int(
date - (date % period)
) # converts into closest round time, minus 5 min if too close from it
if algo.isdigit():
# Use absolute path, to change to your configuration
base_dir = os.path.abspath(os.path.dirname(__file__))
net_dir = os.path.join(
os.path.dirname(base_dir),
'PGPortfolio/train_package/' + algo + '/netfile')
#train_package_path = "/home/michael/cryptofolio/Django/projet/PGPortfolio/train_package/"
#net_dir = train_package_path + algo + "/netfile"
start = end - volume * period # setting start to 30 periods before end
start = int(start - (start % period))
elif algo in ['bcrp', 'best']:
start = end - 90 * volume * period # setting start to 2700 periods before end (special algos)
start = int(start - (start % period))
else:
start = end - volume * period # setting start to 30 periods before end
start = int(start - (start % period))
# Create history matrix
coin_list = create_coinlist(
) # a pandas dataframe with coins (index), pair, volume, price (in BTC)
history = create_history(coin_list, chosen_coins, start, end, period,
features) # history matrix
# Generate new omega
if algo.isdigit():
os.environ["CUDA_VISIBLE_DEVICES"] = ""
with tf.device("/cpu:0"):
myAgent = NNAgent(config=config, restore_dir=net_dir, device="cpu")
omega = myAgent.decide_by_history(history, old_omega)
elif algo in valid_algorithms:
package = importlib.import_module("pgportfolio.tdagent.algorithms." +
algo)
myAgent = getattr(package, algo.upper())()
y = (history[0, :, 1:] / history[0, :, :-1])
y = np.concatenate((np.ones((1, y.shape[1])), y), axis=0)
omega = myAgent.decide_by_history(y.T, old_omega)
else:
print("Invalid algorithm")
omega = None
# Current BTC per Currency price (transformed into Currency per BTC)
v = 1.0 / history[0, :, -1]
v = np.concatenate((np.zeros(1), v), axis=0)
# Round omega to percentage
round_omega = np.around(omega, decimals=1)
# Compute omega in term of currencies, not percentage
portfolio = investment * round_omega * v
return portfolio, round_omega
def __init__(self,
start,
end,
period=86400,
batch_size=50,
volume_average_days=30,
buffer_bias_ratio=0,
market="poloniex",
coin_filter=1,
window_size=50,
feature_number=3,
test_portion=0.15,
portion_reversed=False,
online=False,
is_permed=False):
"""
:param start: Unix time
:param end: Unix time
:param access_period: the data access period of the input matrix.
:param trade_period: the trading period of the agent.
:param period、global_period: the data access period of the global price matrix.
if it is not equal to the access period, there will be inserted observations
:param coin_filter: number of coins that would be selected
:param window_size: periods of input data
:param train_portion: portion of training set
:param is_permed: if False, the sample inside a mini-batch is in order
:param validation_portion: portion of cross-validation set
:param test_portion: portion of test set
:param portion_reversed: if False, the order to sets are [train, validation, test]
else the order is [test, validation, train]
"""
start = int(start)
self.__end = int(end)
self.__coin_no = coin_filter # the number of coins
type_list = get_type_list(feature_number)
self.__features = type_list
self.feature_number = feature_number # the number of features
volume_forward = get_volume_forward(self.__end - start, test_portion,
portion_reversed)
self.__history_manager = gdm.HistoryManager(
coin_number=coin_filter,
end=self.__end,
volume_average_days=volume_average_days,
volume_forward=volume_forward,
online=online)
self.__global_data = self.__history_manager.get_global_panel(
start, self.__end, period=period, features=type_list)
self.__period_length = period
self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis,
columns=self.__global_data.major_axis) #
#print("self.__global_data.minor_axis",self.__global_data.minor_axis)
"""
for example
self.__global_data.minor_axis DatetimeIndex(['2015-12-31 16:00:00', '2015-12-31 16:30:00',
'2015-12-31 17:00:00', '2015-12-31 17:30:00',
'2015-12-31 18:00:00', '2015-12-31 18:30:00',
'2015-12-31 19:00:00', '2015-12-31 19:30:00',
'2015-12-31 20:00:00', '2015-12-31 20:30:00',
...
'2017-12-31 11:30:00', '2017-12-31 12:00:00',
'2017-12-31 12:30:00', '2017-12-31 13:00:00',
'2017-12-31 13:30:00', '2017-12-31 14:00:00',
'2017-12-31 14:30:00', '2017-12-31 15:00:00',
'2017-12-31 15:30:00', '2017-12-31 16:00:00'],
dtype='datetime64[ns]', length=35089, freq=None)
"""
self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no)
self._window_size = window_size
self._num_periods = len(self.__global_data.minor_axis)
self.__divide_data(test_portion, portion_reversed)
self._portion_reversed = portion_reversed
self.__is_permed = is_permed
self.__batch_size = batch_size
self.__delta = 0 # the count of global increased
end_index = self._train_ind[-1]
self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0],
end_index=end_index,
sample_bias=buffer_bias_ratio,
batch_size=self.__batch_size,
coin_number=self.__coin_no,
is_permed=self.__is_permed)
# experience replay buffer
logging.info("the number of training examples is %s"
", of test examples is %s" %
(self._num_train_samples, self._num_test_samples))
logging.debug("the training set is from %s to %s" %
(min(self._train_ind), max(self._train_ind)))
logging.debug("the test set is from %s to %s" %
(min(self._test_ind), max(self._test_ind)))
def __init__(self, start, end, period, batch_size=50, volume_average_days=30, buffer_bias_ratio=0,
market="poloniex", coin_filter=1, window_size=50, feature_number=3, test_portion=0.15,
portion_reversed=False, online=False, is_permed=False):
"""
:param start: Unix time
:param end: Unix time
:param access_period: the data access period of the input matrix.
:param trade_period: the trading period of the agent.
:param global_period: the data access period of the global price matrix.
if it is not equal to the access period, there will be inserted observations
:param coin_filter: number of coins that would be selected
:param window_size: periods of input data
:param train_portion: portion of training set
:param is_permed: if False, the sample inside a mini-batch is in order
:param validation_portion: portion of cross-validation set
:param test_portion: portion of test set
:param portion_reversed: if False, the order to sets are [train, validation, test]
else the order is [test, validation, train]
"""
start = int(start)
self.__end = int(end)
# assert window_size >= MIN_NUM_PERIOD
self.__coin_no = coin_filter
type_list = get_type_list(feature_number)
self.__features = type_list
self.feature_number = feature_number
volume_forward = get_volume_forward(self.__end-start, test_portion, portion_reversed)
self.__history_manager = gdm.HistoryManager(coin_number=coin_filter, end=self.__end,
volume_average_days=volume_average_days,
volume_forward=volume_forward, online=online)
if market == "poloniex":
self.__global_data = self.__history_manager.get_global_panel(start,
self.__end,
period=period,
features=type_list)
else:
raise ValueError("market {} is not valid".format(market))
self.__period_length = period
# portfolio vector memory, [time, assets]
self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis,
columns=self.__global_data.major_axis)
self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no)
self._window_size = window_size
self._num_periods = len(self.__global_data.minor_axis)
self.__divide_data(test_portion, portion_reversed)
self._portion_reversed = portion_reversed
self.__is_permed = is_permed
self.__batch_size = batch_size
self.__delta = 0 # the count of global increased
end_index = self._train_ind[-1]
self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0],
end_index=end_index,
sample_bias=buffer_bias_ratio,
batch_size=self.__batch_size,
coin_number=self.__coin_no,
is_permed=self.__is_permed)
logging.info("the number of training examples is %s"
", of test examples is %s" % (self._num_train_samples, self._num_test_samples))
logging.debug("the training set is from %s to %s" % (min(self._train_ind), max(self._train_ind)))
logging.debug("the test set is from %s to %s" % (min(self._test_ind), max(self._test_ind)))
def __init__(self,
start,
end,
period,
batch_size=50,
volume_average_days=30,
buffer_bias_ratio=0,
market="poloniex",
coin_filter=1,
window_size=50,
feature_number=3,
test_portion=0.15,
portion_reversed=False,
online=False,
is_permed=False):
"""
:param start: Unix time
:param end: Unix time
:param access_period: the data access period of the input matrix.
:param trade_period: the trading period of the agent.
:param global_period: the data access period of the global price matrix.
if it is not equal to the access period, there will be inserted observations
:param coin_filter: number of coins that would be selected
:param window_size: periods of input data
:param train_portion: portion of training set
:param is_permed: if False, the sample inside a mini-batch is in order
:param validation_portion: portion of cross-validation set
:param test_portion: portion of test set
:param portion_reversed: if False, the order to sets are [train, validation, test]
else the order is [test, validation, train]
"""
start = int(start)
self.__end = int(end)
# assert window_size >= MIN_NUM_PERIOD
self.__coin_no = coin_filter
type_list = get_type_list(feature_number)
self.__features = type_list
self.feature_number = feature_number
self.market = market
volume_forward = get_volume_forward(self.__end - start, test_portion,
portion_reversed)
if market == "poloniex":
self.__history_manager = gdm.HistoryManager(
coin_number=coin_filter,
end=self.__end,
volume_average_days=volume_average_days,
volume_forward=volume_forward,
online=online)
self.__global_data = self.__history_manager.get_global_panel(
start, self.__end, period=period, features=type_list)
# If the market is gdax, read directly from the pickle file at GDAX_DIR
elif market == "gdax":
global_data = pd.read_pickle(GDAX_DIR)
# If 4 features are used (close, high, low, open):
if feature_number == 4:
self.__global_data = global_data
# If 3 features are used (close, high, low):
elif feature_number == 3:
self.__global_data = global_data.loc[['close', 'high', 'low']]
# If 2 features are used (close, volume):
elif feature_number == 2:
# gdax does not have volume data
raise NotImplementedError(
"the feature volume is not supported currently")
# If 1 features is used (close):
elif feature_number == 1:
self.__global_data = global_data.loc[['close']]
else:
raise ValueError("feature number could not be %s" %
feature_number)
else:
raise ValueError("market {} is not valid".format(market))
self.__period_length = period
# portfolio vector memory, [time, assets]
self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis,
columns=self.__global_data.major_axis)
self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no)
self._window_size = window_size
self._num_periods = len(self.__global_data.minor_axis)
self.__divide_data(test_portion, portion_reversed)
self._portion_reversed = portion_reversed
self.__is_permed = is_permed
self.__batch_size = batch_size
self.__delta = 0 # the count of global increased
end_index = self._train_ind[-1]
self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0],
end_index=end_index,
sample_bias=buffer_bias_ratio,
batch_size=self.__batch_size,
coin_number=self.__coin_no,
is_permed=self.__is_permed)
logging.info("the number of training examples is %s"
", of test examples is %s" %
(self._num_train_samples, self._num_test_samples))
logging.debug("the training set is from %s to %s" %
(min(self._train_ind), max(self._train_ind)))
logging.debug("the test set is from %s to %s" %
(min(self._test_ind), max(self._test_ind)))