def get_coin_name_list(config, online):
"""
:param online: boolean value to show if connected to internet,
if False, load data from database.
:return : list of coin names
"""
input_config = config["input"]
if not online:
start = parse_time(input_config["start_date"])
end = parse_time(input_config["end_date"])
volume_forward = get_volume_forward(end - start,
input_config["test_portion"]
+ input_config["validation_portion"],
input_config["portion_reversed"])
else:
end = time()
volume_forward = 0
end = end - (end % input_config["trade_period"])
start = end - volume_forward - input_config["volume_average_days"] * DAY
end = end - volume_forward
coins = HistoryManager(input_config["coin_number"], end,
volume_forward=volume_forward,
volume_average_days=input_config["volume_average_days"],
online=online).\
select_coins(start, end)
return coins
def get_coin_name_list(config, online):
"""
:param online: boolean value to show if connected to internet,
if False, load data from database.
:return : list of coin names
"""
input_config = config["input"]
if not online:
start = parse_time(input_config["start_date"])
end = parse_time(input_config["end_date"])
volume_forward = get_volume_forward(
end - start,
input_config["test_portion"] + input_config["validation_portion"],
input_config["portion_reversed"])
else:
end = time()
volume_forward = 0
end = end - (end % input_config["trade_period"])
start = end - volume_forward - input_config["volume_average_days"] * DAY
end = end - volume_forward
coins = HistoryManager(input_config["coin_number"], end,
volume_forward=volume_forward,
volume_average_days=input_config["volume_average_days"],
online=online).\
select_coins(start, end)
return coins
def __init__(self,
start,
end,
period,
batch_size=50,
volume_average_days=30,
buffer_bias_ratio=0,
market='poloniex',
coin_filter=1,
window_size=50,
feature_number=3,
test_portion=0.15,
portion_reserved=False,
online=False,
is_permed=False):
"""
:param start: 開始時間（UNIX時間）
:param end: 終了時間（UNIX時間）
:param period:入力行列にアクセスできる期間
:param batch_size:バッチサイズ
:param volume_average_days:
:param buffer_bias_ratio:
:param market:
:param coin_filter: 使用するコインの数
:param window_size:入力データの数
:param feature_number:特徴量の数。1～4 (2は未実装)
:param test_portion:開始時間・終了時間の間で使用する割合(0～1)?
:param portion_reserved:部分でtestとtrainのどちらを使用するか。
Falseだとtrain, validation, test, Trueだとtest, validation, train。Boolean
:param online:
:param is_permed:
"""
start = int(start)
self.__end = int(end)
# window_sizeがMIN_NUM_PERIOD以上であること
self.__coin_no = coin_filter
type_list = get_type_list(feature_number)
self.__features = type_list
self.feature_number = feature_number
volume_forward = get_volume_forward(self.__end - start, test_portion,
portion_reserved)
self.__history_manager = gdm.HistoryManager(
coin_number=coin_filter,
end=self.__end,
volume_average_days=volume_average_days,
volume_forward=volume_forward,
online=online)
def __init__(self,
start,
end,
period,
batch_size=50,
volume_average_days=30,
buffer_bias_ratio=0,
market="poloniex",
asset_filter=1,
window_size=50,
feature_number=3,
test_portion=0.15,
portion_reversed=False,
online=False,
is_permed=False):
"""
:param start: Unix time
:param end: Unix time
:param access_period: the data access period of the input matrix.
:param trade_period: the trading period of the agent.
:param global_period: the data access period of the global price matrix.
if it is not equal to the access period, there will be inserted observations
:param asset_filter: number of assets that would be selected
:param window_size: periods of input data
:param train_portion: portion of training set
:param is_permed: if False, the sample inside a mini-batch is in order
:param validation_portion: portion of cross-validation set
:param test_portion: portion of test set
:param portion_reversed: if False, the order to sets are [train, validation, test]
else the order is [test, validation, train]
"""
start = int(start)
self.__end = int(end)
# assert window_size >= MIN_NUM_PERIOD
self.__asset_no = asset_filter
type_list = get_type_list(feature_number)
self.__features = type_list
self.feature_number = feature_number
volume_forward = get_volume_forward(self.__end - start, test_portion,
portion_reversed)
self.__history_manager = gdm.HistoryManager(
asset_number=asset_filter,
end=self.__end,
volume_average_days=volume_average_days,
volume_forward=volume_forward)
if market == "poloniex": # poloniex == bitcoin market.
self.__global_data = self.__history_manager.get_global_panel(
start, self.__end, period=period, features=type_list)
else:
raise ValueError("market {} is not valid".format(market))
self.__period_length = period
# portfolio vector memory, [time, assets]
self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis,
columns=self.__global_data.major_axis)
self.__PVM = self.__PVM.fillna(1.0 / self.__asset_no)
self._window_size = window_size
self._num_periods = len(self.__global_data.minor_axis)
self.__divide_data(test_portion, portion_reversed)
self._portion_reversed = portion_reversed
self.__is_permed = is_permed
self.__batch_size = batch_size
self.__delta = 0 # the count of global increased
end_index = self._train_ind[-1]
self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0],
end_index=end_index,
sample_bias=buffer_bias_ratio,
batch_size=self.__batch_size,
asset_number=self.__asset_no,
is_permed=self.__is_permed)
logging.info("the number of training examples is %s"
", of test examples is %s" %
(self._num_train_samples, self._num_test_samples))
logging.debug("the training set is from %s to %s" %
(min(self._train_ind), max(self._train_ind)))
logging.debug("the test set is from %s to %s" %
(min(self._test_ind), max(self._test_ind)))
def __init__(self,
start,
end,
period=86400,
batch_size=50,
volume_average_days=30,
buffer_bias_ratio=0,
market="poloniex",
coin_filter=1,
window_size=50,
feature_number=3,
test_portion=0.15,
portion_reversed=False,
online=False,
is_permed=False):
"""
:param start: Unix time
:param end: Unix time
:param access_period: the data access period of the input matrix.
:param trade_period: the trading period of the agent.
:param period、global_period: the data access period of the global price matrix.
if it is not equal to the access period, there will be inserted observations
:param coin_filter: number of coins that would be selected
:param window_size: periods of input data
:param train_portion: portion of training set
:param is_permed: if False, the sample inside a mini-batch is in order
:param validation_portion: portion of cross-validation set
:param test_portion: portion of test set
:param portion_reversed: if False, the order to sets are [train, validation, test]
else the order is [test, validation, train]
"""
start = int(start)
self.__end = int(end)
self.__coin_no = coin_filter # the number of coins
type_list = get_type_list(feature_number)
self.__features = type_list
self.feature_number = feature_number # the number of features
volume_forward = get_volume_forward(self.__end - start, test_portion,
portion_reversed)
self.__history_manager = gdm.HistoryManager(
coin_number=coin_filter,
end=self.__end,
volume_average_days=volume_average_days,
volume_forward=volume_forward,
online=online)
self.__global_data = self.__history_manager.get_global_panel(
start, self.__end, period=period, features=type_list)
self.__period_length = period
self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis,
columns=self.__global_data.major_axis) #
#print("self.__global_data.minor_axis",self.__global_data.minor_axis)
"""
for example
self.__global_data.minor_axis DatetimeIndex(['2015-12-31 16:00:00', '2015-12-31 16:30:00',
'2015-12-31 17:00:00', '2015-12-31 17:30:00',
'2015-12-31 18:00:00', '2015-12-31 18:30:00',
'2015-12-31 19:00:00', '2015-12-31 19:30:00',
'2015-12-31 20:00:00', '2015-12-31 20:30:00',
...
'2017-12-31 11:30:00', '2017-12-31 12:00:00',
'2017-12-31 12:30:00', '2017-12-31 13:00:00',
'2017-12-31 13:30:00', '2017-12-31 14:00:00',
'2017-12-31 14:30:00', '2017-12-31 15:00:00',
'2017-12-31 15:30:00', '2017-12-31 16:00:00'],
dtype='datetime64[ns]', length=35089, freq=None)
"""
self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no)
self._window_size = window_size
self._num_periods = len(self.__global_data.minor_axis)
self.__divide_data(test_portion, portion_reversed)
self._portion_reversed = portion_reversed
self.__is_permed = is_permed
self.__batch_size = batch_size
self.__delta = 0 # the count of global increased
end_index = self._train_ind[-1]
self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0],
end_index=end_index,
sample_bias=buffer_bias_ratio,
batch_size=self.__batch_size,
coin_number=self.__coin_no,
is_permed=self.__is_permed)
# experience replay buffer
logging.info("the number of training examples is %s"
", of test examples is %s" %
(self._num_train_samples, self._num_test_samples))
logging.debug("the training set is from %s to %s" %
(min(self._train_ind), max(self._train_ind)))
logging.debug("the test set is from %s to %s" %
(min(self._test_ind), max(self._test_ind)))
def __init__(self,
start,
end,
period,
market,
feature_number,
features_list,
batch_size=50,
volume_average_days=30,
buffer_bias_ratio=0,
coin_filter=1,
window_size=50,
test_portion=0.15,
portion_reversed=False,
online=False,
is_permed=False):
"""
:param start: Unix time
:param end: Unix time
:param access_period: the data access period of the input matrix.
:param trade_period: the trading period of the agent.
:param global_period: the data access period of the global price matrix.
if it is not equal to the access period, there will be inserted observations
:param coin_filter: number of coins that would be selected
:param window_size: periods of input data
:param train_portion: portion of training set
:param is_permed: if False, the sample inside a mini-batch is in order
:param validation_portion: portion of cross-validation set
:param test_portion: portion of test set
:param portion_reversed: if False, the order to sets are [train, validation, test]
else the order is [test, validation, train]
"""
# access json file and retrieve any relevant information the user wants to include
with open("./pgportfolio/net_config.json") as file:
config = json.load(file)
config = preprocess_config(config)
market = config["input"]["market_type"]
start = int(start)
self.__end = int(end)
# assert window_size >= MIN_NUM_PERIOD
self.__coin_no = coin_filter
self.__features = features_list
self.feature_number = feature_number
volume_forward = get_volume_forward(self.__end - start, test_portion,
portion_reversed)
if market == "poloniex":
self.__history_manager = gdm.HistoryManager(
coin_number=coin_filter,
end=self.__end,
volume_average_days=volume_average_days,
volume_forward=volume_forward,
online=online)
self.__global_data = self.__history_manager.get_global_dataframe(
start, self.__end, period=period, features=features_list)
elif market == "yahoo":
with open("./pgportfolio/net_config.json") as file:
config = json.load(file)
config = preprocess_config(config)
stock_data = config["input"]["stocks"]
self.__history_manager = sgdm.StockHistoryManager(
coin_number=coin_filter,
end=self.__end,
stocks=stock_data,
volume_average_days=volume_average_days,
volume_forward=volume_forward,
online=online)
self.__global_data = self.__history_manager.get_global_dataframe(
start, self.__end, features_list, stock_data)
elif market == "alphaVantage":
with open("./pgportfolio/net_config.json") as file:
config = json.load(file)
config = preprocess_config(config)
stock_data = config["input"][
"stocks"] # contains user defined securities
api_key = config["input"]["api_key"] # user Alpha Vantage API Key
api_call_limit = config["input"][
"api_call_limit"] # user Alpha Vantage call limit
api_interval = config["input"][
"api_interval"] # time interval for the data
# initiatize stock history manager class
self.__history_manager = avgdm.AlphaVantageHistoryManager(
coin_number=coin_filter,
end=self.__end,
online=online,
stocks=stock_data,
api_key=api_key,
api_call_limit=api_call_limit,
api_interval=api_interval,
volume_average_days=volume_average_days,
volume_forward=volume_forward)
# return a dataframe of all securities data and corresponding tech. ind.
self.__global_data = self.__history_manager.get_global_dataframe(
start, self.__end, online, features_list, stock_data, api_key,
api_call_limit, api_interval)
# fill_dates_alphaVantage()
else:
raise ValueError("market {} is not valid".format(market))
#Go from [coins*features, index] to [features, coins, index]
self.raw = self.__global_data.values.reshape(
len(self.__global_data.index),
len(self.__global_data.columns.levels[0]),
len(self.__global_data.columns.levels[1]),
)
self.raw = self.raw.transpose(2, 1, 0)
self.__period_length = period
# portfolio vector memory, [time, assets]
self.__PVM = pd.DataFrame(
index=self.__global_data.index, #time index
columns=self.__global_data.columns.levels[0]
) #first level is coin names
self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no)
print(self.__PVM)
self._window_size = window_size
self._num_periods = len(self.__global_data.index)
self.__divide_data(test_portion, portion_reversed)
self._portion_reversed = portion_reversed
self.__is_permed = is_permed
self.__batch_size = batch_size
self.__delta = 0 # the count of global increased
end_index = self._train_ind[-1]
self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0],
end_index=end_index,
sample_bias=buffer_bias_ratio,
batch_size=self.__batch_size,
coin_number=self.__coin_no,
is_permed=self.__is_permed)
logging.info("the number of training examples is %s"
", of test examples is %s" %
(self._num_train_samples, self._num_test_samples))
logging.debug("the training set is from %s to %s" %
(min(self._train_ind), max(self._train_ind)))
logging.debug("the test set is from %s to %s" %
(min(self._test_ind), max(self._test_ind)))
def __init__(self, start, end, period, batch_size=50, volume_average_days=30, buffer_bias_ratio=0,
market="poloniex", coin_filter=1, window_size=50, feature_number=3, test_portion=0.15,
portion_reversed=False, online=False, is_permed=False):
"""
:param start: Unix time
:param end: Unix time
:param access_period: the data access period of the input matrix.
:param trade_period: the trading period of the agent.
:param global_period: the data access period of the global price matrix.
if it is not equal to the access period, there will be inserted observations
:param coin_filter: number of coins that would be selected
:param window_size: periods of input data
:param train_portion: portion of training set
:param is_permed: if False, the sample inside a mini-batch is in order
:param validation_portion: portion of cross-validation set
:param test_portion: portion of test set
:param portion_reversed: if False, the order to sets are [train, validation, test]
else the order is [test, validation, train]
"""
start = int(start)
self.__end = int(end)
# assert window_size >= MIN_NUM_PERIOD
self.__coin_no = coin_filter
type_list = get_type_list(feature_number)
self.__features = type_list
self.feature_number = feature_number
volume_forward = get_volume_forward(self.__end-start, test_portion, portion_reversed)
self.__history_manager = gdm.HistoryManager(coin_number=coin_filter, end=self.__end,
volume_average_days=volume_average_days,
volume_forward=volume_forward, online=online)
if market == "poloniex":
self.__global_data = self.__history_manager.get_global_panel(start,
self.__end,
period=period,
features=type_list)
else:
raise ValueError("market {} is not valid".format(market))
self.__period_length = period
# portfolio vector memory, [time, assets]
self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis,
columns=self.__global_data.major_axis)
self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no)
self._window_size = window_size
self._num_periods = len(self.__global_data.minor_axis)
self.__divide_data(test_portion, portion_reversed)
self._portion_reversed = portion_reversed
self.__is_permed = is_permed
self.__batch_size = batch_size
self.__delta = 0 # the count of global increased
end_index = self._train_ind[-1]
self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0],
end_index=end_index,
sample_bias=buffer_bias_ratio,
batch_size=self.__batch_size,
coin_number=self.__coin_no,
is_permed=self.__is_permed)
logging.info("the number of training examples is %s"
", of test examples is %s" % (self._num_train_samples, self._num_test_samples))
logging.debug("the training set is from %s to %s" % (min(self._train_ind), max(self._train_ind)))
logging.debug("the test set is from %s to %s" % (min(self._test_ind), max(self._test_ind)))
def __init__(self,
start,
end,
period,
batch_size=50,
volume_average_days=30,
buffer_bias_ratio=0,
market="poloniex",
coin_filter=1,
window_size=50,
feature_number=3,
test_portion=0.15,
portion_reversed=False,
online=False,
is_permed=False):
"""
:param start: Unix time
:param end: Unix time
:param access_period: the data access period of the input matrix.
:param trade_period: the trading period of the agent.
:param global_period: the data access period of the global price matrix.
if it is not equal to the access period, there will be inserted observations
:param coin_filter: number of coins that would be selected
:param window_size: periods of input data
:param train_portion: portion of training set
:param is_permed: if False, the sample inside a mini-batch is in order
:param validation_portion: portion of cross-validation set
:param test_portion: portion of test set
:param portion_reversed: if False, the order to sets are [train, validation, test]
else the order is [test, validation, train]
"""
start = int(start)
self.__end = int(end)
# assert window_size >= MIN_NUM_PERIOD
self.__coin_no = coin_filter
type_list = get_type_list(feature_number)
self.__features = type_list
self.feature_number = feature_number
self.market = market
volume_forward = get_volume_forward(self.__end - start, test_portion,
portion_reversed)
if market == "poloniex":
self.__history_manager = gdm.HistoryManager(
coin_number=coin_filter,
end=self.__end,
volume_average_days=volume_average_days,
volume_forward=volume_forward,
online=online)
self.__global_data = self.__history_manager.get_global_panel(
start, self.__end, period=period, features=type_list)
# If the market is gdax, read directly from the pickle file at GDAX_DIR
elif market == "gdax":
global_data = pd.read_pickle(GDAX_DIR)
# If 4 features are used (close, high, low, open):
if feature_number == 4:
self.__global_data = global_data
# If 3 features are used (close, high, low):
elif feature_number == 3:
self.__global_data = global_data.loc[['close', 'high', 'low']]
# If 2 features are used (close, volume):
elif feature_number == 2:
# gdax does not have volume data
raise NotImplementedError(
"the feature volume is not supported currently")
# If 1 features is used (close):
elif feature_number == 1:
self.__global_data = global_data.loc[['close']]
else:
raise ValueError("feature number could not be %s" %
feature_number)
else:
raise ValueError("market {} is not valid".format(market))
self.__period_length = period
# portfolio vector memory, [time, assets]
self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis,
columns=self.__global_data.major_axis)
self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no)
self._window_size = window_size
self._num_periods = len(self.__global_data.minor_axis)
self.__divide_data(test_portion, portion_reversed)
self._portion_reversed = portion_reversed
self.__is_permed = is_permed
self.__batch_size = batch_size
self.__delta = 0 # the count of global increased
end_index = self._train_ind[-1]
self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0],
end_index=end_index,
sample_bias=buffer_bias_ratio,
batch_size=self.__batch_size,
coin_number=self.__coin_no,
is_permed=self.__is_permed)
logging.info("the number of training examples is %s"
", of test examples is %s" %
(self._num_train_samples, self._num_test_samples))
logging.debug("the training set is from %s to %s" %
(min(self._train_ind), max(self._train_ind)))
logging.debug("the test set is from %s to %s" %
(min(self._test_ind), max(self._test_ind)))
