def main(): parser = build_parser() options = parser.parse_args() if not os.path.exists('./train_package'): os.makedirs('./train_package') if not os.path.exists('./database'): os.makedirs('./database') if options.mode == 'train': import pgportfolio.autotrain.training if not options.algo: pgportfolio.autotrain.training.train_all(int(options.processes), options.device) else: raise NotImplementedError() elif options.mode == 'generate': import pgportfolio.autotrain.generate as generate logging.basicConfig(level=logging.INFO) generate.add_packages(load_config(), int(options.repeat)) elif options.mode == 'download_data': from pgportfolio.marketdata.datamatrices import DataMatrices with open('./pgportfolio/net_config.json') as file: config =json.load(file) config = preprocess_config(file) # コンフィグのインプットから開始日、終了日を取得 start = time.mktime(datetime.strptime(config["input"]["start_date"], "%Y/%m/%d").timetuple()) end = time.mktime(datetime.strptime(config["input"]["end_date"], "%Y/%m/%d").timetuple())
def decide_by_history(self, x, last_b): if self.last_b is None: from pgportfolio.tools.trade import get_test_data from pgportfolio.tools.configprocess import preprocess_config import json with open("pgportfolio/net_config.json") as file: config = json.load(file) config = preprocess_config(config) data = self.test_relative self.last_b = self.get_weight(data.T) return self.last_b
def decide_by_history(self, x, last_b): if self.last_b is None: from pgportfolio.tools.trade import get_test_data from pgportfolio.tools.configprocess import preprocess_config import json with open("pgportfolio/net_config.json") as file: config = json.load(file) config = preprocess_config(config) data = get_test_data(config) self.last_b = self.get_weight(data.T) return self.last_b
def decide_by_history(self, data, last_b=None): if self.last_b is None: from pgportfolio.tools.trade import get_test_data from pgportfolio.tools.configprocess import preprocess_config import json with open("pgportfolio/net_config.json") as file: config = json.load(file) config = preprocess_config(config) data = get_test_data(config) data = data.T n, m = data.shape tmp_cumprod_ret = np.cumprod(data, axis=0) best_ind = np.argmax(tmp_cumprod_ret[-1,:]) self.last_b = np.zeros(m) self.last_b[best_ind] = 1 return self.last_b.ravel()
def main(): parser = build_parser() options = parser.parse_args() if not os.path.exists("./" + "train_package"): os.makedirs("./" + "train_package") if not os.path.exists("./" + "database"): os.makedirs("./" + "database") if options.mode == "train": import pgportfolio.autotrain.training if not options.algo: pgportfolio.autotrain.training.train_all(int(options.processes), options.device) else: for folder in options.train_floder: raise NotImplementedError() elif options.mode == "generate": import pgportfolio.autotrain.generate as generate logging.basicConfig(level=logging.INFO) generate.add_packages(load_config(), int(options.repeat)) elif options.mode == "download_data": from pgportfolio.marketdata.datamatrices import DataMatrices with open("./pgportfolio/net_config.json") as file: config = json.load(file) config = preprocess_config(config) start = time.mktime(datetime.strptime(config["input"]["start_date"], "%Y/%m/%d").timetuple()) end = time.mktime(datetime.strptime(config["input"]["end_date"], "%Y/%m/%d").timetuple()) DataMatrices(start=start, end=end, feature_number=config["input"]["feature_number"], window_size=config["input"]["window_size"], online=True, period=config["input"]["global_period"], volume_average_days=config["input"]["volume_average_days"], coin_filter=config["input"]["coin_number"], is_permed=config["input"]["is_permed"], test_portion=config["input"]["test_portion"], portion_reversed=config["input"]["portion_reversed"]) elif options.mode == "backtest": config = _config_by_algo(options.algo) _set_logging_by_algo(logging.DEBUG, logging.DEBUG, options.algo, "backtestlog") execute_backtest(options.algo, config) elif options.mode == "save_test_data": # This is used to export the test data save_test_data(load_config(options.folder)) elif options.mode == "plot": logging.basicConfig(level=logging.INFO) algos = options.algos.split(",") if options.labels: labels = options.labels.replace("_"," ") labels = labels.split(",") else: labels = algos plot.plot_backtest(load_config(), algos, labels) elif options.mode == "table": algos = options.algos.split(",") if options.labels: labels = options.labels.replace("_"," ") labels = labels.split(",") else: labels = algos plot.table_backtest(load_config(), algos, labels, format=options.format)
import pgportfolio.autotrain.training if not options_algo: pgportfolio.autotrain.training.train_all(int(options_processes), options_device) else: for folder in options_folder: raise NotImplementedError() elif execute_mode == "generate": import pgportfolio.autotrain.generate as generate logging.basicConfig(level=logging.INFO) generate.add_packages(load_config(), int(options_repeat)) elif execute_mode == "download_data": from pgportfolio.marketdata.datamatrices import DataMatrices with open("./pgportfolio/net_config.json") as file: config = json.load(file) config = preprocess_config(config) start = time.mktime( datetime.strptime(config["input"]["start_date"], "%Y/%m/%d").timetuple()) end = time.mktime( datetime.strptime(config["input"]["end_date"], "%Y/%m/%d").timetuple()) DataMatrices(start=start, end=end, feature_number=config["input"]["feature_number"], window_size=config["input"]["window_size"], online=True, period=config["input"]["global_period"], volume_average_days=config["input"]["volume_average_days"], coin_filter=config["input"]["coin_number"], is_permed=config["input"]["is_permed"], test_portion=config["input"]["test_portion"],
def main(): parser = build_parser() options = parser.parse_args() if not os.path.exists("./" + "train_package"): os.makedirs("./" + "train_package") if not os.path.exists("./" + "database"): os.makedirs("./" + "database") if options.mode == "train": import pgportfolio.autotrain.training if not options.algo: pgportfolio.autotrain.training.train_all(int(options.processes), options.device) else: for folder in options.folder: raise NotImplementedError() elif options.mode == "generate": import pgportfolio.autotrain.generate as generate logging.basicConfig(level=logging.INFO) generate.add_packages(load_config(), int(options.repeat)) elif options.mode == "download_data": from pgportfolio.marketdata.datamatrices import DataMatrices with open("./pgportfolio/net_config.json") as file: config = json.load(file) config = preprocess_config(config) start = time.mktime(datetime.strptime(config["input"]["start_date"], "%Y/%m/%d").timetuple()) end = time.mktime(datetime.strptime(config["input"]["end_date"], "%Y/%m/%d").timetuple()) DataMatrices(start=start, end=end, feature_number=config["input"]["feature_number"], window_size=config["input"]["window_size"], online=True, period=config["input"]["global_period"], volume_average_days=config["input"]["volume_average_days"], coin_filter=config["input"]["coin_number"], is_permed=config["input"]["is_permed"], test_portion=config["input"]["test_portion"], portion_reversed=config["input"]["portion_reversed"]) elif options.mode == "backtest": config = _config_by_algo(options.algo) _set_logging_by_algo(logging.DEBUG, logging.DEBUG, options.algo, "backtestlog") execute_backtest(options.algo, config) elif options.mode == "save_test_data": # This is used to export the test data save_test_data(load_config(options.folder)) elif options.mode == "plot": logging.basicConfig(level=logging.INFO) algos = options.algos.split(",") if options.labels: labels = options.labels.replace("_"," ") labels = labels.split(",") else: labels = algos plot.plot_backtest(load_config(), algos, labels) elif options.mode == "table": algos = options.algos.split(",") if options.labels: labels = options.labels.replace("_"," ") labels = labels.split(",") else: labels = algos plot.table_backtest(load_config(), algos, labels, format=options.format)
def __init__(self, start, end, period, market, feature_number, features_list, batch_size=50, volume_average_days=30, buffer_bias_ratio=0, coin_filter=1, window_size=50, test_portion=0.15, portion_reversed=False, online=False, is_permed=False): """ :param start: Unix time :param end: Unix time :param access_period: the data access period of the input matrix. :param trade_period: the trading period of the agent. :param global_period: the data access period of the global price matrix. if it is not equal to the access period, there will be inserted observations :param coin_filter: number of coins that would be selected :param window_size: periods of input data :param train_portion: portion of training set :param is_permed: if False, the sample inside a mini-batch is in order :param validation_portion: portion of cross-validation set :param test_portion: portion of test set :param portion_reversed: if False, the order to sets are [train, validation, test] else the order is [test, validation, train] """ # access json file and retrieve any relevant information the user wants to include with open("./pgportfolio/net_config.json") as file: config = json.load(file) config = preprocess_config(config) market = config["input"]["market_type"] start = int(start) self.__end = int(end) # assert window_size >= MIN_NUM_PERIOD self.__coin_no = coin_filter self.__features = features_list self.feature_number = feature_number volume_forward = get_volume_forward(self.__end - start, test_portion, portion_reversed) if market == "poloniex": self.__history_manager = gdm.HistoryManager( coin_number=coin_filter, end=self.__end, volume_average_days=volume_average_days, volume_forward=volume_forward, online=online) self.__global_data = self.__history_manager.get_global_dataframe( start, self.__end, period=period, features=features_list) elif market == "yahoo": with open("./pgportfolio/net_config.json") as file: config = json.load(file) config = preprocess_config(config) stock_data = config["input"]["stocks"] self.__history_manager = sgdm.StockHistoryManager( coin_number=coin_filter, end=self.__end, stocks=stock_data, volume_average_days=volume_average_days, volume_forward=volume_forward, online=online) self.__global_data = self.__history_manager.get_global_dataframe( start, self.__end, features_list, stock_data) elif market == "alphaVantage": with open("./pgportfolio/net_config.json") as file: config = json.load(file) config = preprocess_config(config) stock_data = config["input"][ "stocks"] # contains user defined securities api_key = config["input"]["api_key"] # user Alpha Vantage API Key api_call_limit = config["input"][ "api_call_limit"] # user Alpha Vantage call limit api_interval = config["input"][ "api_interval"] # time interval for the data # initiatize stock history manager class self.__history_manager = avgdm.AlphaVantageHistoryManager( coin_number=coin_filter, end=self.__end, online=online, stocks=stock_data, api_key=api_key, api_call_limit=api_call_limit, api_interval=api_interval, volume_average_days=volume_average_days, volume_forward=volume_forward) # return a dataframe of all securities data and corresponding tech. ind. self.__global_data = self.__history_manager.get_global_dataframe( start, self.__end, online, features_list, stock_data, api_key, api_call_limit, api_interval) # fill_dates_alphaVantage() else: raise ValueError("market {} is not valid".format(market)) #Go from [coins*features, index] to [features, coins, index] self.raw = self.__global_data.values.reshape( len(self.__global_data.index), len(self.__global_data.columns.levels[0]), len(self.__global_data.columns.levels[1]), ) self.raw = self.raw.transpose(2, 1, 0) self.__period_length = period # portfolio vector memory, [time, assets] self.__PVM = pd.DataFrame( index=self.__global_data.index, #time index columns=self.__global_data.columns.levels[0] ) #first level is coin names self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no) print(self.__PVM) self._window_size = window_size self._num_periods = len(self.__global_data.index) self.__divide_data(test_portion, portion_reversed) self._portion_reversed = portion_reversed self.__is_permed = is_permed self.__batch_size = batch_size self.__delta = 0 # the count of global increased end_index = self._train_ind[-1] self.__replay_buffer = rb.ReplayBuffer(start_index=self._train_ind[0], end_index=end_index, sample_bias=buffer_bias_ratio, batch_size=self.__batch_size, coin_number=self.__coin_no, is_permed=self.__is_permed) logging.info("the number of training examples is %s" ", of test examples is %s" % (self._num_train_samples, self._num_test_samples)) logging.debug("the training set is from %s to %s" % (min(self._train_ind), max(self._train_ind))) logging.debug("the test set is from %s to %s" % (min(self._test_ind), max(self._test_ind)))
def main(logPath, device): parser = build_parser() options = parser.parse_args() ''' options.folder = oneKey options.stockList = stockList options.featureList = featureList options.start_date = startDate options.end_date = endDate ''' if not os.path.exists("./" + "database"): os.makedirs("./" + "database") #options.repeat = 1 if options.mode == "train": #训练数据 if not options.algo: save_path = logPath + str(options.folder) + "/netfile" # 读取配置文件 with open(logPath + str(options.folder) + "\\net_config.json") as file: config_json = None config_json = json.load(file) config = preprocess_config(config_json) log_file_dir = logPath + str(options.folder) + "/tensorboard" # 定义错误等级 logfile_level = logging.DEBUG console_level = logging.INFO logging.basicConfig(filename=log_file_dir.replace( "tensorboard", "programlog"), level=logfile_level) console = logging.StreamHandler() console.setLevel(console_level) logging.getLogger().addHandler(console) trainer = TraderTrainer(config, options.stockList, options.featureList, options.start_date, options.end_date, save_path=save_path, device=device) #初始化训练器 trainer.train_net(log_file_dir=log_file_dir, index=str(options.folder)) #训练网络 else: for folder in options.folder: raise NotImplementedError() # 生成配置文件到路径中,要想修改配置,直接修改PGPortfolio\pgportfolio\net_config.json elif options.mode == "generate": import pgportfolio.autotrain.generate as generate logging.basicConfig(level=logging.INFO) config_ = load_config() train_dir = logPath generate.add_packages(train_dir, load_config(), int(options.repeat)) elif options.mode == "download_data": from pgportfolio.marketdata.datamatrices import DataMatrices with open("./pgportfolio/net_config.json") as file: config = json.load(file) config = preprocess_config(config) start = time.mktime( datetime.strptime(options.start_date, "%Y/%m/%d").timetuple()) end = time.mktime( datetime.strptime(options.end_date, "%Y/%m/%d").timetuple()) DataMatrices( start=start, end=end, feature_number=len(options.featureList), window_size=config["input"]["window_size"], online=True, period=config["input"]["global_period"], volume_average_days=config["input"]["volume_average_days"], coin_filter=len(options["stockList"]), is_permed=config["input"]["is_permed"], test_portion=config["input"]["test_portion"], portion_reversed=config["input"]["portion_reversed"]) elif options.mode == "backtest": config = _config_by_algo(options.algo) #读取配置文件 _set_logging_by_algo(logging.DEBUG, logging.DEBUG, options.algo, "backtestlog") #设置log的路径 values = execute_backtest(options.algo, config) #执行回测的步数为训练集的长度 elif options.mode == "save_test_data": # This is used to export the test data save_test_data(load_config(options.folder)) #保存测试集数据 elif options.mode == "plot": logging.basicConfig(level=logging.INFO) algos = options.algos.split(",") if options.labels: labels = options.labels.replace("_", " ") labels = labels.split(",") else: labels = algos plot.plot_backtest(load_config(), algos, labels) elif options.mode == "table": algos = options.algos.split(",") if options.labels: labels = options.labels.replace("_", " ") labels = labels.split(",") else: labels = algos plot.table_backtest(load_config(), algos, labels, format=options.format)
import json with open("pgportfolio/net_config.json") as file: config = json.load(file) config = preprocess_config(config) data = get_test_data(config) self.last_b = self.get_weight(data.T) return self.last_b def opt_weights(X, max_leverage=1): x_0 = max_leverage * np.ones(X.shape[1]) / float(X.shape[1]) objective = lambda b: -np.prod(X.dot(b)) cons = ({'type': 'eq', 'fun': lambda b: max_leverage-np.sum(b)},) bnds = [(0., max_leverage)]*len(x_0) res = minimize(objective, x_0, bounds=bnds, constraints=cons, method='slsqp', options={'ftol': 1e-07}) return res.x if __name__ == '__main__': from pgportfolio.tools.backtest import get_test_data from pgportfolio.tools.configprocess import preprocess_config import json with open("pgportfolio/net_config.json") as file: config = json.load(file) config = preprocess_config(config) data = get_test_data(config) bcrp = BCRP() result = bcrp.get_weight(data.T)