def get_test_data(config): """ :return : a 2d numpy array with shape(coin_number, periods), each element the relative price """ config["input"]["feature_number"] = 1 config["input"]["norm_method"] = "relative" config["input"]["global_period"] = config["input"]["global_period"] price_matrix = DataMatrices.create_from_config(config) test_set = price_matrix.get_test_set()["y"][:, 0, :].T test_set = np.concatenate((np.ones((1, test_set.shape[1])), test_set), axis=0) return test_set
def __init__(self, config, fake_data=False, restore_dir=None, save_path=None, device="cpu", agent=None, data_matrices=None): """ :param config: config dictionary :param fake_data: if True will use data generated randomly :param restore_dir: path to the model trained before :param save_path: path to save the model :param device: the device used to train the network :param agent: the nnagent object. If this is provides, the trainer will not create a new agent by itself. Therefore the restore_dir will not affect anything. """ self.config = config self.train_config = config["training"] self.input_config = config["input"] self.save_path = save_path self.best_metric = 0 np.random.seed(config["random_seed"]) self.__window_size = self.input_config["window_size"] self.__coin_number = self.input_config["coin_number"] self.__batch_size = self.train_config["batch_size"] self.__snap_shot = self.train_config["snap_shot"] config["input"]["fake_data"] = fake_data if not data_matrices: data_matrices = DataMatrices.create_from_config(config) self._matrix = data_matrices self.test_set = self._matrix.get_test_set() if not config["training"]["fast_train"]: self.training_set = self._matrix.get_training_set() self.upperbound_validation = 1 self.upperbound_test = 1 tf.set_random_seed(self.config["random_seed"]) self.device = device if agent: self._agent = agent else: if device == "cpu": os.environ["CUDA_VISIBLE_DEVICES"] = "" with tf.device("/cpu:0"): self._agent = NNAgent(config, restore_dir, device) else: self._agent = NNAgent(config, restore_dir, device)
def __init__(self, config, net_dir=None, agent=None, agent_type="nn"): trader.Trader.__init__(self, 0, config, 0, net_dir, initial_krw=1, agent=agent, agent_type=agent_type) if agent_type == "nn": data_matrices = self._rolling_trainer.data_matrices elif agent_type == "traditional": config["input"]["feature_number"] = 1 data_matrices = DataMatrices.create_from_config(config) else: raise ValueError() self.__test_set = data_matrices.get_test_set() self.__test_length = self.__test_set["X"].shape[0] self._total_steps = self.__test_length self.__test_pv = 1.0 self.__test_pc_vector = []
def __init__(self, config, net_dir=None, agent=None, agent_type="nn"): trader.Trader.__init__(self, 0, config, 0, net_dir, initial_BTC=1, agent=agent, agent_type=agent_type) if agent_type == "nn": data_matrices = self._rolling_trainer.data_matrices elif agent_type == "traditional": config["input"]["feature_number"] = 1 data_matrices = DataMatrices.create_from_config(config) else: raise ValueError() self.__test_set = data_matrices.get_test_set() self.__test_length = self.__test_set["X"].shape[0] self._total_steps = self.__test_length self.__test_pv = 1.0 self.__test_pc_vector = []
def __init__(self, config, fake_data=False, restore_dir=None, save_path=None, device="cpu", agent=None): """ :param config: config dictionary :param fake_data: if True will use data generated randomly :param restore_dir: path to the model trained before :param save_path: path to save the model :param device: the device used to train the network :param agent: the nnagent object. If this is provides, the trainer will not create a new agent by itself. Therefore the restore_dir will not affect anything. """ self.config = config self.train_config = config["training"] self.input_config = config["input"] self.save_path = save_path self.best_metric = 0 np.random.seed(config["random_seed"]) self.__window_size = self.input_config["window_size"] self.__coin_number = self.input_config["coin_number"] self.__batch_size = self.train_config["batch_size"] self.__snap_shot = self.train_config["snap_shot"] config["input"]["fake_data"] = fake_data self._matrix = DataMatrices.create_from_config(config) self.test_set = self._matrix.get_test_set() if not config["training"]["fast_train"]: self.training_set = self._matrix.get_training_set() self.upperbound_validation = 1 self.upperbound_test = 1 tf.set_random_seed(self.config["random_seed"]) self.device = device if agent: self._agent = agent else: if device == "cpu": os.environ["CUDA_VISIBLE_DEVICES"] = "" with tf.device("/cpu:0"): self._agent = NNAgent(config, restore_dir, device) else: self._agent = NNAgent(config, restore_dir, device)
def __init__(self, config, stockList, featureList, start_date, end_date, fake_data, net_dir=None, result_path=None, agent=None, agent_type="nn"): trader.Trader.__init__(self, 0, config, stockList, featureList, start_date, end_date, fake_data, 0, net_dir, result_path, initial_cash=1000000, agent=agent, agent_type=agent_type) if agent_type == "nn": data_matrices = self._rolling_trainer.data_matrices elif agent_type == "traditional": config["input"]["feature_number"] = 1 data_matrices = DataMatrices.create_from_config(config) else: raise ValueError() self.__test_set = data_matrices.get_test_set() self.__test_length = self.__test_set["X"].shape[0] #测试集的长度 self.__test_date = data_matrices.get_test_date() self._total_steps = self.__test_length self.__test_pv = 1.0 self.__test_pc_vector = [] self._window_size = config["input"]["window_size"]
def __init__(self, config, net_dir=None, agent=None, agent_type="nn"): trader.Trader.__init__(self, 0, config, 0, net_dir, initial_BTC=1, agent=agent, agent_type=agent_type) if agent_type == "nn": data_matrices = self._rolling_trainer.data_matrices # construct the data and return matrix elif agent_type == "traditional": config["input"]["feature_number"] = 1 # traditional algorithms only need close prices data_matrices = DataMatrices.create_from_config(config) else: raise ValueError() self.__test_set = data_matrices.get_test_set( ) # backtest only needs test set self.__test_length = self.__test_set["X"].shape[0] self._total_steps = self.__test_length self.__test_pv = 1.0 # total capital self.__test_pc_vector = [] #portfolio change ratio self.__turn_over = []
def main(): parser = build_parser() options = parser.parse_args() if not os.path.exists("./" + "train_package"): os.makedirs("./" + "train_package") if not os.path.exists("./" + "database"): os.makedirs("./" + "database") if options.mode == "train": import pgportfolio.autotrain.training if not options.algo: pgportfolio.autotrain.training.train_all(int(options.processes), options.device) else: for folder in options.train_floder: raise NotImplementedError() elif options.mode == "generate": import pgportfolio.autotrain.generate as generate logging.basicConfig(level=logging.INFO) generate.add_packages(load_config(), int(options.repeat)) elif options.mode == "download_data": from pgportfolio.marketdata.datamatrices import DataMatrices with open("./pgportfolio/net_config.json") as file: config = json.load(file) config = preprocess_config(config) start = time.mktime(datetime.strptime(config["input"]["start_date"], "%Y/%m/%d").timetuple()) end = time.mktime(datetime.strptime(config["input"]["end_date"], "%Y/%m/%d").timetuple()) DataMatrices(start=start, end=end, feature_number=config["input"]["feature_number"], window_size=config["input"]["window_size"], online=True, period=config["input"]["global_period"], volume_average_days=config["input"]["volume_average_days"], coin_filter=config["input"]["coin_number"], is_permed=config["input"]["is_permed"], test_portion=config["input"]["test_portion"], portion_reversed=config["input"]["portion_reversed"]) elif options.mode == "backtest": config = _config_by_algo(options.algo) _set_logging_by_algo(logging.DEBUG, logging.DEBUG, options.algo, "backtestlog") execute_backtest(options.algo, config) elif options.mode == "save_test_data": # This is used to export the test data save_test_data(load_config(options.folder)) elif options.mode == "plot": logging.basicConfig(level=logging.INFO) algos = options.algos.split(",") if options.labels: labels = options.labels.replace("_"," ") labels = labels.split(",") else: labels = algos plot.plot_backtest(load_config(), algos, labels) elif options.mode == "table": algos = options.algos.split(",") if options.labels: labels = options.labels.replace("_"," ") labels = labels.split(",") else: labels = algos plot.table_backtest(load_config(), algos, labels, format=options.format)
elif execute_mode == "download_data": from pgportfolio.marketdata.datamatrices import DataMatrices with open("./pgportfolio/net_config.json") as file: config = json.load(file) config = preprocess_config(config) start = time.mktime( datetime.strptime(config["input"]["start_date"], "%Y/%m/%d").timetuple()) end = time.mktime( datetime.strptime(config["input"]["end_date"], "%Y/%m/%d").timetuple()) DataMatrices(start=start, end=end, feature_number=config["input"]["feature_number"], window_size=config["input"]["window_size"], online=True, period=config["input"]["global_period"], volume_average_days=config["input"]["volume_average_days"], coin_filter=config["input"]["coin_number"], is_permed=config["input"]["is_permed"], test_portion=config["input"]["test_portion"], portion_reversed=config["input"]["portion_reversed"]) # elif execute_mode == "backtest": # config = _config_by_algo(options.algo) # _set_logging_by_algo(logging.DEBUG, logging.DEBUG, options.algo, "backtestlog") # execute_backtest(options.algo, config) # elif execute_mode == "save_test_data": # # This is used to export the test data # save_test_data(load_config(options.folder)) # elif execute_mode == "plot": # logging.basicConfig(level=logging.INFO) # algos = options.algos.split(",")
def __init__(self, config, stockList, featureList, start_date, end_date, fake_data=False, restore_dir=None, save_path=None, device="cpu", agent=None): """ :param config: config dictionary :param fake_data: if True will use data generated randomly :param restore_dir: path to the model trained before :param save_path: path to save the model :param device: the device used to train the network :param agent: the nnagent object. If this is provides, the trainer will not create a new agent by itself. Therefore the restore_dir will not affect anything. """ self.config = config self.train_config = config["training"] self.input_config = config["input"] self.save_path = save_path self.best_metric = 0 np.random.seed(config["random_seed"]) self.__window_size = self.input_config["window_size"] self.__batch_size = self.train_config["batch_size"] self.__snap_shot = self.train_config["snap_shot"] config["input"]["fake_data"] = fake_data self.stockList = stockList self.featureList = featureList self.start_date = start_date self.end_date = end_date self.fake_data = fake_data self._matrix = DataMatrices.create_from_config(config, stockList, featureList, start_date, end_date) #数据 self.test_set = self._matrix.get_test_set( ) #测试集 dict:{'X', 'y', 'last_w', 'setw'} # X: (260, 4, 3, 31), y: (260, 4, 3), last_w: (260, 3) # X: (test_length, feature_num, stock_num, time_windows) if not config["training"]["fast_train"]: self.training_set = self._matrix.get_training_set() #训练集 self.upperbound_validation = 1 self.upperbound_test = 1 tf.set_random_seed(self.config["random_seed"]) self.device = device if agent: self._agent = agent else: if device == "cpu": os.environ["CUDA_VISIBLE_DEVICES"] = "" with tf.device("/cpu:0"): self._agent = NNAgent(config, stockList, featureList, restore_dir, device) else: self._agent = NNAgent(config, stockList, featureList, restore_dir, device)
def __init__(self, config): self.__counter = 0 self.__start = parse_time(config["input"]["start_date"]) self.__end = parse_time(config["input"]["end_date"]) self.__number_of_coins = config["input"]["coin_number"] self.__batch_size = config["training"]["batch_size"] self.__window_size = config["input"]["window_size"]+1 span = self.__end - self.__start self.__end = self.__end - config["input"]["test_portion"] * span config2 = copy.deepcopy(config) config2["input"]["global_period"] = 300 self._matrix2 = DataMatrices.create_from_config(config2) self.__weightgarch = pd.DataFrame(index=range(0, self.__batch_size), columns=range(0, self.__number_of_coins)) self.__weightgarch = self.__weightgarch.fillna(1.0 / self.__number_of_coins) training_set = self._matrix2.get_training_set() set = training_set["X"] #good times sets for 3, 26: 1 and 2 are not functioning set5 = set[-5000:, 0, :, 0] set3 = set[-3000:, 0, :, 0] set2 = set[-2000:, 0, :, 0] #self.__abcdefg = set[:, 0] self.__lastvalue1 = set5[0, 0] self.__lastvalue2 = set5[0, 1] self.__lastvalue3 = set2[0, 2] self.__lastvalue4 = set5[0, 3] self.__lastvalue5 = set3[0, 4] self.__lastvalue6 = set3[0, 5] self.__lastvalue7 = set3[0, 6] self.__lastvalue8 = set5[0, 7] self.__lastvalue9 = set5[0, 8] self.__lastvalue10 = set5[0, 9] self.__lastvalue11 = set3[0, 10] #good times set5, bad times set3 #logreturns1 = np.log(set3[1:, 0] / set3[:-1, 0]) logreturns1 = np.log(set5[1:, 0] / set5[:-1, 0]) self.__lastsigma1 = np.sqrt(np.mean(logreturns1 ** 2)) self.__lastlogreturn1 = logreturns1[-1] #good times set5, bad times set3 #logreturns2 = np.log(set3[1:, 1] / set3[:-1, 1]) logreturns2 = np.log(set5[1:, 1] / set5[:-1, 1]) self.__lastsigma2 = np.sqrt(np.mean(logreturns2 ** 2)) self.__lastlogreturn2 = logreturns2[-1] logreturns3 = np.log(set2[1:, 2] / set2[:-1, 2]) self.__lastsigma3 = np.sqrt(np.mean(logreturns3 ** 2)) self.__lastlogreturn3 = logreturns3[-1] logreturns4 = np.log(set5[1:, 3] / set5[:-1, 3]) self.__lastsigma4 = np.sqrt(np.mean(logreturns4 ** 2)) self.__lastlogreturn4 = logreturns4[-1] logreturns5 = np.log(set3[1:, 4] / set3[:-1, 4]) self.__lastsigma5 = np.sqrt(np.mean(logreturns5 ** 2)) self.__lastlogreturn5 = logreturns5[-1] logreturns6 = np.log(set3[1:, 5] / set3[:-1, 5]) self.__lastsigma6 = np.sqrt(np.mean(logreturns6 ** 2)) self.__lastlogreturn6 = logreturns6[-1] logreturns7 = np.log(set3[1:, 6] / set3[:-1, 6]) self.__lastsigma7 = np.sqrt(np.mean(logreturns7 ** 2)) self.__lastlogreturn7 = logreturns7[-1] logreturns8 = np.log(set5[1:, 7] / set5[:-1, 7]) self.__lastsigma8 = np.sqrt(np.mean(logreturns8 ** 2)) self.__lastlogreturn8 = logreturns8[-1] logreturns9 = np.log(set5[1:, 8] / set5[:-1, 8]) self.__lastsigma9 = np.sqrt(np.mean(logreturns9 ** 2)) self.__lastlogreturn9 = logreturns9[-1] logreturns10 = np.log(set5[1:, 9] / set5[:-1, 9]) self.__lastsigma10 = np.sqrt(np.mean(logreturns10 ** 2)) self.__lastlogreturn10 = logreturns10[-1] logreturns11 = np.log(set3[1:, 10] / set3[:-1, 10]) self.__lastsigma11 = np.sqrt(np.mean(logreturns11 ** 2)) self.__lastlogreturn11 = logreturns11[-1] self.__firstsigma1 = self.__lastsigma1 self.__firstsigma2 = self.__lastsigma2 self.__firstsigma3 = self.__lastsigma3 self.__firstsigma4 = self.__lastsigma4 self.__firstsigma5 = self.__lastsigma5 self.__firstsigma6 = self.__lastsigma6 self.__firstsigma7 = self.__lastsigma7 self.__firstsigma8 = self.__lastsigma8 self.__firstsigma9 = self.__lastsigma9 self.__firstsigma10 = self.__lastsigma10 self.__firstsigma11 = self.__lastsigma11 self.__firstlogreturn1 = self.__lastlogreturn1 self.__firstlogreturn2 = self.__lastlogreturn2 self.__firstlogreturn3 = self.__lastlogreturn3 self.__firstlogreturn4 = self.__lastlogreturn4 self.__firstlogreturn5 = self.__lastlogreturn5 self.__firstlogreturn6 = self.__lastlogreturn6 self.__firstlogreturn7 = self.__lastlogreturn7 self.__firstlogreturn8 = self.__lastlogreturn8 self.__firstlogreturn9 = self.__lastlogreturn9 self.__firstlogreturn10 = self.__lastlogreturn10 self.__firstlogreturn11 = self.__lastlogreturn11 self.__firstlastvalue1 = self.__lastvalue1 self.__firstlastvalue2 = self.__lastvalue2 self.__firstlastvalue3 = self.__lastvalue3 self.__firstlastvalue4 = self.__lastvalue4 self.__firstlastvalue5 = self.__lastvalue5 self.__firstlastvalue6 = self.__lastvalue6 self.__firstlastvalue7 = self.__lastvalue7 self.__firstlastvalue8 = self.__lastvalue8 self.__firstlastvalue9 = self.__lastvalue9 self.__firstlastvalue10 = self.__lastvalue10 self.__firstlastvalue11 = self.__lastvalue11 self.__theta1 = self.negative_log_likelihood(logreturns1, (1, 0.5, 0.5)) self.__theta1 = self.fitting(logreturns1) print(self.__theta1) self.__theta2 = self.fitting(logreturns2) print(self.__theta2) self.__theta3 = self.fitting(logreturns3) print(self.__theta3) self.__theta4 = self.fitting(logreturns4) print(self.__theta4) self.__theta5 = self.fitting(logreturns5) print(self.__theta5) self.__theta6 = self.fitting(logreturns6) print(self.__theta6) self.__theta7 = self.fitting(logreturns7) print(self.__theta7) self.__theta8 = self.fitting(logreturns8) print(self.__theta8) self.__theta9 = self.fitting(logreturns9) print(self.__theta9) self.__theta10 = self.fitting(logreturns10) print(self.__theta10) self.__theta11 = self.fitting(logreturns11) print(self.__theta11)
def main(logPath, device): parser = build_parser() options = parser.parse_args() ''' options.folder = oneKey options.stockList = stockList options.featureList = featureList options.start_date = startDate options.end_date = endDate ''' if not os.path.exists("./" + "database"): os.makedirs("./" + "database") #options.repeat = 1 if options.mode == "train": #训练数据 if not options.algo: save_path = logPath + str(options.folder) + "/netfile" # 读取配置文件 with open(logPath + str(options.folder) + "\\net_config.json") as file: config_json = None config_json = json.load(file) config = preprocess_config(config_json) log_file_dir = logPath + str(options.folder) + "/tensorboard" # 定义错误等级 logfile_level = logging.DEBUG console_level = logging.INFO logging.basicConfig(filename=log_file_dir.replace( "tensorboard", "programlog"), level=logfile_level) console = logging.StreamHandler() console.setLevel(console_level) logging.getLogger().addHandler(console) trainer = TraderTrainer(config, options.stockList, options.featureList, options.start_date, options.end_date, save_path=save_path, device=device) #初始化训练器 trainer.train_net(log_file_dir=log_file_dir, index=str(options.folder)) #训练网络 else: for folder in options.folder: raise NotImplementedError() # 生成配置文件到路径中,要想修改配置,直接修改PGPortfolio\pgportfolio\net_config.json elif options.mode == "generate": import pgportfolio.autotrain.generate as generate logging.basicConfig(level=logging.INFO) config_ = load_config() train_dir = logPath generate.add_packages(train_dir, load_config(), int(options.repeat)) elif options.mode == "download_data": from pgportfolio.marketdata.datamatrices import DataMatrices with open("./pgportfolio/net_config.json") as file: config = json.load(file) config = preprocess_config(config) start = time.mktime( datetime.strptime(options.start_date, "%Y/%m/%d").timetuple()) end = time.mktime( datetime.strptime(options.end_date, "%Y/%m/%d").timetuple()) DataMatrices( start=start, end=end, feature_number=len(options.featureList), window_size=config["input"]["window_size"], online=True, period=config["input"]["global_period"], volume_average_days=config["input"]["volume_average_days"], coin_filter=len(options["stockList"]), is_permed=config["input"]["is_permed"], test_portion=config["input"]["test_portion"], portion_reversed=config["input"]["portion_reversed"]) elif options.mode == "backtest": config = _config_by_algo(options.algo) #读取配置文件 _set_logging_by_algo(logging.DEBUG, logging.DEBUG, options.algo, "backtestlog") #设置log的路径 values = execute_backtest(options.algo, config) #执行回测的步数为训练集的长度 elif options.mode == "save_test_data": # This is used to export the test data save_test_data(load_config(options.folder)) #保存测试集数据 elif options.mode == "plot": logging.basicConfig(level=logging.INFO) algos = options.algos.split(",") if options.labels: labels = options.labels.replace("_", " ") labels = labels.split(",") else: labels = algos plot.plot_backtest(load_config(), algos, labels) elif options.mode == "table": algos = options.algos.split(",") if options.labels: labels = options.labels.replace("_", " ") labels = labels.split(",") else: labels = algos plot.table_backtest(load_config(), algos, labels, format=options.format)
else: drawdown_list.append(1.0 - pv_array[i] / max_benefit) return max(drawdown_list) def port_value(pv_array): p = np.array([np.prod(pv_array[:i + 1]) for i in range(pv_array.shape[0])]) return p #%% load config & data config_2 = load_config(2) # config_2['input']['start_date'] = '2015/05/01' # config_2['input']['end_date'] = '2017/04/27' # 自訂區間 data = DataMatrices.create_from_config(config_2) #%% network structure feature_number = config_2["input"]["feature_number"] rows = config_2["input"]["coin_number"] columns = config_2["input"]["window_size"] input_tensor = tf.placeholder(tf.float32, shape=[None, feature_number, rows, columns]) #[None, 3, 11, 31] previous_w = tf.placeholder(tf.float32, shape=[None, rows]) #[None, 11] input_num = tf.placeholder(tf.int32, shape=[]) #[11] y = tf.placeholder(tf.float32, shape=[None, feature_number, rows]) def allint(l):