def DRL_prediction(df, model, name, last_state, iter_num, unique_trade_date, rebalance_window, turbulence_threshold, initial): ### make a prediction based on trained model### ## trading env print("DRLPREDICTION") trade_data = data_split(df, start=unique_trade_date[iter_num - rebalance_window], end=unique_trade_date[iter_num]) env_trade = DummyVecEnv([ lambda: StockEnvTrade(trade_data, turbulence_threshold=turbulence_threshold, initial=initial, previous_state=last_state, model_name=name, iteration=iter_num) ]) obs_trade = env_trade.reset() for i in range(len(trade_data.index.unique())): action, _states = model.predict(obs_trade) obs_trade, rewards, dones, info = env_trade.step(action) if i == (len(trade_data.index.unique()) - 2): # print(env_test.render()) last_state = env_trade.render() df_last_state = pd.DataFrame({'last_state': last_state}) df_last_state.to_csv('results/last_state_{}_{}.csv'.format(name, i), index=False) return last_state
def DRL_prediction_no_rebalance(df, model, name, unique_trade_date, log_interval): ### make a prediction based on trained model### ## trading env print("DRL PREDICTION NO REBALANCE") all_data = data_split(df, start=unique_trade_date[0], end=unique_trade_date[len(unique_trade_date) - 1]) for ticker in all_data["tic"].unique(): trade_data = all_data[all_data["tic"] == ticker] env_trade = DummyVecEnv([ lambda: StockEnvTrade(trade_data, initial=True, model_name=name + "_" + ticker, log_interval=log_interval) ]) obs_trade = env_trade.reset() for i in range(len(trade_data.index.unique())): action, _states = model.predict(obs_trade) print("action: ", action) obs_trade, rewards, dones, info = env_trade.step(action) if i == (len(trade_data.index.unique()) - 2): # print(env_test.render()) last_state = env_trade.render() # df_last_state = pd.DataFrame({'last_state': last_state}) # df_last_state.to_csv('results/last_state_{}_{}.csv'.format(name, i), index=False) return last_state
def train_initial_model(train_func, df, timesteps, model_name, save_path): print("============Start Training Initial Model============") train = data_split(df, start=20090000, end=20151001) env_train = DummyVecEnv([lambda: StockEnvTrain(train)]) return train_func(env_train, model_name, timesteps=timesteps, save_path=save_path)
def train_multitask(df, unique_trade_date, timesteps=10, policy="MlpPolicy", model_name="multitask"): # df of all intermixed values # get out the individual tickers and switch out the dates # timesteps = num training steps per date start = time.time() df = data_split(df, start=unique_trade_date[0], end=unique_trade_date[len(unique_trade_date) - 1]) last_state, initial = [], True model = None for i in range(len(unique_trade_date) - 2): for ticker in df["tic"].unique(): # Interval is every two days so we can optimize on the change in account value start_date, end_date = unique_trade_date[i], unique_trade_date[i + 2] quanta_df = data_split(df, start=start_date, end=end_date) quanta_df = quanta_df[quanta_df["tic"] == ticker] if len(quanta_df.index) < 2: continue quanta_df = quanta_df.reset_index() quanta_env = DummyVecEnv([ lambda: StockEnvTrade(quanta_df, previous_state=last_state, initial=initial, log_interval=1) ]) quanta_env.reset() model = train_PPO_update(model, quanta_env, timesteps, policy=policy) last_state = quanta_env.render() initial = False model.save(f"{config.TRAINED_MODEL_DIR}/{model_name}") end = time.time() print('Training time (Multitask): ', (end - start) / 60, ' minutes') return model
def train_PPO_models(stocks=['./data/TSLA.csv', './data/FB.csv'], tickers=['TSLA', 'FB'], start_date=20130102, end_date=20180101, model_save_path='/content/DeepRL4Stocks/trained_models/'): teachers = [] envs = [] for i in range(len(stocks)): print(i) df = process_yahoo_finance(stocks[i], tickers[i]) train = data_split(df, start=start_date, end=end_date) env_train = DummyVecEnv([lambda: StockEnvTrade(train)]) model = train_PPO(env_train, model_name='PPO_' + tickers[i]) model.save(model_save_path + "PPO_" + tickers[i]) teachers.append(model) envs.append(env_train) return teachers, envs
def test_model(df, model, model_name, turbulence_threshold=140, start=20151001, end=20200707): trade_data = data_split(df, start=start, end=end) env_trade = DummyVecEnv([ lambda: StockEnvTrade(trade_data, turbulence_threshold=turbulence_threshold, initial=True, previous_state=[], model_name=model_name, iteration=0) ]) obs_trade = env_trade.reset() state = None dones = [False for _ in range(env_trade.num_envs)] for i in range(len(trade_data.index.unique())): action, state = model.predict(obs_trade, state=state, mask=dones) obs_trade, rewards, dones, info = env_trade.step(action) return info[0]
def run_ensemble_strategy( df, unique_trade_date, rebalance_window, validation_window ) -> None: """Ensemble Strategy that combines PPO, A2C and DDPG""" # for ensemble model, it's necessary to feed the last state # of the previous model to the current model as the initial state last_state_ensemble = [] ppo_sharpe_list = [] ddpg_sharpe_list = [] a2c_sharpe_list = [] model_used = [] # based on the analysis of the in-sample data # turbulence_threshold = 140 insample_turbulence = df[ (df.datadate < config.VALIDATION_START_DATE - 1) & (df.datadate >= config.TRAINING_START_DATE) ] insample_turbulence = insample_turbulence.drop_duplicates(subset=["datadate"]) insample_turbulence_threshold = np.quantile( insample_turbulence.turbulence.values, 0.90 ) start = time.time() for i in range( rebalance_window + validation_window, len(unique_trade_date), rebalance_window ): ## initial state is empty if i - rebalance_window - validation_window == 0: # inital state initial = True else: # previous state initial = False # Tuning turbulence index based on historical data # Turbulence lookback window is one quarter end_date_index = df.index[ df["datadate"] == unique_trade_date[i - rebalance_window - validation_window] ].to_list()[-1] start_date_index = end_date_index - validation_window * 30 + 1 historical_turbulence = df.iloc[start_date_index : (end_date_index + 1), :] # historical_turbulence = df[(df.datadate<unique_trade_date[i - rebalance_window - validation_window]) & (df.datadate>=(unique_trade_date[i - rebalance_window - validation_window - 63]))] historical_turbulence = historical_turbulence.drop_duplicates( subset=["datadate"] ) historical_turbulence_mean = np.mean(historical_turbulence.turbulence.values) if historical_turbulence_mean > insample_turbulence_threshold: # if the mean of the historical data is greater than the 90% quantile of insample turbulence data # then we assume that the current market is volatile, # therefore we set the 90% quantile of insample turbulence data as the turbulence threshold # meaning the current turbulence can't exceed the 90% quantile of insample turbulence data turbulence_threshold = insample_turbulence_threshold else: # if the mean of the historical data is less than the 90% quantile of insample turbulence data # then we tune up the turbulence_threshold, meaning we lower the risk turbulence_threshold = np.quantile(insample_turbulence.turbulence.values, 1) style = "[bold #31DDCF]" rprint( Align( f"{style}Turbulence Threshold:[/] {str(turbulence_threshold)}", "center" ) ) ############## Environment Setup starts ############## ## training env train = data_split( df, start=config.TRAINING_START_DATE, end=unique_trade_date[i - rebalance_window - validation_window], ) env_train = DummyVecEnv([lambda: StockEnvTrain(train)]) ## validation env validation = data_split( df, start=unique_trade_date[i - rebalance_window - validation_window], end=unique_trade_date[i - rebalance_window], ) env_val = DummyVecEnv( [ lambda: StockEnvValidation( validation, turbulence_threshold=turbulence_threshold, iteration=i ) ] ) obs_val = env_val.reset() ############## Environment Setup ends ############## ############## Training and Validation starts ############## table = Table( title=f"Training from 20090000 to {unique_trade_date[i - rebalance_window - validation_window]}", expand=True, ) table.add_column("Mode Name", justify="center") table.add_column("Sharpe Ratio") table.add_column("Training Time") with Live(table, auto_refresh=False) as live: model_a2c, a2c_training_time = train_A2C( env_train, model_name="A2C_30k_dow_{}".format(i), timesteps=30000 ) DRL_validation( model=model_a2c, test_data=validation, test_env=env_val, test_obs=obs_val, ) sharpe_a2c = get_validation_sharpe(i) table.add_row("A2C", str(sharpe_a2c), f"{a2c_training_time} minutes") live.update(table, refresh=True) model_ppo, ppo_training_time = train_PPO( env_train, model_name="PPO_100k_dow_{}".format(i), timesteps=100000 ) DRL_validation( model=model_ppo, test_data=validation, test_env=env_val, test_obs=obs_val, ) sharpe_ppo = get_validation_sharpe(i) table.add_row("PPO", str(sharpe_ppo), f"{ppo_training_time} minutes") live.update(table, refresh=True) model_ddpg, ddpg_training_time = train_DDPG( env_train, model_name="DDPG_10k_dow_{}".format(i), timesteps=10000 ) # model_ddpg = train_TD3(env_train, model_name="DDPG_10k_dow_{}".format(i), timesteps=20000) DRL_validation( model=model_ddpg, test_data=validation, test_env=env_val, test_obs=obs_val, ) sharpe_ddpg = get_validation_sharpe(i) table.add_row("DDPG", str(sharpe_ddpg), f"{ddpg_training_time} minutes") live.update(table, refresh=True) ppo_sharpe_list.append(sharpe_ppo) a2c_sharpe_list.append(sharpe_a2c) ddpg_sharpe_list.append(sharpe_ddpg) # Model Selection based on sharpe ratio if (sharpe_ppo >= sharpe_a2c) & (sharpe_ppo >= sharpe_ddpg): model_ensemble = model_ppo model_used.append("PPO") elif (sharpe_a2c > sharpe_ppo) & (sharpe_a2c > sharpe_ddpg): model_ensemble = model_a2c model_used.append("A2C") else: model_ensemble = model_ddpg model_used.append("DDPG") ############## Training and Validation ends ############## ############## Trading starts ############## # print("Used Model: ", model_ensemble) last_state_ensemble = DRL_prediction( df=df, model=model_ensemble, name="ensemble", last_state=last_state_ensemble, iter_num=i, unique_trade_date=unique_trade_date, rebalance_window=rebalance_window, turbulence_threshold=turbulence_threshold, initial=initial, ) print("\n\n") # print("============Trading Done============") ############## Trading ends ############## end = time.time() print("Ensemble Strategy took: ", (end - start) / 60, " minutes")