def train_one():
env_kwargs, processed = prepare_data()
# Training & Trading data split
train = data_split(processed, config.START_DATE, config.START_TRADE_DATE)
trade = data_split(processed, config.START_TRADE_DATE, config.END_DATE)
e_train_gym = StockTradingEnv(df=train, **env_kwargs)
e_trade_gym = StockTradingEnv(df=trade,
turbulence_threshold=250,
**env_kwargs)
env_train, _ = e_train_gym.get_sb_env()
agent = DRLAgent(env=env_train)
print("==============Model Training===========")
model = agent.get_model(config.CURRENT_MODEL)
model = agent.train_model(model=model, total_timesteps=80000)
path = f"{config.TRAINED_MODEL_DIR}/model"
model.save(path)
model = model.load(path)
print("==============Start Trading===========")
df_account_value, df_actions = DRLAgent.DRL_prediction(
model=model, environment=e_trade_gym)
log_account(df_account_value, df_actions)
def new_test():
processed = pd.read_csv(
os.path.abspath('./me/datasets/new_data_with_techs_turb.csv'),
index_col=0)
train = data_split(processed, '2009-01-01', '2018-01-01')
trade = data_split(processed, '2018-01-01', '2021-01-01')
stock_dimension = len(train.tic.unique())
state_space = 1 + 2 * stock_dimension + len(
config.TECHNICAL_INDICATORS_LIST) * stock_dimension
print(f"Stock Dimension: {stock_dimension}, State Space: {state_space}")
env_kwargs = {
"hmax": 100,
"initial_amount": 1000000,
"transaction_cost_pct": 0.001,
"state_space": state_space,
"stock_dim": stock_dimension,
"tech_indicator_list": config.TECHNICAL_INDICATORS_LIST,
"action_space": stock_dimension,
"reward_scaling": 1e-4
}
e_train_gym = StockTradingEnv(df=train, **env_kwargs)
env_train, _ = e_train_gym.get_sb_env()
log_dir = "me/tmp/"
os.makedirs(log_dir, exist_ok=True)
env_train.envs[0] = Monitor(env_train.envs[0], log_dir)
agent = DRLAgent(env=env_train)
model_a2c = agent.get_model("a2c", verbose=0)
trained_a2c = agent.train_model(model=model_a2c,
tb_log_name='a2c',
total_timesteps=100000)
data_turbulence = processed[(processed.date < '2018-01-01')
& (processed.date >= '2009-01-01')]
insample_turbulence = data_turbulence.drop_duplicates(subset=['date'])
turbulence_threshold = np.quantile(insample_turbulence.turbulence.values,
1)
e_trade_gym = StockTradingEnv(df=trade,
turbulence_threshold=380,
**env_kwargs)
env_trade, obs_trade = e_trade_gym.get_sb_env()
print("BEGIN PREDICTION")
df_account_value, df_actions = DRLAgent.DRL_prediction(model=trained_a2c,
test_data=trade,
test_env=env_trade,
test_obs=obs_trade)
print(df_account_value)
print("END PREDICTION")
def predict():
env_kwargs, processed = prepare_data()
trade = data_split(processed, config.START_TRADE_DATE, config.END_DATE)
e_trade_gym = StockTradingEnv(df=trade, **env_kwargs)
path = config.TRAINED_MODEL_DIR + "/model"
trained_sac = eval(config.CURRENT_MODEL.upper() + ".load(path)")
df_account_value, df_actions = DRLAgent.DRL_prediction(
model=trained_sac, environment=e_trade_gym)
log_account(df_account_value, df_actions)
def get_trade_results(env_setup, model):
# # --------------- Trading
env_trade, obs_trade = env_setup.create_env_trading(
data=trading_data, env_class=StockEnvTrade, turbulence_threshold=230)
# # --------------- Predict
df_account_value, df_actions = DRLAgent.DRL_prediction(
model=model,
test_data=trading_data,
test_env=env_trade,
test_obs=obs_trade)
return df_account_value, df_actions
def train_one():
"""
train an agent
"""
print("==============Start Fetching Data===========")
df = YahooDownloader(start_date=config.START_DATE,
end_date=config.END_DATE,
ticker_list=config.DOW_30_TICKER).fetch_data()
print("==============Start Feature Engineering===========")
df = FeatureEngineer(df,
feature_number=5,
use_technical_indicator=True,
use_turbulence=True).preprocess_data()
train = data_split(df, config.START_DATE, config.START_TRADE_DATE)
trade = data_split(df, config.START_TRADE_DATE, config.END_DATE)
env_setup = EnvSetup(stock_dim=len(train.tic.unique()))
env_train = env_setup.create_env_training(data=train,
env_class=StockEnvTrain)
agent = DRLAgent(env=env_train)
print("==============Model Training===========")
now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')
a2c_params_tuning = {
'n_steps': 5,
'ent_coef': 0.005,
'learning_rate': 0.0007,
'verbose': 0,
'timesteps': 100000
}
model_a2c = agent.train_A2C(model_name="A2C_{}".format(now),
model_params=a2c_params_tuning)
print("==============Start Trading===========")
env_trade, obs_trade = env_setup.create_env_trading(
data=trade, env_class=StockEnvTrade, turbulence_threshold=250)
df_account_value = DRLAgent.DRL_prediction(model=model_a2c,
test_data=trade,
test_env=env_trade,
test_obs=obs_trade)
df_account_value.to_csv("./" + config.RESULTS_DIR + "/" + now + '.csv')
print("==============Get Backtest Results===========")
perf_stats_all = BackTestStats(df_account_value)
perf_stats_all = pd.DataFrame(perf_stats_all)
perf_stats_all.to_csv("./" + config.RESULTS_DIR + "/perf_stats_all_" +
now + '.csv')
# # DRL model needs to update periodically in order to take full advantage of the data, ideally we need to retrain our model yearly, quarterly, or monthly. We also need to tune the parameters along the way, # in this notebook I only use the in-sample data from 2009-01 to 2018-12 to tune the parameters once, so there is some alpha decay here as the length of trade date extends. # # Numerous hyperparameters – e.g. the learning rate, the total number of samples to train on – influence the learning process and are usually determined by testing some variations. # In[61]: trade.head() # In[192]: e_trade_gym.hmax = 2500 # In[193]: print(len(e_trade_gym.dates)) # In[194]: df_account_value, df_actions = DRLAgent.DRL_prediction(model=model,environment = e_trade_gym) # In[195]: df_account_value.shape # In[196]: df_account_value.head(50) # <a id='6'></a> # # Part 7: Backtest Our Strategy # Backtesting plays a key role in evaluating the performance of a trading strategy. Automated backtesting tool is preferred because it reduces the human error. We usually use the Quantopian pyfolio package to backtest our trading strategies. It is easy to use and consists of various individual plots that provide a comprehensive image of the performance of a trading strategy. # <a id='6.1'></a> # ## 7.1 BackTestStats # pass in df_account_value, this information is stored in env class
def train_one():
"""
train an agent
"""
print("==============Start Fetching Data===========")
df = YahooDownloader(start_date=config.START_DATE,
end_date=config.END_DATE,
ticker_list=config.SP_500_TICKER).fetch_data()
print("==============Start Feature Engineering===========")
df = FeatureEngineer(df, use_technical_indicator=True,
use_turbulence=True).preprocess_data()
# Training & Trade data split
train = data_split(df, config.START_DATE, config.START_TRADE_DATE)
trade = data_split(df, config.START_TRADE_DATE, config.END_DATE)
# data normalization
#feaures_list = list(train.columns)
#feaures_list.remove('date')
#feaures_list.remove('tic')
#feaures_list.remove('close')
#print(feaures_list)
#data_normaliser = preprocessing.StandardScaler()
#train[feaures_list] = data_normaliser.fit_transform(train[feaures_list])
#trade[feaures_list] = data_normaliser.fit_transform(trade[feaures_list])
# calculate state action space
stock_dimension = len(train.tic.unique())
state_space = 1 + 2 * stock_dimension + len(
config.TECHNICAL_INDICATORS_LIST) * stock_dimension
env_setup = EnvSetup(stock_dim=stock_dimension,
state_space=state_space,
hmax=100,
initial_amount=3000,
transaction_cost_pct=0.001)
env_train = env_setup.create_env_training(data=train,
env_class=StockEnvTrain)
agent = DRLAgent(env=env_train)
print("==============Model Training===========")
now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')
a2c_params_tuning = {
'n_steps': 5,
'ent_coef': 0.005,
'learning_rate': 0.0007,
'verbose': 0,
'timesteps': 80000
}
model = agent.train_A2C(model_name="A2C_{}".format(now),
model_params=a2c_params_tuning)
print("==============Start Trading===========")
env_trade, obs_trade = env_setup.create_env_trading(
data=trade, env_class=StockEnvTrade, turbulence_threshold=250)
df_account_value, df_actions = DRLAgent.DRL_prediction(model=model,
test_data=trade,
test_env=env_trade,
test_obs=obs_trade)
df_account_value.to_csv("./" + config.RESULTS_DIR + "/df_account_value_" +
now + '.csv')
df_actions.to_csv("./" + config.RESULTS_DIR + "/df_actions_" + now +
'.csv')
print("==============Get Backtest Results===========")
perf_stats_all = BackTestStats(df_account_value)
perf_stats_all = pd.DataFrame(perf_stats_all)
perf_stats_all.to_csv("./" + config.RESULTS_DIR + "/perf_stats_all_" +
now + '.csv')
trade.head()
# In[25]:
## make a prediction and get the account value change
trade = data_split(data_df, start = '2019-01-01', end = '2021-01-01')
e_trade_gym = StockTradingEnv(df = trade, **env_kwargs)
env_trade, obs_trade = e_trade_gym.get_sb_env()
df_account_value, df_actions = DRLAgent.DRL_prediction(model=trained_sac,
test_data = trade,
test_env = env_trade,
test_obs = obs_trade)
# <a id='6'></a>
# # Part 7: Backtesting Performance
# Backtesting plays a key role in evaluating the performance of a trading strategy. Automated backtesting tool is preferred because it reduces the human error. We usually use the Quantopian pyfolio package to backtest our trading strategies. It is easy to use and consists of various individual plots that provide a comprehensive image of the performance of a trading strategy.
# <a id='6.1'></a>
# ## 7.1 BackTestStats
# pass in df_account_value, this information is stored in env class
#
# In[26]:
def train_one():
"""
train an agent
"""
print("==============Start Fetching Data===========")
df = YahooDownloader(
start_date=config.START_DATE,
end_date=config.END_DATE,
ticker_list=config.DOW_30_TICKER,
).fetch_data()
print("==============Start Feature Engineering===========")
fe = FeatureEngineer(
use_technical_indicator=True,
tech_indicator_list=config.TECHNICAL_INDICATORS_LIST,
use_turbulence=True,
user_defined_feature=False,
)
processed = fe.preprocess_data(df)
# Training & Trading data split
train = data_split(processed, config.START_DATE, config.START_TRADE_DATE)
trade = data_split(processed, config.START_TRADE_DATE, config.END_DATE)
# calculate state action space
stock_dimension = len(train.tic.unique())
state_space = (1 + 2 * stock_dimension +
len(config.TECHNICAL_INDICATORS_LIST) * stock_dimension)
env_kwargs = {
"hmax": 100,
"initial_amount": 1000000,
"buy_cost_pct": 0.001,
"sell_cost_pct": 0.001,
"state_space": state_space,
"stock_dim": stock_dimension,
"tech_indicator_list": config.TECHNICAL_INDICATORS_LIST,
"action_space": stock_dimension,
"reward_scaling": 1e-4
}
e_train_gym = StockTradingEnv(df=train, **env_kwargs)
e_trade_gym = StockTradingEnv(df=trade,
turbulence_threshold=250,
**env_kwargs)
env_train, _ = e_train_gym.get_sb_env()
env_trade, obs_trade = e_trade_gym.get_sb_env()
agent = DRLAgent(env=env_train)
print("==============Model Training===========")
now = datetime.datetime.now().strftime("%Y%m%d-%Hh%M")
model_sac = agent.get_model("sac")
trained_sac = agent.train_model(model=model_sac,
tb_log_name="sac",
total_timesteps=80000)
print("==============Start Trading===========")
df_account_value, df_actions = DRLAgent.DRL_prediction(model=trained_sac,
test_data=trade,
test_env=env_trade,
test_obs=obs_trade)
df_account_value.to_csv("./" + config.RESULTS_DIR + "/df_account_value_" +
now + ".csv")
df_actions.to_csv("./" + config.RESULTS_DIR + "/df_actions_" + now +
".csv")
print("==============Get Backtest Results===========")
perf_stats_all = BackTestStats(df_account_value)
perf_stats_all = pd.DataFrame(perf_stats_all)
perf_stats_all.to_csv("./" + config.RESULTS_DIR + "/perf_stats_all_" +
now + ".csv")
def train_one():
"""
train an agent
"""
print("==============Start Fetching Data===========")
df = YahooDownloader(
start_date=config.START_DATE,
end_date=config.END_DATE,
ticker_list=['FXAIX'],
).fetch_data()
print("==============Start Feature Engineering===========")
fe = FeatureEngineer(
use_technical_indicator=True,
tech_indicator_list=config.TECHNICAL_INDICATORS_LIST,
use_turbulence=True,
user_defined_feature=False,
)
processed = fe.preprocess_data(df)
# Training & Trading data split
train = data_split(processed, config.START_DATE, config.START_TRADE_DATE)
trade = data_split(processed, config.START_TRADE_DATE, config.END_DATE)
# calculate state action space
stock_dimension = len(train.tic.unique())
state_space = (1 + 2 * stock_dimension +
len(config.TECHNICAL_INDICATORS_LIST) * stock_dimension)
env_kwargs = {
"hmax": 100,
"initial_amount": 1000000,
"buy_cost_pct": 0.001,
"sell_cost_pct": 0.001,
"state_space": state_space,
"stock_dim": stock_dimension,
"tech_indicator_list": config.TECHNICAL_INDICATORS_LIST,
"action_space": stock_dimension,
"reward_scaling": 1e-4
}
e_train_gym = StockTradingEnv(df=train, **env_kwargs)
e_trade_gym = StockTradingEnv(df=trade,
turbulence_threshold=250,
**env_kwargs)
env_train, _ = e_train_gym.get_sb_env()
env_trade, obs_trade = e_trade_gym.get_sb_env()
agent = DRLAgent(env=env_train)
print("==============Model Training===========")
now = datetime.datetime.now().strftime("%Y%m%d-%Hh%M")
user_input = input('train model? 1 train 0 don\'t train')
if user_input == 1:
model_sac = agent.get_model("sac")
trained_sac = agent.train_model(model=model_sac,
tb_log_name="sac",
total_timesteps=8000)
trained_sac.save("../models/sac_8k" + df.tic[0] + "_frl")
else:
trained_sac = SAC.load('../models/sac_80k_msft_working')
print("==============Start Trading===========")
df_account_value, df_actions = DRLAgent.DRL_prediction(
trained_sac, e_trade_gym)
df_account_value.to_csv("../" + config.RESULTS_DIR +
"/SAC_df_account_value_" + df.tic[0] + "_" + now +
".csv")
df_actions.to_csv("../" + config.RESULTS_DIR + "/SAC_df_actions_" +
df.tic[0] + "_" + now + ".csv")
# print("==============Get Backtest Results===========")
perf_stats_all = backtest_stats(df_account_value)
perf_stats_all = pd.DataFrame(perf_stats_all)
perf_stats_all.to_csv("../" + config.RESULTS_DIR + "/SAC_perf_stats_all_" +
df.tic[0] + "_" + now + ".csv")
#plot acc value
actions = df_actions['actions']
x = np.arange(0, df_account_value['account_value'].shape[0])
y = df_account_value['account_value']
points = np.array([x, y]).T.reshape(-1, 1, 2)
segments = np.concatenate([points[:-1], points[1:]], axis=1)
fig, axs = plt.subplots(2, 1, sharex=True, sharey=False)
# plt.plot(x, y)
# Use a boundary norm instead
cmap = ListedColormap(['r', 'g', 'b'])
norm = BoundaryNorm([-100, -0.1, 0.1, 100], cmap.N)
lc = LineCollection(segments, cmap=cmap, norm=norm)
lc.set_array(actions)
lc.set_linewidth(2)
line = axs[0].add_collection(lc)
# fig.colorbar(line, ax=axs)
axs[1].set_xlabel('Trading Day (' + 'From ' + config.START_TRADE_DATE +
" to " + config.END_DATE + ')')
axs[0].set_ylabel('Account Value (10000 of USD)')
axs[0].set_title("Trading Test on " + df.tic[0])
axs[0].set_xlim(x.min(), x.max())
axs[0].set_ylim(y.min(), y.max())
custom_lines = [
Line2D([0], [0], color=cmap(0.), lw=4),
Line2D([0], [0], color=cmap(.5), lw=4),
Line2D([0], [0], color=cmap(1.), lw=4)
]
# lines = ax.plot(data)
axs[0].legend(custom_lines, ['Sell', 'Hold', 'Buy'])
#plot stock value
tx = np.arange(0, df_account_value['account_value'].shape[0])
ty = trade['close']
plt.ylabel('Price (USD)')
plt.title(df.tic[0] + " Closing Price")
plt.plot(tx, ty)
plt.savefig("../" + config.RESULTS_DIR + "/plots/"
"SAC_plot_" + df.tic[0] + "_" + now + ".png")
def train_one():
"""
train an agent
"""
print("==============Start Fetching Data===========")
df = YahooDownloader(
start_date=config.START_DATE,
end_date=config.END_DATE,
ticker_list=config.DOW_30_TICKER,
).fetch_data()
print("==============Start Feature Engineering===========")
fe = FeatureEngineer(
use_technical_indicator=True,
tech_indicator_list=config.TECHNICAL_INDICATORS_LIST,
use_turbulence=True,
user_defined_feature=False,
)
processed = fe.preprocess_data(df)
list_ticker = processed["tic"].unique().tolist()
list_date = list(
pd.date_range(processed['date'].min(),
processed['date'].max()).astype(str))
combination = list(itertools.product(list_date, list_ticker))
processed_full = pd.DataFrame(combination,
columns=["date",
"tic"]).merge(processed,
on=["date", "tic"],
how="left")
processed_full = processed_full[processed_full['date'].isin(
processed['date'])]
processed_full = processed_full.sort_values(['date', 'tic'])
processed_full = processed_full.fillna(0)
# Training & Trading data split
train = data_split(processed_full, config.START_DATE,
config.START_TRADE_DATE)
trade = data_split(processed_full, config.START_TRADE_DATE,
config.END_DATE)
# calculate state action space
stock_dimension = len(train.tic.unique())
state_space = (1 + 2 * stock_dimension +
len(config.TECHNICAL_INDICATORS_LIST) * stock_dimension)
env_kwargs = {
"hmax": 100,
"initial_amount": 1000000,
"buy_cost_pct": 0.001,
"sell_cost_pct": 0.001,
"state_space": state_space,
"stock_dim": stock_dimension,
"tech_indicator_list": config.TECHNICAL_INDICATORS_LIST,
"action_space": stock_dimension,
"reward_scaling": 1e-4
}
e_train_gym = StockTradingEnv(df=train, **env_kwargs)
env_train, _ = e_train_gym.get_sb_env()
agent = DRLAgent(env=env_train)
print("==============Model Training===========")
now = datetime.datetime.now().strftime("%Y%m%d-%Hh%M")
model_sac = agent.get_model("sac")
trained_sac = agent.train_model(model=model_sac,
tb_log_name="sac",
total_timesteps=80000)
print("==============Start Trading===========")
e_trade_gym = StockTradingEnv(df=trade,
turbulence_threshold=250,
**env_kwargs)
df_account_value, df_actions = DRLAgent.DRL_prediction(
model=trained_sac, environment=e_trade_gym)
df_account_value.to_csv("./" + config.RESULTS_DIR + "/df_account_value_" +
now + ".csv")
df_actions.to_csv("./" + config.RESULTS_DIR + "/df_actions_" + now +
".csv")
print("==============Get Backtest Results===========")
perf_stats_all = backtest_stats(df_account_value)
perf_stats_all = pd.DataFrame(perf_stats_all)
perf_stats_all.to_csv("./" + config.RESULTS_DIR + "/perf_stats_all_" +
now + ".csv")
def train_one(fetch=False):
"""
train an agent
"""
if fetch:
df = fetch_and_store()
else:
df = load()
counts = df[['date', 'tic']].groupby(['date']).count().tic
assert counts.min() == counts.max()
print("==============Start Feature Engineering===========")
fe = FeatureEngineer(
use_technical_indicator=True,
tech_indicator_list=config.TECHNICAL_INDICATORS_LIST,
use_turbulence=True,
# use_turbulence=False,
user_defined_feature=False,
)
processed = fe.preprocess_data(df)
# Training & Trading data split
start_date, trade_date, end_date = calculate_split(df,
start=config.START_DATE)
print(start_date, trade_date, end_date)
train = data_split(processed, start_date, trade_date)
trade = data_split(processed, trade_date, end_date)
print(
f'\n******\nRunning from {start_date} to {end_date} for:\n{", ".join(config.CRYPTO_TICKER)}\n******\n'
)
# calculate state action space
stock_dimension = len(train.tic.unique())
state_space = (1 + (2 * stock_dimension) +
(len(config.TECHNICAL_INDICATORS_LIST) * stock_dimension))
env_kwargs = {
"hmax": 100,
"initial_amount": 100000,
"buy_cost_pct": 0.0026,
"sell_cost_pct": 0.0026,
"state_space": state_space,
"stock_dim": stock_dimension,
"tech_indicator_list": config.TECHNICAL_INDICATORS_LIST,
"action_space": stock_dimension,
"reward_scaling": 1e-4
}
e_train_gym = StockTradingEnv(df=train, **env_kwargs)
e_trade_gym = StockTradingEnv(df=trade,
turbulence_threshold=250,
make_plots=True,
**env_kwargs)
env_train, _ = e_train_gym.get_sb_env()
env_trade, obs_trade = e_trade_gym.get_sb_env()
agent = DRLAgent(env=env_train)
print("==============Model Training===========")
now = datetime.datetime.now().strftime(config.DATETIME_FMT)
model_sac = agent.get_model("sac")
trained_sac = agent.train_model(
model=model_sac,
tb_log_name="sac",
# total_timesteps=100
total_timesteps=80000)
print("==============Start Trading===========")
df_account_value, df_actions = DRLAgent.DRL_prediction(
# model=trained_sac, test_data=trade, test_env=env_trade, test_obs=obs_trade
trained_sac,
e_trade_gym)
df_account_value.to_csv(
f"./{config.RESULTS_DIR}/df_account_value_{now}.csv")
df_actions.to_csv(f"./{config.RESULTS_DIR}/df_actions_{now}.csv")
df_txns = pd.DataFrame(e_trade_gym.transactions,
columns=['date', 'amount', 'price', 'symbol'])
df_txns = df_txns.set_index(pd.DatetimeIndex(df_txns['date'], tz=pytz.utc))
df_txns.to_csv(f'./{config.RESULTS_DIR}/df_txns_{now}.csv')
df_positions = pd.DataFrame(e_trade_gym.positions,
columns=['date', 'cash'] +
config.CRYPTO_TICKER)
df_positions = df_positions.set_index(
pd.DatetimeIndex(df_positions['date'],
tz=pytz.utc)).drop(columns=['date'])
df_positions['cash'] = df_positions.astype(
{col: np.float64
for col in df_positions.columns})
df_positions.to_csv(f'./{config.RESULTS_DIR}/df_positions_{now}.csv')
print("==============Get Backtest Results===========")
perf_stats_all = backtest_stats(df_account_value,
transactions=df_txns,
positions=df_positions)
perf_stats_all = pd.DataFrame(perf_stats_all)
perf_stats_all.to_csv(f"./{config.RESULTS_DIR}/perf_stats_all_{now}.csv")
backtest_plot(df_account_value,
baseline_start=trade_date,
baseline_end=end_date,
positions=df_positions,
transactions=df_txns)