def get_yahoo_data(start, end): df = YahooDownloader(start_date=start, end_date=end, ticker_list=config.DOW_30_TICKER).fetch_data() df.sort_values(['date', 'tic'], ignore_index=True) x = df.tic.unique() templ = [] # get intersection data, smallest data for name, group in df.groupby('date'): g = group.tic.unique() if len(templ) == 0: templ = [i for i in g if i in x] else: templ = [i for i in g if i in templ] data_merge = pd.DataFrame(columns=list(df.columns)) x = np.array(templ).reshape(-1, 1) temp_df = pd.DataFrame.from_records(x, columns=['tic']) for name, group in df.groupby('date'): temp_df['date'] = name result_outer = pd.merge(group, temp_df, on=['date', 'tic']) result_outer = result_outer.sort_values(['date', 'tic'], ignore_index=True) assert len(result_outer) == len(temp_df.tic.unique()) data_merge = data_merge.append(result_outer) df = data_merge fe = FeatureEngineer(use_technical_indicator=True, tech_indicator_list=config.TECHNICAL_INDICATORS_LIST, use_turbulence=True, user_defined_feature=False) processed = fe.preprocess_data(df) processed.sort_values(['date', 'tic'], ignore_index=True) return processed
os.makedirs("./" + config.RESULTS_DIR) config.START_DATE config.END_DATE print(config.DOW_30_TICKER) ############# DOWNLOAD DATA ############# df = YahooDownloader(start_date='2008-01-01', end_date='2021-01-01', ticker_list=config.DOW_30_TICKER).fetch_data() df.shape df.sort_values(['date', 'tic'], ignore_index=True).head() df.info() ############# PERFORM FEATURE ENGINEERING ############# fe = FeatureEngineer(use_technical_indicator=True, tech_indicator_list=config.TECHNICAL_INDICATORS_LIST, use_turbulence=True, user_defined_feature=False) processed = fe.preprocess_data(df) list_ticker = processed["tic"].unique().tolist() list_date = list( pd.date_range(processed['date'].min(), processed['date'].max()).astype(str))
def generate_data(rollouts, data_dir, noise_type): # pylint: disable=R0914 """ Generates data """ assert exists(data_dir), "The data directory does not exist..." df = YahooDownloader(start_date = '2009-01-01', end_date = '2021-01-01', ticker_list = ['AAPL']).fetch_data() df.sort_values(['date','tic'],ignore_index=True) fe = FeatureEngineer( use_technical_indicator=True, tech_indicator_list = config.TECHNICAL_INDICATORS_LIST, use_turbulence=True, user_defined_feature = False) processed = fe.preprocess_data(df) list_ticker = processed["tic"].unique().tolist() list_date = list(pd.date_range(processed['date'].min(),processed['date'].max()).astype(str)) combination = list(itertools.product(list_date,list_ticker)) processed_full = pd.DataFrame(combination,columns=["date","tic"]).merge(processed,on=["date","tic"],how="left") processed_full = processed_full[processed_full['date'].isin(processed['date'])] processed_full = processed_full.sort_values(['date','tic']) processed_full = processed_full.fillna(0) processed_full.sort_values(['date','tic'],ignore_index=True) train = data_split(processed_full, '2009-01-01','2019-01-01') trade = data_split(processed_full, '2019-01-01','2021-01-01') stock_dimension = len(train.tic.unique()) state_space = 1 + 2*stock_dimension + len(config.TECHNICAL_INDICATORS_LIST)*stock_dimension env_kwargs = { "hmax": 100, "initial_amount": 1000000, # "buy_cost_pct": 0.001i, # "sell_cost_pct": 0.001, "transaction_cost_pct": 0.001, "state_space": state_space, "stock_dim": stock_dimension, "tech_indicator_list": config.TECHNICAL_INDICATORS_LIST, "action_space": stock_dimension, "reward_scaling": 1e-4 } e_train_gym = StockTradingEnv(df = train, **env_kwargs) env_train, _ = e_train_gym.get_sb_env() env = env_train # env = gym.make("CarRacing-v0") seq_len = 10000 for i in range(rollouts): env.reset() # env.env.viewer.window.dispatch_events() if noise_type == 'white': a_rollout = [env.action_space.sample() for _ in range(seq_len)] elif noise_type == 'brown': a_rollout = sample_continuous_policy(env.action_space, seq_len, 1. / 50) s_rollout = [] r_rollout = [] d_rollout = [] t = 0 while True: action = a_rollout[t] t += 1 s, r, done, _ = env.step(action) # env.env.viewer.window.dispatch_events() s_rollout += [s] r_rollout += [r] d_rollout += [done] if done: print("> End of rollout {}, {} frames...".format(i, len(s_rollout))) np.savez(join(data_dir, 'rollout_{}'.format(i)), observations=np.array(s_rollout), rewards=np.array(r_rollout), actions=np.array(a_rollout), terminals=np.array(d_rollout)) break