def preprocess_px(): create_folders() tickers = get_tickers() preprocess_data(tickers, get_config().DATA_PATH, get_config().HIST_BEG, get_config().HIST_END, get_config().DATA_NPZ_PATH, get_config().DATA_FEATURES)
def download_data_for_all_stocks(): def create_folders(): if not os.path.exists(get_config().DATA_FOLDER_PATH): os.makedirs(get_config().DATA_FOLDER_PATH) get_config().TICKER = 'ALL_STOCKS' get_config().reload() create_folders() download_data(stocks, get_config().DATA_PATH, get_config().HIST_BEG, get_config().HIST_END) preprocess_data(stocks, get_config().DATA_PATH, get_config().HIST_BEG, get_config().HIST_END, get_config().DATA_NPZ_PATH, get_config().DATA_FEATURES)
# 2015-10-30 # stocks = ['HPQ'] # stocks = ['BAC'] def create_folders(): if not os.path.exists(get_config().DATA_FOLDER_PATH): os.makedirs(get_config().DATA_FOLDER_PATH) net = NetShiva() for stock in stocks: print("Processing %s stock" % stock) get_config().TICKER = stock get_config().reload() _tickers = [stock] create_folders() download_data(_tickers, get_config().DATA_PATH, get_config().HIST_BEG, get_config().HIST_END) preprocess_data(_tickers, get_config().DATA_PATH, get_config().HIST_BEG, get_config().HIST_END, get_config().DATA_NPZ_PATH, get_config().DATA_FEATURES) train(net)
from config import get_config from tickers import get_nyse_nasdaq_tickers, get_nyse_tickers, get_nasdaq_tickers, get_snp_tickers from download_utils import preprocess_data tickers = get_snp_tickers() preprocess_data(tickers, "data/prices_snp.csv", get_config().HIST_BEG, get_config().HIST_END, "data/snp.npz", get_config().ADJ_PX_FEATURES) # tickers = get_nasdaq_tickers() # preprocess_data(tickers, "data/prices.csv", get_config().HIST_BEG, get_config().HIST_END, "data/nasdaq.npz", get_config().ADJ_PX_FEATURES) # # tickers = get_nyse_nasdaq_tickers() # preprocess_data(tickers, "data/prices.csv", get_config().HIST_BEG, get_config().HIST_END, "data/nyse_nasdaq.npz", get_config().ADJ_PX_FEATURES) # # tickers = get_nyse_tickers() # preprocess_data(tickers, "data/prices.csv", get_config().HIST_BEG, get_config().HIST_END, "data/nyse.npz", get_config().ADJ_PX_FEATURES)
with open('./data/prediction.csv', 'w', newline='') as f: writer = csv.writer(f) writer.writerow(('ticker', 'exchange', 'long prob', 'class', '1w', '1d', '*', '#', 'hp', '1w px', '1d px', '* px', '# px', 'hp px', '1wr pct', '1dr pct', 'hpr pct', '1d v', '1w avg v')) idx = 0 for ticker in tickers: print("Processing %s" % ticker) get_config().TICKER = ticker get_config().reload() create_folders() _tickers = [ticker] download_data(_tickers, get_config().DATA_PATH, START_DATE, END_DATE) preprocess_data(_tickers, get_config().DATA_PATH, START_DATE, END_DATE, get_config().DATA_NPZ_PATH, get_config().DATA_FEATURES) env = Env() beg_idx, end_idx, raw_dates, raw_week_days, tradeable_mask, px, input = get_net_data( env, START_DATE, PREDICTION_DATE) ds_size = end_idx - beg_idx + 1 net.load_weights(get_config().WEIGHTS_PATH, get_config().MAX_EPOCH) state = net.zero_state(1) _input = input _labels = np.zeros((1, ds_size)) _mask = np.zeros((1, ds_size)) state, loss, predictions = net.eval(state, _input, _labels, _mask) pred_ret = predictions[0, -1, 0]
def preprocess_snp_px(): tickers = get_snp_hitorical_components_tickers() preprocess_data(tickers, 'data/snp/snp_px.csv', get_config().HIST_BEG, get_config().HIST_END, 'data/snp/snp_px.npz', get_config().DATA_FEATURES)
# YYYY-MM-DD PREDICTION_DATE = datetime.datetime.strptime('2017-09-08', '%Y-%m-%d').date() OPEN_POS_DATE = datetime.datetime.strptime('2017-09-08', '%Y-%m-%d').date() HPR_DATE = datetime.datetime.strptime('2017-09-15', '%Y-%m-%d').date() # PREDICTION_DATE = datetime.datetime.strptime('2017-08-04', '%Y-%m-%d').date() # OPEN_POS_DATE = datetime.datetime.strptime('2017-08-04', '%Y-%m-%d').date() # HPR_DATE = datetime.datetime.strptime('2017-08-04', '%Y-%m-%d').date() START_DATE = PREDICTION_DATE - datetime.timedelta(days=(NUM_WEEKS + 2) * 7) END_DATE = HPR_DATE ticker_exch_map = get_snp_tickers_exch_map() tickers = list(ticker_exch_map.keys()) download_data(tickers, 'data/history.csv', START_DATE, END_DATE, 50) preprocess_data(tickers, 'data/history.csv', START_DATE, END_DATE, 'data/history.npz', get_config().ADJ_PX_FEATURES) tickers, raw_dt, raw_data = load_npz_data('data/history.npz') tradable_mask = get_tradable_stocks_mask(raw_data) tradable_stocks_per_day = tradable_mask[:, :].sum(0) trading_day_mask = tradable_stocks_per_day > get_config( ).MIN_STOCKS_TRADABLE_PER_TRADING_DAY def ib_convert_ticker(ticker): ticker = ticker.replace('-', ' ') ticker = ticker.replace('.', ' ') return ticker