def preprocess_px():
create_folders()
tickers = get_tickers()
preprocess_data(tickers,
get_config().DATA_PATH,
get_config().HIST_BEG,
get_config().HIST_END,
get_config().DATA_NPZ_PATH,
get_config().DATA_FEATURES)
def download_data_for_all_stocks():
def create_folders():
if not os.path.exists(get_config().DATA_FOLDER_PATH):
os.makedirs(get_config().DATA_FOLDER_PATH)
get_config().TICKER = 'ALL_STOCKS'
get_config().reload()
create_folders()
download_data(stocks,
get_config().DATA_PATH,
get_config().HIST_BEG,
get_config().HIST_END)
preprocess_data(stocks,
get_config().DATA_PATH,
get_config().HIST_BEG,
get_config().HIST_END,
get_config().DATA_NPZ_PATH,
get_config().DATA_FEATURES)
# 2015-10-30
# stocks = ['HPQ']
# stocks = ['BAC']
def create_folders():
if not os.path.exists(get_config().DATA_FOLDER_PATH):
os.makedirs(get_config().DATA_FOLDER_PATH)
net = NetShiva()
for stock in stocks:
print("Processing %s stock" % stock)
get_config().TICKER = stock
get_config().reload()
_tickers = [stock]
create_folders()
download_data(_tickers,
get_config().DATA_PATH,
get_config().HIST_BEG,
get_config().HIST_END)
preprocess_data(_tickers,
get_config().DATA_PATH,
get_config().HIST_BEG,
get_config().HIST_END,
get_config().DATA_NPZ_PATH,
get_config().DATA_FEATURES)
train(net)
from config import get_config
from tickers import get_nyse_nasdaq_tickers, get_nyse_tickers, get_nasdaq_tickers, get_snp_tickers
from download_utils import preprocess_data
tickers = get_snp_tickers()
preprocess_data(tickers, "data/prices_snp.csv",
get_config().HIST_BEG,
get_config().HIST_END, "data/snp.npz",
get_config().ADJ_PX_FEATURES)
# tickers = get_nasdaq_tickers()
# preprocess_data(tickers, "data/prices.csv", get_config().HIST_BEG, get_config().HIST_END, "data/nasdaq.npz", get_config().ADJ_PX_FEATURES)
#
# tickers = get_nyse_nasdaq_tickers()
# preprocess_data(tickers, "data/prices.csv", get_config().HIST_BEG, get_config().HIST_END, "data/nyse_nasdaq.npz", get_config().ADJ_PX_FEATURES)
#
# tickers = get_nyse_tickers()
# preprocess_data(tickers, "data/prices.csv", get_config().HIST_BEG, get_config().HIST_END, "data/nyse.npz", get_config().ADJ_PX_FEATURES)
with open('./data/prediction.csv', 'w', newline='') as f:
writer = csv.writer(f)
writer.writerow(('ticker', 'exchange', 'long prob', 'class', '1w', '1d',
'*', '#', 'hp', '1w px', '1d px', '* px', '# px', 'hp px',
'1wr pct', '1dr pct', 'hpr pct', '1d v', '1w avg v'))
idx = 0
for ticker in tickers:
print("Processing %s" % ticker)
get_config().TICKER = ticker
get_config().reload()
create_folders()
_tickers = [ticker]
download_data(_tickers, get_config().DATA_PATH, START_DATE, END_DATE)
preprocess_data(_tickers,
get_config().DATA_PATH, START_DATE, END_DATE,
get_config().DATA_NPZ_PATH,
get_config().DATA_FEATURES)
env = Env()
beg_idx, end_idx, raw_dates, raw_week_days, tradeable_mask, px, input = get_net_data(
env, START_DATE, PREDICTION_DATE)
ds_size = end_idx - beg_idx + 1
net.load_weights(get_config().WEIGHTS_PATH, get_config().MAX_EPOCH)
state = net.zero_state(1)
_input = input
_labels = np.zeros((1, ds_size))
_mask = np.zeros((1, ds_size))
state, loss, predictions = net.eval(state, _input, _labels, _mask)
pred_ret = predictions[0, -1, 0]
def preprocess_snp_px():
tickers = get_snp_hitorical_components_tickers()
preprocess_data(tickers, 'data/snp/snp_px.csv', get_config().HIST_BEG, get_config().HIST_END, 'data/snp/snp_px.npz',
get_config().DATA_FEATURES)
# YYYY-MM-DD
PREDICTION_DATE = datetime.datetime.strptime('2017-09-08', '%Y-%m-%d').date()
OPEN_POS_DATE = datetime.datetime.strptime('2017-09-08', '%Y-%m-%d').date()
HPR_DATE = datetime.datetime.strptime('2017-09-15', '%Y-%m-%d').date()
# PREDICTION_DATE = datetime.datetime.strptime('2017-08-04', '%Y-%m-%d').date()
# OPEN_POS_DATE = datetime.datetime.strptime('2017-08-04', '%Y-%m-%d').date()
# HPR_DATE = datetime.datetime.strptime('2017-08-04', '%Y-%m-%d').date()
START_DATE = PREDICTION_DATE - datetime.timedelta(days=(NUM_WEEKS + 2) * 7)
END_DATE = HPR_DATE
ticker_exch_map = get_snp_tickers_exch_map()
tickers = list(ticker_exch_map.keys())
download_data(tickers, 'data/history.csv', START_DATE, END_DATE, 50)
preprocess_data(tickers, 'data/history.csv', START_DATE, END_DATE,
'data/history.npz',
get_config().ADJ_PX_FEATURES)
tickers, raw_dt, raw_data = load_npz_data('data/history.npz')
tradable_mask = get_tradable_stocks_mask(raw_data)
tradable_stocks_per_day = tradable_mask[:, :].sum(0)
trading_day_mask = tradable_stocks_per_day > get_config(
).MIN_STOCKS_TRADABLE_PER_TRADING_DAY
def ib_convert_ticker(ticker):
ticker = ticker.replace('-', ' ')
ticker = ticker.replace('.', ' ')
return ticker
