def main():
import timeit
for event_date in event_dates:
#print " aggr_close:", self.df.ix[self.cur_loc, 'aggr_close'], " bb_lower:", self.df.ix[self.cur_loc, 'bb_lower']
print event_date
event_datetime_obj = datetime.strptime(event_date + " " + event_start_time, '%Y-%m-%d %H:%M:%S')
symbol = mkdt_utils.getSymbol(instrument_root, event_datetime_obj)
df = mkdt_utils.getMarketDataFrameForTradingDate(event_date, instrument_root, symbol, "100ms")
if isinstance(df.head(1).time[0], basestring):
df.time = df.time.apply(mkdt_utils.str_to_dt)
df.set_index(df['time'], inplace=True)
start_dt = datetime.strptime(event_date + " " + trading_start_time, '%Y-%m-%d %H:%M:%S')
max_entry_dt = datetime.strptime(event_date + " " + max_entry_time, '%Y-%m-%d %H:%M:%S')
stop_dt = datetime.strptime(event_date + " " + trading_stop_time, '%Y-%m-%d %H:%M:%S')
dollar_value_per_price_level = 12.5 # specific to Euro
Sim = simulator.Simulator(df)
Strat = Strategy(df, start_dt, max_entry_dt, stop_dt, Sim, dollar_value_per_price_level, spike_5s_pred[event_date])
Sim.initStrategy(Strat)
log_df = Strat.start()
dir_path = '/local/disk1/temp_eg/log_run/'
filename = 'log_run_' + event_name + '_' + instrument_root + '_' + event_date.replace('-', '')
store_filename = dir_path + filename + '.h5'
store = pd.HDFStore(store_filename)
store['log_df'] = log_df
store.close()
print "FINISHED"
def main():
import timeit
for event_date in event_dates:
print event_date
event_datetime_obj = datetime.strptime(event_date + " " + event_start_time, '%Y-%m-%d %H:%M:%S')
symbol = mkdt_utils.getSymbol(instrument_root, event_datetime_obj)
df = mkdt_utils.getMarketDataFrameForTradingDate(event_date, instrument_root, symbol, "100ms")
if isinstance(df.head(1).time[0], basestring):
df.time = df.time.apply(mkdt_utils.str_to_dt)
df.set_index(df['time'], inplace=True)
start_event_dt = event_datetime_obj
start_dt = datetime.strptime(event_date + " " + trading_start_time, '%Y-%m-%d %H:%M:%S')
max_entry_dt = datetime.strptime(event_date + " " + max_entry_time, '%Y-%m-%d %H:%M:%S')
stop_dt = datetime.strptime(event_date + " " + trading_stop_time, '%Y-%m-%d %H:%M:%S')
# create dataframe for new features
ema_feats_df = pd.DataFrame(index = df['time'])
ema_feats_df['time'] = df['time']
ema_feats_df = computeEMASignals(df, ema_feats_df, event_date, start_event_dt, start_dt, max_entry_dt, stop_dt, spike_5s[event_date], spike_5s_pred[event_date])
dir_path = '/local/disk1/temp_eg/ema_features/'
filename = 'ema_feats_' + event_name + '_' + instrument_root + '_' + event_date.replace('-', '')
store_filename = dir_path + filename + '.h5'
store = pd.HDFStore(store_filename)
store['ema_feats_df'] = ema_feats_df
store.close()
print "completed processing date: ", event_date
def main():
import timeit
for event_date in event_dates:
print event_date
event_datetime_obj = datetime.strptime(event_date + " " + event_start_time, '%Y-%m-%d %H:%M:%S')
symbol = mkdt_utils.getSymbol(instrument_root, event_datetime_obj)
df = mkdt_utils.getMarketDataFrameForTradingDate(event_date, instrument_root, symbol, "100ms")
if isinstance(df.head(1).time[0], basestring):
df.time = df.time.apply(mkdt_utils.str_to_dt)
df.set_index(df['time'], inplace=True)
data_npar = df[df_cols_to_npar_map].values
# create separate dataframe for features
feats_df = pd.DataFrame(index = df['time'])
feats_df['time'] = df['time']
start_dt = datetime.strptime(event_date + " " + trading_start_time, '%Y-%m-%d %H:%M:%S')
max_entry_dt = datetime.strptime(event_date + " " + max_entry_time, '%Y-%m-%d %H:%M:%S')
stop_dt = datetime.strptime(event_date + " " + trading_stop_time, '%Y-%m-%d %H:%M:%S')
feats_df = computeFeatures(df, feats_df, start_dt, max_entry_dt, stop_dt, event_date)
dir_path = '/local/disk1/temp_eg/features/'
filename = 'feats_' + event_name + '_' + instrument_root + '_' + event_date.replace('-', '')
store_filename = dir_path + filename + '.h5'
store = pd.HDFStore(store_filename)
store['feats_df'] = feats_df
store.close()
def main():
import timeit
for event_date in event_dates:
event_datetime_obj = datetime.strptime(event_date + " " + event_start_time, '%Y-%m-%d %H:%M:%S')
symbol = mkdt_utils.getSymbol(instrument_root, event_datetime_obj)
df = mkdt_utils.getMarketDataFrameForTradingDate(event_date, instrument_root, symbol, "100ms")
if isinstance(df.head(1).time[0], basestring):
df.time = df.time.apply(mkdt_utils.str_to_dt)
df.set_index(df['time'], inplace=True)
t00 = timeit.default_timer()
ask_size_npar = df[['ask_size_0', 'ask_size_1', 'ask_size_2', 'ask_size_3', 'ask_size_4']].values
bid_size_npar = df[['bid_size_0', 'bid_size_1', 'bid_size_2', 'bid_size_3', 'bid_size_4']].values
top_prices_npar = df[['top_ask_price', 'top_bid_price']].values
time_at_loc_npar = df['time'].values
print time_at_loc_npar[0], time_at_loc_npar[1]
t01 = timeit.default_timer()
print "t01-t00=" + "\t" + str(t01-t00)
trades_df = pd.DataFrame(index = df['time'])
trades_df['time'] = df['time']
start_dt = datetime.strptime(event_date + " " + trading_start_time, '%Y-%m-%d %H:%M:%S')
max_entry_dt = datetime.strptime(event_date + " " + max_entry_time, '%Y-%m-%d %H:%M:%S')
stop_dt = datetime.strptime(event_date + " " + trading_stop_time, '%Y-%m-%d %H:%M:%S')
trades_df = computeMarkouts(df, trades_df, start_dt, max_entry_dt, stop_dt, ask_size_npar, bid_size_npar, top_prices_npar, time_at_loc_npar)
#print trades_df.loc[trades_df['time'] == df.ix[9013, 'time']]
dir_path = '/local/disk1/temp_eg/trade_util_labels/'
trade_name = event_name + '_' + instrument_root + '_' + event_date.replace('-', '')
store_filename = dir_path + trade_name + '.h5'
store = pd.HDFStore(store_filename)
store['trades_df'] = trades_df
store.close()
def main():
import timeit
for event_date in event_dates:
print event_date
event_datetime_obj = datetime.strptime(event_date + " " + event_start_time, '%Y-%m-%d %H:%M:%S')
symbol = mkdt_utils.getSymbol(instrument_root, event_datetime_obj)
df = mkdt_utils.getMarketDataFrameForTradingDate(event_date, instrument_root, symbol, "100ms")
if isinstance(df.head(1).time[0], basestring):
df.time = df.time.apply(mkdt_utils.str_to_dt)
df.set_index(df['time'], inplace=True)
start_dt = datetime.strptime(event_date + " " + trading_start_time, '%Y-%m-%d %H:%M:%S')
max_entry_dt = datetime.strptime(event_date + " " + max_entry_time, '%Y-%m-%d %H:%M:%S')
stop_dt = datetime.strptime(event_date + " " + trading_stop_time, '%Y-%m-%d %H:%M:%S')
dollar_value_per_price_level = 12.5 # specific to Euro
Sim = simulator.Simulator(df)
Strat = Strategy(df, start_dt, max_entry_dt, stop_dt, Sim, dollar_value_per_price_level, spike_5s_pred[event_date])
Sim.initStrategy(Strat)
log_df = Strat.start()
dir_path = '/local/disk1/temp_eg/log_run/'
filename = 'log_run_' + event_name + '_' + instrument_root + '_' + event_date.replace('-', '')
store_filename = dir_path + filename + '.h5'
store = pd.HDFStore(store_filename)
store['log_df'] = log_df
store.close()
print "FINISHED"
def readMarketDataForDate(instrument_root, event_date, event_datetime_obj):
try:
symbol = mkdt_utils.getSymbol(instrument_root, event_datetime_obj)
except:
symbol = symbols_bkp[event_date]
df = mkdt_utils.getMarketDataFrameForTradingDate(event_date, instrument_root, symbol, "100ms")
if isinstance(df.head(1).time[0], basestring):
df.time = df.time.apply(mkdt_utils.str_to_dt)
df.set_index(df['time'], inplace=True)
return symbol, df
def main():
import timeit
for event_date in event_dates:
event_datetime_obj = datetime.strptime(
event_date + " " + event_start_time, '%Y-%m-%d %H:%M:%S')
symbol = mkdt_utils.getSymbol(instrument_root, event_datetime_obj)
df = mkdt_utils.getMarketDataFrameForTradingDate(
event_date, instrument_root, symbol, "100ms")
if isinstance(df.head(1).time[0], basestring):
df.time = df.time.apply(mkdt_utils.str_to_dt)
df.set_index(df['time'], inplace=True)
t00 = timeit.default_timer()
ask_size_npar = df[[
'ask_size_0', 'ask_size_1', 'ask_size_2', 'ask_size_3',
'ask_size_4'
]].values
bid_size_npar = df[[
'bid_size_0', 'bid_size_1', 'bid_size_2', 'bid_size_3',
'bid_size_4'
]].values
top_prices_npar = df[['top_ask_price', 'top_bid_price']].values
time_at_loc_npar = df['time'].values
print time_at_loc_npar[0], time_at_loc_npar[1]
t01 = timeit.default_timer()
print "t01-t00=" + "\t" + str(t01 - t00)
trades_df = pd.DataFrame(index=df['time'])
trades_df['time'] = df['time']
start_dt = datetime.strptime(event_date + " " + trading_start_time,
'%Y-%m-%d %H:%M:%S')
max_entry_dt = datetime.strptime(event_date + " " + max_entry_time,
'%Y-%m-%d %H:%M:%S')
stop_dt = datetime.strptime(event_date + " " + trading_stop_time,
'%Y-%m-%d %H:%M:%S')
trades_df = computeMarkouts(df, trades_df, start_dt, max_entry_dt,
stop_dt, ask_size_npar, bid_size_npar,
top_prices_npar, time_at_loc_npar)
#print trades_df.loc[trades_df['time'] == df.ix[9013, 'time']]
dir_path = '/local/disk1/temp_eg/trade_util_labels/'
trade_name = event_name + '_' + instrument_root + '_' + event_date.replace(
'-', '')
store_filename = dir_path + trade_name + '.h5'
store = pd.HDFStore(store_filename)
store['trades_df'] = trades_df
store.close()
def main():
print "starting..."
import timeit
results = []
for event_date in event_dates:
print event_date
event_datetime_obj = datetime.strptime(event_date + " " + event_start_time, '%Y-%m-%d %H:%M:%S')
symbol = mkdt_utils.getSymbol(instrument_root, event_datetime_obj)
df = mkdt_utils.getMarketDataFrameForTradingDate(event_date, instrument_root, symbol, "100ms")
if isinstance(df.head(1).time[0], basestring):
df.time = df.time.apply(mkdt_utils.str_to_dt)
df.set_index(df['time'], inplace=True)
start_dt = datetime.strptime(event_date + " " + trading_start_time, '%Y-%m-%d %H:%M:%S')
trading_stop_time = trading_stop_times[event_durations[event_date]]
stop_dt = datetime.strptime(event_date + " " + trading_stop_time, '%Y-%m-%d %H:%M:%S')
opposite_stacker_off_dt = datetime.strptime(event_date + " " + opposite_stacker_off_time, '%Y-%m-%d %H:%M:%S')
entry_stop_time_dt = datetime.strptime(event_date + " " + entry_stop_time, '%Y-%m-%d %H:%M:%S')
Sim = simulator4.Simulator(df, symbol, min_price_increment)
Strat = Strategy(df, event_datetime_obj, start_dt, stop_dt, opposite_stacker_off_dt, entry_stop_time_dt, Sim, dollar_value_per_price_level)
Sim.initStrategy(Strat)
Strat.start()
event_results = {'pnl': Strat._pnl, 'pos': Strat._pos, 'vol': Strat._vol}
results.append({'event_date': event_date, 'event_results': event_results, 'best_potential_pnl': Strat.best_potential_pnl, 'max_abs_pos': Strat.max_abs_pos})
print "FINISHED:", event_date
print " pos: ", Strat._pos
print " PnL: ", Strat._pnl
print " vol: ", Strat._vol
print
print "--------"
print "RESULTS:"
print "--------"
#print "event_date" + "\t" + "PnL" + "\t\t" + "pos" + "\t" + "vol" + "\t" + "max_abs_pos" + "\t" + "best_potential_pnl"
#for result in results:
# print result['event_date'] + "\t" + str(result['event_results']['pnl']) + "\t\t" + str(result['event_results']['pos']) + "\t" \
# + str(result['event_results']['vol']) + "\t" + str(result['max_abs_pos']) + "\t" + str(result['best_potential_pnl'])
print "event_date".ljust(16) + "PnL".ljust(20) + "pos".ljust(12) + "vol".ljust(12) + "max_abs_pos".ljust(16) + "best_potential_pnl".ljust(20)
for result in results:
print result['event_date'].ljust(16) + str(result['event_results']['pnl']).ljust(20) + str(result['event_results']['pos']).ljust(12) \
+ str(result['event_results']['vol']).ljust(12) + str(result['max_abs_pos']).ljust(16) + str(result['best_potential_pnl']).ljust(20)
def main():
import timeit
for event_date in event_dates:
print event_date
event_datetime_obj = datetime.strptime(
event_date + " " + event_start_time, '%Y-%m-%d %H:%M:%S')
symbol = mkdt_utils.getSymbol(instrument_root, event_datetime_obj)
df = mkdt_utils.getMarketDataFrameForTradingDate(
event_date, instrument_root, symbol, "100ms")
df.set_index(df['time'], inplace=True)
#print type(df)
#print df.describe()
start_dt = datetime.strptime(event_date + " " + trading_start_time,
'%Y-%m-%d %H:%M:%S')
max_entry_dt = datetime.strptime(event_date + " " + max_entry_time,
'%Y-%m-%d %H:%M:%S')
stop_dt = datetime.strptime(event_date + " " + trading_stop_time,
'%Y-%m-%d %H:%M:%S')
# retrieve base features
base_feats_df = retrieveBaseFeatsForDate(event_date, start_dt,
max_entry_dt, stop_dt)
#create dataframe for abstract features
abstr_feats_df = pd.DataFrame(index=df['time'])
abstr_feats_df['time'] = df['time']
#print type(abstr_feats_df)
abstr_feats_df = computeAbstractFeatures(df, base_feats_df,
abstr_feats_df)
#print type(abstr_feats_df)
#print abstr_feats_df.describe()
#print abstr_feats_df.head(50)
dir_path = '/local/disk1/temp_eg/abstr_features/'
filename = 'abstr_feats_' + event_name + '_' + instrument_root + '_' + event_date.replace(
'-', '')
store_filename = dir_path + filename + '.h5'
store = pd.HDFStore(store_filename)
store['abstr_feats_df'] = abstr_feats_df
store.close()
print "completed processing date: ", event_date
def main():
import timeit
for event_date in event_dates:
print event_date
event_datetime_obj = datetime.strptime(event_date + " " + event_start_time, '%Y-%m-%d %H:%M:%S')
symbol = mkdt_utils.getSymbol(instrument_root, event_datetime_obj)
df = mkdt_utils.getMarketDataFrameForTradingDate(event_date, instrument_root, symbol, "100ms")
df.set_index(df['time'], inplace=True)
#print type(df)
#print df.describe()
start_dt = datetime.strptime(event_date + " " + trading_start_time, '%Y-%m-%d %H:%M:%S')
max_entry_dt = datetime.strptime(event_date + " " + max_entry_time, '%Y-%m-%d %H:%M:%S')
stop_dt = datetime.strptime(event_date + " " + trading_stop_time, '%Y-%m-%d %H:%M:%S')
# retrieve base features
base_feats_df = retrieveBaseFeatsForDate(event_date, start_dt, max_entry_dt, stop_dt)
#create dataframe for abstract features
abstr_feats_df = pd.DataFrame(index = df['time'])
abstr_feats_df['time'] = df['time']
#print type(abstr_feats_df)
abstr_feats_df = computeAbstractFeatures(df, base_feats_df, abstr_feats_df)
#print type(abstr_feats_df)
#print abstr_feats_df.describe()
#print abstr_feats_df.head(50)
dir_path = '/local/disk1/temp_eg/abstr_features/'
filename = 'abstr_feats_' + event_name + '_' + instrument_root + '_' + event_date.replace('-', '')
store_filename = dir_path + filename + '.h5'
store = pd.HDFStore(store_filename)
store['abstr_feats_df'] = abstr_feats_df
store.close()
print "completed processing date: ", event_date
def main():
print "starting..."
import timeit
results = []
for event_date in event_dates:
print event_date
global event_start_time
if event_date in event_dates_special_times:
my_event_start_time = event_dates_special_times[event_date]
else:
my_event_start_time = event_start_time
event_datetime_obj = datetime.strptime(event_date + " " + my_event_start_time, '%Y-%m-%d %H:%M:%S')
try:
symbol = mkdt_utils.getSymbol(instrument_root, event_datetime_obj)
except:
symbol = symbols_bkp[event_date]
df = mkdt_utils.getMarketDataFrameForTradingDate(event_date, instrument_root, symbol, "100ms")
if isinstance(df.head(1).time[0], basestring):
df.time = df.time.apply(mkdt_utils.str_to_dt)
df.set_index(df['time'], inplace=True)
start_dt = event_datetime_obj - timedelta(seconds=30)
stop_dt = event_datetime_obj + timedelta(minutes=22) #12
opposite_stacker_off_dt = event_datetime_obj + timedelta(seconds=30)
entry_stop_time_dt = event_datetime_obj + timedelta(seconds=30)
Sim = simulator4.Simulator(df, symbol, min_price_increment)
Strat = Strategy(df, event_datetime_obj, start_dt, stop_dt, opposite_stacker_off_dt, entry_stop_time_dt, Sim, dollar_value_per_price_level)
Sim.initStrategy(Strat)
Strat.start()
event_results = {'pnl': Strat._pnl, 'pos': Strat._pos, 'vol': Strat._vol}
results.append({'event_date': event_date, 'event_results': event_results, 'best_potential_pnl': Strat.best_potential_pnl, 'max_abs_pos': Strat.max_abs_pos})
pnl_results = {}
print "FINISHED:", event_date
print " pos: ", Strat._pos
print " PnL: ", Strat._pnl
print " vol: ", Strat._vol
print
print "--------"
print "RESULTS:"
print "--------"
#print "event_date" + "\t" + "PnL" + "\t\t" + "pos" + "\t" + "vol" + "\t" + "max_abs_pos" + "\t" + "best_potential_pnl"
#for result in results:
# print result['event_date'] + "\t" + str(result['event_results']['pnl']) + "\t\t" + str(result['event_results']['pos']) + "\t" \
# + str(result['event_results']['vol']) + "\t" + str(result['max_abs_pos']) + "\t" + str(result['best_potential_pnl'])
print "event_date".ljust(16) + "PnL".ljust(20) + "pos".ljust(12) + "vol".ljust(12) + "max_abs_pos".ljust(16) + "best_potential_pnl".ljust(20)
for result in results:
print result['event_date'].ljust(16) + str(result['event_results']['pnl']).ljust(20) + str(result['event_results']['pos']).ljust(12) \
+ str(result['event_results']['vol']).ljust(12) + str(result['max_abs_pos']).ljust(16) + str(result['best_potential_pnl']).ljust(20)
pnl_results[result['event_date']] = result['event_results']['pnl']
print pnl_results