def load_data(self, assets):
contlist = {}
exp_dates = {}
dbconfig = self.config.get('dbconfig', dbaccess.hist_dbconfig)
cnx = dbaccess.connect(**dbconfig)
for i, prod in enumerate(assets):
cont_mth, exch = dbaccess.prod_main_cont_exch(prod)
self.contlist[prod], _ = misc.contract_range(
prod, exch, cont_mth, self.start_date, self.end_date)
self.exp_dates[prod] = [
misc.contract_expiry(cont) for cont in contlist[prod]
]
edates = [
misc.day_shift(d, self.config['rollrule'])
for d in exp_dates[prod]
]
sdates = [
misc.day_shift(d, self.sim_period) for d in exp_dates[prod]
]
self.data_store[prod] = {}
for cont, sd, ed in zip(contlist[prod], sdates, edates):
if self.sim_freq == 'd':
tmp_df = dbaccess.load_daily_data_to_df(
cnx, 'fut_min', cont, sd, ed)
else:
minid_start = 1500
minid_end = 2114
if prod in misc.night_session_markets:
minid_start = 300
tmp_df = dbaccess.load_min_data_to_df(
cnx, 'fut_min', cont, sd, ed, minid_start, minid_end)
misc.cleanup_mindata(tmp_df, prod)
tmp_df['contract'] = cont
self.data_store[prod][cont] = tmp_df
cnx.close()
def load_data(self, assets):
contlist = {}
exp_dates = {}
dbconfig = self.config.get('dbconfig', dbaccess.hist_dbconfig)
cnx = dbaccess.connect(**dbconfig)
for i, prod in enumerate(assets):
cont_mth, exch = dbaccess.prod_main_cont_exch(prod)
self.contlist[prod], _ = misc.contract_range(prod, exch, cont_mth, self.start_date, self.end_date)
self.exp_dates[prod] = [misc.contract_expiry(cont) for cont in contlist[prod]]
edates = [ misc.day_shift(d, self.config['rollrule']) for d in exp_dates[prod] ]
sdates = [ misc.day_shift(d, self.sim_period) for d in exp_dates[prod] ]
self.data_store[prod] = {}
for cont, sd, ed in zip(contlist[prod], sdates, edates):
if self.sim_freq == 'd':
tmp_df = dbaccess.load_daily_data_to_df(cnx, 'fut_min', cont, sd, ed)
else:
minid_start = 1500
minid_end = 2114
if prod in misc.night_session_markets:
minid_start = 300
tmp_df = dbaccess.load_min_data_to_df(cnx, 'fut_min', cont, sd, ed, minid_start, minid_end)
misc.cleanup_mindata(tmp_df, prod)
tmp_df['contract'] = cont
self.data_store[prod][cont] = tmp_df
cnx.close()
def load_data(self, idx):
asset = self.sim_assets[idx]
for prod in asset:
mdf = misc.nearby(prod, self.config['nearby'], self.config['start_date'], self.config['end_date'],
self.config['rollrule'], 'm', need_shift = self.need_shift, database=self.config.get('database', 'hist_data'))
mdf = misc.cleanup_mindata(mdf, prod)
self.min_data[prod] = mdf
def get_cont_data(asset,
start_date,
end_date,
freq='1m',
nearby=1,
rollrule='-10b'):
cnx = dbaccess.connect(**dbaccess.hist_dbconfig)
if nearby == 0:
mdf = dbaccess.load_min_data_to_df(cnx,
'fut_min',
asset,
start_date,
end_date,
minid_start=300,
minid_end=2114,
database='hist_data')
mdf['contract'] = asset
else:
mdf = misc.nearby(asset,
nearby,
start_date,
end_date,
rollrule,
'm',
need_shift=True,
database='hist_data')
mdf = misc.cleanup_mindata(mdf, asset)
xdf = dh.conv_ohlc_freq(mdf,
freq,
extra_cols=['contract'],
bar_func=dh.bar_conv_func2)
return xdf
def load_data(self, idx):
asset = self.sim_assets[idx]
for prod in asset:
if prod in self.data_store:
continue
ticker = prod
if '$' not in ticker:
ticker_sp = [ticker, 'spot']
else:
ticker_sp = ticker.split('$')
ticker = ticker_sp[0]
postfix = '_daily'
if self.sim_freq == 'm':
postfix = '_min'
dbtable = ticker_sp[-1] + postfix
if ticker_sp[-1] in ['spot']:
field_id = 'spotID'
elif ticker_sp[-1] in ['ccy']:
field_id = 'instID'
if len(ticker_sp) > 2:
nb = int(ticker_sp[1])
if len(ticker_sp) > 3:
rollrule = ticker_sp[2]
else:
rollrule = '-1b'
df = misc.nearby(ticker,
nb,
self.config['start_date'],
self.config['end_date'],
rollrule,
self.sim_freq,
need_shift=self.need_shift,
database=self.config.get(
'dbconfig',
dbaccess.dbconfig)['database'])
else:
cnx = dbaccess.connect(
**self.config.get('dbconfig', dbaccess.dbconfig))
if self.sim_freq == 'd':
df = dbaccess.load_daily_data_to_df(
cnx,
dbtable,
ticker,
self.config['start_date'],
self.config['end_date'],
index_col='date',
field=field_id)
else:
minid_start = 1500
minid_end = 2114
if ticker in misc.night_session_markets:
minid_start = 300
df = dbaccess.load_min_data_to_df(
cnx, dbtable, ticker, self.config['start_date'],
self.config['end_date'], minid_start, minid_end)
df['contract'] = ticker
if self.sim_freq == 'm':
df = misc.cleanup_mindata(df, ticker)
df.columns = [(prod, col) for col in df.columns]
self.data_store[prod] = df
def load_data(self, idx):
asset = self.sim_assets[idx]
for prod in asset:
df = misc.nearby(prod, self.config['nearby'], self.config['start_date'], self.config['end_date'],
self.config['rollrule'], self.sim_freq, need_shift = self.need_shift)
if self.sim_freq == 'm':
df = misc.cleanup_mindata(df, prod)
self.data_store[prod] = df
def conv_db_htick2min(db_table,
inst_file,
out_table='hist_fut_min',
database='hist_data',
dstep=10):
conf_dict = {}
instIDs = []
if inst_file == '':
instIDs = get_col_dist_values(database + '.' + db_table, 'instID', {})
conf_dict = {'instIDs': instIDs}
try:
inst_file = 'instID_file.json'
with open(inst_file, 'w') as ofile:
json.dump(conf_dict, ofile)
except:
pass
else:
with open(inst_file, 'r') as infile:
conf_dict = json.load(infile)
instIDs = conf_dict['instIDs']
dbconfig = copy.deepcopy(dbaccess.dbconfig)
dbconfig['database'] = database
cnx = dbaccess.connect(**dbconfig)
for inst in instIDs:
field_dict = {'instID': "\'" + inst + "\'"}
datestr_list = get_col_dist_values(database + '.' + db_table, 'date',
field_dict)
mdata = pd.DataFrame()
prod = misc.inst2product(inst)
exch = misc.inst2exch(inst)
num_run = (len(datestr_list) + dstep - 1) / dstep
for idx in range(num_run):
s_idx = idx * dstep
e_idx = min((idx + 1) * dstep - 1, len(datestr_list) - 1)
sdate = datetime.datetime.strptime(datestr_list[s_idx],
"%Y-%m-%d").date()
edate = datetime.datetime.strptime(datestr_list[e_idx],
"%Y-%m-%d").date()
df = load_hist_tick(db_table, inst, sdate, edate)
mdf = conv_ohlc_freq(df, '1Min')
mdf['min_id'] = (
(mdf.index.hour + 6) % 24) * 100 + mdf.index.minute
mdf = misc.cleanup_mindata(mdf, prod)
mdf.index.name = 'datetime'
mdf['instID'] = inst
mdf['exch'] = exch
mdf = mdf.reset_index()
mdf.set_index(['instID', 'exch', 'datetime'], inplace=True)
mdf.to_sql(name=out_table,
flavor='mysql',
con=cnx,
if_exists='append')
cnx.commit()
print inst, sdate, edate, len(mdf)
cnx.close()
return
def get_cont_data(asset, start_date, end_date, freq = '1m', nearby = 1, rollrule = '-10b'):
cnx = dbaccess.connect(**dbaccess.hist_dbconfig)
if nearby == 0:
mdf = dbaccess.load_min_data_to_df(cnx, 'fut_min', asset, start_date, end_date, minid_start = 300, minid_end = 2114, database = 'hist_data')
mdf['contract'] = asset
else:
mdf = misc.nearby(asset, nearby, start_date, end_date, rollrule, 'm', need_shift=True, database = 'hist_data')
mdf = misc.cleanup_mindata(mdf, asset)
xdf = dh.conv_ohlc_freq(mdf, freq, extra_cols = ['contract'], bar_func = dh.bar_conv_func2)
return xdf
def load_data(self, idx):
asset = self.sim_assets[idx]
for prod in asset:
df = misc.nearby(prod,
self.config['nearby'],
self.config['start_date'],
self.config['end_date'],
self.config['rollrule'],
self.sim_freq,
need_shift=self.need_shift)
if self.sim_freq == 'm':
df = misc.cleanup_mindata(df, prod)
self.data_store[prod] = df
def load_data(self, idx):
asset = self.sim_assets[idx]
for prod in asset:
if prod in self.data_store:
continue
ticker = prod
if '$' not in ticker:
ticker_sp = [ticker, 'spot']
else:
ticker_sp = ticker.split('$')
ticker = ticker_sp[0]
postfix = '_daily'
if self.sim_freq == 'm':
postfix = '_min'
dbtable = ticker_sp[-1] + postfix
if ticker_sp[-1] in ['spot']:
field_id = 'spotID'
elif ticker_sp[-1] in ['ccy']:
field_id = 'instID'
if len(ticker_sp) > 2:
nb = int(ticker_sp[1])
if len(ticker_sp) > 3:
rollrule = ticker_sp[2]
else:
rollrule = '-1b'
df = misc.nearby(ticker, nb, self.config['start_date'], self.config['end_date'], rollrule,
self.sim_freq, need_shift = self.need_shift,
database = self.config.get('dbconfig', dbaccess.dbconfig)['database'])
else:
cnx = dbaccess.connect(**self.config.get('dbconfig', dbaccess.dbconfig))
if self.sim_freq == 'd':
df = dbaccess.load_daily_data_to_df(cnx, dbtable, ticker,
self.config['start_date'], self.config['end_date'], index_col='date',
field = field_id)
else:
minid_start = 1500
minid_end = 2114
if ticker in misc.night_session_markets:
minid_start = 300
df = dbaccess.load_min_data_to_df(cnx, dbtable, ticker, self.config['start_date'],
self.config['end_date'], minid_start, minid_end)
df['contract'] = ticker
if self.sim_freq == 'm':
df = misc.cleanup_mindata(df, ticker)
df.columns = [(prod, col) for col in df.columns]
self.data_store[prod] = df