def import_datayes_daily_data(start_date, end_date, cont_list = [], is_replace = False):
numdays = (end_date - start_date).days + 1
date_list = [start_date + datetime.timedelta(days=x) for x in range(0, numdays) ]
date_list = [ d for d in date_list if (d.weekday()< 5) and (d not in misc.CHN_Holidays)]
for d in date_list:
cnt = 0
dstring = d.strftime('%Y%m%d')
ts.set_token(misc.datayes_token)
mkt = ts.Market()
df = mkt.MktFutd(tradeDate = dstring)
if len(df.ticker) == 0:
continue
for cont in df.ticker:
if (len(cont_list) > 0) and (cont not in cont_list):
continue
data = df[df.ticker==cont]
if len(data) == 0:
print 'no data for %s for %s' % (cont, dstring)
else:
data_dict = {}
data_dict['date'] = d
data_dict['open'] = float(data.openPrice)
data_dict['close'] = float(data.closePrice)
data_dict['high'] = float(data.highestPrice)
data_dict['low'] = float(data.lowestPrice)
data_dict['volume'] = int(data.turnoverVol)
data_dict['openInterest'] = int(data.openInt)
if data_dict['volume'] > 0:
cnt += 1
db.insert_daily_data(cont, data_dict, is_replace = is_replace, dbtable = 'fut_daily')
print 'date=%s, insert count = %s' % (d, cnt)
def fix_daily_by_tick(contlist, sdate, edate, is_forced=False):
res = {}
for inst in contlist:
product = misc.inst2product(inst)
start_tick = 1500000
end_tick = 2100000
if product in misc.night_session_markets:
start_tick = 300000
elif product in ['IF', 'TF']:
start_tick = 1515000
end_tick = 2115000
ddf = mysqlaccess.load_daily_data_to_df('fut_daily', inst, sdate,
edate)
tdf = mysqlaccess.load_tick_to_df('fut_tick',
inst,
sdate,
edate,
start_tick=start_tick,
end_tick=end_tick)
for d in list(set(tdf.date)):
if (is_forced) or (d not in ddf.index) or (ddf.ix(d, 'open') == 0):
df = tdf[tdf['date'] == d].sort(['tick_id'])
ddata = {}
ddata['date'] = d
ddata['open'] = float(df.iloc[0].price)
ddata['close'] = float(df.iloc[-1].price)
ddata['high'] = float(df.iloc[-1].high)
ddata['low'] = float(df.iloc[-1].low)
ddata['volume'] = int(df.iloc[-1].volume)
ddata['openInterest'] = int(df.iloc[-1].openInterest)
print inst, ddata
res[(inst, d)] = ddata
mysqlaccess.insert_daily_data(inst, ddata, is_forced)
return res
def fix_daily_data(contlist, sdate, edate):
for inst in contlist:
ddf = mysqlaccess.load_daily_data_to_df('fut_daily', inst, sdate,
edate)
mdf = mysqlaccess.load_min_data_to_df('fut_min',
inst,
sdate,
edate,
minid_start=300,
minid_end=2115)
dailydata = data_handler.conv_ohlc_freq(mdf, 'D')
for dd in dailydata.index:
d = dd.date()
dslice = dailydata.ix[dd]
if d not in ddf.index:
ddata = {}
ddata['date'] = d
ddata['open'] = float(dslice.open)
ddata['close'] = float(dslice.close)
ddata['high'] = float(dslice.high)
ddata['low'] = float(dslice.low)
ddata['volume'] = int(dslice.volume)
ddata['openInterest'] = int(dslice.openInterest)
print inst, ddata
mysqlaccess.insert_daily_data(inst, ddata)
def fix_daily_by_tick(contlist, sdate, edate, is_forced=False):
res = {}
for inst in contlist:
product = misc.inst2product(inst)
start_tick= 1500000
end_tick = 2100000
if product in misc.night_session_markets:
start_tick = 300000
elif product in ['IF','TF']:
start_tick = 1515000
end_tick = 2115000
ddf = mysqlaccess.load_daily_data_to_df('fut_daily', inst, sdate, edate)
tdf = mysqlaccess.load_tick_to_df('fut_tick', inst, sdate, edate, start_tick=start_tick, end_tick = end_tick)
for d in list(set(tdf.date)):
if (is_forced) or (d not in ddf.index) or (ddf.ix(d, 'open')==0):
df = tdf[tdf['date']==d].sort(['tick_id'])
ddata = {}
ddata['date'] = d
ddata['open'] = float(df.iloc[0].price)
ddata['close'] = float(df.iloc[-1].price)
ddata['high'] = float(df.iloc[-1].high)
ddata['low'] = float(df.iloc[-1].low)
ddata['volume'] = int(df.iloc[-1].volume)
ddata['openInterest'] = int(df.iloc[-1].openInterest)
print inst, ddata
res[(inst,d)] = ddata
mysqlaccess.insert_daily_data(inst, ddata, is_forced)
return res
def load_csv_to_db(edate,
save_loc='C:\\dev\\data\\',
freq='m',
is_replace=False):
cont_list = misc.filter_main_cont(edate, False)
if freq not in ['m', 'd']:
return False
for cont in cont_list:
if freq == 'm':
filename = save_loc + cont + '_min.csv'
else:
filename = save_loc + cont + '_daily.csv'
if not os.path.isfile(filename):
continue
data_reader = csv.reader(file(filename, 'rb'))
mindata_list = []
for idx, line in enumerate(data_reader):
if idx > 0:
if 'nan' in [line[0], line[2], line[3], line[4], line[6]]:
continue
min_data = {}
min_data['volume'] = int(float(line[0]))
if min_data['volume'] <= 0: continue
dtime = datetime.datetime.strptime(line[1],
'%Y-%m-%d %H:%M:%S.%f')
if freq == 'm':
min_data['datetime'] = dtime.replace(microsecond=0)
else:
min_data['date'] = dtime.date()
min_data['high'] = float(line[2])
min_data['low'] = float(line[3])
min_data['close'] = float(line[4])
if line[5] == 'nan':
oi = 0
else:
oi = int(float(line[5]))
min_data['openInterest'] = oi
min_data['open'] = float(line[6])
if freq == 'm':
min_data['min_id'] = get_min_id(dtime)
trading_date = dtime.date()
if min_data['min_id'] < 600:
trading_date = misc.day_shift(trading_date, '1b')
min_data['date'] = trading_date
mindata_list.append(min_data)
else:
print cont
db.insert_daily_data(cont,
min_data,
is_replace=is_replace,
dbtable='fut_daily')
if freq == 'm':
print cont
db.bulkinsert_min_data(cont, mindata_list, is_replace=is_replace)
return True
def write_mkt_data(self, event):
inst = event.dict['instID']
type = event.dict['type']
data = event.dict['data']
if type == EVENT_MIN_BAR:
mysqlaccess.insert_min_data(inst, data, dbtable = self.min_db_table)
elif type == EVENT_TICK:
mysqlaccess.bulkinsert_tick_data(inst, data, dbtable = self.tick_db_table)
elif type == EVENT_MKTDATA_EOD:
mysqlaccess.insert_daily_data(inst, data, dbtable = self.daily_db_table)
else:
pass
def write_mkt_data(self, event):
inst = event.dict['instID']
type = event.dict['type']
data = event.dict['data']
if type == EVENT_MIN_BAR:
mysqlaccess.insert_min_data(inst, data, dbtable = self.min_db_table)
elif type == EVENT_TICK:
mysqlaccess.bulkinsert_tick_data(inst, data, dbtable = self.tick_db_table)
elif type == EVENT_MKTDATA_EOD:
mysqlaccess.insert_daily_data(inst, data, dbtable = self.daily_db_table)
else:
pass
def fix_daily_data(contlist, sdate, edate):
for inst in contlist:
ddf = mysqlaccess.load_daily_data_to_df('fut_daily', inst, sdate, edate)
mdf = mysqlaccess.load_min_data_to_df('fut_min', inst, sdate, edate, minid_start=300, minid_end = 2115)
dailydata = data_handler.conv_ohlc_freq(mdf, 'D')
for dd in dailydata.index:
d = dd.date()
dslice = dailydata.ix[dd]
if d not in ddf.index:
ddata = {}
ddata['date'] = d
ddata['open'] = float(dslice.open)
ddata['close'] = float(dslice.close)
ddata['high'] = float(dslice.high)
ddata['low'] = float(dslice.low)
ddata['volume'] = int(dslice.volume)
ddata['openInterest'] = int(dslice.openInterest)
print inst, ddata
mysqlaccess.insert_daily_data(inst, ddata)