def validate_db_data(tday, filter=False):
all_insts = misc.filter_main_cont(tday, filter)
data_count = {}
inst_list = {'min': [], 'daily': []}
for instID in all_insts:
df = mysqlaccess.load_daily_data_to_df('fut_daily', instID, tday, tday)
if len(df) <= 0:
inst_list['daily'].append(instID)
elif (df.close[-1] == 0) or (df.high[-1]
== 0) or (df.low[-1]
== 0) or df.open[-1] == 0:
inst_list['daily'].append(instID)
df = mysqlaccess.load_min_data_to_df('fut_min',
instID,
tday,
tday,
minid_start=300,
minid_end=2115,
database='blueshale')
if len(df) <= 100:
output = instID + ':' + str(len(df))
inst_list['min'].append(output)
elif df.min_id < 2055:
output = instID + ': end earlier'
inst_list['min'].append(output)
print inst_list
def load_csv_to_db(edate,
save_loc='C:\\dev\\data\\',
freq='m',
is_replace=False):
cnx = dbaccess.connect(**dbaccess.dbconfig)
cont_list = misc.filter_main_cont(edate, False)
if freq not in ['m', 'd']:
return False
for cont in cont_list:
if freq == 'm':
filename = save_loc + cont + '_min.csv'
else:
filename = save_loc + cont + '_daily.csv'
if not os.path.isfile(filename):
continue
data_reader = csv.reader(file(filename, 'rb'))
mindata_list = []
for idx, line in enumerate(data_reader):
if idx > 0:
if 'nan' in [line[0], line[2], line[3], line[4], line[6]]:
continue
min_data = {}
min_data['volume'] = int(float(line[0]))
if min_data['volume'] <= 0: continue
dtime = datetime.datetime.strptime(line[1],
'%Y-%m-%d %H:%M:%S.%f')
if freq == 'm':
min_data['datetime'] = dtime.replace(microsecond=0)
else:
min_data['date'] = dtime.date()
min_data['high'] = float(line[2])
min_data['low'] = float(line[3])
min_data['close'] = float(line[4])
if line[5] == 'nan':
oi = 0
else:
oi = int(float(line[5]))
min_data['openInterest'] = oi
min_data['open'] = float(line[6])
if freq == 'm':
min_data['min_id'] = get_min_id(dtime)
trading_date = dtime.date()
if min_data['min_id'] < 600:
trading_date = misc.day_shift(trading_date, '1b')
min_data['date'] = trading_date
mindata_list.append(min_data)
else:
print cont
dbaccess.insert_daily_data(cont,
min_data,
is_replace=is_replace,
dbtable='fut_daily')
if freq == 'm':
print cont
dbaccess.bulkinsert_min_data(cont,
mindata_list,
is_replace=is_replace)
return True
def save(name, config_file, tday, filter):
base.config_logging(name + "\\" + name + ".log", level=logging.DEBUG,
format = '%(name)s:%(funcName)s:%(lineno)d:%(asctime)s %(levelname)s %(message)s',
to_console = True,
console_level = logging.INFO)
scur_day = datetime.datetime.strptime(tday, '%Y%m%d').date()
filter_flag = (int(filter)>0)
with open(config_file, 'r') as infile:
config = json.load(infile)
save_agent = agent.SaveAgent(name = name, tday = scur_day, config = config)
curr_insts = misc.filter_main_cont(tday, filter_flag)
for inst in curr_insts:
save_agent.add_instrument(inst)
try:
save_agent.restart()
while 1:
time.sleep(1)
except KeyboardInterrupt:
save_agent.exit()
def validate_db_data(tday, filter = False):
all_insts = misc.filter_main_cont(tday, filter)
data_count = {}
inst_list = {'min': [], 'daily': [] }
cnx = dbaccess.connect(**dbaccess.dbconfig)
for instID in all_insts:
df = dbaccess.load_daily_data_to_df(cnx, 'fut_daily', instID, tday, tday)
if len(df) <= 0:
inst_list['daily'].append(instID)
elif (df.close[-1] == 0) or (df.high[-1] == 0) or (df.low[-1] == 0) or df.open[-1] == 0:
inst_list['daily'].append(instID)
df = dbaccess.load_min_data_to_df(cnx, 'fut_min', instID, tday, tday, minid_start=300, minid_end=2115)
if len(df) <= 100:
output = instID + ':' + str(len(df))
inst_list['min'].append(output)
elif df.min_id < 2055:
output = instID + ': end earlier'
inst_list['min'].append(output)
print inst_list
def save(name, config_file, tday, filter):
base.config_logging(
name + "\\" + name + ".log",
level=logging.DEBUG,
format=
'%(name)s:%(funcName)s:%(lineno)d:%(asctime)s %(levelname)s %(message)s',
to_console=True,
console_level=logging.INFO)
scur_day = datetime.datetime.strptime(tday, '%Y%m%d').date()
filter_flag = (int(filter) > 0)
with open(config_file, 'r') as infile:
config = json.load(infile)
save_agent = agent.SaveAgent(name=name, tday=scur_day, config=config)
curr_insts = misc.filter_main_cont(tday, filter_flag)
for inst in curr_insts:
save_agent.add_instrument(inst)
try:
save_agent.restart()
while 1:
time.sleep(1)
except KeyboardInterrupt:
save_agent.exit()
