def get_code_search_loop(num_input, code='', timed=60, dayl='10', ave=None):
# if not status:
#
if cct.get_work_time():
if code == num_input:
get_today_tick_ave(code, ave)
else:
ave = get_multiday_ave_compare(num_input, dayl)
time.sleep(timed)
return ave
def get_code_search_loop(num_input, code='', timed=60, dayl='10', ave=None):
# if not status:
#
if cct.get_work_time():
if code == num_input:
get_today_tick_ave(code, ave)
else:
ave = get_multiday_ave_compare(num_input, dayl)
time.sleep(timed)
return ave
df_count = len(df)
now_count = len(top_now)
del df
gc.collect()
radio_t = cct.get_work_time_ratio()
time_Rt = time.time()
# top_now = top_now[top_now.buy > 0]
time_d = time.time()
if time_d - time_s > delay_time:
status_change = True
time_s = time.time()
top_all = pd.DataFrame()
else:
status_change = False
# print ("Buy>0:%s" % len(top_now[top_now['buy'] > 0])),
if len(top_now) > 10 or cct.get_work_time():
# if len(top_now) > 10 and len(top_now[:20][top_now[:20]['buy'] > 0]) > 3:
# if len(top_now) > 10 and not top_now[:1].buy.values == 0:
# top_now=top_now[top_now['percent']>=0]
if len(top_all) == 0:
top_all = top_now
# top_all['llow'] = 0
# top_all['lastp'] = 0
# top_all = top_all[top_all.buy > 0]
codelist = top_all.index.tolist()
log.info('toTDXlist:%s' % len(codelist))
tdxdata = tdd.get_tdx_all_day_LastDF(codelist)
log.debug("TdxLastP: %s %s" % (len(tdxdata), tdxdata.columns.values))
tdxdata.rename(columns={'low': 'llow'}, inplace=True)
tdxdata.rename(columns={'high': 'lhigh'}, inplace=True)
tdxdata.rename(columns={'close': 'lastp'}, inplace=True)
# top_now = tdd.getSinaAlldf(market='混改', filename='mnbk',vol=ct.json_countVol, type=ct.json_countType)
# top_dif = top_now
# top_now.to_hdf("testhdf5", 'marketDD', format='table', complevel=9)
now_count = len(top_now)
radio_t = cct.get_work_time_ratio()
# top_now = top_now[top_now.buy > 0]
time_d = time.time()
if time_d - time_s > delay_time:
status_change = True
time_s = time.time()
top_all = pd.DataFrame()
else:
status_change = False
# print ("Buy>0:%s" % len(top_now[top_now['buy'] > 0])),
if len(top_now) > 10 or cct.get_work_time():
time_Rt = time.time()
if len(top_all) == 0 and len(lastpTDX_DF) == 0:
cct.get_terminal_Position(position=sys.argv[0])
time_Rt = time.time()
top_all, lastpTDX_DF = tdd.get_append_lastp_to_df(
top_now,
lastpTDX_DF=None,
dl=duration_date,
end=end_date,
ptype=ptype,
filter=filter,
power=ct.lastPower,
lastp=False,
newdays=newdays)
def get_tdx_append_now_df(code, type="f", start=None, end=None):
# start=cct.day8_to_day10(start)
# end=cct.day8_to_day10(end)
df = get_tdx_Exp_day_to_df(code, type, start, end).sort_index(ascending=True)
# print df[:1]
if not end == None:
if not end == df.index[-1]:
print (end, df.index[-1])
return df
today = cct.get_today()
if len(df) > 0:
tdx_last_day = df.index[-1]
duration = cct.get_today_duration(tdx_last_day)
log.debug("duration:%s" % duration)
else:
tdx_last_day = None
duration = 1
log.debug("tdx_last_day:%s" % tdx_last_day)
index_status = False
if code == "999999":
code = "sh"
index_status = True
elif code.startswith("399"):
index_status = True
for k in INDEX_LIST.keys():
if INDEX_LIST[k].find(code) > 0:
code = k
log.debug("duration:%s" % duration)
if duration >= 1:
ds = ts.get_hist_data(code, start=tdx_last_day, end=today)
if ds is not None and len(ds) > 1:
ds = ds[: len(ds) - 1]
ds["code"] = code
ds["vol"] = 0
ds = ds.loc[:, ["code", "open", "high", "low", "close", "vol", "volume"]]
ds.rename(columns={"volume": "amount"}, inplace=True)
ds.sort_index(ascending=True, inplace=True)
log.debug("ds:%s" % ds[:1])
df = df.append(ds)
# pd.concat([df,ds],axis=0, join='outer')
# result=pd.concat([df,ds])
if cct.get_work_time() and not index_status:
dm = rl.get_sina_Market_json("all", showtime=False).set_index("code")
# dm=dm.drop_duplicates()
log.debug("dm:%s" % dm[-1:])
dm.rename(columns={"volume": "amount", "trade": "close"}, inplace=True)
# c_name=dm.loc[code,['name']]
dm_code = dm.loc[code, ["open", "high", "low", "close", "amount"]]
log.debug("dm_code:%s" % dm_code["amount"])
dm_code["amount"] = round(float(dm_code["amount"]) / 100, 2)
dm_code["code"] = code
dm_code["vol"] = 0
# dm_code['date']=today
dm_code.name = today
df = df.append(dm_code)
# df['name']=c_name
# log.debug("c_name:%s"%(c_name))
log.debug("df[-3:]:%s" % (df[-2:]))
df["name"] = dm.loc[code, "name"]
log.debug("df:%s" % df[-2:])
return df
def get_tdx_append_now_df_api(code, type="f", start=None, end=None):
import sina_data
# start=cct.day8_to_day10(start)
# end=cct.day8_to_day10(end)
df = get_tdx_Exp_day_to_df(code, type, start, end).sort_index(ascending=True)
# print df[:1]
if not end == None:
if end < df.index[-1]:
print (end, df.index[-1])
return df
today = cct.get_today()
if len(df) > 0:
tdx_last_day = df.index[-1]
duration = cct.get_today_duration(tdx_last_day)
log.debug("duration:%s" % duration)
else:
tdx_last_day = None
duration = 1
log.debug("tdx_last_day:%s" % tdx_last_day)
index_status = False
if code == "999999":
code = "sh"
index_status = True
elif code.startswith("399"):
index_status = True
for k in INDEX_LIST.keys():
if INDEX_LIST[k].find(code) > 0:
code = k
log.debug("duration:%s" % duration)
if duration >= 1:
try:
ds = ts.get_hist_data(code, start=tdx_last_day, end=today)
# ds = ts.get_h_data('000001', start=tdx_last_day, end=today,index=index_status)
# df.index = pd.to_datetime(df.index)
except (IOError, EOFError, Exception) as e:
print "Error duration", e
if index_status:
code_t = INDEX_LIST[code][2:]
ds = ts.get_h_data(code_t, start=tdx_last_day, end=today, index=index_status)
df.index = pd.to_datetime(df.index)
if ds is not None and len(ds) > 1:
ds = ds[: len(ds) - 1]
if index_status:
code_t = INDEX_LIST[code][2:]
if code == "sh":
code = "999999"
else:
code = code_t
ds["code"] = code
else:
ds["code"] = code
# ds['vol'] = 0
ds = ds.loc[:, ["code", "open", "high", "low", "close", "volume", "amount"]]
# ds.rename(columns={'volume': 'amount'}, inplace=True)
ds.rename(columns={"volume": "vol"}, inplace=True)
ds.sort_index(ascending=True, inplace=True)
log.debug("ds:%s" % ds[:1])
df = df.append(ds)
# pd.concat([df,ds],axis=0, join='outer')
# result=pd.concat([df,ds])
if cct.get_work_time():
# dm = rl.get_sina_Market_json('all').set_index('code')
if index_status:
code = INDEX_LIST[code]
dm = sina_data.Sina().get_stock_code_data(code).set_index("code")
if dm is not None and not dm.empty:
# dm=dm.drop_duplicates()
log.debug("dm:%s" % dm[-1:])
dm.rename(columns={"volume": "amount", "turnover": "vol"}, inplace=True)
# c_name=dm.loc[code,['name']]
dm_code = dm.loc[:, ["open", "high", "low", "close", "amount"]]
log.debug("dm_code:%s" % dm_code["amount"])
dm_code["amount"] = round(float(dm_code["amount"]) / 100, 2)
dm_code["code"] = code
dm_code["vol"] = 0
# dm_code['date']=today
dm_code.name = today
df = df.append(dm_code)
# df['name']=c_name
# log.debug("c_name:%s"%(c_name))
log.debug("df[-3:]:%s" % (df[-2:]))
# df['name'] = dm.loc[code, 'name']
log.debug("df:%s" % df[-2:])
return df
print rl.format_for_print(top_all[:10])
# print "staus",status
if status:
for code in top_all[:10].index:
code = re.findall("(\d+)", code)
if len(code) > 0:
code = code[0]
kind = sl.get_multiday_ave_compare_silent(code)
top_all = top_bak
del top_bak
gc.collect()
else:
print "no data"
int_time = cct.get_now_time_int()
if cct.get_work_time():
if int_time < 930:
while 1:
time.sleep(60)
if cct.get_now_time_int() < 931:
time.sleep(60)
print ".",
else:
top_all = pd.DataFrame()
time_s = time.time()
print "."
break
else:
time.sleep(60)
elif cct.get_work_duration():
while 1:
top_now = tdd.getSinaAlldf(market='all',
vol=ct.json_countVol,
vtype=ct.json_countType)
# top_now = tdd.getSinaAlldf(market=u'次新股',filename='cxg', vol=ct.json_countVol, vtype=ct.json_countType)
now_count = len(top_now)
radio_t = cct.get_work_time_ratio()
# top_now = top_now[top_now.buy > 0]
time_d = time.time()
if time_d - time_s > delay_time:
status_change = True
time_s = time.time()
top_all = pd.DataFrame()
else:
status_change = False
# print ("Buy>0:%s" % len(top_now[top_now['buy'] > 0])),
if len(top_now) > 0 or cct.get_work_time():
# time_Rt = time.time()
if len(top_all) == 0 and len(lastpTDX_DF) == 0:
cct.get_terminal_Position(position=sys.argv[0])
# time_Rt = time.time()
print "term:%s" % (cct.get_terminal_Position(
cmd='DurationDn.py')),
if cct.get_terminal_Position(cmd='DurationDn.py') > 1:
top_all, lastpTDX_DF = tdd.get_append_lastp_to_df(
top_now,
lastpTDX_DF=None,
dl=duration_date,
end=end_date,
ptype=ptype,
filter=filter,
# str.isdigit()是用来判断字符串是否纯粹由数字组成
elif not num_input or not len(num_input) == 6:
print("Please input 6 code:or exit")
num_input = ''
if num_input:
if ave == None:
ave = get_code_search_loop(num_input, code, dayl=days)
else:
ave = get_code_search_loop(num_input,
code,
dayl=days,
ave=ave)
code = num_input
int_time = cct.get_now_time_int()
if cct.get_work_time():
if int_time < 1000:
cct.sleep(60)
else:
cct.sleep(ct.duration_sleep_time)
int_time = cct.get_now_time_int()
if cct.get_work_time():
if int_time < 930:
while 1:
cct.sleep(60)
if cct.get_now_time_int() < 931:
cct.sleep(60)
print ".",
else:
# cct.sleep(random.randint(0, 30))
# print "."
def all(self):
self.stockcode = StockCode()
self.stock_code_path = self.stockcode.stock_code_path
self.stock_codes = self.stockcode.get_stock_codes()
self.load_stock_codes()
# print "stocks:",len(self.stock_codes)
self.stock_codes = [elem for elem in self.stock_codes if elem.startswith(('6','30','00'))]
h5 = h5a.load_hdf_db(self.hdf_name,self.table, code_l=self.stock_codes)
# print "stocks:",len(self.stock_codes)
if h5 is not None and len(h5) > 0:
o_time = h5[h5.timel <> 0].timel
if len(o_time) > 0:
o_time = o_time[0]
l_time = time.time() - o_time
sina_limit_time = ct.sina_limit_time
sina_time_status = (cct.get_work_day_status() and 915 < cct.get_now_time_int() < 926)
# return_hdf_status = not cct.get_work_day_status() or (cct.get_work_day_status() and (cct.get_work_time() and l_time < sina_limit_time))
return_hdf_status = not cct.get_work_time() or (cct.get_work_time() and l_time < sina_limit_time)
log.info("915:%s sina_time:%0.2f limit:%s"%(sina_time_status,l_time,sina_limit_time))
if sina_time_status and l_time < 8:
log.info("open 915 hdf ok:%s"%(len(h5)))
return h5
elif not sina_time_status and return_hdf_status:
log.info("return hdf5 data:%s"%(len(h5)))
return h5
else:
log.info("no return hdf5:%s"%(len(h5)))
# self.stock_with_exchange_list = list(
# map(lambda stock_code: ('sh%s' if stock_code.startswith(('5', '6', '9')) else 'sz%s') % stock_code,
# self.stock_codes))
self.stock_with_exchange_list = list(
map(lambda stock_code: ('sh%s' if stock_code.startswith(('6')) else 'sz%s') % stock_code,
self.stock_codes))
self.stock_list = []
self.request_num = len(self.stock_with_exchange_list) // self.max_num
for range_start in range(self.request_num):
num_start = self.max_num * range_start
num_end = self.max_num * (range_start + 1)
request_list = ','.join(
self.stock_with_exchange_list[num_start:num_end])
self.stock_list.append(request_list)
# print len(self.stock_with_exchange_list), num_end
if len(self.stock_with_exchange_list) > num_end:
request_list = ','.join(
self.stock_with_exchange_list[num_end:])
self.stock_list.append(request_list)
self.request_num += 1
# a = 0
# for x in range(self.request_num):
# print x
# i = len(self.stock_list[x].split(','))
# print i
# a += i
# print a
log.debug('all:%s' % len(self.stock_list))
# log.error('all:%s req:%s' %
# (len(self.stock_list), len(self.stock_list)))
return self.get_stock_data()
def load_hdf_db(fname,
table='all',
code_l=None,
timelimit=True,
index=False,
limit_time=ct.h5_limit_time,
dratio_limit=0.12):
time_t = time.time()
global RAMDISK_KEY, INIT_LOG_Error
if not RAMDISK_KEY < 1:
return None
df = None
dd = None
if code_l is not None:
if table is not None:
with SafeHDFStore(fname) as store:
if store is not None:
if '/' + table in store.keys():
dd = store[table]
if dd is not None and len(dd) > 0:
if index:
code_l = map((lambda x: str(1000000 - int(x))
if x.startswith('0') else x), code_l)
dif_co = list(set(dd.index) & set(code_l))
dratio = (float(len(code_l)) - float(len(dif_co))) / \
float(len(code_l))
# if dratio < 0.1 or len(dd) > 3100:
if dratio < dratio_limit:
log.info("find all:%s :%s %0.2f" %
(len(code_l), len(code_l) - len(dif_co), dratio))
if timelimit and len(dd) > 0:
dd = dd.loc[dif_co]
o_time = dd[dd.timel <> 0].timel.tolist()
# if fname == 'powerCompute':
# o_time = sorted(set(o_time),reverse=True)
o_time = sorted(set(o_time), reverse=False)
o_time = [time.time() - t_x for t_x in o_time]
if len(dd) > 0:
# if len(dd) > 0 and (not cct.get_work_time() or len(o_time) <= ct.h5_time_l_count):
l_time = np.mean(o_time)
return_hdf_status = (not cct.get_work_time()) or (
cct.get_work_time() and l_time < limit_time)
# return_hdf_status = l_time < limit_time
# print return_hdf_status,l_time,limit_time
if return_hdf_status:
df = dd
log.info(
"return hdf: %s timel:%s l_t:%s hdf ok:%s"
% (fname, len(o_time), l_time, len(df)))
else:
log.error(
"%s %s o_time:%s %s" %
(fname, table, len(o_time),
[time.time() - t_x for t_x in o_time[:3]]))
log.info('fname:%s l_time:%s' %
(fname, [time.time() - t_x
for t_x in o_time]))
else:
df = dd.loc[dif_co]
else:
if len(code_l
) > ct.h5_time_l_count * 10 and INIT_LOG_Error < 5:
# INIT_LOG_Error += 1
log.error(
"fn:%s cl:%s h5:%s don't find:%s dra:%0.2f log_err:%s"
% (fname, len(code_l), len(dd), len(code_l) -
len(dif_co), dratio, INIT_LOG_Error))
else:
log.error("%s is not find %s" % (fname, table))
else:
# h5 = get_hdf5_file(fname,wr_mode='r')
if table is not None:
with SafeHDFStore(fname) as store:
if store is not None:
if '/' + table in store.keys():
dd = store[table]
if dd is not None and len(dd) > 0:
if timelimit:
if dd is not None and len(dd) > 0:
o_time = dd[dd.timel <> 0].timel.tolist()
o_time = sorted(set(o_time))
o_time = [time.time() - t_x for t_x in o_time]
if len(o_time) > 0:
l_time = np.mean(o_time)
# l_time = time.time() - l_time
# return_hdf_status = not cct.get_work_day_status() or not cct.get_work_time() or (cct.get_work_day_status() and (cct.get_work_time() and l_time < limit_time))
return_hdf_status = not cct.get_work_time() or (
cct.get_work_time() and l_time < limit_time)
log.info("return_hdf_status:%s time:%0.2f" %
(return_hdf_status, l_time))
if return_hdf_status:
log.info("return hdf5 data:%s o_time:%s" %
(len(dd), len(o_time)))
df = dd
else:
log.info("no return time hdf5:%s" % (len(dd)))
log.info('fname:%s l_time:%s' %
(fname, [time.time() - t_x
for t_x in o_time]))
else:
df = dd
else:
log.error("%s is not find %s" % (fname, table))
else:
log.error("% / table is Init None:%s" (fname, table))
if df is not None and len(df) > 0:
df = df.fillna(0)
if 'timel' in df.columns:
time_list = df.timel.tolist()
# time_list = sorted(set(time_list),key = time_list.index)
time_list = sorted(set(time_list))
# log.info("test:%s"%(sorted(set(time_list),key = time_list.index)))
if time_list is not None and len(time_list) > 0:
df['timel'] = time_list[0]
log.info("load hdf times:%s" %
([time.time() - t_x for t_x in time_list]))
log.info("load_hdf_time:%0.2f" % (time.time() - time_t))
# if df is not None and len(df) > 1:
# df = df.drop_duplicates()
return df
