def get_code_search_loop(num_input, code='', timed=60, dayl='10', ave=None):
# if not status:
#
if cct.get_work_time():
if code == num_input:
get_today_tick_ave(code, ave)
else:
ave = get_multiday_ave_compare(num_input, dayl)
time.sleep(timed)
return ave
# top_now = tdd.getSinaAlldf(market='���', filename='mnbk',vol=ct.json_countVol, vtype=ct.json_countType)
# top_dif = top_now
# top_now.to_hdf("testhdf5", 'marketDD', format='table', complevel=9)
now_count = len(top_now)
radio_t = cct.get_work_time_ratio()
# top_now = top_now[top_now.buy > 0]
time_d = time.time()
if time_d - time_s > delay_time:
status_change = True
time_s = time.time()
top_all = pd.DataFrame()
else:
status_change = False
# print ("Buy>0:%s" % len(top_now[top_now['buy'] > 0])),
if len(top_now) > 10 or cct.get_work_time():
time_Rt = time.time()
if len(top_all) == 0 and len(lastpTDX_DF) == 0:
cct.get_terminal_Position(position=sys.argv[0])
time_Rt = time.time()
top_all, lastpTDX_DF = tdd.get_append_lastp_to_df(
top_now, lastpTDX_DF=None, dl=duration_date, end=end_date, ptype=ptype, filter=filter, power=ct.lastPower, lastp=False, newdays=newdays, resample=resample)
log.debug("len:%s" % (len(top_all)))
# codelist = top_all.index.tolist()
# log.info('toTDXlist:%s' % len(codelist))
# # tdxdata = tdd.get_tdx_all_day_LastDF(codelist,dt=duration_date,ptype=ptype)
# # print "duration_date:%s ptype=%s filter:%s"%(duration_date, ptype,filter)
# # tdxdata = tdd.get_tdx_exp_all_LastDF(codelist, dt=duration_date, end=end_date,ptype=ptype,filter=filter)
# power=True
# log.debug("TdxLastP: %s %s" % (len(tdxdata), tdxdata.columns.values))
def format_response_data(self, index=False):
stocks_detail = ''.join(self.stock_data)
# print stocks_detail
result = self.grep_stock_detail.finditer(stocks_detail)
# stock_dict = dict()
list_s = []
for stock_match_object in result:
stock = stock_match_object.groups()
# print stock
# print stock
# fn=(lambda x:x)
# list.append(map(fn,stock))
# df = pd.DataFrame(list,columns=ct.SINA_Total_Columns)
# list_s.append({'code'})
list_s.append({
'code': stock[0],
'name': stock[1],
'open': float(stock[2]),
'close': float(stock[3]),
'now': float(stock[4]),
'high': float(stock[5]),
'low': float(stock[6]),
'buy': float(stock[7]),
'sell': float(stock[8]),
'volume': int(stock[9]),
'turnover': float(stock[10]),
# 'amount': float(stock[10]),
'b1_v': int(stock[11]),
'b1': float(stock[12]),
'b2_v': int(stock[13]),
'b2': float(stock[14]),
'b3_v': int(stock[15]),
'b3': float(stock[16]),
'b4_v': int(stock[17]),
'b4': float(stock[18]),
'b5_v': int(stock[19]),
'b5': float(stock[20]),
'a1_v': int(stock[21]),
'a1': float(stock[22]),
'a2_v': int(stock[23]),
'a2': float(stock[24]),
'a3_v': int(stock[25]),
'a3': float(stock[26]),
'a4_v': int(stock[27]),
'a4': float(stock[28]),
'a5_v': int(stock[29]),
'a5': float(stock[30]),
'dt': (stock[31]),
'ticktime': (stock[32])
})
# print list_s
# df = pd.DataFrame.from_dict(stock_dict,columns=ct.SINA_Total_Columns)
df = pd.DataFrame(list_s, columns=ct.SINA_Total_Columns)
# if self.index_status and cct.get_work_time():
# if self.index_status:
# if cct.get_work_time() or (cct.get_now_time_int() > 915) :
# df = df.drop('close', axis=1)
df.rename(columns={'close': 'llastp'}, inplace=True)
df['b1_vv'] = df['b1_v']
if (cct.get_now_time_int() > 915 and cct.get_now_time_int() < 926):
# df.rename(columns={'buy': 'close'}, inplace=True)
df['close'] = df['buy']
df['low'] = df['buy']
df['volume'] = ((df['b1_v'] + df['b2_v'])).map(lambda x: x)
# df['b1_v'] = ((df['b1_v'] + df['b2_v']) / 100 / 10000).map(lambda x: round(x, 1) + 0.01)
# df['b1_v'] = ((df['b1_v']) / 100 / 10000).map(lambda x: round(x, 1) + 0.01)
# df['b1_vv'] = map(lambda x: round(x / 100 / 10000, 1) + 0.01, df['b1_v'])
elif (cct.get_now_time_int() > 830 and cct.get_now_time_int() <= 915):
# df.rename(columns={'buy': 'close'}, inplace=True)
df['buy'] = df['llastp']
df['close'] = df['buy']
df['low'] = df['buy']
# df['b1_v'] = ((df['b1_v']) / df['volume'] * 100).map(lambda x: round(x, 1))
else:
# df['b1_v'] = ((df['b1_v']) / df['volume'] * 100).map(lambda x: round(x, 1))
# df.rename(columns={'now': 'close'}, inplace=True)
df['close'] = df['now']
df['nvol'] = df['volume']
df = df.drop_duplicates('code')
# df = df.loc[:, ct.SINA_Total_Columns_Clean]
# df = df.loc[:, ct.SINA_Total_Columns]
# df.rename(columns={'turnover': 'amount'}, inplace=True)
df = df.fillna(0)
# df = df.sort_values(by='code', ascending=0)
df = df.set_index('code')
if index:
df.index = map(
(lambda x: str(1000000 - int(x)) if x.startswith('0') else x),
df.index)
# print ("Market-df:%s %s time: %s" % (
# cct.get_now_time()))
log.info("hdf:all%s %s" % (len(df), len(self.stock_codes)))
dd = df.copy()
h5_fname = 'sina_MultiIndex_data'
h5_table = 'all' + '_' + str(ct.sina_limit_time)
if cct.get_now_time_int() >= 925 and not index and len(
df) > 3000 and cct.get_work_time():
time_s = time.time()
df.index = df.index.astype(str)
df.ticktime = df.ticktime.astype(str)
# df.ticktime = map(lambda x: int(x.replace(':', '')), df.ticktime)
df.ticktime = map(lambda x, y: str(x) + ' ' + str(y), df.dt,
df.ticktime)
df.ticktime = pd.to_datetime(df.ticktime,
format='%Y-%m-%d %H:%M:%S')
# df = df.loc[:, ['open', 'high', 'low', 'close', 'llastp', 'volume', 'ticktime']]
df = df.loc[:, [
'close', 'high', 'low', 'llastp', 'volume', 'ticktime'
]]
if 'code' not in df.columns:
df = df.reset_index()
if 'dt' in df.columns:
df = df.drop(['dt'], axis=1)
# df.dt = df.dt.astype(str)
if 'name' in df.columns:
# df.name = df.name.astype(str)
df = df.drop(['name'], axis=1)
df = df.set_index(['code', 'ticktime'])
h5a.write_hdf_db(h5_fname,
df,
table=h5_table,
index=False,
baseCount=500,
append=False,
MultiIndex=True)
log.info("hdf5 class all :%s time:%0.2f" %
(len(df), time.time() - time_s))
if 'nlow' not in df.columns or 'nhigh' not in df.columns or (
cct.get_work_time() and 925 < cct.get_now_time_int() < 1500):
# if 'nlow' not in df.columns or 'nhigh' not in df.columns or cct.get_work_time():
h5 = h5a.load_hdf_db(h5_fname, h5_table, timelimit=False)
time_s = time.time()
if cct.get_work_time() and cct.get_now_time_int() <= 1000:
run_col = ['low', 'high', 'close']
startime = '09:25:00'
# endtime = '10:00:00'
endtime = '09:45:00'
dd = self.get_col_agg_df(h5, dd, run_col, all_func, startime,
endtime)
startime = '09:25:00'
# endtime = '10:00:00'
endtime = '10:00:00'
run_col = {'close': 'std'}
dd = self.get_col_agg_df(h5, dd, run_col, run_col, startime,
endtime)
dd.rename(columns={'std': 'nstd'}, inplace=True)
if dd is not None and len(
dd
) > 0 and 'nclose' in dd.columns and 'nstd' in dd.columns:
for co in ['nclose', 'nstd']:
dd[co] = dd[co].apply(lambda x: round(x, 2))
else:
run_col = ['low', 'high']
startime = '09:25:00'
# endtime = '10:00:00'
endtime = '09:40:00'
dd = self.get_col_agg_df(h5, dd, run_col, all_func, startime,
endtime)
startime = '09:25:00'
endtime = '15:01:00'
run_col = ['close']
# h5 = cct.get_limit_multiIndex_Group(h5, freq='15T', col=run_col,start=startime, end=endtime)
dd = self.get_col_agg_df(h5, dd, run_col, all_func, startime,
endtime)
run_col = {'close': 'std'}
dd = self.get_col_agg_df(h5, dd, run_col, run_col, startime,
endtime)
dd.rename(columns={'std': 'nstd'}, inplace=True)
if dd is not None and len(
dd
) > 0 and 'nclose' in dd.columns and 'nstd' in dd.columns:
for co in ['nclose', 'nstd']:
dd[co] = dd[co].apply(lambda x: round(x, 2))
log.info("agg_df_all_time:%0.2f" % (time.time() - time_s))
# top_temp[:1][['high','nhigh','low','nlow','close','nclose','llastp']]
h5a.write_hdf_db(self.hdf_name, dd, self.table, index=index)
log.info("wr end:%0.2f" % (time.time() - self.start_t))
return dd
def all(self):
self.stockcode = StockCode()
self.stock_code_path = self.stockcode.stock_code_path
self.stock_codes = self.stockcode.get_stock_codes()
self.load_stock_codes()
# print "stocks:",len(self.stock_codes)
self.stock_codes = [
elem for elem in self.stock_codes
if elem.startswith(('6', '30', '00'))
]
time_s = time.time()
h5 = h5a.load_hdf_db(self.hdf_name,
self.table,
code_l=self.stock_codes,
limit_time=self.sina_limit_time)
log.info("h5a stocksTime:%0.2f" % (time.time() - time_s))
if h5 is not None and len(h5) > 0:
o_time = h5[h5.timel <> 0].timel
if len(o_time) > 0:
o_time = o_time[0]
l_time = time.time() - o_time
sina_limit_time = ct.sina_limit_time
sina_time_status = (cct.get_work_day_status()
and 915 < cct.get_now_time_int() < 926)
# return_hdf_status = not cct.get_work_day_status() or (cct.get_work_day_status() and (cct.get_work_time() and l_time < sina_limit_time))
return_hdf_status = not cct.get_work_time() or (
cct.get_work_time() and l_time < sina_limit_time)
log.info("915:%s sina_time:%0.2f limit:%s" %
(sina_time_status, l_time, sina_limit_time))
if sina_time_status and l_time < 6:
log.info("open 915 hdf ok:%s" % (len(h5)))
return h5
elif not sina_time_status and return_hdf_status:
log.info("return hdf5 data:%s" % (len(h5)))
return h5
else:
log.info("no return hdf5:%s" % (len(h5)))
else:
log.info("no return hdf5:%s" %
(len(h5) if h5 is not None else 'None'))
# self.stock_with_exchange_list = list(
# map(lambda stock_code: ('sh%s' if stock_code.startswith(('5', '6', '9')) else 'sz%s') % stock_code,
# self.stock_codes))
self.stock_with_exchange_list = list(
map(
lambda stock_code: ('sh%s' if stock_code.startswith(
('6')) else 'sz%s') % stock_code, self.stock_codes))
self.stock_list = []
self.request_num = len(self.stock_with_exchange_list) // self.max_num
for range_start in range(self.request_num):
num_start = self.max_num * range_start
num_end = self.max_num * (range_start + 1)
request_list = ','.join(
self.stock_with_exchange_list[num_start:num_end])
self.stock_list.append(request_list)
# print len(self.stock_with_exchange_list), num_end
if len(self.stock_with_exchange_list) > num_end:
request_list = ','.join(self.stock_with_exchange_list[num_end:])
self.stock_list.append(request_list)
self.request_num += 1
# a = 0
# for x in range(self.request_num):
# print x
# i = len(self.stock_list[x].split(','))
# print i
# a += i
# print a
log.debug('all:%s' % len(self.stock_list))
# log.error('all:%s req:%s' %
# (len(self.stock_list), len(self.stock_list)))
return self.get_stock_data()
clean=True)
log.info('agg_df_Row:%.2f h5:%s endtime:%s' %
((time.time() - time_n), len(h5), endtime))
return dd
h5_fname = 'sina_MultiIndex_data'
# h5_fname = 'sina_multi_index'
h5_table = 'all_10'
time_s = time.time()
h5 = h5a.load_hdf_db(h5_fname,
table=h5_table,
code_l=None,
timelimit=False,
dratio_limit=0.12)
print 'h5:', len(h5)
if cct.get_work_time() and cct.get_now_time_int() <= 1000:
run_col = ['low', 'high', 'close']
startime = None
# endtime = '10:00:00'
endtime = '09:45:00'
dd = get_col_agg_df_Test(h5, dd, run_col, all_func, startime, endtime)
else:
run_col = ['low', 'high']
startime = None
# endtime = '10:00:00'
endtime = '09:45:00'
dd = get_col_agg_df_Test(h5, dd, run_col, all_func, startime, endtime)
startime = '09:30:00'
endtime = '15:01:00'
# run_col = ['close']
def load_hdf_db(fname,
table='all',
code_l=None,
timelimit=True,
index=False,
limit_time=ct.h5_limit_time,
dratio_limit=ct.dratio_limit,
MultiIndex=False):
"""[summary]
[load hdf ]
Parameters
----------
fname : {[type]}
[description]
table : {str}, optional
[description] (the default is 'all', which [default_description])
code_l : {[type]}, optional
[description] (the default is None, which [default_description])
timelimit : {bool}, optional
[description] (the default is True, which [default_description])
index : {bool}, optional
[description] (the default is False, which [default_description])
limit_time : {[type]}, optional
[description] (the default is ct.h5_limit_time, which [default_description])
dratio_limit : {[type]}, optional
[description] (the default is ct.dratio_limit, which [default_description])
MultiIndex : {bool}, optional
[description] (the default is False, which [default_description])
Returns
-------
[dataframe]
[description]
"""
time_t = time.time()
global RAMDISK_KEY, INIT_LOG_Error
if not RAMDISK_KEY < 1:
return None
df = None
dd = None
table = table.replace('/', '') if table.find('/') >= 0 else table
if code_l is not None:
if table is not None:
with SafeHDFStore(fname) as store:
if store is not None:
try:
if '/' + table in list(store.keys()):
dd = store[table]
except AttributeError as e:
store.close()
os.remove(store.filename)
log.error("AttributeError:%s %s" % (fname, e))
log.error("Remove File:%s" % (fname))
except Exception as e:
print("Exception:%s name:%s" % (fname, e))
else:
pass
finally:
pass
if dd is not None and len(dd) > 0:
if not MultiIndex:
if index:
code_l = list(
map((lambda x: str(1000000 - int(x))
if x.startswith('0') else x), code_l))
dif_co = list(set(dd.index) & set(code_l))
if len(code_l) > 0:
dratio=(float(len(code_l)) - float(len(dif_co))) / \
float(len(code_l))
else:
dratio = 0
# if dratio < 0.1 or len(dd) > 3100:
log.info("find all:%s :%s %0.2f" %
(len(code_l), len(code_l) - len(dif_co), dratio))
if timelimit and len(dd) > 0:
dd = dd.loc[dif_co]
o_time = dd[dd.timel != 0].timel.tolist()
# if fname == 'powerCompute':
# o_time = sorted(set(o_time),reverse=True)
o_time = sorted(set(o_time), reverse=False)
o_time = [time.time() - t_x for t_x in o_time]
if len(dd) > 0:
# if len(dd) > 0 and (not cct.get_work_time() or len(o_time) <= ct.h5_time_l_count):
l_time = np.mean(o_time)
return_hdf_status = (not cct.get_work_time()) or (
cct.get_work_time() and l_time < limit_time)
# return_hdf_status = l_time < limit_time
# print return_hdf_status,l_time,limit_time
if return_hdf_status:
# df=dd
df = dd.loc[dif_co]
log.info(
"return hdf: %s timel:%s l_t:%s hdf ok:%s"
% (fname, len(o_time), l_time, len(df)))
else:
log.error(
"%s %s o_time:%s %s" %
(fname, table, len(o_time),
[time.time() - t_x for t_x in o_time[:3]]))
log.info('fname:%s l_time:%s' %
(fname, [time.time() - t_x
for t_x in o_time]))
else:
df = dd.loc[dif_co]
if dratio > dratio_limit:
if len(
code_l
) > ct.h5_time_l_count * 10 and INIT_LOG_Error < 5:
# INIT_LOG_Error += 1
log.info(
"fn:%s cl:%s h5:%s don't find:%s dra:%0.2f log_err:%s"
% (fname, len(code_l), len(dd), len(code_l) -
len(dif_co), dratio, INIT_LOG_Error))
return None
# if dratio < dratio_limit:
# log.info("find all:%s :%s %0.2f" %
# (len(code_l), len(code_l) - len(dif_co), dratio))
# if timelimit and len(dd) > 0:
# dd=dd.loc[dif_co]
# o_time=dd[dd.timel <> 0].timel.tolist()
# # if fname == 'powerCompute':
# # o_time = sorted(set(o_time),reverse=True)
# o_time=sorted(set(o_time), reverse=False)
# o_time=[time.time() - t_x for t_x in o_time]
# if len(dd) > 0:
# # if len(dd) > 0 and (not cct.get_work_time() or len(o_time) <= ct.h5_time_l_count):
# l_time=np.mean(o_time)
# return_hdf_status=(not cct.get_work_time()) or (
# cct.get_work_time() and l_time < limit_time)
# # return_hdf_status = l_time < limit_time
# # print return_hdf_status,l_time,limit_time
# if return_hdf_status:
# # df=dd
# dd.loc[dif_co]
# log.info("return hdf: %s timel:%s l_t:%s hdf ok:%s" % (
# fname, len(o_time), l_time, len(df)))
# else:
# log.error("%s %s o_time:%s %s" % (fname, table, len(
# o_time), [time.time() - t_x for t_x in o_time[:3]]))
# log.info('fname:%s l_time:%s' %
# (fname, [time.time() - t_x for t_x in o_time]))
# else:
# df=dd.loc[dif_co]
# else:
# if len(code_l) > ct.h5_time_l_count * 10 and INIT_LOG_Error < 5:
# # INIT_LOG_Error += 1
# log.error("fn:%s cl:%s h5:%s don't find:%s dra:%0.2f log_err:%s" % (
# fname, len(code_l), len(dd), len(code_l) - len(dif_co), dratio, INIT_LOG_Error))
else:
df = dd.loc[dd.index.isin(code_l, level='code')]
else:
log.error("%s is not find %s" % (fname, table))
else:
# h5 = get_hdf5_file(fname,wr_mode='r')
if table is not None:
with SafeHDFStore(fname) as store:
# if store is not None:
# if '/' + table in store.keys():
# try:
# dd=store[table]
# except Exception as e:
# print ("%s fname:%s"%(e,fname))
# cct.sleep(ct.sleep_time)
if store is not None:
try:
if '/' + table in list(store.keys()):
dd = store[table]
except AttributeError as e:
store.close()
os.remove(store.filename)
log.error("AttributeError:%s %s" % (fname, e))
log.error("Remove File:%s" % (fname))
except Exception as e:
log.error("Exception:%s %s" % (fname, e))
print("Exception:%s name:%s" % (fname, e))
else:
pass
finally:
pass
if dd is not None and len(dd) > 0:
if timelimit:
if dd is not None and len(dd) > 0:
o_time = dd[dd.timel != 0].timel.tolist()
o_time = sorted(set(o_time))
o_time = [time.time() - t_x for t_x in o_time]
if len(o_time) > 0:
l_time = np.mean(o_time)
# l_time = time.time() - l_time
# return_hdf_status = not cct.get_work_day_status() or not
# cct.get_work_time() or (cct.get_work_day_status() and
# (cct.get_work_time() and l_time < limit_time))
return_hdf_status = not cct.get_work_time() or (
cct.get_work_time() and l_time < limit_time)
log.info("return_hdf_status:%s time:%0.2f" %
(return_hdf_status, l_time))
if return_hdf_status:
log.info("return hdf5 data:%s o_time:%s" %
(len(dd), len(o_time)))
df = dd
else:
log.info("no return time hdf5:%s" % (len(dd)))
log.info('fname:%s l_time:%s' %
(fname, [time.time() - t_x
for t_x in o_time]))
else:
df = dd
else:
log.error("%s is not find %s" % (fname, table))
# else:
# log.error("%s / table is Init None:%s"(fname, table))
if df is not None and len(df) > 0:
df = df.fillna(0)
if 'timel' in df.columns:
time_list = df.timel.tolist()
# time_list = sorted(set(time_list),key = time_list.index)
time_list = sorted(set(time_list))
# log.info("test:%s"%(sorted(set(time_list),key = time_list.index)))
if time_list is not None and len(time_list) > 0:
df['timel'] = time_list[0]
log.info("load hdf times:%s" %
([time.time() - t_x for t_x in time_list]))
log.info("load_hdf_time:%0.2f" % (time.time() - time_t))
if df is not None:
df = df[~df.index.duplicated(keep='last')]
if fname.find('MultiIndex') > 0 and 'volume' in df.columns:
count_drop = len(df)
df = df.drop_duplicates()
# df = df.drop_duplicates('volume',keep='last')
dratio = round((float(len(df))) / float(count_drop), 2)
log.debug("all:%s drop:%s dratio:%.2f" %
(int(count_drop / 100), int(len(df) / 100), dratio))
if dratio < 0.8:
log.error("MultiIndex drop_duplicates:%s %s dr:%s" %
(count_drop, len(df), dratio))
if isinstance(df.index, pd.core.index.MultiIndex):
write_hdf_db(fname,
df,
table=table,
index=index,
MultiIndex=True,
rewrite=True)
# df = df.drop_duplicates()
return df
# top_now = tdd.getSinaAlldf(market='雄安新区', filename='xaxq', vol=ct.json_countVol, vtype=ct.json_countType)
# top_now = tdd.getSinaAlldf(market=u'京津冀', filename='beijing', vol=ct.json_countVol, vtype=ct.json_countType)
# top_now = tdd.getSinaAlldf(market='sh', vol=ct.json_countVol, vtype=ct.json_countType)
now_count = len(top_now)
radio_t = cct.get_work_time_ratio()
# top_now = top_now[top_now.buy > 0]`
time_d = time.time()
if time_d - time_s > delay_time:
status_change = True
time_s = time.time()
top_all = pd.DataFrame()
else:
status_change = False
# print ("Buy>0:%s" % len(top_now[top_now['buy'] > 0])),
if len(top_now) > 10 or cct.get_work_time():
time_Rt = time.time()
if len(top_all) == 0 and len(lastpTDX_DF) == 0:
cct.get_terminal_Position(position=sys.argv[0])
time_Rt = time.time()
print(("term:%s" % (cct.get_terminal_Position(cmd='DurationCXDN.py')),))
if cct.get_terminal_Position(cmd='DurationCXDN.py') > 1:
top_all, lastpTDX_DF = tdd.get_append_lastp_to_df(
top_now, lastpTDX_DF=None, dl=duration_date, end=end_date, ptype=ptype, filter=filter, power=ct.lastPower, lastp=False, newdays=newdays, checknew=True,resample=resample)
else:
newdays = 15
top_all, lastpTDX_DF = tdd.get_append_lastp_to_df(
top_now, lastpTDX_DF=None, dl=duration_date, end=end_date, ptype=ptype, filter=filter, power=ct.lastPower, lastp=False, newdays=newdays, checknew=True, resample=resample)
ct.Monitor_format_trade, market_sort_value[:2])
print cct.format_for_print(
top_temp.loc[:, ct_MonitorMarket_Values][:10])
# print cct.format_for_print(top_all[:10])
if status:
for code in top_all[:10].index:
code = re.findall('(\d+)', code)
if len(code) > 0:
code = code[0]
kind = sl.get_multiday_ave_compare_silent(code)
else:
# print top_now[:10]
print "\tNo data"
int_time = cct.get_now_time_int()
if cct.get_work_time():
if int_time < 930:
while 1:
cct.sleep(60)
if cct.get_now_time_int() < 931:
cct.sleep(60)
print ".",
else:
top_all = pd.DataFrame()
time_s = time.time()
print "."
break
else:
cct.sleep(60)
elif cct.get_work_duration():
while 1:
def getBollFilter(df=None,
boll=ct.bollFilter,
duration=ct.PowerCountdl,
filter=True,
ma5d=True,
dl=14,
percent=True,
resample='d',
ene=False,
upper=False,
down=False,
indexdff=True,
cuminTrend=False,
top10=True):
# drop_cxg = cct.GlobalValues().getkey('dropcxg')
# if len(drop_cxg) >0:
# log.info("stf drop_cxg:%s"%(len(drop_cxg)))
# drop_cxg = list(set(drop_cxg))
# drop_t = [ co for co in drop_cxg if co in df.index]
# if len(drop_t) > 0:
# df = df.drop(drop_t,axis=0)top
# log.error("stf drop_cxg:%s"%(len(drop_t)))
# hvdu max量天数 lvdu min天数 hv max量 lv 量
# fib < 2 (max) fibl > 2 lvdu > 3 (hvdu > ? or volume < 5)? ene
time_s = time.time()
# df['upper'] = map(lambda x: round((1 + 11.0 / 100) * x, 1), df.ma10d)
# df['lower'] = map(lambda x: round((1 - 9.0 / 100) * x, 1), df.ma10d)
# df['ene'] = map(lambda x, y: round((x + y) / 2, 1), df.upper, df.lower)
'''
df = df[(df.close > df.ma20d) | (df.close > df.ene)]
if filter:
if 'boll' in df.columns:
df = df[(df.boll >= boll) | (df.buy > df.upper)]
'''
# if 'nlow' in df.columns:
# df = df[ ((df.nlow >= df.llastp) & (df.nlow >= df.low)) | ((df.nlow <= df.low) & (df.open == df.nlow))]
# if cct.get_work_time() and 'df2' in df.columns:
# if 915 < cct.get_now_time_int() < 930:
# df['cumnow'] = map(lambda x, y: 1 if x - y > 0 else 0, df.close, df.cumaxe)
# else:
# df['cumnow'] = map(lambda x, y: 1 if x - y > 0 else 0, df.high, df.cumaxe)
# df['df2'] = (df['df2'] + df['cumnow']).map(lambda x: int(x))
# df = df[df.df2 > 1]
# else:
# df = df[df.df2 > 0]
radio_t = cct.get_work_time_ratio()
df['lvolr%s' % (resample)] = df['volume']
# df['volume'] = (map(lambda x, y: round(x / y / radio_t, 1), df.nvol.values, df.lvolume.values))
if cct.get_now_time_int() > 1530:
df['volume'] = (map(
lambda x, y, z: round((x / y / radio_t)
if z < 9.9 else (x / y), 1), df.nvol.values,
df.ma5vol.values, df.percent.values))
else:
df['volume'] = (map(
lambda x, y, z: round((x / y / radio_t)
if z < 9.9 else (x / y), 1), df.nvol.values,
df.lvol.values, df.percent.values))
# df['volume'] = (map(lambda x, y,z: round((x / y / radio_t) if z < 9.9 else (x / y) , 1), df.nvol.values, df.ma5vol.values,df.percent.values))
# df['volume'] = (map(lambda x, y: round(x / y / radio_t, 1), df.nvol.values, df.lastv1d.values))
indexfibl = cct.GlobalValues().getkey('indexfibl')
sort_value = cct.GlobalValues().getkey('market_sort_value')
market_key = cct.GlobalValues().getkey('market_key')
market_value = cct.GlobalValues().getkey('market_value')
# if int(market_value) > 1 and 930 < cct.get_now_time_int():
df = compute_perd_value(df, market_value, 'perc')
df = compute_perd_value(df, market_value, 'per')
if sort_value <> 'percent' and (market_key in ['2', '3']
and market_value not in ['1']):
# print("sort_value:%s,market_key:%s ,market_value:%s" %
# (sort_value, market_key, market_value))
if market_key is not None and market_value is not None:
# if market_key == '3':
if market_key == '3':
market_value = int(market_value)
log.info("stf market_key:%s" % (market_key))
idx_k = cct.get_col_in_columns(df, 'perc%sd', market_value)
# filter percd > idx
# df= df[(df[("perc%sd" % (idx_k))] >= idx_k) | (df[("perc%sd" % (idx_k))]< -idx_k)]
# if int(market_value) > 1 and 930 < cct.get_now_time_int():
# df= compute_perd_value(df, market_value, 'perc')
# df= compute_perd_value(df, market_value, 'per')
# elif market_key == '2':
# if int(market_value) > 1 and 915 < cct.get_now_time_int():
# # df['per%d'%(market_value)] = compute_perd_value(df,market_value)
# df= compute_perd_value(df, market_value,'per')
# df= compute_perd_value(df, market_value, 'perc')
# if sort_value <> 'percent' and (market_key in ['2','3'] and market_value not in ['1'] ):
# if indexfibl is not None:
# df = df[df.fibl >= (indexfibl - 1)]
# else:
# indexfibl = 1
# if cuminTrend:
# cuminfibl = cct.GlobalValues().getkey('cuminfibl')
# if cuminfibl is None:
# cuminfibl = cct.get_index_fibl()
# if cuminfibl > 1:
# df = df[df.df2 >= cuminfibl]
# print df.loc['300107'][['close','cumine','llastp','cumaxe','cumaxc']]
# df = df[ (df.cumine == df.cumaxc) | (df.close >= df.cumine) | (df.llastp >=df.cumaxc) | (df.close >= df.cumaxe) ]
# else:
# upper = True
if not down:
if upper:
# upper_count = len( df[ ( ( df.buy > df.upper)| ( df.lastp0d > df.upper) | ( df.lastp1d > df.upper) | ( df.lastp2d > df.upper) | ( df.lastp3d > df.upper))])
# # ene_count = len(df[df.buy > df.upper])
# if upper_count > 30:
# df = df[ ( ( df.buy > df.upper)| ( df.lastp0d > df.upper) | ( df.lastp1d > df.upper) | ( df.lastp2d > df.upper) | ( df.lastp3d > df.upper))]
# else:
# df = df[ ( (( df.buy > df.ene ) ) | ( df.buy > df.upper)| ( df.lasth0d > df.upper) | ( df.lasth1d > df.upper) | ( df.lasth2d > df.upper) | ( df.lasth3d > df.upper))]
df = df[(df.low >= df.upper) | (df.close >= df.upper) |
(df.per1d >= 9.5) | (df.lastp1d == df.hmax) |
((df.lastp1d <> df.hmax) & (df.high > df.hmax))]
else:
if 935 < cct.get_now_time_int() < 1400:
df = df[((df.buy > df.ene) | (df.llastp > df.ene) |
(df.llastp >= df.max5) | (df.high >= df.max5) |
(df.max5 >= df.upper))]
else:
df = df[((df.buy > df.ene) | (df.llastp > df.ene) |
(df.llastp >= df.max5) | (df.high >= df.max5) |
(df.max5 >= df.upper))]
# df['cumins'] = round(cum_counts.index[0], 2)
# df['cumine'] = round(cumdf[-1], 2)
# df['cumax']
# df.rename(columns={'cumin': 'df2'}, inplace=True)
if df is None:
print "dataframe is None"
return None
else:
# top10 = df[ (df.percent >= 9.99) & (df.b1_v > df.a1_v)]
if resample in ['d', 'w']:
# if not top10 or (cct.get_now_time_int() < 950 or cct.get_now_time_int() > 1502):
# df.loc[((df.b1_v > df.a1_v) & (df.percent > 9)), 'percent'] = 10.1
# df.loc[((df.percent >= 9.97) & (df.percent < 10.1)), 'percent'] = 10
# else:
# df.loc[((df.b1_v > df.a1_v) & (df.percent > 9)), 'percent'] = 10.1
# df.loc[((df.percent >= 9.97) & (df.percent < 10.1)), 'percent'] = 10
# df.loc[df.per1d >= 9.99, 'per1d'] = 10
df['percent'] = df['percent'].apply(lambda x: round(x, 2))
# time_ss = time.time()
perc_col = [co for co in df.columns if co.find('perc') == 0]
per_col = [co for co in df.columns if co.find('per') == 0]
# per_col = list(set(per_col) - set(perc_col) - set(['per1d', 'perlastp']))
per_col = list(
set(per_col) - set(perc_col) - set(['per1d', 'perlastp']))
perc_col.remove('percent')
for co in perc_col:
df[co] = df[co].apply(lambda x: round(x, 2))
# da, down_zero, down_dn, percent_l = 1, 0, 0, 2
# da, down_zero, down_dn, percent_l = 1, 0, -1, 1
# df['perc_n'] = map((lambda h, lh, l, ll, c, lc: (1 if (h - lh) > 0 else down_dn) + (1 if c - lc > 0 else down_dn) + (1 if (l - ll) > 0 else down_dn) + (2 if (c - lh) > 0 else down_zero) + (2 if (l - lc) > 0 else down_zero) + (0 if (h - lc) > 0 else down_dn)), df['high'], df['lasth%sd' % da], df['low'], df['lastl%sd' % da], df['close'],df['lastp%sd' % da])
# df['perc_n'] = map((lambda c, lc: (1 if (c - lc) > 0 else down_zero) + (1 if (c - lc) / lc * 100 > 3 else down_zero) +
# (down_dn if (c - lc) / lc * 100 < -3 else down_zero)), df['close'], df['lastp%sd' % da])
# idx_rnd = random.randint(0, len(df) - 10) if len(df) > 10 else 0
# print "idx_rnd",idx_rnd,df.ix[idx_rnd].lastp0d ,df.ix[idx_rnd].close,df.ix[idx_rnd].lastp0d != df.ix[idx_rnd].close
# if not upper:
# if cct.get_work_time() or df.ix[idx_rnd].lastp0d <> df.ix[idx_rnd].close:
# nowd, per1d = 0, 1
# if 'nlow' in df.columns:
# df['perc_n'] = map(cct.func_compute_percd2, df['close'], df['llastp'], df['open'],df['lasto%sd' % nowd], df['lasth%sd' %
# (nowd)], df['lastl%sd' % (nowd)], df['nhigh'], df['nlow'],df['nvol']/radio_t,df['lastv%sd'%(nowd)], df['hmax'],df['df2'])
# else:
# df['perc_n'] = map(cct.func_compute_percd2, df['close'], df['llastp'], df['open'],df['lasto%sd' % nowd], df['lasth%sd' %
# (nowd)], df['lastl%sd' % (nowd)], df['high'], df['low'],df['nvol']/radio_t,df['lastv%sd'%(nowd)], df['hmax'],df['df2'])
# else:
# nowd, per1d = 1, 2
# # print df['per%sd' % da+1], df['lastp%sd' % (da)], df['lasth%sd' % (da)], df['lastl%sd' % (da)], df['high'], df['low']
# df['perc_n'] = map(cct.func_compute_percd2, df['close'], df['llastp'], df['open'],df['lasto%sd' % nowd], df['lasth%sd' %
# (nowd)], df['lastl%sd' % (nowd)], df['high'], df['low'], df['hmax'])
'''
idx_rnd = random.randint(0, len(df) - 10) if len(df) > 10 else 0
if not upper:
# if cct.get_work_time() or df.ix[idx_rnd].lastp1d <> df.ix[idx_rnd].close:
if cct.get_work_time() :
nowd, per1d=1, 1
if 'nlow' in df.columns:
df['perc_n']=map(cct.func_compute_percd2020, df['open'], df['close'], df['nhigh'], df['nlow'], df['lasto%sd' % nowd], df['llastp'],
df['lasth%sd' % (nowd)], df['lastl%sd' % (nowd)], df['ma5d'], df['ma10d'], df['nvol'] / radio_t, df['lastv%sd' % (nowd)])
else:
df['perc_n']=map(cct.func_compute_percd2020, df['open'], df['close'], df['high'], df['low'], df['lasto%sd' % nowd], df['llastp'],
df['lasth%sd' % (nowd)], df['lastl%sd' % (nowd)], df['ma5d'], df['ma10d'], df['nvol'] / radio_t, df['lastv%sd' % (nowd)])
else:
nowd, per1d=1, 1
# print df['per%sd' % da+1], df['lastp%sd' % (da)], df['lasth%sd' % (da)], df['lastl%sd' % (da)], df['high'], df['low']
df['perc_n']=map(cct.func_compute_percd2020, df['open'], df['close'], df['high'], df['low'], df['lasto%sd' % nowd], df['llastp'],
df['lasth%sd' % (nowd)], df['lastl%sd' % (nowd)], df['ma5d'], df['ma10d'], df['nvol'] / radio_t, df['lastv%sd' % (nowd)])
for co in perc_col:
df[co] = (df[co] + df['perc_n']).map(lambda x: int(x))
'''
# for co in per_col:
# df[co] = (df[co] + df['percent']).map(lambda x: int(x))
# print "percT:%.2f"%(time.time()-time_ss)
else:
df['percent'] = (map(
lambda x, y: round((x - y) / y * 100, 1)
if int(y) > 0 else 0, df.buy, df.lastp1d))
# df.loc['002204'].percent
# -15.4
# ipdb> df.loc['002204'].ma10d
# 3.2000000000000002
# ipdb> df.loc['002204'].ma5d
# 3.2999999999999998
if 'fib' not in df.columns:
df['fib'] = 0
# else:
co2int = ['boll', 'op', 'ratio', 'fib', 'fibl', 'df2']
# co2int.extend([co for co in df.columns.tolist()
# if co.startswith('perc') and co.endswith('d')])
co2int.extend(['top10', 'topR'])
co2int = [inx for inx in co2int if inx in df.columns]
for co in co2int:
df[co] = df[co].astype(int)
# if down:
# # if 'nlow' in df.columns:
# # if 'max5' in df.columns:
# # # df = df[(df.hmax >= df.ene) & (df.nlow >= df.low) & (df.low <> df.high) & (((df.low < df.hmax) & (df.high > df.cmean)) | (df.high > df.max5))]
# # df = df[ (((df.buy > df.ene) & (df.volume / radio_t > 1.5)) ) | (((df.lastv1d > df.lvol * 1.5) & (df.lastv1d > df.lvol * 1.5)) & (df.lastp1d > df.ene) )]
# # else:
# # df = df[ (((df.buy > df.ene) & (df.volume / radio_t > 1.5)) ) | (((df.lastv1d > df.lvol * 1.5) ) )]
# # else:
# # # df = df[ (df.max5 >= df.ene) & (df.low <> df.high) & (((df.low < df.hmax) & (df.close > df.cmean)) | (df.high > df.cmean))]
# # df = df[ (((df.buy > df.ene) & (df.volume / radio_t > 1.5)) ) | (((df.lastp1d > df.ene) ) | ((df.buy > df.max5)|(df.buy > df.ene)) | (df.max5 >= df.ene) | (df.high > df.cmean) )]
# return df
if cct.get_work_time() and 'b1_v' in df.columns and 'nvol' in df.columns:
df = df[(df.b1_v > 0) | (df.nvol > 0)]
if (cct.get_now_time_int() > 915 and cct.get_now_time_int() < 926):
df['b1_v'] = df['volume']
else:
dd = df[df.percent < 10]
dd['b1_v'] = dd['volume']
df = cct.combine_dataFrame(df, dd.loc[:, ['b1_v']])
# print "t:%0.2f"%(time.time()-time_ss)
# if int(market_value) < 3 and 930 < cct.get_now_time_int():
# if 'nlow' in df.columns:
# # df = df[((df.per1d < 3) & (df.nlow >= df.llastp)) | ((df.per1d > 5) & (df.buy > df.llastp))]
# df = df[((df.per1d < 3) & (df.nlow >= df.llastp)) | ((df.per1d > 5) & (df.buy > df.llastp))]
# else:
# df = df[((df.per1d < 3) & (df.open > df.llastp)) | ((df.per1d > 5) & (df.buy > df.llastp))]
# else:
# if 'fib' in df.columns:
# df = df[df.fib <= 5]
# log.error("perc%sd"%(market_value))
# elif market_key is not None and market_key == '2':
# if market_value in ['1', '2']:
# df['dff'] = (map(lambda x, y, z: round((x + (y if z > 20 else 3 * y)), 1), df.dff.values, df.volume.values, df.ratio.values))
# df['df2'] = (map(lambda x, y, z: w=round((x - y) / z * 100, 1), df.high.values, df.low.values, df.llastp.values))
# df['df2'] = (map(func_compute_df2, df.close.values, df.llastp.values,df.high.values, df.low.values,df.ratio.values))
# df['df2'] = (map(func_compute_df2, df.close.values, df.llastp.values,df.high.values, df.low.values))
'''
filter_up= 'ma5d'
filter_dn= 'ma10d'
if ma5d and filter_up in df.columns:
if filter_dn in df.columns:
if market_key is not None and market_key == '3':
df= df[(df.buy >= df[filter_up]) & (
df.buy * 1.02 >= df[filter_dn] * ct.changeRatio)]
else:
df= df[(df.buy >= df[filter_up])]
else:
df= df[df.buy >= df[filter_up] * ct.changeRatio]
'''
# df = df[(df.lvolume > df.lvol * 0.9) & (df.lvolume > df.lowvol * 1.1)]
# if 'nlow' in df.columns and 932 < cct.get_now_time_int() < 1030:
if ene:
pass
if 'nclose' in df.columns:
# df = df[ ((df.buy > df.ene) ) | ((df.buy > df.upper) & ( df.nclose > df.cmean))]
df = df[((df.buy > df.llastp)) |
((df['nclose'] > df['ene'] * ct.changeRatio) &
(df['percent'] > -3) & (df.volume > 1.5)) |
((df['nclose'] > df['upper'] * ct.changeRatio) &
(df['buy'] > df['upper'] * ct.changeRatioUp)) |
((df['llastp'] > df['upper']) &
(df['nclose'] > df['upper']))]
else:
df = df[((df.buy > df.llastp)) | ((df.buy > df.ene)) |
((df.buy > df.upper) & (df.close > df.cmean))]
if 'nlow' in df.columns and 930 < cct.get_now_time_int():
# for col in ['nhigh', 'nclose', 'nlow','nstd']:
# df[col] = df[col].apply(lambda x: round(x, 2))
if 'nhigh' in df.columns and 'nclose' in df.columns:
# ncloseRatio = map(lambda x, y: x * ct.changeRatio if x * ct.changeRatio > y else y, df.nclose, df.nlow)
df['ncloseRatio'] = map(lambda x: x * 0.99, df.nclose)
df['nopenRatio'] = map(lambda x: x * 0.99, df.open)
# nhighRatio = map(lambda x, y: x * ct.changeRatio if x * ct.changeRatio > y else y, df.nhigh, df.nclose)
# if cct.get_now_time_int() > ct.nlow_limit_time:
# name open low high close nclose nlow nhigh nstd ticktime
# code
# 601899 紫金矿业 4.25 4.24 4.69 4.69 4.52 4.24 4.42 0.12 15:00:00
# 603917 合力科技 27.95 27.69 31.54 31.54 30.46 27.69 30.00 0.70 15:00:00
# 300713 英可瑞 81.00 81.00 89.82 89.77 87.53 80.90 86.36 1.75 15:02:03
# 000933 神火股份 9.07 9.02 10.00 10.00 9.71 9.02 9.44 0.24 15:02:03
# 600050 中国联通 6.49 6.37 6.51 6.41 6.44 6.44 6.51 0.02 15:00:00
# 002350 北京科锐 10.17 10.17 10.75 10.23 10.31 10.17 10.75 0.07 15:02:03
# 603363 傲农生物 17.49 16.91 18.35 18.35 17.35 17.18 18.00 0.34 15:00:00
# 000868 安凯客车 8.77 8.53 8.82 8.82 8.82 8.53 8.82 0.01 15:02:03
# 002505 大康农业 2.91 2.86 2.99 2.89 2.91 2.90 2.99 0.02 15:02:03
# 601939 建设银行 7.50 7.41 7.61 7.55 7.50 7.43 7.52 0.05 15:00:00
# 600392 盛和资源 16.85 16.81 18.63 18.63 17.84 16.81 17.51 0.69 15:00:00 (11, 41)
# 603676 卫信康 17.44 16.95 17.96 17.82 17.34 17.03 17.77 0.18 15:00:00
if 'nstd' in df.columns:
df['stdv'] = map(lambda x, y: round(x / y * 100, 1),
df.nstd, df.open)
# top_temp[:40][['nstd','stdv','name','volume','dff','percent']]
# top_temp.loc['603363'][['nstd','stdv','name','volume','high','nhigh','percent','open','nclose']]
# df['stdv'] = map(lambda x, y, z: round(z / (x / y * 100), 1), df.nstd, df.open, df.volume)
# df.loc['002906'][['open','nclose','close','ncloseRatio','nopenRatio','low','nlow']]
df = df[(df.low > df.upper) |
(((df.nopenRatio <= df.nclose) & (df.low >= df.nlow) &
(df.close >= df.ncloseRatio)) |
((df.nopenRatio <= df.nclose) &
(df.close >= df.ncloseRatio) &
(df.high >= df.nhigh)))]
# else:
# df = df[(((df.low >= df.nlow) & (df.close >= ncloseRatio)) | ((df.close >= ncloseRatio) & (df.close > nhighRatio)))]
else:
df = df[((df.low >= df.nlow) &
(df.close > df.llastp * ct.changeRatio))]
# if 'nhigh' in df.columns and 'nclose' in df.columns:
# if cct.get_now_time_int() > ct.nlow_limit_time:
# df = df[(df.low >= df.nlow) & ((df.open > df.llastp * ct.changeRatio) & (df.nclose > df.llastp * ct.changeRatio)) &
# (((df.low >= df.nlow) & (df.close >= df.nclose)) | ((df.close >= df.nclose) & (df.close > df.nhigh * ct.changeRatio) & (df.high >= df.nhigh)))]
# else:
# df = df[((df.open > df.llastp * ct.changeRatio) & (df.close > df.llastp * ct.changeRatio)) &
# (((df.low >= df.nlow) & (df.close >= df.nclose)) | ((df.close >= df.nclose) & (df.close > df.nhigh * ct.changeRatio)))]
# else:
# df = df[((df.low >= df.nlow) & (df.close > df.llastp))]
if filter:
if cct.get_now_time_int() > 915 and cct.get_now_time_int() <= 1000:
df = df[((df.buy > df.llastp)) |
(df.buy > df.hmax * ct.changeRatio) |
(df.buy < df.lmin * ct.changeRatioUp)]
# df = df[(df.buy > df.hmax * ct.changeRatio) & (df.low > df.llastp)]
# df = df[(df.buy > df.cmean * ct.changeRatioUp) | (df.open > df.llastp)]
# df = df[df.buy > df.cmean]
elif cct.get_now_time_int() > 1000 and cct.get_now_time_int(
) <= 1430:
df = df[((df.buy > df.llastp)) |
(df.buy > df.hmax * ct.changeRatio) |
(df.buy < df.lmin * ct.changeRatioUp)]
# df = df[(df.buy > df.hmax * ct.changeRatio) | (df.low > df.llastp)]
# df = df[(df.buy > df.hmax * ct.changeRatio) & (df.low > df.llastp)]
# df = df[(df.buy > df.cmean * ct.changeRatioUp) | (df.open > df.llastp)]
# df = df[df.buy > df.cmean]
else:
df = df[((df.buy > df.llastp)) |
(df.buy > df.hmax * ct.changeRatio) |
(df.buy < df.lmin * ct.changeRatioUp)]
# df = df[(df.buy > df.hmax * ct.changeRatio) | (df.low > df.llastp)]
# df = df[(df.buy > df.hmax * ct.changeRatio) & (df.low > df.llastp)]
# df = df[(df.buy > df.cmean * ct.changeRatioUp) | (df.open > df.llastp)]
# df = df[df.buy > df.cmean]
if 'vstd' in df.columns:
# df = df[(df.lvol * df.volume > (df.vstd + df.lvol)) | ((df.percent > -10) & (df.hv / df.lv > 1.5))]
# dd = df.loc['000760'] dd.hv/dd.lv,dd.volume
df['hvRatio'] = map(
lambda x, y: round(x / y / cct.get_work_time_ratio(), 1),
df.hv, df.lv)
df['volRatio'] = (map(lambda x, y: round(x / y / radio_t, 1),
df.nvol.values, df.lv.values))
# df = df[(df.lvol * df.volume > (df.vstd + df.lvol)) | ((df.percent > -5) & (df.hv / df.lv > 3))]
# if 1000 < cct.get_now_time_int() < 1450:
# df = df[ (3 < df.ratio) & (df.ratio < 20)]
if 'nclose' in df.columns:
# df = df[((df.hvRatio > df.volRatio) & ((df.buy > df.lastp) & (df.nclose > df.lastp))) | ((
df = df[(((df.buy > df.lastp) & (df.nclose > df.lastp))) |
(((df.percent > -5) &
((df.vstd + df.lvol) * 1.2 < df.nvol) &
(df.nvol > (df.vstd + df.lvol))))]
else:
# df = df[((df.hvRatio > df.volRatio) & ((df.buy > df.lastp))) | ((
df = df[(((df.buy > df.lastp))) |
(((df.percent > -5) &
((df.vstd + df.lvol) * 1.2 < df.nvol) &
(df.nvol > (df.vstd + df.lvol))))]
# [dd.lvol * dd.volume > (dd.vstd + dd.lvol) | dd.lvol * dd.volume >(dd.ldvolume + dd.vstd]
# print df.loc['000801']
if percent:
pass
'''
if 'stdv' in df.columns and 926 < cct.get_now_time_int():
# df = df[((df.volume > 2 * cct.get_work_time_ratio()) & (df.percent > -3)) | ((df.stdv < 1) &
# (df.percent > 2)) | ((df.lvolume > df.lvol * 0.9) & (df.lvolume > df.lowvol * 1.1))]
df_index = tdd.getSinaIndexdf()
if isinstance(df_index, type(pd.DataFrame())):
df_index['volume'] = (map(lambda x, y: round(x / y / radio_t, 1), df_index.nvol.values, df_index.lastv1d.values))
index_vol = df_index.loc['999999'].volume
if 'percent' in df_index.columns and '999999' in df_index.index:
index_percent = df_index.loc['999999'].percent
else:
# import ipdb;ipdb.set_trace()
log.error("index_percent" is None)
index_percent = 0
index_boll = df_index.loc['999999'].boll
index_op = df_index.loc['999999'].op
# df = df[((df.percent > -3) | (df.volume > index_vol)) & (df.percent > index_percent)]
# df = df[((df.percent > -1) | (df.volume > index_vol * 1.5)) & (df.percent > index_percent) & (df.boll >= index_boll)]
df = df[((df.percent > -1) | (df.volume > index_vol * 1.5)) & (df.percent > index_percent)]
else:
print ("df_index is Series",df_index.T)
else:
df = df[((df.volume > 1.2 * cct.get_work_time_ratio()) & (df.percent > -3))]
'''
# df = df[((df['buy'] >= df['ene'])) | ((df['buy'] < df['ene']) & (df['low'] > df['lower'])) | ((df['buy'] > df['upper']) & (df['low'] > df['upper']))]
# df = df[(( df['ene'] * ct.changeRatio < df['open']) & (df['buy'] > df['ene'] * ct.changeRatioUp)) | ((df['low'] > df['upper']) & (df['close'] > df['ene']))]
# df = df[(( df['ene'] < df['open']) & (df['buy'] < df['ene'] * ct.changeRatioUp))]
# df = df[(df.per1d > 9) | (df.per2d > 4) | (df.per3d > 6)]
# df = df[(df.per1d > 0) | (df.per2d > 4) | (df.per3d > 6)]
# time_ss=time.time()
# codel = df.index.tolist()
# dm = tdd.get_sina_data_df(codel)
# results = cct.to_mp_run_async(getab.Get_BBANDS, codel,'d',5,duration,dm)
# bolldf = pd.DataFrame(results, columns=['code','boll'])
# bolldf = bolldf.set_index('code')
# df = cct.combine_dataFrame(df, bolldf)
# print "bollt:%0.2f"%(time.time()-time_ss),
per3d_l = 2
percent_l = -1
op_l = 3
# if 'boll' in df.columns:
# if 915 < cct.get_now_time_int() < 950:
# # df = df[(df.boll >= boll) | ((df.percent > percent_l) & (df.op > 4)) | ((df.percent > percent_l) & (df.per3d > per3d_l))]
# # df = df[((df.percent > percent_l) & (df.op > 4)) | ((df.percent > percent_l) & (df.per3d > per3d_l))]
# pass
# elif 950 < cct.get_now_time_int() < 1501:
# # df = df[(df.boll >= boll) | ((df.low <> 0) & (df.open == df.low) & (((df.percent > percent_l) & (df.op > op_l)) | ((df.percent > percent_l) & (df.per3d > per3d_l))))]
# # df = df[(df.boll >= boll) & ((df.low <> 0) & (df.open >= df.low *
# # ct.changeRatio) & (((df.percent > percent_l)) | ((df.percent >
# # percent_l) & (df.per3d > per3d_l))))]
# df = df[(df.boll >= boll)]
# # else:
# df = df[(df.boll >= boll) | ((df.low <> 0) & (df.open == df.low) &
# (((df.percent > percent_l) & (df.op > op_l)) | ((df.percent > percent_l)
# & (df.per3d > per3d_l))))]
# if 945 < cct.get_now_time_int() and market_key is None:
# df.loc[df.percent >= 9.94, 'percent'] = -10
# else:
# # df['upper'] = map(lambda x: round((1 + 11.0 / 100) * x, 1), df.ma10d)
# # df['lower'] = map(lambda x: round((1 - 9.0 / 100) * x, 1), df.ma10d)
# # df['ene'] = map(lambda x, y: round((x + y) / 2, 1), df.upper, df.lower)
# # df = df[((df['buy'] >= df['ene'])) | ((df['buy'] < df['ene']) & (df['low'] > df['lower'])) | ((df['buy'] > df['upper']) & (df['low'] > df['upper']))]
# # df = df[(( df['ene'] * ct.changeRatio < df['open']) & (df['buy'] > df['ene'] * ct.changeRatioUp)) | ((df['low'] > df['upper']) & (df['close'] > df['ene']))]
# if 'nclose' in df.columns:
# # df = df[((df['nclose'] > df['ene'] * ct.changeRatio) & (df['percent'] > -3) & (df.volume > 1.5)) |
# # ((df['nclose'] > df['upper'] * ct.changeRatio) & (df['buy'] > df['upper'] * ct.changeRatioUp)) | ((df['llastp'] > df['upper']) & (df['nclose'] > df['upper']))]
# df = df[(df.nclose > df.cmean)]
# else:
# df = df[(df.buy > df.cmean)]
print("bo:%0.1f" % (time.time() - time_s)),
df['ral'] = (map(lambda x, y: round((x + y), 1), df.percent, df.ral))
df['ra'] = (map(lambda x, y: round((x + y), 1), df.percent, df.ra))
return df
def get_sina_all_json_dd(vol='0', type='0', num='10000', retry_count=3, pause=0.001):
start_t = time.time()
# url="http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?page=1&num=50&sort=changepercent&asc=0&node=sh_a&symbol="
# SINA_REAL_PRICE_DD = '%s%s/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?page=1&num=%s&sort=changepercent&asc=0&node=%s&symbol=%s'
#http://vip.stock.finance.sina.com.cn/quotes_service/view/cn_bill_sum.php
#http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/CN_Bill.GetBillList?num=10000&page=1&sort=ticktime&asc=0&volume=100000&type=0
#http://vip.stock.finance.sina.com.cn/quotes_service/view/cn_bill_all.php?num=100&page=1&sort=ticktime&asc=0&volume=100000&type=0
"""
一次性获取最近一个日交易日所有股票的交易数据
return
-------
DataFrame
属性：代码，名称，涨跌幅，现价，开盘价，最高价，最低价，最日收盘价，成交量，换手率
"""
# return ''
h5_fname = 'get_sina_all_dd'
h5_table = 'all'+'_'+ct.DD_VOL_List[str(vol)]+'_'+str(num)
limit_time = ct.sina_dd_limit_time
h5 = h5a.load_hdf_db(h5_fname, table=h5_table,limit_time=limit_time)
if h5 is not None and not h5.empty and len(h5) > 500 and 'timel' in h5.columns:
o_time = h5[h5.timel != 0].timel
if len(o_time) > 0:
o_time = o_time[0]
l_time = time.time() - o_time
return_hdf_status = not cct.get_work_time() or (cct.get_work_time() and l_time < limit_time)
if return_hdf_status:
log.info("load hdf data:%s %s %s"%(h5_fname,h5_table,len(h5)))
return h5
# ct._write_head()
url_list = _get_sina_json_dd_url(vol, type, num)
# print url_list
df = pd.DataFrame()
# import ipdb;ipdb.set_trace()
# data['code'] = symbol
# df = df.append(data, ignore_index=True)
if len(url_list)>0:
log.info("json_dd_url:%s"%url_list[0])
for url in url_list:
# df = df.append(_parsing_sina_dd_price_json(url))
data = _parsing_sina_dd_price_json(url)
# if data is not None and not data.empty and len(data) > 0:
if len(data) > 5:
df = df.append(data)
# else:
# log.error("data:%s"%(data))
# data = cct.to_mp_run(_parsing_sina_dd_price_json, url_list)
# data = cct.to_mp_run_async(_parsing_sina_dd_price_json, url_list)
# if len(url_list)>cct.get_cpu_count():
# divs=cct.get_cpu_count()
# else:
# divs=len(url_list)
#
# if len(url_list)>=divs:
# print len(url_list),
# dl=cct.get_div_list(url_list,divs)
# data=cct.to_mp_run_async(cct.to_asyncio_run,dl,_parsing_sina_dd_price_json)
# else:
# data=cct.to_asyncio_run(url_list,_parsing_sina_dd_price_json)
# data = cct.to_asyncio_run(url_list, _parsing_sina_dd_price_json) //return df list
# if len(data)>50:
# df = df.append(data, ignore_index=True)
# # log.debug("dd.columns:%s" % df.columns.values)
# #['code' 'name' 'ticktime' 'price' 'volume' 'prev_price' 'kind']
# log.debug("get_sina_all_json_dd:%s" % df[:1])
if df is not None and not df.empty and len(df) > 50:
# for i in range(2, ct.PAGE_NUM[0]):
# newdf = _parsing_dayprice_json(i)
# df = df.append(newdf, ignore_index=True)
# print "dd",
time_drop=time.time()
df['couts']=df.groupby(['code'])['code'].transform('count')
# df=df[(df['kind'] == 'U')]
df=df.sort_values(by='couts',ascending=0)
df=df.drop_duplicates('code')
print("djdf:%0.1f"%(time.time()-time_drop), end=' ')
# df=df[df.price >df.prev_price]
log.info("sina-DD:%s" % df[:1])
# df = df.loc[:, ['code','name', 'couts', 'kind', 'prev_price']]
# print df.columns
df = df.loc[:, ['code','name', 'couts', 'kind', 'prev_price','ticktime']]
df.code=df.code.apply(lambda x:str(x).replace('sh','') if str(x).startswith('sh') else str(x).replace('sz',''))
if len(df) > 0:
df = df.set_index('code')
h5 = h5a.write_hdf_db(h5_fname, df, table=h5_table,append=False)
log.info("get_sina_all_json_dd:%s"%(len(df)))
print((" dd-df:%0.2f" % ((time.time() - start_t))), end=' ')
return df
else:
print()
print(("url:%s no data json-df:%0.2f"%(url_list[0],(time.time() - start_t))), end=' ')
return ''
else:
print(("Url None json-df:%0.2f "%((time.time() - start_t))), end=' ')
return ''
def get_sina_Market_json(market='all', showtime=True, num='100', retry_count=3, pause=0.001):
start_t = time.time()
# qq stock api
# http://qt.gtimg.cn/q=sz000858,sh600199
# http://blog.csdn.net/ustbhacker/article/details/8365756
# url="http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?page=1&num=50&sort=changepercent&asc=0&node=sh_a&symbol="
# SINA_REAL_PRICE_DD = '%s%s/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?page=1&num=%s&sort=changepercent&asc=0&node=%s&symbol=%s'
"""
一次性获取最近一个日交易日所有股票的交易数据
return
-------
DataFrame
属性：代码，名称，涨跌幅，现价，开盘价，最高价，最低价，最日收盘价，成交量，换手率
"""
# ct._write_head()
h5_fname = 'get_sina_all_ratio'
# h5_table = 'all'
h5_table = 'all'+'_'+str(num)
# if market == 'all':
limit_time = ct.sina_dd_limit_time
h5 = h5a.load_hdf_db(h5_fname, table=h5_table,limit_time=limit_time)
if h5 is not None and len(h5) > 0 and 'timel' in h5.columns:
o_time = h5[h5.timel != 0].timel
if len(h5) < 2000:
log.error("h5 not full data")
o_time = []
if len(o_time) > 0:
o_time = o_time[0]
l_time = time.time() - o_time
# return_hdf_status = not cct.get_work_day_status() or not cct.get_work_time() or (cct.get_work_day_status() and cct.get_work_time() and l_time < limit_time)
return_hdf_status = not cct.get_work_time() or (cct.get_work_time() and l_time < limit_time)
if return_hdf_status:
log.info("load hdf data:%s %s %s"%(h5_fname,h5_table,len(h5)))
dd = None
if market == 'all':
co_inx = [inx for inx in h5.index if str(inx).startswith(('6','30','00'))]
elif market == 'sh':
co_inx = [inx for inx in h5.index if str(inx).startswith(('6'))]
elif market == 'sz':
co_inx = [inx for inx in h5.index if str(inx).startswith(('00'))]
elif market == 'cyb':
co_inx = [inx for inx in h5.index if str(inx).startswith(('30'))]
else:
log.error('market is not Find:%s'%(market))
codel = cct.read_to_blocknew(market)
co_inx = [inx for inx in codel if inx in h5.index]
dd = h5.loc[co_inx]
if len(dd) > 100:
log.info("return sina_ratio:%s"%(len(dd)))
return dd
# else:
# market = 'all'
if h5 is None or market=='all':
url_list=[]
# for m in ct.SINA_Market_KEY.values():
for m in ['sh_a','sz_a']:
mlist=_get_sina_Market_url(m, num=num)
for l in mlist:url_list.append(l)
# print url_list
else:
url_list=_get_sina_Market_url(ct.SINA_Market_KEY[market], num=num)
# print url_list
# log.debug("Market_jsonURL: %s" % url_list[0])
# print url_list
# print "url:",url_list
df = pd.DataFrame()
# data['code'] = symbol
# df = df.append(data, ignore_index=True)
# results = cct.to_mp_run(_parsing_Market_price_json, url_list)
# urltest ='http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?page=1&num=100&sort=changepercent&asc=0&node=sh_a&symbol='
if len(url_list) > 0:
log.debug("cct.to_asyncio_run:_parsing_Market_price_json:%s"%(url_list[0]))
results = cct.to_asyncio_run(url_list, _parsing_Market_price_json)
else:
results = []
if len(results)>0:
df = df.append(results, ignore_index=True)
# df['volume']= df['volume'].apply(lambda x:x/100)
# print df.columns
if 'ratio' in df.columns:
df['ratio']=df['ratio'].apply(lambda x:round(float(x),1))
df['percent']=df['percent'].apply(lambda x:round(float(x),2))
# if cct.get_now_time_int() > 915 and cct.get_now_time_int() < 926:
# df = df[(df.buy > 0)]
# else:
# df = df[(df.trade > 0)]
# print df[:1]
# for url in url_list:
# # print url
# data = _parsing_Market_price_json(url)
# # print data[:1]
# df = df.append(data, ignore_index=True)
# # break
if df is not None and len(df) > 0:
# for i in range(2, ct.PAGE_NUM[0]):
# newdf = _parsing_dayprice_json(i)
# df = df.append(newdf, ignore_index=True)
# print len(df.index)
# print type(df)
if 'code' in df.columns:
df=df.drop_duplicates('code')
df = df.set_index('code')
# if market == 'all':
# append_status = False #(new all don't append main )
# else:
# append_status = True
if market=='all':
h5 = h5a.write_hdf_db(h5_fname, df, table=h5_table,append=False)
else:
h5 = h5a.write_hdf_db(h5_fname, df, table=h5_table,append=True)
if showtime: print(("Market-df:%s %s" % (format((time.time() - start_t), '.1f'), len(df))), end=' ')
if market == 'all':
co_inx = [inx for inx in df.index if str(inx).startswith(('6','30','00'))]
df = df.loc[co_inx]
elif market == 'sh':
co_inx = [inx for inx in df.index if str(inx).startswith(('6'))]
df = df.loc[co_inx]
elif market == 'sz':
co_inx = [inx for inx in df.index if str(inx).startswith(('00'))]
df = df.loc[co_inx]
elif market == 'cyb':
co_inx = [inx for inx in df.index if str(inx).startswith(('30'))]
df = df.loc[co_inx]
return df
else:
if showtime:print(("no data Market-df:%s" % (format((time.time() - start_t), '.2f'))))
log.error("no data Market-df:%s"%(url_list[0]))
return []
# if dratio < 0.1 or len(dd) > 3100:
log.info("find all:%s :%s %0.2f" %
(len(code_l), len(code_l) - len(dif_co), dratio))
if timelimit and len(dd) > 0:
dd = dd.loc[dif_co]
o_time = dd[dd.timel <> 0].timel.tolist()
# if fname == 'powerCompute':
# o_time = sorted(set(o_time),reverse=True)
o_time = sorted(set(o_time), reverse=False)
o_time = [time.time() - t_x for t_x in o_time]
if len(dd) > 0:
# if len(dd) > 0 and (not cct.get_work_time() or len(o_time) <= ct.h5_time_l_count):
l_time = np.mean(o_time)
return_hdf_status = (not cct.get_work_time()) or (
cct.get_work_time() and l_time < limit_time)
# return_hdf_status = l_time < limit_time
# print return_hdf_status,l_time,limit_time
if return_hdf_status:
# df=dd
df = dd.loc[dif_co]
log.info(
"return hdf: %s timel:%s l_t:%s hdf ok:%s"
% (fname, len(o_time), l_time, len(df)))
else:
log.error(
"%s %s o_time:%s %s" %
(fname, table, len(o_time),
[time.time() - t_x for t_x in o_time[:3]]))
log.info('fname:%s l_time:%s' %
