def filterPowerCount(df, count=200, down=False, duration=2):
nowint = cct.get_now_time_int()
df.sort_values('percent', ascending=False, inplace=True)
top_temp = df[:count].copy()
if 915 < nowint <= 945:
if nowint <= 935:
top_temp = df[(df.buy > df.llastp) | (df.percent > df['per%sd' %
(duration)])]
else:
top_temp = df[((df.open > df.llastp) & (df.low >= df.llastp)) |
(df.percent > df['per%sd' % (duration)])]
if len(top_temp) > 0:
return top_temp
else:
log.error("915 open>llastp is None")
return df[:count].copy()
if 930 < nowint <= 945:
top_high = df[((df.open > df.llastp) & (df.low >= df.llastp))]
elif 945 < nowint:
if 'nlow' in df.columns:
# top_high = df[ (df.nlow >= df.llastp) | (df.close >= df.nlow)]
top_high = df[((df.nlow >= df.llastp) & (df.nlow >= df.low)) |
((df.nlow <= df.low) & (df.open == df.nlow))]
else:
top_high = df[((df.open > df.llastp) & (df.low > df.llastp)) |
(df.percent > df['per%sd' % (duration)])]
else:
top_high = None
if top_high is not None:
top_high['upper'] = map(lambda x: round((1 + 11.0 / 100) * x, 1),
top_high.ma10d)
top_high['lower'] = map(lambda x: round((1 - 9.0 / 100) * x, 1),
top_high.ma10d)
top_high['ene'] = map(lambda x, y: round((x + y) / 2, 1),
top_high.upper, top_high.lower)
if 930 < nowint < 1500:
radio_t = cct.get_work_time_ratio()
else:
radio_t = 1
top_high = top_high[(((top_high.buy > top_high.ene) &
(top_high.volume / radio_t > 2))) |
(((top_high.lastv1d > top_high.lvol * 1.5)) &
(top_high.lastp1d > top_high.ene))]
if len(top_high) > count:
top_high = top_high[(top_high.low > top_high.ene) |
((top_high.percent > 1) &
(top_high.volume > 2))]
top_temp = cct.combine_dataFrame(top_temp,
top_high,
col=None,
compare=None,
append=True,
clean=True)
return top_temp
def get_col_agg_df_Test(h5,
dd,
run_col,
all_func,
startime,
endtime,
freq=None):
if isinstance(run_col, list):
now_col = [
all_func[co] for co in run_col if co in list(all_func.keys())
]
else:
now_col = [
all_func[co] for co in list(run_col.keys())
if co in list(all_func.keys())
]
now_func = cct.from_list_to_dict(run_col, all_func)
if h5 is not None and len(h5) > len(dd):
time_n = time.time()
# h5 = cct.get_limit_multiIndex_Group(h5, freq=freq,end=endtime)
ts = time.time()
if freq is None:
h5 = cct.get_limit_multiIndex_Row(h5,
col=run_col,
start=startime,
end=endtime)
else:
h5 = cct.get_limit_multiIndex_freq(h5,
freq=freq,
col=run_col,
start=startime,
end=endtime)
h5 = h5.groupby(level=[0]).tail(1)
# h5 = h5.groupby(level=[0]).tail(1)
print("s:", round(time.time() - ts, 2), len(h5))
if h5 is not None and len(h5) > 0:
h5 = h5.reset_index().set_index('code')
h5.rename(columns=now_func, inplace=True)
# log.info("get_limit_multiIndex_Row:%s endtime:%s" % (len(h5), endtime))
#h5 = h5.drop(['ticktime'], axis=1)
h5 = h5.loc[:, now_col]
dd = cct.combine_dataFrame(dd,
h5,
col=None,
compare=None,
append=False,
clean=True)
log.info('agg_df_Row:%.2f h5:%s endtime:%s' %
((time.time() - time_n), len(h5), endtime))
return dd
else:
if 'couts' in top_now.columns.values:
if not 'couts' in top_all.columns.values:
top_all['couts'] = 0
top_all['prev_p'] = 0
# for symbol in top_now.index:
# if 'couts' in top_now.columns.values:
# top_all.loc[symbol, ct.columns_now] = top_now.loc[symbol, ct.columns_now]
# else:
# # top_now.loc[symbol, 'dff'] = round(
# # ((float(top_now.loc[symbol, 'buy']) - float(
# # top_all.loc[symbol, 'lastp'])) / float(top_all.loc[symbol, 'lastp']) * 100),
# # 1)
# top_all.loc[symbol, ct.columns_now] = top_now.loc[symbol, ct.columns_now]
top_all = cct.combine_dataFrame(top_all, top_now, col=None)
top_dif = top_all.copy()
market_sort_value, market_sort_value_key = ct.get_market_sort_value_key(st_key_sort, top_all=top_all)
log.debug('top_dif:%s' % (len(top_dif)))
if 'trade' in top_dif.columns:
top_dif['buy'] = (
map(lambda x, y: y if int(x) == 0 else x, top_dif['buy'].values, top_dif['trade'].values))
# log.debug('top_dif:%s'%(len(top_dif)))
if ct.checkfilter and cct.get_now_time_int() > 915 and cct.get_now_time_int() < ct.checkfilter_end_timeDu:
top_dif = top_dif[top_dif.low > top_dif.llow * ct.changeRatio]
# top_dif = top_dif[top_dif.buy >= top_dif.lhigh * ct.changeRatio]
log.debug('top_dif:%s' % (len(top_dif)))
def write_hdf_db(fname,
df,
table='all',
index=False,
complib='blosc',
baseCount=500,
append=True,
MultiIndex=False,
rewrite=False):
"""[summary]
[description]
Parameters
----------
fname : {[type]}
[description]
df : {[type]}
[description]
table : {str}, optional
[description] (the default is 'all', which [default_description])
index : {bool}, optional
[description] (the default is False, which [default_description])
complib : {str}, optional
[description] (the default is 'blosc', which [default_description])
baseCount : {number}, optional
[description] (the default is 500, which [default_description])
append : {bool}, optional
[description] (the default is True, which [default_description])
MultiIndex : {bool}, optional
[description] (the default is False, which [default_description])
Returns
-------
[type]
[description]
"""
if 'code' in df.columns:
df = df.set_index('code')
# write_status = False
time_t = time.time()
# if not os.path.exists(cct.get_ramdisk_dir()):
# log.info("NO RamDisk")
# return False
df = df.fillna(0)
df = df[~df.index.duplicated(keep='first')]
code_subdf = df.index.tolist()
global RAMDISK_KEY
if not RAMDISK_KEY < 1:
return df
if not MultiIndex:
df['timel'] = time.time()
if not rewrite:
if df is not None and not df.empty and table is not None:
# h5 = get_hdf5_file(fname,wr_mode='r')
tmpdf = []
with SafeHDFStore(fname) as store:
if store is not None:
if '/' + table in list(store.keys()):
tmpdf = store[table]
tmpdf = tmpdf[~tmpdf.index.duplicated(keep='first')]
if not MultiIndex:
if index:
# log.error("debug index:%s %s %s"%(df,index,len(df)))
df.index = list(
map((lambda x: str(1000000 - int(x))
if x.startswith('0') else x), df.index))
if tmpdf is not None and len(tmpdf) > 0:
if 'code' in tmpdf.columns:
tmpdf = tmpdf.set_index('code')
if 'code' in df.columns:
df = df.set_index('code')
diff_columns = set(df.columns) - set(tmpdf.columns)
if len(diff_columns) != 0:
log.error("columns diff:%s" % (diff_columns))
limit_t = time.time()
df['timel'] = limit_t
# df_code = df.index.tolist()
df = cct.combine_dataFrame(tmpdf,
df,
col=None,
append=append)
if not append:
df['timel'] = time.time()
elif fname == 'powerCompute':
o_time = df[df.timel < limit_t].timel.tolist()
o_time = sorted(set(o_time), reverse=False)
if len(o_time) >= ct.h5_time_l_count:
o_time = [time.time() - t_x for t_x in o_time]
o_timel = len(o_time)
o_time = np.mean(o_time)
if (o_time) > ct.h5_power_limit_time:
df['timel'] = time.time()
log.error("%s %s o_time:%.1f timel:%s" %
(fname, table, o_time, o_timel))
# df=cct.combine_dataFrame(tmpdf, df, col=None,append=False)
log.info("read hdf time:%0.2f" % (time.time() - time_t))
else:
# if index:
# df.index = map((lambda x:str(1000000-int(x)) if x.startswith('0') else x),df.index)
log.info("h5 None hdf reindex time:%0.2f" %
(time.time() - time_t))
else:
# df.loc[df.index.isin(['000002','000001'], level='code')]
# df.loc[(df.index.get_level_values('code')== 600004)]
# df.loc[(df.index.get_level_values('code')== '600199')]
# da.swaplevel(0, 1, axis=0).loc['2017-05-25']
# df.loc[(600004,20170414),:]
# df.xs(20170425,level='date')
# df.index.get_level_values('code').unique()
# df.index.get_loc(600006)
# slice(58, 87, None)
# df.index.get_loc_level(600006)
# da.swaplevel(0, 1, axis=0).loc['2017-05-25']
# da.reorder_levels([1,0], axis=0)
# da.sort_index(level=0, axis=0,ascending=False
# setting: dfm.index.is_lexsorted() dfm = dfm.sort_index() da.loc[('000001','2017-05-12'):('000005','2017-05-25')]
# da.groupby(level=1).mean()
# da.index.get_loc('000005') da.iloc[slice(22,33,None)]
# mask = totals['dirty']+totals['swap'] > 1e7 result =
# mask.loc[mask]
# store.remove('key_name', where='<where clause>')
# tmpdf = tmpdf[~tmpdf.index.duplicated(keep='first')]
# df = df[~df.index.duplicated(keep='first')]
if not rewrite and tmpdf is not None and len(tmpdf) > 0:
# multi_code = tmpdf.index.get_level_values('code').unique().tolist()
multi_code = tmpdf.index.get_level_values(
'code').unique().tolist()
df_multi_code = df.index.get_level_values(
'code').unique().tolist()
dratio = cct.get_diff_dratio(multi_code, df_multi_code)
if dratio < ct.dratio_limit:
comm_code = list(set(df_multi_code) & set(multi_code))
# print df_multi_code,multi_code,comm_code,len(comm_code)
inx_key = comm_code[random.randint(
0,
len(comm_code) - 1)]
if inx_key in df.index.get_level_values('code'):
now_time = df.loc[inx_key].index[-1]
tmp_time = tmpdf.loc[inx_key].index[-1]
if now_time == tmp_time:
log.debug(
"%s %s Multi out %s hdf5:%s No Wri!!!" %
(fname, table, inx_key, now_time))
return False
elif dratio == 1:
print(("newData ratio:%s all:%s" % (dratio, len(df))))
else:
log.debug(
"dratio:%s main:%s new:%s %s %s Multi All Wri" %
(dratio, len(multi_code), len(df_multi_code),
fname, table))
# da.drop(('000001','2017-05-11'))
else:
log.debug("%s %s Multi rewrite:%s Wri!!!" %
(fname, table, rewrite))
time_t = time.time()
if df is not None and not df.empty and table is not None:
if df is not None and not df.empty and len(df) > 0:
dd = df.dtypes.to_frame()
if 'object' in dd.values:
dd = dd[dd == 'object'].dropna()
col = dd.index.tolist()
log.info("col:%s" % (col))
if not MultiIndex:
df[col] = df[col].astype(str)
df.index = df.index.astype(str)
df = df.fillna(0)
# else:
# print col
# for co in col:
# print ('object:%s'%(co))
# df = df.drop(col,axis=1)
# df[co] = df[co].apply()
# recordStringInHDF5(h5file, h5file.root, 'mrtamb',u'\u266b Hey Mr. Tambourine Man \u266b')
with SafeHDFStore(fname) as h5:
df = df.fillna(0)
if h5 is not None:
if '/' + table in list(h5.keys()):
if not MultiIndex:
# if MultiIndex and rewrite:
# src_code = h5[table].index.get_level_values('code').unique().tolist()
# new_code = df.index.get_level_values('code').unique().tolist()
# diff_code = list(set(new_code) - set(src_code))
# dratio = cct.get_diff_dratio(new_code, src_code)
# print dratio,len(diff_code)
# import ipdb;ipdb.set_trace()
# df = pd.concat([df, h5[table]], axis=0)
# df = df.index.drop_duplicates()
# df[df.index.get_level_values('code') not in diff_code ]
h5.remove(table)
# h5[table]=df
h5.put(table,
df,
format='table',
index=False,
complib=complib,
data_columns=True)
# h5.put(table, df, format='table',index=False, data_columns=True, append=False)
else:
if rewrite:
h5.remove(table)
h5.put(table,
df,
format='table',
index=False,
complib=complib,
data_columns=True,
append=True)
# h5.append(table, df, format='table', append=True,data_columns=True, dropna=None)
else:
if not MultiIndex:
# h5[table]=df
h5.put(table,
df,
format='table',
index=False,
complib=complib,
data_columns=True)
# h5.put(table, df, format='table',index=False, data_columns=True, append=False)
else:
h5.put(table,
df,
format='table',
index=False,
complib=complib,
data_columns=True,
append=True)
# h5.append(table, df, format='table', append=True, data_columns=True, dropna=None)
# h5[table]=df
h5.flush()
else:
log.error("HDFile is None,Pls check:%s" % (fname))
log.info("write hdf time:%0.2f" % (time.time() - time_t))
return True
top_all = tdd.get_append_lastp_to_df(top_now, lastpTDX_DF)
# top_all = top_all.merge(lastpTDX_DF, left_index=True, right_index=True, how='left')
# top_all = top_all[top_all['llow'] > 0]
else:
# top_now = top_now[top_now.trade >= top_now.high * 0.98]
# for symbol in top_now.index:
# # code = rl._symbol_to_code(symbol)
# if symbol in top_all.index:
# top_all.loc[symbol,['name', 'percent', 'dff', 'couts', 'trade', 'high', 'open', 'low', 'ratio', 'volume',
# 'prev_p']] = top_now.loc[symbol,['name', 'percent', 'dff', 'couts', 'trade', 'high', 'open', 'low', 'ratio', 'volume',
# 'prev_p']]
# else:
# top_all.append(top_now.loc[symbol])
top_all = cct.combine_dataFrame(top_all, top_now, col=None)
# top_bak = top_all
# top_all['buy'] = (
# map(lambda x, y: y if int(x) == 0 else x, top_all['buy'].values, top_all['trade'].values))
codelist = top_all.index.tolist()
if len(codelist) > 0:
# log.info('toTDXlist:%s' % len(codelist))
# top_all = tdd.get_append_lastp_to_df(top_all)
# log.info("df:%s" % top_all[:1])
radio_t = cct.get_work_time_ratio()
log.debug("Second:vol/vol/:%s" % radio_t)
# top_all['volume'] = top_all['volume'].apply(lambda x: round(x / radio_t, 1))
log.debug("top_diff:vol")
top_all['volume'] = (map(
lambda x, y: round(x / y / radio_t, 1),
# top_now.loc[symbol, 'dff'] = count_n - count_a
# if status_change:
# # top_all.loc[symbol] = top_now.loc[symbol]
# top_all.loc[symbol, ['name', 'percent', 'dff', 'couts', 'trade', 'high', 'open', 'low', 'ratio', 'volume',
# 'prev_p']] = top_now.loc[symbol, ['name', 'percent', 'dff', 'couts', 'trade', 'high', 'open', 'low', 'ratio', 'volume',
# 'prev_p']]
# else:
# top_all.loc[symbol, ['percent', 'dff']] = top_now.loc[
# symbol, ['percent', 'dff']]
# # top_all.loc[symbol, 'trade':] = top_now.loc[symbol, 'trade':]
# top_all.loc[symbol, ['trade', 'high', 'open', 'low', 'ratio', 'volume',
# 'prev_p']] = top_now.loc[symbol, ['trade', 'high', 'open', 'low', 'ratio', 'volume',
# 'prev_p']]
# else:
# top_all.append(top_now.loc[symbol])
top_all = cct.combine_dataFrame(
top_all, top_now, col='couts', compare='dff')
# top_all=top_all.sort_values(by=['dff','percent','couts'],ascending=[0,0,1])
# top_all=top_all.sort_values(by=['dff','ratio','percent','couts'],ascending=[0,1,0,1])
# top_all=top_all.sort_values(by=['dff','percent','couts','ratio'],ascending=[0,0,1,1])
# top_all[(top_all.upperT > 3) & (top_all.top10 >2) &(top_all.close > top_all.upper*0.98) & (top_all.close < top_all.upper *1.05)]
# top_all[(top_all.upperT > 3) & (top_all.top10 >2) &(top_all.close > top_all.upper*0.98) & (top_all.close < top_all.upper *1.05) &(top_all.lastp1d > top_all.upper)].name
# cct.write_to_blocknew(block_path, dd.index.tolist())
# writecode = "cct.write_to_blocknew(block_path, dd.index.tolist())"
top_bak = top_all
# market_sort_value, market_sort_value_key = ct.get_market_sort_value_key(st_key_sort, top_all=top_all)
codelist = top_all.index.tolist()
if len(codelist) > 0:
# log.info('toTDXlist:%s' % len(codelist))
# tdxdata = tdd.get_tdx_all_day_LastDF(codelist)
top_all = tdd.get_append_lastp_to_df(top_now, lastpTDX_DF)
else:
if 'couts' in top_now.columns.values:
if not 'couts' in top_all.columns.values:
top_all['couts'] = 0
top_all['prev_p'] = 0
# for symbol in top_now.index:
# if symbol in top_all.index and top_now.loc[symbol, 'buy'] <> 0:
# if 'couts' in top_now.columns.values:
# top_all.loc[symbol, 'buy':'prev_p'] = top_now.loc[
# symbol, 'buy':'prev_p']
# else:
# top_all.loc[symbol, 'buy':'low'] = top_now.loc[
# symbol, 'buy':'low']
top_all = cct.combine_dataFrame(top_all, top_now, col=None)
top_dif = top_all.copy()
market_sort_value, market_sort_value_key = ct.get_market_sort_value_key(
st_key_sort, top_all=top_all)
top_dif = top_dif[top_dif.lvol > ct.LvolumeSize]
# if top_dif[:1].llow.values <> 0:
# if not (cct.get_now_time_int() > 915 and cct.get_now_time_int() <= 925):
if len(top_dif[:5][top_dif[:5]['buy'] > 0]) > 3:
log.debug('diff2-0-buy>0')
if cct.get_now_time_int() > 915 and cct.get_now_time_int(
) < ct.checkfilter_end_time:
top_dif = top_dif[top_dif.low > top_dif.llow *
ct.changeRatio]
log.debug('dif4 open>low0.99:%s' % len(top_dif))
# top_dif['buy'] = (map(lambda x, y: y if int(x) == 0 else x, top_dif['buy'].values, top_dif['trade'].values))
def getBollFilter(df=None,
boll=ct.bollFilter,
duration=ct.PowerCountdl,
filter=True,
ma5d=True,
dl=14,
percent=True,
resample='d',
ene=False,
upper=False,
down=False,
indexdff=True,
cuminTrend=False,
top10=True):
# drop_cxg = cct.GlobalValues().getkey('dropcxg')
# if len(drop_cxg) >0:
# log.info("stf drop_cxg:%s"%(len(drop_cxg)))
# drop_cxg = list(set(drop_cxg))
# drop_t = [ co for co in drop_cxg if co in df.index]
# if len(drop_t) > 0:
# df = df.drop(drop_t,axis=0)top
# log.error("stf drop_cxg:%s"%(len(drop_t)))
# hvdu max量天数 lvdu min天数 hv max量 lv 量
# fib < 2 (max) fibl > 2 lvdu > 3 (hvdu > ? or volume < 5)? ene
time_s = time.time()
# df['upper'] = map(lambda x: round((1 + 11.0 / 100) * x, 1), df.ma10d)
# df['lower'] = map(lambda x: round((1 - 9.0 / 100) * x, 1), df.ma10d)
# df['ene'] = map(lambda x, y: round((x + y) / 2, 1), df.upper, df.lower)
'''
df = df[(df.close > df.ma20d) | (df.close > df.ene)]
if filter:
if 'boll' in df.columns:
df = df[(df.boll >= boll) | (df.buy > df.upper)]
'''
# if 'nlow' in df.columns:
# df = df[ ((df.nlow >= df.llastp) & (df.nlow >= df.low)) | ((df.nlow <= df.low) & (df.open == df.nlow))]
# if cct.get_work_time() and 'df2' in df.columns:
# if 915 < cct.get_now_time_int() < 930:
# df['cumnow'] = map(lambda x, y: 1 if x - y > 0 else 0, df.close, df.cumaxe)
# else:
# df['cumnow'] = map(lambda x, y: 1 if x - y > 0 else 0, df.high, df.cumaxe)
# df['df2'] = (df['df2'] + df['cumnow']).map(lambda x: int(x))
# df = df[df.df2 > 1]
# else:
# df = df[df.df2 > 0]
radio_t = cct.get_work_time_ratio()
df['lvolr%s' % (resample)] = df['volume']
# df['volume'] = (map(lambda x, y: round(x / y / radio_t, 1), df.nvol.values, df.lvolume.values))
if cct.get_now_time_int() > 1530:
df['volume'] = (map(
lambda x, y, z: round((x / y / radio_t)
if z < 9.9 else (x / y), 1), df.nvol.values,
df.ma5vol.values, df.percent.values))
else:
df['volume'] = (map(
lambda x, y, z: round((x / y / radio_t)
if z < 9.9 else (x / y), 1), df.nvol.values,
df.lvol.values, df.percent.values))
# df['volume'] = (map(lambda x, y,z: round((x / y / radio_t) if z < 9.9 else (x / y) , 1), df.nvol.values, df.ma5vol.values,df.percent.values))
# df['volume'] = (map(lambda x, y: round(x / y / radio_t, 1), df.nvol.values, df.lastv1d.values))
indexfibl = cct.GlobalValues().getkey('indexfibl')
sort_value = cct.GlobalValues().getkey('market_sort_value')
market_key = cct.GlobalValues().getkey('market_key')
market_value = cct.GlobalValues().getkey('market_value')
# if int(market_value) > 1 and 930 < cct.get_now_time_int():
df = compute_perd_value(df, market_value, 'perc')
df = compute_perd_value(df, market_value, 'per')
if sort_value <> 'percent' and (market_key in ['2', '3']
and market_value not in ['1']):
# print("sort_value:%s,market_key:%s ,market_value:%s" %
# (sort_value, market_key, market_value))
if market_key is not None and market_value is not None:
# if market_key == '3':
if market_key == '3':
market_value = int(market_value)
log.info("stf market_key:%s" % (market_key))
idx_k = cct.get_col_in_columns(df, 'perc%sd', market_value)
# filter percd > idx
# df= df[(df[("perc%sd" % (idx_k))] >= idx_k) | (df[("perc%sd" % (idx_k))]< -idx_k)]
# if int(market_value) > 1 and 930 < cct.get_now_time_int():
# df= compute_perd_value(df, market_value, 'perc')
# df= compute_perd_value(df, market_value, 'per')
# elif market_key == '2':
# if int(market_value) > 1 and 915 < cct.get_now_time_int():
# # df['per%d'%(market_value)] = compute_perd_value(df,market_value)
# df= compute_perd_value(df, market_value,'per')
# df= compute_perd_value(df, market_value, 'perc')
# if sort_value <> 'percent' and (market_key in ['2','3'] and market_value not in ['1'] ):
# if indexfibl is not None:
# df = df[df.fibl >= (indexfibl - 1)]
# else:
# indexfibl = 1
# if cuminTrend:
# cuminfibl = cct.GlobalValues().getkey('cuminfibl')
# if cuminfibl is None:
# cuminfibl = cct.get_index_fibl()
# if cuminfibl > 1:
# df = df[df.df2 >= cuminfibl]
# print df.loc['300107'][['close','cumine','llastp','cumaxe','cumaxc']]
# df = df[ (df.cumine == df.cumaxc) | (df.close >= df.cumine) | (df.llastp >=df.cumaxc) | (df.close >= df.cumaxe) ]
# else:
# upper = True
if not down:
if upper:
# upper_count = len( df[ ( ( df.buy > df.upper)| ( df.lastp0d > df.upper) | ( df.lastp1d > df.upper) | ( df.lastp2d > df.upper) | ( df.lastp3d > df.upper))])
# # ene_count = len(df[df.buy > df.upper])
# if upper_count > 30:
# df = df[ ( ( df.buy > df.upper)| ( df.lastp0d > df.upper) | ( df.lastp1d > df.upper) | ( df.lastp2d > df.upper) | ( df.lastp3d > df.upper))]
# else:
# df = df[ ( (( df.buy > df.ene ) ) | ( df.buy > df.upper)| ( df.lasth0d > df.upper) | ( df.lasth1d > df.upper) | ( df.lasth2d > df.upper) | ( df.lasth3d > df.upper))]
df = df[(df.low >= df.upper) | (df.close >= df.upper) |
(df.per1d >= 9.5) | (df.lastp1d == df.hmax) |
((df.lastp1d <> df.hmax) & (df.high > df.hmax))]
else:
if 935 < cct.get_now_time_int() < 1400:
df = df[((df.buy > df.ene) | (df.llastp > df.ene) |
(df.llastp >= df.max5) | (df.high >= df.max5) |
(df.max5 >= df.upper))]
else:
df = df[((df.buy > df.ene) | (df.llastp > df.ene) |
(df.llastp >= df.max5) | (df.high >= df.max5) |
(df.max5 >= df.upper))]
# df['cumins'] = round(cum_counts.index[0], 2)
# df['cumine'] = round(cumdf[-1], 2)
# df['cumax']
# df.rename(columns={'cumin': 'df2'}, inplace=True)
if df is None:
print "dataframe is None"
return None
else:
# top10 = df[ (df.percent >= 9.99) & (df.b1_v > df.a1_v)]
if resample in ['d', 'w']:
# if not top10 or (cct.get_now_time_int() < 950 or cct.get_now_time_int() > 1502):
# df.loc[((df.b1_v > df.a1_v) & (df.percent > 9)), 'percent'] = 10.1
# df.loc[((df.percent >= 9.97) & (df.percent < 10.1)), 'percent'] = 10
# else:
# df.loc[((df.b1_v > df.a1_v) & (df.percent > 9)), 'percent'] = 10.1
# df.loc[((df.percent >= 9.97) & (df.percent < 10.1)), 'percent'] = 10
# df.loc[df.per1d >= 9.99, 'per1d'] = 10
df['percent'] = df['percent'].apply(lambda x: round(x, 2))
# time_ss = time.time()
perc_col = [co for co in df.columns if co.find('perc') == 0]
per_col = [co for co in df.columns if co.find('per') == 0]
# per_col = list(set(per_col) - set(perc_col) - set(['per1d', 'perlastp']))
per_col = list(
set(per_col) - set(perc_col) - set(['per1d', 'perlastp']))
perc_col.remove('percent')
for co in perc_col:
df[co] = df[co].apply(lambda x: round(x, 2))
# da, down_zero, down_dn, percent_l = 1, 0, 0, 2
# da, down_zero, down_dn, percent_l = 1, 0, -1, 1
# df['perc_n'] = map((lambda h, lh, l, ll, c, lc: (1 if (h - lh) > 0 else down_dn) + (1 if c - lc > 0 else down_dn) + (1 if (l - ll) > 0 else down_dn) + (2 if (c - lh) > 0 else down_zero) + (2 if (l - lc) > 0 else down_zero) + (0 if (h - lc) > 0 else down_dn)), df['high'], df['lasth%sd' % da], df['low'], df['lastl%sd' % da], df['close'],df['lastp%sd' % da])
# df['perc_n'] = map((lambda c, lc: (1 if (c - lc) > 0 else down_zero) + (1 if (c - lc) / lc * 100 > 3 else down_zero) +
# (down_dn if (c - lc) / lc * 100 < -3 else down_zero)), df['close'], df['lastp%sd' % da])
# idx_rnd = random.randint(0, len(df) - 10) if len(df) > 10 else 0
# print "idx_rnd",idx_rnd,df.ix[idx_rnd].lastp0d ,df.ix[idx_rnd].close,df.ix[idx_rnd].lastp0d != df.ix[idx_rnd].close
# if not upper:
# if cct.get_work_time() or df.ix[idx_rnd].lastp0d <> df.ix[idx_rnd].close:
# nowd, per1d = 0, 1
# if 'nlow' in df.columns:
# df['perc_n'] = map(cct.func_compute_percd2, df['close'], df['llastp'], df['open'],df['lasto%sd' % nowd], df['lasth%sd' %
# (nowd)], df['lastl%sd' % (nowd)], df['nhigh'], df['nlow'],df['nvol']/radio_t,df['lastv%sd'%(nowd)], df['hmax'],df['df2'])
# else:
# df['perc_n'] = map(cct.func_compute_percd2, df['close'], df['llastp'], df['open'],df['lasto%sd' % nowd], df['lasth%sd' %
# (nowd)], df['lastl%sd' % (nowd)], df['high'], df['low'],df['nvol']/radio_t,df['lastv%sd'%(nowd)], df['hmax'],df['df2'])
# else:
# nowd, per1d = 1, 2
# # print df['per%sd' % da+1], df['lastp%sd' % (da)], df['lasth%sd' % (da)], df['lastl%sd' % (da)], df['high'], df['low']
# df['perc_n'] = map(cct.func_compute_percd2, df['close'], df['llastp'], df['open'],df['lasto%sd' % nowd], df['lasth%sd' %
# (nowd)], df['lastl%sd' % (nowd)], df['high'], df['low'], df['hmax'])
'''
idx_rnd = random.randint(0, len(df) - 10) if len(df) > 10 else 0
if not upper:
# if cct.get_work_time() or df.ix[idx_rnd].lastp1d <> df.ix[idx_rnd].close:
if cct.get_work_time() :
nowd, per1d=1, 1
if 'nlow' in df.columns:
df['perc_n']=map(cct.func_compute_percd2020, df['open'], df['close'], df['nhigh'], df['nlow'], df['lasto%sd' % nowd], df['llastp'],
df['lasth%sd' % (nowd)], df['lastl%sd' % (nowd)], df['ma5d'], df['ma10d'], df['nvol'] / radio_t, df['lastv%sd' % (nowd)])
else:
df['perc_n']=map(cct.func_compute_percd2020, df['open'], df['close'], df['high'], df['low'], df['lasto%sd' % nowd], df['llastp'],
df['lasth%sd' % (nowd)], df['lastl%sd' % (nowd)], df['ma5d'], df['ma10d'], df['nvol'] / radio_t, df['lastv%sd' % (nowd)])
else:
nowd, per1d=1, 1
# print df['per%sd' % da+1], df['lastp%sd' % (da)], df['lasth%sd' % (da)], df['lastl%sd' % (da)], df['high'], df['low']
df['perc_n']=map(cct.func_compute_percd2020, df['open'], df['close'], df['high'], df['low'], df['lasto%sd' % nowd], df['llastp'],
df['lasth%sd' % (nowd)], df['lastl%sd' % (nowd)], df['ma5d'], df['ma10d'], df['nvol'] / radio_t, df['lastv%sd' % (nowd)])
for co in perc_col:
df[co] = (df[co] + df['perc_n']).map(lambda x: int(x))
'''
# for co in per_col:
# df[co] = (df[co] + df['percent']).map(lambda x: int(x))
# print "percT:%.2f"%(time.time()-time_ss)
else:
df['percent'] = (map(
lambda x, y: round((x - y) / y * 100, 1)
if int(y) > 0 else 0, df.buy, df.lastp1d))
# df.loc['002204'].percent
# -15.4
# ipdb> df.loc['002204'].ma10d
# 3.2000000000000002
# ipdb> df.loc['002204'].ma5d
# 3.2999999999999998
if 'fib' not in df.columns:
df['fib'] = 0
# else:
co2int = ['boll', 'op', 'ratio', 'fib', 'fibl', 'df2']
# co2int.extend([co for co in df.columns.tolist()
# if co.startswith('perc') and co.endswith('d')])
co2int.extend(['top10', 'topR'])
co2int = [inx for inx in co2int if inx in df.columns]
for co in co2int:
df[co] = df[co].astype(int)
# if down:
# # if 'nlow' in df.columns:
# # if 'max5' in df.columns:
# # # df = df[(df.hmax >= df.ene) & (df.nlow >= df.low) & (df.low <> df.high) & (((df.low < df.hmax) & (df.high > df.cmean)) | (df.high > df.max5))]
# # df = df[ (((df.buy > df.ene) & (df.volume / radio_t > 1.5)) ) | (((df.lastv1d > df.lvol * 1.5) & (df.lastv1d > df.lvol * 1.5)) & (df.lastp1d > df.ene) )]
# # else:
# # df = df[ (((df.buy > df.ene) & (df.volume / radio_t > 1.5)) ) | (((df.lastv1d > df.lvol * 1.5) ) )]
# # else:
# # # df = df[ (df.max5 >= df.ene) & (df.low <> df.high) & (((df.low < df.hmax) & (df.close > df.cmean)) | (df.high > df.cmean))]
# # df = df[ (((df.buy > df.ene) & (df.volume / radio_t > 1.5)) ) | (((df.lastp1d > df.ene) ) | ((df.buy > df.max5)|(df.buy > df.ene)) | (df.max5 >= df.ene) | (df.high > df.cmean) )]
# return df
if cct.get_work_time() and 'b1_v' in df.columns and 'nvol' in df.columns:
df = df[(df.b1_v > 0) | (df.nvol > 0)]
if (cct.get_now_time_int() > 915 and cct.get_now_time_int() < 926):
df['b1_v'] = df['volume']
else:
dd = df[df.percent < 10]
dd['b1_v'] = dd['volume']
df = cct.combine_dataFrame(df, dd.loc[:, ['b1_v']])
# print "t:%0.2f"%(time.time()-time_ss)
# if int(market_value) < 3 and 930 < cct.get_now_time_int():
# if 'nlow' in df.columns:
# # df = df[((df.per1d < 3) & (df.nlow >= df.llastp)) | ((df.per1d > 5) & (df.buy > df.llastp))]
# df = df[((df.per1d < 3) & (df.nlow >= df.llastp)) | ((df.per1d > 5) & (df.buy > df.llastp))]
# else:
# df = df[((df.per1d < 3) & (df.open > df.llastp)) | ((df.per1d > 5) & (df.buy > df.llastp))]
# else:
# if 'fib' in df.columns:
# df = df[df.fib <= 5]
# log.error("perc%sd"%(market_value))
# elif market_key is not None and market_key == '2':
# if market_value in ['1', '2']:
# df['dff'] = (map(lambda x, y, z: round((x + (y if z > 20 else 3 * y)), 1), df.dff.values, df.volume.values, df.ratio.values))
# df['df2'] = (map(lambda x, y, z: w=round((x - y) / z * 100, 1), df.high.values, df.low.values, df.llastp.values))
# df['df2'] = (map(func_compute_df2, df.close.values, df.llastp.values,df.high.values, df.low.values,df.ratio.values))
# df['df2'] = (map(func_compute_df2, df.close.values, df.llastp.values,df.high.values, df.low.values))
'''
filter_up= 'ma5d'
filter_dn= 'ma10d'
if ma5d and filter_up in df.columns:
if filter_dn in df.columns:
if market_key is not None and market_key == '3':
df= df[(df.buy >= df[filter_up]) & (
df.buy * 1.02 >= df[filter_dn] * ct.changeRatio)]
else:
df= df[(df.buy >= df[filter_up])]
else:
df= df[df.buy >= df[filter_up] * ct.changeRatio]
'''
# df = df[(df.lvolume > df.lvol * 0.9) & (df.lvolume > df.lowvol * 1.1)]
# if 'nlow' in df.columns and 932 < cct.get_now_time_int() < 1030:
if ene:
pass
if 'nclose' in df.columns:
# df = df[ ((df.buy > df.ene) ) | ((df.buy > df.upper) & ( df.nclose > df.cmean))]
df = df[((df.buy > df.llastp)) |
((df['nclose'] > df['ene'] * ct.changeRatio) &
(df['percent'] > -3) & (df.volume > 1.5)) |
((df['nclose'] > df['upper'] * ct.changeRatio) &
(df['buy'] > df['upper'] * ct.changeRatioUp)) |
((df['llastp'] > df['upper']) &
(df['nclose'] > df['upper']))]
else:
df = df[((df.buy > df.llastp)) | ((df.buy > df.ene)) |
((df.buy > df.upper) & (df.close > df.cmean))]
if 'nlow' in df.columns and 930 < cct.get_now_time_int():
# for col in ['nhigh', 'nclose', 'nlow','nstd']:
# df[col] = df[col].apply(lambda x: round(x, 2))
if 'nhigh' in df.columns and 'nclose' in df.columns:
# ncloseRatio = map(lambda x, y: x * ct.changeRatio if x * ct.changeRatio > y else y, df.nclose, df.nlow)
df['ncloseRatio'] = map(lambda x: x * 0.99, df.nclose)
df['nopenRatio'] = map(lambda x: x * 0.99, df.open)
# nhighRatio = map(lambda x, y: x * ct.changeRatio if x * ct.changeRatio > y else y, df.nhigh, df.nclose)
# if cct.get_now_time_int() > ct.nlow_limit_time:
# name open low high close nclose nlow nhigh nstd ticktime
# code
# 601899 紫金矿业 4.25 4.24 4.69 4.69 4.52 4.24 4.42 0.12 15:00:00
# 603917 合力科技 27.95 27.69 31.54 31.54 30.46 27.69 30.00 0.70 15:00:00
# 300713 英可瑞 81.00 81.00 89.82 89.77 87.53 80.90 86.36 1.75 15:02:03
# 000933 神火股份 9.07 9.02 10.00 10.00 9.71 9.02 9.44 0.24 15:02:03
# 600050 中国联通 6.49 6.37 6.51 6.41 6.44 6.44 6.51 0.02 15:00:00
# 002350 北京科锐 10.17 10.17 10.75 10.23 10.31 10.17 10.75 0.07 15:02:03
# 603363 傲农生物 17.49 16.91 18.35 18.35 17.35 17.18 18.00 0.34 15:00:00
# 000868 安凯客车 8.77 8.53 8.82 8.82 8.82 8.53 8.82 0.01 15:02:03
# 002505 大康农业 2.91 2.86 2.99 2.89 2.91 2.90 2.99 0.02 15:02:03
# 601939 建设银行 7.50 7.41 7.61 7.55 7.50 7.43 7.52 0.05 15:00:00
# 600392 盛和资源 16.85 16.81 18.63 18.63 17.84 16.81 17.51 0.69 15:00:00 (11, 41)
# 603676 卫信康 17.44 16.95 17.96 17.82 17.34 17.03 17.77 0.18 15:00:00
if 'nstd' in df.columns:
df['stdv'] = map(lambda x, y: round(x / y * 100, 1),
df.nstd, df.open)
# top_temp[:40][['nstd','stdv','name','volume','dff','percent']]
# top_temp.loc['603363'][['nstd','stdv','name','volume','high','nhigh','percent','open','nclose']]
# df['stdv'] = map(lambda x, y, z: round(z / (x / y * 100), 1), df.nstd, df.open, df.volume)
# df.loc['002906'][['open','nclose','close','ncloseRatio','nopenRatio','low','nlow']]
df = df[(df.low > df.upper) |
(((df.nopenRatio <= df.nclose) & (df.low >= df.nlow) &
(df.close >= df.ncloseRatio)) |
((df.nopenRatio <= df.nclose) &
(df.close >= df.ncloseRatio) &
(df.high >= df.nhigh)))]
# else:
# df = df[(((df.low >= df.nlow) & (df.close >= ncloseRatio)) | ((df.close >= ncloseRatio) & (df.close > nhighRatio)))]
else:
df = df[((df.low >= df.nlow) &
(df.close > df.llastp * ct.changeRatio))]
# if 'nhigh' in df.columns and 'nclose' in df.columns:
# if cct.get_now_time_int() > ct.nlow_limit_time:
# df = df[(df.low >= df.nlow) & ((df.open > df.llastp * ct.changeRatio) & (df.nclose > df.llastp * ct.changeRatio)) &
# (((df.low >= df.nlow) & (df.close >= df.nclose)) | ((df.close >= df.nclose) & (df.close > df.nhigh * ct.changeRatio) & (df.high >= df.nhigh)))]
# else:
# df = df[((df.open > df.llastp * ct.changeRatio) & (df.close > df.llastp * ct.changeRatio)) &
# (((df.low >= df.nlow) & (df.close >= df.nclose)) | ((df.close >= df.nclose) & (df.close > df.nhigh * ct.changeRatio)))]
# else:
# df = df[((df.low >= df.nlow) & (df.close > df.llastp))]
if filter:
if cct.get_now_time_int() > 915 and cct.get_now_time_int() <= 1000:
df = df[((df.buy > df.llastp)) |
(df.buy > df.hmax * ct.changeRatio) |
(df.buy < df.lmin * ct.changeRatioUp)]
# df = df[(df.buy > df.hmax * ct.changeRatio) & (df.low > df.llastp)]
# df = df[(df.buy > df.cmean * ct.changeRatioUp) | (df.open > df.llastp)]
# df = df[df.buy > df.cmean]
elif cct.get_now_time_int() > 1000 and cct.get_now_time_int(
) <= 1430:
df = df[((df.buy > df.llastp)) |
(df.buy > df.hmax * ct.changeRatio) |
(df.buy < df.lmin * ct.changeRatioUp)]
# df = df[(df.buy > df.hmax * ct.changeRatio) | (df.low > df.llastp)]
# df = df[(df.buy > df.hmax * ct.changeRatio) & (df.low > df.llastp)]
# df = df[(df.buy > df.cmean * ct.changeRatioUp) | (df.open > df.llastp)]
# df = df[df.buy > df.cmean]
else:
df = df[((df.buy > df.llastp)) |
(df.buy > df.hmax * ct.changeRatio) |
(df.buy < df.lmin * ct.changeRatioUp)]
# df = df[(df.buy > df.hmax * ct.changeRatio) | (df.low > df.llastp)]
# df = df[(df.buy > df.hmax * ct.changeRatio) & (df.low > df.llastp)]
# df = df[(df.buy > df.cmean * ct.changeRatioUp) | (df.open > df.llastp)]
# df = df[df.buy > df.cmean]
if 'vstd' in df.columns:
# df = df[(df.lvol * df.volume > (df.vstd + df.lvol)) | ((df.percent > -10) & (df.hv / df.lv > 1.5))]
# dd = df.loc['000760'] dd.hv/dd.lv,dd.volume
df['hvRatio'] = map(
lambda x, y: round(x / y / cct.get_work_time_ratio(), 1),
df.hv, df.lv)
df['volRatio'] = (map(lambda x, y: round(x / y / radio_t, 1),
df.nvol.values, df.lv.values))
# df = df[(df.lvol * df.volume > (df.vstd + df.lvol)) | ((df.percent > -5) & (df.hv / df.lv > 3))]
# if 1000 < cct.get_now_time_int() < 1450:
# df = df[ (3 < df.ratio) & (df.ratio < 20)]
if 'nclose' in df.columns:
# df = df[((df.hvRatio > df.volRatio) & ((df.buy > df.lastp) & (df.nclose > df.lastp))) | ((
df = df[(((df.buy > df.lastp) & (df.nclose > df.lastp))) |
(((df.percent > -5) &
((df.vstd + df.lvol) * 1.2 < df.nvol) &
(df.nvol > (df.vstd + df.lvol))))]
else:
# df = df[((df.hvRatio > df.volRatio) & ((df.buy > df.lastp))) | ((
df = df[(((df.buy > df.lastp))) |
(((df.percent > -5) &
((df.vstd + df.lvol) * 1.2 < df.nvol) &
(df.nvol > (df.vstd + df.lvol))))]
# [dd.lvol * dd.volume > (dd.vstd + dd.lvol) | dd.lvol * dd.volume >(dd.ldvolume + dd.vstd]
# print df.loc['000801']
if percent:
pass
'''
if 'stdv' in df.columns and 926 < cct.get_now_time_int():
# df = df[((df.volume > 2 * cct.get_work_time_ratio()) & (df.percent > -3)) | ((df.stdv < 1) &
# (df.percent > 2)) | ((df.lvolume > df.lvol * 0.9) & (df.lvolume > df.lowvol * 1.1))]
df_index = tdd.getSinaIndexdf()
if isinstance(df_index, type(pd.DataFrame())):
df_index['volume'] = (map(lambda x, y: round(x / y / radio_t, 1), df_index.nvol.values, df_index.lastv1d.values))
index_vol = df_index.loc['999999'].volume
if 'percent' in df_index.columns and '999999' in df_index.index:
index_percent = df_index.loc['999999'].percent
else:
# import ipdb;ipdb.set_trace()
log.error("index_percent" is None)
index_percent = 0
index_boll = df_index.loc['999999'].boll
index_op = df_index.loc['999999'].op
# df = df[((df.percent > -3) | (df.volume > index_vol)) & (df.percent > index_percent)]
# df = df[((df.percent > -1) | (df.volume > index_vol * 1.5)) & (df.percent > index_percent) & (df.boll >= index_boll)]
df = df[((df.percent > -1) | (df.volume > index_vol * 1.5)) & (df.percent > index_percent)]
else:
print ("df_index is Series",df_index.T)
else:
df = df[((df.volume > 1.2 * cct.get_work_time_ratio()) & (df.percent > -3))]
'''
# df = df[((df['buy'] >= df['ene'])) | ((df['buy'] < df['ene']) & (df['low'] > df['lower'])) | ((df['buy'] > df['upper']) & (df['low'] > df['upper']))]
# df = df[(( df['ene'] * ct.changeRatio < df['open']) & (df['buy'] > df['ene'] * ct.changeRatioUp)) | ((df['low'] > df['upper']) & (df['close'] > df['ene']))]
# df = df[(( df['ene'] < df['open']) & (df['buy'] < df['ene'] * ct.changeRatioUp))]
# df = df[(df.per1d > 9) | (df.per2d > 4) | (df.per3d > 6)]
# df = df[(df.per1d > 0) | (df.per2d > 4) | (df.per3d > 6)]
# time_ss=time.time()
# codel = df.index.tolist()
# dm = tdd.get_sina_data_df(codel)
# results = cct.to_mp_run_async(getab.Get_BBANDS, codel,'d',5,duration,dm)
# bolldf = pd.DataFrame(results, columns=['code','boll'])
# bolldf = bolldf.set_index('code')
# df = cct.combine_dataFrame(df, bolldf)
# print "bollt:%0.2f"%(time.time()-time_ss),
per3d_l = 2
percent_l = -1
op_l = 3
# if 'boll' in df.columns:
# if 915 < cct.get_now_time_int() < 950:
# # df = df[(df.boll >= boll) | ((df.percent > percent_l) & (df.op > 4)) | ((df.percent > percent_l) & (df.per3d > per3d_l))]
# # df = df[((df.percent > percent_l) & (df.op > 4)) | ((df.percent > percent_l) & (df.per3d > per3d_l))]
# pass
# elif 950 < cct.get_now_time_int() < 1501:
# # df = df[(df.boll >= boll) | ((df.low <> 0) & (df.open == df.low) & (((df.percent > percent_l) & (df.op > op_l)) | ((df.percent > percent_l) & (df.per3d > per3d_l))))]
# # df = df[(df.boll >= boll) & ((df.low <> 0) & (df.open >= df.low *
# # ct.changeRatio) & (((df.percent > percent_l)) | ((df.percent >
# # percent_l) & (df.per3d > per3d_l))))]
# df = df[(df.boll >= boll)]
# # else:
# df = df[(df.boll >= boll) | ((df.low <> 0) & (df.open == df.low) &
# (((df.percent > percent_l) & (df.op > op_l)) | ((df.percent > percent_l)
# & (df.per3d > per3d_l))))]
# if 945 < cct.get_now_time_int() and market_key is None:
# df.loc[df.percent >= 9.94, 'percent'] = -10
# else:
# # df['upper'] = map(lambda x: round((1 + 11.0 / 100) * x, 1), df.ma10d)
# # df['lower'] = map(lambda x: round((1 - 9.0 / 100) * x, 1), df.ma10d)
# # df['ene'] = map(lambda x, y: round((x + y) / 2, 1), df.upper, df.lower)
# # df = df[((df['buy'] >= df['ene'])) | ((df['buy'] < df['ene']) & (df['low'] > df['lower'])) | ((df['buy'] > df['upper']) & (df['low'] > df['upper']))]
# # df = df[(( df['ene'] * ct.changeRatio < df['open']) & (df['buy'] > df['ene'] * ct.changeRatioUp)) | ((df['low'] > df['upper']) & (df['close'] > df['ene']))]
# if 'nclose' in df.columns:
# # df = df[((df['nclose'] > df['ene'] * ct.changeRatio) & (df['percent'] > -3) & (df.volume > 1.5)) |
# # ((df['nclose'] > df['upper'] * ct.changeRatio) & (df['buy'] > df['upper'] * ct.changeRatioUp)) | ((df['llastp'] > df['upper']) & (df['nclose'] > df['upper']))]
# df = df[(df.nclose > df.cmean)]
# else:
# df = df[(df.buy > df.cmean)]
print("bo:%0.1f" % (time.time() - time_s)),
df['ral'] = (map(lambda x, y: round((x + y), 1), df.percent, df.ral))
df['ra'] = (map(lambda x, y: round((x + y), 1), df.percent, df.ra))
return df
def get_market_price_sina_dd_realTime(dp='',vol='0',type='0'):
'''
input df count and merge price to df
'''
if len(dp)==0:
dp=get_sina_Market_json()
if len(dp)>10:
# df=df.dropna('index')
# df=df.drop_duplicates('code')
# dm=pd.merge(df,dp,on='name',how='left')
# print(type(dp))
# dp=dp.drop_duplicates('code')
log.info("Market_realTime:%s"%len(dp))
# dp=dp.set_index('code')
dp=dp.fillna(0)
dp=dp.dropna('index')
# if dp[:1].volume.values >0:
# log.debug("dp.volume>0:%s"%dp[:1].volume.values)
# dp['volume']=dp['volume'].apply(lambda x:round(float(x)/100,1))
# dp=dp.loc[:,'trade':].astype(float)
log.info("DP:%s" % dp[:1].open)
# if len(dp[:10][dp[:10]['buy'] > 0]) > 2 and len(dp[:10][dp[:10]['percent'] == 0]) > 2:
# if 'close' in dp.columns:
# if len(dp[:5][dp[:5]['close'] > 0]) > 2:
# dp['percent'] = (map(lambda x, y: round((x - y) / y * 100, 1), dp['buy'].values, dp['close'].values))
# log.info("DP-1-percent==0:%s" % dp[:1].percent)
# dp.loc[dp.percent>9.9,'percent']=10
dp['dff']=0
df=get_sina_all_json_dd(vol,type)
if len(df)>5:
# print df[df.couts>0][:2]
dm = cct.combine_dataFrame(dp,df)
log.info("top_now:main:%s subobject:%s dm:%s "%(len(dp),len(df),len(dm)))
# dm=pd.merge(dp,df,on='name',how='left')
log.debug("dmMerge:%s"%dm.columns)
# print dm[dm.couts>0][:2]
dm.couts=dm.couts.fillna(0)
dm.prev_price=dm.prev_price.fillna(0.0)
dm.couts=dm.couts.astype(int)
# dm=dm.drop_duplicates('code')
# dm=dm.set_index('code')
dm.rename(columns={'prev_price': 'prev_p'}, inplace=True)
# print dm.sort_values(by=['couts','percent','dff','ratio'],ascending=[0,0,0,1])[:2]
# dm=dm.fillna(int(0))
# dm.ratio=dm.ratio
# dm=dm.loc[:,ct.SINA_Market_Clean_UP_Columns]
else:
# if len(dp) > 0 and 'code' in dp.columns:
if len(dp) > 0:
if 'code' in dp.columns:dp=dp.set_index('code')
dp['couts'] = 0
dp['prev_p'] = 0
dm = dp
else:
log.error('dp is None')
dm = ''
# dm=dp.loc[:,ct.SINA_Market_Clean_Columns]
# dm=dp.loc[:,ct.SINA_Market_Clean_UP_Columns]
# dm['prev_p']=0.0
# ['name','buy','dff','percent','ratio','high','open','volume','low','couts']
#['name','buy','dff','percent','trade','high','ratio','volume','couts']
else:
dm=''
# print type(dm)
return dm
def get_dfcfw_rzrq_SHSZ2(url=ct.DFCFW_RZYE2sh):
data = {}
log.info("rzrq:%s" % (url))
dd = pd.DataFrame()
# rzdata = cct.get_url_data(url)
#shrzrq2
rzdata = cct.get_url_data_R(ct.DFCFW_RZYE2sh)
# rzdata = rzdata.replace('var HIoUWbQY=','')
# rz_dic = re.findall('{"result"[\D\d]+?}', rzdata.encode('utf8'))
rz_dic = re.findall('{"result":[\D\d]+.', rzdata.encode('utf8'))[0]
rz_dic = rz_dic.replace(';', '')
# ct.DFCFW_RZYE2sh
rzdata_dic = json.loads(rz_dic)
rzdata_list = (rzdata_dic['result']['data'])
df = pd.DataFrame(rzdata_list, columns=ct.dfcfw_rzye_columns2sh)
# ['DIM_DATE','RZYEZB', 'RQCHL10D', 'RQCHL3D', 'RQYE', 'RQMCL', 'RQYL', 'R]
# df.index = pd.to_datetime(df.index,format='%Y-%m-%d')
# rz_dic = re.findall('{"tdate"[\D\d]+?}', rzdata.encode('utf8'))
df.rename(columns={'DIM_DATE': 'tdate'}, inplace=True)
# RZYE->rzye_hs rzye_h none rzye_h none
df.rename(columns={'RZYE': 'sh'}, inplace=True)
# df.rename(columns={'rzye_h': 'sh'}, inplace=True)
# df.rename(columns={'rzye_h': 'sz'}, inplace=True)
df.tdate = df.tdate.apply(lambda x: x[:10])
df = df.set_index('tdate')
dd = df.loc[:, ['sh']]
#szrzrq
rzdata = cct.get_url_data_R(ct.DFCFW_RZYE2sz)
# rzdata = rzdata.replace('var HIoUWbQY=','')
# rz_dic = re.findall('{"result"[\D\d]+?}', rzdata.encode('utf8'))
rz_dic = re.findall('{"result":[\D\d]+.', rzdata.encode('utf8'))[0]
rz_dic = rz_dic.replace(';', '')
# ct.DFCFW_RZYE2sh
rzdata_dic = json.loads(rz_dic)
rzdata_list = (rzdata_dic['result']['data'])
df = pd.DataFrame(rzdata_list, columns=ct.dfcfw_rzye_columns2sh)
# ['DIM_DATE','RZYEZB', 'RQCHL10D', 'RQCHL3D', 'RQYE', 'RQMCL', 'RQYL', 'R]
# df.index = pd.to_datetime(df.index,format='%Y-%m-%d')
# rz_dic = re.findall('{"tdate"[\D\d]+?}', rzdata.encode('utf8'))
df.rename(columns={'DIM_DATE': 'tdate'}, inplace=True)
# RZYE->rzye_hs rzye_h none rzye_h none
df.rename(columns={'RZYE': 'sz'}, inplace=True)
# df.rename(columns={'rzye_h': 'sh'}, inplace=True)
# df.rename(columns={'rzye_h': 'sz'}, inplace=True)
df.tdate = df.tdate.apply(lambda x: x[:10])
df = df.set_index('tdate')
dd = cct.combine_dataFrame(dd, df.loc[:, ['sz']])
df = dd
df['sh'] = df['sh'].apply(lambda x: round(
(float(x) / 1000 / 1000 / 100), 2))
df['sz'] = df['sz'].apply(lambda x: round(
(float(x) / 1000 / 1000 / 100), 2))
df['all'] = map(lambda x, y: round(x + y, 1), df.sh.values, df.sz.values)
# data=get_tzrq(url,today)
# yestoday = cct.last_tddate(1)
# log.debug(today)
# beforeyesterday = cct.last_tddate(days=2)
def get_days_data(days=1, df=None):
rzrq_status = 1
# data=''
da = 0
i = 0
data2 = ''
while rzrq_status:
for x in range(days, 10):
yestoday = cct.last_tddate(x)
if yestoday in df.index:
data2 = df.loc[yestoday]
# log.info("yestoday:%s data:%s" % (yestoday, data2))
break
# print da
else:
log.error("%s:None" % (yestoday))
rzrq_status = 0
return data2
data1 = get_days_data(1, df)
data2 = get_days_data(2, df)
# data = df.loc[yestoday]
# data2 = df.loc[beforeyesterday]
# log.info("data1:%s,data2:%s", data1, data2)
if len(data2) > 0:
# print data1
data['all'] = round(data1.loc['all'], 2)
data['sh'] = round(data1.loc['sh'], 2)
data['sz'] = round(data1.loc['sz'], 2)
data['dff'] = round(data1.loc['all'] - data2.loc['all'], 2)
data['shrz'] = round(data1.loc['sh'] - data2.loc['sh'], 2)
data['szrz'] = round(data1.loc['sz'] - data2.loc['sz'], 2)
else:
log.error("df.index:%s" % (df.index.values[0]))
data['dff'] = 'error'
data['all'] = 0
data['sh'] = 0
data['sz'] = 0
data['shrz'] = 0
data['szrz'] = 0
if len(data) == 0:
log.error("Fund_f NO Url:%s" % url)
return data
