duration_date = args.start
if len(str(duration_date)) < 4:
du_date = tdd.get_duration_Index_date('999999', dl=int(duration_date))
# print duration_date
ct.PowerCountdl = duration_date
set_duration_console(du_date)
time_s = time.time()
status = False
top_all = pd.DataFrame()
lastpTDX_DF = pd.DataFrame()
elif st.startswith('w') or st.startswith('a'):
args = cct.writeArgmain().parse_args(st.split())
codew = stf.WriteCountFilter(top_temp, duration=duration_date, writecount=args.dl)
if args.code == 'a':
cct.write_to_blocknew(block_path, codew)
# sl.write_to_blocknew(all_diffpath, codew)
else:
cct.write_to_blocknew(block_path, codew, False)
# sl.write_to_blocknew(all_diffpath, codew, False)
print "wri ok:%s" % block_path
cct.sleeprandom(ct.duration_sleep_time / 2)
elif st.startswith('sh'):
while 1:
input = cct.cct_raw_input("code:")
if len(input) >= 6:
args = parser.parse_args(input.split())
if len(str(args.code)) == 6:
# print args.code
if args.code in top_temp.index.values:
lhg.get_linear_model_histogram(args.code, start=top_temp.loc[args.code, 'date'],
def get_roll_mean_all(single=True,
tdx=False,
app=True,
duration=100,
ma_250_l=1.02,
ma_250_h=1.11):
# df = tdd.search_Tdx_multi_data_duration('tdx_all_df_300', 'all_300', df=None,code_l=code_list, start=start, end=None, freq=None, col=None, index='date')
block_path = tdd.get_tdx_dir_blocknew() + '060.blk'
if not app and cct.get_file_size(
block_path) > 100 and cct.creation_date_duration(block_path) == 0:
print "It's Today Update"
return True
code_list = sina_data.Sina().market('all').index.tolist()
print "all code:", len(code_list)
if duration < 300:
h5_fname = 'tdx_all_df' + '_' + str(300)
h5_table = 'all' + '_' + str(300)
else:
h5_fname = 'tdx_all_df' + '_' + str(900)
h5_table = 'all' + '_' + str(900)
# df = tdd.search_Tdx_multi_data_duration('tdx_all_df_300', 'all_300', df=None,code_l=code_list, start='20150501', end=None, freq=None, col=None, index='date')
df = tdd.search_Tdx_multi_data_duration(h5_fname,
h5_table,
df=None,
code_l=code_list,
start=None,
end=None,
freq=None,
col=None,
index='date')
# df = tdd.search_Tdx_multi_data_duration(h5_fname, h5_table, df=None,code_l=code_list, start=None, end=None, freq=None, col=None, index='date',tail=1)
code_uniquelist = df.index.get_level_values('code').unique()
code_select = code_uniquelist[random.randint(0, len(code_uniquelist) - 1)]
print round(time.time() - time_s, 2), df.index.get_level_values(
'code').unique().shape, code_select, df.loc[code_select].shape
# df.groupby(level=[0]),df.index.get_level_values(0)
# len(df.index.get_level_values('code').unique())
# df = df[~df.index.duplicated(keep='first')]
dfs = df
def get_groupby_mean_median_close(dfs):
groupd = dfs.groupby(level=[0])
df = groupd['close'].agg({'median': 'median', 'mean': 'mean'})
df['close'] = groupd.tail(1).reset_index().set_index(['code'])['close']
# dfs['mean'] = groupd['close'].agg('mean')
# dfs['median'] = groupd['close'].agg('median')
# dfs = dfs.fillna(0)
# idx = pd.IndexSlice
# mask = ( (dfs['mean'] > dfs['median'])
# & (dfs['close'] > dfs['mean'])
# )
# df=dfs.loc[idx[mask, :]]
df = df[(df['mean'] > df['median']) & (df['close'] > df['mean'])]
# dt_low = None
# if dl == 1:
# dfs = groupd.tail(1)
# print("dfs tail1")
# else:
# dl = 30
# dindex = tdd.get_tdx_Exp_day_to_df(
# '999999', dl=dl).sort_index(ascending=False)
# dt = tdd.get_duration_price_date('999999', df=dindex)
# dt = dindex[dindex.index >= dt].index.values
# dt_low = dt[-1]
# dtlen = len(dt) if len(dt) >0 else 1
# dfs = groupd.tail(dtlen)
# print("dfs tail:%s dt:%s"%(dtlen,dt))
# dfs = get_multi_date_duration(dfs,dt[-1])
return df
groupd = dfs.groupby(level=[0])
# rollma = ['5','10','60','100','200']
# rollma = ['5','10','250']
if duration < 300:
rollma = ['10']
else:
rollma = ['10', '250']
rollma.extend([str(duration)])
# import ipdb;ipdb.set_trace()
# df.loc['300130'][:2]
# dfs['mean'] = groupd['close'].agg('mean')
# dfs['median'] = groupd['close'].agg('median')
for da in rollma:
cumdays = int(da)
dfs['ma%d' % cumdays] = groupd['close'].apply(pd.rolling_mean, cumdays)
if cumdays == 10:
dfs['upper'] = dfs['ma%d' % cumdays].apply(lambda x: round(
(1 + 11.0 / 100) * x, 1))
dfs['lower'] = dfs['ma%d' % cumdays].apply(lambda x: round(
(1 - 9.0 / 100) * x, 1))
dfs['ene'] = map(lambda x, y: round((x + y) / 2, 1), dfs['upper'],
dfs['lower'])
# df['upper'] = map(lambda x: round((1 + 11.0 / 100) * x, 1), df.ma10d)
# df['lower'] = map(lambda x: round((1 - 9.0 / 100) * x, 1), df.ma10d)
# df['ene'] = map(lambda x, y: round((x + y) / 2, 1), df.upper, df.lower)
# dfs['amount%d'%cumdays] = groupd['amount'].apply(pd.rolling_mean, cumdays)
# df.ix[df.index.levels[0]]
#df.ix[df.index[len(df.index)-1][0]] #last row
# dfs = tdd.search_Tdx_multi_data_duration(df=dfs,code_l=code_list, start='20170918', end='20170918', freq=None, col=None, index='date')
# print dfs[:1],len(dfs)
# groupd.agg({'low': 'min'})
# '''idx mask filter'''
# '''
dt_low = None
df_idx = None
if single:
dfs = groupd.tail(1)
print("dfs tail1")
else:
dl = 30
dindex = tdd.get_tdx_Exp_day_to_df('999999',
dl=dl).sort_index(ascending=False)
dt = tdd.get_duration_price_date('999999', df=dindex)
dt = dindex[dindex.index >= dt].index.values
dt_low = dt[-1]
dtlen = len(dt) if len(dt) > 0 else 1
dfs = groupd.tail(dtlen)
# import ipdb;ipdb.set_trace()
df_idx = get_groupby_mean_median_close(dfs)
print("dfs tail:%s dt:%s" % (dtlen, dt))
dfs = get_multi_date_duration(dfs, dt[-1])
# groupd2 = dfs.groupby(level=[0])
# dfs['ma%d'%cumdays] = groupd['close'].apply(pd.rolling_mean, cumdays)
# dfs.reset_index().groupby(['code'])['date'].transform('count')
single = True
dfs = dfs.fillna(0)
idx = pd.IndexSlice
# mask = (dfs[('ma%s')%(rollma[0])] > dfs[('ma%s')%(rollma[1])]) & (dfs[('ma%s')%(rollma[-1])] > 0) & (dfs[('close')] > dfs[('ma%s')%(rollma[0])]) & (dfs[('close')] > dfs[('ma%s')%(rollma[-1])])
# mask = (dfs[('ma%s')%(rollma[0])] > dfs[('ma%s')%(rollma[1])]) & (dfs[('ma%s')%(rollma[-1])] > 0) & (dfs[('close')] > dfs[('ma%s')%(rollma[1])]) & (dfs[('close')] > dfs[('ma%s')%(rollma[-1])])
# mask = (dfs[('ma%s')%(rollma[0])] > dfs[('ma%s')%(rollma[1])]) & (dfs[('ma%s')%(rollma[-1])] > 0) & (dfs[('close')] > dfs[('ma%s')%(rollma[-1])])
# mask = ( (dfs[('ma%s')%(rollma[0])] > 0) & (dfs[('ma%s')%(rollma[-1])] > 0) & (dfs[('close')] > dfs[('ma%s')%(rollma[-1])]) & (dfs[('close')] > dfs[('ma%s')%(rollma[0])]))
# mask = ( (dfs[('ma%s')%(rollma[0])] > 0) & (dfs[('ma%s')%(rollma[-1])] > 0)
# & (dfs[('close')] > dfs[('ma%s')%(rollma[-1])]*ma_250_l)
# & (dfs[('close')] < dfs[('ma%s')%(rollma[-1])]*ma_250_h)
# & (dfs[('close')] > dfs[('ma%s')%(rollma[0])]))
# & (dfs['mean'] > dfs['median'])
# & (dfs['close'] > dfs['mean'])
if len(rollma) > 1:
mask = ((dfs[('ma%s') % (rollma[0])] > 0) & (dfs[('ma%s') %
(rollma[-1])] > 0)
& (dfs[('ma%s') % (rollma[0])] > dfs[('ma%s') % (rollma[-1])])
& (dfs[('close')] > dfs[('ma%s') % (rollma[0])])
& (dfs[('close')] > dfs[('ma%s') % (rollma[-1])] * ma_250_h)
& ((dfs[('close')] > dfs['ene']) |
(dfs[('close')] > dfs['upper'])))
else:
mask = ((dfs[('ma%s') % (rollma[0])] > 0)
& (dfs[('close')] > dfs[('ma%s') % (rollma[0])])
& ((dfs[('close')] > dfs['ene']) |
(dfs[('close')] > dfs['upper'])))
# mask = ((dfs[('close')] > dfs[('ma%s')%(rollma[-1])]))
df = dfs.loc[idx[mask, :]]
df = get_multi_code_count(df)
print(df.couts[:5])
# import ipdb;ipdb.set_trace()
# df.sort_values(by='couts',ascending=0)
# groupd.first()[:2],groupd.last()[:2]
# groupd = df250.groupby(level=[0])
# '''
# groupd.transform(lambda x: x.iloc[-1])
# groupd.last()
# groupd.apply(lambda x: x.close > x.ma250)
# df.shape,df.sort_index(ascending=False)[:5]
# ?groupd.agg
# groupd = df.groupby(level=[0])
# groupd['close'].apply(pd.rolling_mean, 250, min_periods=1)
#ex:# Group df by df.platoon, then apply a rolling mean lambda function to df.casualties
# df.groupby('Platoon')['Casualties'].apply(lambda x:x.rolling(center=False,window=2).mean())
code_uniquelist = df.index.get_level_values('code').unique()
code_select = code_uniquelist[random.randint(0, len(code_uniquelist) - 1)]
if app:
print round(time.time() - time_s, 2), 's', df.index.get_level_values(
'code').unique().shape, code_select, df.loc[code_select][-1:]
if single:
# groupd = df.groupby(level=[0])
if tdx:
# block_path = tdd.get_tdx_dir_blocknew() + '060.blk'
# if cct.get_work_time():
# codew = df[df.date == cct.get_today()].index.tolist()
if dt_low is not None:
groupd2 = df.groupby(level=[0])
df = groupd2.tail(1)
df = df.reset_index().set_index('code')
# import ipdb;ipdb.set_trace()
# df = df[(df.date >= dt_low) & (df.date <= cct.get_today())]
dd = df[(df.date == dt_low)]
df = df[(df.date >= cct.last_tddate(1))]
# import ipdb;ipdb.set_trace()
print("df:%s df_idx:%s" % (len(df), len(df_idx)))
if df_idx is not None and len(df_idx) > 0:
df = df.loc[df_idx.index, :].dropna()
print("Main Down dd :%s MainUP df:%s couts std:%0.1f " %
(len(dd), len(df), df.couts.std()))
# print df.date.mode()[0]
df = df.sort_values(by='couts', ascending=1)
df = df[df.couts > df.couts.std()]
# df = df[(df.date >= df.date.mode()[0]) & (df.date <= cct.get_today())]
codew = df.index.tolist()
if app:
print round(time.time() - time_s, 2), 'groupd2', len(df)
else:
df = df.reset_index().set_index('code')
df = df[(df.date >= cct.last_tddate(days=10))
& (df.date <= cct.get_today())]
codew = df.index.tolist()
top_temp = tdd.get_sina_datadf_cnamedf(codew, df)
top_temp = top_temp[(~top_temp.index.str.contains('688'))
& (~top_temp.name.str.contains('ST'))]
codew = top_temp.index.tolist()
#clean st and 688
if app:
hdf5_wri = cct.cct_raw_input("rewrite code [Y] or append [N]:")
if hdf5_wri == 'y' or hdf5_wri == 'Y':
append_status = False
else:
append_status = True
else:
append_status = False
if len(codew) > 10:
cct.write_to_blocknew(block_path,
codew,
append_status,
doubleFile=False,
keep_last=0)
print "write:%s block_path:%s" % (len(codew), block_path)
else:
print "write error:%s block_path:%s" % (len(codew), block_path)
# df['date'] = df['date'].apply(lambda x:(x.replace('-','')))
# df['date'] = df['date'].astype(int)
# print df.loc[code_select].T,df.shape
MultiIndex = False
else:
MultiIndex = True
h5a.write_hdf_db('all300',
df,
table='roll200',
index=False,
baseCount=500,
append=False,
MultiIndex=MultiIndex)
return df
print("timeSearch:%s" % (time.time() - time_s))
print("gold:%s %s :%s" % (mk, len(gold_f), gold_f.iloc[:1, 0:2]))
print("down:%s %s :%s" % (mk, len(down_f), down_f.iloc[:1, 0:2]))
return gold_f, down_f
# result = cct.to_mp_run(get_market_status, ['sh', 'sz'])
# df = pd.DataFrame()
# for x in result:
# df = df.append(x)
# print len(df)
# df.iloc[:, 0:2].to_csv('stock-Line-%s.csv'%today, encoding='utf8')
type = 'l'
start = '2016-01-18'
# start = None
market = 'sh'
df_u, df_d = get_market_status(market, type, start)
if not type == 'l':
if len(df_u) > 0:
print("Type: %s wri gold:%s %s" %
(type, len(df_u.index.tolist()), block_path))
df_u.iloc[:, 0:2].to_csv('stock-Line-%s.csv' % today,
encoding='utf8')
cct.write_to_blocknew(block_path, df_u.index.tolist(), False)
else:
if len(df_u) > 0:
print("Type: %s wri down:%s %s" %
(type, len(df_d.index.tolist()), block_path_d))
df_d.iloc[:, 0:2].to_csv('stock-Line-down-%s.csv' % today,
encoding='utf8')
cct.write_to_blocknew(block_path_d, df_d.index.tolist(), False)
and cct.get_now_time_int() < 2400):
while 1:
if cct.get_now_time_int(
) > 1502 and cct.get_now_time_int() < 1510:
print ".",
cct.sleep(60)
elif cct.get_now_time_int() < 2400:
print ".",
print "write dm to file"
if cct.get_work_day_status():
tdd.Write_market_all_day_mp('all')
top_temp = cct.GlobalValues().getkey('top_max')
codew = stf.WriteCountFilter(top_temp,
writecount='all')
cct.write_to_blocknew(block_path,
codew,
append=False,
doubleFile=False)
break
else:
print "."
raise KeyboardInterrupt("Stop Time")
# st = cct.cct_raw_input("status:[go(g),clear(c),quit(q,e)]:")
# if len(st) == 0:
# status = False
# elif st.lower() == 'g' or st.lower() == 'go':
# status = True
# num_input = ''
# ave = None
# code = ''
# elif len(st) == 6:
