duration_date = args.start if len(str(duration_date)) < 4: du_date = tdd.get_duration_Index_date('999999', dl=int(duration_date)) # print duration_date ct.PowerCountdl = duration_date set_duration_console(du_date) time_s = time.time() status = False top_all = pd.DataFrame() lastpTDX_DF = pd.DataFrame() elif st.startswith('w') or st.startswith('a'): args = cct.writeArgmain().parse_args(st.split()) codew = stf.WriteCountFilter(top_temp, duration=duration_date, writecount=args.dl) if args.code == 'a': cct.write_to_blocknew(block_path, codew) # sl.write_to_blocknew(all_diffpath, codew) else: cct.write_to_blocknew(block_path, codew, False) # sl.write_to_blocknew(all_diffpath, codew, False) print "wri ok:%s" % block_path cct.sleeprandom(ct.duration_sleep_time / 2) elif st.startswith('sh'): while 1: input = cct.cct_raw_input("code:") if len(input) >= 6: args = parser.parse_args(input.split()) if len(str(args.code)) == 6: # print args.code if args.code in top_temp.index.values: lhg.get_linear_model_histogram(args.code, start=top_temp.loc[args.code, 'date'],
def get_roll_mean_all(single=True, tdx=False, app=True, duration=100, ma_250_l=1.02, ma_250_h=1.11): # df = tdd.search_Tdx_multi_data_duration('tdx_all_df_300', 'all_300', df=None,code_l=code_list, start=start, end=None, freq=None, col=None, index='date') block_path = tdd.get_tdx_dir_blocknew() + '060.blk' if not app and cct.get_file_size( block_path) > 100 and cct.creation_date_duration(block_path) == 0: print "It's Today Update" return True code_list = sina_data.Sina().market('all').index.tolist() print "all code:", len(code_list) if duration < 300: h5_fname = 'tdx_all_df' + '_' + str(300) h5_table = 'all' + '_' + str(300) else: h5_fname = 'tdx_all_df' + '_' + str(900) h5_table = 'all' + '_' + str(900) # df = tdd.search_Tdx_multi_data_duration('tdx_all_df_300', 'all_300', df=None,code_l=code_list, start='20150501', end=None, freq=None, col=None, index='date') df = tdd.search_Tdx_multi_data_duration(h5_fname, h5_table, df=None, code_l=code_list, start=None, end=None, freq=None, col=None, index='date') # df = tdd.search_Tdx_multi_data_duration(h5_fname, h5_table, df=None,code_l=code_list, start=None, end=None, freq=None, col=None, index='date',tail=1) code_uniquelist = df.index.get_level_values('code').unique() code_select = code_uniquelist[random.randint(0, len(code_uniquelist) - 1)] print round(time.time() - time_s, 2), df.index.get_level_values( 'code').unique().shape, code_select, df.loc[code_select].shape # df.groupby(level=[0]),df.index.get_level_values(0) # len(df.index.get_level_values('code').unique()) # df = df[~df.index.duplicated(keep='first')] dfs = df def get_groupby_mean_median_close(dfs): groupd = dfs.groupby(level=[0]) df = groupd['close'].agg({'median': 'median', 'mean': 'mean'}) df['close'] = groupd.tail(1).reset_index().set_index(['code'])['close'] # dfs['mean'] = groupd['close'].agg('mean') # dfs['median'] = groupd['close'].agg('median') # dfs = dfs.fillna(0) # idx = pd.IndexSlice # mask = ( (dfs['mean'] > dfs['median']) # & (dfs['close'] > dfs['mean']) # ) # df=dfs.loc[idx[mask, :]] df = df[(df['mean'] > df['median']) & (df['close'] > df['mean'])] # dt_low = None # if dl == 1: # dfs = groupd.tail(1) # print("dfs tail1") # else: # dl = 30 # dindex = tdd.get_tdx_Exp_day_to_df( # '999999', dl=dl).sort_index(ascending=False) # dt = tdd.get_duration_price_date('999999', df=dindex) # dt = dindex[dindex.index >= dt].index.values # dt_low = dt[-1] # dtlen = len(dt) if len(dt) >0 else 1 # dfs = groupd.tail(dtlen) # print("dfs tail:%s dt:%s"%(dtlen,dt)) # dfs = get_multi_date_duration(dfs,dt[-1]) return df groupd = dfs.groupby(level=[0]) # rollma = ['5','10','60','100','200'] # rollma = ['5','10','250'] if duration < 300: rollma = ['10'] else: rollma = ['10', '250'] rollma.extend([str(duration)]) # import ipdb;ipdb.set_trace() # df.loc['300130'][:2] # dfs['mean'] = groupd['close'].agg('mean') # dfs['median'] = groupd['close'].agg('median') for da in rollma: cumdays = int(da) dfs['ma%d' % cumdays] = groupd['close'].apply(pd.rolling_mean, cumdays) if cumdays == 10: dfs['upper'] = dfs['ma%d' % cumdays].apply(lambda x: round( (1 + 11.0 / 100) * x, 1)) dfs['lower'] = dfs['ma%d' % cumdays].apply(lambda x: round( (1 - 9.0 / 100) * x, 1)) dfs['ene'] = map(lambda x, y: round((x + y) / 2, 1), dfs['upper'], dfs['lower']) # df['upper'] = map(lambda x: round((1 + 11.0 / 100) * x, 1), df.ma10d) # df['lower'] = map(lambda x: round((1 - 9.0 / 100) * x, 1), df.ma10d) # df['ene'] = map(lambda x, y: round((x + y) / 2, 1), df.upper, df.lower) # dfs['amount%d'%cumdays] = groupd['amount'].apply(pd.rolling_mean, cumdays) # df.ix[df.index.levels[0]] #df.ix[df.index[len(df.index)-1][0]] #last row # dfs = tdd.search_Tdx_multi_data_duration(df=dfs,code_l=code_list, start='20170918', end='20170918', freq=None, col=None, index='date') # print dfs[:1],len(dfs) # groupd.agg({'low': 'min'}) # '''idx mask filter''' # ''' dt_low = None df_idx = None if single: dfs = groupd.tail(1) print("dfs tail1") else: dl = 30 dindex = tdd.get_tdx_Exp_day_to_df('999999', dl=dl).sort_index(ascending=False) dt = tdd.get_duration_price_date('999999', df=dindex) dt = dindex[dindex.index >= dt].index.values dt_low = dt[-1] dtlen = len(dt) if len(dt) > 0 else 1 dfs = groupd.tail(dtlen) # import ipdb;ipdb.set_trace() df_idx = get_groupby_mean_median_close(dfs) print("dfs tail:%s dt:%s" % (dtlen, dt)) dfs = get_multi_date_duration(dfs, dt[-1]) # groupd2 = dfs.groupby(level=[0]) # dfs['ma%d'%cumdays] = groupd['close'].apply(pd.rolling_mean, cumdays) # dfs.reset_index().groupby(['code'])['date'].transform('count') single = True dfs = dfs.fillna(0) idx = pd.IndexSlice # mask = (dfs[('ma%s')%(rollma[0])] > dfs[('ma%s')%(rollma[1])]) & (dfs[('ma%s')%(rollma[-1])] > 0) & (dfs[('close')] > dfs[('ma%s')%(rollma[0])]) & (dfs[('close')] > dfs[('ma%s')%(rollma[-1])]) # mask = (dfs[('ma%s')%(rollma[0])] > dfs[('ma%s')%(rollma[1])]) & (dfs[('ma%s')%(rollma[-1])] > 0) & (dfs[('close')] > dfs[('ma%s')%(rollma[1])]) & (dfs[('close')] > dfs[('ma%s')%(rollma[-1])]) # mask = (dfs[('ma%s')%(rollma[0])] > dfs[('ma%s')%(rollma[1])]) & (dfs[('ma%s')%(rollma[-1])] > 0) & (dfs[('close')] > dfs[('ma%s')%(rollma[-1])]) # mask = ( (dfs[('ma%s')%(rollma[0])] > 0) & (dfs[('ma%s')%(rollma[-1])] > 0) & (dfs[('close')] > dfs[('ma%s')%(rollma[-1])]) & (dfs[('close')] > dfs[('ma%s')%(rollma[0])])) # mask = ( (dfs[('ma%s')%(rollma[0])] > 0) & (dfs[('ma%s')%(rollma[-1])] > 0) # & (dfs[('close')] > dfs[('ma%s')%(rollma[-1])]*ma_250_l) # & (dfs[('close')] < dfs[('ma%s')%(rollma[-1])]*ma_250_h) # & (dfs[('close')] > dfs[('ma%s')%(rollma[0])])) # & (dfs['mean'] > dfs['median']) # & (dfs['close'] > dfs['mean']) if len(rollma) > 1: mask = ((dfs[('ma%s') % (rollma[0])] > 0) & (dfs[('ma%s') % (rollma[-1])] > 0) & (dfs[('ma%s') % (rollma[0])] > dfs[('ma%s') % (rollma[-1])]) & (dfs[('close')] > dfs[('ma%s') % (rollma[0])]) & (dfs[('close')] > dfs[('ma%s') % (rollma[-1])] * ma_250_h) & ((dfs[('close')] > dfs['ene']) | (dfs[('close')] > dfs['upper']))) else: mask = ((dfs[('ma%s') % (rollma[0])] > 0) & (dfs[('close')] > dfs[('ma%s') % (rollma[0])]) & ((dfs[('close')] > dfs['ene']) | (dfs[('close')] > dfs['upper']))) # mask = ((dfs[('close')] > dfs[('ma%s')%(rollma[-1])])) df = dfs.loc[idx[mask, :]] df = get_multi_code_count(df) print(df.couts[:5]) # import ipdb;ipdb.set_trace() # df.sort_values(by='couts',ascending=0) # groupd.first()[:2],groupd.last()[:2] # groupd = df250.groupby(level=[0]) # ''' # groupd.transform(lambda x: x.iloc[-1]) # groupd.last() # groupd.apply(lambda x: x.close > x.ma250) # df.shape,df.sort_index(ascending=False)[:5] # ?groupd.agg # groupd = df.groupby(level=[0]) # groupd['close'].apply(pd.rolling_mean, 250, min_periods=1) #ex:# Group df by df.platoon, then apply a rolling mean lambda function to df.casualties # df.groupby('Platoon')['Casualties'].apply(lambda x:x.rolling(center=False,window=2).mean()) code_uniquelist = df.index.get_level_values('code').unique() code_select = code_uniquelist[random.randint(0, len(code_uniquelist) - 1)] if app: print round(time.time() - time_s, 2), 's', df.index.get_level_values( 'code').unique().shape, code_select, df.loc[code_select][-1:] if single: # groupd = df.groupby(level=[0]) if tdx: # block_path = tdd.get_tdx_dir_blocknew() + '060.blk' # if cct.get_work_time(): # codew = df[df.date == cct.get_today()].index.tolist() if dt_low is not None: groupd2 = df.groupby(level=[0]) df = groupd2.tail(1) df = df.reset_index().set_index('code') # import ipdb;ipdb.set_trace() # df = df[(df.date >= dt_low) & (df.date <= cct.get_today())] dd = df[(df.date == dt_low)] df = df[(df.date >= cct.last_tddate(1))] # import ipdb;ipdb.set_trace() print("df:%s df_idx:%s" % (len(df), len(df_idx))) if df_idx is not None and len(df_idx) > 0: df = df.loc[df_idx.index, :].dropna() print("Main Down dd :%s MainUP df:%s couts std:%0.1f " % (len(dd), len(df), df.couts.std())) # print df.date.mode()[0] df = df.sort_values(by='couts', ascending=1) df = df[df.couts > df.couts.std()] # df = df[(df.date >= df.date.mode()[0]) & (df.date <= cct.get_today())] codew = df.index.tolist() if app: print round(time.time() - time_s, 2), 'groupd2', len(df) else: df = df.reset_index().set_index('code') df = df[(df.date >= cct.last_tddate(days=10)) & (df.date <= cct.get_today())] codew = df.index.tolist() top_temp = tdd.get_sina_datadf_cnamedf(codew, df) top_temp = top_temp[(~top_temp.index.str.contains('688')) & (~top_temp.name.str.contains('ST'))] codew = top_temp.index.tolist() #clean st and 688 if app: hdf5_wri = cct.cct_raw_input("rewrite code [Y] or append [N]:") if hdf5_wri == 'y' or hdf5_wri == 'Y': append_status = False else: append_status = True else: append_status = False if len(codew) > 10: cct.write_to_blocknew(block_path, codew, append_status, doubleFile=False, keep_last=0) print "write:%s block_path:%s" % (len(codew), block_path) else: print "write error:%s block_path:%s" % (len(codew), block_path) # df['date'] = df['date'].apply(lambda x:(x.replace('-',''))) # df['date'] = df['date'].astype(int) # print df.loc[code_select].T,df.shape MultiIndex = False else: MultiIndex = True h5a.write_hdf_db('all300', df, table='roll200', index=False, baseCount=500, append=False, MultiIndex=MultiIndex) return df
print("timeSearch:%s" % (time.time() - time_s)) print("gold:%s %s :%s" % (mk, len(gold_f), gold_f.iloc[:1, 0:2])) print("down:%s %s :%s" % (mk, len(down_f), down_f.iloc[:1, 0:2])) return gold_f, down_f # result = cct.to_mp_run(get_market_status, ['sh', 'sz']) # df = pd.DataFrame() # for x in result: # df = df.append(x) # print len(df) # df.iloc[:, 0:2].to_csv('stock-Line-%s.csv'%today, encoding='utf8') type = 'l' start = '2016-01-18' # start = None market = 'sh' df_u, df_d = get_market_status(market, type, start) if not type == 'l': if len(df_u) > 0: print("Type: %s wri gold:%s %s" % (type, len(df_u.index.tolist()), block_path)) df_u.iloc[:, 0:2].to_csv('stock-Line-%s.csv' % today, encoding='utf8') cct.write_to_blocknew(block_path, df_u.index.tolist(), False) else: if len(df_u) > 0: print("Type: %s wri down:%s %s" % (type, len(df_d.index.tolist()), block_path_d)) df_d.iloc[:, 0:2].to_csv('stock-Line-down-%s.csv' % today, encoding='utf8') cct.write_to_blocknew(block_path_d, df_d.index.tolist(), False)
and cct.get_now_time_int() < 2400): while 1: if cct.get_now_time_int( ) > 1502 and cct.get_now_time_int() < 1510: print ".", cct.sleep(60) elif cct.get_now_time_int() < 2400: print ".", print "write dm to file" if cct.get_work_day_status(): tdd.Write_market_all_day_mp('all') top_temp = cct.GlobalValues().getkey('top_max') codew = stf.WriteCountFilter(top_temp, writecount='all') cct.write_to_blocknew(block_path, codew, append=False, doubleFile=False) break else: print "." raise KeyboardInterrupt("Stop Time") # st = cct.cct_raw_input("status:[go(g),clear(c),quit(q,e)]:") # if len(st) == 0: # status = False # elif st.lower() == 'g' or st.lower() == 'go': # status = True # num_input = '' # ave = None # code = '' # elif len(st) == 6: