def get_code_search_loop(num_input, code='', timed=60, dayl='10', ave=None): # if not status: # if cct.get_work_time(): if code == num_input: get_today_tick_ave(code, ave) else: ave = get_multiday_ave_compare(num_input, dayl) time.sleep(timed) return ave
# top_now = tdd.getSinaAlldf(market='���', filename='mnbk',vol=ct.json_countVol, vtype=ct.json_countType) # top_dif = top_now # top_now.to_hdf("testhdf5", 'marketDD', format='table', complevel=9) now_count = len(top_now) radio_t = cct.get_work_time_ratio() # top_now = top_now[top_now.buy > 0] time_d = time.time() if time_d - time_s > delay_time: status_change = True time_s = time.time() top_all = pd.DataFrame() else: status_change = False # print ("Buy>0:%s" % len(top_now[top_now['buy'] > 0])), if len(top_now) > 10 or cct.get_work_time(): time_Rt = time.time() if len(top_all) == 0 and len(lastpTDX_DF) == 0: cct.get_terminal_Position(position=sys.argv[0]) time_Rt = time.time() top_all, lastpTDX_DF = tdd.get_append_lastp_to_df( top_now, lastpTDX_DF=None, dl=duration_date, end=end_date, ptype=ptype, filter=filter, power=ct.lastPower, lastp=False, newdays=newdays, resample=resample) log.debug("len:%s" % (len(top_all))) # codelist = top_all.index.tolist() # log.info('toTDXlist:%s' % len(codelist)) # # tdxdata = tdd.get_tdx_all_day_LastDF(codelist,dt=duration_date,ptype=ptype) # # print "duration_date:%s ptype=%s filter:%s"%(duration_date, ptype,filter) # # tdxdata = tdd.get_tdx_exp_all_LastDF(codelist, dt=duration_date, end=end_date,ptype=ptype,filter=filter) # power=True # log.debug("TdxLastP: %s %s" % (len(tdxdata), tdxdata.columns.values))
def format_response_data(self, index=False): stocks_detail = ''.join(self.stock_data) # print stocks_detail result = self.grep_stock_detail.finditer(stocks_detail) # stock_dict = dict() list_s = [] for stock_match_object in result: stock = stock_match_object.groups() # print stock # print stock # fn=(lambda x:x) # list.append(map(fn,stock)) # df = pd.DataFrame(list,columns=ct.SINA_Total_Columns) # list_s.append({'code'}) list_s.append({ 'code': stock[0], 'name': stock[1], 'open': float(stock[2]), 'close': float(stock[3]), 'now': float(stock[4]), 'high': float(stock[5]), 'low': float(stock[6]), 'buy': float(stock[7]), 'sell': float(stock[8]), 'volume': int(stock[9]), 'turnover': float(stock[10]), # 'amount': float(stock[10]), 'b1_v': int(stock[11]), 'b1': float(stock[12]), 'b2_v': int(stock[13]), 'b2': float(stock[14]), 'b3_v': int(stock[15]), 'b3': float(stock[16]), 'b4_v': int(stock[17]), 'b4': float(stock[18]), 'b5_v': int(stock[19]), 'b5': float(stock[20]), 'a1_v': int(stock[21]), 'a1': float(stock[22]), 'a2_v': int(stock[23]), 'a2': float(stock[24]), 'a3_v': int(stock[25]), 'a3': float(stock[26]), 'a4_v': int(stock[27]), 'a4': float(stock[28]), 'a5_v': int(stock[29]), 'a5': float(stock[30]), 'dt': (stock[31]), 'ticktime': (stock[32]) }) # print list_s # df = pd.DataFrame.from_dict(stock_dict,columns=ct.SINA_Total_Columns) df = pd.DataFrame(list_s, columns=ct.SINA_Total_Columns) # if self.index_status and cct.get_work_time(): # if self.index_status: # if cct.get_work_time() or (cct.get_now_time_int() > 915) : # df = df.drop('close', axis=1) df.rename(columns={'close': 'llastp'}, inplace=True) df['b1_vv'] = df['b1_v'] if (cct.get_now_time_int() > 915 and cct.get_now_time_int() < 926): # df.rename(columns={'buy': 'close'}, inplace=True) df['close'] = df['buy'] df['low'] = df['buy'] df['volume'] = ((df['b1_v'] + df['b2_v'])).map(lambda x: x) # df['b1_v'] = ((df['b1_v'] + df['b2_v']) / 100 / 10000).map(lambda x: round(x, 1) + 0.01) # df['b1_v'] = ((df['b1_v']) / 100 / 10000).map(lambda x: round(x, 1) + 0.01) # df['b1_vv'] = map(lambda x: round(x / 100 / 10000, 1) + 0.01, df['b1_v']) elif (cct.get_now_time_int() > 830 and cct.get_now_time_int() <= 915): # df.rename(columns={'buy': 'close'}, inplace=True) df['buy'] = df['llastp'] df['close'] = df['buy'] df['low'] = df['buy'] # df['b1_v'] = ((df['b1_v']) / df['volume'] * 100).map(lambda x: round(x, 1)) else: # df['b1_v'] = ((df['b1_v']) / df['volume'] * 100).map(lambda x: round(x, 1)) # df.rename(columns={'now': 'close'}, inplace=True) df['close'] = df['now'] df['nvol'] = df['volume'] df = df.drop_duplicates('code') # df = df.loc[:, ct.SINA_Total_Columns_Clean] # df = df.loc[:, ct.SINA_Total_Columns] # df.rename(columns={'turnover': 'amount'}, inplace=True) df = df.fillna(0) # df = df.sort_values(by='code', ascending=0) df = df.set_index('code') if index: df.index = map( (lambda x: str(1000000 - int(x)) if x.startswith('0') else x), df.index) # print ("Market-df:%s %s time: %s" % ( # cct.get_now_time())) log.info("hdf:all%s %s" % (len(df), len(self.stock_codes))) dd = df.copy() h5_fname = 'sina_MultiIndex_data' h5_table = 'all' + '_' + str(ct.sina_limit_time) if cct.get_now_time_int() >= 925 and not index and len( df) > 3000 and cct.get_work_time(): time_s = time.time() df.index = df.index.astype(str) df.ticktime = df.ticktime.astype(str) # df.ticktime = map(lambda x: int(x.replace(':', '')), df.ticktime) df.ticktime = map(lambda x, y: str(x) + ' ' + str(y), df.dt, df.ticktime) df.ticktime = pd.to_datetime(df.ticktime, format='%Y-%m-%d %H:%M:%S') # df = df.loc[:, ['open', 'high', 'low', 'close', 'llastp', 'volume', 'ticktime']] df = df.loc[:, [ 'close', 'high', 'low', 'llastp', 'volume', 'ticktime' ]] if 'code' not in df.columns: df = df.reset_index() if 'dt' in df.columns: df = df.drop(['dt'], axis=1) # df.dt = df.dt.astype(str) if 'name' in df.columns: # df.name = df.name.astype(str) df = df.drop(['name'], axis=1) df = df.set_index(['code', 'ticktime']) h5a.write_hdf_db(h5_fname, df, table=h5_table, index=False, baseCount=500, append=False, MultiIndex=True) log.info("hdf5 class all :%s time:%0.2f" % (len(df), time.time() - time_s)) if 'nlow' not in df.columns or 'nhigh' not in df.columns or ( cct.get_work_time() and 925 < cct.get_now_time_int() < 1500): # if 'nlow' not in df.columns or 'nhigh' not in df.columns or cct.get_work_time(): h5 = h5a.load_hdf_db(h5_fname, h5_table, timelimit=False) time_s = time.time() if cct.get_work_time() and cct.get_now_time_int() <= 1000: run_col = ['low', 'high', 'close'] startime = '09:25:00' # endtime = '10:00:00' endtime = '09:45:00' dd = self.get_col_agg_df(h5, dd, run_col, all_func, startime, endtime) startime = '09:25:00' # endtime = '10:00:00' endtime = '10:00:00' run_col = {'close': 'std'} dd = self.get_col_agg_df(h5, dd, run_col, run_col, startime, endtime) dd.rename(columns={'std': 'nstd'}, inplace=True) if dd is not None and len( dd ) > 0 and 'nclose' in dd.columns and 'nstd' in dd.columns: for co in ['nclose', 'nstd']: dd[co] = dd[co].apply(lambda x: round(x, 2)) else: run_col = ['low', 'high'] startime = '09:25:00' # endtime = '10:00:00' endtime = '09:40:00' dd = self.get_col_agg_df(h5, dd, run_col, all_func, startime, endtime) startime = '09:25:00' endtime = '15:01:00' run_col = ['close'] # h5 = cct.get_limit_multiIndex_Group(h5, freq='15T', col=run_col,start=startime, end=endtime) dd = self.get_col_agg_df(h5, dd, run_col, all_func, startime, endtime) run_col = {'close': 'std'} dd = self.get_col_agg_df(h5, dd, run_col, run_col, startime, endtime) dd.rename(columns={'std': 'nstd'}, inplace=True) if dd is not None and len( dd ) > 0 and 'nclose' in dd.columns and 'nstd' in dd.columns: for co in ['nclose', 'nstd']: dd[co] = dd[co].apply(lambda x: round(x, 2)) log.info("agg_df_all_time:%0.2f" % (time.time() - time_s)) # top_temp[:1][['high','nhigh','low','nlow','close','nclose','llastp']] h5a.write_hdf_db(self.hdf_name, dd, self.table, index=index) log.info("wr end:%0.2f" % (time.time() - self.start_t)) return dd
def all(self): self.stockcode = StockCode() self.stock_code_path = self.stockcode.stock_code_path self.stock_codes = self.stockcode.get_stock_codes() self.load_stock_codes() # print "stocks:",len(self.stock_codes) self.stock_codes = [ elem for elem in self.stock_codes if elem.startswith(('6', '30', '00')) ] time_s = time.time() h5 = h5a.load_hdf_db(self.hdf_name, self.table, code_l=self.stock_codes, limit_time=self.sina_limit_time) log.info("h5a stocksTime:%0.2f" % (time.time() - time_s)) if h5 is not None and len(h5) > 0: o_time = h5[h5.timel <> 0].timel if len(o_time) > 0: o_time = o_time[0] l_time = time.time() - o_time sina_limit_time = ct.sina_limit_time sina_time_status = (cct.get_work_day_status() and 915 < cct.get_now_time_int() < 926) # return_hdf_status = not cct.get_work_day_status() or (cct.get_work_day_status() and (cct.get_work_time() and l_time < sina_limit_time)) return_hdf_status = not cct.get_work_time() or ( cct.get_work_time() and l_time < sina_limit_time) log.info("915:%s sina_time:%0.2f limit:%s" % (sina_time_status, l_time, sina_limit_time)) if sina_time_status and l_time < 6: log.info("open 915 hdf ok:%s" % (len(h5))) return h5 elif not sina_time_status and return_hdf_status: log.info("return hdf5 data:%s" % (len(h5))) return h5 else: log.info("no return hdf5:%s" % (len(h5))) else: log.info("no return hdf5:%s" % (len(h5) if h5 is not None else 'None')) # self.stock_with_exchange_list = list( # map(lambda stock_code: ('sh%s' if stock_code.startswith(('5', '6', '9')) else 'sz%s') % stock_code, # self.stock_codes)) self.stock_with_exchange_list = list( map( lambda stock_code: ('sh%s' if stock_code.startswith( ('6')) else 'sz%s') % stock_code, self.stock_codes)) self.stock_list = [] self.request_num = len(self.stock_with_exchange_list) // self.max_num for range_start in range(self.request_num): num_start = self.max_num * range_start num_end = self.max_num * (range_start + 1) request_list = ','.join( self.stock_with_exchange_list[num_start:num_end]) self.stock_list.append(request_list) # print len(self.stock_with_exchange_list), num_end if len(self.stock_with_exchange_list) > num_end: request_list = ','.join(self.stock_with_exchange_list[num_end:]) self.stock_list.append(request_list) self.request_num += 1 # a = 0 # for x in range(self.request_num): # print x # i = len(self.stock_list[x].split(',')) # print i # a += i # print a log.debug('all:%s' % len(self.stock_list)) # log.error('all:%s req:%s' % # (len(self.stock_list), len(self.stock_list))) return self.get_stock_data()
clean=True) log.info('agg_df_Row:%.2f h5:%s endtime:%s' % ((time.time() - time_n), len(h5), endtime)) return dd h5_fname = 'sina_MultiIndex_data' # h5_fname = 'sina_multi_index' h5_table = 'all_10' time_s = time.time() h5 = h5a.load_hdf_db(h5_fname, table=h5_table, code_l=None, timelimit=False, dratio_limit=0.12) print 'h5:', len(h5) if cct.get_work_time() and cct.get_now_time_int() <= 1000: run_col = ['low', 'high', 'close'] startime = None # endtime = '10:00:00' endtime = '09:45:00' dd = get_col_agg_df_Test(h5, dd, run_col, all_func, startime, endtime) else: run_col = ['low', 'high'] startime = None # endtime = '10:00:00' endtime = '09:45:00' dd = get_col_agg_df_Test(h5, dd, run_col, all_func, startime, endtime) startime = '09:30:00' endtime = '15:01:00' # run_col = ['close']
def load_hdf_db(fname, table='all', code_l=None, timelimit=True, index=False, limit_time=ct.h5_limit_time, dratio_limit=ct.dratio_limit, MultiIndex=False): """[summary] [load hdf ] Parameters ---------- fname : {[type]} [description] table : {str}, optional [description] (the default is 'all', which [default_description]) code_l : {[type]}, optional [description] (the default is None, which [default_description]) timelimit : {bool}, optional [description] (the default is True, which [default_description]) index : {bool}, optional [description] (the default is False, which [default_description]) limit_time : {[type]}, optional [description] (the default is ct.h5_limit_time, which [default_description]) dratio_limit : {[type]}, optional [description] (the default is ct.dratio_limit, which [default_description]) MultiIndex : {bool}, optional [description] (the default is False, which [default_description]) Returns ------- [dataframe] [description] """ time_t = time.time() global RAMDISK_KEY, INIT_LOG_Error if not RAMDISK_KEY < 1: return None df = None dd = None table = table.replace('/', '') if table.find('/') >= 0 else table if code_l is not None: if table is not None: with SafeHDFStore(fname) as store: if store is not None: try: if '/' + table in list(store.keys()): dd = store[table] except AttributeError as e: store.close() os.remove(store.filename) log.error("AttributeError:%s %s" % (fname, e)) log.error("Remove File:%s" % (fname)) except Exception as e: print("Exception:%s name:%s" % (fname, e)) else: pass finally: pass if dd is not None and len(dd) > 0: if not MultiIndex: if index: code_l = list( map((lambda x: str(1000000 - int(x)) if x.startswith('0') else x), code_l)) dif_co = list(set(dd.index) & set(code_l)) if len(code_l) > 0: dratio=(float(len(code_l)) - float(len(dif_co))) / \ float(len(code_l)) else: dratio = 0 # if dratio < 0.1 or len(dd) > 3100: log.info("find all:%s :%s %0.2f" % (len(code_l), len(code_l) - len(dif_co), dratio)) if timelimit and len(dd) > 0: dd = dd.loc[dif_co] o_time = dd[dd.timel != 0].timel.tolist() # if fname == 'powerCompute': # o_time = sorted(set(o_time),reverse=True) o_time = sorted(set(o_time), reverse=False) o_time = [time.time() - t_x for t_x in o_time] if len(dd) > 0: # if len(dd) > 0 and (not cct.get_work_time() or len(o_time) <= ct.h5_time_l_count): l_time = np.mean(o_time) return_hdf_status = (not cct.get_work_time()) or ( cct.get_work_time() and l_time < limit_time) # return_hdf_status = l_time < limit_time # print return_hdf_status,l_time,limit_time if return_hdf_status: # df=dd df = dd.loc[dif_co] log.info( "return hdf: %s timel:%s l_t:%s hdf ok:%s" % (fname, len(o_time), l_time, len(df))) else: log.error( "%s %s o_time:%s %s" % (fname, table, len(o_time), [time.time() - t_x for t_x in o_time[:3]])) log.info('fname:%s l_time:%s' % (fname, [time.time() - t_x for t_x in o_time])) else: df = dd.loc[dif_co] if dratio > dratio_limit: if len( code_l ) > ct.h5_time_l_count * 10 and INIT_LOG_Error < 5: # INIT_LOG_Error += 1 log.info( "fn:%s cl:%s h5:%s don't find:%s dra:%0.2f log_err:%s" % (fname, len(code_l), len(dd), len(code_l) - len(dif_co), dratio, INIT_LOG_Error)) return None # if dratio < dratio_limit: # log.info("find all:%s :%s %0.2f" % # (len(code_l), len(code_l) - len(dif_co), dratio)) # if timelimit and len(dd) > 0: # dd=dd.loc[dif_co] # o_time=dd[dd.timel <> 0].timel.tolist() # # if fname == 'powerCompute': # # o_time = sorted(set(o_time),reverse=True) # o_time=sorted(set(o_time), reverse=False) # o_time=[time.time() - t_x for t_x in o_time] # if len(dd) > 0: # # if len(dd) > 0 and (not cct.get_work_time() or len(o_time) <= ct.h5_time_l_count): # l_time=np.mean(o_time) # return_hdf_status=(not cct.get_work_time()) or ( # cct.get_work_time() and l_time < limit_time) # # return_hdf_status = l_time < limit_time # # print return_hdf_status,l_time,limit_time # if return_hdf_status: # # df=dd # dd.loc[dif_co] # log.info("return hdf: %s timel:%s l_t:%s hdf ok:%s" % ( # fname, len(o_time), l_time, len(df))) # else: # log.error("%s %s o_time:%s %s" % (fname, table, len( # o_time), [time.time() - t_x for t_x in o_time[:3]])) # log.info('fname:%s l_time:%s' % # (fname, [time.time() - t_x for t_x in o_time])) # else: # df=dd.loc[dif_co] # else: # if len(code_l) > ct.h5_time_l_count * 10 and INIT_LOG_Error < 5: # # INIT_LOG_Error += 1 # log.error("fn:%s cl:%s h5:%s don't find:%s dra:%0.2f log_err:%s" % ( # fname, len(code_l), len(dd), len(code_l) - len(dif_co), dratio, INIT_LOG_Error)) else: df = dd.loc[dd.index.isin(code_l, level='code')] else: log.error("%s is not find %s" % (fname, table)) else: # h5 = get_hdf5_file(fname,wr_mode='r') if table is not None: with SafeHDFStore(fname) as store: # if store is not None: # if '/' + table in store.keys(): # try: # dd=store[table] # except Exception as e: # print ("%s fname:%s"%(e,fname)) # cct.sleep(ct.sleep_time) if store is not None: try: if '/' + table in list(store.keys()): dd = store[table] except AttributeError as e: store.close() os.remove(store.filename) log.error("AttributeError:%s %s" % (fname, e)) log.error("Remove File:%s" % (fname)) except Exception as e: log.error("Exception:%s %s" % (fname, e)) print("Exception:%s name:%s" % (fname, e)) else: pass finally: pass if dd is not None and len(dd) > 0: if timelimit: if dd is not None and len(dd) > 0: o_time = dd[dd.timel != 0].timel.tolist() o_time = sorted(set(o_time)) o_time = [time.time() - t_x for t_x in o_time] if len(o_time) > 0: l_time = np.mean(o_time) # l_time = time.time() - l_time # return_hdf_status = not cct.get_work_day_status() or not # cct.get_work_time() or (cct.get_work_day_status() and # (cct.get_work_time() and l_time < limit_time)) return_hdf_status = not cct.get_work_time() or ( cct.get_work_time() and l_time < limit_time) log.info("return_hdf_status:%s time:%0.2f" % (return_hdf_status, l_time)) if return_hdf_status: log.info("return hdf5 data:%s o_time:%s" % (len(dd), len(o_time))) df = dd else: log.info("no return time hdf5:%s" % (len(dd))) log.info('fname:%s l_time:%s' % (fname, [time.time() - t_x for t_x in o_time])) else: df = dd else: log.error("%s is not find %s" % (fname, table)) # else: # log.error("%s / table is Init None:%s"(fname, table)) if df is not None and len(df) > 0: df = df.fillna(0) if 'timel' in df.columns: time_list = df.timel.tolist() # time_list = sorted(set(time_list),key = time_list.index) time_list = sorted(set(time_list)) # log.info("test:%s"%(sorted(set(time_list),key = time_list.index))) if time_list is not None and len(time_list) > 0: df['timel'] = time_list[0] log.info("load hdf times:%s" % ([time.time() - t_x for t_x in time_list])) log.info("load_hdf_time:%0.2f" % (time.time() - time_t)) if df is not None: df = df[~df.index.duplicated(keep='last')] if fname.find('MultiIndex') > 0 and 'volume' in df.columns: count_drop = len(df) df = df.drop_duplicates() # df = df.drop_duplicates('volume',keep='last') dratio = round((float(len(df))) / float(count_drop), 2) log.debug("all:%s drop:%s dratio:%.2f" % (int(count_drop / 100), int(len(df) / 100), dratio)) if dratio < 0.8: log.error("MultiIndex drop_duplicates:%s %s dr:%s" % (count_drop, len(df), dratio)) if isinstance(df.index, pd.core.index.MultiIndex): write_hdf_db(fname, df, table=table, index=index, MultiIndex=True, rewrite=True) # df = df.drop_duplicates() return df
# top_now = tdd.getSinaAlldf(market='雄安新区', filename='xaxq', vol=ct.json_countVol, vtype=ct.json_countType) # top_now = tdd.getSinaAlldf(market=u'京津冀', filename='beijing', vol=ct.json_countVol, vtype=ct.json_countType) # top_now = tdd.getSinaAlldf(market='sh', vol=ct.json_countVol, vtype=ct.json_countType) now_count = len(top_now) radio_t = cct.get_work_time_ratio() # top_now = top_now[top_now.buy > 0]` time_d = time.time() if time_d - time_s > delay_time: status_change = True time_s = time.time() top_all = pd.DataFrame() else: status_change = False # print ("Buy>0:%s" % len(top_now[top_now['buy'] > 0])), if len(top_now) > 10 or cct.get_work_time(): time_Rt = time.time() if len(top_all) == 0 and len(lastpTDX_DF) == 0: cct.get_terminal_Position(position=sys.argv[0]) time_Rt = time.time() print(("term:%s" % (cct.get_terminal_Position(cmd='DurationCXDN.py')),)) if cct.get_terminal_Position(cmd='DurationCXDN.py') > 1: top_all, lastpTDX_DF = tdd.get_append_lastp_to_df( top_now, lastpTDX_DF=None, dl=duration_date, end=end_date, ptype=ptype, filter=filter, power=ct.lastPower, lastp=False, newdays=newdays, checknew=True,resample=resample) else: newdays = 15 top_all, lastpTDX_DF = tdd.get_append_lastp_to_df( top_now, lastpTDX_DF=None, dl=duration_date, end=end_date, ptype=ptype, filter=filter, power=ct.lastPower, lastp=False, newdays=newdays, checknew=True, resample=resample)
ct.Monitor_format_trade, market_sort_value[:2]) print cct.format_for_print( top_temp.loc[:, ct_MonitorMarket_Values][:10]) # print cct.format_for_print(top_all[:10]) if status: for code in top_all[:10].index: code = re.findall('(\d+)', code) if len(code) > 0: code = code[0] kind = sl.get_multiday_ave_compare_silent(code) else: # print top_now[:10] print "\tNo data" int_time = cct.get_now_time_int() if cct.get_work_time(): if int_time < 930: while 1: cct.sleep(60) if cct.get_now_time_int() < 931: cct.sleep(60) print ".", else: top_all = pd.DataFrame() time_s = time.time() print "." break else: cct.sleep(60) elif cct.get_work_duration(): while 1:
def getBollFilter(df=None, boll=ct.bollFilter, duration=ct.PowerCountdl, filter=True, ma5d=True, dl=14, percent=True, resample='d', ene=False, upper=False, down=False, indexdff=True, cuminTrend=False, top10=True): # drop_cxg = cct.GlobalValues().getkey('dropcxg') # if len(drop_cxg) >0: # log.info("stf drop_cxg:%s"%(len(drop_cxg))) # drop_cxg = list(set(drop_cxg)) # drop_t = [ co for co in drop_cxg if co in df.index] # if len(drop_t) > 0: # df = df.drop(drop_t,axis=0)top # log.error("stf drop_cxg:%s"%(len(drop_t))) # hvdu max量天数 lvdu min天数 hv max量 lv 量 # fib < 2 (max) fibl > 2 lvdu > 3 (hvdu > ? or volume < 5)? ene time_s = time.time() # df['upper'] = map(lambda x: round((1 + 11.0 / 100) * x, 1), df.ma10d) # df['lower'] = map(lambda x: round((1 - 9.0 / 100) * x, 1), df.ma10d) # df['ene'] = map(lambda x, y: round((x + y) / 2, 1), df.upper, df.lower) ''' df = df[(df.close > df.ma20d) | (df.close > df.ene)] if filter: if 'boll' in df.columns: df = df[(df.boll >= boll) | (df.buy > df.upper)] ''' # if 'nlow' in df.columns: # df = df[ ((df.nlow >= df.llastp) & (df.nlow >= df.low)) | ((df.nlow <= df.low) & (df.open == df.nlow))] # if cct.get_work_time() and 'df2' in df.columns: # if 915 < cct.get_now_time_int() < 930: # df['cumnow'] = map(lambda x, y: 1 if x - y > 0 else 0, df.close, df.cumaxe) # else: # df['cumnow'] = map(lambda x, y: 1 if x - y > 0 else 0, df.high, df.cumaxe) # df['df2'] = (df['df2'] + df['cumnow']).map(lambda x: int(x)) # df = df[df.df2 > 1] # else: # df = df[df.df2 > 0] radio_t = cct.get_work_time_ratio() df['lvolr%s' % (resample)] = df['volume'] # df['volume'] = (map(lambda x, y: round(x / y / radio_t, 1), df.nvol.values, df.lvolume.values)) if cct.get_now_time_int() > 1530: df['volume'] = (map( lambda x, y, z: round((x / y / radio_t) if z < 9.9 else (x / y), 1), df.nvol.values, df.ma5vol.values, df.percent.values)) else: df['volume'] = (map( lambda x, y, z: round((x / y / radio_t) if z < 9.9 else (x / y), 1), df.nvol.values, df.lvol.values, df.percent.values)) # df['volume'] = (map(lambda x, y,z: round((x / y / radio_t) if z < 9.9 else (x / y) , 1), df.nvol.values, df.ma5vol.values,df.percent.values)) # df['volume'] = (map(lambda x, y: round(x / y / radio_t, 1), df.nvol.values, df.lastv1d.values)) indexfibl = cct.GlobalValues().getkey('indexfibl') sort_value = cct.GlobalValues().getkey('market_sort_value') market_key = cct.GlobalValues().getkey('market_key') market_value = cct.GlobalValues().getkey('market_value') # if int(market_value) > 1 and 930 < cct.get_now_time_int(): df = compute_perd_value(df, market_value, 'perc') df = compute_perd_value(df, market_value, 'per') if sort_value <> 'percent' and (market_key in ['2', '3'] and market_value not in ['1']): # print("sort_value:%s,market_key:%s ,market_value:%s" % # (sort_value, market_key, market_value)) if market_key is not None and market_value is not None: # if market_key == '3': if market_key == '3': market_value = int(market_value) log.info("stf market_key:%s" % (market_key)) idx_k = cct.get_col_in_columns(df, 'perc%sd', market_value) # filter percd > idx # df= df[(df[("perc%sd" % (idx_k))] >= idx_k) | (df[("perc%sd" % (idx_k))]< -idx_k)] # if int(market_value) > 1 and 930 < cct.get_now_time_int(): # df= compute_perd_value(df, market_value, 'perc') # df= compute_perd_value(df, market_value, 'per') # elif market_key == '2': # if int(market_value) > 1 and 915 < cct.get_now_time_int(): # # df['per%d'%(market_value)] = compute_perd_value(df,market_value) # df= compute_perd_value(df, market_value,'per') # df= compute_perd_value(df, market_value, 'perc') # if sort_value <> 'percent' and (market_key in ['2','3'] and market_value not in ['1'] ): # if indexfibl is not None: # df = df[df.fibl >= (indexfibl - 1)] # else: # indexfibl = 1 # if cuminTrend: # cuminfibl = cct.GlobalValues().getkey('cuminfibl') # if cuminfibl is None: # cuminfibl = cct.get_index_fibl() # if cuminfibl > 1: # df = df[df.df2 >= cuminfibl] # print df.loc['300107'][['close','cumine','llastp','cumaxe','cumaxc']] # df = df[ (df.cumine == df.cumaxc) | (df.close >= df.cumine) | (df.llastp >=df.cumaxc) | (df.close >= df.cumaxe) ] # else: # upper = True if not down: if upper: # upper_count = len( df[ ( ( df.buy > df.upper)| ( df.lastp0d > df.upper) | ( df.lastp1d > df.upper) | ( df.lastp2d > df.upper) | ( df.lastp3d > df.upper))]) # # ene_count = len(df[df.buy > df.upper]) # if upper_count > 30: # df = df[ ( ( df.buy > df.upper)| ( df.lastp0d > df.upper) | ( df.lastp1d > df.upper) | ( df.lastp2d > df.upper) | ( df.lastp3d > df.upper))] # else: # df = df[ ( (( df.buy > df.ene ) ) | ( df.buy > df.upper)| ( df.lasth0d > df.upper) | ( df.lasth1d > df.upper) | ( df.lasth2d > df.upper) | ( df.lasth3d > df.upper))] df = df[(df.low >= df.upper) | (df.close >= df.upper) | (df.per1d >= 9.5) | (df.lastp1d == df.hmax) | ((df.lastp1d <> df.hmax) & (df.high > df.hmax))] else: if 935 < cct.get_now_time_int() < 1400: df = df[((df.buy > df.ene) | (df.llastp > df.ene) | (df.llastp >= df.max5) | (df.high >= df.max5) | (df.max5 >= df.upper))] else: df = df[((df.buy > df.ene) | (df.llastp > df.ene) | (df.llastp >= df.max5) | (df.high >= df.max5) | (df.max5 >= df.upper))] # df['cumins'] = round(cum_counts.index[0], 2) # df['cumine'] = round(cumdf[-1], 2) # df['cumax'] # df.rename(columns={'cumin': 'df2'}, inplace=True) if df is None: print "dataframe is None" return None else: # top10 = df[ (df.percent >= 9.99) & (df.b1_v > df.a1_v)] if resample in ['d', 'w']: # if not top10 or (cct.get_now_time_int() < 950 or cct.get_now_time_int() > 1502): # df.loc[((df.b1_v > df.a1_v) & (df.percent > 9)), 'percent'] = 10.1 # df.loc[((df.percent >= 9.97) & (df.percent < 10.1)), 'percent'] = 10 # else: # df.loc[((df.b1_v > df.a1_v) & (df.percent > 9)), 'percent'] = 10.1 # df.loc[((df.percent >= 9.97) & (df.percent < 10.1)), 'percent'] = 10 # df.loc[df.per1d >= 9.99, 'per1d'] = 10 df['percent'] = df['percent'].apply(lambda x: round(x, 2)) # time_ss = time.time() perc_col = [co for co in df.columns if co.find('perc') == 0] per_col = [co for co in df.columns if co.find('per') == 0] # per_col = list(set(per_col) - set(perc_col) - set(['per1d', 'perlastp'])) per_col = list( set(per_col) - set(perc_col) - set(['per1d', 'perlastp'])) perc_col.remove('percent') for co in perc_col: df[co] = df[co].apply(lambda x: round(x, 2)) # da, down_zero, down_dn, percent_l = 1, 0, 0, 2 # da, down_zero, down_dn, percent_l = 1, 0, -1, 1 # df['perc_n'] = map((lambda h, lh, l, ll, c, lc: (1 if (h - lh) > 0 else down_dn) + (1 if c - lc > 0 else down_dn) + (1 if (l - ll) > 0 else down_dn) + (2 if (c - lh) > 0 else down_zero) + (2 if (l - lc) > 0 else down_zero) + (0 if (h - lc) > 0 else down_dn)), df['high'], df['lasth%sd' % da], df['low'], df['lastl%sd' % da], df['close'],df['lastp%sd' % da]) # df['perc_n'] = map((lambda c, lc: (1 if (c - lc) > 0 else down_zero) + (1 if (c - lc) / lc * 100 > 3 else down_zero) + # (down_dn if (c - lc) / lc * 100 < -3 else down_zero)), df['close'], df['lastp%sd' % da]) # idx_rnd = random.randint(0, len(df) - 10) if len(df) > 10 else 0 # print "idx_rnd",idx_rnd,df.ix[idx_rnd].lastp0d ,df.ix[idx_rnd].close,df.ix[idx_rnd].lastp0d != df.ix[idx_rnd].close # if not upper: # if cct.get_work_time() or df.ix[idx_rnd].lastp0d <> df.ix[idx_rnd].close: # nowd, per1d = 0, 1 # if 'nlow' in df.columns: # df['perc_n'] = map(cct.func_compute_percd2, df['close'], df['llastp'], df['open'],df['lasto%sd' % nowd], df['lasth%sd' % # (nowd)], df['lastl%sd' % (nowd)], df['nhigh'], df['nlow'],df['nvol']/radio_t,df['lastv%sd'%(nowd)], df['hmax'],df['df2']) # else: # df['perc_n'] = map(cct.func_compute_percd2, df['close'], df['llastp'], df['open'],df['lasto%sd' % nowd], df['lasth%sd' % # (nowd)], df['lastl%sd' % (nowd)], df['high'], df['low'],df['nvol']/radio_t,df['lastv%sd'%(nowd)], df['hmax'],df['df2']) # else: # nowd, per1d = 1, 2 # # print df['per%sd' % da+1], df['lastp%sd' % (da)], df['lasth%sd' % (da)], df['lastl%sd' % (da)], df['high'], df['low'] # df['perc_n'] = map(cct.func_compute_percd2, df['close'], df['llastp'], df['open'],df['lasto%sd' % nowd], df['lasth%sd' % # (nowd)], df['lastl%sd' % (nowd)], df['high'], df['low'], df['hmax']) ''' idx_rnd = random.randint(0, len(df) - 10) if len(df) > 10 else 0 if not upper: # if cct.get_work_time() or df.ix[idx_rnd].lastp1d <> df.ix[idx_rnd].close: if cct.get_work_time() : nowd, per1d=1, 1 if 'nlow' in df.columns: df['perc_n']=map(cct.func_compute_percd2020, df['open'], df['close'], df['nhigh'], df['nlow'], df['lasto%sd' % nowd], df['llastp'], df['lasth%sd' % (nowd)], df['lastl%sd' % (nowd)], df['ma5d'], df['ma10d'], df['nvol'] / radio_t, df['lastv%sd' % (nowd)]) else: df['perc_n']=map(cct.func_compute_percd2020, df['open'], df['close'], df['high'], df['low'], df['lasto%sd' % nowd], df['llastp'], df['lasth%sd' % (nowd)], df['lastl%sd' % (nowd)], df['ma5d'], df['ma10d'], df['nvol'] / radio_t, df['lastv%sd' % (nowd)]) else: nowd, per1d=1, 1 # print df['per%sd' % da+1], df['lastp%sd' % (da)], df['lasth%sd' % (da)], df['lastl%sd' % (da)], df['high'], df['low'] df['perc_n']=map(cct.func_compute_percd2020, df['open'], df['close'], df['high'], df['low'], df['lasto%sd' % nowd], df['llastp'], df['lasth%sd' % (nowd)], df['lastl%sd' % (nowd)], df['ma5d'], df['ma10d'], df['nvol'] / radio_t, df['lastv%sd' % (nowd)]) for co in perc_col: df[co] = (df[co] + df['perc_n']).map(lambda x: int(x)) ''' # for co in per_col: # df[co] = (df[co] + df['percent']).map(lambda x: int(x)) # print "percT:%.2f"%(time.time()-time_ss) else: df['percent'] = (map( lambda x, y: round((x - y) / y * 100, 1) if int(y) > 0 else 0, df.buy, df.lastp1d)) # df.loc['002204'].percent # -15.4 # ipdb> df.loc['002204'].ma10d # 3.2000000000000002 # ipdb> df.loc['002204'].ma5d # 3.2999999999999998 if 'fib' not in df.columns: df['fib'] = 0 # else: co2int = ['boll', 'op', 'ratio', 'fib', 'fibl', 'df2'] # co2int.extend([co for co in df.columns.tolist() # if co.startswith('perc') and co.endswith('d')]) co2int.extend(['top10', 'topR']) co2int = [inx for inx in co2int if inx in df.columns] for co in co2int: df[co] = df[co].astype(int) # if down: # # if 'nlow' in df.columns: # # if 'max5' in df.columns: # # # df = df[(df.hmax >= df.ene) & (df.nlow >= df.low) & (df.low <> df.high) & (((df.low < df.hmax) & (df.high > df.cmean)) | (df.high > df.max5))] # # df = df[ (((df.buy > df.ene) & (df.volume / radio_t > 1.5)) ) | (((df.lastv1d > df.lvol * 1.5) & (df.lastv1d > df.lvol * 1.5)) & (df.lastp1d > df.ene) )] # # else: # # df = df[ (((df.buy > df.ene) & (df.volume / radio_t > 1.5)) ) | (((df.lastv1d > df.lvol * 1.5) ) )] # # else: # # # df = df[ (df.max5 >= df.ene) & (df.low <> df.high) & (((df.low < df.hmax) & (df.close > df.cmean)) | (df.high > df.cmean))] # # df = df[ (((df.buy > df.ene) & (df.volume / radio_t > 1.5)) ) | (((df.lastp1d > df.ene) ) | ((df.buy > df.max5)|(df.buy > df.ene)) | (df.max5 >= df.ene) | (df.high > df.cmean) )] # return df if cct.get_work_time() and 'b1_v' in df.columns and 'nvol' in df.columns: df = df[(df.b1_v > 0) | (df.nvol > 0)] if (cct.get_now_time_int() > 915 and cct.get_now_time_int() < 926): df['b1_v'] = df['volume'] else: dd = df[df.percent < 10] dd['b1_v'] = dd['volume'] df = cct.combine_dataFrame(df, dd.loc[:, ['b1_v']]) # print "t:%0.2f"%(time.time()-time_ss) # if int(market_value) < 3 and 930 < cct.get_now_time_int(): # if 'nlow' in df.columns: # # df = df[((df.per1d < 3) & (df.nlow >= df.llastp)) | ((df.per1d > 5) & (df.buy > df.llastp))] # df = df[((df.per1d < 3) & (df.nlow >= df.llastp)) | ((df.per1d > 5) & (df.buy > df.llastp))] # else: # df = df[((df.per1d < 3) & (df.open > df.llastp)) | ((df.per1d > 5) & (df.buy > df.llastp))] # else: # if 'fib' in df.columns: # df = df[df.fib <= 5] # log.error("perc%sd"%(market_value)) # elif market_key is not None and market_key == '2': # if market_value in ['1', '2']: # df['dff'] = (map(lambda x, y, z: round((x + (y if z > 20 else 3 * y)), 1), df.dff.values, df.volume.values, df.ratio.values)) # df['df2'] = (map(lambda x, y, z: w=round((x - y) / z * 100, 1), df.high.values, df.low.values, df.llastp.values)) # df['df2'] = (map(func_compute_df2, df.close.values, df.llastp.values,df.high.values, df.low.values,df.ratio.values)) # df['df2'] = (map(func_compute_df2, df.close.values, df.llastp.values,df.high.values, df.low.values)) ''' filter_up= 'ma5d' filter_dn= 'ma10d' if ma5d and filter_up in df.columns: if filter_dn in df.columns: if market_key is not None and market_key == '3': df= df[(df.buy >= df[filter_up]) & ( df.buy * 1.02 >= df[filter_dn] * ct.changeRatio)] else: df= df[(df.buy >= df[filter_up])] else: df= df[df.buy >= df[filter_up] * ct.changeRatio] ''' # df = df[(df.lvolume > df.lvol * 0.9) & (df.lvolume > df.lowvol * 1.1)] # if 'nlow' in df.columns and 932 < cct.get_now_time_int() < 1030: if ene: pass if 'nclose' in df.columns: # df = df[ ((df.buy > df.ene) ) | ((df.buy > df.upper) & ( df.nclose > df.cmean))] df = df[((df.buy > df.llastp)) | ((df['nclose'] > df['ene'] * ct.changeRatio) & (df['percent'] > -3) & (df.volume > 1.5)) | ((df['nclose'] > df['upper'] * ct.changeRatio) & (df['buy'] > df['upper'] * ct.changeRatioUp)) | ((df['llastp'] > df['upper']) & (df['nclose'] > df['upper']))] else: df = df[((df.buy > df.llastp)) | ((df.buy > df.ene)) | ((df.buy > df.upper) & (df.close > df.cmean))] if 'nlow' in df.columns and 930 < cct.get_now_time_int(): # for col in ['nhigh', 'nclose', 'nlow','nstd']: # df[col] = df[col].apply(lambda x: round(x, 2)) if 'nhigh' in df.columns and 'nclose' in df.columns: # ncloseRatio = map(lambda x, y: x * ct.changeRatio if x * ct.changeRatio > y else y, df.nclose, df.nlow) df['ncloseRatio'] = map(lambda x: x * 0.99, df.nclose) df['nopenRatio'] = map(lambda x: x * 0.99, df.open) # nhighRatio = map(lambda x, y: x * ct.changeRatio if x * ct.changeRatio > y else y, df.nhigh, df.nclose) # if cct.get_now_time_int() > ct.nlow_limit_time: # name open low high close nclose nlow nhigh nstd ticktime # code # 601899 紫金矿业 4.25 4.24 4.69 4.69 4.52 4.24 4.42 0.12 15:00:00 # 603917 合力科技 27.95 27.69 31.54 31.54 30.46 27.69 30.00 0.70 15:00:00 # 300713 英可瑞 81.00 81.00 89.82 89.77 87.53 80.90 86.36 1.75 15:02:03 # 000933 神火股份 9.07 9.02 10.00 10.00 9.71 9.02 9.44 0.24 15:02:03 # 600050 中国联通 6.49 6.37 6.51 6.41 6.44 6.44 6.51 0.02 15:00:00 # 002350 北京科锐 10.17 10.17 10.75 10.23 10.31 10.17 10.75 0.07 15:02:03 # 603363 傲农生物 17.49 16.91 18.35 18.35 17.35 17.18 18.00 0.34 15:00:00 # 000868 安凯客车 8.77 8.53 8.82 8.82 8.82 8.53 8.82 0.01 15:02:03 # 002505 大康农业 2.91 2.86 2.99 2.89 2.91 2.90 2.99 0.02 15:02:03 # 601939 建设银行 7.50 7.41 7.61 7.55 7.50 7.43 7.52 0.05 15:00:00 # 600392 盛和资源 16.85 16.81 18.63 18.63 17.84 16.81 17.51 0.69 15:00:00 (11, 41) # 603676 卫信康 17.44 16.95 17.96 17.82 17.34 17.03 17.77 0.18 15:00:00 if 'nstd' in df.columns: df['stdv'] = map(lambda x, y: round(x / y * 100, 1), df.nstd, df.open) # top_temp[:40][['nstd','stdv','name','volume','dff','percent']] # top_temp.loc['603363'][['nstd','stdv','name','volume','high','nhigh','percent','open','nclose']] # df['stdv'] = map(lambda x, y, z: round(z / (x / y * 100), 1), df.nstd, df.open, df.volume) # df.loc['002906'][['open','nclose','close','ncloseRatio','nopenRatio','low','nlow']] df = df[(df.low > df.upper) | (((df.nopenRatio <= df.nclose) & (df.low >= df.nlow) & (df.close >= df.ncloseRatio)) | ((df.nopenRatio <= df.nclose) & (df.close >= df.ncloseRatio) & (df.high >= df.nhigh)))] # else: # df = df[(((df.low >= df.nlow) & (df.close >= ncloseRatio)) | ((df.close >= ncloseRatio) & (df.close > nhighRatio)))] else: df = df[((df.low >= df.nlow) & (df.close > df.llastp * ct.changeRatio))] # if 'nhigh' in df.columns and 'nclose' in df.columns: # if cct.get_now_time_int() > ct.nlow_limit_time: # df = df[(df.low >= df.nlow) & ((df.open > df.llastp * ct.changeRatio) & (df.nclose > df.llastp * ct.changeRatio)) & # (((df.low >= df.nlow) & (df.close >= df.nclose)) | ((df.close >= df.nclose) & (df.close > df.nhigh * ct.changeRatio) & (df.high >= df.nhigh)))] # else: # df = df[((df.open > df.llastp * ct.changeRatio) & (df.close > df.llastp * ct.changeRatio)) & # (((df.low >= df.nlow) & (df.close >= df.nclose)) | ((df.close >= df.nclose) & (df.close > df.nhigh * ct.changeRatio)))] # else: # df = df[((df.low >= df.nlow) & (df.close > df.llastp))] if filter: if cct.get_now_time_int() > 915 and cct.get_now_time_int() <= 1000: df = df[((df.buy > df.llastp)) | (df.buy > df.hmax * ct.changeRatio) | (df.buy < df.lmin * ct.changeRatioUp)] # df = df[(df.buy > df.hmax * ct.changeRatio) & (df.low > df.llastp)] # df = df[(df.buy > df.cmean * ct.changeRatioUp) | (df.open > df.llastp)] # df = df[df.buy > df.cmean] elif cct.get_now_time_int() > 1000 and cct.get_now_time_int( ) <= 1430: df = df[((df.buy > df.llastp)) | (df.buy > df.hmax * ct.changeRatio) | (df.buy < df.lmin * ct.changeRatioUp)] # df = df[(df.buy > df.hmax * ct.changeRatio) | (df.low > df.llastp)] # df = df[(df.buy > df.hmax * ct.changeRatio) & (df.low > df.llastp)] # df = df[(df.buy > df.cmean * ct.changeRatioUp) | (df.open > df.llastp)] # df = df[df.buy > df.cmean] else: df = df[((df.buy > df.llastp)) | (df.buy > df.hmax * ct.changeRatio) | (df.buy < df.lmin * ct.changeRatioUp)] # df = df[(df.buy > df.hmax * ct.changeRatio) | (df.low > df.llastp)] # df = df[(df.buy > df.hmax * ct.changeRatio) & (df.low > df.llastp)] # df = df[(df.buy > df.cmean * ct.changeRatioUp) | (df.open > df.llastp)] # df = df[df.buy > df.cmean] if 'vstd' in df.columns: # df = df[(df.lvol * df.volume > (df.vstd + df.lvol)) | ((df.percent > -10) & (df.hv / df.lv > 1.5))] # dd = df.loc['000760'] dd.hv/dd.lv,dd.volume df['hvRatio'] = map( lambda x, y: round(x / y / cct.get_work_time_ratio(), 1), df.hv, df.lv) df['volRatio'] = (map(lambda x, y: round(x / y / radio_t, 1), df.nvol.values, df.lv.values)) # df = df[(df.lvol * df.volume > (df.vstd + df.lvol)) | ((df.percent > -5) & (df.hv / df.lv > 3))] # if 1000 < cct.get_now_time_int() < 1450: # df = df[ (3 < df.ratio) & (df.ratio < 20)] if 'nclose' in df.columns: # df = df[((df.hvRatio > df.volRatio) & ((df.buy > df.lastp) & (df.nclose > df.lastp))) | (( df = df[(((df.buy > df.lastp) & (df.nclose > df.lastp))) | (((df.percent > -5) & ((df.vstd + df.lvol) * 1.2 < df.nvol) & (df.nvol > (df.vstd + df.lvol))))] else: # df = df[((df.hvRatio > df.volRatio) & ((df.buy > df.lastp))) | (( df = df[(((df.buy > df.lastp))) | (((df.percent > -5) & ((df.vstd + df.lvol) * 1.2 < df.nvol) & (df.nvol > (df.vstd + df.lvol))))] # [dd.lvol * dd.volume > (dd.vstd + dd.lvol) | dd.lvol * dd.volume >(dd.ldvolume + dd.vstd] # print df.loc['000801'] if percent: pass ''' if 'stdv' in df.columns and 926 < cct.get_now_time_int(): # df = df[((df.volume > 2 * cct.get_work_time_ratio()) & (df.percent > -3)) | ((df.stdv < 1) & # (df.percent > 2)) | ((df.lvolume > df.lvol * 0.9) & (df.lvolume > df.lowvol * 1.1))] df_index = tdd.getSinaIndexdf() if isinstance(df_index, type(pd.DataFrame())): df_index['volume'] = (map(lambda x, y: round(x / y / radio_t, 1), df_index.nvol.values, df_index.lastv1d.values)) index_vol = df_index.loc['999999'].volume if 'percent' in df_index.columns and '999999' in df_index.index: index_percent = df_index.loc['999999'].percent else: # import ipdb;ipdb.set_trace() log.error("index_percent" is None) index_percent = 0 index_boll = df_index.loc['999999'].boll index_op = df_index.loc['999999'].op # df = df[((df.percent > -3) | (df.volume > index_vol)) & (df.percent > index_percent)] # df = df[((df.percent > -1) | (df.volume > index_vol * 1.5)) & (df.percent > index_percent) & (df.boll >= index_boll)] df = df[((df.percent > -1) | (df.volume > index_vol * 1.5)) & (df.percent > index_percent)] else: print ("df_index is Series",df_index.T) else: df = df[((df.volume > 1.2 * cct.get_work_time_ratio()) & (df.percent > -3))] ''' # df = df[((df['buy'] >= df['ene'])) | ((df['buy'] < df['ene']) & (df['low'] > df['lower'])) | ((df['buy'] > df['upper']) & (df['low'] > df['upper']))] # df = df[(( df['ene'] * ct.changeRatio < df['open']) & (df['buy'] > df['ene'] * ct.changeRatioUp)) | ((df['low'] > df['upper']) & (df['close'] > df['ene']))] # df = df[(( df['ene'] < df['open']) & (df['buy'] < df['ene'] * ct.changeRatioUp))] # df = df[(df.per1d > 9) | (df.per2d > 4) | (df.per3d > 6)] # df = df[(df.per1d > 0) | (df.per2d > 4) | (df.per3d > 6)] # time_ss=time.time() # codel = df.index.tolist() # dm = tdd.get_sina_data_df(codel) # results = cct.to_mp_run_async(getab.Get_BBANDS, codel,'d',5,duration,dm) # bolldf = pd.DataFrame(results, columns=['code','boll']) # bolldf = bolldf.set_index('code') # df = cct.combine_dataFrame(df, bolldf) # print "bollt:%0.2f"%(time.time()-time_ss), per3d_l = 2 percent_l = -1 op_l = 3 # if 'boll' in df.columns: # if 915 < cct.get_now_time_int() < 950: # # df = df[(df.boll >= boll) | ((df.percent > percent_l) & (df.op > 4)) | ((df.percent > percent_l) & (df.per3d > per3d_l))] # # df = df[((df.percent > percent_l) & (df.op > 4)) | ((df.percent > percent_l) & (df.per3d > per3d_l))] # pass # elif 950 < cct.get_now_time_int() < 1501: # # df = df[(df.boll >= boll) | ((df.low <> 0) & (df.open == df.low) & (((df.percent > percent_l) & (df.op > op_l)) | ((df.percent > percent_l) & (df.per3d > per3d_l))))] # # df = df[(df.boll >= boll) & ((df.low <> 0) & (df.open >= df.low * # # ct.changeRatio) & (((df.percent > percent_l)) | ((df.percent > # # percent_l) & (df.per3d > per3d_l))))] # df = df[(df.boll >= boll)] # # else: # df = df[(df.boll >= boll) | ((df.low <> 0) & (df.open == df.low) & # (((df.percent > percent_l) & (df.op > op_l)) | ((df.percent > percent_l) # & (df.per3d > per3d_l))))] # if 945 < cct.get_now_time_int() and market_key is None: # df.loc[df.percent >= 9.94, 'percent'] = -10 # else: # # df['upper'] = map(lambda x: round((1 + 11.0 / 100) * x, 1), df.ma10d) # # df['lower'] = map(lambda x: round((1 - 9.0 / 100) * x, 1), df.ma10d) # # df['ene'] = map(lambda x, y: round((x + y) / 2, 1), df.upper, df.lower) # # df = df[((df['buy'] >= df['ene'])) | ((df['buy'] < df['ene']) & (df['low'] > df['lower'])) | ((df['buy'] > df['upper']) & (df['low'] > df['upper']))] # # df = df[(( df['ene'] * ct.changeRatio < df['open']) & (df['buy'] > df['ene'] * ct.changeRatioUp)) | ((df['low'] > df['upper']) & (df['close'] > df['ene']))] # if 'nclose' in df.columns: # # df = df[((df['nclose'] > df['ene'] * ct.changeRatio) & (df['percent'] > -3) & (df.volume > 1.5)) | # # ((df['nclose'] > df['upper'] * ct.changeRatio) & (df['buy'] > df['upper'] * ct.changeRatioUp)) | ((df['llastp'] > df['upper']) & (df['nclose'] > df['upper']))] # df = df[(df.nclose > df.cmean)] # else: # df = df[(df.buy > df.cmean)] print("bo:%0.1f" % (time.time() - time_s)), df['ral'] = (map(lambda x, y: round((x + y), 1), df.percent, df.ral)) df['ra'] = (map(lambda x, y: round((x + y), 1), df.percent, df.ra)) return df
def get_sina_all_json_dd(vol='0', type='0', num='10000', retry_count=3, pause=0.001): start_t = time.time() # url="http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?page=1&num=50&sort=changepercent&asc=0&node=sh_a&symbol=" # SINA_REAL_PRICE_DD = '%s%s/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?page=1&num=%s&sort=changepercent&asc=0&node=%s&symbol=%s' #http://vip.stock.finance.sina.com.cn/quotes_service/view/cn_bill_sum.php #http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/CN_Bill.GetBillList?num=10000&page=1&sort=ticktime&asc=0&volume=100000&type=0 #http://vip.stock.finance.sina.com.cn/quotes_service/view/cn_bill_all.php?num=100&page=1&sort=ticktime&asc=0&volume=100000&type=0 """ 一次性获取最近一个日交易日所有股票的交易数据 return ------- DataFrame 属性:代码,名称,涨跌幅,现价,开盘价,最高价,最低价,最日收盘价,成交量,换手率 """ # return '' h5_fname = 'get_sina_all_dd' h5_table = 'all'+'_'+ct.DD_VOL_List[str(vol)]+'_'+str(num) limit_time = ct.sina_dd_limit_time h5 = h5a.load_hdf_db(h5_fname, table=h5_table,limit_time=limit_time) if h5 is not None and not h5.empty and len(h5) > 500 and 'timel' in h5.columns: o_time = h5[h5.timel != 0].timel if len(o_time) > 0: o_time = o_time[0] l_time = time.time() - o_time return_hdf_status = not cct.get_work_time() or (cct.get_work_time() and l_time < limit_time) if return_hdf_status: log.info("load hdf data:%s %s %s"%(h5_fname,h5_table,len(h5))) return h5 # ct._write_head() url_list = _get_sina_json_dd_url(vol, type, num) # print url_list df = pd.DataFrame() # import ipdb;ipdb.set_trace() # data['code'] = symbol # df = df.append(data, ignore_index=True) if len(url_list)>0: log.info("json_dd_url:%s"%url_list[0]) for url in url_list: # df = df.append(_parsing_sina_dd_price_json(url)) data = _parsing_sina_dd_price_json(url) # if data is not None and not data.empty and len(data) > 0: if len(data) > 5: df = df.append(data) # else: # log.error("data:%s"%(data)) # data = cct.to_mp_run(_parsing_sina_dd_price_json, url_list) # data = cct.to_mp_run_async(_parsing_sina_dd_price_json, url_list) # if len(url_list)>cct.get_cpu_count(): # divs=cct.get_cpu_count() # else: # divs=len(url_list) # # if len(url_list)>=divs: # print len(url_list), # dl=cct.get_div_list(url_list,divs) # data=cct.to_mp_run_async(cct.to_asyncio_run,dl,_parsing_sina_dd_price_json) # else: # data=cct.to_asyncio_run(url_list,_parsing_sina_dd_price_json) # data = cct.to_asyncio_run(url_list, _parsing_sina_dd_price_json) //return df list # if len(data)>50: # df = df.append(data, ignore_index=True) # # log.debug("dd.columns:%s" % df.columns.values) # #['code' 'name' 'ticktime' 'price' 'volume' 'prev_price' 'kind'] # log.debug("get_sina_all_json_dd:%s" % df[:1]) if df is not None and not df.empty and len(df) > 50: # for i in range(2, ct.PAGE_NUM[0]): # newdf = _parsing_dayprice_json(i) # df = df.append(newdf, ignore_index=True) # print "dd", time_drop=time.time() df['couts']=df.groupby(['code'])['code'].transform('count') # df=df[(df['kind'] == 'U')] df=df.sort_values(by='couts',ascending=0) df=df.drop_duplicates('code') print("djdf:%0.1f"%(time.time()-time_drop), end=' ') # df=df[df.price >df.prev_price] log.info("sina-DD:%s" % df[:1]) # df = df.loc[:, ['code','name', 'couts', 'kind', 'prev_price']] # print df.columns df = df.loc[:, ['code','name', 'couts', 'kind', 'prev_price','ticktime']] df.code=df.code.apply(lambda x:str(x).replace('sh','') if str(x).startswith('sh') else str(x).replace('sz','')) if len(df) > 0: df = df.set_index('code') h5 = h5a.write_hdf_db(h5_fname, df, table=h5_table,append=False) log.info("get_sina_all_json_dd:%s"%(len(df))) print((" dd-df:%0.2f" % ((time.time() - start_t))), end=' ') return df else: print() print(("url:%s no data json-df:%0.2f"%(url_list[0],(time.time() - start_t))), end=' ') return '' else: print(("Url None json-df:%0.2f "%((time.time() - start_t))), end=' ') return ''
def get_sina_Market_json(market='all', showtime=True, num='100', retry_count=3, pause=0.001): start_t = time.time() # qq stock api # http://qt.gtimg.cn/q=sz000858,sh600199 # http://blog.csdn.net/ustbhacker/article/details/8365756 # url="http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?page=1&num=50&sort=changepercent&asc=0&node=sh_a&symbol=" # SINA_REAL_PRICE_DD = '%s%s/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?page=1&num=%s&sort=changepercent&asc=0&node=%s&symbol=%s' """ 一次性获取最近一个日交易日所有股票的交易数据 return ------- DataFrame 属性:代码,名称,涨跌幅,现价,开盘价,最高价,最低价,最日收盘价,成交量,换手率 """ # ct._write_head() h5_fname = 'get_sina_all_ratio' # h5_table = 'all' h5_table = 'all'+'_'+str(num) # if market == 'all': limit_time = ct.sina_dd_limit_time h5 = h5a.load_hdf_db(h5_fname, table=h5_table,limit_time=limit_time) if h5 is not None and len(h5) > 0 and 'timel' in h5.columns: o_time = h5[h5.timel != 0].timel if len(h5) < 2000: log.error("h5 not full data") o_time = [] if len(o_time) > 0: o_time = o_time[0] l_time = time.time() - o_time # return_hdf_status = not cct.get_work_day_status() or not cct.get_work_time() or (cct.get_work_day_status() and cct.get_work_time() and l_time < limit_time) return_hdf_status = not cct.get_work_time() or (cct.get_work_time() and l_time < limit_time) if return_hdf_status: log.info("load hdf data:%s %s %s"%(h5_fname,h5_table,len(h5))) dd = None if market == 'all': co_inx = [inx for inx in h5.index if str(inx).startswith(('6','30','00'))] elif market == 'sh': co_inx = [inx for inx in h5.index if str(inx).startswith(('6'))] elif market == 'sz': co_inx = [inx for inx in h5.index if str(inx).startswith(('00'))] elif market == 'cyb': co_inx = [inx for inx in h5.index if str(inx).startswith(('30'))] else: log.error('market is not Find:%s'%(market)) codel = cct.read_to_blocknew(market) co_inx = [inx for inx in codel if inx in h5.index] dd = h5.loc[co_inx] if len(dd) > 100: log.info("return sina_ratio:%s"%(len(dd))) return dd # else: # market = 'all' if h5 is None or market=='all': url_list=[] # for m in ct.SINA_Market_KEY.values(): for m in ['sh_a','sz_a']: mlist=_get_sina_Market_url(m, num=num) for l in mlist:url_list.append(l) # print url_list else: url_list=_get_sina_Market_url(ct.SINA_Market_KEY[market], num=num) # print url_list # log.debug("Market_jsonURL: %s" % url_list[0]) # print url_list # print "url:",url_list df = pd.DataFrame() # data['code'] = symbol # df = df.append(data, ignore_index=True) # results = cct.to_mp_run(_parsing_Market_price_json, url_list) # urltest ='http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?page=1&num=100&sort=changepercent&asc=0&node=sh_a&symbol=' if len(url_list) > 0: log.debug("cct.to_asyncio_run:_parsing_Market_price_json:%s"%(url_list[0])) results = cct.to_asyncio_run(url_list, _parsing_Market_price_json) else: results = [] if len(results)>0: df = df.append(results, ignore_index=True) # df['volume']= df['volume'].apply(lambda x:x/100) # print df.columns if 'ratio' in df.columns: df['ratio']=df['ratio'].apply(lambda x:round(float(x),1)) df['percent']=df['percent'].apply(lambda x:round(float(x),2)) # if cct.get_now_time_int() > 915 and cct.get_now_time_int() < 926: # df = df[(df.buy > 0)] # else: # df = df[(df.trade > 0)] # print df[:1] # for url in url_list: # # print url # data = _parsing_Market_price_json(url) # # print data[:1] # df = df.append(data, ignore_index=True) # # break if df is not None and len(df) > 0: # for i in range(2, ct.PAGE_NUM[0]): # newdf = _parsing_dayprice_json(i) # df = df.append(newdf, ignore_index=True) # print len(df.index) # print type(df) if 'code' in df.columns: df=df.drop_duplicates('code') df = df.set_index('code') # if market == 'all': # append_status = False #(new all don't append main ) # else: # append_status = True if market=='all': h5 = h5a.write_hdf_db(h5_fname, df, table=h5_table,append=False) else: h5 = h5a.write_hdf_db(h5_fname, df, table=h5_table,append=True) if showtime: print(("Market-df:%s %s" % (format((time.time() - start_t), '.1f'), len(df))), end=' ') if market == 'all': co_inx = [inx for inx in df.index if str(inx).startswith(('6','30','00'))] df = df.loc[co_inx] elif market == 'sh': co_inx = [inx for inx in df.index if str(inx).startswith(('6'))] df = df.loc[co_inx] elif market == 'sz': co_inx = [inx for inx in df.index if str(inx).startswith(('00'))] df = df.loc[co_inx] elif market == 'cyb': co_inx = [inx for inx in df.index if str(inx).startswith(('30'))] df = df.loc[co_inx] return df else: if showtime:print(("no data Market-df:%s" % (format((time.time() - start_t), '.2f')))) log.error("no data Market-df:%s"%(url_list[0])) return []
# if dratio < 0.1 or len(dd) > 3100: log.info("find all:%s :%s %0.2f" % (len(code_l), len(code_l) - len(dif_co), dratio)) if timelimit and len(dd) > 0: dd = dd.loc[dif_co] o_time = dd[dd.timel <> 0].timel.tolist() # if fname == 'powerCompute': # o_time = sorted(set(o_time),reverse=True) o_time = sorted(set(o_time), reverse=False) o_time = [time.time() - t_x for t_x in o_time] if len(dd) > 0: # if len(dd) > 0 and (not cct.get_work_time() or len(o_time) <= ct.h5_time_l_count): l_time = np.mean(o_time) return_hdf_status = (not cct.get_work_time()) or ( cct.get_work_time() and l_time < limit_time) # return_hdf_status = l_time < limit_time # print return_hdf_status,l_time,limit_time if return_hdf_status: # df=dd df = dd.loc[dif_co] log.info( "return hdf: %s timel:%s l_t:%s hdf ok:%s" % (fname, len(o_time), l_time, len(df))) else: log.error( "%s %s o_time:%s %s" % (fname, table, len(o_time), [time.time() - t_x for t_x in o_time[:3]])) log.info('fname:%s l_time:%s' %