def pool(self): pool = Pool(processes=4) shareDict = Manager().list([]) Rice = tick_csv_Rice() Rice.setParam(self.cvsParam) ticks = Rice.exitTicks() print(ticks) for rc in ticks: # if rc in exists: # continue map = rc.split("_") for kp in self.klinePeriods: #self.start(map, kp, shareDict) try: pool.apply_async(self.start, (map, kp, shareDict)) pass except Exception as e: print(e) continue # break pool.close() pool.join()
def start(self, codes, time0, kt, shareDict): print("子进程启动:", self.cindex, codes, time.time() - time0) self.klineType = kt # 主力合约 self.codes = codes self.mCodes = mCodes = [n + '88' for n in codes] # 查询获得配置 - 费率和每手单量 self.Base = future_baseInfo() for doc in self.Base.getInfo(codes): self.baseInfo[doc["code"] + '88'] = doc cs0, cs1 = self.baseInfo[mCodes[0]], self.baseInfo[mCodes[1]] if cs0["nightEnd"] != cs1["nightEnd"]: return None # 计算tick 导致的滑点 sh = [self.baseInfo[d + '88']['tick_size'] for d in codes] # 子进程共享类 self.Rice = tick_csv_Rice() self.Train = train_future() self.Total = train_total() self.Total.tablename = self.total_tablename # 查询获得N分钟K线 dfs_l = self.Rice.kline(mCodes, period=self.klineType, start=self.startDate, end=self.endDate) # 获得1分钟K线作为滚动线 dfs = self.Rice.kline(mCodes, period='1m', start=self.startDate, end=self.endDate) # 按时间截取并调整 dfs = self.dateAdjust(codes, dfs, sh) print('kline load:', mCodes, len(dfs[mCodes[0]]), len(dfs[mCodes[1]])) # 根据配置文件获取最佳交易手数对 self.iniVolume = round( self.iniAmount / cs0["lastPrice"] / cs0["contract_multiplier"], 0) # 分参数执行 results = [] for period in self.periodList: for scale in self.scaleList: for ss in self.shiftScaleList: self.shiftScale = ss self.shift = [sh[i] * ss for i in range(len(sh))] doc = self.total(dfs, dfs_l, scale=scale, period=period) if doc is None or doc['amount'] == 0: continue logger.info((doc['uid'], doc['income'], " time:", time.time() - time0)) results.append(doc) # 结果写入共享阈 #shareDict.append(doc) self.Total.insertAll(results)
def start(self, codes, time0, kt, shareDict): print("子进程启动:", self.cindex, codes, time.time() - time0) self.klineType = kt # 主力合约 self.codes = codes self.mCodes = mCodes = [n + '88' for n in codes] # 查询获得配置 - 费率和每手单量 self.Base = future_baseInfo() for doc in self.Base.getInfo(codes): self.baseInfo[doc["code"] + '88'] = doc cs0 = self.baseInfo[mCodes[0]] # 计算tick 导致的滑点 sh = [self.baseInfo[d + '88']['tick_size'] for d in codes] self.shift = [sh[i] * self.shiftScale for i in range(len(sh))] # 子进程共享类 self.Rice = tick_csv_Rice() self.Rice.setTimeArea(cs0["nightEnd"]) self.Train = train_future() self.Total = train_total() self.Total.tablename = self.total_tablename # 查询获得N分钟K线 dfs_l = self.Rice.kline(mCodes, period=self.klineType, start=self.startDate, end=self.endDate, pre=60) # 获得1分钟K线作为滚动线 dfs = self.Rice.kline(mCodes, period='1m', start=self.startDate, end=self.endDate, pre=0) # 按时间截取并调整 # dfs= self.dateAdjust(codes, dfs, sh) print('kline load:', mCodes, len(dfs[mCodes[0]])) # 根据配置文件获取最佳交易手数对 self.iniVolume = round( self.iniAmount / cs0["lastPrice"] / cs0["contract_multiplier"], 0) # 分参数执行 results = [] for period in self.periodList: docs = self.total(dfs, dfs_l, period=period) if docs is None or len(docs) == 0: continue logger.info( (self.codes, period, self.klineType, len(docs), " time:", time.time() - time0)) results.extend(docs) # 结果写入共享阈 self.Total.insertAll(results)
def start(self, codes, share): print("子进程:", codes) self.time0 = time0 = time.time() self.Base = future_baseInfo() self.Rice = tick_csv_Rice() self.Rice.shareDict = share self.Rice.basePath = self.basePath self.Rice.cvs_start = self.cvs_start self.Rice.cvs_end = self.cvs_end self.Rice.dd = self.dd self.new_docs = [] self.codes = codes self.baseInfo = {} self.filePath = self.basePath + "%s.csv" % "_".join(codes) # 查询获得配置 - 费率和每手单量 for doc in self.Base.getInfo(codes): self.baseInfo[doc["code"]] = doc cs0, cs1 = self.baseInfo[codes[0]], self.baseInfo[codes[1]] if cs0["nightEnd"] != cs1["nightEnd"]: return None # 设置结束时间,用于数据清洗 try: res = self.Rice.curKlineByCsv(codes, period=1, callback=self.new_tick) except Exception as e: print('error', codes, len(self.new_docs), e ) res = True if res and len(self.new_docs) > 0 : # 保存文件 df = pd.DataFrame(self.new_docs) # 数据整理 df = self.detail(df) print(public.getDatetime(), " 已保存", codes, "耗时:", time.time()-time0) df.to_csv(self.filePath, columns=self.columns1, index=0) self.new_docs= [] return True
def reduce(self, c, share): Rice = tick_csv_Rice() Base = future_baseInfo() Rice.codes = [c] s = self.dd file = self.basePath + "%s%s.csv" % (c, s[0]) df = pd.read_csv(file) # 设置结束时间,用于数据清洗 for doc in Base.getInfo([c]): ne = doc["nightEnd"] Rice.setTimeArea(ne) df = Rice.clean(df) len0 = len(df) df = df[(df['datetime'] > self.cvs_start) & (df['datetime'] < self.cvs_end)] print(c,len0, len(df)) df.to_csv(file, columns= Rice.cvs_columns, index=0)
def Pool(self): time0 = time.time() pool = Pool(processes=4) shareDict = Manager().list([]) # 从现有ticks文件获取列表 Rice = tick_csv_Rice() Rice.setParam(model_future_quickTickTest().cvsParam) ticks = Rice.exitTicks() # 已存在的ticks for cs in ticks: rs = cs.split("_") for kt in self.klineTypeList: #self.start(rs, time0, kt , shareDict) try: pool.apply_async(self.start, (rs, time0, kt, shareDict)) pass except Exception as e: print(e) continue pool.close() pool.join()
def compare(self, codes, share): # ticks columns =['datetime','price_t','p_l_t','p_h_t'] self.Rice = tick_csv_Rice() ticks = self.Rice.getTicks(codes) ticks = ticks[ticks['isnew'] == 1] cvs_start = ticks["datetime"].values[0] cvs_end = ticks["datetime"].values[-1] # key = "_".join(codes) ticks["datetime"] = ticks["datetime"].apply(lambda x : x[0:-4] if x.find('.')> 0 else x) ticks["datetime"] = ticks["datetime"].str.replace(':01$|:02$|:03$|:04$',':00') ticks["price_t"] = ticks["close"] ticks["p_l_t"] = ticks["a1"] / ticks["n_b1"] ticks["p_h_t"] = ticks["b1"] / ticks["n_a1"] ticks = self.dfRound(ticks,5,['close','p_l','p_h']) ticks.to_csv(self.basePath + "tick_%s.csv" % key, columns=columns, index=0) Base = future_baseInfo() mCodes = [ c+'88' for c in codes] (b0, b1) =(doc['tick_size'] for doc in Base.getInfo(codes)) print(b0,b1) dfs = self.Rice.kline(mCodes, start=cvs_start, end=cvs_end) df0, df1= dfs[mCodes[0]], dfs[mCodes[1]] df0 = df0[(df0.index >= cvs_start) & (df0.index <= cvs_end)] df0["datetime"] = df0.index df0["datetime"] = df0["datetime"].apply(str) c0, c1 = df0["close"], df1["close"] df0["price"] = c0 / c1 shift = ((c0+b0)/(c1-b1)-(c0-b0)/(c1+b1))/2 df0["p_l"] = c0/c1 + shift df0["p_h"] = c0/c1 - shift df0["p_l_1"] = (c0+b0/2)/(c1-b1/2) df0["p_h_1"] = (c0-b0/2)/(c1+b1/2) df0 = self.dfRound(df0, 5, ['price', 'p_l', 'p_h', 'p_l_1', 'p_h_1']) df0 =pd.merge(df0, ticks, on='datetime', how='left') columns = ['datetime_t', 'datetime', 'price_t', 'price', 'p_l_t', 'p_l' , 'p_l_1', 'p_h_t', 'p_h' , 'p_h_1'] res = {} res['p_l'] = df0['p_l'].mean() - df0['p_l_t'].mean() res['p_l_1'] = df0['p_l_1'].mean() - df0['p_l_t'].mean() res['p_h'] = df0['p_h'].mean() - df0['p_h_t'].mean() res['p_h_1'] = df0['p_h_1'].mean() - df0['p_h_t'].mean() print(res) df0.to_csv(self.basePath + "total_%s.csv" % key, columns=columns, index=0)
def start(self, map, kPeriod, shareDict): # 每个进程单独初始化对象 self.time0 = time.time() self.Record = train_future() # 数据库存储 self.Record.tablename = self.tablename self.Rice = tick_csv_Rice() self.Rice.setParam(self.cvsParam) self.Rice.shareDict = self.shareDict = shareDict self.Rice.kPeriod = self.kPeriod = kPeriod self.Base = future_baseInfo() self.timeArea = [("09:00:00", "11:30:00"), ("13:30:00", "15:00:00")] self.codes = map[0:2] # 查询基础配置 for doc in self.Base.getInfo(self.codes): self.config[doc["code"]] = doc c0, c1 = self.config[self.codes[0]], self.config[self.codes[1]] # 设置交易对的收盘时间,用于计算收盘时间并计算止损 self.Rice.setTimeArea(doc["nightEnd"]) # 根据配置文件获取最佳交易手数对 self.iniVolume = round( self.iniAmount / c0["lastPrice"] / c0["contract_multiplier"], 0) # 设置shift区间 [t, c] = [[self.config[d][key] for d in self.codes] for key in ["tick_size", "lastPrice"]] self.shift = ((c[0] + t[0]) / (c[1] - t[1]) - (c[0] - t[0]) / (c[1] + t[1])) / self.shiftScale logger.info(("model_future_detect start: %s " % "_".join(self.codes), self.uidKey, self.Rice.TimeArea)) # 启动进程 ticks = self.Rice.getTicks(self.codes) # tick类型选择 if self.tickType != 'none': ticks = ticks[ticks[self.tickType] == 1] ticks["p_l"] = ticks["a1"] / ticks["n_b1"] ticks["p_h"] = ticks["b1"] / ticks["n_a1"] indexR, records = 0, [] # 及时K线 for index, row in ticks.iterrows(): #print(row["datetime"]) res = self.onTick(row, isdelay=self.tickType != 'isnew') if len(res) > 0: records.extend(res) # 阶段性存入到数据库,并清空 if len(records) > 50: self.Record.insertAll(records) indexR += len(records) print(public.getDatetime(), " insert into record:", self.codes, self.kPeriod, indexR) records = [] # 将剩下的写入数据库 if len(records) > 0: self.Record.insertAll(records)