def pool(self):
pool = Pool(processes=4)
shareDict = Manager().list([])
Rice = tick_csv_Rice()
Rice.setParam(self.cvsParam)
ticks = Rice.exitTicks()
print(ticks)
for rc in ticks:
# if rc in exists:
# continue
map = rc.split("_")
for kp in self.klinePeriods:
#self.start(map, kp, shareDict)
try:
pool.apply_async(self.start, (map, kp, shareDict))
pass
except Exception as e:
print(e)
continue
# break
pool.close()
pool.join()
def start(self, codes, time0, kt, shareDict):
print("子进程启动:", self.cindex, codes, time.time() - time0)
self.klineType = kt
# 主力合约
self.codes = codes
self.mCodes = mCodes = [n + '88' for n in codes]
# 查询获得配置 - 费率和每手单量
self.Base = future_baseInfo()
for doc in self.Base.getInfo(codes):
self.baseInfo[doc["code"] + '88'] = doc
cs0, cs1 = self.baseInfo[mCodes[0]], self.baseInfo[mCodes[1]]
if cs0["nightEnd"] != cs1["nightEnd"]: return None
# 计算tick 导致的滑点
sh = [self.baseInfo[d + '88']['tick_size'] for d in codes]
# 子进程共享类
self.Rice = tick_csv_Rice()
self.Train = train_future()
self.Total = train_total()
self.Total.tablename = self.total_tablename
# 查询获得N分钟K线
dfs_l = self.Rice.kline(mCodes,
period=self.klineType,
start=self.startDate,
end=self.endDate)
# 获得1分钟K线作为滚动线
dfs = self.Rice.kline(mCodes,
period='1m',
start=self.startDate,
end=self.endDate)
# 按时间截取并调整
dfs = self.dateAdjust(codes, dfs, sh)
print('kline load:', mCodes, len(dfs[mCodes[0]]), len(dfs[mCodes[1]]))
# 根据配置文件获取最佳交易手数对
self.iniVolume = round(
self.iniAmount / cs0["lastPrice"] / cs0["contract_multiplier"], 0)
# 分参数执行
results = []
for period in self.periodList:
for scale in self.scaleList:
for ss in self.shiftScaleList:
self.shiftScale = ss
self.shift = [sh[i] * ss for i in range(len(sh))]
doc = self.total(dfs, dfs_l, scale=scale, period=period)
if doc is None or doc['amount'] == 0: continue
logger.info((doc['uid'], doc['income'], " time:",
time.time() - time0))
results.append(doc)
# 结果写入共享阈
#shareDict.append(doc)
self.Total.insertAll(results)
def start(self, codes, time0, kt, shareDict):
print("子进程启动:", self.cindex, codes, time.time() - time0)
self.klineType = kt
# 主力合约
self.codes = codes
self.mCodes = mCodes = [n + '88' for n in codes]
# 查询获得配置 - 费率和每手单量
self.Base = future_baseInfo()
for doc in self.Base.getInfo(codes):
self.baseInfo[doc["code"] + '88'] = doc
cs0 = self.baseInfo[mCodes[0]]
# 计算tick 导致的滑点
sh = [self.baseInfo[d + '88']['tick_size'] for d in codes]
self.shift = [sh[i] * self.shiftScale for i in range(len(sh))]
# 子进程共享类
self.Rice = tick_csv_Rice()
self.Rice.setTimeArea(cs0["nightEnd"])
self.Train = train_future()
self.Total = train_total()
self.Total.tablename = self.total_tablename
# 查询获得N分钟K线
dfs_l = self.Rice.kline(mCodes,
period=self.klineType,
start=self.startDate,
end=self.endDate,
pre=60)
# 获得1分钟K线作为滚动线
dfs = self.Rice.kline(mCodes,
period='1m',
start=self.startDate,
end=self.endDate,
pre=0)
# 按时间截取并调整
# dfs= self.dateAdjust(codes, dfs, sh)
print('kline load:', mCodes, len(dfs[mCodes[0]]))
# 根据配置文件获取最佳交易手数对
self.iniVolume = round(
self.iniAmount / cs0["lastPrice"] / cs0["contract_multiplier"], 0)
# 分参数执行
results = []
for period in self.periodList:
docs = self.total(dfs, dfs_l, period=period)
if docs is None or len(docs) == 0: continue
logger.info(
(self.codes, period, self.klineType, len(docs), " time:",
time.time() - time0))
results.extend(docs)
# 结果写入共享阈
self.Total.insertAll(results)
def start(self, codes, share):
print("子进程:", codes)
self.time0 = time0 = time.time()
self.Base = future_baseInfo()
self.Rice = tick_csv_Rice()
self.Rice.shareDict = share
self.Rice.basePath = self.basePath
self.Rice.cvs_start = self.cvs_start
self.Rice.cvs_end = self.cvs_end
self.Rice.dd = self.dd
self.new_docs = []
self.codes = codes
self.baseInfo = {}
self.filePath = self.basePath + "%s.csv" % "_".join(codes)
# 查询获得配置 - 费率和每手单量
for doc in self.Base.getInfo(codes):
self.baseInfo[doc["code"]] = doc
cs0, cs1 = self.baseInfo[codes[0]], self.baseInfo[codes[1]]
if cs0["nightEnd"] != cs1["nightEnd"]: return None
# 设置结束时间,用于数据清洗
try:
res = self.Rice.curKlineByCsv(codes, period=1, callback=self.new_tick)
except Exception as e:
print('error', codes, len(self.new_docs), e )
res = True
if res and len(self.new_docs) > 0 :
# 保存文件
df = pd.DataFrame(self.new_docs)
# 数据整理
df = self.detail(df)
print(public.getDatetime(), " 已保存", codes, "耗时:", time.time()-time0)
df.to_csv(self.filePath, columns=self.columns1, index=0)
self.new_docs= []
return True
def reduce(self, c, share):
Rice = tick_csv_Rice()
Base = future_baseInfo()
Rice.codes = [c]
s = self.dd
file = self.basePath + "%s%s.csv" % (c, s[0])
df = pd.read_csv(file)
# 设置结束时间,用于数据清洗
for doc in Base.getInfo([c]):
ne = doc["nightEnd"]
Rice.setTimeArea(ne)
df = Rice.clean(df)
len0 = len(df)
df = df[(df['datetime'] > self.cvs_start) & (df['datetime'] < self.cvs_end)]
print(c,len0, len(df))
df.to_csv(file, columns= Rice.cvs_columns, index=0)
def Pool(self):
time0 = time.time()
pool = Pool(processes=4)
shareDict = Manager().list([])
# 从现有ticks文件获取列表
Rice = tick_csv_Rice()
Rice.setParam(model_future_quickTickTest().cvsParam)
ticks = Rice.exitTicks() # 已存在的ticks
for cs in ticks:
rs = cs.split("_")
for kt in self.klineTypeList:
#self.start(rs, time0, kt , shareDict)
try:
pool.apply_async(self.start, (rs, time0, kt, shareDict))
pass
except Exception as e:
print(e)
continue
pool.close()
pool.join()
def compare(self, codes, share):
# ticks
columns =['datetime','price_t','p_l_t','p_h_t']
self.Rice = tick_csv_Rice()
ticks = self.Rice.getTicks(codes)
ticks = ticks[ticks['isnew'] == 1]
cvs_start = ticks["datetime"].values[0]
cvs_end = ticks["datetime"].values[-1]
#
key = "_".join(codes)
ticks["datetime"] = ticks["datetime"].apply(lambda x : x[0:-4] if x.find('.')> 0 else x)
ticks["datetime"] = ticks["datetime"].str.replace(':01$|:02$|:03$|:04$',':00')
ticks["price_t"] = ticks["close"]
ticks["p_l_t"] = ticks["a1"] / ticks["n_b1"]
ticks["p_h_t"] = ticks["b1"] / ticks["n_a1"]
ticks = self.dfRound(ticks,5,['close','p_l','p_h'])
ticks.to_csv(self.basePath + "tick_%s.csv" % key, columns=columns, index=0)
Base = future_baseInfo()
mCodes = [ c+'88' for c in codes]
(b0, b1) =(doc['tick_size'] for doc in Base.getInfo(codes))
print(b0,b1)
dfs = self.Rice.kline(mCodes, start=cvs_start, end=cvs_end)
df0, df1= dfs[mCodes[0]], dfs[mCodes[1]]
df0 = df0[(df0.index >= cvs_start) & (df0.index <= cvs_end)]
df0["datetime"] = df0.index
df0["datetime"] = df0["datetime"].apply(str)
c0, c1 = df0["close"], df1["close"]
df0["price"] = c0 / c1
shift = ((c0+b0)/(c1-b1)-(c0-b0)/(c1+b1))/2
df0["p_l"] = c0/c1 + shift
df0["p_h"] = c0/c1 - shift
df0["p_l_1"] = (c0+b0/2)/(c1-b1/2)
df0["p_h_1"] = (c0-b0/2)/(c1+b1/2)
df0 = self.dfRound(df0, 5, ['price', 'p_l', 'p_h', 'p_l_1', 'p_h_1'])
df0 =pd.merge(df0, ticks, on='datetime', how='left')
columns = ['datetime_t', 'datetime', 'price_t', 'price',
'p_l_t', 'p_l' , 'p_l_1', 'p_h_t', 'p_h' , 'p_h_1']
res = {}
res['p_l'] = df0['p_l'].mean() - df0['p_l_t'].mean()
res['p_l_1'] = df0['p_l_1'].mean() - df0['p_l_t'].mean()
res['p_h'] = df0['p_h'].mean() - df0['p_h_t'].mean()
res['p_h_1'] = df0['p_h_1'].mean() - df0['p_h_t'].mean()
print(res)
df0.to_csv(self.basePath + "total_%s.csv" % key, columns=columns, index=0)
def start(self, map, kPeriod, shareDict):
# 每个进程单独初始化对象
self.time0 = time.time()
self.Record = train_future()
# 数据库存储
self.Record.tablename = self.tablename
self.Rice = tick_csv_Rice()
self.Rice.setParam(self.cvsParam)
self.Rice.shareDict = self.shareDict = shareDict
self.Rice.kPeriod = self.kPeriod = kPeriod
self.Base = future_baseInfo()
self.timeArea = [("09:00:00", "11:30:00"), ("13:30:00", "15:00:00")]
self.codes = map[0:2]
# 查询基础配置
for doc in self.Base.getInfo(self.codes):
self.config[doc["code"]] = doc
c0, c1 = self.config[self.codes[0]], self.config[self.codes[1]]
# 设置交易对的收盘时间,用于计算收盘时间并计算止损
self.Rice.setTimeArea(doc["nightEnd"])
# 根据配置文件获取最佳交易手数对
self.iniVolume = round(
self.iniAmount / c0["lastPrice"] / c0["contract_multiplier"], 0)
# 设置shift区间
[t, c] = [[self.config[d][key] for d in self.codes]
for key in ["tick_size", "lastPrice"]]
self.shift = ((c[0] + t[0]) / (c[1] - t[1]) - (c[0] - t[0]) /
(c[1] + t[1])) / self.shiftScale
logger.info(("model_future_detect start: %s " % "_".join(self.codes),
self.uidKey, self.Rice.TimeArea))
# 启动进程
ticks = self.Rice.getTicks(self.codes)
# tick类型选择
if self.tickType != 'none':
ticks = ticks[ticks[self.tickType] == 1]
ticks["p_l"] = ticks["a1"] / ticks["n_b1"]
ticks["p_h"] = ticks["b1"] / ticks["n_a1"]
indexR, records = 0, []
# 及时K线
for index, row in ticks.iterrows():
#print(row["datetime"])
res = self.onTick(row, isdelay=self.tickType != 'isnew')
if len(res) > 0:
records.extend(res)
# 阶段性存入到数据库，并清空
if len(records) > 50:
self.Record.insertAll(records)
indexR += len(records)
print(public.getDatetime(), " insert into record:", self.codes,
self.kPeriod, indexR)
records = []
# 将剩下的写入数据库
if len(records) > 0:
self.Record.insertAll(records)
