def prepareData(self,codeList,startDate,endDate):
tradedays=TradedayDataProcess.getTradedays(startDate,endDate)
myfactor=tickFactorsProcess()
for day in tradedays:
data=myfactor.parallelizationGetDataByDate(codeList,day)
data=data[(data['tick']>='093000000') & (data['tick']<'145700000')]
tickColumns=[ 'code', 'date', 'tick', 'lastPrice', 'S1', 'S2','S3', 'S4', 'S5', 'S6', 'S7','S8', 'S9', 'S10', 'B1', 'B2', 'B3', 'B4','B5', 'B6', 'B7', 'B8', 'B9', 'B10', 'SV1', 'SV2', 'SV3', 'SV4', 'SV5','SV6', 'SV7', 'SV8', 'SV9', 'SV10', 'BV1', 'BV2', 'BV3', 'BV4', 'BV5','BV6', 'BV7', 'BV8', 'BV9', 'BV10', 'volume', 'amount','volumeIncrease', 'amountIncrease', 'midPrice']
#dailyColumns=['increaseToday','closeStd20','ts_closeStd20','preClose','is300','is500']
mycolumns=list(set(data.columns).difference(set(tickColumns)))
mycolumns=mycolumns+['code', 'date', 'tick']
data=data[mycolumns]
#print(data.shape)
errorData=data[data[mycolumns].isna().sum(axis=1)>0]
if errorData.shape[0]>0:
logger.warning(f'factorData of date {day} has Nan!!!')
#对部分因子进行去极值操作
#data=self.dataSelect(data,0.2)
#逐日存储数据
fileName=os.path.join(self.path,str(day)+'.h5')
exists=os.path.exists(fileName)
if exists==True:
os.remove(fileName)
try:
with pd.HDFStore(fileName,'a',complib='blosc:zstd',append=True,complevel=9) as store:
store.append('data',data,append=True,format="table",data_columns=data.columns)
except Exception as excp:
logger.error(f'{fileName} error! {excp}')
pass
pass
def singleCode(self, code, startDate, endDate, parameters):
days = list(TradedayDataProcess().getTradedays(startDate, endDate))
tick = tickFactorsProcess()
select = [
'code', 'date', 'tick', 'B1', 'S1', 'midIncreasePrevious3m',
'closeStd20', 'ts_closeStd20', 'ts_buySellForceChange',
'ts_buySellVolumeRatio5', 'ts_buySellVolumeRatio2', 'preClose',
'increaseToday', 'weight300', 'weight500'
]
trade = []
for day in days:
data = tick.getTickDataAndFactorsByDateFromLocalFile(code, day)
if data.shape[0] > 0:
trade0 = self.__strategy(data, select)
trade.append(trade0)
pass
if trade == []:
trade = pd.DataFrame()
else:
trade = pd.concat(trade)
return trade
pass
def reverse_singleCode(self, code, startDate, endDate):
days = list(TradedayDataProcess().getTradedays(startDate, endDate))
#factors=['closeStd','index','marketValue','industry']
dailyRepo = dailyFactorsProcess()
dailyFactor = dailyRepo.getSingleStockDailyFactors(
code, startDate, endDate)
dailyKLine = KLineDataProcess('daily')
dailyData = dailyKLine.getDataByDate(code, startDate, endDate)
if dailyData.empty == True:
logger.error(
f'there is no data of {code} from {startDate} to {endDate}')
return pd.DataFrame()
tick = TickDataProcess()
tickfactor = tickFactorsProcess()
mydata = []
position = 0
profit = 0
myStatusList = []
myTradeList = []
myStatus = {}
myTrade = {}
positionYesterday = 0
select1 = []
select2 = []
selectall = []
for today in days:
#logger.info(f'{code} in {today} start!')
todayInfo = dailyFactor[dailyFactor['date'] == today]
if todayInfo.empty == True:
logger.error(
f'there is no factor data of {code} in date {today}')
continue
pass
todayKLine = dailyData[dailyData['date'] == today]
if todayKLine.empty == True:
logger.error(
f'there is no KLine data of {code} in date {today}')
continue
pass
myStatus['date'] = today
myStatus['closeStd20'] = todayInfo['closeStd20'].iloc[0]
myStatus['weight50'] = todayInfo['weight50'].iloc[0]
myStatus['weight300'] = todayInfo['weight300'].iloc[0]
myStatus['weight500'] = todayInfo['weight500'].iloc[0]
myStatus['ts_closeStd20'] = todayInfo['closeStd20'].iloc[0]
myStatus['adjFactor'] = todayKLine['adjFactor'].iloc[0]
myStatus['preClose'] = todayKLine['preClose'].iloc[0]
positionNow = positionYesterday
if (todayInfo.empty
== False) & (todayKLine['status'].iloc[0] != '停牌'):
#tickData=tick.getDataByDateFromLocalFile(code,today)
tickData = tick.getTickShotDataFromInfluxdbServer(code, today)
#factors=tickfactor.getDataByDateFromLocalFile(code,today,'buySellForce')
factors = tickfactor.getFactorsUsedByDateFromLocalFile(
code, today)
all = pd.merge(tickData,
factors,
how='left',
left_index=True,
right_index=True)
all['closeStd20'] = todayInfo['closeStd20'].iloc[0]
all['ts_closeStd20'] = todayInfo['ts_closeStd20'].iloc[0]
all['adjFactor'] = todayKLine['adjFactor'].iloc[0]
all['preClose'] = todayKLine['preClose'].iloc[0]
all['increaseToday'] = all['midPrice'] / all['preClose'] - 1
all['midIncreasePrevious5m'] = all['midPrice'] / all[
'midPrice'].shift(60) - 1
#select0=all[all['increaseToday']>2*all['closeStd20']]
selectall.append(all)
select0 = all[all['midIncreasePrevious5m'] > 0.5 *
all['closeStd20']]
select1.append(select0)
select0 = all[all['midIncreasePrevious5m'] < -0.5 *
all['closeStd20']]
select2.append(select0)
pass
else:
logger.warning(f'There is no data of {code} in {today}')
mycolumns = [
'midIncreaseNext1m', 'midIncreaseNext5m', 'ts_buyForceIncrease',
'ts_sellForceIncrease', 'ts_buySellVolumeRatio5',
'midIncreasePrevious3m', 'differenceHighLow',
'ts_differenceHighLow', 'differenceMidVwap',
'ts_differenceMidVwap', 'midStd60', 'ts_midStd60'
]
select1 = pd.concat(select1)
select2 = pd.concat(select2)
selectall = pd.concat(selectall)
selectall = selectall[mycolumns]
s = select1[(select1['ts_sellForceIncrease'] > 0.6)
& (select1['ts_buySellVolumeRatio5'] > 0.6) &
(select1['increaseToday'] > select1['closeStd20'])]
print(s['midIncreaseNext5m'].mean())
ss = select2[(select2['ts_buySellVolumeRatio5'] > 0.6)
& (select2['increaseToday'] < -select2['closeStd20'])]
print(ss['midIncreaseNext5m'].mean())
return mydata
########################################################################
def singleCode(self,
code: str,
startDate: str,
endDate: str,
parameters=[]):
days = list(TradedayDataProcess().getTradedays(startDate, endDate))
tickFactors = tickFactorsProcess()
tick = TickDataProcess()
daily = dailyFactorsProcess()
dailyKLine = KLineDataProcess('daily')
trade = []
for day in days:
#tickData=tick.getDataByDateFromLocalFile(code,day)
tickData = tick.getTickShotDataFromInfluxdbServer(code, day)
if tickData.shape[0] == 0:
continue
data = tickFactors.getTickFactorsOnlyByDateFromLocalFile(code, day)
data = pd.merge(data,
tickData,
how='left',
left_index=True,
right_index=True)
dailyData = daily.getSingleStockDailyFactors(code, day, day)
for col in dailyData.columns:
if col not in ['date', 'code']:
data[col] = dailyData[col].iloc[0]
dailyKLineData = dailyKLine.getDataByDate(code, day, day)
data['preClose'] = dailyKLineData['preClose'].iloc[0]
data['increaseToday'] = data['midPrice'] / data['preClose'] - 1
if np.isnan(data['weight300'].iloc[0]) == True:
continue
maxPosition = round(
data['weight300'].iloc[0] * 100000000 /
data['preClose'].iloc[0] / 100, -2)
features = [
'buyForce', 'sellForce', 'buySellForceChange', 'buySellSpread',
'differenceHighLow3m', 'midToVwap', 'midToVwap3m',
'midPrice3mIncrease', 'midPriceBV3m', 'midInPrevious3m',
'midStd60', 'increaseToday', 'closeStd20',
'buySellVolumeRatio2', 'buySellWeightedVolumeRatio2',
'buySellVolumeRatio5', 'buySellWeightedVolumeRatio5',
'buySellVolumeRatio10', 'buySellWeightedVolumeRatio10'
]
A = data[features]
A = self.dataSelect(A, 0.2)
A = A.values
maxWeight = np.array([
0.03218688, -0.0121024, -0.00970715, 0.48172206, 0.42610642,
0.10048948, -0.05574053, 0.08212702, -0.12357012, -0.00123216,
0.09529259, 0.00509518, 0.14970625, -0.00291313, 0.00402094,
-0.00452788, 0.00286216, 0.0020172, -0.00235546
])
minWeight = np.array([
-0.00385887, -0.01163938, 0.0043455, -0.01114819, -0.34286923,
0.08314041, 0.00154458, 0.12249813, -0.02194375, -0.00038749,
-0.02217015, 0.00610296, -0.09264385, -0.0020065, 0.00249547,
-0.00324293, 0.00501176, 0.00389697, -0.00294958
])
maxIntercept = 0.00079871
minIntercept = -0.00155935
mymax = A.dot(maxWeight) + maxIntercept
mymin = A.dot(minWeight) + minIntercept
data['maxPredict'] = mymax
data['minPredict'] = mymin
data['maxPredict'] = data['maxPredict'].ewm(span=2,
ignore_na=True,
adjust=True).mean()
data['minPredict'] = data['minPredict'].ewm(span=2,
ignore_na=True,
adjust=True).mean()
data['midPredict'] = (data['maxPredict'] + data['minPredict']) / 2
m = data[[
'midIncreaseMinNext5m', 'midIncreaseMaxNext5m', 'maxPredict',
'minPredict', 'midPredict'
]]
print(m.corr())
#long=data[(data['maxPredict']>0.01)]['midIncreaseMaxNext5m'].mean()-data['midIncreaseMaxNext5m'].mean()
#short=data[(data['minPredict']<-0.01)]['midIncreaseMinNext5m'].mean()-data['midIncreaseMinNext5m'].mean()
#print(long)
#print(short)
mycolumns = list(tickData.columns)
mycolumns.append('maxPredict')
mycolumns.append('minPredict')
data = data[mycolumns]
parameters = {
'maxPosition': maxPosition,
'longOpen': 0.015,
'shortOpen': -0.015,
'longClose': 0.01,
'shortClose': -0.01,
'transactionRatio': 0.2
}
#trade0=self.strategy(data,parameters)
#trade.append(trade0)
pass
if len(trade) == 0:
trade = pd.DataFrame()
else:
trade = pd.concat(trade)
trade['code'] = code
trade['fee'] = trade['price'] * 0.0001
selectBuy = trade['direction'] == 'buy'
selectSell = trade['direction'] == 'sell'
trade.loc[selectSell, 'fee'] = (trade['fee'] +
trade['price'] * 0.001)[selectSell]
trade.loc[selectBuy,
'cashChange'] = ((-trade['price'] - trade['fee']) *
trade['volume'])[selectBuy]
trade.loc[selectSell,
'cashChange'] = ((trade['price'] - trade['fee']) *
trade['volume'])[selectSell]
trade['amount'] = trade['price'] * trade['volume']
return trade
pass
def singleCode(self,
code: str,
startDate: str,
endDate: str,
parameters=[]):
days = list(TradedayDataProcess().getTradedays(startDate, endDate))
tickFactors = tickFactorsProcess()
tick = TickDataProcess()
daily = dailyFactorsProcess()
dailyKLine = KLineDataProcess('daily')
file = os.path.join(LocalFileAddress, 'tmp',
'dnn001_midIncreaseNext5m.h5')
trade = []
for day in days:
tickData = tick.getDataByDateFromLocalFile(code, day)
if tickData.shape[0] == 0:
continue
data = tickFactors.getTickFactorsOnlyByDateFromLocalFile(code, day)
data = pd.merge(data,
tickData,
how='left',
left_index=True,
right_index=True)
dailyData = daily.getSingleStockDailyFactors(code, day, day)
for col in dailyData.columns:
if col not in ['date', 'code']:
data[col] = dailyData[col].iloc[0]
dailyKLineData = dailyKLine.getDataByDate(code, day, day)
data['preClose'] = dailyKLineData['preClose'].iloc[0]
data['increaseToday'] = data['midPrice'] / data['preClose'] - 1
if np.isnan(data['weight300'].iloc[0]) == True:
continue
maxPosition = round(
data['weight300'].iloc[0] * 100000000 /
data['preClose'].iloc[0] / 100, -2)
features = [
'buyForce', 'sellForce', 'buySellForceChange', 'buySellSpread',
'differenceHighLow3m', 'midToVwap', 'midToVwap3m',
'midPrice3mIncrease', 'midPriceBV3m', 'midInPrevious3m',
'midStd60', 'increaseToday', 'closeStd20',
'buySellVolumeRatio2', 'buySellWeightedVolumeRatio2',
'buySellVolumeRatio5', 'buySellWeightedVolumeRatio5',
'buySellVolumeRatio10', 'buySellWeightedVolumeRatio10'
]
A = data[features]
A = self.dataSelect(A, 0.2)
A = A.values
warnings.filterwarnings('ignore')
model = keras.models.load_model(file)
predictArray = model.predict(A, verbose=0)
mymin = predictArray[:, 0]
mymax = predictArray[:, 1]
data['maxPredict'] = mymax
data['minPredict'] = mymin
data['maxPredict'] = data['maxPredict'].ewm(span=2,
ignore_na=True,
adjust=True).mean()
data['minPredict'] = data['minPredict'].ewm(span=2,
ignore_na=True,
adjust=True).mean()
data['midPredict'] = (data['maxPredict'] + data['minPredict']) / 2
m = data[[
'midIncreaseMinNext5m', 'midIncreaseMaxNext5m', 'maxPredict',
'minPredict', 'midPredict'
]]
pd.set_option('display.max_rows', None)
#print(m.corr())
#long=data[(data['maxPredict']>0.003)]['midIncreaseMaxNext5m'].mean()-data['midIncreaseMaxNext5m'].mean()
#short=data[(data['minPredict']<-0.003)]['midIncreaseMinNext5m'].mean()-data['midIncreaseMinNext5m'].mean()
#print(long)
#print(short)
mycolumns = list(tickData.columns)
mycolumns.append('maxPredict')
mycolumns.append('minPredict')
mycolumns.append('midPredict')
mycolumns.append('increaseToday')
mycolumns.append('midInPrevious3m')
data = data[mycolumns]
parameters = {
'maxPosition': maxPosition,
'longOpen': 0.015,
'shortOpen': -0.015,
'longClose': 0.01,
'shortClose': -0.01,
'transactionRatio': 0.1
}
trade0 = self.strategy(data, parameters)
trade.append(trade0)
pass
if len(trade) == 0:
trade = pd.DataFrame()
else:
trade = pd.concat(trade)
if trade.shape[0] == 0:
return pd.DataFrame()
trade['code'] = code
trade['fee'] = trade['price'] * 0.0001
selectBuy = trade['direction'] == 'buy'
selectSell = trade['direction'] == 'sell'
trade.loc[selectSell, 'fee'] = (trade['fee'] +
trade['price'] * 0.001)[selectSell]
trade.loc[selectBuy,
'cashChange'] = ((-trade['price'] - trade['fee']) *
trade['volume'])[selectBuy]
trade.loc[selectSell,
'cashChange'] = ((trade['price'] - trade['fee']) *
trade['volume'])[selectSell]
trade['amount'] = trade['price'] * trade['volume']
return trade
pass