def getParallelResult(strategyParameter,resultpath,parasetlist,paranum):
strategyName = strategyParameter['strategyName']
exchange_id = strategyParameter['exchange_id']
sec_id = strategyParameter['sec_id']
K_MIN = strategyParameter['K_MIN']
startdate = strategyParameter['startdate']
enddate = strategyParameter['enddate']
symbol = '.'.join([exchange_id, sec_id])
# ======================数据准备==============================================
# 取合约信息
symbolInfo = DC.SymbolInfo(symbol)
# 取跨合约数据
contractswaplist = DC.getContractSwaplist(symbol)
swaplist = np.array(contractswaplist.swaputc)
# 取K线数据
rawdata = DC.getBarData(symbol, K_MIN, startdate + ' 00:00:00', enddate + ' 23:59:59').reset_index(drop=True)
foldername = ' '.join([strategyName, exchange_id, sec_id, str(K_MIN)])
try:
os.chdir(resultpath)
os.mkdir(foldername)
except:
print ("%s folder already exsist!" %foldername)
os.chdir(foldername)
# 多进程优化，启动一个对应CPU核心数量的进程池
pool = multiprocessing.Pool(multiprocessing.cpu_count() - 1)
l = []
resultlist = pd.DataFrame(columns=
['Setname', 'MA_Short', 'MA_Long','DMI_N', 'opentimes', 'end_cash', 'SR', 'Annual','Sharpe','DrawBack',
'max_single_loss_rate'])
for i in range(0, paranum):
setname = parasetlist.ix[i, 'Setname']
ma_short = parasetlist.ix[i, 'MA_Short']
ma_long = parasetlist.ix[i, 'MA_Long']
dmi_n = parasetlist.ix[i,'DMI_N']
paraSet = {
'Setname':setname,
'MA_Short': ma_short,
'MA_Long': ma_long,
'DMI_N':dmi_n,
'DMI_M':6
}
l.append(pool.apply_async(getResult, (symbolInfo, K_MIN, setname, rawdata, paraSet, swaplist)))
pool.close()
pool.join()
# 显示结果
i = 0
for res in l:
resultlist.loc[i] = res.get()
i += 1
print resultlist
finalresults=("%s %d finalresults.csv"%(symbol,K_MIN))
resultlist.to_csv(finalresults)
return resultlist
def calOprResult(strategyName,rawpath,symbolinfo,K_MIN,nextmonth,columns,barxmdic,positionRatio,initialCash,indexcols,indexcolsFlag,resultfilesuffix='result.csv'):
'''
根据灰区的取值，取出各灰区的操作列表，组成目标集组的操作表，并计算各个评价指标
:return:
'''
symbol=symbolinfo.domain_symbol
graydf=pd.read_csv(rawpath+'ForwardOprAnalyze\\'+strategyName+' '+symbol+str(K_MIN)+'multiTargetForwardSetname.csv',index_col='Group')
cols = graydf.columns.tolist()[3:]
cols.append(nextmonth)
groupResult = []
closetime_col = columns['closetime_col']
closeindex_col = columns['closeindex_col']
closeprice_col = columns['closeprice_col']
closeutc_col = columns['closeutc_col']
retr_col = columns['retr_col']
ret_col = columns['ret_col']
cash_col = columns['cash_col']
hands_col = columns['hands_col']
for i in range(graydf.shape[0]):
gray=graydf.iloc[i]
oprdf = pd.DataFrame(columns=['opentime', 'openutc', 'openindex', 'openprice', closetime_col, closeutc_col, closeindex_col,
closeprice_col, 'tradetype', ret_col, retr_col, 'symbol'])
print gray.name,gray.Target,gray.Windows
for l in range(len(cols)-1):
startmonth=cols[l]
endmonth=cols[l+1]
setname=gray[startmonth]
oprdf=pd.concat([oprdf,getOprlistByMonth(strategyName,rawpath,symbol,K_MIN,setname,startmonth,endmonth,columns,resultfilesuffix)])
oprdf=oprdf.reset_index(drop=True)
oprdf['commission_fee'], oprdf['per earn'], oprdf[cash_col], oprdf[hands_col] = RS.calcResult(
oprdf,
symbolinfo,
initialCash,
positionRatio, ret_col)
tofilename=('%s %s%d_%s_win%d_oprResult.csv'%(strategyName,symbol,K_MIN,gray.Target,gray.Windows))
oprdf.to_csv(rawpath+'ForwardOprAnalyze\\'+tofilename)
symbolDomainDic = symbolinfo.amendSymbolDomainDicByOpr(oprdf, closeutc_col=closeutc_col)
barxm = DC.getDomainbarByDomainSymbol(symbolinfo.getSymbolList(), barxmdic, symbolDomainDic)
dailyK = DC.generatDailyClose(barxm) # 生成按日的K线
dR = RS.dailyReturn(symbolinfo, oprdf, dailyK, initialCash) # 计算生成每日结果
dR.calDailyResult()
tofilename = ('%s %s%d_%s_win%d_oprdailyResult.csv' % (strategyName, symbol, K_MIN, gray.Target, gray.Windows))
dR.dailyClose.to_csv(rawpath+'ForwardOprAnalyze\\'+tofilename)
r=RS.getStatisticsResult(oprdf,indexcolsFlag,indexcols,dR.dailyClose)
groupResult.append([gray.name,gray.Target,gray.Windows]+r)
groupResultDf=pd.DataFrame(groupResult,columns=['Group','Target','Windows']+indexcols)
groupResultDf.to_csv(rawpath+'ForwardOprAnalyze\\'+strategyName+' '+symbol+'_'+str(K_MIN)+'_groupOprResult.csv', index=False)
pass
def calDailyReturn():
'''基于已有的opr结果，重新补充计算dailyReturn'''
startdate = '2016-01-01'
enddate = '2018-04-30'
symbol = 'SHFE.RB'
K_MIN = 3600
symbolinfo = DC.SymbolInfo(symbol, startdate, enddate)
strategyName = Parameter.strategyName
#rawdata = DC.getBarData(symbol, K_MIN, startdate + ' 00:00:00', enddate + ' 23:59:59').reset_index(drop=True)
#dailyK = DC.generatDailyClose(rawdata)
bardic = DC.getBarBySymbolList(symbol, symbolinfo.getSymbolList(), K_MIN,
startdate, enddate)
upperpath = DC.getUpperPath(Parameter.folderLevel)
resultpath = upperpath + Parameter.resultFolderName
os.chdir(resultpath)
foldername = ' '.join(
[strategyName, Parameter.exchange_id, Parameter.sec_id,
str(K_MIN)])
#foldername = ' '.join([strategyName, Parameter.exchange_id, Parameter.sec_id, str(K_MIN)])+'\\DynamicStopLoss-18.0\\'
#foldername = ' '.join([strategyName, Parameter.exchange_id, Parameter.sec_id, str(K_MIN)])+'\\OnceWinNoLoss8.0\\'
#foldername = ' '.join([strategyName, Parameter.exchange_id, Parameter.sec_id, str(K_MIN)])+'\\dsl_-0.020ownl_0.008\\'
os.chdir(foldername)
parasetlist = pd.read_csv(resultpath + Parameter.parasetname)
paranum = parasetlist.shape[0]
filesuffix = 'result.csv'
#filesuffix = 'resultDSL_by_tick.csv'
#filesuffix = 'resultOWNL_by_tick.csv'
#filesuffix = 'result_multiSLT.csv'
indexcols = Parameter.ResultIndexDic
resultList = []
for i in range(paranum):
setname = parasetlist.ix[i, 'Setname']
print setname
oprdf = pd.read_csv(strategyName + ' ' + symbolinfo.domain_symbol +
str(K_MIN) + ' ' + setname + ' ' + filesuffix)
symbolDomainDic = symbolinfo.amendSymbolDomainDicByOpr(oprdf)
bars = DC.getDomainbarByDomainSymbol(symbolinfo.getSymbolList(),
bardic, symbolDomainDic)
dailyK = DC.generatDailyClose(bars)
dR = RS.dailyReturn(symbolinfo, oprdf, dailyK, Parameter.initialCash)
dR.calDailyResult()
dR.dailyClose.to_csv(strategyName + ' ' + symbolinfo.domain_symbol +
str(K_MIN) + ' ' + setname + ' daily' +
filesuffix)
results = RS.getStatisticsResult(oprdf, False, indexcols,
dR.dailyClose)
print results
resultList.append([setname] + results) # 在这里附上setname
resultdf = pd.DataFrame(resultList, columns=['Setname'] + indexcols)
resultdf.to_csv("%s %s %d finalresults.csv" %
(strategyName, symbol, K_MIN))
def volumeProcess1m(symbol, startdate, enddate):
'''
将tick数据中的交易量和持仓量数据转换成1m级别,保存到volume data文件夹中
:param symbol:
:param startdate:
:param enddate:
:return:
'''
#读取tick数据
exchange_id, sec_id = symbol.split('.', 1)
tradedates = DC.getTradedates(exchange_id, startdate, enddate)['strtime']
firstdayTick = DC.getTickByDate(symbol, startdate)
last_delta_position = firstdayTick['cum_position'].iloc[0]
last_cum_position_long = 0
last_cum_position_short = 0
last_delta_volume = firstdayTick['cum_volume'].iloc[0]
results = pd.DataFrame()
#1.先将tick数据处理
for td in tradedates:
print("process tick data of %s %s" % (symbol, td))
tickdata = DC.getTickByDate(symbol, td)
result,last_delta_position,last_cum_position_long,last_cum_position_short,last_delta_volume=\
volumeProcess(tickdata,last_delta_position,last_cum_position_long,last_cum_position_short,last_delta_volume)
results = pd.concat([results, result])
#2.取1m数据进行对齐
bardata = DC.getBarData(symbol=symbol,
K_MIN=60,
starttime=startdate + ' 00:00:00',
endtime=enddate + ' 23:59:59')
bardata.set_index('utc_time', inplace=True)
results = results.loc[bardata.index]
results['bar_type'] = bardata['bar_type']
results['exchange'] = bardata['exchange']
results['sec_id'] = bardata['sec_id']
results['strtime'] = bardata['strtime']
results['strendtime'] = bardata['strendtime']
results['utc_endtime'] = bardata['utc_endtime']
results['close'] = bardata['close']
results.reset_index(drop=False, inplace=True)
os.chdir(DC.VOLUME_DATA_PATH)
try:
os.mkdir(symbol)
except:
print("%s folder already exist!")
results.to_csv("\\%s\\%s %d_volume.csv" % (symbol, symbol, 60))
print("%s tick volume processed from %s to %s!" %
(symbol, startdate, enddate))
def getForward(strategyName, symbolinfo, K_MIN, parasetlist, rawdatapath,
startdate, enddate, nextmonth, windowsSet, colslist,
result_para_dic, indexcolsFlag, resultfilesuffix):
symbol = symbolinfo.domain_symbol
forwordresultpath = rawdatapath + '\\ForwardResults\\'
forwardrankpath = rawdatapath + '\\ForwardRank\\'
monthlist = [
datetime.strftime(x, '%Y-%m')
for x in list(pd.date_range(start=startdate, end=enddate, freq='M'))
]
monthlist.append(nextmonth)
os.chdir(rawdatapath)
try:
os.mkdir('ForwardResults')
except:
print 'ForwardResults already exist!'
try:
os.mkdir('ForwardRank')
except:
print 'ForwardRank already exist!'
try:
os.mkdir('ForwardOprAnalyze')
except:
print 'ForwardOprAnalyze already exist!'
starttime = datetime.now()
print starttime
# 多进程优化，启动一个对应CPU核心数量的进程池
initialCash = result_para_dic['initialCash']
positionRatio = result_para_dic['positionRatio']
pool = multiprocessing.Pool(multiprocessing.cpu_count() - 1)
l = []
for whiteWindows in windowsSet:
# l.append(mtf.runPara(strategyName,whiteWindows, symbolinfo, K_MIN, parasetlist, monthlist, rawdatapath, forwordresultpath, forwardrankpath, colslist, resultfilesuffix))
l.append(
pool.apply_async(
mtf.runPara,
(strategyName, whiteWindows, symbolinfo, K_MIN, parasetlist,
monthlist, rawdatapath, forwordresultpath, forwardrankpath,
colslist, resultfilesuffix)))
pool.close()
pool.join()
mtf.calGrayResult(strategyName, symbol, K_MIN, windowsSet, forwardrankpath,
rawdatapath)
indexcols = Parameter.ResultIndexDic
#rawdata = DC.getBarData(symbol, K_MIN, monthlist[12] + '-01 00:00:00', enddate + ' 23:59:59').reset_index(drop=True)
cols = [
'open', 'high', 'low', 'close', 'strtime', 'utc_time', 'utc_endtime'
]
barxmdic = DC.getBarDic(symbolinfo, K_MIN, cols)
mtf.calOprResult(strategyName, rawdatapath, symbolinfo, K_MIN, nextmonth,
colslist, barxmdic, positionRatio, initialCash, indexcols,
indexcolsFlag, resultfilesuffix)
endtime = datetime.now()
print starttime
print endtime
def add_max_period_cash_to_finalresult():
upperpath = DC.getUpperPath(Parameter.folderLevel)
resultpath = upperpath + Parameter.resultFolderName
os.chdir(resultpath)
para_set_list = pd.read_csv('ParameterSet_HullRsi.csv')['Setname'].tolist()
multi_symbol_df = pd.read_excel('HullRsiWin_symbol_KMIN_set.xlsx')
for n, row in multi_symbol_df.iterrows():
strategy_name = row['strategyName']
exchange_id = row['exchange_id']
sec_id = row['sec_id']
bar_type = row['K_MIN']
folder_name = "%s %s %s %d\\" % (strategy_name, exchange_id, sec_id,
bar_type)
result_file_name = "%s %s.%s %d finalresults.csv" % (
strategy_name, exchange_id, sec_id, bar_type)
#print result_file_name
result_df = pd.read_csv(folder_name + result_file_name,
index_col='Setname')
for setname in para_set_list:
set_file_name = "%s %s.%s%d %s result.csv" % (
strategy_name, exchange_id, sec_id, bar_type, setname)
print set_file_name
set_result = pd.read_csv(folder_name + set_file_name)
max_own_cash = set_result['own cash'].max()
result_df.ix[setname, 'max_own_cash'] = max_own_cash
result_df.to_csv(folder_name + result_file_name)
pass
def set_generator():
setlist = []
i = 0
for n1 in range(15, 36, 5):
for m1 in range(6, 17, 4):
# for m2 in range(3, 15, 3):
for m2 in [3, 6, 9]:
#for n in range(3, 16, 3):
for n in [6, 10, 14, 18]:
#for ma_n in range(20, 51, 10):
for tunnel_n in [8, 14, 20, 26]:
setname = "Set%d N1_%d M1_%d M2_%d N_%d Tunnel_N_%d" % (
i, n1, m1, m2, n, tunnel_n)
l = [setname, n1, m1, m2, n, tunnel_n]
setlist.append(l)
i += 1
setpd = pd.DataFrame(
setlist, columns=['Setname', 'N1', 'M1', 'M2', 'N', 'Tunnel_N'])
#setpd['RSI1_UP'] = 70
#setpd['RSI1_DOWN'] = 30
upperpath = DC.getUpperPath(Parameter.folderLevel)
resultpath = upperpath + Parameter.resultFolderName
setpd.to_csv(resultpath + Parameter.parasetname)
def stat_multi_symbol_result():
upperpath = DC.getUpperPath(Parameter.folderLevel)
resultpath = upperpath + Parameter.resultFolderName
os.chdir(resultpath)
multi_symbol_df = pd.read_excel('HullRsiWin_symbol_KMIN_set.xlsx')
for n, row in multi_symbol_df.iterrows():
strategy_name = row['strategyName']
exchange_id = row['exchange_id']
sec_id = row['sec_id']
bar_type = row['K_MIN']
folder_name = "%s %s %s %d\\" % (strategy_name, exchange_id, sec_id,
bar_type)
result_file_name = "%s %s.%s %d finalresults.csv" % (
strategy_name, exchange_id, sec_id, bar_type)
print result_file_name
result_df = pd.read_csv(folder_name + result_file_name)
multi_symbol_df.ix[n, 'OprTimes'] = result_df['OprTimes'].mean()
multi_symbol_df.ix[n, 'Annual_max'] = result_df['Annual'].max()
multi_symbol_df.ix[n, 'Annual_avg'] = result_df['Annual'].mean()
multi_symbol_df.ix[n, 'EndCash_avg'] = result_df['EndCash'].mean()
multi_symbol_df.ix[n, 'EndCash_max'] = result_df['EndCash'].max()
multi_symbol_df.ix[
n, 'own_cash_max_max'] = result_df['max_own_cash'].max()
multi_symbol_df.ix[
n, 'own_cash_max_avg'] = result_df['max_own_cash'].mean()
multi_symbol_df.ix[n, 'Sharpe_max'] = result_df['Sharpe'].max()
multi_symbol_df.ix[n, 'SR_avg'] = result_df['SR'].mean()
multi_symbol_df.ix[n, 'DR_avg'] = result_df['DrawBack'].mean()
multi_symbol_df.ix[
n, 'SingleEarn_avg'] = result_df['MaxSingleEarnRate'].mean()
multi_symbol_df.ix[
n, 'SingleLoss_avg'] = result_df['MaxSingleLossRate'].mean()
multi_symbol_df.ix[
n, 'ProfitLossRate_avg'] = result_df['ProfitLossRate'].mean()
multi_symbol_df.to_csv('HullRsiWin_symbol_KMIN_set2_result.csv')
def plot_parameter_result_pic(
multi_sybmol_file_name="multi_symbol_1st_xu.xlsx"):
"""绘制finalresult结果中参数对应的end cash和max own cash的分布柱状图"""
upperpath = DC.getUpperPath(Parameter.folderLevel)
resultpath = upperpath + Parameter.resultFolderName
os.chdir(resultpath)
symbol_sellected = pd.read_excel(multi_sybmol_file_name)
for n, rows in symbol_sellected.iterrows():
fig = plt.figure(figsize=(6, 12))
exchange = rows['exchange']
sec = rows['sec']
bar_type = rows['bar_type']
folder_name = "%s %s %s %d\\" % (Parameter.strategyName, exchange, sec,
bar_type)
final_result_file = pd.read_csv(
folder_name + "%s %s.%s %d finalresults.csv" %
(Parameter.strategyName, exchange, sec, bar_type))
para_file = pd.read_csv(
folder_name + "%s %s %d %s.csv" %
(exchange, sec, bar_type, Parameter.parasetname))
para_name_list = ['MACD_Short', 'MACD_Long', 'MACD_M', 'MA_N']
for i in range(len(para_name_list)):
para_name = para_name_list[i]
final_result_file[para_name_list] = para_file[para_name_list]
grouped = final_result_file.groupby(para_name)
end_cash_grouped = grouped['EndCash'].mean()
p = plt.subplot(len(para_name_list), 1, i + 1)
p.set_title(para_name)
p.bar(end_cash_grouped.index.tolist(), end_cash_grouped.values)
print end_cash_grouped
fig.savefig('%s %s %s %d_para_distribute.png' %
(Parameter.strategyName, exchange, sec, bar_type),
dip=500)
def resultCalc():
'''
重新统计多品种多周期的回测结果
将各文件夹中的finalResult文件汇总到一起
:return:
'''
upperpath=DC.getUpperPath(Parameter.folderLevel)
resultpath = upperpath + Parameter.resultFolderName
os.chdir(resultpath)
allsymbolresult = pd.DataFrame(columns=
['Setname', 'MA_Short', 'MA_Long', 'KDJ_N', 'DMI_N', 'opentimes', 'end_cash',
'SR', 'Annual', 'Sharpe', 'DrawBack', 'max_single_loss_rate',
'strategyName', 'exchange_id', 'sec_id', 'K_MIN'])
symbolset = pd.read_excel(resultpath + Parameter.symbol_KMIN_set_filename)
symbolsetNum = symbolset.shape[0]
for i in range(symbolsetNum):
strategyName=symbolset.ix[i, 'strategyName']
exchangeid = symbolset.ix[i, 'exchange_id']
secid = symbolset.ix[i, 'sec_id']
symbol = '.'.join([exchangeid, secid])
K_MIN=symbolset.ix[i, 'K_MIN']
foldername=' '.join([strategyName, exchangeid, secid, str(K_MIN)])
print ("collecting %s %d final results"%(symbol,K_MIN))
result = pd.read_csv(foldername+'\\'+"%s %d finalresults.csv" % (symbol, K_MIN))
result['strategyName']=strategyName
result['exchange_id']=exchangeid
result['sec_id'] = secid
result['K_MIN'] = K_MIN
allsymbolresult=pd.concat([allsymbolresult,result])
allsymbolresult.to_csv(Parameter.strategyName + "_symbol_KMIN_results.csv")
pass
def re_concat_multi_symbol_final_result():
"""
重新汇总多品种回测的final_result结果，自动从symbol_KMIN_set.xlsx文件读取品种列表
:return:
"""
upperpath = DC.getUpperPath(Parameter.folderLevel)
resultpath = upperpath + Parameter.resultFolderName
os.chdir(resultpath)
multi_symbol_df = pd.read_excel('%s_symbol_KMIN_set.xlsx' %
Parameter.strategyName)
all_final_result_list = []
for n, row in multi_symbol_df.iterrows():
strategy_name = row['strategyName']
exchange_id = row['exchange_id']
sec_id = row['sec_id']
bar_type = row['K_MIN']
symbol_folder_name = "%s %s %s %d\\" % (strategy_name, exchange_id,
sec_id, bar_type)
bt_folder = "%s.%s %d backtesting\\" % (exchange_id, sec_id, bar_type)
result_file_name = "%s %s.%s %d finalresults.csv" % (
strategy_name, exchange_id, sec_id, bar_type)
print result_file_name
final_result_df = pd.read_csv(symbol_folder_name + result_file_name)
final_result_df['strategy_name'] = strategy_name
final_result_df['exchange_id'] = exchange_id
final_result_df['sec_id'] = sec_id
final_result_df['ba_type'] = bar_type
all_final_result_list.append(final_result_df)
multi_symbol_result_df = pd.concat(all_final_result_list)
multi_symbol_result_df.to_csv('%s_symbol_KMIN_set_result.csv' %
Parameter.strategyName)
def calc_single_backtest_final_result(domain_symbol, bar_type):
"""
计算单个品种回测结果的汇总finalresult文件
:param domain_symbol: 主力合约编号
:param bar_type: 周期
:return:
"""
upperpath = DC.getUpperPath(Parameter.folderLevel)
resultpath = upperpath + Parameter.resultFolderName
symbol_folder = domain_symbol.replace('.', ' ') + ' ' + str(bar_type)
os.chdir(resultpath + symbol_folder)
bt_folder = "%s %d backtesting\\" % (domain_symbol, bar_type)
parasetlist = pd.read_csv("%s %d %s" % (domain_symbol.replace(
'.', ' '), bar_type, Parameter.parasetname))['Setname'].tolist()
indexcols = Parameter.ResultIndexDic
strategy_name = Parameter.strategyName
resultlist = pd.DataFrame(columns=['Setname'] + indexcols)
i = 0
for setname in parasetlist:
print setname
result = pd.read_csv((bt_folder + strategy_name + ' ' + domain_symbol +
str(bar_type) + ' ' + setname + ' result.csv'))
dailyClose = pd.read_csv(
(bt_folder + strategy_name + ' ' + domain_symbol + str(bar_type) +
' ' + setname + ' dailyresult.csv'))
results = RS.getStatisticsResult(result, False, indexcols, dailyClose)
resultlist.loc[i] = [setname] + results #在这里附上setname
i += 1
finalresults = ("%s %s %d finalresults.csv" %
(strategy_name, domain_symbol, bar_type))
resultlist.to_csv(finalresults)
def getParallelResult(domain_symbol, bar_type, parasetlist):
strategyName = 'LvyiWin_wave'
# ======================数据准备==============================================
symbolinfo = DC.SymbolInfo(domain_symbol)
rawdataDic = DC.getBarBySymbolList(domain_symbol,
symbolinfo.getSymbolList(), bar_type)
foldername = ' '.join([strategyName, domain_symbol, str(bar_type)])
try:
os.mkdir(foldername)
except:
print("%s folder already exsist!" % foldername)
os.chdir(foldername)
# 多进程优化，启动一个对应CPU核心数量的进程池
pool = multiprocessing.Pool(multiprocessing.cpu_count() - 1)
l = []
resultlist = pd.DataFrame(columns=[
'Setname', 'wave_time_mean', 'wave_time_median', 'wave_time_75quanter',
'wave_height_mean', 'wave_height_median', 'wave_height_75quanter'
])
ma_short_list = parasetlist['MA_Short'].drop_duplicates()
ma_long_list = parasetlist['MA_Long'].drop_duplicates()
for ma_short in ma_short_list:
for ma_long in ma_long_list:
paraset = {
'Setname': "ms%d_ml%d" % (ma_short, ma_long),
'MA_Short': ma_short,
'MA_Long': ma_long
}
#l.append(LvyiWin_wave_cal(strategyName, symbolinfo, bar_type, rawdataDic, paraset))
l.append(
pool.apply_async(
LvyiWin_wave_cal,
(strategyName, symbolinfo, bar_type, rawdataDic, paraset)))
pool.close()
pool.join()
# 显示结果
i = 0
for res in l:
resultlist.loc[i] = res.get()
i += 1
finalresults = ("%s %s %d finalresults.csv" %
(strategyName, domain_symbol, bar_type))
resultlist.to_csv(finalresults)
return resultlist
def remove_polar():
symbol = 'SHFE.RB'
symbolinfo = DC.SymbolInfo(symbol)
folder = 'D:\\002 MakeLive\myquant\LvyiWin\Results\LvyiWin SHFE RB 600\dsl_-0.018ownl_0.008\ForwardOprAnalyze\\'
filename = 'LvyiWin SHFE.RB600_Rank4_win1_oprResult.csv'
opr = pd.read_csv(folder+filename)
opr = RS.opr_result_remove_polar(opr, new_cols=True)
opr['commission_fee_rp'], opr['per earn_rp'], opr['own cash_rp'], opr['hands_rp'] = RS.calcResult(result=opr, symbolinfo=symbolinfo, initialCash=200000, positionRatio=1, ret_col='new_ret')
opr.to_csv(folder+'LvyiWin SHFE.RB600_Rank4_win1_oprResult_remove_polar.csv')
def multiStopLosslCal_remove_polar(stratetyName, symbolInfo, K_MIN, setname,
stopLossTargetDictList, barxmdic,
positionRatio, initialCash, tofolder,
indexcols):
print 'setname:', setname
symbol = symbolInfo.domain_symbol
oprdf = pd.read_csv(stratetyName + ' ' + symbol + str(K_MIN) + ' ' +
setname + ' result.csv')
oprdf = RS.opr_result_remove_polar(oprdf=oprdf, new_cols=True)
symbolDomainDic = symbolInfo.amendSymbolDomainDicByOpr(oprdf)
barxm = DC.getDomainbarByDomainSymbol(symbolInfo.getSymbolList(), barxmdic,
symbolDomainDic)
dailyK = DC.generatDailyClose(barxm)
oprdf['new_commission_fee'], oprdf['new_per earn'], oprdf[
'new_own cash'], oprdf['new_hands'] = RS.calcResult(oprdf,
symbolInfo,
initialCash,
positionRatio,
ret_col='new_ret')
oprdf.to_csv(tofolder + '\\' + stratetyName + ' ' + symbol + str(K_MIN) +
' ' + setname + ' result_multiSLT_remove_polar.csv',
index=False)
#计算统计结果
slWorkNum = oprdf.loc[oprdf['closetype'] != 'Normal'].shape[0]
olddailydf = pd.read_csv(stratetyName + ' ' + symbol + str(K_MIN) + ' ' +
setname + ' dailyresult.csv',
index_col='date')
oldr = RS.getStatisticsResult(oprdf, False, indexcols, olddailydf)
dR = RS.dailyReturn(symbolInfo, oprdf, dailyK, initialCash) # 计算生成每日结果
dR.calDailyResult()
dR.dailyClose.to_csv(tofolder + '\\' + stratetyName + ' ' + symbol +
str(K_MIN) + ' ' + setname +
' dailyresult_multiSLT_remove_polar.csv')
newr = RS.getStatisticsResult(oprdf, True, indexcols, dR.dailyClose)
print newr
return [
setname,
tofolder,
slWorkNum,
] + oldr + newr
def sortOpr(strategyList):
resultDf = pd.DataFrame()
for st in strategyList:
symbol=st['symbol']
symbolinfo=DC.SymbolInfo(symbol)
if st['new']:
opendf = pd.read_csv(st['filePath'] + st['fileName'], usecols=new_openheader)
closedf = pd.read_csv(st['filePath'] + st['fileName'], usecols=new_closeheaders)
opendf.rename(columns={'opentime': 'oprtime',
'openutc': 'oprutc',
'openindex': 'oprindex',
'openprice': 'oprprice',
'new_ret': 'ret',
'new_ret_r': 'ret_r'}, inplace=True)
opendf['oprtype'] = 1 # 1表示开仓
closedf.rename(columns={'new_closetime': 'oprtime',
'new_closeutc': 'oprutc',
'new_closeindex': 'oprindex',
'new_closeprice': 'oprprice',
'new_ret': 'ret',
'new_ret_r': 'ret_r'}, inplace=True)
closedf['oprtype'] = -1 # -1表示平仓
else:
opendf = pd.read_csv(st['filePath'] + st['fileName'], usecols=openheader)
closedf = pd.read_csv(st['filePath'] + st['fileName'], usecols=closeheaders)
opendf.rename(columns={'opentime': 'oprtime',
'openutc': 'oprutc',
'openindex': 'oprindex',
'openprice': 'oprprice',
'ret': 'ret',
'ret_r': 'ret_r'}, inplace=True)
opendf['oprtype'] = 1 # 1表示开仓
closedf.rename(columns={'closetime': 'oprtime',
'closeutc': 'oprutc',
'closeindex': 'oprindex',
'closeprice': 'oprprice',
'ret': 'ret',
'ret_r': 'ret_r'}, inplace=True)
closedf['oprtype'] = -1 # -1表示平仓
df = pd.concat([opendf, closedf])
df.set_index('oprutc', inplace=True)
df['strategy'] = st['name']
df['positionRate'] = st['positionRate']
df['multiplier'] = symbolinfo.getMultiplier() # 乘数
df['poundgeType'], df['poundgeFee'], df['poundgeRate'] = symbolinfo.getPoundage() # 手续费率
df['marginRatio'] = symbolinfo.getMarginRatio() # 保证金率
resultDf = pd.concat([resultDf, df])
resultDf.sort_index(inplace=True)
resultDf.reset_index(drop=False, inplace=True)
#resultDf.to_csv('resultCostatistics.csv')
#print resultDf.head(10)
return resultDf
def remove_polar():
symbol = 'DCE.J'
symbolinfo = DC.SymbolInfo(symbol)
folder = 'D:\\002 MakeLive\myquant\HopeWin\Results\HopeMacdMaWin DCE J 3600\ForwardOprAnalyze\\'
filename = 'HopeMacdMaWin DCE.J3600_Rank3_win9_oprResult.csv'
opr = pd.read_csv(folder + filename)
opr = RS.opr_result_remove_polar(opr)
opr['commission_fee_rp'], opr['per earn_rp'], opr['own cash_rp'], opr[
'hands_rp'] = RS.calcResult(result=opr,
symbolinfo=symbolinfo,
initialCash=200000,
positionRatio=1,
ret_col='new_ret')
opr.to_csv(folder +
'HopeMacdMaWin DCE.J3600_Rank3_win9_oprResult_remove_polar.csv')
def riceConcateToBar(bar_type, enddate):
os.chdir('D:\\002 MakeLive\DataCollection\\ricequant data\\')
contractlist = pd.read_excel(DC.PUBLIC_DATA_PATH + 'Contract.xlsx')['Contract']
for symbol in contractlist:
print symbol
ricefolder = "riceToMyquant %s\\" % (symbol)
ricedata = pd.read_csv(ricefolder + "%s %d_%s.csv" % (symbol, bar_type, enddate))
bardata = DC.getBarData(symbol, bar_type, starttime='2016-05-01 00:00:00', endtime='2018-03-31 23:59:59')
bardata = pd.concat([ricedata, bardata])
tofilename = ("%s %d.csv" % (symbol, bar_type))
bardata.reset_index(drop=True, inplace=True)
bardata = bardata.drop('Unnamed: 0', axis=1)
bardata = bardata.drop('Unnamed: 0.1', axis=1)
bardata.to_csv(DC.BAR_DATA_PATH + symbol + '\\' + tofilename)
pass
def parasetGenerator():
upperpath=DC.getUpperPath(Parameter.folderLevel)
resultpath = upperpath + Parameter.resultFolderName
setlist = []
i = 0
for ms in [5, 6, 7]:
for ml in [24,25,26, 27, 28, 29, 30]:
for kn in [6, 8, 20, 22, 24, 26, 28, 30]:
for dn in [28, 30, 32, 34, 36]:
setname = 'Set' + str(i) + ' MS' + str(ms) + ' ML' + str(ml) + ' KN' + str(kn) + ' DN' + str(dn)
l = [setname, ms, ml, kn, dn]
setlist.append(l)
i += 1
setpd = pd.DataFrame(setlist, columns=['Setname', 'MA_Short', 'MA_Long', 'KDJ_N', 'DMI_N'])
setpd.to_csv(resultpath+'ParameterOptSet_simple.csv')
def calc_singal_close_final_result(domain_symbol, bar_type, sl_type,
folder_name):
"""
计算单个品种单个止损结果的汇总finalresult文件
:param domain_symbol: 主力合约编号
:param bar_type: 周期
:param sl_type: 止损类型 'DSL', 'OWNL', 'FRSL', 'ATR', 'GOWNL'
:param folder_name: 结果文件存放文件夹名
:return:
"""
upperpath = DC.getUpperPath(Parameter.folderLevel)
resultpath = upperpath + Parameter.resultFolderName
symbol_folder = domain_symbol.replace('.', ' ') + ' ' + str(bar_type)
os.chdir(resultpath + symbol_folder)
bt_folder = "%s %d backtesting\\" % (domain_symbol, bar_type)
parasetlist = pd.read_csv(resultpath +
Parameter.parasetname)['Setname'].tolist()
strategy_name = Parameter.strategyName
indexcols = Parameter.ResultIndexDic
file_name_suffix = '%s_by_tick.csv' % sl_type
new_indexcols = []
for i in indexcols:
new_indexcols.append('new_' + i)
resultdf = pd.DataFrame(columns=['setname', 'sl_target', 'worknum'] +
indexcols + new_indexcols)
i = 0
for setname in parasetlist:
print setname
worknum = 0
olddailydf = pd.read_csv(bt_folder + strategy_name + ' ' +
domain_symbol + str(bar_type) + ' ' +
setname + ' dailyresult.csv',
index_col='date')
opr_file_name = "\\%s %s%d %s result%s" % (
strategy_name, domain_symbol, bar_type, setname, file_name_suffix)
oprdf = pd.read_csv(folder_name + opr_file_name)
oldr = RS.getStatisticsResult(oprdf, False, indexcols, olddailydf)
opr_dialy_k_file_name = "\\%s %s%d %s dailyresult%s" % (
strategy_name, domain_symbol, bar_type, setname, file_name_suffix)
dailyClose = pd.read_csv(folder_name + opr_dialy_k_file_name)
newr = RS.getStatisticsResult(oprdf, True, indexcols, dailyClose)
resultdf.loc[i] = [setname, folder_name, worknum] + oldr + newr
i += 1
resultdf.to_csv(
"%s\\%s %s%d finalresult_%s.csv" %
(folder_name, strategy_name, domain_symbol, bar_type, folder_name))
def getParallelResult(symbolinfo, K_MIN, backtest_startdate, backtest_enddate,
setname, para, contractswaplist):
rawdata = DC.getBarData(
symbolinfo.symbol,
K_MIN,
backtest_startdate,
backtest_enddate,
).reset_index(drop=True)
result, df, closeopr, results = LvyiWin.LvyiWin(symbolinfo, rawdata, para,
contractswaplist)
r = [
setname, para['MA_Short'], para['MA_Long'], para['KDJ_N'],
para['DMI_N'], results['opentimes'], results['end_cash'],
results['SR'], results['Annual'], results['Sharpe'],
results['DrawBack'], results['max_single_loss_rate']
]
print setname + " finished"
result.to_csv(symbolinfo.symbol + str(K_MIN) + ' ' + setname +
' result.csv')
del result
return r
def set_generator():
setlist = []
i = 0
for ms in range(5, 14, 1):
for ml in range(16, 29, 2):
for mm in range(5, 16, 2):
setname = 'Set' + str(i) + ' MS' + str(ms) + ' ML' + str(
ml) + ' MM' + str(mm)
l = [setname, ms, ml, mm]
setlist.append(l)
i += 1
setpd = pd.DataFrame(
setlist, columns=['Setname', 'MACD_Short', 'MACD_Long', 'MACD_M'])
setpd['BIAS_N'] = 40
setpd['BIAS_UP'] = 3
setpd['BIAS_DOWN'] = -3
upperpath = DC.getUpperPath(Parameter.folderLevel)
resultpath = upperpath + Parameter.resultFolderName
setpd.to_csv(resultpath + Parameter.parasetname)
def re_calc_finalresult():
upperpath = DC.getUpperPath(Parameter.folderLevel)
resultpath = upperpath + Parameter.resultFolderName
symbol_folder = "%s\\%s %s %s %d\\" % (resultpath, Parameter.strategyName,
Parameter.exchange_id,
Parameter.sec_id, Parameter.K_MIN)
os.chdir(symbol_folder)
atrsl_folder = '10_4.0_16_2.0\\' # 止损文件夹名
parasetlist = pd.read_csv(resultpath +
Parameter.parasetname)['Setname'].tolist()
strategyName = Parameter.strategyName
symbol = "%s.%s" % (Parameter.exchange_id, Parameter.sec_id)
K_MIN = Parameter.K_MIN
indexcols = Parameter.ResultIndexDic
new_indexcols = []
for i in indexcols:
new_indexcols.append('new_' + i)
resultdf = pd.DataFrame(columns=['setname', 'atr_sl_target', 'worknum'] +
indexcols + new_indexcols)
#resultdf['No'] = range(len(parasetlist))
i = 0
for setname in parasetlist:
print setname
worknum = 0
olddailydf = pd.read_csv(strategyName + ' ' + symbol + str(K_MIN) +
' ' + setname + ' dailyresult.csv',
index_col='date')
opr_file_name = "%s %s%d %s resultATR_by_tick.csv" % (
strategyName, symbol, K_MIN, setname)
oprdf = pd.read_csv(atrsl_folder + opr_file_name)
oldr = RS.getStatisticsResult(oprdf, False, indexcols, olddailydf)
opr_dialy_k_file_name = "%s %s%d %s dailyresultATR_by_tick.csv" % (
strategyName, symbol, K_MIN, setname)
dailyClose = pd.read_csv(atrsl_folder + opr_dialy_k_file_name)
newr = RS.getStatisticsResult(oprdf, True, indexcols, dailyClose)
resultdf.loc[i] = [setname, '10_4.0_16_2.0', worknum] + oldr + newr
i += 1
resultdf.to_csv("%s%s %s%d finalresult_atrsl10_4.0_16_2.0.csv" %
(atrsl_folder, strategyName, symbol, K_MIN))
def collection_by_symbol():
domain_symbol = 'CFFEX.IF'
symbol_list = ['IF1807', 'IF1808', 'IF1809']
print("start collect:%s" % domain_symbol)
symbolinfo = DC.SymbolInfo(domain_symbol)
folder = "rqdata_raw_%s" % domain_symbol
try:
os.mkdir(folder)
except:
pass
for symbol in symbol_list:
startdate, enddate = symbolinfo.getSymbolLifeDate(symbol)
bardata = get_price(symbol,
frequency='1m',
start_date=startdate,
end_date=enddate)
bardata.to_csv(folder + "\\rqdata_raw_%s_of_%s_1m.csv" %
(symbol, domain_symbol))
print("%s of %s from %s to %s collection finished" %
(symbol, domain_symbol, startdate, enddate))
print("finish collect:%s" % domain_symbol)
print("all work done!")
def re_concat_atrsl_result():
upperpath = DC.getUpperPath(Parameter.folderLevel)
resultpath = upperpath + Parameter.resultFolderName
symbol_folder = "%s\\%s %s %s %d\\" % (resultpath, Parameter.strategyName,
Parameter.exchange_id,
Parameter.sec_id, Parameter.K_MIN)
os.chdir(symbol_folder)
atrsl_para_name_list = []
atr_pendant_n_list = Parameter.atr_pendant_n_list_close
atr_pendan_rate_list = Parameter.atr_pendant_rate_list_close
atr_yoyo_n_list = Parameter.atr_yoyo_n_list_close
atr_yoyo_rate_list = Parameter.atr_yoyo_rate_list_close
for atr_pendant_n in atr_pendant_n_list:
for atr_pendant_rate in atr_pendan_rate_list:
for atr_yoyo_n in atr_yoyo_n_list:
for atr_yoyo_rate in atr_yoyo_rate_list:
atrsl_para_name_list.append(
'%d_%.1f_%d_%.1f' % (atr_pendant_n, atr_pendant_rate,
atr_yoyo_n, atr_yoyo_rate))
final_result_name_0 = "%s %s.%s%d finalresult_atrsl%s.csv" % (
Parameter.strategyName, Parameter.exchange_id, Parameter.sec_id,
Parameter.K_MIN, atrsl_para_name_list[0])
final_result_file = pd.read_csv(
"%s\\%s" % (atrsl_para_name_list[0], final_result_name_0))
for atr_para_name in atrsl_para_name_list[1:]:
final_result_name = "%s %s.%s%d finalresult_atrsl%s.csv" % (
Parameter.strategyName, Parameter.exchange_id, Parameter.sec_id,
Parameter.K_MIN, atr_para_name)
final_result_file = pd.concat([
final_result_file,
pd.read_csv("%s\\%s" % (atr_para_name, final_result_name))
])
final_result_file.to_csv("%s %s.%s%d finalresult_atrsl_reconcat.csv" %
(Parameter.strategyName, Parameter.exchange_id,
Parameter.sec_id, Parameter.K_MIN))
def collection_domain_by_symbol():
# 抓取全品种全合约数据
domain_list = pd.read_excel(DC.PUBLIC_DATA_PATH +
'domainMap.xlsx')['symbol']
for domain_symbol in domain_list:
print("start collect:%s" % domain_symbol)
symbolinfo = DC.SymbolInfo(domain_symbol)
folder = "rqdata_raw_%s" % domain_symbol
try:
os.mkdir(folder)
except:
pass
for symbol in symbolinfo.getSymbolList():
startdate, enddate = symbolinfo.getSymbolLifeDate(symbol)
bardata = get_price(symbol,
frequency='1d',
start_date=startdate,
end_date=enddate)
bardata.to_csv(folder + "\\rqdata_raw_%s_of_%s_1d.csv" %
(symbol, domain_symbol))
print("%s of %s from %s to %s collection finished" %
(symbol, domain_symbol, startdate, enddate))
print("finish collect:%s" % domain_symbol)
print("all work done!")
newfolder = ''
for sltp in sltset:
newfolder += (sltp['name'] + '_%.3f' % (sltp['sltValue']))
rawdatapath = folderpath + newfolder + '\\'
df = mtf.getMonthParameter(strategyName, startmonth, endmonth,
symbolinfo, K_MIN, parasetlist, rawdatapath,
colslist, resultfilesuffix)
filenamehead = ("%s%s_%s_%d_%s_parameter_%s" %
(rawdatapath, strategyName, symbolinfo.domain_symbol,
K_MIN, endmonth, newfolder))
df.to_csv(filenamehead + '.csv')
if __name__ == '__main__':
# 文件路径
upperpath = DC.getUpperPath(Parameter.folderLevel)
resultpath = upperpath + Parameter.resultFolderName
# 取参数集
parasetlist = pd.read_csv(resultpath + Parameter.parasetname)
paranum = parasetlist.shape[0]
#生成月份列表，取开始月
newmonth = Parameter.enddate[:7]
month_n = Parameter.month_n
monthlist = [
datetime.strftime(x, '%Y-%m') for x in list(
pd.date_range(
start=Parameter.startdate, end=newmonth + '-01', freq='M'))
]
startmonth = monthlist[-month_n]
# ======================================参数配置==================================================
(strategyName, symbol, K_MIN, newfolder))
resultdf.to_csv(newfolder + '\\' + resultfilename, index=False)
allresultname = ''
for slt in sltlist:
allresultname += slt['name']
# allresultdf['cashDelta'] = allresultdf['new_endcash'] - allresultdf['old_endcash']
allresultdf.to_csv("%s %s%d finalresult_multiSLT_%s.csv" %
(strategyName, symbol, K_MIN, allresultname),
index=False)
pass
if __name__ == '__main__':
# 文件路径
upperpath = DC.getUpperPath(Parameter.folderLevel)
resultpath = upperpath + Parameter.resultFolderName
# 取参数集
parasetlist = pd.read_csv(resultpath + Parameter.parasetname)
paranum = parasetlist.shape[0]
#indexcols
indexcols = Parameter.ResultIndexDic
# 参数设置
strategyParameterSet = []
if not Parameter.symbol_KMIN_opt_swtich:
# 单品种单周期模式
paradic = {
'strategyName': Parameter.strategyName,
tickworknum=tickdf.loc[tickdf['new_closeutc']!=tickdf['closeutc'],'new_ret'].count()
totaloprnum=ownldf.shape[0]
simuTickRetDelta=tickretdelta-retdelta
return [setname,simuTickRetDelta,tickworknum,dslworknum,totaloprnum,tickretdelta,retdelta,tickworkretdelta,workretdelta,tickworkretsum,workdslretsum,
tickworkoriretsum,workdsloriretsum,tickretsum,dslretsum,oriretsum]
if __name__ =='__main__':
#参数配置
exchange_id = 'SHFE'
sec_id='RB'
symbol = '.'.join([exchange_id, sec_id])
K_MIN = 600
topN=4500
pricetick=DC.getPriceTick(symbol)
slip=pricetick
starttime='2017-09-01'
endtime='2017-12-11'
tickstarttime='2017-10-01'
tickendtime='2017-12-01'
tickstartutc = float(time.mktime(time.strptime(tickstarttime + ' 00:00:00', "%Y-%m-%d %H:%M:%S")))
tickendutc = float(time.mktime(time.strptime(tickendtime + ' 23:59:59', "%Y-%m-%d %H:%M:%S")))
# 优化参数
# stoplossStep=-0.002
# stoplossList = np.arange(-0.02, -0.04, stoplossStep)
stoplossList = [-0.022]
winSwitchList = [0.009]
# 文件路径
exchange_id = 'DCE'
sec_id = 'I'
K_MIN = 3600
symbol = '.'.join([exchange_id, sec_id])
startdate = '2016-01-01'
enddate = '2017-12-31'
# MACD参数
MACD_SHORT = 6
MACD_LONG = 30
MACD_M = 9
# MA参数
MA_N = 30
# 文件路径
upperpath = DC.getUpperPath(2)
foldername = ' '.join([strategyName, exchange_id, sec_id, str(K_MIN)])
resultpath = upperpath + "\\Results\\"
os.chdir(resultpath)
try:
os.mkdir(foldername)
except:
print ("%s folder already exsist!" % foldername)
os.chdir(foldername)
# ======================数据准备==============================================
# 取参数集
parasetlist = pd.read_csv(resultpath + 'MACDParameterSet1.csv')
paranum = parasetlist.shape[0]
# 取合约信息
symbolInfo = DC.SymbolInfo(symbol)