def kline_merge_strategy(k_min = 30,start_date = '20200102',symbols=['IC2001.CFE']):
'''
用tick合成一天之内分钟级别的k线
example:
---------
k_min = 30 : tick合成30minK
kcut_res_data = K_cut.kline_merge_strategy(k_min = 30,start_date = '20200102',symbols=['IC2001.CFE'])
'''
tick_data_batch = D.query_tsdata(datasource='future_tick', symbols=symbols,\
fields='price',start=start_date, end=start_date, delivery_mode='batch_by_symbol')
for d in tick_data_batch:
tick_data = d
tick_data = tick_data[start_date + ' 09:30:00':start_date + ' 15:00:00']
kcut_res_data = K_cut.aggre_tick(tick_data = tick_data,m = 60*k_min,n = 60*k_min)
#调整输出格式
kcut_res_data=kcut_res_data.reset_index()
kcut_res_data=kcut_res_data.rename(columns={'Symbol':'code','index':'DATETIME'})
kcut_res_data['DATETIME'] = kcut_res_data['DATETIME'].apply(lambda x:pd.Timestamp(x.strftime('%Y-%m-%d %H:%M:%S')))
kcut_res_data['code'] = symbols[0][:-1]
kcut_res_data.drop('label',axis = 1,inplace = True)
kcut_res_data.sort_values('DATETIME',inplace = True)
#检查是否有重复行
if any(kcut_res_data['DATETIME'].duplicated()):
print('warning: kline_merge_strategy输出有重复时间戳')
return
else:
pass
return kcut_res_data
def long_Kline_merge(start = '20190102',end = '20191231',k_min = 1):
'''
获取从start到end这段时间内，合成k_min分钟K线。
'''
# 获取交易日
trading_day_data = D.query_ext_factor(datasource='wind_eod_index_price', symbols=['000001.SH'],\
fields=['S_DQ_CLOSE'], start=start, end=end)
trading_day_data.reset_index(inplace = True)
trading_day_data.rename({'ts':'trading_day'},axis = 1,inplace= True)
trading_day_data =trading_day_data['trading_day']
trading_day_data = trading_day_data.apply(lambda x :x.strftime('%Y%m%d') )
#获取 IC
IC_data = backtest.get_IC_date(start = start,end = end)
#使用class，并concat
kline_df = pd.DataFrame()
for start_date in trading_day_data:
symbols = []
symbols.append(IC_data[(IC_data['ENDDATE'] >= start_date) &
(IC_data['STARTDATE'] <= start_date)]['FS_MAPPING_WINDCODE'].iloc[0])
try:
kline= K_cut.kline_merge_strategy(k_min = k_min,start_date = start_date,symbols=symbols)
except TypeError:
pass
else:
kline_df = pd.concat([kline_df,kline]).reset_index(drop = True)
return kline_df
def get_lab_kline(start = '20190102',end = '20191231',k_min_high_level = 1,code = 'IC00.CFE'):
'''
获取lab平台上的股指期货K_line数据
example：
---------
prices = DataProcess.get_lab_kline(start = '20190102',end = '20191231',k_min_high_level = 1,code = 'IC00.CFE')
'''
data = D.query_tsdata(datasource=f'future_kline_{k_min_high_level}min', symbols=ContFutureSymbolFilter(code), fields='*', start=start, end=end)
prices_list = []
for df in data:
# #xhq modify
# df.index = pd.Series(df.index).shift(-1)
# df = df.iloc[:-1,:]
# ###
prices_list.append(df)
prices_df = pd.concat(prices_list)
prices = prices_df[['Symbol', 'open', 'high', 'low', 'close']]
prices.reset_index(inplace=True)
prices = prices.rename(columns={'index':'DATETIME', 'Symbol': 'code'})
return prices
# -*- coding: utf-8 -*-
# @Time : 2020/7/3 20:33
# @Author ：Xie Huiqin
# @File : pre_part.py
# @Software : orient_lab/jupyter notebook
import pandas as pd
import numpy as np
import datetime
from market_timing_lab import MeanReversion
from algoqi.data import D
D.load()
from backtest_class import backtest
class interval:
"""
Some idea from 缠中说禅.
Examples
---------
import numpy as np
import pandas as pd
from market_timing_lab import MeanReversion
from algoqi.data import D
from pre_part import *
D.load()
#IC2006.CF，1minK分钟数据
data=D.query_tsdata(datasource='future_kline_1min', symbols=['IC2006.CF'], fields=['open','high','low','close'],
from algoqi.data import D, plotutils
D.switch_zxzx_client(access_key='e7e2de63f95674632824ed6b2be8dd1f', secret_key='d43ca617fe7048e4ccfb158dde8e1c32')
D.load()
import pandas as pd
import numpy as np
class ContFutureSymbolFilter(D.FutContContractFilter):
'''
当月合约筛选器
examle：
---------
symbols=ContFutureSymbolFilter(code)
'''
@staticmethod
def future_suffix_replace(symbol_str: str):
return symbol_str if '.CFE' not in symbol_str[0] else [symbol_str[0].replace(".CFE", ".CF")]
def symbols_during(self, start_dt: str, end_dt: str):
main_contracts_df = self._query_main_contract(symbols=self.symbols, start=start_dt, end=end_dt)
if not main_contracts_df.empty:
cont_list = []
for i in range(len(main_contracts_df)):
sub_cont = (main_contracts_df.iloc[i].symbol, main_contracts_df.iloc[i].start,
main_contracts_df.iloc[i].end, self.future_suffix_replace([main_contracts_df.iloc[i].mapping_symbol]))
cont_list.append(sub_cont)
return cont_list
class DataProcess:
@staticmethod
def get_lab_kline(start = '20190102',end = '20191231',k_min_high_level = 1,code = 'IC00.CFE'):
def bt_long(py_path='./final_5min.py',
code='IC00.CFE',
start='20190102',
end='20191231',
N=5,
per_size=1,
save=False,
save_path='./final_5min.pkl',
save_name='final_5min',
k_min_high_level=1,
k_min_low_level=1,
use_tick_merge=False):
'''
return：
-----------
trades_df: 交易记录
logs_df: 日志记录
target
-----------
做长期回测，如一年
parameter
-----------
py_path：策略文件路径
code：获取当月合约，统一使用'IC00.CFE'
start：回测开始时间（必须是交易日）
end: 回测结束时间
N: 时间切割策略中，回测使用的K线数据做几次切割。如：N = 5（使用5minK线）代表将5minK线切割5次，生成5套5minK线数据。
per_size: 单次交易购买的合约数，一般情况下，使用per_size = 1
save: 是否存储回测过程中交易和日志的中间结果:trades_df,logs_df
save_path: 交易和日志的中间结果的存储路径
save_name: 建议使用策略文件名，或者在策略文件名上加一些尾注
k_min_high_level : （int）在第三买卖点策略中标识计算中枢的K线级别，如k_min_high_level =30 表示计算中枢的高级K线是30min K线
k_min_low_level : （int）在第三买卖点策略中标识计算笔的K线级别
use_tick_merge : use_tick_merge = True，使用tick数据合成K线；use_tick_merge = False，使用平台提供的K线数据。
'''
symbol = code # 回测品种
beg_date = start # 回测开始时间,开始时间不能是周末
end_date = end # 回测结束时间
symbols_list = D.CONT_FUT('IC00.CFE').symbols_during(
beg_date, end_date)
start = [symbols_list[i][1] for i in range(0, len(symbols_list))]
end = [symbols_list[i][2] for i in range(0, len(symbols_list))]
code = [
symbols_list[i][3][0] + 'E' for i in range(0, len(symbols_list))
]
res_trades = [] # 构造空的dataframe
res_log = []
for i in range(0, len(symbols_list)):
if i > 0: j = i - 1
else: j = 0
params_dict = {
'symbol': code[i],
'per_size': per_size,
'start': start[i],
'latest_end': end[j],
'first_start': start[0],
'last_end': end[-1],
'end': end[i],
'N': N,
'long_short': 'long',
'py_name': save_name,
'k_min_high_level': k_min_high_level,
'k_min_low_level': k_min_low_level,
'use_tick_merge': use_tick_merge
}
def params_func(*args, **kwargs):
return {
'symbol': code[i],
'per_size': per_size,
'start': start[i],
'latest_end': end[j],
'first_start': start[0],
'last_end': end[-1],
'end': end[i],
'N': N,
'long_short': 'long',
'py_name': save_name,
'k_min_high_level': k_min_high_level,
'k_min_low_level': k_min_low_level,
'use_tick_merge': use_tick_merge
}
print('\n', start[i], '-', end[i])
config = BacktestConfig(sub_config=[])
config.start_ts = start[i]
config.end_ts = end[i]
# config.fields = ["B1", "S1"]
config.fields = "*"
config.split = None
config.datasource = "future_tick"
# config.init_positions = [("AU1912", LongShortType.LONG, 0), \\("AU1912", LongShortType.SHORT, 0)]
config.init_positions = []
config.symbols = [code[i]] #这里是列表，一定要注意数据结构
config.param = params_dict
res = AQ.backtest_local(py_path,
config,
params_dict=params_func,
convert_to_df=True)
res_log.append(res.logs)
res_trades.append(res.trades)
#中间存储
if save:
trades_df0 = pd.concat(res_trades)
logs_df0 = pd.concat(res_log)
try:
loc = save_path.rindex('/')
save_path_trades = list(save_path)
save_path_trades.insert(loc + 1, 'trades_')
save_path_trades = ''.join(save_path_trades)
save_path_logs = list(save_path)
save_path_logs.insert(loc + 1, 'logs_')
save_path_logs = ''.join(save_path_logs)
except Exception as e:
raise ValueError(r'Save_path must contain at least one /')
trades_df0.to_pickle(save_path_trades)
logs_df0.to_pickle(save_path_logs)
trades_df = pd.concat(res_trades) #输出的是list，要转化为dataframe
logs_df = pd.concat(res_log) #输出的是list，要转化为dataframe
return trades_df, logs_df
def on_init(init_param, ctx: Context, U: UserSpace):
ctx.log("initialize params") # 日志
long_or_short: str = init_param['long_short']
if long_or_short == 'long':
symbol = init_param['symbol']
symbols = symbol[:-1] # .CF
else:
# tick数据是.CFE;1min_kline是.CF
symbol: str = init_param['symbol'] # 初始化参数
# 'IC00.CFE'
symbols = symbol
symbol = symbol + 'E'
U.py_name :str = init_param['py_name']
U.first_start: str = init_param['first_start']
U.last_end: str = init_param['last_end']
U.start: str = init_param['start']
#latest_end:本次合约之前一个合约的结束日期
U.latest_end : str = init_param['latest_end']
U.end: str = init_param['end']
U.per_size: int = init_param['per_size']
U.k_min_high_level: int = init_param['k_min_high_level']
U.k_min_low_level: int = init_param['k_min_low_level']
U.use_tick_merge = init_param['use_tick_merge']
# N :int = init_param['N']
ctx.subscribe_tick([symbol], fields=[TickFields.TRADE_PRICE,
TickFields.B1, TickFields.S1,
TickFields.SV1, TickFields.BV1])
ctx.subscribe_time("09:45:00", "market_start") # 开盘前16min不开仓
ctx.subscribe_time("11:30:00", "market_suspend")
ctx.subscribe_time("13:00:00", "market_start")
ctx.subscribe_time("15:00:00", "market_almost_close") # 收盘前15min完全平仓 修改：收盘前30min只平仓不开仓
today = ctx.today().strftime('%Y%m%d')
if today == U.start:
# #使用tick合成的一段时间K线（pkl文件）， 导入pkl，加快运行速度【数据1】
# U.raw_prices = pd.read_pickle(f'./第三买卖点/{U.k_min_low_level}minK_20190102_20191231.pkl')
# U.raw_prices_high_level = pd.read_pickle(f'./第三买卖点/{U.k_min_high_level}minK_20190102_20191231.pkl')
#使用平台数据【数据2】或者使用【数据3】
pass
# 【数据1】
# now_date = ctx.today().strftime('%Y-%m-%d')
# next_date = (ctx.today() + timedelta(days=1)).strftime('%Y-%m-%d')
# prices = U.raw_prices[(U.raw_prices['DATETIME'] <= next_date) & (U.raw_prices['DATETIME'] >= now_date)]
# prices_high_level = U.raw_prices_high_level[(U.raw_prices_high_level['DATETIME'] <= next_date)
# & (U.raw_prices_high_level['DATETIME'] >= now_date)]
if U.use_tick_merge:
# 使用tick合成的K线，每日计算【数据3】
prices = K_cut.kline_merge_strategy(k_min = U.k_min_low_level,start_date = today,symbols=[symbol])
prices_high_level = K_cut.kline_merge_strategy(k_min = U.k_min_high_level,start_date = today,symbols=[symbol])
else:
# 【数据2】
curDate = ctx.today().strftime('%Y%m%d')
data = D.query_tsdata(datasource=f'future_kline_{U.k_min_low_level}min', symbols=[symbols], fields = ['open','high','low','close'],\
start = curDate, end=curDate, delivery_mode=D.DataDeliveryMode.DIRECT)
data_high_level = D.query_tsdata(datasource=f'future_kline_{U.k_min_high_level}min', symbols=[symbols], fields = ['open','high','low','close'],\
start = curDate, end=curDate, delivery_mode=D.DataDeliveryMode.DIRECT)
try:
# 检查是否发生数据缺失，缺失则跳过
data = data.reset_index()
data_high_level = data_high_level.reset_index()
except AttributeError:
print('\n',curDate,'：数据缺失，跳过该日。')
return
prices = data.rename(columns={'Symbol': 'code', 'index': 'DATETIME'})
prices_high_level = data_high_level.rename(columns={'Symbol': 'code', 'index': 'DATETIME'})
# print(prices_high_level.iloc[:10,:])
if len(prices) == 0 or len(prices_high_level) == 0:
return
###input###
# 计算每天前15min的数据
if today == U.start:
if today == U.first_start:
#创建存储文件夹
try:
import os
os.mkdir(f'./第三买卖点/strategy_save/{U.py_name}_{U.first_start}_{U.last_end}')
except FileExistsError:
pass
U.high_level_begin_time = prices_high_level['DATETIME'].iloc[0].strftime('%Y-%m-%d %H:%M:%S')
# 回测第一天，计算初始值
U.last_trading_point_time = 'init_trading_point_time'
U.last_open = 'init_open'
# 初始化中枢和三买df
U.pivot_df = pd.DataFrame()
U.three_point_df = pd.DataFrame()
prices_init = prices[prices['DATETIME'] <= U.high_level_begin_time]
prices_init_high_level = prices_high_level[prices_high_level['DATETIME'] <= U.high_level_begin_time]
prices_adj, prices_part, prices_adj_high_level, prices_part_high_level, pivot_latest, trading_point_level_3 \
= three_point.three_point_init_level2(prices_init, prices_init_high_level)
U.pivot_df = U.pivot_df.append(pd.DataFrame({U.high_level_begin_time: pivot_latest}).T)
if trading_point_level_3 != {}:
U.three_point_df = U.three_point_df.append(
pd.DataFrame({U.high_level_begin_time: trading_point_level_3}).T)
if pd.Timestamp(U.first_start + ' 09:45:00').strftime('%Y-%m-%d %H:%M:%S') > U.high_level_begin_time:
prices_init = prices[(U.high_level_begin_time < prices['DATETIME'] )&(prices['DATETIME'] <= today+ ' 09:45:00')]
# print('\n*****',prices_init)
for datetime in prices_init['DATETIME']:
prices_latest = prices.loc[prices['DATETIME'] == datetime, :]
prices_latest_high_level = prices_high_level.loc[prices_high_level['DATETIME'] == datetime, :]
prices_adj, prices_part, prices_adj_high_level, prices_part_high_level, pivot_latest, trading_point_level_3 = \
three_point.three_point_latest_level2_v0827(prices_latest, prices_latest_high_level, prices_adj,
prices_part, \
prices_adj_high_level, prices_part_high_level, pivot_latest,
trading_point_level_3)
U.pivot_df = U.pivot_df.append(pd.DataFrame({datetime.strftime('%Y-%m-%d %H:%M:%S'): pivot_latest}).T)
if trading_point_level_3 != {}:
U.three_point_df = U.three_point_df.append(
pd.DataFrame({datetime.strftime('%Y-%m-%d %H:%M:%S'): trading_point_level_3}).T)
else:
pass
U.compare_time = max(pd.Timestamp(U.first_start + ' 09:45:00').strftime('%Y-%m-%d %H:%M:%S'),U.high_level_begin_time)
else:
# 承接上一产品信息
prices_adj = pd.read_pickle(
f'./第三买卖点/strategy_save/{U.py_name}_{U.first_start}_{U.last_end}/prices_adj_{U.py_name}_{U.first_start}_{U.latest_end}.pkl')
prices_part = pd.read_pickle(
f'./第三买卖点/strategy_save/{U.py_name}_{U.first_start}_{U.last_end}/prices_part_{U.py_name}_{U.first_start}_{U.latest_end}.pkl')
prices_adj_high_level = pd.read_pickle(
f'./第三买卖点/strategy_save/{U.py_name}_{U.first_start}_{U.last_end}/prices_adj_high_level_{U.py_name}_{U.first_start}_{U.latest_end}.pkl')
prices_part_high_level = pd.read_pickle(
f'./第三买卖点/strategy_save/{U.py_name}_{U.first_start}_{U.last_end}/prices_part_high_level_{U.py_name}_{U.first_start}_{U.latest_end}.pkl')
pivot_latest_series = pd.read_pickle(
f'./第三买卖点/strategy_save/{U.py_name}_{U.first_start}_{U.last_end}/pivot_latest_{U.py_name}_{U.first_start}_{U.latest_end}.pkl')
# pivot_latest = pivot_latest_series.to_dict()
pivot_latest = strategy.dict_no_nat(pivot_latest_series)
trading_point_series = pd.read_pickle(
f'./第三买卖点/strategy_save/{U.py_name}_{U.first_start}_{U.last_end}/trading_point_level_3_{U.py_name}_{U.first_start}_{U.latest_end}.pkl')
# trading_point_level_3 = trading_point_series.to_dict()
trading_point_level_3 = strategy.dict_no_nat(trading_point_series)
last_point_series = pd.read_pickle(
f'./第三买卖点/strategy_save/{U.py_name}_{U.first_start}_{U.last_end}/last_trading_point_time_{U.py_name}_{U.first_start}_{U.latest_end}.pkl')
U.last_trading_point_time = last_point_series.iloc[0]
U.last_open = last_point_series.iloc[1]
U.can_close = last_point_series.iloc[2]
U.last_pivot = last_point_series.iloc[3]
# print('\n','in_pivot:',U.last_pivot,'\n',last_point_series)
U.pivot_df = pd.read_pickle(f'./第三买卖点/strategy_save/{U.py_name}_{U.first_start}_{U.last_end}/pivot_df_{U.py_name}_{U.first_start}_{U.latest_end}.pkl')
U.three_point_df = pd.read_pickle(
f'./第三买卖点/strategy_save/{U.py_name}_{U.first_start}_{U.last_end}/three_point_df_{U.py_name}_{U.first_start}_{U.latest_end}.pkl')
prices_init = prices[prices['DATETIME'] < today + ' 09:45:00']
for datetime in prices_init['DATETIME']:
prices_latest = prices.loc[prices['DATETIME'] == datetime, :]
prices_latest_high_level = prices_high_level.loc[prices_high_level['DATETIME'] == datetime, :]
prices_adj, prices_part, prices_adj_high_level, prices_part_high_level, pivot_latest, trading_point_level_3 = \
three_point.three_point_latest_level2(prices_latest, prices_latest_high_level, prices_adj,
prices_part, \
prices_adj_high_level, prices_part_high_level, pivot_latest,
trading_point_level_3)
U.pivot_df = U.pivot_df.append(pd.DataFrame({datetime.strftime('%Y-%m-%d %H:%M:%S'): pivot_latest}).T)
if trading_point_level_3 != {}:
U.three_point_df = U.three_point_df.append(
pd.DataFrame({datetime.strftime('%Y-%m-%d %H:%M:%S'): trading_point_level_3}).T)
elif today != U.start:
# 承接同一产品中上一日的信息
prices_adj = U.prices_adj
prices_part = U.prices_part
prices_adj_high_level = U.prices_adj_high_level
prices_part_high_level = U.prices_part_high_level
pivot_latest = U.pivot_latest
trading_point_level_3 = U.trading_point_level_3
prices_init = prices[prices['DATETIME'] < today + ' 09:45:00']
for datetime in prices_init['DATETIME']:
prices_latest = prices.loc[prices['DATETIME'] == datetime, :]
prices_latest_high_level = prices_high_level.loc[prices_high_level['DATETIME'] == datetime, :]
prices_adj, prices_part, prices_adj_high_level, prices_part_high_level, pivot_latest, trading_point_level_3 = \
three_point.three_point_latest_level2(prices_latest, prices_latest_high_level, prices_adj, prices_part, \
prices_adj_high_level, prices_part_high_level, pivot_latest,
trading_point_level_3)
U.pivot_df = U.pivot_df.append(pd.DataFrame({datetime.strftime('%Y-%m-%d %H:%M:%S'): pivot_latest}).T)
if trading_point_level_3 != {}:
U.three_point_df = U.three_point_df.append(
pd.DataFrame({datetime.strftime('%Y-%m-%d %H:%M:%S'): trading_point_level_3}).T)
U.prices = prices
U.prices_high_level = prices_high_level
U.current_state = "WAITING"
# U.per_size = per_size
U.symbol = symbol # .CFE
U.symbols = symbols # .CF
U.prices_adj = prices_adj
U.prices_part = prices_part
U.prices_adj_high_level = prices_adj_high_level
U.prices_part_high_level = prices_part_high_level
U.pivot_latest = pivot_latest
U.trading_point_level_3 = trading_point_level_3
U.date = 'date_init'
U.today = today