def get_kline_data(self):
kline_data = GetKlineData()
all_stock_data = kline_data.cache_all_stock_data()
self.kline = kline_data.get_market_data(all_stock_data,
stock_code=self.stock_list,
field=['close'])
return self.kline
def get_benchmark_net_asset_value(self, period, benchmark):
data_class = GetKlineData()
benchmark_close = None
if period == Period.DAILY.value:
start_time = Environment.benchmark_index[0]
end_time = Environment.benchmark_index[-1]
benchmark_close = data_class.get_market_data(
Environment.index_daily_data,
stock_code=[benchmark],
field=['close'],
start=start_time,
end=end_time)
elif period == Period.ONE_MIN.value:
start_time = millisecond_to_date(
millisecond=Environment.benchmark_index[0],
format='%Y-%m-%d %H:%M:%S')
end_time = millisecond_to_date(
millisecond=Environment.benchmark_index[-1],
format='%Y-%m-%d %H:%M:%S')
benchmark_close = data_class.get_market_data(
Environment.index_daily_data,
stock_code=[benchmark],
field=['close'],
start=start_time,
end=end_time)
benchmark_close = list(benchmark_close)
benchmark_net_asset_value = [
current_close / benchmark_close[0]
for current_close in benchmark_close
]
return benchmark_net_asset_value
def show_backtesting_indicator(cls, event):
benchmark = event.event_data_dict['strategy_data'].benchmark
account = event.event_data_dict['strategy_data'].account
data_class = GetKlineData()
start_time = Environment.benchmark_index[0]
end_time = Environment.benchmark_index[-1]
account_df = Environment.backtesting_record_account
benchmark_df = data_class.get_market_data(Environment.index_daily_data, stock_code=[benchmark],
field=['close'], ).to_frame(name='close')
account_df = account_df[account_df.index.get_level_values(1) == account[0]]
account_df.reset_index(level='account_id', drop=True, inplace=True)
net_value_analysis_obj = NetValueAnalysis(account_df, benchmark_df, start_time, end_time)
net_analysis_result = net_value_analysis_obj.cal_net_analysis_result()
for i in net_analysis_result:
Environment.logger.info(i, net_analysis_result[i])
# 持仓数据转pandas
position_data_df = Environment.backtesting_record_position
position_data_df = position_data_df[position_data_df.index.get_level_values(1) == account[0]]
position_analysis_obj = PositionAnalysis(position_data_df)
position_analysis_result = position_analysis_obj.cal_position_analysis_result()
for i in position_analysis_result:
Environment.logger.info(i, position_analysis_result[i])
def update_a_share_capitalization(self):
"""
保存 总股本,总市值, 流通股本,流通市值 四个hdf5
:return:
"""
with MongoConnect(self.database):
a_share_capitalization = AShareCapitalization.objects().as_pymongo()
field_list = ['security_code', 'change_date', 'total_share', 'float_share', 'float_a_share',
'float_b_share', 'float_h_share']
self.a_share_capitalization = pd.DataFrame(list(a_share_capitalization)).reindex(columns=field_list)
kline_object = GetKlineData()
market_close_data = kline_object.cache_all_stock_data()['close']
index = list(set(market_close_data.index).union(set(self.a_share_capitalization['change_date'])))
share_capitalization_grouped = self.a_share_capitalization.groupby('security_code')
share_capitalization = pd.DataFrame(index=index)
for i in share_capitalization_grouped:
data = i[1].reindex(['change_date', 'total_share'], axis=1).sort_values('change_date').set_index(
'change_date')
try:
share_capitalization[i[0]] = data
except ValueError:
# 有四只票 change date 重复,需要手工清洗修正
# print(data)
share_capitalization[i[0]] = data[data.index.duplicated()]
share_capitalization = share_capitalization.fillna(method='ffill').reindex(market_close_data.index)
return share_capitalization.multiply(10000) * market_close_data
def save_a_share_adj_factor_right(self):
"""
取当日收盘价，作为转、送的股价，
再计算复权因子更新到AShareExRightDividend, 复权因子adj_factor
比例 = 送股比例 + 转增比例 + 缩减比例
单次复权因子 = 股权登记日收盘价 * (1 + 比例 + 配股比例 + 增发比例) /
(股权登记日收盘价 - 派息比例 + 股权登记日收盘价 * 比例 + 配股价格 * 配股比例 + 增发价格 * 增发比例)
:return:
"""
kline_object = GetKlineData()
all_market_data = kline_object.cache_all_stock_data()
with MongoConnect(self.database):
self.data = pd.DataFrame(
AShareExRightDividend.objects.as_pymongo())
self.data['close'] = self.data.apply(
lambda x: self.get_adj_day_close(x['security_code'], x[
'ex_date'], all_market_data),
axis=1)
self.data = self.data.fillna(0)
ratio = self.data['bonus_share_ratio'] + self.data[
'conversed_ratio'] + self.data['consolidate_split_ratio']
self.data['adj_factor'] = self.data['close'] * (
1 + ratio + self.data['rightsissue_ratio'] +
self.data['seo_ratio']
) / (self.data['close'] - self.data['cash_dividend_ratio'] +
self.data['close'] * ratio + self.data['rightsissue_price'] *
self.data['rightsissue_ratio'] +
self.data['seo_price'] * self.data['seo_ratio'])
folder_name = LocalDataFolderName.ADJ_FACTOR.value
path = LocalDataPath.path + folder_name + '/'
self.data = self.data.reindex(
columns=['security_code', 'ex_date', 'adj_factor'])
self.data.set_index(["ex_date"], inplace=True)
self.data.sort_index(inplace=True)
calendar_obj = GetCalendar()
calendar = calendar_obj.get_calendar('SZ')
backward_factor = pd.DataFrame(index=calendar)
adj_factor = pd.DataFrame(index=calendar)
data_dict = dict(
list(self.data.groupby(self.data['security_code'])))
for security_code, adj_data in data_dict.items():
backward_factor[security_code] = self.cal_backward_factor(
adj_data['adj_factor'])
adj_factor[security_code] = adj_data['adj_factor']
backward_factor.replace([np.inf, 0], np.nan, inplace=True)
backward_factor.fillna(method='ffill', inplace=True)
backward_factor.fillna(1, inplace=True)
backward_factor = backward_factor.reindex(
columns=all_market_data['close'].columns, fill_value=1)
save_data_to_hdf5(path, AdjustmentFactor.BACKWARD_ADJ_FACTOR.value,
backward_factor)
save_data_to_hdf5(path, AdjustmentFactor.FROWARD_ADJ_FACTOR.value,
backward_factor.div(backward_factor.iloc[-1]))
def update_a_share_capitalization(self):
"""
保存 总股本,总市值, 流通股本,流通市值 四个hdf5
:return:
"""
with MongoConnect(self.database):
a_share_capitalization = AShareCapitalization.objects().as_pymongo(
)
field_list = [
'security_code', 'change_date', 'total_share', 'float_share',
'float_a_share', 'float_b_share', 'float_h_share'
]
self.a_share_capitalization = pd.DataFrame(
list(a_share_capitalization)).reindex(columns=field_list)
kline_object = GetKlineData()
market_close_data = kline_object.cache_all_stock_data()['close']
index = list(
set(market_close_data.index).union(
set(self.a_share_capitalization['change_date'])))
index.sort()
share_capitalization_grouped = self.a_share_capitalization.groupby(
'security_code')
total_share = pd.DataFrame({})
float_a_share = pd.DataFrame({})
for i in share_capitalization_grouped:
data = i[1].sort_values('change_date').set_index('change_date')
try:
total_share[i[0]] = data['total_share'].reindex(index)
float_a_share[i[0]] = data['float_a_share'].reindex(index)
except ValueError:
# 有四只票 change date 重复,需要手工清洗修正
# print(data[data.index.duplicated()])
total_share[i[0]] = data[
data.index.duplicated()]['total_share'].reindex(index)
float_a_share[i[0]] = data[data.index.duplicated(
)]['float_a_share'].reindex(index)
total_share = total_share.fillna(method='ffill').reindex(
market_close_data.index)
float_a_share = float_a_share.fillna(method='ffill').reindex(
market_close_data.index)
total_share_value = total_share.multiply(10000) * market_close_data
float_a_share_value = float_a_share.multiply(
10000) * market_close_data
folder_name = LocalDataFolderName.INDICATOR_EVERYDAY.value
path = LocalDataPath.path + folder_name + '/'
save_data_to_hdf5(path, 'total_share', total_share)
save_data_to_hdf5(path, 'float_a_share', float_a_share)
save_data_to_hdf5(path, 'total_share_value', total_share_value)
save_data_to_hdf5(path, 'float_a_share_value', float_a_share_value)
def update_market_data(cls, event):
current_date = event.event_data_dict["strategy_data"].time_tag
data_class = GetKlineData()
stock_code_list = []
for position_data in Environment.bar_position_data_list:
stock_code_list.append(position_data.instrument + "." +
position_data.exchange)
stock_code_list = list(set(stock_code_list))
cls.current_close_price_all = data_class.get_market_data(
Environment.daily_data,
stock_code=stock_code_list,
field=["close"],
start=current_date,
end=current_date)
def __init__(self):
self._run_mode = RunMode.BACKTESTING.value
self._account = []
self._capital = 1000000
self._start = datetime(2017, 1, 1)
self._end = datetime(2018, 1, 1)
self._benchmark = '000300.SH'
self._period = Period.DAILY.value # 后续支持1min 3min 5min 等多周期
self._universe = []
self._rights_adjustment = RightsAdjustment.FROWARD.value
self._time_tag = 0
# 数据缓存开关
self._daily_data_cache = False
self._one_min_data_cache = False
# 取数据
self._get_data = GetKlineData()
self.bar_index = 0
def __init__(self, factor, factor_name):
self.factor = factor
self.factor_name = factor_name
market_data = GetKlineData() \
.cache_all_stock_data(dividend_type=RightsAdjustment.BACKWARD.value, field=['close'])['close'] \
.reindex(factor.index) \
.reindex(factor.columns, axis=1)
self.ic_decay = 20
column_list = [factor_name + '_' + str(i + 1) for i in range(self.ic_decay)]
self.stock_return_dict = {i + 1: market_data.pct_change(periods=i + 1) for i in range(self.ic_decay)}
# IC信号衰减计算，index 是时间序列， columns是decay周期，[1, self.ic_decay], 闭区间
self.ic_df = pd.DataFrame(columns=column_list)
self.p_value_df = pd.DataFrame(columns=column_list)
# IC均值、 IC标准差、 IC_IR比率、 IC > 0 占比、 | IC | > 0.02 占比(绝对值)、 偏度、 峰度
index_list = ['ic_mean', 'ic_std', 'ic_ir', 'ic_ratio', 'ic_abs_ratio', 'ic_skewness', 'ic_kurtosis']
self.ic_result = pd.DataFrame(index=index_list, columns=column_list)
factor_t_value=factor_t_value,
# 单因子检测的T值的统计值，'t_value_mean': 绝对值均值, 't_value_greater_two':绝对值序列大于2的占比
factor_t_value_statistics=self.factor_t_value_statistics,
# 净值分析结果
net_analysis_result=self.net_analysis_result)
doc.save()
if __name__ == '__main__':
factor_name = 'factor_ma10'
path = LocalDataPath.path + LocalDataFolderName.FACTOR.value + '/'
factor_ma5 = get_local_data(path, factor_name + '.h5')
# 指数数据不全，需要删一部分因子数据
factor_ma5 = factor_ma5[factor_ma5.index < datetime.datetime(2020, 1, 1)]
kline_object = GetKlineData()
market_data = kline_object.cache_all_stock_data(
dividend_type=RightsAdjustment.BACKWARD.value, field=['close'])
market_close_data = kline_object.get_market_data(market_data,
field=['close'])
# 指数行情，沪深300代替
all_index_data = kline_object.cache_all_index_data()
benchmark_df = kline_object.get_market_data(
all_index_data, stock_code=['000300.SH'],
field=['close']).to_frame(name='close')
# 沪深300 的日线，有脏数据，后续单独处理
if datetime.datetime(2016, 1, 1) in benchmark_df.index:
benchmark_df = benchmark_df.drop(datetime.datetime(2016, 1, 1))
regression_analysis_obj = RegressionAnalysis(factor_ma5, 'factor_ma5',
market_close_data,
# 偏度，峰度
net_skewness, net_kurtosis = self.cal_skew_kurt(
self.net_value_df['profit_ratio'])
net_analysis_result['net_skewness'] = net_skewness
net_analysis_result['net_kurtosis'] = net_kurtosis
benchmark_skewness, benchmark_kurtosis = self.cal_skew_kurt(
self.benchmark_df['profit_ratio'])
net_analysis_result['benchmark_skewness'] = benchmark_skewness
net_analysis_result['benchmark_kurtosis'] = benchmark_kurtosis
return net_analysis_result
if __name__ == '__main__':
start_time = datetime(2018, 1, 2)
end_time = datetime(2018, 12, 28)
kline_object = GetKlineData()
#
# 指数行情，沪深300代替
all_index_data = kline_object.cache_all_index_data()
benchmark_df = kline_object.get_market_data(all_index_data, stock_code=['000300.SH'], field=['close'], ) \
.to_frame(name='close')
# 策略净值数据,index 为 datetime,取单个账户分析，后续可做多个账户
net_value_df = pd.read_csv('account_data.csv', index_col=0)
net_value_df.index = pd.DatetimeIndex(net_value_df.index)
net_value_single_account_df = pd.DataFrame({})
for i in net_value_df.groupby('account_id'):
net_value_single_account_df = i[1]
break
net_value_analysis_obj = NetValueAnalysis(net_value_single_account_df,
benchmark_df, start_time,
class StrategyBase(metaclass=ABCMeta):
def __init__(self, strategy_name='ma_strategy'):
self._strategy_name = strategy_name
self._run_mode = RunMode.BACKTESTING.value
self._account = []
self._capital = 1000000
self._start = datetime(2017, 1, 1)
self._end = datetime(2018, 1, 1)
self._benchmark = '000300.SH'
self._period = Period.DAILY.value # 后续支持1min 3min 5min 等多周期
self._universe = []
self._rights_adjustment = RightsAdjustment.FROWARD.value
self._time_tag = 0
# 数据缓存开关
self._daily_data_cache = False
self._one_min_data_cache = False
# 取数据
self._get_data = GetKlineData()
self.bar_index = 0
@property
def strategy_name(self):
return self._strategy_name
@strategy_name.setter
def strategy_name(self, value):
self._strategy_name = value
@property
def run_mode(self):
return self._run_mode
@run_mode.setter
def run_mode(self, value):
self._run_mode = value
@property
def account(self):
return self._account
@account.setter
def account(self, value):
self._account = value
@property
def capital(self):
return self._capital
@capital.setter
def capital(self, value):
self._capital = value
@property
def start(self):
return self._start
@start.setter
def start(self, value):
self._start = value
@property
def end(self):
return self._end
@end.setter
def end(self, value):
self._end = value
@property
def benchmark(self):
return self._benchmark
@benchmark.setter
def benchmark(self, value):
self._benchmark = value
@property
def period(self):
return self._period
@period.setter
def period(self, value):
self._period = value
@property
def universe(self):
return self._universe
@universe.setter
def universe(self, value):
self._universe.extend(value)
@property
def rights_adjustment(self):
return self._rights_adjustment
@rights_adjustment.setter
def rights_adjustment(self, value):
self._rights_adjustment = value
@property
def time_tag(self):
return self._time_tag
@time_tag.setter
def time_tag(self, value):
self._time_tag = value
@property
def daily_data_cache(self):
return self._daily_data_cache
@daily_data_cache.setter
def daily_data_cache(self, value):
self._daily_data_cache = value
@property
def one_min_data_cache(self):
return self._one_min_data_cache
@one_min_data_cache.setter
def one_min_data_cache(self, value):
self._one_min_data_cache = value
# 回测滑点设置
def set_slippage(self,
stock_type=StockType.STOCK.value,
slippage_type=SlippageType.SLIPPAGE_FIX.value,
value=0):
Environment.slippage_dict[stock_type] = {
'slippage_type': slippage_type,
'value': value
}
# 回测手续费和印花税
def set_commission(self,
stock_type=StockType.STOCK.value,
tax=0,
open_commission=0,
close_commission=0,
close_today_commission=0,
min_commission=0):
Environment.commission_dict[stock_type] = {
'tax': tax,
'open_commission': open_commission,
'close_commission': close_commission,
'close_today_commission': close_today_commission,
'min_commission': min_commission
}
def run(self, save_trade_record=False):
self.initialize()
# 初始化 account_data
if self.account:
for account in self.account:
Environment.current_account_data = AccountData()
Environment.current_account_data.account_id = generate_random_id.generate_random_id(
account)
Environment.current_account_data.total_balance = self.capital[
account]
Environment.current_account_data.available = self.capital[
account]
# Environment.logger(Environment.current_account_data.account_id, Environment.current_account_data.available)
Environment.bar_account_data_list.append(
Environment.current_account_data)
# if self.run_mode == RunMode.TRADE.value:
# self.end = self._get_data.get_end_time_tag(benchmark=self.benchmark, period=Period.DAILY.value)
# 缓存数据开关，和bar_index的计算
if self.period == Period.DAILY.value:
self.daily_data_cache = True
elif self.period == Period.ONE_MIN.value:
self.one_min_data_cache = True
#
security_list = copy.copy(self.universe)
security_list = list(set(security_list))
if self._daily_data_cache:
Environment.daily_data = self._get_data.cache_all_stock_data(
dividend_type=self.rights_adjustment)
Environment.index_daily_data = self._get_data.cache_all_index_data(
)
if self.one_min_data_cache:
Environment.one_min_data = self._get_data.cache_all_stock_data(
period=Period.ONE_MIN.value)
if self.period == Period.DAILY.value:
Environment.benchmark_index = [
i for i in Environment.index_daily_data['close'][
self.benchmark].index if self.start <= i <= self.end
]
elif self.period == Period.ONE_MIN.value:
Environment.benchmark_index = [
data_transfer.date_to_millisecond(str(int(i)), '%Y%m%d') for i
in Environment.one_min_data['open'].ix[self.benchmark].index
if i >= data_transfer.date_str_to_int(self.start)
]
self.bar_index = 0
while True:
try:
# Environment.logger(self.time_tag, Environment.benchmark_index)
self.time_tag = Environment.benchmark_index[self.bar_index]
except IndexError:
if self.run_mode == RunMode.BACKTESTING.value:
if save_trade_record:
run_backtesting_analysis_engine(self)
break
# elif self.run_mode == RunMode.TRADE.value:
# 读取最新tick, 更新最新的分钟或者日线
# if 读取最新tick, 更新最新的分钟或者日线 == done:
# daily_data.append(new_day_data)
# self.bar_index += 1
# benchmark_index.append(new_day_time_tag)
else:
run_bar_engine(self)
@abstractmethod
def initialize(self):
"""
设置运行模式，回测、实盘
设置on_bar运行周期，分钟线、日线
设置缓存历史数据范围、实时行情订阅
设置回测的各种设置。手续费、滑点、回测区间等
:return:
"""
pass
def on_bar(self, event):
"""
支持分钟线、日线的处理
:param event:
:return:
"""
Environment.logger('abstractmethod handle_bar')
pass
def on_quote(self, event):
"""
level1行情数据的推送会自动触发该方法的调用,每只股票数据推送都调用
:param event:
:return:
"""
pass
def on_transaction(self, event):
"""
逐笔成交行情数据的推送会自动触发该方法的调用,每只股票数据推送都调用
:param event:
:return:
"""
pass
def on_entrust(self, event):
"""
逐笔委托行情数据的推送会自动触发该方法的调用,每只股票数据推送都调用
:param event:
:return:
"""
pass
def on_order(self, event):
"""
策略订单成交信息的更新会自动触发该方法的调用。
:param event:
:return:
"""
pass
def on_cancel_order(self, event):
"""
策略订单撤单信息的更新会自动触发该方法的调用。
:param event:
:return:
"""
pass
def on_trade(self, event):
"""
策略订单委托信息的更新会自动触发该方法的调用。
:param event:
:return:
"""
pass
def on_account(self, event):
"""
策略账户信息的更新会自动触发该方法的调用。支持多账户，股票，期货
:param event:
:return:
"""
pass
ic_df = self.ic_df.copy()
p_value_df = self.p_value_df.copy()
ic_df.index = ic_df.index.format()
p_value_df.index = p_value_df.index.format()
doc = FactorIcAnalysisResult(
factor_name=factor_name,
# 因子数据开始时间
begin_date=self.factor.index[0],
# 因子数据结束时间
end_date=self.factor.index[-1],
# IC信号衰减计算，index 是时间序列， columns是decay周期，[1, self.ic_decay], 闭区间
ic=ic_df,
# p值信号衰减计算，index 是时间序列， columns是decay周期，[1, self.ic_decay], 闭区间
p_value=p_value_df,
ic_result=self.ic_result)
doc.save()
if __name__ == '__main__':
path = LocalDataPath.path + LocalDataFolderName.FACTOR.value + '/'
factor_name = 'factor_ma10'
factor_ma5 = get_local_data(path, factor_name + '.h5')
market_close_data = GetKlineData().cache_all_stock_data(
dividend_type=RightsAdjustment.BACKWARD.value,
field=['close'])['close']
ic_analysis_obj = IcAnalysis(factor_ma5, factor_name, market_close_data)
ic_analysis_obj.cal_ic_df(method='spearmanr')
ic_analysis_obj.cal_ic_indicator()
ic_analysis_obj.save_ic_analysis_result(factor_name)
