def test_backtest_analyze():
ta = ana.AlphaAnalyzer()
dv = DataView()
dv.load_dataview(folder_path=dataview_dir_path)
ta.initialize(dataview=dv, file_folder=backtest_result_dir_path)
print "process trades..."
ta.process_trades()
print "get daily stats..."
ta.get_daily()
print "calc strategy return..."
ta.get_returns(consider_commission=True)
# position change info is huge!
# print "get position change..."
# ta.get_pos_change_info()
selected_sec = list(ta.universe)[:2]
if len(selected_sec) > 0:
print "Plot single securities PnL"
for symbol in selected_sec:
df_daily = ta.daily.get(symbol, None)
if df_daily is not None:
ana.plot_trades(df_daily, symbol=symbol, save_folder=backtest_result_dir_path)
print "Plot strategy PnL..."
ta.plot_pnl(backtest_result_dir_path)
print "generate report..."
static_folder = fileio.join_relative_path("trade/analyze/static")
ta.gen_report(source_dir=static_folder, template_fn='report_template.html',
out_folder=backtest_result_dir_path,
selected=selected_sec)
def save_dataview(sub_folder='test_dataview'):
ds = RemoteDataService()
dv = DataView()
props = {
'start_date':
20141114,
'end_date':
20160327,
'universe':
'000300.SH',
'fields': (
'open,high,low,close,vwap,volume,turnover,'
# + 'pb,net_assets,'
+ 's_fa_eps_basic,oper_exp,tot_profit,int_income'),
'freq':
1
}
dv.init_from_config(props, ds)
dv.prepare_data()
factor_formula = 'close > Ts_Max(close, 20)' # 20 days new high
factor_name = 'new_high'
dv.add_formula(factor_name, factor_formula, is_quarterly=False)
dv.save_dataview(
folder_path=fileio.join_relative_path('../output/prepared'),
sub_folder=sub_folder)
def test_backtest_analyze():
ta = ana.AlphaAnalyzer()
dv = DataView()
dv.load_dataview(folder_path=dataview_dir_path)
ta.initialize(dataview=dv, file_folder=backtest_result_dir_path)
ta.do_analyze(result_dir=backtest_result_dir_path, selected_sec=list(ta.universe)[:3])
def test_alpha_strategy_dataview():
save_dataview()
dv = DataView()
dv.load_dataview(folder_path=dataview_dir_path)
props = {
"start_date": dv.start_date,
"end_date": dv.end_date,
"period": "week",
"days_delay": 0,
"init_balance": 1e8,
"position_ratio": 0.7,
'commission_rate': 0.0
}
trade_api = AlphaTradeApi()
bt = AlphaBacktestInstance()
risk_model = model.FactorRiskModel()
signal_model = model.FactorRevenueModel()
cost_model = model.SimpleCostModel()
stock_selector = model.StockSelector()
signal_model.add_signal(name='my_factor', func=my_factor)
cost_model.consider_cost(name='my_commission',
func=my_commission,
options={'myrate': 1e-2})
stock_selector.add_filter(name='total_profit_growth', func=my_selector)
stock_selector.add_filter(name='no_new_stocks',
func=my_selector_no_new_stocks)
strategy = AlphaStrategy(revenue_model=signal_model,
stock_selector=stock_selector,
cost_model=cost_model,
risk_model=risk_model,
pc_method='factor_value_weight')
pm = PortfolioManager()
# strategy = AlphaStrategy(revenue_model=signal_model, pc_method='factor_value_weight')
# strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='market_value_weight')
# strategy = AlphaStrategy()
context = model.AlphaContext(dataview=dv,
trade_api=trade_api,
instance=bt,
strategy=strategy,
pm=pm)
for mdl in [risk_model, signal_model, cost_model, stock_selector]:
mdl.register_context(context)
bt.init_from_config(props)
bt.run_alpha()
bt.save_results(folder_path=backtest_result_dir_path)
def test_alpha_strategy_dataview():
dv_subfolder_name = 'test_dataview'
save_dataview(sub_folder=dv_subfolder_name)
dv = DataView()
fullpath = fileio.join_relative_path('../output/prepared',
dv_subfolder_name)
dv.load_dataview(folder=fullpath)
props = {
"benchmark": "000300.SH",
# "symbol": ','.join(dv.symbol),
"universe": ','.join(dv.symbol),
"start_date": dv.start_date,
"end_date": dv.end_date,
"period": "month",
"days_delay": 0,
"init_balance": 1e9,
"position_ratio": 0.7,
}
gateway = DailyStockSimGateway()
gateway.init_from_config(props)
context = model.Context()
context.register_gateway(gateway)
context.register_trade_api(gateway)
context.register_dataview(dv)
risk_model = model.FactorRiskModel()
signal_model = model.FactorRevenueModel_dv()
cost_model = model.SimpleCostModel()
risk_model.register_context(context)
signal_model.register_context(context)
cost_model.register_context(context)
signal_model.register_func('my_factor', my_factor)
signal_model.activate_func({'my_factor': {}})
cost_model.register_func('my_commission', my_commission)
cost_model.activate_func({'my_commission': {'myrate': 1e-2}})
strategy = DemoAlphaStrategy(risk_model, signal_model, cost_model)
# strategy.active_pc_method = 'equal_weight'
# strategy.active_pc_method = 'mc'
strategy.active_pc_method = 'factor_value_weight'
bt = AlphaBacktestInstance_dv()
bt.init_from_config(props, strategy, context=context)
bt.run_alpha()
bt.save_results(fileio.join_relative_path('../output/'))
def test_write():
ds = RemoteDataService()
ds.init_from_config()
dv = DataView()
secs = '600030.SH,000063.SZ,000001.SZ'
props = {
'start_date': 20160601,
'end_date': 20170601,
'symbol': secs,
'fields': 'open,close,high,low,volume,pb,net_assets,pcf_ncf',
'freq': 1
}
dv.init_from_config(props, data_api=ds)
dv.prepare_data()
assert dv.data_d.shape == (281, 48)
assert dv.dates.shape == (281, )
# TODO
"""
PerformanceWarning:
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->mixed,key->block1_values] [items->[('000001.SZ', 'int_income'), ('000001.SZ', 'less_handling_chrg_comm_exp'), ('000001.SZ', 'net_int_income'), ('000001.SZ', 'oper_exp'), ('000001.SZ', 'symbol'), ('000063.SZ', 'int_income'), ('000063.SZ', 'less_handling_chrg_comm_exp'), ('000063.SZ', 'net_int_income'), ('000063.SZ', 'oper_exp'), ('000063.SZ', 'symbol'), ('600030.SH', 'int_income'), ('600030.SH', 'less_handling_chrg_comm_exp'), ('600030.SH', 'net_int_income'), ('600030.SH', 'oper_exp'), ('600030.SH', 'symbol')]]
"""
dv.save_dataview(folder_path=daily_path)
def test_alpha_strategy_dataview():
dv = DataView()
dv.load_dataview(folder_path=dataview_dir_path)
props = {
"benchmark": "000300.SH",
"universe": ','.join(dv.symbol),
"start_date": dv.start_date,
"end_date": dv.end_date,
"period": "month",
"days_delay": 0,
"init_balance": 1e8,
"position_ratio": 1.0,
}
trade_api = AlphaTradeApi()
trade_api.init_from_config(props)
def selector_growth(context, user_options=None):
growth_rate = context.snapshot['net_profit_growth']
return (growth_rate >= 0.2) & (growth_rate <= 4)
def selector_pe(context, user_options=None):
pe_ttm = context.snapshot['pe_ttm']
return (pe_ttm >= 10) & (pe_ttm <= 20)
stock_selector = model.StockSelector()
stock_selector.add_filter(name='net_profit_growth', func=selector_growth)
stock_selector.add_filter(name='pe', func=selector_pe)
strategy = AlphaStrategy(stock_selector=stock_selector,
pc_method='equal_weight')
pm = PortfolioManager()
bt = AlphaBacktestInstance()
context = model.Context(dataview=dv,
instance=bt,
strategy=strategy,
trade_api=trade_api,
pm=pm)
stock_selector.register_context(context)
bt.init_from_config(props)
bt.run_alpha()
bt.save_results(folder_path=backtest_result_dir_path)
def test_remote_data_service_industry():
from jaqs.data.align import align
import pandas as pd
ds = RemoteDataService()
arr = ds.get_index_comp(index='000300.SH', start_date=20130101, end_date=20170505)
df = ds.get_industry_raw(symbol=','.join(arr), type_='ZZ')
df = df.astype(dtype={'in_date': int})
# df_ann = df.loc[:, ['in_date', 'symbol']]
# df_ann = df_ann.set_index(['symbol', 'in_date'])
# df_ann = df_ann.unstack(level='symbol')
from jaqs.data.dataview import DataView
dic_sec = DataView._group_df_to_dict(df, by='symbol')
dic_sec = {sec: df.reset_index() for sec, df in dic_sec.viewitems()}
df_ann = pd.concat([df.loc[:, 'in_date'].rename(sec) for sec, df in dic_sec.viewitems()], axis=1)
df_value = pd.concat([df.loc[:, 'industry1_code'].rename(sec) for sec, df in dic_sec.viewitems()], axis=1)
dates_arr = ds.get_trade_date(20140101, 20170505)
res = align(df_value, df_ann, dates_arr)
# df_ann = df.pivot(index='in_date', columns='symbol', values='in_date')
# df_value = df.pivot(index=None, columns='symbol', values='industry1_code')
def align_single_df(df_one_sec):
df_value = df_one_sec.loc[:, ['industry1_code']]
df_ann = df_one_sec.loc[:, ['in_date']]
res = align(df_value, df_ann, dates_arr)
return res
# res_list = [align_single_df(df) for sec, df in dic_sec.viewitems()]
res_list = [align_single_df(df) for sec, df in dic_sec.items()[:10]]
res = pd.concat(res_list, axis=1)
def test_save_dataview():
ds = RemoteDataService()
ds.init_from_config()
dv = DataView()
props = {'start_date': 20170201, 'end_date': 20171001, 'universe': '000300.SH',
'fields': ('float_mv,sw2,sw1'),
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
factor_formula = 'GroupQuantile(float_mv, sw1, 10)'
dv.add_formula('gq30', factor_formula, is_quarterly=False)
dv.save_dataview(folder_path=dataview_dir_path)
def test_q_add_formula():
dv = DataView()
folder_path = '../output/prepared/20160609_20170601_freq=1D'
dv.load_dataview(folder_path=quarterly_path)
nrows, ncols = dv.data_d.shape
n_securities = len(dv.data_d.columns.levels[0])
formula = 'total_oper_rev / close'
dv.add_formula('myvar1', formula, is_quarterly=False)
df1 = dv.get_ts('myvar1')
assert not df1.empty
formula2 = 'Delta(oper_exp * myvar1 - open, 3)'
dv.add_formula('myvar2', formula2, is_quarterly=False)
df2 = dv.get_ts('myvar2')
assert not df2.empty
def _set_fields(self):
"确定查询字段，同时也确定mongoDB的db_name"
dv = DataView()
fields_init_config = {
'reference_daily_fields': dv.reference_daily_fields
# 此处可能增加新的字段，只要是qunatos的dataview支持的字段
}
self.fields = fields_init_config.get('reference_daily_fields')
def test_alpha_strategy_dataview():
dv = DataView()
dv.load_dataview(folder_path=dataview_dir_path)
props = {
"start_date": dv.start_date,
"end_date": dv.end_date,
"period": "week",
"days_delay": 0,
"init_balance": 1e8,
"position_ratio": 0.7,
'commission_rate': 0.0
}
trade_api = AlphaTradeApi()
bt = AlphaBacktestInstance()
stock_selector = model.StockSelector()
stock_selector.add_filter(name='myselector', func=my_selector)
strategy = AlphaStrategy(stock_selector=stock_selector,
pc_method='equal_weight')
pm = PortfolioManager()
context = model.AlphaContext(dataview=dv, trade_api=trade_api,
instance=bt, strategy=strategy, pm=pm)
store = pd.HDFStore(ic_weight_hd5_path)
factorList = fileio.read_json(custom_data_path)
context.ic_weight = store['ic_weight']
context.factorList = factorList
store.close()
for mdl in [stock_selector]:
mdl.register_context(context)
bt.init_from_config(props)
bt.run_alpha()
bt.save_results(folder_path=backtest_result_dir_path)
def test_alpha_strategy_dataview():
dv = DataView()
dv.load_dataview(folder_path=dataview_dir_path)
props = {
"benchmark": "000300.SH",
"universe": ','.join(dv.symbol),
"start_date": 20170131,
"end_date": dv.end_date,
"period": "month",
"days_delay": 0,
"init_balance": 1e9,
"position_ratio": 1.0,
}
trade_api = AlphaTradeApi()
def singal_gq30(context, user_options=None):
import numpy as np
res = np.power(context.snapshot['gq30'], 8)
return res
signal_model = model.FactorRevenueModel()
signal_model.add_signal('signal_gq30', singal_gq30)
strategy = AlphaStrategy(revenue_model=signal_model, pc_method='factor_value_weight')
pm = PortfolioManager()
bt = AlphaBacktestInstance()
context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm)
signal_model.register_context(context)
bt.init_from_config(props)
bt.run_alpha()
bt.save_results(folder_path=backtest_result_dir_path)
def test_add_formula():
dv = DataView()
dv.load_dataview(folder_path=daily_path)
nrows, ncols = dv.data_d.shape
n_securities = len(dv.data_d.columns.levels[0])
formula = 'Delta(high - close, 1)'
dv.add_formula('myvar1', formula, is_quarterly=False)
assert dv.data_d.shape == (nrows, ncols + 1 * n_securities)
formula2 = 'myvar1 - close'
dv.add_formula('myvar2', formula2, is_quarterly=False)
assert dv.data_d.shape == (nrows, ncols + 2 * n_securities)
def test_save_dataview():
ds = RemoteDataService()
ds.init_from_config()
dv = DataView()
props = {
'start_date': 20170101,
'end_date': 20171001,
'universe': '000300.SH',
'fields': 'pe_ttm,net_profit_incl_min_int_inc',
'freq': 1
}
dv.init_from_config(props, ds)
dv.prepare_data()
factor_formula = 'Return(net_profit_incl_min_int_inc, 4)'
factor_name = 'net_profit_growth'
dv.add_formula(factor_name, factor_formula, is_quarterly=True)
dv.save_dataview(folder_path=dataview_dir_path)
def test_add_formula():
dv = DataView()
folder_path = '../output/prepared/20160601_20170601_freq=1D'
dv.load_dataview(folder=folder_path)
nrows, ncols = dv.data_d.shape
n_securities = len(dv.data_d.columns.levels[0])
formula = 'Delta(high - close, 1)'
dv.add_formula('myvar1', formula, is_quarterly=False)
assert dv.data_d.shape == (nrows, ncols + 1 * n_securities)
formula2 = 'myvar1 - close'
dv.add_formula('myvar2', formula2, is_quarterly=False)
assert dv.data_d.shape == (nrows, ncols + 2 * n_securities)
def test_alpha_strategy_dataview():
dv = DataView()
dv.load_dataview(folder_path=dataview_dir_path)
props = {
"start_date": dv.start_date,
"end_date": dv.end_date,
"period": "week",
"days_delay": 0,
"init_balance": 1e8,
"position_ratio": 1.0,
}
gateway = AlphaTradeApi()
gateway.init_from_config(props)
context = model.Context(dataview=dv, gateway=gateway)
stock_selector = model.StockSelector()
stock_selector.add_filter(name='myselector', func=my_selector)
signal_model = model.FactorRevenueModel()
signal_model.add_signal(name='signalsize', func=signal_size)
strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='factor_value_weight',
revenue_model=signal_model)
pm = PortfolioManager()
bt = AlphaBacktestInstance()
context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm)
for mdl in [signal_model, stock_selector]:
mdl.register_context(context)
bt.init_from_config(props)
bt.run_alpha()
bt.save_results(folder_path=backtest_result_dir_path)
def store_ic_weight():
"""
Calculate IC weight and save it to file
"""
dv = DataView()
dv.load_dataview(folder_path=dataview_dir_path)
factorList = ['TO', 'BP', 'REVS20', 'float_mv_factor']
orthFactor_dic = {}
for factor in factorList:
orthFactor_dic[factor] = {}
# add the orthogonalized factor to dataview
for trade_date in dv.dates:
snapshot = dv.get_snapshot(trade_date)
factorPanel = snapshot[factorList]
factorPanel = factorPanel.dropna()
if len(factorPanel) != 0:
orthfactorPanel = Schmidt(factorPanel)
orthfactorPanel.columns = [x + '_adj' for x in factorList]
snapshot = pd.merge(left=snapshot, right=orthfactorPanel,
left_index=True, right_index=True, how='left')
for factor in factorList:
orthFactor_dic[factor][trade_date] = snapshot[factor]
for factor in factorList:
dv.append_df(pd.DataFrame(orthFactor_dic[factor]).T, field_name=factor + '_adj', is_quarterly=False)
dv.save_dataview(dataview_dir_path)
factorList_adj = [x + '_adj' for x in factorList]
fileio.save_json(factorList_adj, custom_data_path)
w = get_ic_weight(dv)
store = pd.HDFStore(ic_weight_hd5_path)
store['ic_weight'] = w
store.close()
def test_alpha_strategy_dataview():
dv = DataView()
dv.load_dataview(folder_path=dataview_dir_path)
props = {
"benchmark": "000300.SH",
"universe": ','.join(dv.symbol),
"start_date": dv.start_date,
"end_date": dv.end_date,
"period": "month",
"days_delay": 0,
"init_balance": 1e8,
"position_ratio": 1.0,
}
trade_api = AlphaTradeApi()
context = model.Context(dataview=dv, gateway=trade_api)
stock_selector = model.StockSelector(context)
stock_selector.add_filter(name='myrank', func=my_selector)
strategy = AlphaStrategy(stock_selector=stock_selector,
pc_method='equal_weight')
pm = PortfolioManager()
bt = AlphaBacktestInstance()
context = model.Context(dataview=dv,
instance=bt,
strategy=strategy,
trade_api=trade_api,
pm=pm)
stock_selector.register_context(context)
bt.init_from_config(props)
bt.run_alpha()
bt.save_results(folder_path=backtest_result_dir_path)
def test_add_formula_directly():
from jaqs.data.dataservice import RemoteDataService
ds = RemoteDataService()
dv = DataView()
secs = '600030.SH,000063.SZ,000001.SZ'
props = {'start_date': 20160601, 'end_date': 20170601, 'symbol': secs,
'fields': 'open,close',
'freq': 1}
dv.init_from_config(props, data_api=ds)
dv.prepare_data()
dv.add_formula("myfactor", 'close / open', is_quarterly=False)
assert dv.data_d.shape == (281, 33)
def save_dataview():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
start_date = get_index_basic_information()[2]
end_date = get_index_basic_information()[3]
props = {
'universe': index,
'start_date': start_date,
'end_date': end_date,
'fields': fields,
'freq': 1
}
dv.init_from_config(props, data_api=ds)
dv.prepare_data()
dv.save_dataview(folder_path=dataview_dir_path)
def calculate_pe_pb_of_index_single_day(date):
dv = DataView()
dv.load_dataview(folder_path=dataview_dir_path)
# 计算指数pe和pb的中位数、等权数
data = dv.get_snapshot(date, symbol='', fields='pe_ttm,pb')
# 判断数据质量，如果非nan数据占比超过2%，则抛出异常
if len(data.dropna(how='any')) / len(data) <= 0.98:
raise Exception('Nan of Data is too much.')
else:
data.dropna(how='any', inplace=True)
# 计算成分股个数
N = len(data)
# 计算中位数，以倒数排序可以去掉负数的影响
pe_median = 1 / ((1 / data['pe_ttm']).median())
pb_median = 1 / ((1 / data['pb']).quantile(0.5))
# 计算等权，即调和平均数
pe_equal = N / (1 / data['pe_ttm']).sum()
pb_equal = N / (1 / data['pb']).sum()
print(data)
print(date, pe_median, pe_equal, pb_median, pb_equal)
return (date, pe_median, pe_equal, pb_median, pb_equal)
def test_add_field():
dv = DataView()
dv.load_dataview(folder_path=daily_path)
nrows, ncols = dv.data_d.shape
n_securities = len(dv.data_d.columns.levels[0])
ds = RemoteDataService()
ds.init_from_config()
dv.add_field('total_share', ds)
assert dv.data_d.shape == (nrows, ncols + 1 * n_securities)
def test_add_field():
dv = DataView()
folder_path = '../output/prepared/20160601_20170601_freq=1D'
dv.load_dataview(folder=folder_path)
nrows, ncols = dv.data_d.shape
n_securities = len(dv.data_d.columns.levels[0])
from jaqs.data.dataservice import RemoteDataService
ds = RemoteDataService()
dv.add_field('share_amount', ds)
assert dv.data_d.shape == (nrows, ncols + 1 * n_securities)
def test_q():
from jaqs.data.dataservice import RemoteDataService
ds = RemoteDataService()
dv = DataView()
secs = '600030.SH,000063.SZ,000001.SZ'
props = {'start_date': 20160609, 'end_date': 20170601, 'universe': '000300.SH', 'symbol': secs,
'fields': ('open,close,'
+ 'pb,net_assets,'
+ 'total_oper_rev,oper_exp,'
+ 'cash_paid_invest,'
+ 'capital_stk,'
+ 'roe'), 'freq': 1}
dv.init_from_config(props, data_api=ds)
dv.prepare_data()
folder_path = '../output/prepared'
dv.save_dataview(folder_path=folder_path)
def test_q_add_field():
dv = DataView()
dv.load_dataview(folder_path=quarterly_path)
nrows, ncols = dv.data_q.shape
n_securities = len(dv.data_d.columns.levels[0])
ds = RemoteDataService()
ds.init_from_config()
dv.add_field('net_inc_other_ops', ds)
"""
dv.add_field('oper_rev', ds)
dv.add_field('turnover', ds)
"""
assert dv.data_q.shape == (nrows, ncols + 1 * n_securities)
def test_q_add_field():
dv = DataView()
folder_path = '../output/prepared/20160609_20170601_freq=1D'
dv.load_dataview(folder=folder_path)
nrows, ncols = dv.data_q.shape
n_securities = len(dv.data_d.columns.levels[0])
from jaqs.data.dataservice import RemoteDataService
ds = RemoteDataService()
dv.add_field('net_inc_other_ops', ds)
"""
dv.add_field('oper_rev', ds)
dv.add_field('turnover', ds)
"""
assert dv.data_q.shape == (nrows, ncols + 1 * n_securities)
def _download_data(self):
"使用quantos的dataview，下载单个股票的给定字段的数据，返回pd.DataFrame"
dv = DataView()
# fields = ','.join(list(dv.reference_daily_fields))
props = {
'symbol': self.symbol,
'fields': self.fields,
'start_date': self.start_date,
'end_date': self.end_date,
'freq': 1
}
dv.init_from_config(props=props, data_api=self._remote_data_service)
dv.prepare_data()
self._dataview_data = dv.data_d
def test_dataview_universe():
from jaqs.data.dataservice import RemoteDataService
ds = RemoteDataService()
dv = DataView()
props = {'start_date': 20170227, 'end_date': 20170327, 'universe': '000016.SH',
# 'symbol': 'rb1710.SHF,rb1801.SHF',
'fields': ('open,high,low,close,vwap,volume,turnover,'
+ 'roe,net_assets,'
+ 'total_oper_rev,oper_exp,tot_profit,int_income'
),
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
def test_q():
ds = RemoteDataService()
ds.init_from_config()
dv = DataView()
secs = '600030.SH,000063.SZ,000001.SZ'
props = {
'start_date':
20160609,
'end_date':
20170601,
'symbol':
secs,
'fields':
('open,close,' + 'pb,net_assets,' + 'total_oper_rev,oper_exp,' +
'cash_paid_invest,' + 'capital_stk,' + 'roe'),
'freq':
1
}
dv.init_from_config(props, data_api=ds)
dv.prepare_data()
dv.save_dataview(folder_path=quarterly_path)