def test_backtest_analyze(): ta = ana.AlphaAnalyzer() dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) ta.initialize(dataview=dv, file_folder=backtest_result_dir_path) print "process trades..." ta.process_trades() print "get daily stats..." ta.get_daily() print "calc strategy return..." ta.get_returns(consider_commission=True) # position change info is huge! # print "get position change..." # ta.get_pos_change_info() selected_sec = list(ta.universe)[:2] if len(selected_sec) > 0: print "Plot single securities PnL" for symbol in selected_sec: df_daily = ta.daily.get(symbol, None) if df_daily is not None: ana.plot_trades(df_daily, symbol=symbol, save_folder=backtest_result_dir_path) print "Plot strategy PnL..." ta.plot_pnl(backtest_result_dir_path) print "generate report..." static_folder = fileio.join_relative_path("trade/analyze/static") ta.gen_report(source_dir=static_folder, template_fn='report_template.html', out_folder=backtest_result_dir_path, selected=selected_sec)
def save_dataview(sub_folder='test_dataview'): ds = RemoteDataService() dv = DataView() props = { 'start_date': 20141114, 'end_date': 20160327, 'universe': '000300.SH', 'fields': ( 'open,high,low,close,vwap,volume,turnover,' # + 'pb,net_assets,' + 's_fa_eps_basic,oper_exp,tot_profit,int_income'), 'freq': 1 } dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'close > Ts_Max(close, 20)' # 20 days new high factor_name = 'new_high' dv.add_formula(factor_name, factor_formula, is_quarterly=False) dv.save_dataview( folder_path=fileio.join_relative_path('../output/prepared'), sub_folder=sub_folder)
def test_backtest_analyze(): ta = ana.AlphaAnalyzer() dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) ta.initialize(dataview=dv, file_folder=backtest_result_dir_path) ta.do_analyze(result_dir=backtest_result_dir_path, selected_sec=list(ta.universe)[:3])
def test_alpha_strategy_dataview(): save_dataview() dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "start_date": dv.start_date, "end_date": dv.end_date, "period": "week", "days_delay": 0, "init_balance": 1e8, "position_ratio": 0.7, 'commission_rate': 0.0 } trade_api = AlphaTradeApi() bt = AlphaBacktestInstance() risk_model = model.FactorRiskModel() signal_model = model.FactorRevenueModel() cost_model = model.SimpleCostModel() stock_selector = model.StockSelector() signal_model.add_signal(name='my_factor', func=my_factor) cost_model.consider_cost(name='my_commission', func=my_commission, options={'myrate': 1e-2}) stock_selector.add_filter(name='total_profit_growth', func=my_selector) stock_selector.add_filter(name='no_new_stocks', func=my_selector_no_new_stocks) strategy = AlphaStrategy(revenue_model=signal_model, stock_selector=stock_selector, cost_model=cost_model, risk_model=risk_model, pc_method='factor_value_weight') pm = PortfolioManager() # strategy = AlphaStrategy(revenue_model=signal_model, pc_method='factor_value_weight') # strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='market_value_weight') # strategy = AlphaStrategy() context = model.AlphaContext(dataview=dv, trade_api=trade_api, instance=bt, strategy=strategy, pm=pm) for mdl in [risk_model, signal_model, cost_model, stock_selector]: mdl.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def test_alpha_strategy_dataview(): dv_subfolder_name = 'test_dataview' save_dataview(sub_folder=dv_subfolder_name) dv = DataView() fullpath = fileio.join_relative_path('../output/prepared', dv_subfolder_name) dv.load_dataview(folder=fullpath) props = { "benchmark": "000300.SH", # "symbol": ','.join(dv.symbol), "universe": ','.join(dv.symbol), "start_date": dv.start_date, "end_date": dv.end_date, "period": "month", "days_delay": 0, "init_balance": 1e9, "position_ratio": 0.7, } gateway = DailyStockSimGateway() gateway.init_from_config(props) context = model.Context() context.register_gateway(gateway) context.register_trade_api(gateway) context.register_dataview(dv) risk_model = model.FactorRiskModel() signal_model = model.FactorRevenueModel_dv() cost_model = model.SimpleCostModel() risk_model.register_context(context) signal_model.register_context(context) cost_model.register_context(context) signal_model.register_func('my_factor', my_factor) signal_model.activate_func({'my_factor': {}}) cost_model.register_func('my_commission', my_commission) cost_model.activate_func({'my_commission': {'myrate': 1e-2}}) strategy = DemoAlphaStrategy(risk_model, signal_model, cost_model) # strategy.active_pc_method = 'equal_weight' # strategy.active_pc_method = 'mc' strategy.active_pc_method = 'factor_value_weight' bt = AlphaBacktestInstance_dv() bt.init_from_config(props, strategy, context=context) bt.run_alpha() bt.save_results(fileio.join_relative_path('../output/'))
def test_write(): ds = RemoteDataService() ds.init_from_config() dv = DataView() secs = '600030.SH,000063.SZ,000001.SZ' props = { 'start_date': 20160601, 'end_date': 20170601, 'symbol': secs, 'fields': 'open,close,high,low,volume,pb,net_assets,pcf_ncf', 'freq': 1 } dv.init_from_config(props, data_api=ds) dv.prepare_data() assert dv.data_d.shape == (281, 48) assert dv.dates.shape == (281, ) # TODO """ PerformanceWarning: your performance may suffer as PyTables will pickle object types that it cannot map directly to c-types [inferred_type->mixed,key->block1_values] [items->[('000001.SZ', 'int_income'), ('000001.SZ', 'less_handling_chrg_comm_exp'), ('000001.SZ', 'net_int_income'), ('000001.SZ', 'oper_exp'), ('000001.SZ', 'symbol'), ('000063.SZ', 'int_income'), ('000063.SZ', 'less_handling_chrg_comm_exp'), ('000063.SZ', 'net_int_income'), ('000063.SZ', 'oper_exp'), ('000063.SZ', 'symbol'), ('600030.SH', 'int_income'), ('600030.SH', 'less_handling_chrg_comm_exp'), ('600030.SH', 'net_int_income'), ('600030.SH', 'oper_exp'), ('600030.SH', 'symbol')]] """ dv.save_dataview(folder_path=daily_path)
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "benchmark": "000300.SH", "universe": ','.join(dv.symbol), "start_date": dv.start_date, "end_date": dv.end_date, "period": "month", "days_delay": 0, "init_balance": 1e8, "position_ratio": 1.0, } trade_api = AlphaTradeApi() trade_api.init_from_config(props) def selector_growth(context, user_options=None): growth_rate = context.snapshot['net_profit_growth'] return (growth_rate >= 0.2) & (growth_rate <= 4) def selector_pe(context, user_options=None): pe_ttm = context.snapshot['pe_ttm'] return (pe_ttm >= 10) & (pe_ttm <= 20) stock_selector = model.StockSelector() stock_selector.add_filter(name='net_profit_growth', func=selector_growth) stock_selector.add_filter(name='pe', func=selector_pe) strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='equal_weight') pm = PortfolioManager() bt = AlphaBacktestInstance() context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) stock_selector.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def test_remote_data_service_industry(): from jaqs.data.align import align import pandas as pd ds = RemoteDataService() arr = ds.get_index_comp(index='000300.SH', start_date=20130101, end_date=20170505) df = ds.get_industry_raw(symbol=','.join(arr), type_='ZZ') df = df.astype(dtype={'in_date': int}) # df_ann = df.loc[:, ['in_date', 'symbol']] # df_ann = df_ann.set_index(['symbol', 'in_date']) # df_ann = df_ann.unstack(level='symbol') from jaqs.data.dataview import DataView dic_sec = DataView._group_df_to_dict(df, by='symbol') dic_sec = {sec: df.reset_index() for sec, df in dic_sec.viewitems()} df_ann = pd.concat([df.loc[:, 'in_date'].rename(sec) for sec, df in dic_sec.viewitems()], axis=1) df_value = pd.concat([df.loc[:, 'industry1_code'].rename(sec) for sec, df in dic_sec.viewitems()], axis=1) dates_arr = ds.get_trade_date(20140101, 20170505) res = align(df_value, df_ann, dates_arr) # df_ann = df.pivot(index='in_date', columns='symbol', values='in_date') # df_value = df.pivot(index=None, columns='symbol', values='industry1_code') def align_single_df(df_one_sec): df_value = df_one_sec.loc[:, ['industry1_code']] df_ann = df_one_sec.loc[:, ['in_date']] res = align(df_value, df_ann, dates_arr) return res # res_list = [align_single_df(df) for sec, df in dic_sec.viewitems()] res_list = [align_single_df(df) for sec, df in dic_sec.items()[:10]] res = pd.concat(res_list, axis=1)
def test_save_dataview(): ds = RemoteDataService() ds.init_from_config() dv = DataView() props = {'start_date': 20170201, 'end_date': 20171001, 'universe': '000300.SH', 'fields': ('float_mv,sw2,sw1'), 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'GroupQuantile(float_mv, sw1, 10)' dv.add_formula('gq30', factor_formula, is_quarterly=False) dv.save_dataview(folder_path=dataview_dir_path)
def test_q_add_formula(): dv = DataView() folder_path = '../output/prepared/20160609_20170601_freq=1D' dv.load_dataview(folder_path=quarterly_path) nrows, ncols = dv.data_d.shape n_securities = len(dv.data_d.columns.levels[0]) formula = 'total_oper_rev / close' dv.add_formula('myvar1', formula, is_quarterly=False) df1 = dv.get_ts('myvar1') assert not df1.empty formula2 = 'Delta(oper_exp * myvar1 - open, 3)' dv.add_formula('myvar2', formula2, is_quarterly=False) df2 = dv.get_ts('myvar2') assert not df2.empty
def _set_fields(self): "确定查询字段,同时也确定mongoDB的db_name" dv = DataView() fields_init_config = { 'reference_daily_fields': dv.reference_daily_fields # 此处可能增加新的字段,只要是qunatos的dataview支持的字段 } self.fields = fields_init_config.get('reference_daily_fields')
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "start_date": dv.start_date, "end_date": dv.end_date, "period": "week", "days_delay": 0, "init_balance": 1e8, "position_ratio": 0.7, 'commission_rate': 0.0 } trade_api = AlphaTradeApi() bt = AlphaBacktestInstance() stock_selector = model.StockSelector() stock_selector.add_filter(name='myselector', func=my_selector) strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='equal_weight') pm = PortfolioManager() context = model.AlphaContext(dataview=dv, trade_api=trade_api, instance=bt, strategy=strategy, pm=pm) store = pd.HDFStore(ic_weight_hd5_path) factorList = fileio.read_json(custom_data_path) context.ic_weight = store['ic_weight'] context.factorList = factorList store.close() for mdl in [stock_selector]: mdl.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "benchmark": "000300.SH", "universe": ','.join(dv.symbol), "start_date": 20170131, "end_date": dv.end_date, "period": "month", "days_delay": 0, "init_balance": 1e9, "position_ratio": 1.0, } trade_api = AlphaTradeApi() def singal_gq30(context, user_options=None): import numpy as np res = np.power(context.snapshot['gq30'], 8) return res signal_model = model.FactorRevenueModel() signal_model.add_signal('signal_gq30', singal_gq30) strategy = AlphaStrategy(revenue_model=signal_model, pc_method='factor_value_weight') pm = PortfolioManager() bt = AlphaBacktestInstance() context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) signal_model.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def test_add_formula(): dv = DataView() dv.load_dataview(folder_path=daily_path) nrows, ncols = dv.data_d.shape n_securities = len(dv.data_d.columns.levels[0]) formula = 'Delta(high - close, 1)' dv.add_formula('myvar1', formula, is_quarterly=False) assert dv.data_d.shape == (nrows, ncols + 1 * n_securities) formula2 = 'myvar1 - close' dv.add_formula('myvar2', formula2, is_quarterly=False) assert dv.data_d.shape == (nrows, ncols + 2 * n_securities)
def test_save_dataview(): ds = RemoteDataService() ds.init_from_config() dv = DataView() props = { 'start_date': 20170101, 'end_date': 20171001, 'universe': '000300.SH', 'fields': 'pe_ttm,net_profit_incl_min_int_inc', 'freq': 1 } dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'Return(net_profit_incl_min_int_inc, 4)' factor_name = 'net_profit_growth' dv.add_formula(factor_name, factor_formula, is_quarterly=True) dv.save_dataview(folder_path=dataview_dir_path)
def test_add_formula(): dv = DataView() folder_path = '../output/prepared/20160601_20170601_freq=1D' dv.load_dataview(folder=folder_path) nrows, ncols = dv.data_d.shape n_securities = len(dv.data_d.columns.levels[0]) formula = 'Delta(high - close, 1)' dv.add_formula('myvar1', formula, is_quarterly=False) assert dv.data_d.shape == (nrows, ncols + 1 * n_securities) formula2 = 'myvar1 - close' dv.add_formula('myvar2', formula2, is_quarterly=False) assert dv.data_d.shape == (nrows, ncols + 2 * n_securities)
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "start_date": dv.start_date, "end_date": dv.end_date, "period": "week", "days_delay": 0, "init_balance": 1e8, "position_ratio": 1.0, } gateway = AlphaTradeApi() gateway.init_from_config(props) context = model.Context(dataview=dv, gateway=gateway) stock_selector = model.StockSelector() stock_selector.add_filter(name='myselector', func=my_selector) signal_model = model.FactorRevenueModel() signal_model.add_signal(name='signalsize', func=signal_size) strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='factor_value_weight', revenue_model=signal_model) pm = PortfolioManager() bt = AlphaBacktestInstance() context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) for mdl in [signal_model, stock_selector]: mdl.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def store_ic_weight(): """ Calculate IC weight and save it to file """ dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) factorList = ['TO', 'BP', 'REVS20', 'float_mv_factor'] orthFactor_dic = {} for factor in factorList: orthFactor_dic[factor] = {} # add the orthogonalized factor to dataview for trade_date in dv.dates: snapshot = dv.get_snapshot(trade_date) factorPanel = snapshot[factorList] factorPanel = factorPanel.dropna() if len(factorPanel) != 0: orthfactorPanel = Schmidt(factorPanel) orthfactorPanel.columns = [x + '_adj' for x in factorList] snapshot = pd.merge(left=snapshot, right=orthfactorPanel, left_index=True, right_index=True, how='left') for factor in factorList: orthFactor_dic[factor][trade_date] = snapshot[factor] for factor in factorList: dv.append_df(pd.DataFrame(orthFactor_dic[factor]).T, field_name=factor + '_adj', is_quarterly=False) dv.save_dataview(dataview_dir_path) factorList_adj = [x + '_adj' for x in factorList] fileio.save_json(factorList_adj, custom_data_path) w = get_ic_weight(dv) store = pd.HDFStore(ic_weight_hd5_path) store['ic_weight'] = w store.close()
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "benchmark": "000300.SH", "universe": ','.join(dv.symbol), "start_date": dv.start_date, "end_date": dv.end_date, "period": "month", "days_delay": 0, "init_balance": 1e8, "position_ratio": 1.0, } trade_api = AlphaTradeApi() context = model.Context(dataview=dv, gateway=trade_api) stock_selector = model.StockSelector(context) stock_selector.add_filter(name='myrank', func=my_selector) strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='equal_weight') pm = PortfolioManager() bt = AlphaBacktestInstance() context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) stock_selector.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def test_add_formula_directly(): from jaqs.data.dataservice import RemoteDataService ds = RemoteDataService() dv = DataView() secs = '600030.SH,000063.SZ,000001.SZ' props = {'start_date': 20160601, 'end_date': 20170601, 'symbol': secs, 'fields': 'open,close', 'freq': 1} dv.init_from_config(props, data_api=ds) dv.prepare_data() dv.add_formula("myfactor", 'close / open', is_quarterly=False) assert dv.data_d.shape == (281, 33)
def save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() start_date = get_index_basic_information()[2] end_date = get_index_basic_information()[3] props = { 'universe': index, 'start_date': start_date, 'end_date': end_date, 'fields': fields, 'freq': 1 } dv.init_from_config(props, data_api=ds) dv.prepare_data() dv.save_dataview(folder_path=dataview_dir_path)
def calculate_pe_pb_of_index_single_day(date): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) # 计算指数pe和pb的中位数、等权数 data = dv.get_snapshot(date, symbol='', fields='pe_ttm,pb') # 判断数据质量,如果非nan数据占比超过2%,则抛出异常 if len(data.dropna(how='any')) / len(data) <= 0.98: raise Exception('Nan of Data is too much.') else: data.dropna(how='any', inplace=True) # 计算成分股个数 N = len(data) # 计算中位数,以倒数排序可以去掉负数的影响 pe_median = 1 / ((1 / data['pe_ttm']).median()) pb_median = 1 / ((1 / data['pb']).quantile(0.5)) # 计算等权,即调和平均数 pe_equal = N / (1 / data['pe_ttm']).sum() pb_equal = N / (1 / data['pb']).sum() print(data) print(date, pe_median, pe_equal, pb_median, pb_equal) return (date, pe_median, pe_equal, pb_median, pb_equal)
def test_add_field(): dv = DataView() dv.load_dataview(folder_path=daily_path) nrows, ncols = dv.data_d.shape n_securities = len(dv.data_d.columns.levels[0]) ds = RemoteDataService() ds.init_from_config() dv.add_field('total_share', ds) assert dv.data_d.shape == (nrows, ncols + 1 * n_securities)
def test_add_field(): dv = DataView() folder_path = '../output/prepared/20160601_20170601_freq=1D' dv.load_dataview(folder=folder_path) nrows, ncols = dv.data_d.shape n_securities = len(dv.data_d.columns.levels[0]) from jaqs.data.dataservice import RemoteDataService ds = RemoteDataService() dv.add_field('share_amount', ds) assert dv.data_d.shape == (nrows, ncols + 1 * n_securities)
def test_q(): from jaqs.data.dataservice import RemoteDataService ds = RemoteDataService() dv = DataView() secs = '600030.SH,000063.SZ,000001.SZ' props = {'start_date': 20160609, 'end_date': 20170601, 'universe': '000300.SH', 'symbol': secs, 'fields': ('open,close,' + 'pb,net_assets,' + 'total_oper_rev,oper_exp,' + 'cash_paid_invest,' + 'capital_stk,' + 'roe'), 'freq': 1} dv.init_from_config(props, data_api=ds) dv.prepare_data() folder_path = '../output/prepared' dv.save_dataview(folder_path=folder_path)
def test_q_add_field(): dv = DataView() dv.load_dataview(folder_path=quarterly_path) nrows, ncols = dv.data_q.shape n_securities = len(dv.data_d.columns.levels[0]) ds = RemoteDataService() ds.init_from_config() dv.add_field('net_inc_other_ops', ds) """ dv.add_field('oper_rev', ds) dv.add_field('turnover', ds) """ assert dv.data_q.shape == (nrows, ncols + 1 * n_securities)
def test_q_add_field(): dv = DataView() folder_path = '../output/prepared/20160609_20170601_freq=1D' dv.load_dataview(folder=folder_path) nrows, ncols = dv.data_q.shape n_securities = len(dv.data_d.columns.levels[0]) from jaqs.data.dataservice import RemoteDataService ds = RemoteDataService() dv.add_field('net_inc_other_ops', ds) """ dv.add_field('oper_rev', ds) dv.add_field('turnover', ds) """ assert dv.data_q.shape == (nrows, ncols + 1 * n_securities)
def _download_data(self): "使用quantos的dataview,下载单个股票的给定字段的数据,返回pd.DataFrame" dv = DataView() # fields = ','.join(list(dv.reference_daily_fields)) props = { 'symbol': self.symbol, 'fields': self.fields, 'start_date': self.start_date, 'end_date': self.end_date, 'freq': 1 } dv.init_from_config(props=props, data_api=self._remote_data_service) dv.prepare_data() self._dataview_data = dv.data_d
def test_dataview_universe(): from jaqs.data.dataservice import RemoteDataService ds = RemoteDataService() dv = DataView() props = {'start_date': 20170227, 'end_date': 20170327, 'universe': '000016.SH', # 'symbol': 'rb1710.SHF,rb1801.SHF', 'fields': ('open,high,low,close,vwap,volume,turnover,' + 'roe,net_assets,' + 'total_oper_rev,oper_exp,tot_profit,int_income' ), 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data()
def test_q(): ds = RemoteDataService() ds.init_from_config() dv = DataView() secs = '600030.SH,000063.SZ,000001.SZ' props = { 'start_date': 20160609, 'end_date': 20170601, 'symbol': secs, 'fields': ('open,close,' + 'pb,net_assets,' + 'total_oper_rev,oper_exp,' + 'cash_paid_invest,' + 'capital_stk,' + 'roe'), 'freq': 1 } dv.init_from_config(props, data_api=ds) dv.prepare_data() dv.save_dataview(folder_path=quarterly_path)