def save_dataview(): ds = RemoteDataService() ds.init_from_config() dv = DataView() props = { 'start_date': 20140101, 'end_date': 20171001, 'universe': '000300.SH', 'fields': 'volume,turnover,float_mv,pb,total_mv', 'freq': 1 } dv.init_from_config(props, ds) dv.prepare_data() # for convenience to check limit reachers dv.add_formula('limit_reached', 'Abs((open - Delay(close, 1)) / Delay(close, 1)) > 0.095', is_quarterly=False) dv.add_formula('random', 'StdDev(volume, 20)', is_quarterly=False) dv.add_formula('momentum', 'Return(close_adj, 20)', is_quarterly=False) # dv.add_formula('size', '', is_quarterly=False) dv.save_dataview(dataview_folder)
def save_dataview(): ds = RemoteDataService() ds.init_from_config() dv = DataView() props = {'start_date': 20150101, 'end_date': 20170930, 'universe': '000905.SH', 'fields': ('turnover,float_mv,close_adj,pe,pb'), 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'Cutoff(Standardize(turnover / 10000 / float_mv), 2)' dv.add_formula('TO', factor_formula, is_quarterly=False) factor_formula = 'Cutoff(Standardize(1/pb), 2)' dv.add_formula('BP', factor_formula, is_quarterly=False) factor_formula = 'Cutoff(Standardize(Return(close_adj, 20)), 2)' dv.add_formula('REVS20', factor_formula, is_quarterly=False) factor_formula = 'Cutoff(Standardize(Log(float_mv)), 2)' dv.add_formula('float_mv_factor', factor_formula, is_quarterly=False) factor_formula = 'Delay(Return(close_adj, 1), -1)' dv.add_formula('NextRet', factor_formula, is_quarterly=False) dv.save_dataview(folder_path=dataview_dir_path)
def test_backtest_analyze(): ta = ana.AlphaAnalyzer() dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) ta.initialize(dataview=dv, file_folder=backtest_result_dir_path) print "process trades..." ta.process_trades() print "get daily stats..." ta.get_daily() print "calc strategy return..." ta.get_returns(consider_commission=True) # position change info is huge! # print "get position change..." # ta.get_pos_change_info() selected_sec = list(ta.universe)[:2] if len(selected_sec) > 0: print "Plot single securities PnL" for symbol in selected_sec: df_daily = ta.daily.get(symbol, None) if df_daily is not None: ana.plot_trades(df_daily, symbol=symbol, save_folder=backtest_result_dir_path) print "Plot strategy PnL..." ta.plot_pnl(backtest_result_dir_path) print "generate report..." static_folder = fileio.join_relative_path("trade/analyze/static") ta.gen_report(source_dir=static_folder, template_fn='report_template.html', out_folder=backtest_result_dir_path, selected=selected_sec)
def save_dataview(sub_folder='test_dataview'): ds = RemoteDataService() dv = DataView() props = { 'start_date': 20141114, 'end_date': 20160327, 'universe': '000300.SH', 'fields': ( 'open,high,low,close,vwap,volume,turnover,' # + 'pb,net_assets,' + 's_fa_eps_basic,oper_exp,tot_profit,int_income'), 'freq': 1 } dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'close > Ts_Max(close, 20)' # 20 days new high factor_name = 'new_high' dv.add_formula(factor_name, factor_formula, is_quarterly=False) dv.save_dataview( folder_path=fileio.join_relative_path('../output/prepared'), sub_folder=sub_folder)
def test_write(): ds = RemoteDataService() ds.init_from_config() dv = DataView() secs = '600030.SH,000063.SZ,000001.SZ' props = { 'start_date': 20160601, 'end_date': 20170601, 'symbol': secs, 'fields': 'open,close,high,low,volume,pb,net_assets,pcf_ncf', 'freq': 1 } dv.init_from_config(props, data_api=ds) dv.prepare_data() assert dv.data_d.shape == (281, 48) assert dv.dates.shape == (281, ) # TODO """ PerformanceWarning: your performance may suffer as PyTables will pickle object types that it cannot map directly to c-types [inferred_type->mixed,key->block1_values] [items->[('000001.SZ', 'int_income'), ('000001.SZ', 'less_handling_chrg_comm_exp'), ('000001.SZ', 'net_int_income'), ('000001.SZ', 'oper_exp'), ('000001.SZ', 'symbol'), ('000063.SZ', 'int_income'), ('000063.SZ', 'less_handling_chrg_comm_exp'), ('000063.SZ', 'net_int_income'), ('000063.SZ', 'oper_exp'), ('000063.SZ', 'symbol'), ('600030.SH', 'int_income'), ('600030.SH', 'less_handling_chrg_comm_exp'), ('600030.SH', 'net_int_income'), ('600030.SH', 'oper_exp'), ('600030.SH', 'symbol')]] """ dv.save_dataview(folder_path=daily_path)
def save_dataview(): ds = RemoteDataService() ds.init_from_config() dv = DataView() props = { 'start_date': 20170101, 'end_date': 20171030, 'universe': '000300.SH', 'fields': ( 'open,high,low,close,vwap,volume,turnover,sw1,' # + 'pb,net_assets,' + 'eps_basic,total_mv,tot_profit,int_income'), 'freq': 1 } dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'close >= Delay(Ts_Max(close, 20), 1)' # 20 days new high factor_name = 'new_high' dv.add_formula(factor_name, factor_formula, is_quarterly=False) dv.add_formula('total_profit_growth', formula='Return(tot_profit, 4)', is_quarterly=True) dv.save_dataview(folder_path=dataview_dir_path)
def test_save_dataview(): ds = RemoteDataService() ds.init_from_config() dv = DataView() props = { 'start_date': 20170101, 'end_date': 20171001, 'universe': '000300.SH', 'fields': ('float_mv,pb,pe_ttm,sw2'), 'freq': 1 } dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'GroupQuantile(-float_mv, sw2, 10)' dv.add_formula('rank_mv', factor_formula, is_quarterly=False) factor_formula = 'GroupQuantile(If(pb >= 0.2, pb, 100), sw2, 10)' dv.add_formula('rank_pb', factor_formula, is_quarterly=False) factor_formula = 'GroupQuantile(If(pe_ttm >= 3, pe_ttm, 9999.0), sw2, 10)' dv.add_formula('rank_pe', factor_formula, is_quarterly=False) dv.save_dataview(folder_path=dataview_dir_path)
def test_save_dataview(): ds = RemoteDataService() ds.init_from_config() dv = DataView() props = {'start_date': 20150101, 'end_date': 20170930, 'universe': '000905.SH', 'fields': ('tot_cur_assets,tot_cur_liab,inventories,pre_pay,deferred_exp,' 'eps_basic,ebit,pe,pb,float_mv,sw1'), 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'pe < 30' dv.add_formula('pe_condition', factor_formula, is_quarterly=False) factor_formula = 'pb < 3' dv.add_formula('pb_condition', factor_formula, is_quarterly=False) factor_formula = 'Return(eps_basic, 4) > 0' dv.add_formula('eps_condition', factor_formula, is_quarterly=True) factor_formula = 'Return(ebit, 4) > 0' dv.add_formula('ebit_condition', factor_formula, is_quarterly=True) factor_formula = 'tot_cur_assets/tot_cur_liab > 2' dv.add_formula('current_condition', factor_formula, is_quarterly=True) factor_formula = '(tot_cur_assets - inventories - pre_pay - deferred_exp)/tot_cur_liab > 1' dv.add_formula('quick_condition', factor_formula, is_quarterly=True) dv.add_formula('mv_rank', 'Rank(float_mv)', is_quarterly=False) dv.save_dataview(folder_path=dataview_dir_path)
def test_backtest_analyze(): ta = ana.AlphaAnalyzer() dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) ta.initialize(dataview=dv, file_folder=backtest_result_dir_path) ta.do_analyze(result_dir=backtest_result_dir_path, selected_sec=list(ta.universe)[:3])
def _set_fields(self): "确定查询字段,同时也确定mongoDB的db_name" dv = DataView() fields_init_config = { 'reference_daily_fields': dv.reference_daily_fields # 此处可能增加新的字段,只要是qunatos的dataview支持的字段 } self.fields = fields_init_config.get('reference_daily_fields')
def test_add_field(): dv = DataView() dv.load_dataview(folder_path=daily_path) nrows, ncols = dv.data_d.shape n_securities = len(dv.data_d.columns.levels[0]) ds = RemoteDataService() ds.init_from_config() dv.add_field('total_share', ds) assert dv.data_d.shape == (nrows, ncols + 1 * n_securities)
def test_alpha_strategy_dataview(): save_dataview() dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "start_date": dv.start_date, "end_date": dv.end_date, "period": "week", "days_delay": 0, "init_balance": 1e8, "position_ratio": 0.7, 'commission_rate': 0.0 } trade_api = AlphaTradeApi() bt = AlphaBacktestInstance() risk_model = model.FactorRiskModel() signal_model = model.FactorRevenueModel() cost_model = model.SimpleCostModel() stock_selector = model.StockSelector() signal_model.add_signal(name='my_factor', func=my_factor) cost_model.consider_cost(name='my_commission', func=my_commission, options={'myrate': 1e-2}) stock_selector.add_filter(name='total_profit_growth', func=my_selector) stock_selector.add_filter(name='no_new_stocks', func=my_selector_no_new_stocks) strategy = AlphaStrategy(revenue_model=signal_model, stock_selector=stock_selector, cost_model=cost_model, risk_model=risk_model, pc_method='factor_value_weight') pm = PortfolioManager() # strategy = AlphaStrategy(revenue_model=signal_model, pc_method='factor_value_weight') # strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='market_value_weight') # strategy = AlphaStrategy() context = model.AlphaContext(dataview=dv, trade_api=trade_api, instance=bt, strategy=strategy, pm=pm) for mdl in [risk_model, signal_model, cost_model, stock_selector]: mdl.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def test_alpha_strategy_dataview(): dv_subfolder_name = 'test_dataview' save_dataview(sub_folder=dv_subfolder_name) dv = DataView() fullpath = fileio.join_relative_path('../output/prepared', dv_subfolder_name) dv.load_dataview(folder=fullpath) props = { "benchmark": "000300.SH", # "symbol": ','.join(dv.symbol), "universe": ','.join(dv.symbol), "start_date": dv.start_date, "end_date": dv.end_date, "period": "month", "days_delay": 0, "init_balance": 1e9, "position_ratio": 0.7, } gateway = DailyStockSimGateway() gateway.init_from_config(props) context = model.Context() context.register_gateway(gateway) context.register_trade_api(gateway) context.register_dataview(dv) risk_model = model.FactorRiskModel() signal_model = model.FactorRevenueModel_dv() cost_model = model.SimpleCostModel() risk_model.register_context(context) signal_model.register_context(context) cost_model.register_context(context) signal_model.register_func('my_factor', my_factor) signal_model.activate_func({'my_factor': {}}) cost_model.register_func('my_commission', my_commission) cost_model.activate_func({'my_commission': {'myrate': 1e-2}}) strategy = DemoAlphaStrategy(risk_model, signal_model, cost_model) # strategy.active_pc_method = 'equal_weight' # strategy.active_pc_method = 'mc' strategy.active_pc_method = 'factor_value_weight' bt = AlphaBacktestInstance_dv() bt.init_from_config(props, strategy, context=context) bt.run_alpha() bt.save_results(fileio.join_relative_path('../output/'))
def test_add_field(): dv = DataView() folder_path = '../output/prepared/20160601_20170601_freq=1D' dv.load_dataview(folder=folder_path) nrows, ncols = dv.data_d.shape n_securities = len(dv.data_d.columns.levels[0]) from jaqs.data.dataservice import RemoteDataService ds = RemoteDataService() dv.add_field('share_amount', ds) assert dv.data_d.shape == (nrows, ncols + 1 * n_securities)
def analyze_signal(): # -------------------------------------------------------------------------------- # Step.1 load dataview dv = DataView() dv.load_dataview(dataview_folder) # -------------------------------------------------------------------------------- # Step.2 calculate mask (to mask those ill data points) trade_status = dv.get_ts('trade_status') mask_sus = trade_status == u'停牌'.encode('utf-8') df_index_member = dv.get_ts('index_member') mask_index_member = ~(df_index_member > 0) dv.add_formula('limit_reached', 'Abs((open - Delay(close, 1)) / Delay(close, 1)) > 0.095', is_quarterly=False) df_limit_reached = dv.get_ts('limit_reached') mask_limit_reached = df_limit_reached > 0 mask_all = np.logical_or( mask_sus, np.logical_or(mask_index_member, mask_limit_reached)) # -------------------------------------------------------------------------------- # Step.3 get signal, benchmark and price data # dv.add_formula('illi_daily', '(high - low) * 1000000000 / turnover', is_quarterly=False) # dv.add_formula('illi', 'Ewma(illi_daily, 11)', is_quarterly=False) # dv.add_formula('size', 'Log(float_mv)', is_quarterly=False) # dv.add_formula('value', '-1.0/pb', is_quarterly=False) # dv.add_formula('liquidity', 'Ts_Mean(volume, 22) / float_mv', is_quarterly=False) dv.add_formula('divert', '- Correlation(vwap_adj, volume, 10)', is_quarterly=False) signal = dv.get_ts('divert').shift(1, axis=0) # avoid look-ahead bias price = dv.get_ts('close_adj') price_bench = dv.data_benchmark # Step.4 analyze! my_period = 5 obj = signaldigger.digger.SignalDigger( output_folder=jutil.join_relative_path('../output'), output_format='pdf') obj.process_signal_before_analysis( signal, price=price, mask=mask_all, n_quantiles=5, period=my_period, benchmark_price=price_bench, ) res = obj.create_full_report()
def test_add_formula(): dv = DataView() dv.load_dataview(folder_path=daily_path) nrows, ncols = dv.data_d.shape n_securities = len(dv.data_d.columns.levels[0]) formula = 'Delta(high - close, 1)' dv.add_formula('myvar1', formula, is_quarterly=False) assert dv.data_d.shape == (nrows, ncols + 1 * n_securities) formula2 = 'myvar1 - close' dv.add_formula('myvar2', formula2, is_quarterly=False) assert dv.data_d.shape == (nrows, ncols + 2 * n_securities)
def test_q_add_field(): dv = DataView() dv.load_dataview(folder_path=quarterly_path) nrows, ncols = dv.data_q.shape n_securities = len(dv.data_d.columns.levels[0]) ds = RemoteDataService() ds.init_from_config() dv.add_field('net_inc_other_ops', ds) """ dv.add_field('oper_rev', ds) dv.add_field('turnover', ds) """ assert dv.data_q.shape == (nrows, ncols + 1 * n_securities)
def test_add_formula(): dv = DataView() folder_path = '../output/prepared/20160601_20170601_freq=1D' dv.load_dataview(folder=folder_path) nrows, ncols = dv.data_d.shape n_securities = len(dv.data_d.columns.levels[0]) formula = 'Delta(high - close, 1)' dv.add_formula('myvar1', formula, is_quarterly=False) assert dv.data_d.shape == (nrows, ncols + 1 * n_securities) formula2 = 'myvar1 - close' dv.add_formula('myvar2', formula2, is_quarterly=False) assert dv.data_d.shape == (nrows, ncols + 2 * n_securities)
def test_q_add_field(): dv = DataView() folder_path = '../output/prepared/20160609_20170601_freq=1D' dv.load_dataview(folder=folder_path) nrows, ncols = dv.data_q.shape n_securities = len(dv.data_d.columns.levels[0]) from jaqs.data.dataservice import RemoteDataService ds = RemoteDataService() dv.add_field('net_inc_other_ops', ds) """ dv.add_field('oper_rev', ds) dv.add_field('turnover', ds) """ assert dv.data_q.shape == (nrows, ncols + 1 * n_securities)
def _download_data(self): "使用quantos的dataview,下载单个股票的给定字段的数据,返回pd.DataFrame" dv = DataView() # fields = ','.join(list(dv.reference_daily_fields)) props = { 'symbol': self.symbol, 'fields': self.fields, 'start_date': self.start_date, 'end_date': self.end_date, 'freq': 1 } dv.init_from_config(props=props, data_api=self._remote_data_service) dv.prepare_data() self._dataview_data = dv.data_d
def test_add_formula_directly(): from jaqs.data.dataservice import RemoteDataService ds = RemoteDataService() dv = DataView() secs = '600030.SH,000063.SZ,000001.SZ' props = {'start_date': 20160601, 'end_date': 20170601, 'symbol': secs, 'fields': 'open,close', 'freq': 1} dv.init_from_config(props, data_api=ds) dv.prepare_data() dv.add_formula("myfactor", 'close / open', is_quarterly=False) assert dv.data_d.shape == (281, 33)
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "benchmark": "000300.SH", "universe": ','.join(dv.symbol), "start_date": dv.start_date, "end_date": dv.end_date, "period": "month", "days_delay": 0, "init_balance": 1e8, "position_ratio": 1.0, } trade_api = AlphaTradeApi() trade_api.init_from_config(props) def selector_growth(context, user_options=None): growth_rate = context.snapshot['net_profit_growth'] return (growth_rate >= 0.2) & (growth_rate <= 4) def selector_pe(context, user_options=None): pe_ttm = context.snapshot['pe_ttm'] return (pe_ttm >= 10) & (pe_ttm <= 20) stock_selector = model.StockSelector() stock_selector.add_filter(name='net_profit_growth', func=selector_growth) stock_selector.add_filter(name='pe', func=selector_pe) strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='equal_weight') pm = PortfolioManager() bt = AlphaBacktestInstance() context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) stock_selector.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def test_dataview_universe(): from jaqs.data.dataservice import RemoteDataService ds = RemoteDataService() dv = DataView() props = {'start_date': 20170227, 'end_date': 20170327, 'universe': '000016.SH', # 'symbol': 'rb1710.SHF,rb1801.SHF', 'fields': ('open,high,low,close,vwap,volume,turnover,' + 'roe,net_assets,' + 'total_oper_rev,oper_exp,tot_profit,int_income' ), 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data()
def test_q_add_formula(): dv = DataView() folder_path = '../output/prepared/20160609_20170601_freq=1D' dv.load_dataview(folder_path=quarterly_path) nrows, ncols = dv.data_d.shape n_securities = len(dv.data_d.columns.levels[0]) formula = 'total_oper_rev / close' dv.add_formula('myvar1', formula, is_quarterly=False) df1 = dv.get_ts('myvar1') assert not df1.empty formula2 = 'Delta(oper_exp * myvar1 - open, 3)' dv.add_formula('myvar2', formula2, is_quarterly=False) df2 = dv.get_ts('myvar2') assert not df2.empty
def test_save_dataview(): ds = RemoteDataService() ds.init_from_config() dv = DataView() props = {'start_date': 20170201, 'end_date': 20171001, 'universe': '000300.SH', 'fields': ('float_mv,sw2,sw1'), 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'GroupQuantile(float_mv, sw1, 10)' dv.add_formula('gq30', factor_formula, is_quarterly=False) dv.save_dataview(folder_path=dataview_dir_path)
def save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() start_date = get_index_basic_information()[2] end_date = get_index_basic_information()[3] props = { 'universe': index, 'start_date': start_date, 'end_date': end_date, 'fields': fields, 'freq': 1 } dv.init_from_config(props, data_api=ds) dv.prepare_data() dv.save_dataview(folder_path=dataview_dir_path)
def test_load(): dv = DataView() folder_path = '../output/prepared/20160601_20170601_freq=1D' dv.load_dataview(folder=folder_path) assert dv.start_date == 20160601 and set(dv.symbol) == set('000001.SZ,600030.SH,000063.SZ'.split(',')) # test get_snapshot snap1 = dv.get_snapshot(20170504, symbol='600030.SH,000063.SZ', fields='close,pb') assert snap1.shape == (2, 2) assert set(snap1.columns.values) == {'close', 'pb'} assert set(snap1.index.values) == {'600030.SH', '000063.SZ'} # test get_ts ts1 = dv.get_ts('close', symbol='600030.SH,000063.SZ', start_date=20170101, end_date=20170302) assert ts1.shape == (38, 2) assert set(ts1.columns.values) == {'600030.SH', '000063.SZ'} assert ts1.index.values[-1] == 20170302
def test_write_future(): ds = RemoteDataService() ds.init_from_config() dv = DataView() secs = 'rb1710.SHF,j1710.DCE' props = { 'start_date': 20170401, 'end_date': 20170901, 'symbol': secs, 'fields': 'open,close,high,low,volume,oi', 'freq': 1, 'all_price': False } dv.init_from_config(props, data_api=ds) dv.prepare_data() assert dv.data_d.shape == (145, 14)
def store_ic_weight(): """ Calculate IC weight and save it to file """ dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) factorList = ['TO', 'BP', 'REVS20', 'float_mv_factor'] orthFactor_dic = {} for factor in factorList: orthFactor_dic[factor] = {} # add the orthogonalized factor to dataview for trade_date in dv.dates: snapshot = dv.get_snapshot(trade_date) factorPanel = snapshot[factorList] factorPanel = factorPanel.dropna() if len(factorPanel) != 0: orthfactorPanel = Schmidt(factorPanel) orthfactorPanel.columns = [x + '_adj' for x in factorList] snapshot = pd.merge(left=snapshot, right=orthfactorPanel, left_index=True, right_index=True, how='left') for factor in factorList: orthFactor_dic[factor][trade_date] = snapshot[factor] for factor in factorList: dv.append_df(pd.DataFrame(orthFactor_dic[factor]).T, field_name=factor + '_adj', is_quarterly=False) dv.save_dataview(dataview_dir_path) factorList_adj = [x + '_adj' for x in factorList] fileio.save_json(factorList_adj, custom_data_path) w = get_ic_weight(dv) store = pd.HDFStore(ic_weight_hd5_path) store['ic_weight'] = w store.close()
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "start_date": dv.start_date, "end_date": dv.end_date, "period": "week", "days_delay": 0, "init_balance": 1e8, "position_ratio": 0.7, 'commission_rate': 0.0 } trade_api = AlphaTradeApi() bt = AlphaBacktestInstance() stock_selector = model.StockSelector() stock_selector.add_filter(name='myselector', func=my_selector) strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='equal_weight') pm = PortfolioManager() context = model.AlphaContext(dataview=dv, trade_api=trade_api, instance=bt, strategy=strategy, pm=pm) store = pd.HDFStore(ic_weight_hd5_path) factorList = fileio.read_json(custom_data_path) context.ic_weight = store['ic_weight'] context.factorList = factorList store.close() for mdl in [stock_selector]: mdl.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)