def do_livetrade(): dv = DataView() dv.load_dataview(folder_path=dataview_store_folder) props = {"period": "day", "strategy_no": 1044, "init_balance": 1e6} props.update(data_config) props.update(trade_config) strategy = AlphaStrategy(pc_method='market_value_weight') pm = PortfolioManager() bt = AlphaLiveTradeInstance() trade_api = RealTimeTradeApi(props) ds = RemoteDataService() context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm, data_api=ds) bt.init_from_config(props) bt.run_alpha() goal_positions = strategy.goal_positions print("Length of goal positions:", len(goal_positions)) task_id, msg = trade_api.goal_portfolio(goal_positions) print(task_id, msg)
def analyze_event(): # -------------------------------------------------------------------------------- # Step.1 load dataview dv = DataView() dv.load_dataview(dataview_folder) # -------------------------------------------------------------------------------- # Step.2 calculate mask (to mask those ill data points) mask_limit_reached = dv.get_ts('mask_limit_reached') mask_index_member = dv.get_ts('mask_index_member') mask_sus = dv.get_ts('mask_sus') mask_all = np.logical_or(mask_sus, np.logical_or(mask_index_member, mask_limit_reached)) # -------------------------------------------------------------------------------- # Step.3 get signal, benchmark and price data price = dv.get_ts('close_adj') price_bench = dv.data_benchmark dv.add_formula('in_', '(Delay(index_weight, 1) == 0) && (index_weight > 0)', is_quarterly=False) signal = dv.get_ts('in_').shift(1, axis=0) # avoid look-ahead bias # Step.4 analyze! obj = SignalDigger(output_folder='../../output', output_format='pdf') obj.create_binary_event_report(signal, price, mask_all, price_bench, periods=[20, 60, 121, 242], group_by=None)
def test_backtest_analyze(): ta = ana.AlphaAnalyzer() dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) ta.initialize(dataview=dv, file_folder=backtest_result_dir_path) ta.do_analyze(result_dir=backtest_result_dir_path, selected_sec=list(ta.universe)[:3])
def simple_test_signal(): dv = DataView() dv.load_dataview(dataview_folder) dv.add_formula('open_jump', 'open_adj / Delay(close_adj, 1)', is_quarterly=False) # good analyze_signal(dv, 'open_jump', 'pdf') print("Signal return & IC test finished.")
def test_add_field(): dv = DataView() dv.load_dataview(folder_path=daily_path) nrows, ncols = dv.data_d.shape n_securities = len(dv.data_d.columns.levels[0]) ds = RemoteDataService() ds.init_from_config(data_config) dv.add_field('total_share', ds) assert dv.data_d.shape == (nrows, ncols + 1 * n_securities)
def test_backtest_analyze(): ta = ana.AlphaAnalyzer() dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) ta.initialize(dataview=dv, file_folder=backtest_result_dir_path) ta.do_analyze(result_dir=backtest_result_dir_path, selected_sec=list(ta.universe)[:3])
def test_alpha_strategy_dataview(): save_dataview() dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "start_date": dv.start_date, "end_date": dv.end_date, "period": "week", "days_delay": 0, "init_balance": 1e8, "position_ratio": 0.7, 'commission_rate': 0.0 } trade_api = AlphaTradeApi() bt = AlphaBacktestInstance() risk_model = model.FactorRiskModel() signal_model = model.FactorSignalModel() cost_model = model.SimpleCostModel() stock_selector = model.StockSelector() signal_model.add_signal(name='my_factor', func=my_factor) cost_model.consider_cost(name='my_commission', func=my_commission, options={'myrate': 1e-2}) stock_selector.add_filter(name='total_profit_growth', func=my_selector) stock_selector.add_filter(name='no_new_stocks', func=my_selector_no_new_stocks) strategy = AlphaStrategy(signal_model=signal_model, stock_selector=stock_selector, cost_model=cost_model, risk_model=risk_model, pc_method='factor_value_weight') pm = PortfolioManager() # strategy = AlphaStrategy(signal_model=signal_model, pc_method='factor_value_weight') # strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='market_value_weight') # strategy = AlphaStrategy() context = model.AlphaContext(dataview=dv, trade_api=trade_api, instance=bt, strategy=strategy, pm=pm) for mdl in [risk_model, signal_model, cost_model, stock_selector]: mdl.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def save_data(): """ This function fetches data from remote server and stores them locally. Then we can use local data to do back-test. """ dataview_props = { 'start_date': 20170101, # Start and end date of back-test 'end_date': 20171030, 'symbol': '000300.SH', # 'universe': UNIVERSE, # Investment universe and performance benchmark # 'benchmark': '000300.SH', 'fields': 'open,high,low,close,volume', # Data fields that we need 'freq': 1 # freq = 1 means we use daily data. Please do not change this. } # RemoteDataService communicates with a remote server to fetch data ds = RemoteDataService() # Use username and password in data_config to login ds.init_from_config(data_config) # DataView utilizes RemoteDataService to get various data and store them dv = DataView() dv.init_from_config(dataview_props, ds) dv.prepare_data() print(dv.prepare_data()) dv.save_dataview(folder_path=dataview_store_folder)
def analyze_backtest_results(): # Analyzer help us calculate various trade statistics according to trade results. # All the calculation results will be stored as its members. ta = ana.AlphaAnalyzer() dv = DataView() dv.load_dataview(folder_path=dataview_store_folder) ta.initialize(dataview=dv, file_folder=backtest_result_folder) ta.do_analyze(result_dir=backtest_result_folder, selected_sec=list(ta.universe)[:3])
def analyze_backtest_results(): # Analyzer help us calculate various trade statistics according to trade results. # All the calculation results will be stored as its members. ta = ana.AlphaAnalyzer() dv = DataView() dv.load_dataview(folder_path=dataview_store_folder) ta.initialize(dataview=dv, file_folder=backtest_result_folder) ta.do_analyze(result_dir=backtest_result_folder, selected_sec=list(ta.universe)[:3])
def test_write_future(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() secs = 'rb1710.SHF,j1710.DCE' props = {'start_date': 20170401, 'end_date': 20170901, 'symbol': secs, 'fields': 'open,close,high,low,volume,oi', 'freq': 1, 'all_price': False} dv.init_from_config(props, data_api=ds) dv.prepare_data() assert dv.data_d.shape == (145, 14)
def test_q_add_field(): dv = DataView() dv.load_dataview(folder_path=quarterly_path) nrows, ncols = dv.data_q.shape n_securities = len(dv.data_d.columns.levels[0]) ds = RemoteDataService() ds.init_from_config(data_config) dv.add_field('net_inc_other_ops', ds) """ dv.add_field('oper_rev', ds) dv.add_field('turnover', ds) """ assert dv.data_q.shape == (nrows, ncols + 1 * n_securities)
def analyze_event(): # -------------------------------------------------------------------------------- # Step.1 load dataview dv = DataView() dv.load_dataview(dataview_folder) # -------------------------------------------------------------------------------- # Step.3 get signal, benchmark and price data target_symbol = '600519.SH' price = dv.get_ts('close_adj', symbol=target_symbol) dv.add_formula('in_', 'open_adj / Delay(close_adj, 1)', is_quarterly=False) signal = dv.get_ts('in_', symbol=target_symbol).shift(1, axis=0) # avoid look-ahead bias # Step.4 analyze! obj = SignalDigger(output_folder='../../output', output_format='pdf') obj.create_single_signal_report(signal, price, [1, 5, 9, 21], 6, mask=None, buy_condition={'cond1': {'column': 'quantile', 'filter': lambda x: x > 3, 'hold': 5}, 'cond2': {'column': 'quantile', 'filter': lambda x: x > 5, 'hold': 5}, 'cond3': {'column': 'quantile', 'filter': lambda x: x > 5, 'hold': 9}, })
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "benchmark": "000300.SH", "universe": ','.join(dv.symbol), "start_date": dv.start_date, "end_date": dv.end_date, "period": "month", "days_delay": 0, "init_balance": 1e8, "position_ratio": 1.0, } props.update(data_config) props.update(trade_config) trade_api = AlphaTradeApi() trade_api.init_from_config(props) def selector_growth(context, user_options=None): growth_rate = context.snapshot['net_profit_growth'] return (growth_rate >= 0.2) & (growth_rate <= 4) def selector_pe(context, user_options=None): pe_ttm = context.snapshot['pe_ttm'] return (pe_ttm >= 5) & (pe_ttm <= 80) stock_selector = model.StockSelector() stock_selector.add_filter(name='net_profit_growth', func=selector_growth) stock_selector.add_filter(name='pe', func=selector_pe) strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='equal_weight') pm = PortfolioManager() bt = AlphaBacktestInstance() context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) stock_selector.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "benchmark": BENCHMARK, "universe": ','.join(dv.symbol), "start_date": dv.start_date, "end_date": dv.end_date, "period": "day", "days_delay": 0, "init_balance": 1e8, "position_ratio": 1.0, "strategy_no": 44 } props.update(data_config) props.update(trade_config) stock_selector = model.StockSelector() stock_selector.add_filter(name='rank_ret_top10', func=my_selector) strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='equal_weight') pm = PortfolioManager() if is_backtest: bt = AlphaBacktestInstance() trade_api = AlphaTradeApi() ds = None else: bt = AlphaLiveTradeInstance() trade_api = RealTimeTradeApi(props) ds = RemoteDataService() context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm, data_api=ds) stock_selector.register_context(context) bt.init_from_config(props) bt.run_alpha() if is_backtest: bt.save_results(folder_path=backtest_result_dir_path) else: goal_positions = strategy.goal_positions print(goal_positions)
def load_data(symbol): dv = DataView() dv.load_dataview(folder_path=dataview_store_folder) df = pd.DataFrame() df['close'] = dv.get_ts('close', symbol=symbol, start_date=20080101, end_date=20171231)[symbol] df['open'] = dv.get_ts('open', symbol=symbol, start_date=20080101, end_date=20171231)[symbol] df['high'] = dv.get_ts('high', symbol=symbol, start_date=20080101, end_date=20171231)[symbol] df['low'] = dv.get_ts('low', symbol=symbol, start_date=20080101, end_date=20171231)[symbol] df = df.dropna() return df
def test_init_data(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = { 'start_date': 20150101, 'end_date': 20170930, 'universe': '000905.SH', 'fields': ('turnover,float_mv,close_adj,pe,pb'), 'freq': 1 } dv.init_from_config(props, ds) dv.prepare_data() # # factor_formula = 'Cutoff(Standardize(turnover / 10000 / float_mv), 2)' # dv.add_formula('TO', factor_formula, is_quarterly=False) # # factor_formula = 'Cutoff(Standardize(1/pb), 2)' # dv.add_formula('BP', factor_formula, is_quarterly=False) # # factor_formula = 'Cutoff(Standardize(Return(close_adj, 20)), 2)' # dv.add_formula('REVS20', factor_formula, is_quarterly=False) # # factor_formula = 'Cutoff(Standardize(Log(float_mv)), 2)' # dv.add_formula('float_mv_factor', factor_formula, is_quarterly=False) # # factor_formula = 'Delay(Return(close_adj, 1), -1)' # dv.add_formula('NextRet', factor_formula, is_quarterly=False) dv.save_dataview(folder_path=dataview_dir_path)
def test_write(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() secs = '600030.SH,000063.SZ,000001.SZ' props = { 'start_date': 20160601, 'end_date': 20170601, 'symbol': secs, 'fields': 'open,close,high,low,volume,pb,net_assets,pcf_ncf', 'freq': 1 } dv.init_from_config(props, data_api=ds) dv.prepare_data() assert dv.data_d.shape == (281, 48) assert dv.dates.shape == (281, ) # TODO """ PerformanceWarning: your performance may suffer as PyTables will pickle object types that it cannot map directly to c-types [inferred_type->mixed,key->block1_values] [items->[('000001.SZ', 'int_income'), ('000001.SZ', 'less_handling_chrg_comm_exp'), ('000001.SZ', 'net_int_income'), ('000001.SZ', 'oper_exp'), ('000001.SZ', 'symbol'), ('000063.SZ', 'int_income'), ('000063.SZ', 'less_handling_chrg_comm_exp'), ('000063.SZ', 'net_int_income'), ('000063.SZ', 'oper_exp'), ('000063.SZ', 'symbol'), ('600030.SH', 'int_income'), ('600030.SH', 'less_handling_chrg_comm_exp'), ('600030.SH', 'net_int_income'), ('600030.SH', 'oper_exp'), ('600030.SH', 'symbol')]] """ dv.save_dataview(folder_path=daily_path)
def save_data(): """ This function fetches data from remote server and stores them locally. Then we can use local data to do back-test. """ dataview_props = {# Start and end date of back-test 'start_date': 20170101, 'end_date': 20171231, # Investment universe and performance benchmark 'universe': UNIVERSE, 'benchmark': '000300.SH', #业绩比较基准(benchmark):沪深300指数 000300.SH # Data fields that we need 'fields': 'total_mv,turnover', # freq = 1 means we use daily data. Please do not change this. 'freq': 1} # RemoteDataService communicates with a remote server to fetch data ds = RemoteDataService() # Use username and password in data_config to login ds.init_from_config(data_config) # DataView utilizes RemoteDataService to get various data and store them dv = DataView() dv.init_from_config(dataview_props, ds) dv.prepare_data() dv.save_dataview(folder_path=dataview_store_folder)
def add(formula, name): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) dv.add_formula(name, formula, is_quarterly=False, formula_func_name_style='lower') dv.save_dataview(folder_path=dataview_dir_path)
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) #回测参数选择 props = { "benchmark": "000905.SH", "universe": ','.join(dv.symbol), "start_date": 20170605, "end_date": 20180807, "period": "day", "days_delay": 0, "init_balance": 1e9, "position_ratio": 1.0, "commission_rate": 0.0015, #手续费 "n_periods": 2 } props.update(data_config) props.update(trade_config) trade_api = AlphaTradeApi() signal_model = model.FactorSignalModel() #添加信号 signal_model.add_signal('alpha3', alpha) #在使用新因子时,alpha3应改为新因子的名称 stock_selector = model.StockSelector() stock_selector.add_filter(name='myselector', func=my_selector) strategy = AlphaStrategy(stock_selector=stock_selector, signal_model=signal_model, pc_method='factor_value_weight') pm = PortfolioManager() bt = AlphaBacktestInstance() context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) for mdl in [signal_model, stock_selector]: mdl.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "benchmark": "000300.SH", "universe": ','.join(dv.symbol), "start_date": dv.start_date, "end_date": dv.end_date, "period": "month", "days_delay": 0, "init_balance": 1e8, "position_ratio": 1.0, } props.update(data_config) props.update(trade_config) trade_api = AlphaTradeApi() trade_api.init_from_config(props) def selector_growth(context, user_options=None): growth_rate = context.snapshot['net_profit_growth'] return (growth_rate >= 0.2) & (growth_rate <= 4) def selector_pe(context, user_options=None): pe_ttm = context.snapshot['pe_ttm'] return (pe_ttm >= 10) & (pe_ttm <= 20) stock_selector = model.StockSelector() stock_selector.add_filter(name='net_profit_growth', func=selector_growth) stock_selector.add_filter(name='pe', func=selector_pe) strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='equal_weight') pm = PortfolioManager() bt = AlphaBacktestInstance() context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) stock_selector.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def test_save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = {'start_date': 20170201, 'end_date': 20171001, 'universe': '000300.SH', 'fields': ('float_mv,sw2,sw1'), 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'GroupQuantile(float_mv, sw1, 10)' dv.add_formula('gq30', factor_formula, is_quarterly=False) dv.save_dataview(folder_path=dataview_dir_path)
def test_q_add_formula(): dv = DataView() folder_path = '../output/prepared/20160609_20170601_freq=1D' dv.load_dataview(folder_path=quarterly_path) nrows, ncols = dv.data_d.shape n_securities = len(dv.data_d.columns.levels[0]) formula = 'total_oper_rev / close' dv.add_formula('myvar1', formula, is_quarterly=False) df1 = dv.get_ts('myvar1') assert not df1.empty formula2 = 'Delta(oper_exp * myvar1 - open, 3)' dv.add_formula('myvar2', formula2, is_quarterly=False) df2 = dv.get_ts('myvar2') assert not df2.empty
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "benchmark": BENCHMARK, "universe": ','.join(dv.symbol), "start_date": dv.start_date, "end_date": dv.end_date, "period": "day", "days_delay": 0, "init_balance": 1e8, "position_ratio": 1.0, "strategy_no": 44 } props.update(data_config) props.update(trade_config) stock_selector = model.StockSelector() stock_selector.add_filter(name='rank_ret_top10', func=my_selector) strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='equal_weight') pm = PortfolioManager() if is_backtest: bt = AlphaBacktestInstance() trade_api = AlphaTradeApi() ds = None else: bt = AlphaLiveTradeInstance() trade_api = RealTimeTradeApi(props) ds = RemoteDataService() context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm, data_api=ds) stock_selector.register_context(context) bt.init_from_config(props) bt.run_alpha() if is_backtest: bt.save_results(folder_path=backtest_result_dir_path) else: goal_positions = strategy.goal_positions print(goal_positions)
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "benchmark": "000300.SH", "universe": ','.join(dv.symbol), "start_date": 20170131, "end_date": dv.end_date, "period": "month", "days_delay": 0, "init_balance": 1e9, "position_ratio": 1.0, } props.update(data_config) props.update(trade_config) trade_api = AlphaTradeApi() def singal_gq30(context, user_options=None): import numpy as np res = np.power(context.snapshot['gq30'], 8) return res signal_model = model.FactorSignalModel() signal_model.add_signal('signal_gq30', singal_gq30) strategy = AlphaStrategy(signal_model=signal_model, pc_method='factor_value_weight') pm = PortfolioManager() bt = AlphaBacktestInstance() context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) signal_model.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def test_load(): dv = DataView() dv.load_dataview(folder_path=daily_path) assert dv.start_date == 20160601 and set(dv.symbol) == set('000001.SZ,600030.SH,000063.SZ'.split(',')) # test get_snapshot snap1 = dv.get_snapshot(20170504, symbol='600030.SH,000063.SZ', fields='close,pb') assert snap1.shape == (2, 2) assert set(snap1.columns.values) == {'close', 'pb'} assert set(snap1.index.values) == {'600030.SH', '000063.SZ'} # test get_ts ts1 = dv.get_ts('close', symbol='600030.SH,000063.SZ', start_date=20170101, end_date=20170302) assert ts1.shape == (38, 2) assert set(ts1.columns.values) == {'600030.SH', '000063.SZ'} assert ts1.index.values[-1] == 20170302
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "start_date": dv.start_date, "end_date": dv.end_date, "period": "week", "days_delay": 0, "init_balance": 1e8, 'commission_rate': 0.0 } props.update(data_config) props.update(trade_config) trade_api = AlphaTradeApi() bt = AlphaBacktestInstance() stock_selector = model.StockSelector() stock_selector.add_filter(name='myselector', func=my_selector) strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='equal_weight') pm = PortfolioManager() context = model.AlphaContext(dataview=dv, trade_api=trade_api, instance=bt, strategy=strategy, pm=pm) store = pd.HDFStore(ic_weight_hd5_path) factorList = jutil.read_json(custom_data_path) context.ic_weight = store['ic_weight'] context.factorList = factorList store.close() for mdl in [stock_selector]: mdl.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def test_save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = {'start_date': 20170101, 'end_date': 20171001, 'universe': '000300.SH', 'fields': 'pe_ttm,net_profit_incl_min_int_inc', 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'Return(net_profit_incl_min_int_inc, 4)' factor_name = 'net_profit_growth' dv.add_formula(factor_name, factor_formula, is_quarterly=True) dv.save_dataview(folder_path=dataview_dir_path)
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "benchmark": "000300.SH", "universe": ','.join(dv.symbol), "start_date": 20170131, "end_date": dv.end_date, "period": "month", "days_delay": 0, "init_balance": 1e9, "position_ratio": 1.0, } props.update(data_config) props.update(trade_config) trade_api = AlphaTradeApi() def singal_gq30(context, user_options=None): import numpy as np res = np.power(context.snapshot['gq30'], 8) return res signal_model = model.FactorSignalModel() signal_model.add_signal('signal_gq30', singal_gq30) strategy = AlphaStrategy(signal_model=signal_model, pc_method='factor_value_weight') pm = PortfolioManager() bt = AlphaBacktestInstance() context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) signal_model.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def get_dv(start=20170101, end=20180101): import warnings warnings.filterwarnings("ignore") try: import jaqs_fxdayu jaqs_fxdayu.patch_all() from jaqs.data import DataView from jaqs_fxdayu.data.dataservice import LocalDataService except ImportError as e: _handle_import_exception(e) ds = LocalDataService(fp=get_data_root()) ZZ800_id = ds.query_index_member("000906.SH", start, end) stock_symbol = list(set(ZZ800_id)) dv_props = { 'start_date': start, 'end_date': end, 'symbol': ','.join(stock_symbol), 'fields': "", 'freq': 1, "prepare_fields": True } dv = DataView() dv.init_from_config(dv_props, data_api=ds) dv.prepare_data() hs300_benchmark = dv.data_api.daily("000300.SH", dv.extended_start_date_d, dv.end_date, fields='trade_date,close') dv.data_benchmark = hs300_benchmark[0][['trade_date', 'close']].set_index('trade_date') return dv
def save_data(): """ This function fetches data from remote server and stores them locally. Then we can use local data to do back-test. """ dataview_props = {# Start and end date of back-test 'start_date': 20170101, 'end_date': 20171030, # Investment universe and performance benchmark 'universe': UNIVERSE, 'benchmark': '000300.SH', # Data fields that we need 'fields': 'total_mv,turnover', # freq = 1 means we use daily data. Please do not change this. 'freq': 1} # RemoteDataService communicates with a remote server to fetch data ds = RemoteDataService() # Use username and password in data_config to login ds.init_from_config(data_config) # DataView utilizes RemoteDataService to get various data and store them dv = DataView() dv.init_from_config(dataview_props, ds) dv.prepare_data() dv.save_dataview(folder_path=dataview_store_folder)
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "start_date": dv.start_date, "end_date": dv.end_date, "period": "week", "days_delay": 0, "init_balance": 1e8, 'commission_rate': 0.0 } props.update(data_config) props.update(trade_config) trade_api = AlphaTradeApi() bt = AlphaBacktestInstance() stock_selector = model.StockSelector() stock_selector.add_filter(name='myselector', func=my_selector) strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='equal_weight') pm = PortfolioManager() context = model.AlphaContext(dataview=dv, trade_api=trade_api, instance=bt, strategy=strategy, pm=pm) store = pd.HDFStore(ic_weight_hd5_path) factorList = jutil.read_json(custom_data_path) context.ic_weight = store['ic_weight'] context.factorList = factorList store.close() for mdl in [stock_selector]: mdl.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def test_optimizer(): from jaqs_fxdayu.research import Optimizer dv = DataView() dv.load_dataview(dataview_folder) mask = mask_index_member(dv) can_enter, can_exit = limit_up_down(dv) price = dv.get_ts('close_adj') high = dv.get_ts('high_adj') low = dv.get_ts('low_adj') price_bench = dv.data_benchmark optimizer = Optimizer(dataview=dv, formula='- Correlation(vwap_adj, volume, LEN)', params={"LEN": range(2, 4, 1)}, name='divert', price=price, high=high, low=low, benchmark_price=price_bench, # =None求绝对收益 #=price_bench求相对收益 period=30, n_quantiles=5, mask=mask, can_enter=can_enter, can_exit=can_exit, commission=0.0008, # 手续费 默认0.0008 is_event=False, # 是否是事件(0/1因子) is_quarterly=False) # 是否是季度因子 默认为False ret_best = optimizer.enumerate_optimizer(target_type="top_quantile_ret", # 优化目标类型 target="Ann. IR", # 优化目标 in_sample_range=[20140101, 20160101], # 样本内范围 默认为None,在全样本上优化 ascending=False) # 是否按优化目标升序排列(从小到大)
def get_dv(start=20170101, end=20180101): import jaqs_fxdayu jaqs_fxdayu.patch_all() from jaqs.data import DataView from jaqs_fxdayu.data.dataservice import LocalDataService import warnings warnings.filterwarnings("ignore") #-------------------------------------------------------- #define factor_list = [ 'volume', 'tot_profit', 'plus_non_oper_rev', 'less_non_oper_exp', 'net_cash_flows_oper_act', 'total_oper_rev', 'tot_oper_cost', 'fin_exp', 'less_selling_dist_exp', 'less_gerl_admin_exp', 'oper_rev', 'less_oper_cost', 'turnover' ] check_factor = ','.join(factor_list) dataview_folder = 'G:\GSICE\DATA\data' ds = LocalDataService(fp=dataview_folder) ZZ800_id = ds.query_index_member("000906.SH", start, end) stock_symbol = list(set(ZZ800_id)) dv_props = { 'start_date': start, 'end_date': end, 'symbol': ','.join(stock_symbol), 'fields': check_factor, 'freq': 1, "prepare_fields": True } dv = DataView() dv.init_from_config(dv_props, data_api=ds) dv.prepare_data() data_config = { "remote.data.address": "tcp://data.tushare.org:8910", "remote.data.username": "******", "remote.data.password": "******" } from jaqs_fxdayu.data import RemoteDataService Ds = RemoteDataService() Ds.init_from_config(data_config) dv.add_field('pe', Ds) return dv
def store_ic_weight(): """ Calculate IC weight and save it to file """ dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) factorList = ['TO', 'BP', 'REVS20', 'float_mv_factor'] orthFactor_dic = {} for factor in factorList: orthFactor_dic[factor] = {} # add the orthogonalized factor to dataview for trade_date in dv.dates: snapshot = dv.get_snapshot(trade_date) factorPanel = snapshot[factorList] factorPanel = factorPanel.dropna() if len(factorPanel) != 0: orthfactorPanel = Schmidt(factorPanel) orthfactorPanel.columns = [x + '_adj' for x in factorList] snapshot = pd.merge(left=snapshot, right=orthfactorPanel, left_index=True, right_index=True, how='left') for factor in factorList: orthFactor_dic[factor][trade_date] = snapshot[factor] for factor in factorList: dv.append_df(pd.DataFrame(orthFactor_dic[factor]).T, field_name=factor + '_adj', is_quarterly=False) dv.save_dataview(dataview_dir_path) factorList_adj = [x + '_adj' for x in factorList] jutil.save_json(factorList_adj, custom_data_path) w = get_ic_weight(dv) store = pd.HDFStore(ic_weight_hd5_path) store['ic_weight'] = w store.close()
def test_q_add_formula(): dv = DataView() folder_path = '../output/prepared/20160609_20170601_freq=1D' dv.load_dataview(folder_path=quarterly_path) nrows, ncols = dv.data_d.shape n_securities = len(dv.data_d.columns.levels[0]) formula = 'total_oper_rev / close' dv.add_formula('myvar1', formula, is_quarterly=False) df1 = dv.get_ts('myvar1') assert not df1.empty formula2 = 'Delta(oper_exp * myvar1 - open, 3)' dv.add_formula('myvar2', formula2, is_quarterly=False) df2 = dv.get_ts('myvar2') assert not df2.empty
def do_backtest(): # Load local data file that we just stored. dv = DataView() dv.load_dataview(folder_path=dataview_store_folder) backtest_props = {# start and end date of back-test "start_date": dv.start_date, "end_date": dv.end_date, # re-balance period length "period": "month", # benchmark and universe "benchmark": dv.benchmark, "universe": dv.universe, # Amount of money at the start of back-test "init_balance": 1e8, # Amount of money at the start of back-test "position_ratio": 1.0, } backtest_props.update(data_config) backtest_props.update(trade_config) # We use trade_api to send orders trade_api = AlphaTradeApi() # This is our strategy strategy = AlphaStrategy(pc_method='market_value_weight') # PortfolioManager helps us to manage tasks, orders and calculate positions pm = PortfolioManager() # BacktestInstance is in charge of running the back-test bt = AlphaBacktestInstance() # Public variables are stored in context. We can also store anything in it context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) bt.init_from_config(backtest_props) bt.run_alpha() # After finishing back-test, we save trade results into a folder bt.save_results(folder_path=backtest_result_folder)
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "start_date": dv.start_date, "end_date": dv.end_date, "period": "week", "days_delay": 0, "init_balance": 1e8, "position_ratio": 1.0, } props.update(data_config) props.update(trade_config) trade_api = AlphaTradeApi() stock_selector = model.StockSelector() stock_selector.add_filter(name='myselector', func=my_selector) signal_model = model.FactorSignalModel() signal_model.add_signal(name='signalsize', func=signal_size) strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='factor_value_weight', signal_model=signal_model) pm = PortfolioManager() bt = AlphaBacktestInstance() context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) for mdl in [signal_model, stock_selector]: mdl.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def test_add_formula(): dv = DataView() dv.load_dataview(folder_path=daily_path) nrows, ncols = dv.data_d.shape n_securities = len(dv.data_d.columns.levels[0]) formula = 'Delta(high - close, 1)' dv.add_formula('myvar1', formula, is_quarterly=False) assert dv.data_d.shape == (nrows, ncols + 1 * n_securities) formula2 = 'myvar1 - close' dv.add_formula('myvar2', formula2, is_quarterly=False) assert dv.data_d.shape == (nrows, ncols + 2 * n_securities)
def save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = {'start_date': 20160101, 'end_date': 20171001, 'universe': '000300.SH', 'fields': 'volume,turnover', 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() dv.save_dataview(dataview_folder)
def test_save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() print(DataView) props = {'start_date': 20170501, 'end_date': 20171001, 'universe': '000016.SH', 'fields': 'volume,pb,pe,ps,float_mv,sw1', 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() dv.save_dataview(dataview_folder)
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "symbol": dv.symbol, "universe": ','.join(dv.symbol), "start_date": dv.start_date, "end_date": dv.end_date, "period": "week", "days_delay": 0, "init_balance": 1e7, "position_ratio": 1.0, "commission_rate": 2E-4 # 手续费万2 } props.update(data_config) props.update(trade_config) trade_api = AlphaTradeApi() signal_model = model.FactorSignalModel() signal_model.add_signal('stockWeight', stockWeight) strategy = AlphaStrategy(signal_model=signal_model, pc_method='factor_value_weight') pm = PortfolioManager() bt = AlphaBacktestInstance() context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) signal_model.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def get_dv(start=20170101, end=20180101): import jaqs_fxdayu jaqs_fxdayu.patch_all() from jaqs.data import DataView from jaqs_fxdayu.data.dataservice import LocalDataService import warnings warnings.filterwarnings("ignore") #-------------------------------------------------------- #define factor_list = ['PB', 'turnover_ratio', 'volume', 'pe_ttm'] check_factor = ','.join(factor_list) dataview_folder = r'E:\BaiduNetdiskDownload\data\data' ds = LocalDataService(fp=dataview_folder) ZZ800_id = ds.query_index_member("000906.SH", start, end) stock_symbol = list(set(ZZ800_id)) dv_props = { 'start_date': start, 'end_date': end, 'symbol': ','.join(stock_symbol), 'fields': check_factor, 'freq': 1, "prepare_fields": True } dv = DataView() dv.init_from_config(dv_props, data_api=ds) dv.prepare_data() return dv
def get_dv(start = 20170101,end = 20180101): import jaqs_fxdayu jaqs_fxdayu.patch_all() from jaqs.data import DataView from jaqs_fxdayu.data.dataservice import LocalDataService import warnings warnings.filterwarnings("ignore") #-------------------------------------------------------- #define factor_list = ['TSEPToTotalCapital','alpha107','TRIX5_J','OperatingRevenueGrowRate_J','LossVariance60','BIAS60_J','alpha110','DIZ_J'] check_factor = ','.join(factor_list) dataview_folder = r'E:/data/data' ds = LocalDataService(fp = dataview_folder) ZZ800_id = ds.query_index_member("000906.SH", start, end) stock_symbol = list(set(ZZ800_id)) dv_props = {'start_date': start, 'end_date': end, 'symbol':','.join(stock_symbol), 'fields': check_factor, 'freq': 1, "prepare_fields": True} dv = DataView() dv.init_from_config(dv_props, data_api=ds) dv.prepare_data() return dv
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "start_date": dv.start_date, "end_date": dv.end_date, "period": "week", "days_delay": 0, "init_balance": 1e8, "position_ratio": 1.0, } props.update(data_config) props.update(trade_config) trade_api = AlphaTradeApi() stock_selector = model.StockSelector() stock_selector.add_filter(name='myselector', func=my_selector) signal_model = model.FactorSignalModel() signal_model.add_signal(name='signalsize', func=signal_size) strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='factor_value_weight', signal_model=signal_model) pm = PortfolioManager() bt = AlphaBacktestInstance() context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) for mdl in [signal_model, stock_selector]: mdl.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def do_backtest(): # Load local data file that we just stored. dv = DataView() dv.load_dataview(folder_path=dataview_store_folder) backtest_props = {# start and end date of back-test "start_date": dv.start_date, "end_date": dv.end_date, # re-balance period length "period": "month", # benchmark and universe "benchmark": dv.benchmark, "universe": dv.universe, # Amount of money at the start of back-test "init_balance": 1e8, # Amount of money at the start of back-test "position_ratio": 1.0, } backtest_props.update(data_config) backtest_props.update(trade_config) # We use trade_api to send orders trade_api = AlphaTradeApi() # This is our strategy strategy = AlphaStrategy(pc_method='market_value_weight') # PortfolioManager helps us to manage tasks, orders and calculate positions pm = PortfolioManager() # BacktestInstance is in charge of running the back-test bt = AlphaBacktestInstance() # Public variables are stored in context. We can also store anything in it context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) bt.init_from_config(backtest_props) bt.run_alpha() # After finishing back-test, we save trade results into a folder bt.save_results(folder_path=backtest_result_folder)
def test_alpha_strategy_dataview(): dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "benchmark": "000300.SH", "universe": ','.join(dv.symbol), "start_date": dv.start_date, "end_date": dv.end_date, "period": "month", "days_delay": 0, "init_balance": 1e8, "position_ratio": 1.0, } props.update(data_config) props.update(trade_config) trade_api = AlphaTradeApi() stock_selector = model.StockSelector() stock_selector.add_filter(name='myrank', func=my_selector) strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='equal_weight') pm = PortfolioManager() bt = AlphaBacktestInstance() context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) stock_selector.register_context(context) bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path)
def test_add_formula(): dv = DataView() dv.load_dataview(folder_path=daily_path) nrows, ncols = dv.data_d.shape n_securities = len(dv.data_d.columns.levels[0]) formula = 'Delta(high - close, 1)' dv.add_formula('myvar1', formula, is_quarterly=False) assert dv.data_d.shape == (nrows, ncols + 1 * n_securities) formula2 = 'myvar1 - close' dv.add_formula('myvar2', formula2, is_quarterly=False) assert dv.data_d.shape == (nrows, ncols + 2 * n_securities)
def test_add_formula_directly(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() secs = '600030.SH,000063.SZ,000001.SZ' props = {'start_date': 20160601, 'end_date': 20170601, 'symbol': secs, 'fields': 'open,close', 'freq': 1} dv.init_from_config(props, data_api=ds) dv.prepare_data() dv.add_formula("myfactor", 'close / open', is_quarterly=False) assert dv.data_d.shape == (281, 39)
def store_ic_weight(): """ Calculate IC weight and save it to file """ dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) factorList = ['TO', 'BP', 'REVS20', 'float_mv_factor'] orthFactor_dic = {} for factor in factorList: orthFactor_dic[factor] = {} # add the orthogonalized factor to dataview for trade_date in dv.dates: snapshot = dv.get_snapshot(trade_date) factorPanel = snapshot[factorList] factorPanel = factorPanel.dropna() if len(factorPanel) != 0: orthfactorPanel = Schmidt(factorPanel) orthfactorPanel.columns = [x + '_adj' for x in factorList] snapshot = pd.merge(left=snapshot, right=orthfactorPanel, left_index=True, right_index=True, how='left') for factor in factorList: orthFactor_dic[factor][trade_date] = snapshot[factor] for factor in factorList: dv.append_df(pd.DataFrame(orthFactor_dic[factor]).T, field_name=factor + '_adj', is_quarterly=False) dv.save_dataview(dataview_dir_path) factorList_adj = [x + '_adj' for x in factorList] jutil.save_json(factorList_adj, custom_data_path) w = get_ic_weight(dv) store = pd.HDFStore(ic_weight_hd5_path) store['ic_weight'] = w store.close()
def test_q(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() secs = '600030.SH,000063.SZ,000001.SZ' props = {'start_date': 20160609, 'end_date': 20170601, 'symbol': secs, 'fields': ('open,close,' + 'pb,net_assets,' + 'total_oper_rev,oper_exp,' + 'cash_paid_invest,' + 'capital_stk,' + 'roe'), 'freq': 1} dv.init_from_config(props, data_api=ds) dv.prepare_data() dv.save_dataview(folder_path=quarterly_path)
def test_write(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() secs = '600030.SH,000063.SZ,000001.SZ' props = {'start_date': 20160601, 'end_date': 20170601, 'symbol': secs, 'fields': 'open,close,high,low,volume,pb,net_assets,pcf_ncf', 'freq': 1} dv.init_from_config(props, data_api=ds) dv.prepare_data() assert dv.data_d.shape == (281, 48) assert dv.dates.shape == (281, ) # TODO """ PerformanceWarning: your performance may suffer as PyTables will pickle object types that it cannot map directly to c-types [inferred_type->mixed,key->block1_values] [items->[('000001.SZ', 'int_income'), ('000001.SZ', 'less_handling_chrg_comm_exp'), ('000001.SZ', 'net_int_income'), ('000001.SZ', 'oper_exp'), ('000001.SZ', 'symbol'), ('000063.SZ', 'int_income'), ('000063.SZ', 'less_handling_chrg_comm_exp'), ('000063.SZ', 'net_int_income'), ('000063.SZ', 'oper_exp'), ('000063.SZ', 'symbol'), ('600030.SH', 'int_income'), ('600030.SH', 'less_handling_chrg_comm_exp'), ('600030.SH', 'net_int_income'), ('600030.SH', 'oper_exp'), ('600030.SH', 'symbol')]] """ dv.save_dataview(folder_path=daily_path)
def test_save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = {'start_date': 20170901, 'end_date': 20171129, 'universe': BENCHMARK, 'fields': 'close,volume,sw1', 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() dv.add_formula('ret', 'Return(close_adj, 20)', is_quarterly=False) dv.add_formula('rank_ret', 'Rank(ret)', is_quarterly=False) dv.save_dataview(folder_path=dataview_dir_path)
def test_save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = {'start_date': 20170101, 'end_date': 20171001, 'universe': '000300.SH', 'fields': ('float_mv,pb,pe_ttm,sw2'), 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'GroupQuantile(-float_mv, sw2, 10)' dv.add_formula('rank_mv', factor_formula, is_quarterly=False) factor_formula = 'GroupQuantile(If(pb >= 0.2, pb, 100), sw2, 10)' dv.add_formula('rank_pb', factor_formula, is_quarterly=False) factor_formula = 'GroupQuantile(If(pe_ttm >= 3, pe_ttm, 9999.0), sw2, 10)' dv.add_formula('rank_pe', factor_formula, is_quarterly=False) dv.save_dataview(folder_path=dataview_dir_path)
def save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = {'start_date': 20150101, 'end_date': 20170930, 'universe': '000905.SH', 'fields': ('turnover,float_mv,close_adj,pe,pb'), 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'Cutoff(Standardize(turnover / 10000 / float_mv), 2)' dv.add_formula('TO', factor_formula, is_quarterly=False) factor_formula = 'Cutoff(Standardize(1/pb), 2)' dv.add_formula('BP', factor_formula, is_quarterly=False) factor_formula = 'Cutoff(Standardize(Return(close_adj, 20)), 2)' dv.add_formula('REVS20', factor_formula, is_quarterly=False) factor_formula = 'Cutoff(Standardize(Log(float_mv)), 2)' dv.add_formula('float_mv_factor', factor_formula, is_quarterly=False) factor_formula = 'Delay(Return(close_adj, 1), -1)' dv.add_formula('NextRet', factor_formula, is_quarterly=False) dv.save_dataview(folder_path=dataview_dir_path)