def save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = {'start_date': 20160101, 'end_date': 20171001, 'universe': '000300.SH', 'fields': 'volume,turnover', 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() # for convenience to check limit reachers dv.add_formula('limit_reached', 'Abs((open - Delay(close, 1)) / Delay(close, 1)) > 0.095', is_quarterly=False) dv.add_formula('mask_limit_reached', 'limit_reached > 0', is_quarterly=False) dv.add_formula('mask_index_member', '!(index_member > 0)', is_quarterly=False) trade_status = dv.get_ts('trade_status') mask_sus = trade_status == u'停牌' dv.append_df(mask_sus, 'mask_sus', is_quarterly=False) # dv.add_formula('size', '', is_quarterly=False) dv.save_dataview(dataview_folder)
def prepare_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) symbols = ['600036.SH', '000001.SZ'] dv_props = {'symbol': ','.join(symbols), 'start_date': backtest_props['start_date'], 'end_date': backtest_props['end_date'], 'benchmark': '000300.SH'} dv = EventDataView() dv.init_from_config(dv_props, ds) dv.prepare_data() import pandas as pd # target security diff_cols = ['open', 'high', 'low', 'close'] df0 = dv.get_symbol(symbols[0], fields=','.join(diff_cols)) df1 = dv.get_symbol(symbols[1], fields=','.join(diff_cols)) df_diff = df0 - df1 # calculate signal df_signal = pd.concat([df0[['close']], df1[['close']], df_diff[['close']]], axis=1) df_signal.columns = symbols + ['diff'] roll = df_signal['diff'].rolling(window=10) df_signal.loc[:, 'signal'] = (df_signal['diff'] - roll.mean()) / roll.std() dv.append_df_symbol(df_diff, '001.JZ') dv.data_custom = df_signal dv.save_dataview(dataview_dir_path)
def save_data(): """ This function fetches data from remote server and stores them locally. Then we can use local data to do back-test. """ dataview_props = {# Start and end date of back-test 'start_date': 20170101, 'end_date': 20171030, # Investment universe and performance benchmark 'universe': UNIVERSE, 'benchmark': '000300.SH', # Data fields that we need 'fields': 'total_mv,turnover', # freq = 1 means we use daily data. Please do not change this. 'freq': 1} # RemoteDataService communicates with a remote server to fetch data ds = RemoteDataService() # Use username and password in data_config to login ds.init_from_config(data_config) # DataView utilizes RemoteDataService to get various data and store them dv = DataView() dv.init_from_config(dataview_props, ds) dv.prepare_data() dv.save_dataview(folder_path=dataview_store_folder)
def save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = {'start_date': 20150101, 'end_date': 20170930, 'universe': '000905.SH', 'fields': ('turnover,float_mv,close_adj,pe,pb'), 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'Cutoff(Standardize(turnover / 10000 / float_mv), 2)' dv.add_formula('TO', factor_formula, is_quarterly=False) factor_formula = 'Cutoff(Standardize(1/pb), 2)' dv.add_formula('BP', factor_formula, is_quarterly=False) factor_formula = 'Cutoff(Standardize(Return(close_adj, 20)), 2)' dv.add_formula('REVS20', factor_formula, is_quarterly=False) factor_formula = 'Cutoff(Standardize(Log(float_mv)), 2)' dv.add_formula('float_mv_factor', factor_formula, is_quarterly=False) factor_formula = 'Delay(Return(close_adj, 1), -1)' dv.add_formula('NextRet', factor_formula, is_quarterly=False) dv.save_dataview(folder_path=dataview_dir_path)
def test_save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = {'start_date': 20150101, 'end_date': 20170930, 'universe': '000905.SH', 'fields': ('tot_cur_assets,tot_cur_liab,inventories,pre_pay,deferred_exp,' 'eps_basic,ebit,pe,pb,float_mv,sw1'), 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'pe < 30' dv.add_formula('pe_condition', factor_formula, is_quarterly=False) factor_formula = 'pb < 3' dv.add_formula('pb_condition', factor_formula, is_quarterly=False) factor_formula = 'Return(eps_basic, 4) > 0' dv.add_formula('eps_condition', factor_formula, is_quarterly=True) factor_formula = 'Return(ebit, 4) > 0' dv.add_formula('ebit_condition', factor_formula, is_quarterly=True) factor_formula = 'tot_cur_assets/tot_cur_liab > 2' dv.add_formula('current_condition', factor_formula, is_quarterly=True) factor_formula = '(tot_cur_assets - inventories - pre_pay - deferred_exp)/tot_cur_liab > 1' dv.add_formula('quick_condition', factor_formula, is_quarterly=True) dv.add_formula('mv_rank', 'Rank(float_mv)', is_quarterly=False) dv.save_dataview(folder_path=dataview_dir_path)
def test_dataview_universe(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = {'start_date': 20170227, 'end_date': 20170327, 'universe': '000016.SH', # 'symbol': 'rb1710.SHF,rb1801.SHF', 'fields': ('open,high,low,close,vwap,volume,turnover,' + 'sw1,zz2,' + 'roe,net_assets,' + 'total_oper_rev,oper_exp,tot_profit,int_income' ), 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() data_bench = dv.data_benchmark.copy() dv.data_benchmark = data_bench try: dv.data_benchmark = data_bench.iloc[3:] except ValueError: pass dv.remove_field('roe,net_assets') dv.remove_field('close')
def analyze(): ta = ana.EventAnalyzer() ds = RemoteDataService() ds.init_from_config(data_config) ta.initialize(data_server_=ds, file_folder=result_dir_path) ta.do_analyze(result_dir=result_dir_path, selected_sec=props['symbol'].split(','))
def test_add_field(): dv = DataView() dv.load_dataview(folder_path=daily_path) nrows, ncols = dv.data_d.shape n_securities = len(dv.data_d.columns.levels[0]) ds = RemoteDataService() ds.init_from_config(data_config) dv.add_field('total_share', ds) assert dv.data_d.shape == (nrows, ncols + 1 * n_securities)
def test_bar(): from jaqs.data import RemoteDataService from jaqs.trade.common import QUOTE_TYPE ds = RemoteDataService() ds.init_from_config(data_config) df_quotes, msg = ds.bar(symbol='rb1710.SHF,hc1710.SHF', start_time=200000, end_time=160000, trade_date=20170704, freq=QUOTE_TYPE.MIN) bar_list = Bar.create_from_df(df_quotes) bar = bar_list[0] print(str(bar))
def test_inst_manager(): ds = RemoteDataService() ds.init_from_config(data_config) mgr = InstManager(ds) mgr.load_instruments() sym = '000001.SZ' inst_obj = mgr.get_instrument(sym) assert inst_obj.market == 'SZ' assert inst_obj.symbol == sym assert inst_obj.multiplier == 1 assert inst_obj.inst_type == 1
def test_write_future(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() secs = 'rb1710.SHF,j1710.DCE' props = {'start_date': 20170401, 'end_date': 20170901, 'symbol': secs, 'fields': 'open,close,high,low,volume,oi', 'freq': 1, 'all_price': False} dv.init_from_config(props, data_api=ds) dv.prepare_data() assert dv.data_d.shape == (145, 14)
def save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = {'start_date': 20160101, 'end_date': 20171001, 'universe': '000300.SH', 'fields': 'volume,turnover', 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() dv.save_dataview(dataview_folder)
def test_align(): # ------------------------------------------------------------------------------------- # input and pre-process demo data ds = RemoteDataService() ds.init_from_config(data_config) raw, msg = ds.query_lb_fin_stat('income', '600000.SH', 20151225, 20170501, 'oper_rev') assert msg == '0,' idx_list = ['report_date', 'symbol'] raw_idx = raw.set_index(idx_list) raw_idx.sort_index(axis=0, level=idx_list, inplace=True) df_ann = raw_idx.loc[pd.IndexSlice[:, :], 'ann_date'] df_ann = df_ann.unstack(level=1) df_value = raw_idx.loc[pd.IndexSlice[:, :], 'oper_rev'] df_value = df_value.unstack(level=1) date_arr = ds.query_trade_dates(20160101, 20170501) df_close = pd.DataFrame(index=date_arr, columns=df_value.columns, data=1e3) # ------------------------------------------------------------------------------------- # demo usage of parser parser = Parser() parser.register_function('Myfunc', lambda x: x * 0 + 1) # simultaneously test register function and align expr_formula = 'signal / Myfunc(close)' expression = parser.parse(expr_formula) for i in range(100): df_res = parser.evaluate({'signal': df_value, 'close': df_close}, df_ann, date_arr) # ------------------------------------------------------------------------------------- sec = '600000.SH' """ # print to validate results print "\n======Expression Formula:\n{:s}".format(expr_formula) print "\n======Report date, ann_date and evaluation value:" tmp = pd.concat([df_ann.loc[:, sec], df_value.loc[:, sec]], axis=1) tmp.columns = ['df_ann', 'df_value'] print tmp print "\n======Selection of result of expansion:" print "20161028 {:.4f}".format(df_res.loc[20161028, sec]) print "20161031 {:.4f}".format(df_res.loc[20161031, sec]) print "20170427 {:.4f}".format(df_res.loc[20170427, sec]) """ assert abs(df_res.loc[20161028, sec] - 82172000000) < 1 assert abs(df_res.loc[20161031, sec] - 120928000000) < 1 assert abs(df_res.loc[20170427, sec] - 42360000000) < 1
def test_q_add_field(): dv = DataView() dv.load_dataview(folder_path=quarterly_path) nrows, ncols = dv.data_q.shape n_securities = len(dv.data_d.columns.levels[0]) ds = RemoteDataService() ds.init_from_config(data_config) dv.add_field('net_inc_other_ops', ds) """ dv.add_field('oper_rev', ds) dv.add_field('turnover', ds) """ assert dv.data_q.shape == (nrows, ncols + 1 * n_securities)
def do_analyze(): from jaqs.trade.analyze.analyze import TradeRecordEmptyError ta = ana.EventAnalyzer() ds = RemoteDataService() ds.init_from_config(data_config) try: ta.initialize(data_server_=ds, file_folder=result_dir_path) ta.do_analyze(result_dir=result_dir_path, selected_sec=[]) except TradeRecordEmptyError: pass
def test_add_formula_directly(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() secs = '600030.SH,000063.SZ,000001.SZ' props = {'start_date': 20160601, 'end_date': 20170601, 'symbol': secs, 'fields': 'open,close', 'freq': 1} dv.init_from_config(props, data_api=ds) dv.prepare_data() dv.add_formula("myfactor", 'close / open', is_quarterly=False) assert dv.data_d.shape == (281, 39)
def test_save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = {'start_date': 20170901, 'end_date': 20171129, 'universe': BENCHMARK, 'fields': 'close,volume,sw1', 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() dv.add_formula('ret', 'Return(close_adj, 20)', is_quarterly=False) dv.add_formula('rank_ret', 'Rank(ret)', is_quarterly=False) dv.save_dataview(folder_path=dataview_dir_path)
def test_save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = {'start_date': 20170201, 'end_date': 20171001, 'universe': '000300.SH', 'fields': ('float_mv,sw2,sw1'), 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'GroupQuantile(float_mv, sw1, 10)' dv.add_formula('gq30', factor_formula, is_quarterly=False) dv.save_dataview(folder_path=dataview_dir_path)
def test_q(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() secs = '600030.SH,000063.SZ,000001.SZ' props = {'start_date': 20160609, 'end_date': 20170601, 'symbol': secs, 'fields': ('open,close,' + 'pb,net_assets,' + 'total_oper_rev,oper_exp,' + 'cash_paid_invest,' + 'capital_stk,' + 'roe'), 'freq': 1} dv.init_from_config(props, data_api=ds) dv.prepare_data() dv.save_dataview(folder_path=quarterly_path)
def test_save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = {'start_date': 20170101, 'end_date': 20171001, 'universe': '000300.SH', 'fields': 'pe_ttm,net_profit_incl_min_int_inc', 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'Return(net_profit_incl_min_int_inc, 4)' factor_name = 'net_profit_growth' dv.add_formula(factor_name, factor_formula, is_quarterly=True) dv.save_dataview(folder_path=dataview_dir_path)
def run_strategy(): start_date = 20150501 end_date = 20171030 index = '399975.SZ' ds = RemoteDataService() ds.init_from_config(data_config) symbol_list = ds.query_index_member(index, start_date, start_date) # add the benchmark index to the last position of symbol_list symbol_list.append(index) props = {"symbol": ','.join(symbol_list), "start_date": start_date, "end_date": end_date, "bar_type": "1d", "init_balance": 1e7, "std multiplier": 1.5, "m": 10, "n": 60, "commission_rate": 2E-4} props.update(data_config) props.update(trade_config) tapi = BacktestTradeApi() ins = EventBacktestInstance() strat = SectorRolling() pm = PortfolioManager() context = model.Context(data_api=ds, trade_api=tapi, instance=ins, strategy=strat, pm=pm) ins.init_from_config(props) ins.run() ins.save_results(folder_path=result_dir_path) ta = ana.EventAnalyzer() ta.initialize(data_server_=ds, file_folder=result_dir_path) df_bench, _ = ds.daily(index, start_date=start_date, end_date=end_date) ta.data_benchmark = df_bench.set_index('trade_date').loc[:, ['close']] ta.do_analyze(result_dir=result_dir_path, selected_sec=props['symbol'].split(',')[:2])
def test_save_dataview(sub_folder='test_dataview'): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = {'start_date': 20150101, 'end_date': 20170930, 'universe': '000905.SH', 'fields': ('float_mv,tot_shrhldr_eqy_excl_min_int,deferred_tax_assets,sw2'), 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'Quantile(-float_mv,5)' dv.add_formula('rank_mv', factor_formula, is_quarterly=False) factor_formula = 'Quantile(float_mv/(tot_shrhldr_eqy_excl_min_int+deferred_tax_assets), 5)' dv.add_formula('rank_pb', factor_formula, is_quarterly=False) dv.save_dataview(folder_path=dataview_dir_path)
def test_save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = { 'start_date': 20170101, 'end_date': 20171001, 'universe': '000300.SH', 'fields': 'pe_ttm,net_profit_incl_min_int_inc', 'freq': 1 } dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'Return(net_profit_incl_min_int_inc, 4)' factor_name = 'net_profit_growth' dv.add_formula(factor_name, factor_formula, is_quarterly=True) dv.save_dataview(folder_path=dataview_dir_path)
def save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = {'start_date': 20150101, 'end_date': 20171001, 'universe': '000300.SH', 'fields': 'volume,turnover,float_mv,pb,total_mv', 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() trade_status = dv.get_ts('trade_status') mask_sus = trade_status == '停牌' dv.append_df(mask_sus, 'suspended', is_quarterly=False) dv.add_formula('not_index_member', '!index_member', is_quarterly=False) dv.add_formula('limit_reached', 'Abs((open - Delay(close, 1)) / Delay(close, 1)) > 0.095', is_quarterly=False) dv.save_dataview(dataview_folder)
def download_data(): dataview_props = { 'start_date': 20120101, 'end_date': 20181231, 'universe': '000905.SH', # 'symbol':'600030.SH,600104.SH', 'fields': 'open,close,high,low,close_adj,volume', 'freq': 1 } ds = RemoteDataService() ds.init_from_config(data_config) # DataView utilizes RemoteDataService to get various data and store them dv = DataView() dv.init_from_config(dataview_props, ds) dv.prepare_data() factor_formula = 'Delay(Return(close_adj, 2, 0), -2)' dv.add_formula('future_return_2', factor_formula, is_quarterly=False, is_factor=False) factor_formula = 'Delay(Return(close_adj, 3, 0), -3)' dv.add_formula('future_return_3', factor_formula, is_quarterly=False, is_factor=False) factor_formula = 'Delay(Return(close_adj, 4, 0), -4)' dv.add_formula('future_return_4', factor_formula, is_quarterly=False, is_factor=False) factor_formula = 'Delay(Return(close_adj, 5, 0), -5)' dv.add_formula('future_return_5', factor_formula, is_quarterly=False, is_factor=False) dv.save_dataview(folder_path=dataview_store_folder)
def test_save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = {'start_date': 20170101, 'end_date': 20171001, 'universe': '000300.SH', 'fields': ('float_mv,pb,pe_ttm,sw2'), 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'GroupQuantile(-float_mv, sw2, 10)' dv.add_formula('rank_mv', factor_formula, is_quarterly=False) factor_formula = 'GroupQuantile(If(pb >= 0.2, pb, 100), sw2, 10)' dv.add_formula('rank_pb', factor_formula, is_quarterly=False) factor_formula = 'GroupQuantile(If(pe_ttm >= 3, pe_ttm, 9999.0), sw2, 10)' dv.add_formula('rank_pe', factor_formula, is_quarterly=False) dv.save_dataview(folder_path=dataview_dir_path)
def test_write(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() secs = '600030.SH,000063.SZ,000001.SZ' props = {'start_date': 20160601, 'end_date': 20170601, 'symbol': secs, 'fields': 'open,close,high,low,volume,pb,net_assets,pcf_ncf', 'freq': 1} dv.init_from_config(props, data_api=ds) dv.prepare_data() assert dv.data_d.shape == (281, 48) assert dv.dates.shape == (281, ) # TODO """ PerformanceWarning: your performance may suffer as PyTables will pickle object types that it cannot map directly to c-types [inferred_type->mixed,key->block1_values] [items->[('000001.SZ', 'int_income'), ('000001.SZ', 'less_handling_chrg_comm_exp'), ('000001.SZ', 'net_int_income'), ('000001.SZ', 'oper_exp'), ('000001.SZ', 'symbol'), ('000063.SZ', 'int_income'), ('000063.SZ', 'less_handling_chrg_comm_exp'), ('000063.SZ', 'net_int_income'), ('000063.SZ', 'oper_exp'), ('000063.SZ', 'symbol'), ('600030.SH', 'int_income'), ('600030.SH', 'less_handling_chrg_comm_exp'), ('600030.SH', 'net_int_income'), ('600030.SH', 'oper_exp'), ('600030.SH', 'symbol')]] """ dv.save_dataview(folder_path=daily_path)
def my_globals(request): ds = RemoteDataService() ds.init_from_config(data_config) df, msg = ds.daily("000001.SH, 600030.SH, 000300.SH", start_date=20170801, end_date=20170820, fields="open,high,low,close,vwap,preclose") multi_index_names = ['trade_date', 'symbol'] df_multi = df.set_index(multi_index_names, drop=False) df_multi.sort_index(axis=0, level=multi_index_names, inplace=True) dfx = df_multi.loc[pd.IndexSlice[:, :], pd.IndexSlice['close']].unstack() dfy = df_multi.loc[pd.IndexSlice[:, :], pd.IndexSlice['open']].unstack() parser = Parser() request.function.__globals__.update({ 'parser': parser, 'dfx': dfx, 'dfy': dfy })
def test_dataview_universe(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = { 'start_date': 20170227, 'end_date': 20170327, 'universe': '000016.SH', # 'symbol': 'rb1710.SHF,rb1801.SHF', 'fields': ('open,high,low,close,vwap,volume,turnover,' + 'roe,net_assets,' + 'total_oper_rev,oper_exp,tot_profit,int_income'), 'freq': 1 } dv.init_from_config(props, ds) dv.prepare_data()
def test_save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = { 'start_date': 20150101, 'end_date': 20170930, 'universe': '000905.SH', 'fields': ('tot_cur_assets,tot_cur_liab,inventories,pre_pay,deferred_exp,' 'eps_basic,ebit,pe,pb,float_mv,sw1'), 'freq': 1 } dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'pe < 30' dv.add_formula('pe_condition', factor_formula, is_quarterly=False) factor_formula = 'pb < 3' dv.add_formula('pb_condition', factor_formula, is_quarterly=False) factor_formula = 'Return(eps_basic, 4) > 0' dv.add_formula('eps_condition', factor_formula, is_quarterly=True) factor_formula = 'Return(ebit, 4) > 0' dv.add_formula('ebit_condition', factor_formula, is_quarterly=True) factor_formula = 'tot_cur_assets/tot_cur_liab > 2' dv.add_formula('current_condition', factor_formula, is_quarterly=True) factor_formula = '(tot_cur_assets - inventories - pre_pay - deferred_exp)/tot_cur_liab > 1' dv.add_formula('quick_condition', factor_formula, is_quarterly=True) dv.add_formula('mv_rank', 'Rank(float_mv)', is_quarterly=False) dv.save_dataview(folder_path=dataview_dir_path)
def test_calling(): ds = RemoteDataService() ds.init_from_config(data_config) df, msg = ds.data_api.daily(symbol="600832.SH, 600030.SH", start_date=20121026, end_date=20121130, fields=('close,pb,pe,turnover'), adjust_mode="post") print(df) symbol = "600832.SH, 600030.SH" start_date = 20121026 end_date = 20121130 filter_argument = ds._dic2url({ 'symbol': symbol, 'start_date': start_date, 'end_date': end_date }) res, err_msg = ds.query("lb.secDailyIndicator", fields=('float_mv'), filter=filter_argument, orderby="trade_date") print(res)
def test_calendar(): ds = RemoteDataService() ds.init_from_config(data_config) res1 = ds.query_trade_dates(20121224, 20130201) assert len(res1) == 27 day_zero = 20170102 res2 = ds.query_next_trade_date(day_zero) assert res2 == 20170103 res2_last = ds.query_last_trade_date(res2) assert res2_last == 20161230 res3 = ds.query_next_trade_date(20170104) assert res3 == 20170105 res4 = ds.query_last_trade_date(res3) assert res4 == 20170104 res11 = ds.query_trade_dates(20161224, 20170201) assert len(res11) == 23 assert not ds.is_trade_date(20150101) assert not ds.is_trade_date(20130501)
def save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = {'start_date': 20170101, 'end_date': 20171030, 'universe': '000300.SH', 'fields': ('open,high,low,close,vwap,volume,turnover,sw1,' # + 'pb,net_assets,' + 'eps_basic,total_mv,tot_profit,int_income' ), 'freq': 1} dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'close >= Delay(Ts_Max(close, 20), 1)' # 20 days new high factor_name = 'new_high' dv.add_formula(factor_name, factor_formula, is_quarterly=False) dv.add_formula('total_profit_growth', formula='Return(tot_profit, 4)', is_quarterly=True) dv.save_dataview(folder_path=dataview_dir_path)
def test_q(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() secs = '600030.SH,000063.SZ,000001.SZ' props = { 'start_date': 20160609, 'end_date': 20170601, 'symbol': secs, 'fields': ('open,close,' + 'pb,net_assets,' + 'total_oper_rev,oper_exp,' + 'cash_paid_invest,' + 'capital_stk,' + 'roe'), 'freq': 1 } dv.init_from_config(props, data_api=ds) dv.prepare_data() dv.save_dataview(folder_path=quarterly_path)
def test_calendar(): ds = RemoteDataService() ds.init_from_config(data_config) res1 = ds.query_trade_dates(20121224, 20130201) assert len(res1) == 27 day_zero = 20170102 res2 = ds.query_next_trade_date(day_zero) assert res2 == 20170103 res2_last = ds.query_last_trade_date(res2) assert res2_last == 20161230 res3 = ds.query_next_trade_date(20170104) assert res3 == 20170105 res4 = ds.query_last_trade_date(res3) assert res4 == 20170104 res11 = ds.query_trade_dates(20161224, 20170201) assert len(res11) == 23 assert not ds.is_trade_date(20150101) assert not ds.is_trade_date(20130501)
def test_save_dataview(sub_folder='test_dataview'): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() props = { 'start_date': 20150101, 'end_date': 20170930, 'universe': '000905.SH', 'fields': ('float_mv,tot_shrhldr_eqy_excl_min_int,deferred_tax_assets,sw2'), 'freq': 1 } dv.init_from_config(props, ds) dv.prepare_data() factor_formula = 'Quantile(-float_mv,5)' dv.add_formula('rank_mv', factor_formula, is_quarterly=False) factor_formula = 'Quantile(float_mv/(tot_shrhldr_eqy_excl_min_int+deferred_tax_assets), 5)' dv.add_formula('rank_pb', factor_formula, is_quarterly=False) dv.save_dataview(folder_path=dataview_dir_path)
def test_save_dataview(): ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() #dataview参数选择 props = { 'start_date': 20080527, 'end_date': 20180807, 'universe': '000002.SH,399107.SZ', "benchmark": "000905.SH,000905.SH", 'fields': ('open,close,volume,vwap,high,low,turnover'), 'freq': 1 } dv.init_from_config(props, ds) dv.prepare_data() #因子 factor_formula = 'rank(volume)*(ts_sum(close, 5)/5)*(vwap-close)/(high-low)' dv.add_formula('alpha', factor_formula, is_quarterly=False, formula_func_name_style='lower') dv.save_dataview(folder_path=dataview_dir_path)
def save_data(): """ This function fetches data from remote server and stores them locally. Then we can use local data to do back-test. """ dataview_props = {# Start and end date of back-test 'start_date': 20170101, 'end_date': 20171030, # Investment universe and performance benchmark 'universe': UNIVERSE, 'benchmark': '000300.SH', # Data fields that we need 'fields': 'total_mv,turnover', # freq = 1 means we use daily data. Please do not change this. 'freq': 1} # RemoteDataService communicates with a remote server to fetch data ds = RemoteDataService() # Use username and password in data_config to login ds.init_from_config(data_config) # DataView utilizes RemoteDataService to get various data and store them dv = DataView() dv.init_from_config(dataview_props, ds) dv.prepare_data() dv.save_dataview(folder_path=dataview_store_folder)
def test_align(): # ------------------------------------------------------------------------------------- # input and pre-process demo data ds = RemoteDataService() ds.init_from_config(data_config) raw, msg = ds.query_lb_fin_stat('income', '000001.SZ,600000.SH,601328.SH,601988.SH', 20160505, 20170505, fields='oper_rev,oper_cost') #fp = '../output/test_align.csv' #raw = pd.read_csv(fp) idx_list = ['report_date', 'symbol'] raw_idx = raw.set_index(idx_list) raw_idx.sort_index(axis=0, level=idx_list, inplace=True) # ------------------------------------------------------------------------------------- # get DataFrames df_ann = raw_idx.loc[pd.IndexSlice[:, :], 'ann_date'] df_ann = df_ann.unstack(level=1) df_value = raw_idx.loc[pd.IndexSlice[:, :], 'oper_rev'] df_value = df_value.unstack(level=1) # ------------------------------------------------------------------------------------- # get data array and align # date_arr = ds.get_trade_date(20160325, 20170625) date_arr = np.array([20160325, 20160328, 20160329, 20160330, 20160331, 20160401, 20160405, 20160406, 20160407, 20160408, 20160411, 20160412, 20160413, 20160414, 20160415, 20160418, 20160419, 20160420, 20160421, 20160422, 20160425, 20160426, 20160427, 20160428, 20160429, 20160503, 20160504, 20160505, 20160506, 20160509, 20160510, 20160511, 20160512, 20160513, 20160516, 20160517, 20160518, 20160519, 20160520, 20160523, 20160524, 20160525, 20160526, 20160527, 20160530, 20160531, 20160601, 20160602, 20160603, 20160606, 20160607, 20160608, 20160613, 20160614, 20160615, 20160616, 20160617, 20160620, 20160621, 20160622, 20160623, 20160624, 20160627, 20160628, 20160629, 20160630, 20160701, 20160704, 20160705, 20160706, 20160707, 20160708, 20160711, 20160712, 20160713, 20160714, 20160715, 20160718, 20160719, 20160720, 20160721, 20160722, 20160725, 20160726, 20160727, 20160728, 20160729, 20160801, 20160802, 20160803, 20160804, 20160805, 20160808, 20160809, 20160810, 20160811, 20160812, 20160815, 20160816, 20160817, 20160818, 20160819, 20160822, 20160823, 20160824, 20160825, 20160826, 20160829, 20160830, 20160831, 20160901, 20160902, 20160905, 20160906, 20160907, 20160908, 20160909, 20160912, 20160913, 20160914, 20160919, 20160920, 20160921, 20160922, 20160923, 20160926, 20160927, 20160928, 20160929, 20160930, 20161010, 20161011, 20161012, 20161013, 20161014, 20161017, 20161018, 20161019, 20161020, 20161021, 20161024, 20161025, 20161026, 20161027, 20161028, 20161031, 20161101, 20161102, 20161103, 20161104, 20161107, 20161108, 20161109, 20161110, 20161111, 20161114, 20161115, 20161116, 20161117, 20161118, 20161121, 20161122, 20161123, 20161124, 20161125, 20161128, 20161129, 20161130, 20161201, 20161202, 20161205, 20161206, 20161207, 20161208, 20161209, 20161212, 20161213, 20161214, 20161215, 20161216, 20161219, 20161220, 20161221, 20161222, 20161223, 20161226, 20161227, 20161228, 20161229, 20161230, 20170103, 20170104, 20170105, 20170106, 20170109, 20170110, 20170111, 20170112, 20170113, 20170116, 20170117, 20170118, 20170119, 20170120, 20170123, 20170124, 20170125, 20170126, 20170203, 20170206, 20170207, 20170208, 20170209, 20170210, 20170213, 20170214, 20170215, 20170216, 20170217, 20170220, 20170221, 20170222, 20170223, 20170224, 20170227, 20170228, 20170301, 20170302, 20170303, 20170306, 20170307, 20170308, 20170309, 20170310, 20170313, 20170314, 20170315, 20170316, 20170317, 20170320, 20170321, 20170322, 20170323, 20170324, 20170327, 20170328, 20170329, 20170330, 20170331, 20170405, 20170406, 20170407, 20170410, 20170411, 20170412, 20170413, 20170414, 20170417, 20170418, 20170419, 20170420, 20170421, 20170424, 20170425, 20170426, 20170427, 20170428, 20170502, 20170503, 20170504, 20170505, 20170508, 20170509, 20170510, 20170511, 20170512, 20170515, 20170516, 20170517, 20170518, 20170519, 20170522, 20170523, 20170524, 20170525, 20170526, 20170531, 20170601, 20170602, 20170605, 20170606, 20170607, 20170608, 20170609, 20170612, 20170613, 20170614, 20170615, 20170616, 20170619, 20170620, 20170621, 20170622, 20170623]) # df_res = align(df_ann, df_evaluate, date_arr) res_align = align(df_value, df_ann, date_arr) for symbol, ser_value in df_value.iteritems(): ser_ann = df_ann[symbol] ann_date_last = 0 assert res_align.loc[: ser_ann.iat[0]-1, symbol].isnull().all() for i in range(len(ser_value)): value = ser_value.iat[i] ann_date = ser_ann.iat[i] if i+1 >= len(ser_value): ann_date_next = 99999999 else: ann_date_next = ser_ann.iat[i+1] assert (res_align.loc[ann_date: ann_date_next-1, symbol] == value).all()
def test_align(): # ------------------------------------------------------------------------------------- # input and pre-process demo data ds = RemoteDataService() ds.init_from_config(data_config) raw, msg = ds.query_lb_fin_stat('income', '000001.SZ,600000.SH,601328.SH,601988.SH', 20160505, 20170505, fields='oper_rev,oper_cost') #fp = '../output/test_align.csv' #raw = pd.read_csv(fp) idx_list = ['report_date', 'symbol'] raw_idx = raw.set_index(idx_list) raw_idx.sort_index(axis=0, level=idx_list, inplace=True) # ------------------------------------------------------------------------------------- # get DataFrames df_ann = raw_idx.loc[pd.IndexSlice[:, :], 'ann_date'] df_ann = df_ann.unstack(level=1) df_value = raw_idx.loc[pd.IndexSlice[:, :], 'oper_rev'] df_value = df_value.unstack(level=1) # ------------------------------------------------------------------------------------- # get data array and align # date_arr = ds.get_trade_date(20160325, 20170625) date_arr = np.array([ 20160325, 20160328, 20160329, 20160330, 20160331, 20160401, 20160405, 20160406, 20160407, 20160408, 20160411, 20160412, 20160413, 20160414, 20160415, 20160418, 20160419, 20160420, 20160421, 20160422, 20160425, 20160426, 20160427, 20160428, 20160429, 20160503, 20160504, 20160505, 20160506, 20160509, 20160510, 20160511, 20160512, 20160513, 20160516, 20160517, 20160518, 20160519, 20160520, 20160523, 20160524, 20160525, 20160526, 20160527, 20160530, 20160531, 20160601, 20160602, 20160603, 20160606, 20160607, 20160608, 20160613, 20160614, 20160615, 20160616, 20160617, 20160620, 20160621, 20160622, 20160623, 20160624, 20160627, 20160628, 20160629, 20160630, 20160701, 20160704, 20160705, 20160706, 20160707, 20160708, 20160711, 20160712, 20160713, 20160714, 20160715, 20160718, 20160719, 20160720, 20160721, 20160722, 20160725, 20160726, 20160727, 20160728, 20160729, 20160801, 20160802, 20160803, 20160804, 20160805, 20160808, 20160809, 20160810, 20160811, 20160812, 20160815, 20160816, 20160817, 20160818, 20160819, 20160822, 20160823, 20160824, 20160825, 20160826, 20160829, 20160830, 20160831, 20160901, 20160902, 20160905, 20160906, 20160907, 20160908, 20160909, 20160912, 20160913, 20160914, 20160919, 20160920, 20160921, 20160922, 20160923, 20160926, 20160927, 20160928, 20160929, 20160930, 20161010, 20161011, 20161012, 20161013, 20161014, 20161017, 20161018, 20161019, 20161020, 20161021, 20161024, 20161025, 20161026, 20161027, 20161028, 20161031, 20161101, 20161102, 20161103, 20161104, 20161107, 20161108, 20161109, 20161110, 20161111, 20161114, 20161115, 20161116, 20161117, 20161118, 20161121, 20161122, 20161123, 20161124, 20161125, 20161128, 20161129, 20161130, 20161201, 20161202, 20161205, 20161206, 20161207, 20161208, 20161209, 20161212, 20161213, 20161214, 20161215, 20161216, 20161219, 20161220, 20161221, 20161222, 20161223, 20161226, 20161227, 20161228, 20161229, 20161230, 20170103, 20170104, 20170105, 20170106, 20170109, 20170110, 20170111, 20170112, 20170113, 20170116, 20170117, 20170118, 20170119, 20170120, 20170123, 20170124, 20170125, 20170126, 20170203, 20170206, 20170207, 20170208, 20170209, 20170210, 20170213, 20170214, 20170215, 20170216, 20170217, 20170220, 20170221, 20170222, 20170223, 20170224, 20170227, 20170228, 20170301, 20170302, 20170303, 20170306, 20170307, 20170308, 20170309, 20170310, 20170313, 20170314, 20170315, 20170316, 20170317, 20170320, 20170321, 20170322, 20170323, 20170324, 20170327, 20170328, 20170329, 20170330, 20170331, 20170405, 20170406, 20170407, 20170410, 20170411, 20170412, 20170413, 20170414, 20170417, 20170418, 20170419, 20170420, 20170421, 20170424, 20170425, 20170426, 20170427, 20170428, 20170502, 20170503, 20170504, 20170505, 20170508, 20170509, 20170510, 20170511, 20170512, 20170515, 20170516, 20170517, 20170518, 20170519, 20170522, 20170523, 20170524, 20170525, 20170526, 20170531, 20170601, 20170602, 20170605, 20170606, 20170607, 20170608, 20170609, 20170612, 20170613, 20170614, 20170615, 20170616, 20170619, 20170620, 20170621, 20170622, 20170623 ]) # df_res = align(df_ann, df_evaluate, date_arr) res_align = align(df_value, df_ann, date_arr) for symbol, ser_value in df_value.iteritems(): ser_ann = df_ann[symbol] ann_date_last = 0 assert res_align.loc[:ser_ann.iat[0] - 1, symbol].isnull().all() for i in range(len(ser_value)): value = ser_value.iat[i] ann_date = ser_ann.iat[i] if i + 1 >= len(ser_value): ann_date_next = 99999999 else: ann_date_next = ser_ann.iat[i + 1] assert (res_align.loc[ann_date:ann_date_next - 1, symbol] == value).all()
def my_globals(request): ds = RemoteDataService() ds.init_from_config(data_config) request.function.__globals__.update({'ds': ds})
raise exc ''' @pytest.fixture(autouse=True) def my_globals(request): ds = RemoteDataService() ds.init_from_config(data_config) request.function.__globals__.update({'ds': ds}) if __name__ == "__main__": import time t_start = time.time() ds = RemoteDataService() ds.init_from_config(data_config) g = globals() g = {k: v for k, v in g.items() if k.startswith('test_') and callable(v)} for test_name, test_func in g.items(): print("\n==========\nTesting {:s}...".format(test_name)) test_func() print("Test Complete.") t3 = time.time() - t_start print("\n\n\nTime lapsed in total: {:.1f}".format(t3))
def run_strategy(): if is_backtest: """ 回测模式 """ ds = RemoteDataService() ds.init_from_config(data_config) symbol_list = ds.query_index_member(index, start_date, end_date) # symbol_list = ['600887.SH'] # symbol_list = sample(symbol_list, 20) print(symbol_list) # add the benchmark index to the last position of symbol_list symbol_list.append(index) props = {"symbol": ','.join(symbol_list), "holding_Count": 15, "start_date": start_date, "end_date": end_date, "bar_type": "1d", # '1d' "init_balance": 300000, "commission_rate": 2E-4} tapi = BacktestTradeApi() ins = EventBacktestInstance() else: """ 实盘/仿真模式 """ props = {'symbol': '600519.SH', "fast_ma_length": 5, "slow_ma_length": 15, 'strategy.no': 1062} tapi = RealTimeTradeApi(trade_config) ins = EventLiveTradeInstance() props.update(data_config) props.update(trade_config) ds = RemoteDataService() strat = RNNStrategy() pm = PortfolioManager() context = model.Context(data_api=ds, trade_api=tapi, instance=ins, strategy=strat, pm=pm) ins.init_from_config(props) if not is_backtest: ds.subscribe(props['symbol']) ins.run() if not is_backtest: time.sleep(9999) ins.save_results(folder_path=result_dir_path) ta = ana.EventAnalyzer() ds = RemoteDataService() ds.init_from_config(data_config) ta.initialize(data_server_=ds, file_folder=result_dir_path) df_bench, _ = ds.daily(index, start_date=start_date, end_date=end_date) ta.data_benchmark = df_bench.set_index('trade_date').loc[:, ['close']] temp = pd.read_csv(result_dir_path + '/trades.csv') symbols = set(temp['symbol'].unique()) print(symbols) ta.do_analyze(result_dir=result_dir_path, selected_sec=list(symbols))
from jaqs.data import RemoteDataService from jaqs.data import DataView import jaqs.util as jutil from config_path import DATA_CONFIG_PATH, TRADE_CONFIG_PATH data_config = jutil.read_json(DATA_CONFIG_PATH) trade_config = jutil.read_json(TRADE_CONFIG_PATH) #dataview_dir_path = '../../output/test_dataview/dataview' props = { 'start_date': 20170101, 'end_date': 20180516, 'universe': '000905.SH', 'fields': "", 'freq': 1 } ds = RemoteDataService() ds.init_from_config(data_config) dv = DataView() dv.init_from_config(props, ds) #dv.prepare_data() # factor_formula = 'Quantile(-float_mv,5)' # dv.add_formula('rank_mv', factor_formula, is_quarterly=False) # factor_formula = 'Quantile(float_mv/(tot_shrhldr_eqy_excl_min_int+deferred_tax_assets), 5)' # dv.add_formula('rank_pb', factor_formula, is_quarterly=False) # dv.save_dataview(folder_path=dataview_dir_path)
#'roe_pb'表示算法的新名称,'roe/pb'为公式,is_quarterly=False代表是否为季度数据 print(dv.get_ts('roe_pb').head()) #这里用get_ts的方法输入新的名称即可 #5_从数据服务添加新数据至本地 #先设置Config data_config = { "remote.data.address": "tcp://data.tushare.org:8910", #地址统一,暂不做修改 "remote.data.username": "******", #quantos账号(手机号码) #quantos账号的API令牌号码 "remote.data.password": "******" } ds = RemoteDataService() #DataService启动 ds.init_from_config(data_config) #data_config启动 dv.add_field('eps_basic', ds) #添加新数据(eps_basic)至本地(ds) print(dv.get_ts('eps_basic').head()) A = dv.get_ts('eps_basic').head() #get_ts为数据获取 dv.remove_field('eps_basic') #删除数据(eps_basic) dv.add_field('volume', ds) #添加新数据至本地(ds) A = dv.get_ts('volume').head() dv.save_dataview('G:/data/hs300') #保存 dv.save_dataview('G:/data/hs300_1') #这相当于另存为 print(dv.fields) #查看dv中取得的数据 dv.add_field('roe', ds) #添加新数据至本地(ds)
def my_globals(request): ds = RemoteDataService() ds.init_from_config(data_config) request.function.__globals__.update({'ds': ds})