def save_dataview():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
props = {'start_date': 20160101, 'end_date': 20171001, 'universe': '000300.SH',
'fields': 'volume,turnover',
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
# for convenience to check limit reachers
dv.add_formula('limit_reached', 'Abs((open - Delay(close, 1)) / Delay(close, 1)) > 0.095', is_quarterly=False)
dv.add_formula('mask_limit_reached', 'limit_reached > 0', is_quarterly=False)
dv.add_formula('mask_index_member', '!(index_member > 0)', is_quarterly=False)
trade_status = dv.get_ts('trade_status')
mask_sus = trade_status == u'停牌'
dv.append_df(mask_sus, 'mask_sus', is_quarterly=False)
# dv.add_formula('size', '', is_quarterly=False)
dv.save_dataview(dataview_folder)
def prepare_dataview():
ds = RemoteDataService()
ds.init_from_config(data_config)
symbols = ['600036.SH', '000001.SZ']
dv_props = {'symbol': ','.join(symbols),
'start_date': backtest_props['start_date'],
'end_date': backtest_props['end_date'],
'benchmark': '000300.SH'}
dv = EventDataView()
dv.init_from_config(dv_props, ds)
dv.prepare_data()
import pandas as pd
# target security
diff_cols = ['open', 'high', 'low', 'close']
df0 = dv.get_symbol(symbols[0], fields=','.join(diff_cols))
df1 = dv.get_symbol(symbols[1], fields=','.join(diff_cols))
df_diff = df0 - df1
# calculate signal
df_signal = pd.concat([df0[['close']], df1[['close']], df_diff[['close']]], axis=1)
df_signal.columns = symbols + ['diff']
roll = df_signal['diff'].rolling(window=10)
df_signal.loc[:, 'signal'] = (df_signal['diff'] - roll.mean()) / roll.std()
dv.append_df_symbol(df_diff, '001.JZ')
dv.data_custom = df_signal
dv.save_dataview(dataview_dir_path)
def save_data():
"""
This function fetches data from remote server and stores them locally.
Then we can use local data to do back-test.
"""
dataview_props = {# Start and end date of back-test
'start_date': 20170101, 'end_date': 20171030,
# Investment universe and performance benchmark
'universe': UNIVERSE, 'benchmark': '000300.SH',
# Data fields that we need
'fields': 'total_mv,turnover',
# freq = 1 means we use daily data. Please do not change this.
'freq': 1}
# RemoteDataService communicates with a remote server to fetch data
ds = RemoteDataService()
# Use username and password in data_config to login
ds.init_from_config(data_config)
# DataView utilizes RemoteDataService to get various data and store them
dv = DataView()
dv.init_from_config(dataview_props, ds)
dv.prepare_data()
dv.save_dataview(folder_path=dataview_store_folder)
def save_dataview():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
props = {'start_date': 20150101, 'end_date': 20170930, 'universe': '000905.SH',
'fields': ('turnover,float_mv,close_adj,pe,pb'),
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
factor_formula = 'Cutoff(Standardize(turnover / 10000 / float_mv), 2)'
dv.add_formula('TO', factor_formula, is_quarterly=False)
factor_formula = 'Cutoff(Standardize(1/pb), 2)'
dv.add_formula('BP', factor_formula, is_quarterly=False)
factor_formula = 'Cutoff(Standardize(Return(close_adj, 20)), 2)'
dv.add_formula('REVS20', factor_formula, is_quarterly=False)
factor_formula = 'Cutoff(Standardize(Log(float_mv)), 2)'
dv.add_formula('float_mv_factor', factor_formula, is_quarterly=False)
factor_formula = 'Delay(Return(close_adj, 1), -1)'
dv.add_formula('NextRet', factor_formula, is_quarterly=False)
dv.save_dataview(folder_path=dataview_dir_path)
def test_save_dataview():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
props = {'start_date': 20150101, 'end_date': 20170930, 'universe': '000905.SH',
'fields': ('tot_cur_assets,tot_cur_liab,inventories,pre_pay,deferred_exp,'
'eps_basic,ebit,pe,pb,float_mv,sw1'),
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
factor_formula = 'pe < 30'
dv.add_formula('pe_condition', factor_formula, is_quarterly=False)
factor_formula = 'pb < 3'
dv.add_formula('pb_condition', factor_formula, is_quarterly=False)
factor_formula = 'Return(eps_basic, 4) > 0'
dv.add_formula('eps_condition', factor_formula, is_quarterly=True)
factor_formula = 'Return(ebit, 4) > 0'
dv.add_formula('ebit_condition', factor_formula, is_quarterly=True)
factor_formula = 'tot_cur_assets/tot_cur_liab > 2'
dv.add_formula('current_condition', factor_formula, is_quarterly=True)
factor_formula = '(tot_cur_assets - inventories - pre_pay - deferred_exp)/tot_cur_liab > 1'
dv.add_formula('quick_condition', factor_formula, is_quarterly=True)
dv.add_formula('mv_rank', 'Rank(float_mv)', is_quarterly=False)
dv.save_dataview(folder_path=dataview_dir_path)
def test_dataview_universe():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
props = {'start_date': 20170227, 'end_date': 20170327, 'universe': '000016.SH',
# 'symbol': 'rb1710.SHF,rb1801.SHF',
'fields': ('open,high,low,close,vwap,volume,turnover,'
+ 'sw1,zz2,'
+ 'roe,net_assets,'
+ 'total_oper_rev,oper_exp,tot_profit,int_income'
),
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
data_bench = dv.data_benchmark.copy()
dv.data_benchmark = data_bench
try:
dv.data_benchmark = data_bench.iloc[3:]
except ValueError:
pass
dv.remove_field('roe,net_assets')
dv.remove_field('close')
def analyze():
ta = ana.EventAnalyzer()
ds = RemoteDataService()
ds.init_from_config(data_config)
ta.initialize(data_server_=ds, file_folder=result_dir_path)
ta.do_analyze(result_dir=result_dir_path, selected_sec=props['symbol'].split(','))
def test_add_field():
dv = DataView()
dv.load_dataview(folder_path=daily_path)
nrows, ncols = dv.data_d.shape
n_securities = len(dv.data_d.columns.levels[0])
ds = RemoteDataService()
ds.init_from_config(data_config)
dv.add_field('total_share', ds)
assert dv.data_d.shape == (nrows, ncols + 1 * n_securities)
def test_bar():
from jaqs.data import RemoteDataService
from jaqs.trade.common import QUOTE_TYPE
ds = RemoteDataService()
ds.init_from_config(data_config)
df_quotes, msg = ds.bar(symbol='rb1710.SHF,hc1710.SHF', start_time=200000, end_time=160000,
trade_date=20170704, freq=QUOTE_TYPE.MIN)
bar_list = Bar.create_from_df(df_quotes)
bar = bar_list[0]
print(str(bar))
def test_inst_manager():
ds = RemoteDataService()
ds.init_from_config(data_config)
mgr = InstManager(ds)
mgr.load_instruments()
sym = '000001.SZ'
inst_obj = mgr.get_instrument(sym)
assert inst_obj.market == 'SZ'
assert inst_obj.symbol == sym
assert inst_obj.multiplier == 1
assert inst_obj.inst_type == 1
def test_write_future():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
secs = 'rb1710.SHF,j1710.DCE'
props = {'start_date': 20170401, 'end_date': 20170901, 'symbol': secs,
'fields': 'open,close,high,low,volume,oi',
'freq': 1, 'all_price': False}
dv.init_from_config(props, data_api=ds)
dv.prepare_data()
assert dv.data_d.shape == (145, 14)
def save_dataview():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
props = {'start_date': 20160101, 'end_date': 20171001, 'universe': '000300.SH',
'fields': 'volume,turnover',
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
dv.save_dataview(dataview_folder)
def test_align():
# -------------------------------------------------------------------------------------
# input and pre-process demo data
ds = RemoteDataService()
ds.init_from_config(data_config)
raw, msg = ds.query_lb_fin_stat('income', '600000.SH', 20151225, 20170501, 'oper_rev')
assert msg == '0,'
idx_list = ['report_date', 'symbol']
raw_idx = raw.set_index(idx_list)
raw_idx.sort_index(axis=0, level=idx_list, inplace=True)
df_ann = raw_idx.loc[pd.IndexSlice[:, :], 'ann_date']
df_ann = df_ann.unstack(level=1)
df_value = raw_idx.loc[pd.IndexSlice[:, :], 'oper_rev']
df_value = df_value.unstack(level=1)
date_arr = ds.query_trade_dates(20160101, 20170501)
df_close = pd.DataFrame(index=date_arr, columns=df_value.columns, data=1e3)
# -------------------------------------------------------------------------------------
# demo usage of parser
parser = Parser()
parser.register_function('Myfunc', lambda x: x * 0 + 1) # simultaneously test register function and align
expr_formula = 'signal / Myfunc(close)'
expression = parser.parse(expr_formula)
for i in range(100):
df_res = parser.evaluate({'signal': df_value, 'close': df_close}, df_ann, date_arr)
# -------------------------------------------------------------------------------------
sec = '600000.SH'
"""
# print to validate results
print "\n======Expression Formula:\n{:s}".format(expr_formula)
print "\n======Report date, ann_date and evaluation value:"
tmp = pd.concat([df_ann.loc[:, sec], df_value.loc[:, sec]], axis=1)
tmp.columns = ['df_ann', 'df_value']
print tmp
print "\n======Selection of result of expansion:"
print "20161028 {:.4f}".format(df_res.loc[20161028, sec])
print "20161031 {:.4f}".format(df_res.loc[20161031, sec])
print "20170427 {:.4f}".format(df_res.loc[20170427, sec])
"""
assert abs(df_res.loc[20161028, sec] - 82172000000) < 1
assert abs(df_res.loc[20161031, sec] - 120928000000) < 1
assert abs(df_res.loc[20170427, sec] - 42360000000) < 1
def test_q_add_field():
dv = DataView()
dv.load_dataview(folder_path=quarterly_path)
nrows, ncols = dv.data_q.shape
n_securities = len(dv.data_d.columns.levels[0])
ds = RemoteDataService()
ds.init_from_config(data_config)
dv.add_field('net_inc_other_ops', ds)
"""
dv.add_field('oper_rev', ds)
dv.add_field('turnover', ds)
"""
assert dv.data_q.shape == (nrows, ncols + 1 * n_securities)
def do_analyze():
from jaqs.trade.analyze.analyze import TradeRecordEmptyError
ta = ana.EventAnalyzer()
ds = RemoteDataService()
ds.init_from_config(data_config)
try:
ta.initialize(data_server_=ds, file_folder=result_dir_path)
ta.do_analyze(result_dir=result_dir_path, selected_sec=[])
except TradeRecordEmptyError:
pass
def test_add_formula_directly():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
secs = '600030.SH,000063.SZ,000001.SZ'
props = {'start_date': 20160601, 'end_date': 20170601, 'symbol': secs,
'fields': 'open,close',
'freq': 1}
dv.init_from_config(props, data_api=ds)
dv.prepare_data()
dv.add_formula("myfactor", 'close / open', is_quarterly=False)
assert dv.data_d.shape == (281, 39)
def test_save_dataview():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
props = {'start_date': 20170901, 'end_date': 20171129, 'universe': BENCHMARK,
'fields': 'close,volume,sw1',
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
dv.add_formula('ret', 'Return(close_adj, 20)', is_quarterly=False)
dv.add_formula('rank_ret', 'Rank(ret)', is_quarterly=False)
dv.save_dataview(folder_path=dataview_dir_path)
def test_save_dataview():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
props = {'start_date': 20170201, 'end_date': 20171001, 'universe': '000300.SH',
'fields': ('float_mv,sw2,sw1'),
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
factor_formula = 'GroupQuantile(float_mv, sw1, 10)'
dv.add_formula('gq30', factor_formula, is_quarterly=False)
dv.save_dataview(folder_path=dataview_dir_path)
def test_q():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
secs = '600030.SH,000063.SZ,000001.SZ'
props = {'start_date': 20160609, 'end_date': 20170601, 'symbol': secs,
'fields': ('open,close,'
+ 'pb,net_assets,'
+ 'total_oper_rev,oper_exp,'
+ 'cash_paid_invest,'
+ 'capital_stk,'
+ 'roe'), 'freq': 1}
dv.init_from_config(props, data_api=ds)
dv.prepare_data()
dv.save_dataview(folder_path=quarterly_path)
def test_save_dataview():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
props = {'start_date': 20170101, 'end_date': 20171001, 'universe': '000300.SH',
'fields': 'pe_ttm,net_profit_incl_min_int_inc',
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
factor_formula = 'Return(net_profit_incl_min_int_inc, 4)'
factor_name = 'net_profit_growth'
dv.add_formula(factor_name, factor_formula, is_quarterly=True)
dv.save_dataview(folder_path=dataview_dir_path)
def run_strategy():
start_date = 20150501
end_date = 20171030
index = '399975.SZ'
ds = RemoteDataService()
ds.init_from_config(data_config)
symbol_list = ds.query_index_member(index, start_date, start_date)
# add the benchmark index to the last position of symbol_list
symbol_list.append(index)
props = {"symbol": ','.join(symbol_list),
"start_date": start_date,
"end_date": end_date,
"bar_type": "1d",
"init_balance": 1e7,
"std multiplier": 1.5,
"m": 10,
"n": 60,
"commission_rate": 2E-4}
props.update(data_config)
props.update(trade_config)
tapi = BacktestTradeApi()
ins = EventBacktestInstance()
strat = SectorRolling()
pm = PortfolioManager()
context = model.Context(data_api=ds, trade_api=tapi, instance=ins,
strategy=strat, pm=pm)
ins.init_from_config(props)
ins.run()
ins.save_results(folder_path=result_dir_path)
ta = ana.EventAnalyzer()
ta.initialize(data_server_=ds, file_folder=result_dir_path)
df_bench, _ = ds.daily(index, start_date=start_date, end_date=end_date)
ta.data_benchmark = df_bench.set_index('trade_date').loc[:, ['close']]
ta.do_analyze(result_dir=result_dir_path, selected_sec=props['symbol'].split(',')[:2])
def test_save_dataview(sub_folder='test_dataview'):
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
props = {'start_date': 20150101, 'end_date': 20170930, 'universe': '000905.SH',
'fields': ('float_mv,tot_shrhldr_eqy_excl_min_int,deferred_tax_assets,sw2'),
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
factor_formula = 'Quantile(-float_mv,5)'
dv.add_formula('rank_mv', factor_formula, is_quarterly=False)
factor_formula = 'Quantile(float_mv/(tot_shrhldr_eqy_excl_min_int+deferred_tax_assets), 5)'
dv.add_formula('rank_pb', factor_formula, is_quarterly=False)
dv.save_dataview(folder_path=dataview_dir_path)
def test_save_dataview():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
props = {
'start_date': 20170101,
'end_date': 20171001,
'universe': '000300.SH',
'fields': 'pe_ttm,net_profit_incl_min_int_inc',
'freq': 1
}
dv.init_from_config(props, ds)
dv.prepare_data()
factor_formula = 'Return(net_profit_incl_min_int_inc, 4)'
factor_name = 'net_profit_growth'
dv.add_formula(factor_name, factor_formula, is_quarterly=True)
dv.save_dataview(folder_path=dataview_dir_path)
def save_dataview():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
props = {'start_date': 20150101, 'end_date': 20171001, 'universe': '000300.SH',
'fields': 'volume,turnover,float_mv,pb,total_mv',
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
trade_status = dv.get_ts('trade_status')
mask_sus = trade_status == '停牌'
dv.append_df(mask_sus, 'suspended', is_quarterly=False)
dv.add_formula('not_index_member', '!index_member', is_quarterly=False)
dv.add_formula('limit_reached', 'Abs((open - Delay(close, 1)) / Delay(close, 1)) > 0.095', is_quarterly=False)
dv.save_dataview(dataview_folder)
def download_data():
dataview_props = {
'start_date': 20120101,
'end_date': 20181231,
'universe': '000905.SH',
# 'symbol':'600030.SH,600104.SH',
'fields': 'open,close,high,low,close_adj,volume',
'freq': 1
}
ds = RemoteDataService()
ds.init_from_config(data_config)
# DataView utilizes RemoteDataService to get various data and store them
dv = DataView()
dv.init_from_config(dataview_props, ds)
dv.prepare_data()
factor_formula = 'Delay(Return(close_adj, 2, 0), -2)'
dv.add_formula('future_return_2',
factor_formula,
is_quarterly=False,
is_factor=False)
factor_formula = 'Delay(Return(close_adj, 3, 0), -3)'
dv.add_formula('future_return_3',
factor_formula,
is_quarterly=False,
is_factor=False)
factor_formula = 'Delay(Return(close_adj, 4, 0), -4)'
dv.add_formula('future_return_4',
factor_formula,
is_quarterly=False,
is_factor=False)
factor_formula = 'Delay(Return(close_adj, 5, 0), -5)'
dv.add_formula('future_return_5',
factor_formula,
is_quarterly=False,
is_factor=False)
dv.save_dataview(folder_path=dataview_store_folder)
def test_save_dataview():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
props = {'start_date': 20170101, 'end_date': 20171001, 'universe': '000300.SH',
'fields': ('float_mv,pb,pe_ttm,sw2'),
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
factor_formula = 'GroupQuantile(-float_mv, sw2, 10)'
dv.add_formula('rank_mv', factor_formula, is_quarterly=False)
factor_formula = 'GroupQuantile(If(pb >= 0.2, pb, 100), sw2, 10)'
dv.add_formula('rank_pb', factor_formula, is_quarterly=False)
factor_formula = 'GroupQuantile(If(pe_ttm >= 3, pe_ttm, 9999.0), sw2, 10)'
dv.add_formula('rank_pe', factor_formula, is_quarterly=False)
dv.save_dataview(folder_path=dataview_dir_path)
def test_write():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
secs = '600030.SH,000063.SZ,000001.SZ'
props = {'start_date': 20160601, 'end_date': 20170601, 'symbol': secs,
'fields': 'open,close,high,low,volume,pb,net_assets,pcf_ncf',
'freq': 1}
dv.init_from_config(props, data_api=ds)
dv.prepare_data()
assert dv.data_d.shape == (281, 48)
assert dv.dates.shape == (281, )
# TODO
"""
PerformanceWarning:
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->mixed,key->block1_values] [items->[('000001.SZ', 'int_income'), ('000001.SZ', 'less_handling_chrg_comm_exp'), ('000001.SZ', 'net_int_income'), ('000001.SZ', 'oper_exp'), ('000001.SZ', 'symbol'), ('000063.SZ', 'int_income'), ('000063.SZ', 'less_handling_chrg_comm_exp'), ('000063.SZ', 'net_int_income'), ('000063.SZ', 'oper_exp'), ('000063.SZ', 'symbol'), ('600030.SH', 'int_income'), ('600030.SH', 'less_handling_chrg_comm_exp'), ('600030.SH', 'net_int_income'), ('600030.SH', 'oper_exp'), ('600030.SH', 'symbol')]]
"""
dv.save_dataview(folder_path=daily_path)
def my_globals(request):
ds = RemoteDataService()
ds.init_from_config(data_config)
df, msg = ds.daily("000001.SH, 600030.SH, 000300.SH",
start_date=20170801,
end_date=20170820,
fields="open,high,low,close,vwap,preclose")
multi_index_names = ['trade_date', 'symbol']
df_multi = df.set_index(multi_index_names, drop=False)
df_multi.sort_index(axis=0, level=multi_index_names, inplace=True)
dfx = df_multi.loc[pd.IndexSlice[:, :], pd.IndexSlice['close']].unstack()
dfy = df_multi.loc[pd.IndexSlice[:, :], pd.IndexSlice['open']].unstack()
parser = Parser()
request.function.__globals__.update({
'parser': parser,
'dfx': dfx,
'dfy': dfy
})
def test_dataview_universe():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
props = {
'start_date':
20170227,
'end_date':
20170327,
'universe':
'000016.SH',
# 'symbol': 'rb1710.SHF,rb1801.SHF',
'fields':
('open,high,low,close,vwap,volume,turnover,' + 'roe,net_assets,' +
'total_oper_rev,oper_exp,tot_profit,int_income'),
'freq':
1
}
dv.init_from_config(props, ds)
dv.prepare_data()
def test_save_dataview():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
props = {
'start_date':
20150101,
'end_date':
20170930,
'universe':
'000905.SH',
'fields':
('tot_cur_assets,tot_cur_liab,inventories,pre_pay,deferred_exp,'
'eps_basic,ebit,pe,pb,float_mv,sw1'),
'freq':
1
}
dv.init_from_config(props, ds)
dv.prepare_data()
factor_formula = 'pe < 30'
dv.add_formula('pe_condition', factor_formula, is_quarterly=False)
factor_formula = 'pb < 3'
dv.add_formula('pb_condition', factor_formula, is_quarterly=False)
factor_formula = 'Return(eps_basic, 4) > 0'
dv.add_formula('eps_condition', factor_formula, is_quarterly=True)
factor_formula = 'Return(ebit, 4) > 0'
dv.add_formula('ebit_condition', factor_formula, is_quarterly=True)
factor_formula = 'tot_cur_assets/tot_cur_liab > 2'
dv.add_formula('current_condition', factor_formula, is_quarterly=True)
factor_formula = '(tot_cur_assets - inventories - pre_pay - deferred_exp)/tot_cur_liab > 1'
dv.add_formula('quick_condition', factor_formula, is_quarterly=True)
dv.add_formula('mv_rank', 'Rank(float_mv)', is_quarterly=False)
dv.save_dataview(folder_path=dataview_dir_path)
def test_calling():
ds = RemoteDataService()
ds.init_from_config(data_config)
df, msg = ds.data_api.daily(symbol="600832.SH, 600030.SH",
start_date=20121026,
end_date=20121130,
fields=('close,pb,pe,turnover'),
adjust_mode="post")
print(df)
symbol = "600832.SH, 600030.SH"
start_date = 20121026
end_date = 20121130
filter_argument = ds._dic2url({
'symbol': symbol,
'start_date': start_date,
'end_date': end_date
})
res, err_msg = ds.query("lb.secDailyIndicator",
fields=('float_mv'),
filter=filter_argument,
orderby="trade_date")
print(res)
def test_calendar():
ds = RemoteDataService()
ds.init_from_config(data_config)
res1 = ds.query_trade_dates(20121224, 20130201)
assert len(res1) == 27
day_zero = 20170102
res2 = ds.query_next_trade_date(day_zero)
assert res2 == 20170103
res2_last = ds.query_last_trade_date(res2)
assert res2_last == 20161230
res3 = ds.query_next_trade_date(20170104)
assert res3 == 20170105
res4 = ds.query_last_trade_date(res3)
assert res4 == 20170104
res11 = ds.query_trade_dates(20161224, 20170201)
assert len(res11) == 23
assert not ds.is_trade_date(20150101)
assert not ds.is_trade_date(20130501)
def save_dataview():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
props = {'start_date': 20170101, 'end_date': 20171030, 'universe': '000300.SH',
'fields': ('open,high,low,close,vwap,volume,turnover,sw1,'
# + 'pb,net_assets,'
+ 'eps_basic,total_mv,tot_profit,int_income'
),
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
factor_formula = 'close >= Delay(Ts_Max(close, 20), 1)' # 20 days new high
factor_name = 'new_high'
dv.add_formula(factor_name, factor_formula, is_quarterly=False)
dv.add_formula('total_profit_growth', formula='Return(tot_profit, 4)', is_quarterly=True)
dv.save_dataview(folder_path=dataview_dir_path)
def test_q():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
secs = '600030.SH,000063.SZ,000001.SZ'
props = {
'start_date':
20160609,
'end_date':
20170601,
'symbol':
secs,
'fields':
('open,close,' + 'pb,net_assets,' + 'total_oper_rev,oper_exp,' +
'cash_paid_invest,' + 'capital_stk,' + 'roe'),
'freq':
1
}
dv.init_from_config(props, data_api=ds)
dv.prepare_data()
dv.save_dataview(folder_path=quarterly_path)
def test_calendar():
ds = RemoteDataService()
ds.init_from_config(data_config)
res1 = ds.query_trade_dates(20121224, 20130201)
assert len(res1) == 27
day_zero = 20170102
res2 = ds.query_next_trade_date(day_zero)
assert res2 == 20170103
res2_last = ds.query_last_trade_date(res2)
assert res2_last == 20161230
res3 = ds.query_next_trade_date(20170104)
assert res3 == 20170105
res4 = ds.query_last_trade_date(res3)
assert res4 == 20170104
res11 = ds.query_trade_dates(20161224, 20170201)
assert len(res11) == 23
assert not ds.is_trade_date(20150101)
assert not ds.is_trade_date(20130501)
def test_save_dataview(sub_folder='test_dataview'):
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
props = {
'start_date': 20150101,
'end_date': 20170930,
'universe': '000905.SH',
'fields':
('float_mv,tot_shrhldr_eqy_excl_min_int,deferred_tax_assets,sw2'),
'freq': 1
}
dv.init_from_config(props, ds)
dv.prepare_data()
factor_formula = 'Quantile(-float_mv,5)'
dv.add_formula('rank_mv', factor_formula, is_quarterly=False)
factor_formula = 'Quantile(float_mv/(tot_shrhldr_eqy_excl_min_int+deferred_tax_assets), 5)'
dv.add_formula('rank_pb', factor_formula, is_quarterly=False)
dv.save_dataview(folder_path=dataview_dir_path)
def test_save_dataview():
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
#dataview参数选择
props = {
'start_date': 20080527,
'end_date': 20180807,
'universe': '000002.SH,399107.SZ',
"benchmark": "000905.SH,000905.SH",
'fields': ('open,close,volume,vwap,high,low,turnover'),
'freq': 1
}
dv.init_from_config(props, ds)
dv.prepare_data()
#因子
factor_formula = 'rank(volume)*(ts_sum(close, 5)/5)*(vwap-close)/(high-low)'
dv.add_formula('alpha',
factor_formula,
is_quarterly=False,
formula_func_name_style='lower')
dv.save_dataview(folder_path=dataview_dir_path)
def save_data():
"""
This function fetches data from remote server and stores them locally.
Then we can use local data to do back-test.
"""
dataview_props = {# Start and end date of back-test
'start_date': 20170101, 'end_date': 20171030,
# Investment universe and performance benchmark
'universe': UNIVERSE, 'benchmark': '000300.SH',
# Data fields that we need
'fields': 'total_mv,turnover',
# freq = 1 means we use daily data. Please do not change this.
'freq': 1}
# RemoteDataService communicates with a remote server to fetch data
ds = RemoteDataService()
# Use username and password in data_config to login
ds.init_from_config(data_config)
# DataView utilizes RemoteDataService to get various data and store them
dv = DataView()
dv.init_from_config(dataview_props, ds)
dv.prepare_data()
dv.save_dataview(folder_path=dataview_store_folder)
def test_align():
# -------------------------------------------------------------------------------------
# input and pre-process demo data
ds = RemoteDataService()
ds.init_from_config(data_config)
raw, msg = ds.query_lb_fin_stat('income', '000001.SZ,600000.SH,601328.SH,601988.SH',
20160505, 20170505, fields='oper_rev,oper_cost')
#fp = '../output/test_align.csv'
#raw = pd.read_csv(fp)
idx_list = ['report_date', 'symbol']
raw_idx = raw.set_index(idx_list)
raw_idx.sort_index(axis=0, level=idx_list, inplace=True)
# -------------------------------------------------------------------------------------
# get DataFrames
df_ann = raw_idx.loc[pd.IndexSlice[:, :], 'ann_date']
df_ann = df_ann.unstack(level=1)
df_value = raw_idx.loc[pd.IndexSlice[:, :], 'oper_rev']
df_value = df_value.unstack(level=1)
# -------------------------------------------------------------------------------------
# get data array and align
# date_arr = ds.get_trade_date(20160325, 20170625)
date_arr = np.array([20160325, 20160328, 20160329, 20160330, 20160331, 20160401, 20160405, 20160406,
20160407, 20160408, 20160411, 20160412, 20160413, 20160414, 20160415, 20160418,
20160419, 20160420, 20160421, 20160422, 20160425, 20160426, 20160427, 20160428,
20160429, 20160503, 20160504, 20160505, 20160506, 20160509, 20160510, 20160511,
20160512, 20160513, 20160516, 20160517, 20160518, 20160519, 20160520, 20160523,
20160524, 20160525, 20160526, 20160527, 20160530, 20160531, 20160601, 20160602,
20160603, 20160606, 20160607, 20160608, 20160613, 20160614, 20160615, 20160616,
20160617, 20160620, 20160621, 20160622, 20160623, 20160624, 20160627, 20160628,
20160629, 20160630, 20160701, 20160704, 20160705, 20160706, 20160707, 20160708,
20160711, 20160712, 20160713, 20160714, 20160715, 20160718, 20160719, 20160720,
20160721, 20160722, 20160725, 20160726, 20160727, 20160728, 20160729, 20160801,
20160802, 20160803, 20160804, 20160805, 20160808, 20160809, 20160810, 20160811,
20160812, 20160815, 20160816, 20160817, 20160818, 20160819, 20160822, 20160823,
20160824, 20160825, 20160826, 20160829, 20160830, 20160831, 20160901, 20160902,
20160905, 20160906, 20160907, 20160908, 20160909, 20160912, 20160913, 20160914,
20160919, 20160920, 20160921, 20160922, 20160923, 20160926, 20160927, 20160928,
20160929, 20160930, 20161010, 20161011, 20161012, 20161013, 20161014, 20161017,
20161018, 20161019, 20161020, 20161021, 20161024, 20161025, 20161026, 20161027,
20161028, 20161031, 20161101, 20161102, 20161103, 20161104, 20161107, 20161108,
20161109, 20161110, 20161111, 20161114, 20161115, 20161116, 20161117, 20161118,
20161121, 20161122, 20161123, 20161124, 20161125, 20161128, 20161129, 20161130,
20161201, 20161202, 20161205, 20161206, 20161207, 20161208, 20161209, 20161212,
20161213, 20161214, 20161215, 20161216, 20161219, 20161220, 20161221, 20161222,
20161223, 20161226, 20161227, 20161228, 20161229, 20161230, 20170103, 20170104,
20170105, 20170106, 20170109, 20170110, 20170111, 20170112, 20170113, 20170116,
20170117, 20170118, 20170119, 20170120, 20170123, 20170124, 20170125, 20170126,
20170203, 20170206, 20170207, 20170208, 20170209, 20170210, 20170213, 20170214,
20170215, 20170216, 20170217, 20170220, 20170221, 20170222, 20170223, 20170224,
20170227, 20170228, 20170301, 20170302, 20170303, 20170306, 20170307, 20170308,
20170309, 20170310, 20170313, 20170314, 20170315, 20170316, 20170317, 20170320,
20170321, 20170322, 20170323, 20170324, 20170327, 20170328, 20170329, 20170330,
20170331, 20170405, 20170406, 20170407, 20170410, 20170411, 20170412, 20170413,
20170414, 20170417, 20170418, 20170419, 20170420, 20170421, 20170424, 20170425,
20170426, 20170427, 20170428, 20170502, 20170503, 20170504, 20170505, 20170508,
20170509, 20170510, 20170511, 20170512, 20170515, 20170516, 20170517, 20170518,
20170519, 20170522, 20170523, 20170524, 20170525, 20170526, 20170531, 20170601,
20170602, 20170605, 20170606, 20170607, 20170608, 20170609, 20170612, 20170613,
20170614, 20170615, 20170616, 20170619, 20170620, 20170621, 20170622, 20170623])
# df_res = align(df_ann, df_evaluate, date_arr)
res_align = align(df_value, df_ann, date_arr)
for symbol, ser_value in df_value.iteritems():
ser_ann = df_ann[symbol]
ann_date_last = 0
assert res_align.loc[: ser_ann.iat[0]-1, symbol].isnull().all()
for i in range(len(ser_value)):
value = ser_value.iat[i]
ann_date = ser_ann.iat[i]
if i+1 >= len(ser_value):
ann_date_next = 99999999
else:
ann_date_next = ser_ann.iat[i+1]
assert (res_align.loc[ann_date: ann_date_next-1, symbol] == value).all()
def test_align():
# -------------------------------------------------------------------------------------
# input and pre-process demo data
ds = RemoteDataService()
ds.init_from_config(data_config)
raw, msg = ds.query_lb_fin_stat('income',
'000001.SZ,600000.SH,601328.SH,601988.SH',
20160505,
20170505,
fields='oper_rev,oper_cost')
#fp = '../output/test_align.csv'
#raw = pd.read_csv(fp)
idx_list = ['report_date', 'symbol']
raw_idx = raw.set_index(idx_list)
raw_idx.sort_index(axis=0, level=idx_list, inplace=True)
# -------------------------------------------------------------------------------------
# get DataFrames
df_ann = raw_idx.loc[pd.IndexSlice[:, :], 'ann_date']
df_ann = df_ann.unstack(level=1)
df_value = raw_idx.loc[pd.IndexSlice[:, :], 'oper_rev']
df_value = df_value.unstack(level=1)
# -------------------------------------------------------------------------------------
# get data array and align
# date_arr = ds.get_trade_date(20160325, 20170625)
date_arr = np.array([
20160325, 20160328, 20160329, 20160330, 20160331, 20160401, 20160405,
20160406, 20160407, 20160408, 20160411, 20160412, 20160413, 20160414,
20160415, 20160418, 20160419, 20160420, 20160421, 20160422, 20160425,
20160426, 20160427, 20160428, 20160429, 20160503, 20160504, 20160505,
20160506, 20160509, 20160510, 20160511, 20160512, 20160513, 20160516,
20160517, 20160518, 20160519, 20160520, 20160523, 20160524, 20160525,
20160526, 20160527, 20160530, 20160531, 20160601, 20160602, 20160603,
20160606, 20160607, 20160608, 20160613, 20160614, 20160615, 20160616,
20160617, 20160620, 20160621, 20160622, 20160623, 20160624, 20160627,
20160628, 20160629, 20160630, 20160701, 20160704, 20160705, 20160706,
20160707, 20160708, 20160711, 20160712, 20160713, 20160714, 20160715,
20160718, 20160719, 20160720, 20160721, 20160722, 20160725, 20160726,
20160727, 20160728, 20160729, 20160801, 20160802, 20160803, 20160804,
20160805, 20160808, 20160809, 20160810, 20160811, 20160812, 20160815,
20160816, 20160817, 20160818, 20160819, 20160822, 20160823, 20160824,
20160825, 20160826, 20160829, 20160830, 20160831, 20160901, 20160902,
20160905, 20160906, 20160907, 20160908, 20160909, 20160912, 20160913,
20160914, 20160919, 20160920, 20160921, 20160922, 20160923, 20160926,
20160927, 20160928, 20160929, 20160930, 20161010, 20161011, 20161012,
20161013, 20161014, 20161017, 20161018, 20161019, 20161020, 20161021,
20161024, 20161025, 20161026, 20161027, 20161028, 20161031, 20161101,
20161102, 20161103, 20161104, 20161107, 20161108, 20161109, 20161110,
20161111, 20161114, 20161115, 20161116, 20161117, 20161118, 20161121,
20161122, 20161123, 20161124, 20161125, 20161128, 20161129, 20161130,
20161201, 20161202, 20161205, 20161206, 20161207, 20161208, 20161209,
20161212, 20161213, 20161214, 20161215, 20161216, 20161219, 20161220,
20161221, 20161222, 20161223, 20161226, 20161227, 20161228, 20161229,
20161230, 20170103, 20170104, 20170105, 20170106, 20170109, 20170110,
20170111, 20170112, 20170113, 20170116, 20170117, 20170118, 20170119,
20170120, 20170123, 20170124, 20170125, 20170126, 20170203, 20170206,
20170207, 20170208, 20170209, 20170210, 20170213, 20170214, 20170215,
20170216, 20170217, 20170220, 20170221, 20170222, 20170223, 20170224,
20170227, 20170228, 20170301, 20170302, 20170303, 20170306, 20170307,
20170308, 20170309, 20170310, 20170313, 20170314, 20170315, 20170316,
20170317, 20170320, 20170321, 20170322, 20170323, 20170324, 20170327,
20170328, 20170329, 20170330, 20170331, 20170405, 20170406, 20170407,
20170410, 20170411, 20170412, 20170413, 20170414, 20170417, 20170418,
20170419, 20170420, 20170421, 20170424, 20170425, 20170426, 20170427,
20170428, 20170502, 20170503, 20170504, 20170505, 20170508, 20170509,
20170510, 20170511, 20170512, 20170515, 20170516, 20170517, 20170518,
20170519, 20170522, 20170523, 20170524, 20170525, 20170526, 20170531,
20170601, 20170602, 20170605, 20170606, 20170607, 20170608, 20170609,
20170612, 20170613, 20170614, 20170615, 20170616, 20170619, 20170620,
20170621, 20170622, 20170623
])
# df_res = align(df_ann, df_evaluate, date_arr)
res_align = align(df_value, df_ann, date_arr)
for symbol, ser_value in df_value.iteritems():
ser_ann = df_ann[symbol]
ann_date_last = 0
assert res_align.loc[:ser_ann.iat[0] - 1, symbol].isnull().all()
for i in range(len(ser_value)):
value = ser_value.iat[i]
ann_date = ser_ann.iat[i]
if i + 1 >= len(ser_value):
ann_date_next = 99999999
else:
ann_date_next = ser_ann.iat[i + 1]
assert (res_align.loc[ann_date:ann_date_next - 1,
symbol] == value).all()
def my_globals(request):
ds = RemoteDataService()
ds.init_from_config(data_config)
request.function.__globals__.update({'ds': ds})
raise exc
'''
@pytest.fixture(autouse=True)
def my_globals(request):
ds = RemoteDataService()
ds.init_from_config(data_config)
request.function.__globals__.update({'ds': ds})
if __name__ == "__main__":
import time
t_start = time.time()
ds = RemoteDataService()
ds.init_from_config(data_config)
g = globals()
g = {k: v for k, v in g.items() if k.startswith('test_') and callable(v)}
for test_name, test_func in g.items():
print("\n==========\nTesting {:s}...".format(test_name))
test_func()
print("Test Complete.")
t3 = time.time() - t_start
print("\n\n\nTime lapsed in total: {:.1f}".format(t3))
def run_strategy():
if is_backtest:
"""
回测模式
"""
ds = RemoteDataService()
ds.init_from_config(data_config)
symbol_list = ds.query_index_member(index, start_date, end_date)
# symbol_list = ['600887.SH']
# symbol_list = sample(symbol_list, 20)
print(symbol_list)
# add the benchmark index to the last position of symbol_list
symbol_list.append(index)
props = {"symbol": ','.join(symbol_list),
"holding_Count": 15,
"start_date": start_date,
"end_date": end_date,
"bar_type": "1d", # '1d'
"init_balance": 300000,
"commission_rate": 2E-4}
tapi = BacktestTradeApi()
ins = EventBacktestInstance()
else:
"""
实盘/仿真模式
"""
props = {'symbol': '600519.SH',
"fast_ma_length": 5,
"slow_ma_length": 15,
'strategy.no': 1062}
tapi = RealTimeTradeApi(trade_config)
ins = EventLiveTradeInstance()
props.update(data_config)
props.update(trade_config)
ds = RemoteDataService()
strat = RNNStrategy()
pm = PortfolioManager()
context = model.Context(data_api=ds, trade_api=tapi, instance=ins,
strategy=strat, pm=pm)
ins.init_from_config(props)
if not is_backtest:
ds.subscribe(props['symbol'])
ins.run()
if not is_backtest:
time.sleep(9999)
ins.save_results(folder_path=result_dir_path)
ta = ana.EventAnalyzer()
ds = RemoteDataService()
ds.init_from_config(data_config)
ta.initialize(data_server_=ds, file_folder=result_dir_path)
df_bench, _ = ds.daily(index, start_date=start_date, end_date=end_date)
ta.data_benchmark = df_bench.set_index('trade_date').loc[:, ['close']]
temp = pd.read_csv(result_dir_path + '/trades.csv')
symbols = set(temp['symbol'].unique())
print(symbols)
ta.do_analyze(result_dir=result_dir_path, selected_sec=list(symbols))
from jaqs.data import RemoteDataService
from jaqs.data import DataView
import jaqs.util as jutil
from config_path import DATA_CONFIG_PATH, TRADE_CONFIG_PATH
data_config = jutil.read_json(DATA_CONFIG_PATH)
trade_config = jutil.read_json(TRADE_CONFIG_PATH)
#dataview_dir_path = '../../output/test_dataview/dataview'
props = {
'start_date': 20170101,
'end_date': 20180516,
'universe': '000905.SH',
'fields': "",
'freq': 1
}
ds = RemoteDataService()
ds.init_from_config(data_config)
dv = DataView()
dv.init_from_config(props, ds)
#dv.prepare_data()
# factor_formula = 'Quantile(-float_mv,5)'
# dv.add_formula('rank_mv', factor_formula, is_quarterly=False)
# factor_formula = 'Quantile(float_mv/(tot_shrhldr_eqy_excl_min_int+deferred_tax_assets), 5)'
# dv.add_formula('rank_pb', factor_formula, is_quarterly=False)
# dv.save_dataview(folder_path=dataview_dir_path)
#'roe_pb'表示算法的新名称,'roe/pb'为公式,is_quarterly=False代表是否为季度数据
print(dv.get_ts('roe_pb').head()) #这里用get_ts的方法输入新的名称即可
#5_从数据服务添加新数据至本地
#先设置Config
data_config = {
"remote.data.address":
"tcp://data.tushare.org:8910", #地址统一,暂不做修改
"remote.data.username":
"******", #quantos账号(手机号码)
#quantos账号的API令牌号码
"remote.data.password":
"******"
}
ds = RemoteDataService() #DataService启动
ds.init_from_config(data_config) #data_config启动
dv.add_field('eps_basic', ds) #添加新数据(eps_basic)至本地(ds)
print(dv.get_ts('eps_basic').head())
A = dv.get_ts('eps_basic').head() #get_ts为数据获取
dv.remove_field('eps_basic') #删除数据(eps_basic)
dv.add_field('volume', ds) #添加新数据至本地(ds)
A = dv.get_ts('volume').head()
dv.save_dataview('G:/data/hs300') #保存
dv.save_dataview('G:/data/hs300_1') #这相当于另存为
print(dv.fields) #查看dv中取得的数据
dv.add_field('roe', ds) #添加新数据至本地(ds)
def my_globals(request):
ds = RemoteDataService()
ds.init_from_config(data_config)
request.function.__globals__.update({'ds': ds})
