def test_alpha_strategy_dataview():
dv = DataView()
fullpath = fileio.join_relative_path(
'../output/prepared/20141114_20170827_freq=1D')
dv.load_dataview(folder=fullpath)
props = {
"benchmark": "000300.SH",
# "symbol": ','.join(dv.symbol),
"universe": ','.join(dv.symbol),
"start_date": dv.start_date,
"end_date": dv.end_date,
"period": "month",
"days_delay": 0,
"init_balance": 1e9,
"position_ratio": 0.7,
}
gateway = DailyStockSimGateway()
gateway.init_from_config(props)
context = model.Context()
context.register_gateway(gateway)
context.register_trade_api(gateway)
context.register_dataview(dv)
risk_model = model.FactorRiskModel()
signal_model = model.FactorRevenueModel_dv()
cost_model = model.SimpleCostModel()
risk_model.register_context(context)
signal_model.register_context(context)
cost_model.register_context(context)
signal_model.register_func('gtja', gtja_factor_dataview)
signal_model.activate_func({'gtja': {}})
cost_model.register_func('my_commission', my_commission)
cost_model.activate_func({'my_commission': {'myrate': 1e-2}})
strategy = DemoAlphaStrategy(risk_model, signal_model, cost_model)
# strategy.active_pc_method = 'equal_weight'
# strategy.active_pc_method = 'mc'
strategy.active_pc_method = 'factor_value_weight'
bt = AlphaBacktestInstance_dv()
bt.init_from_config(props, strategy, context=context)
bt.run_alpha()
bt.save_results('../output/')
def test_add_formula_quarterly():
dv = DataView()
folder_path = '../output/prepared/20160609_20170601_freq=1D'
dv.load_dataview(folder=folder_path)
nrows, ncols = dv.data_d.shape
n_securities = len(dv.data_d.columns.levels[0])
formula = 'total_oper_rev / close'
dv.add_formula('myvar1', formula, is_quarterly=False)
df1 = dv.get_ts('myvar1')
assert not df1.empty
formula2 = 'Delta(oper_exp * myvar1 - open, 3)'
dv.add_formula('myvar2', formula2, is_quarterly=False)
df2 = dv.get_ts('myvar2')
assert not df2.empty
def test_remote_data_service_industry():
from quantos.data.align import align
import pandas as pd
ds = RemoteDataService()
arr = ds.get_index_comp(index='000300.SH', start_date=20130101, end_date=20170505)
df = ds.get_industry_raw(symbol=','.join(arr), type_='ZZ')
df = df.astype(dtype={'in_date': int})
# df_ann = df.loc[:, ['in_date', 'symbol']]
# df_ann = df_ann.set_index(['symbol', 'in_date'])
# df_ann = df_ann.unstack(level='symbol')
from quantos.data.dataview import DataView
dic_sec = DataView._group_df_to_dict(df, by='symbol')
dic_sec = {sec: df.reset_index() for sec, df in dic_sec.viewitems()}
df_ann = pd.concat([df.loc[:, 'in_date'].rename(sec) for sec, df in dic_sec.viewitems()], axis=1)
df_value = pd.concat([df.loc[:, 'industry1_code'].rename(sec) for sec, df in dic_sec.viewitems()], axis=1)
dates_arr = ds.get_trade_date(20140101, 20170505)
res = align(df_value, df_ann, dates_arr)
print
# df_ann = df.pivot(index='in_date', columns='symbol', values='in_date')
# df_value = df.pivot(index=None, columns='symbol', values='industry1_code')
def align_single_df(df_one_sec):
df_value = df_one_sec.loc[:, ['industry1_code']]
df_ann = df_one_sec.loc[:, ['in_date']]
res = align(df_value, df_ann, dates_arr)
return res
# res_list = [align_single_df(df) for sec, df in dic_sec.viewitems()]
res_list = [align_single_df(df) for sec, df in dic_sec.items()[:10]]
res = pd.concat(res_list, axis=1)
print res
def test_add_formula():
dv = DataView()
folder_path = '../output/prepared/20160601_20170601_freq=1D'
dv.load_dataview(folder=folder_path)
nrows, ncols = dv.data_d.shape
n_securities = len(dv.data_d.columns.levels[0])
formula = 'Delta(high - close, 1)'
dv.add_formula('myvar1', formula, is_quarterly=False)
assert dv.data_d.shape == (nrows, ncols + 1 * n_securities)
formula2 = 'myvar1 - close'
dv.add_formula('myvar2', formula2, is_quarterly=False)
assert dv.data_d.shape == (nrows, ncols + 2 * n_securities)
def test_add_formula_directly():
from quantos.data.dataservice import RemoteDataService
ds = RemoteDataService()
dv = DataView()
secs = '600030.SH,000063.SZ,000001.SZ'
props = {'start_date': 20160601, 'end_date': 20170601, 'symbol': secs,
'fields': 'open,close',
'freq': 1}
dv.init_from_config(props, data_api=ds)
dv.prepare_data()
dv.add_formula("myfactor", 'close / open', is_quarterly=False)
assert dv.data_d.shape == (281, 33)
def save_dataview():
from quantos.data.dataservice import RemoteDataService
ds = RemoteDataService()
dv = DataView()
props = {
'start_date': 20141114,
'end_date': 20161114,
'universe': '000300.SH',
'fields':
'open,close,high,low,volume,turnover,vwap,' + 'oper_rev,oper_exp',
'freq': 1
}
dv.init_from_config(props, data_api=ds)
dv.prepare_data()
dv.save_dataview(folder_path='../output/prepared')
def test_add_field():
dv = DataView()
folder_path = '../output/prepared/20160601_20170601_freq=1D'
dv.load_dataview(folder=folder_path)
nrows, ncols = dv.data_d.shape
n_securities = len(dv.data_d.columns.levels[0])
from quantos.data.dataservice import RemoteDataService
ds = RemoteDataService()
dv.add_field('share_amount', ds)
assert dv.data_d.shape == (nrows, ncols + 1 * n_securities)
def test_quarterly():
from quantos.data.dataservice import RemoteDataService
ds = RemoteDataService()
dv = DataView()
secs = '600030.SH,000063.SZ,000001.SZ'
props = {'start_date': 20160609, 'end_date': 20170601, 'universe': '000300.SH', 'symbol': secs,
'fields': ('open,close,'
+ 'pb,net_assets,'
+ 'total_oper_rev,oper_exp,'
+ 'cash_paid_invest,'
+ 'capital_stk,'
+ 'roe'), 'freq': 1}
dv.init_from_config(props, data_api=ds)
dv.prepare_data()
folder_path = '../output/prepared'
dv.save_dataview(folder_path=folder_path)
def test_add_field_quarterly():
dv = DataView()
folder_path = '../output/prepared/20160609_20170601_freq=1D'
dv.load_dataview(folder=folder_path)
nrows, ncols = dv.data_q.shape
n_securities = len(dv.data_d.columns.levels[0])
from quantos.data.dataservice import RemoteDataService
ds = RemoteDataService()
dv.add_field('net_inc_other_ops', ds)
"""
dv.add_field('oper_rev', ds)
dv.add_field('turnover', ds)
"""
assert dv.data_q.shape == (nrows, ncols + 1 * n_securities)
def test_dataview_universe():
from quantos.data.dataservice import RemoteDataService
ds = RemoteDataService()
dv = DataView()
props = {'start_date': 20170227, 'end_date': 20170327, 'universe': '000016.SH',
# 'symbol': 'rb1710.SHF,rb1801.SHF',
'fields': ('open,high,low,close,vwap,volume,turnover,'
+ 'roe,net_assets,'
+ 'total_oper_rev,oper_exp,tot_profit,int_income'
),
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
def main():
dv = DataView()
fullpath = fileio.join_relative_path('../output/prepared/20141114_20170327_freq=1D')
dv.load_dataview(folder=fullpath)
print dv.fields
factor_formula = '-1 * Rank(Ts_Max(Delta(vwap, 7), 11))' # GTJA
# factor_formula = '-Delta((((close - low) - (high - close)) / (high - low)), 1)'
# factor_formula = '-Delta(close, 5) / close'# / pb' # revert
# factor_formula = 'Delta(tot_profit, 1) / Delay(tot_profit, 1)' # pct change
# factor_formula = '- Delta(close, 3) / Delay(close, 3)'
# factor_formula = 'Delay(total_oper_rev, 1)'
factor_name = 'factor1'
# dv.add_formula(factor_name, factor_formula)
# dv.add_formula('factor2', 'GroupApply(Standardize, GroupApply(Cutoff, gtja, 3.0))')
# dv.add_formula('factor_bool', 'If(factor1 > total_oper_rev, 1.5, 0.5)')
# dv.add_formula('factor2', 'Standardize(factor1)')
factor = dv.get_ts('ret20').shift(1, axis=0) # avoid look-ahead bias
# factor = dv.get_ts('gtja').shift(1, axis=0) # avoid look-ahead bias
price = dv.get_ts('vwap')
price_bench = dv._data_benchmark
trade_status = dv.get_ts('trade_status')
mask_sus = trade_status != u'交易'.encode('utf-8')
df_group = dv.data_group.copy()
from quantos.util import dtutil
df_group.index = dtutil.convert_int_to_datetime(df_group.index)
factor_data = alphalens.utils.get_clean_factor_and_forward_returns(factor, price,
mask_sus=mask_sus, benchmark_price=None,
quantiles=5, periods=[20],
# groupby=df_group.stack(), by_group=False
)
res = alphalens.tears.create_full_tear_sheet(factor_data, long_short=True,
output_format='pdf', verbose=True,
# by_group=True
)
def test_write():
from quantos.data.dataservice import RemoteDataService
ds = RemoteDataService()
dv = DataView()
secs = '600030.SH,000063.SZ,000001.SZ'
props = {'start_date': 20160601, 'end_date': 20170601, 'symbol': secs,
'fields': 'open,close,high,low,volume,pb,net_assets,ncf',
'freq': 1}
dv.init_from_config(props, data_api=ds)
dv.prepare_data()
assert dv.data_d.shape == (281, 42)
assert dv.dates.shape == (281, )
# TODO
"""
PerformanceWarning:
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->mixed,key->block1_values] [items->[('000001.SZ', 'int_income'), ('000001.SZ', 'less_handling_chrg_comm_exp'), ('000001.SZ', 'net_int_income'), ('000001.SZ', 'oper_exp'), ('000001.SZ', 'symbol'), ('000063.SZ', 'int_income'), ('000063.SZ', 'less_handling_chrg_comm_exp'), ('000063.SZ', 'net_int_income'), ('000063.SZ', 'oper_exp'), ('000063.SZ', 'symbol'), ('600030.SH', 'int_income'), ('600030.SH', 'less_handling_chrg_comm_exp'), ('600030.SH', 'net_int_income'), ('600030.SH', 'oper_exp'), ('600030.SH', 'symbol')]]
"""
folder_path = '../output/prepared'
dv.save_dataview(folder_path=folder_path)
def test_load():
dv = DataView()
folder_path = '../output/prepared/20160601_20170601_freq=1D'
dv.load_dataview(folder=folder_path)
assert dv.start_date == 20160601 and set(dv.symbol) == set('000001.SZ,600030.SH,000063.SZ'.split(','))
# test get_snapshot
snap1 = dv.get_snapshot(20170504, symbol='600030.SH,000063.SZ', fields='close,pb')
assert snap1.shape == (2, 2)
assert set(snap1.columns.values) == {'close', 'pb'}
assert set(snap1.index.values) == {'600030.SH', '000063.SZ'}
# test get_ts
ts1 = dv.get_ts('close', symbol='600030.SH,000063.SZ', start_date=20170101, end_date=20170302)
assert ts1.shape == (38, 2)
assert set(ts1.columns.values) == {'600030.SH', '000063.SZ'}
assert ts1.index.values[-1] == 20170302
def test_get_quarterly():
dv = DataView()
folder_path = '../output/prepared/20160609_20170601_freq=1D'
dv.load_dataview(folder=folder_path)
res = dv.get("", 0, 0, 'total_oper_rev')
def _test_append_custom_data():
# --------------------------------------------------------------------------------
# get custom data
ds = RemoteDataService()
# lb.blablabla
df_raw, msg = ds.api.query("lb.secRestricted",
fields="symbol,list_date,lifted_shares,lifted_ratio",
filter="start_date=20170325&end_date=20170525",
orderby="",
data_format='pandas')
assert msg == '0,'
gp = df_raw.groupby(by=['list_date', 'symbol'])
df_multi = gp.agg({'lifted_ratio': np.sum})
df_value = df_multi.unstack(level=1)
df_value.columns = df_value.columns.droplevel(level=0)
# df_value = df_value.fillna(0.0)
# --------------------------------------------------------------------------------
# Format df_custom
dv = DataView()
dv.load_dataview('../output/prepared/20160609_20170601_freq=1D')
df_value = df_value.loc[:, dv.symbol]
df_custom = pd.DataFrame(index=dv.dates, columns=dv.symbol, data=None)
df_custom.loc[df_value.index, df_value.columns] = df_value
df_custom.fillna(0.0, inplace=True)
# --------------------------------------------------------------------------------
# append DataFrame to existed DataView
dv.append_df(df_custom + 1e-3 * np.random.rand(df_custom.shape[1]), field_name='custom')
dv.add_formula('myfactor', 'Rank(custom)')
# --------------------------------------------------------------------------------
# test this factor
factor = dv.get_ts('myfactor')
trade_status = dv.get_ts('trade_status')
close = dv.get_ts('close')
mask_sus = trade_status != u'交易'.encode('utf-8')
factor_data = alphalens.utils.get_clean_factor_and_forward_returns(factor, close, mask_sus=mask_sus, periods=[5])
alphalens.tears.create_full_tear_sheet(factor_data, output_format='pdf')
# encoding: utf-8
import numpy as np
import pandas as pd
from quantos.data.dataview import DataView
from quantos.data.dataservice import RemoteDataService
from quantos.research import alphalens
from quantos.util import fileio
ds = RemoteDataService()
dv = DataView()
props = {'start_date': 20120101, 'end_date': 20170901, 'universe': '000300.SH',
# 'symbol': 'rb1710.SHF,rb1801.SHF',
'fields': ('open,high,low,close,vwap,volume,turnover,'
# + 'pb,net_assets,'
+ 's_fa_eps_basic,oper_exp,tot_profit,int_income'
),
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
dv.add_formula('eps_ret', 'Return(s_fa_eps_basic, 4)', is_quarterly=True)
dv.add_formula('rule1', '(eps_ret > 0.2) && (Delay(eps_ret, 1) > 0.2)', is_quarterly=True)
dv.add_formula('rule2', 'close > Ts_Max(close, 120)', is_quarterly=False)
# dv.add_formula('ytan', 'rule1 && rule2', is_quarterly=False)
dv.add_formula('ret20', 'Delay(Return(close_adj, 20), -20)', is_quarterly=False)
def save_dataview():
# total 130 seconds
ds = RemoteDataService()
dv = DataView()
props = {'start_date': 20141114, 'end_date': 20170327, 'universe': '000300.SH',
# 'symbol': 'rb1710.SHF,rb1801.SHF',
'fields': ('open,high,low,close,vwap,volume,turnover,'
# + 'pb,net_assets,'
+ 'total_oper_rev,oper_exp,tot_profit,int_income'
),
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
factor_formula = '-1 * Rank(Ts_Max(Delta(vwap, 7), 11))' # GTJA
factor_name = 'gtja'
dv.add_formula(factor_name, factor_formula)
dv.add_formula('eps_ret_wrong', 'Return(eps, 3)', is_quarterly=False)
tmp = dv.get_ts('eps_ret_wrong')
dv.add_formula('eps_ret', 'Return(eps, 3)', is_quarterly=True)
tmp = dv.get_ts('eps_ret')
dv.add_formula('look_ahead', 'Delay(Return(close_adj, 5), -5)')
dv.add_formula('ret1', 'Return(close_adj, 1)')
dv.add_formula('ret20', 'Delay(Return(close_adj, 20), -20)')
dv.save_dataview(folder_path=fileio.join_relative_path('../output/prepared'))