def save_dataview():
# total 130 seconds
ds = RemoteDataService()
dv = DataView()
props = {'start_date': 20141114, 'end_date': 20170327, 'universe': '000300.SH',
# 'symbol': 'rb1710.SHF,rb1801.SHF',
'fields': ('open,high,low,close,vwap,volume,turnover,'
# + 'pb,net_assets,'
+ 'total_oper_rev,oper_exp,tot_profit,int_income'
),
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
factor_formula = '-1 * Rank(Ts_Max(Delta(vwap, 7), 11))' # GTJA
factor_name = 'gtja'
dv.add_formula(factor_name, factor_formula)
dv.add_formula('eps_ret_wrong', 'Return(eps, 3)', is_quarterly=False)
tmp = dv.get_ts('eps_ret_wrong')
dv.add_formula('eps_ret', 'Return(eps, 3)', is_quarterly=True)
tmp = dv.get_ts('eps_ret')
dv.add_formula('look_ahead', 'Delay(Return(close_adj, 5), -5)')
dv.add_formula('ret1', 'Return(close_adj, 1)')
dv.add_formula('ret20', 'Delay(Return(close_adj, 20), -20)')
dv.save_dataview(folder_path=fileio.join_relative_path('../output/prepared'))
def test_double_ma():
prop_file_path = fileio.join_relative_path("etc/backtest.json")
print prop_file_path
prop_file = open(prop_file_path, 'r')
props = json.load(prop_file)
enum_props = {'bar_type': common.QUOTE_TYPE}
for k, v in enum_props.iteritems():
props[k] = v.to_enum(props[k])
# strategy = CtaStrategy()
strategy = DoubleMaStrategy()
gateway = BarSimulatorGateway()
data_service = RemoteDataService()
context = model.Context()
context.register_data_api(data_service)
context.register_gateway(gateway)
context.register_trade_api(gateway)
backtest = EventBacktestInstance()
backtest.init_from_config(props, strategy, context=context)
# backtest.run()
backtest.run()
report = backtest.generate_report()
def test_alpha_strategy():
gateway = DailyStockSimGateway()
remote_data_service = RemoteDataService()
prop_file_path = fileio.join_relative_path('etc', 'alpha.json')
props = read_props(prop_file_path)
"""
props = {
"benchmark": "000300.SH",
"universe": "600026.SH,600027.SH,600028.SH,600029.SH,600030.SH,600031.SH",
"period": "week",
"days_delay": 2,
"init_balance": 1e7,
"position_ratio": 0.7,
"start_date": 20120101,
"end_date": 20170601,
}
"""
remote_data_service.init_from_config(props)
remote_data_service.initialize()
gateway.init_from_config(props)
context = model.Context()
context.register_data_api(remote_data_service)
context.register_gateway(gateway)
context.register_trade_api(gateway)
risk_model = model.FactorRiskModel()
signal_model = model.FactorRevenueModel()
cost_model = model.SimpleCostModel()
risk_model.register_context(context)
signal_model.register_context(context)
cost_model.register_context(context)
signal_model.register_func('pb_factor', pb_factor)
signal_model.activate_func({'pb_factor': {'coef': 3.27}})
cost_model.register_func('my_commission', my_commission)
cost_model.activate_func({'my_commission': {'myrate': 1e-2}})
strategy = DemoAlphaStrategy(risk_model, signal_model, cost_model)
# strategy.register_context(context)
# strategy.active_pc_method = 'equal_weight'
strategy.active_pc_method = 'mc'
backtest = AlphaBacktestInstance()
backtest.init_from_config(props, strategy, context=context)
backtest.run_alpha()
backtest.save_results('../output/')
def test_output():
static_folder = fileio.join_relative_path('backtest/analyze/static')
r = Report({'mytitle': 'Test Title', 'mytable': 'Hello World!'},
source_dir=static_folder,
template_fn='test_template.html',
css_fn='blueprint.css',
out_folder='../output')
r.generate_html()
r.output_html()
r.output_pdf()
def create_turnover_tear_sheet(factor_data, output_format='plot'):
"""
Creates a tear sheet for analyzing the turnover properties of a factor.
Parameters
----------
factor_data : pd.DataFrame - MultiIndex
A MultiIndex DataFrame indexed by date (level 0) and asset (level 1),
containing the values for a single alpha factor, forward returns for
each period, the factor quantile/bin that factor value belongs to, and
(optionally) the group the asset belongs to.
"""
turnover_periods = utils.get_forward_returns_columns(factor_data.columns)
quantile_factor = factor_data['factor_quantile']
quantile_turnover = \
{p: pd.concat([perf.quantile_turnover(quantile_factor, q, p)
for q in range(1, int(quantile_factor.max()) + 1)],
axis=1)
for p in turnover_periods}
autocorrelation = pd.concat(
[perf.factor_rank_autocorrelation(factor_data, period) for period in
turnover_periods], axis=1)
if output_format:
plotting.plot_turnover_table(autocorrelation, quantile_turnover)
fr_cols = len(turnover_periods)
columns_wide = 1
rows_when_wide = (((fr_cols - 1) // 1) + 1)
vertical_sections = fr_cols + 3 * rows_when_wide + 2 * fr_cols
gf = GridFigure(rows=vertical_sections, cols=columns_wide)
for period in sorted(quantile_turnover.keys()):
plotting.plot_top_bottom_quantile_turnover(quantile_turnover[period],
period=period,
ax=gf.next_row())
for period in autocorrelation:
plotting.plot_factor_rank_auto_correlation(autocorrelation[period],
period=period,
ax=gf.next_row())
if output_format == 'plot':
plt.show()
# gf.fig.show()
elif output_format == 'pdf':
from quantos.util import fileio
gf.fig.savefig(fileio.join_relative_path('..', 'output', 'turnover_tear_sheet.pdf'))
elif output_format == '':
pass
def test_alpha_strategy_dataview():
dv = DataView()
fullpath = fileio.join_relative_path(
'../output/prepared/20141114_20170827_freq=1D')
dv.load_dataview(folder=fullpath)
props = {
"benchmark": "000300.SH",
# "symbol": ','.join(dv.symbol),
"universe": ','.join(dv.symbol),
"start_date": dv.start_date,
"end_date": dv.end_date,
"period": "month",
"days_delay": 0,
"init_balance": 1e9,
"position_ratio": 0.7,
}
gateway = DailyStockSimGateway()
gateway.init_from_config(props)
context = model.Context()
context.register_gateway(gateway)
context.register_trade_api(gateway)
context.register_dataview(dv)
risk_model = model.FactorRiskModel()
signal_model = model.FactorRevenueModel_dv()
cost_model = model.SimpleCostModel()
risk_model.register_context(context)
signal_model.register_context(context)
cost_model.register_context(context)
signal_model.register_func('gtja', gtja_factor_dataview)
signal_model.activate_func({'gtja': {}})
cost_model.register_func('my_commission', my_commission)
cost_model.activate_func({'my_commission': {'myrate': 1e-2}})
strategy = DemoAlphaStrategy(risk_model, signal_model, cost_model)
# strategy.active_pc_method = 'equal_weight'
# strategy.active_pc_method = 'mc'
strategy.active_pc_method = 'factor_value_weight'
bt = AlphaBacktestInstance_dv()
bt.init_from_config(props, strategy, context=context)
bt.run_alpha()
bt.save_results('../output/')
def test_backtest_analyze():
ta = ana.AlphaAnalyzer()
data_service = RemoteDataService()
out_folder = fileio.join_relative_path("../output")
ta.initialize(data_service, out_folder)
print "process trades..."
ta.process_trades()
print "get daily stats..."
ta.get_daily()
print "calc strategy return..."
ta.get_returns()
# print "get position change..."
# ta.get_pos_change_info()
selected_sec = [] # list(ta.universe)[:3]
if len(selected_sec) > 0:
print "Plot single securities PnL"
for symbol in selected_sec:
df_daily = ta.daily.get(symbol, None)
if df_daily is not None:
ana.plot_trades(df_daily,
symbol=symbol,
save_folder=out_folder)
print "Plot strategy PnL..."
ta.plot_pnl(out_folder)
print "generate report..."
static_folder = fileio.join_relative_path("backtest/analyze/static")
ta.gen_report(source_dir=static_folder,
template_fn='report_template.html',
css_fn='blueprint.css',
out_folder=out_folder,
selected=selected_sec)
def __init__(self):
DataService.__init__(self)
dic = fileio.read_json(
fileio.join_relative_path('etc/data_config.json'))
address = dic.get("remote.address", None)
username = dic.get("remote.username", None)
password = dic.get("remote.password", None)
if address is None or username is None or password is None:
raise ValueError("no address, username or password available!")
self.api = DataApi(address, use_jrpc=False)
self.api.set_timeout(60)
r, msg = self.api.login(username=username, password=password)
if not r:
print msg
else:
print "DataAPI login success.".format(address)
self.REPORT_DATE_FIELD_NAME = 'report_date'
def main():
dv = DataView()
fullpath = fileio.join_relative_path('../output/prepared/20141114_20170327_freq=1D')
dv.load_dataview(folder=fullpath)
print dv.fields
factor_formula = '-1 * Rank(Ts_Max(Delta(vwap, 7), 11))' # GTJA
# factor_formula = '-Delta((((close - low) - (high - close)) / (high - low)), 1)'
# factor_formula = '-Delta(close, 5) / close'# / pb' # revert
# factor_formula = 'Delta(tot_profit, 1) / Delay(tot_profit, 1)' # pct change
# factor_formula = '- Delta(close, 3) / Delay(close, 3)'
# factor_formula = 'Delay(total_oper_rev, 1)'
factor_name = 'factor1'
# dv.add_formula(factor_name, factor_formula)
# dv.add_formula('factor2', 'GroupApply(Standardize, GroupApply(Cutoff, gtja, 3.0))')
# dv.add_formula('factor_bool', 'If(factor1 > total_oper_rev, 1.5, 0.5)')
# dv.add_formula('factor2', 'Standardize(factor1)')
factor = dv.get_ts('ret20').shift(1, axis=0) # avoid look-ahead bias
# factor = dv.get_ts('gtja').shift(1, axis=0) # avoid look-ahead bias
price = dv.get_ts('vwap')
price_bench = dv._data_benchmark
trade_status = dv.get_ts('trade_status')
mask_sus = trade_status != u'交易'.encode('utf-8')
df_group = dv.data_group.copy()
from quantos.util import dtutil
df_group.index = dtutil.convert_int_to_datetime(df_group.index)
factor_data = alphalens.utils.get_clean_factor_and_forward_returns(factor, price,
mask_sus=mask_sus, benchmark_price=None,
quantiles=5, periods=[20],
# groupby=df_group.stack(), by_group=False
)
res = alphalens.tears.create_full_tear_sheet(factor_data, long_short=True,
output_format='pdf', verbose=True,
# by_group=True
)
def test_data_api():
dic = fileio.read_json(fileio.join_relative_path('etc/data_config.json'))
address = dic.get("remote.address", None)
username = dic.get("remote.username", None)
password = dic.get("remote.password", None)
if address is None or username is None or password is None:
raise ValueError("no data service config available!")
api = DataApi(address, use_jrpc=False)
login_msg = api.login(username=username, password=password)
print login_msg
daily, msg = api.daily(
symbol="600030.SH,000002.SZ",
start_date=20170103,
end_date=20170708,
fields="open,high,low,close,volume,last,trade_date,settle")
daily2, msg2 = api.daily(
symbol="600030.SH",
start_date=20170103,
end_date=20170708,
fields="open,high,low,close,volume,last,trade_date,settle")
# err_code, err_msg = msg.split(',')
assert msg == '0,'
assert msg2 == '0,'
assert daily.shape == (248, 8)
assert daily2.shape == (124, 8)
df, msg = api.bar(symbol="600030.SH",
trade_date=20170904,
freq='1m',
start_time=90000,
end_time=150000)
print df.columns
assert df.shape == (240, 15)
print "test passed"
import pandas as pd
from quantos.data.dataview import DataView
from quantos.data.dataservice import RemoteDataService
from quantos.research import alphalens
from quantos.util import fileio
ds = RemoteDataService()
dv = DataView()
props = {'start_date': 20120101, 'end_date': 20170901, 'universe': '000300.SH',
# 'symbol': 'rb1710.SHF,rb1801.SHF',
'fields': ('open,high,low,close,vwap,volume,turnover,'
# + 'pb,net_assets,'
+ 's_fa_eps_basic,oper_exp,tot_profit,int_income'
),
'freq': 1}
dv.init_from_config(props, ds)
dv.prepare_data()
dv.add_formula('eps_ret', 'Return(s_fa_eps_basic, 4)', is_quarterly=True)
dv.add_formula('rule1', '(eps_ret > 0.2) && (Delay(eps_ret, 1) > 0.2)', is_quarterly=True)
dv.add_formula('rule2', 'close > Ts_Max(close, 120)', is_quarterly=False)
# dv.add_formula('ytan', 'rule1 && rule2', is_quarterly=False)
dv.add_formula('ret20', 'Delay(Return(close_adj, 20), -20)', is_quarterly=False)
dv.save_dataview(folder_path=fileio.join_relative_path('../output/prepared'))
def create_information_tear_sheet(factor_data, group_adjust=False, by_group=False,
output_format='plot', verbose=False):
"""
Creates a tear sheet for information analysis of a factor.
Parameters
----------
factor_data : pd.DataFrame - MultiIndex
A MultiIndex DataFrame indexed by date (level 0) and asset (level 1),
containing the values for a single alpha factor, forward returns for
each period, the factor quantile/bin that factor value belongs to, and
(optionally) the group the asset belongs to.
group_adjust : bool
Demean forward returns by group before computing IC.
by_group : bool
If True, perform calcuations, and display graphs separately for
each group.
verbose : bool
Whether return data.
"""
ic = perf.factor_information_coefficient(factor_data, group_adjust)
if output_format:
plotting.plot_information_table(ic)
columns_wide = 2
fr_cols = len(ic.columns)
rows_when_wide = (((fr_cols - 1) // columns_wide) + 1)
vertical_sections = fr_cols + 3 * rows_when_wide + 2 * fr_cols
gf = GridFigure(rows=vertical_sections, cols=columns_wide)
ax_ic_ts = [gf.next_row() for _ in range(fr_cols)]
plotting.plot_ic_ts(ic, ax=ax_ic_ts)
ax_ic_hqq = [gf.next_cell() for _ in range(fr_cols * 2)]
plotting.plot_ic_hist(ic, ax=ax_ic_hqq[::2])
plotting.plot_ic_qq(ic, ax=ax_ic_hqq[1::2])
if not by_group:
mean_monthly_ic = perf.mean_information_coefficient(factor_data,
group_adjust,
by_group,
"M")
if output_format:
ax_monthly_ic_heatmap = [gf.next_cell() for x in range(fr_cols)]
plotting.plot_monthly_ic_heatmap(mean_monthly_ic,
ax=ax_monthly_ic_heatmap)
if by_group:
mean_group_ic = perf.mean_information_coefficient(factor_data,
by_group=True)
if output_format:
plotting.plot_ic_by_group(mean_group_ic, ax=gf.next_row())
if output_format == 'plot':
plt.show()
# gf.fig.show()
elif output_format == 'pdf':
from quantos.util import fileio
gf.fig.savefig(fileio.join_relative_path('..', 'output', 'infomation_tear_sheet.pdf'))
elif output_format == '':
pass
if verbose:
return {'daily_ic': ic}
def create_returns_tear_sheet(factor_data, long_short=True, by_group=False,
output_format='plot', verbose=False):
"""
Creates a tear sheet for returns analysis of a factor.
Parameters
----------
factor_data : pd.DataFrame - MultiIndex
A MultiIndex DataFrame indexed by date (level 0) and asset (level 1),
containing the values for a single alpha factor, forward returns for
each period, the factor quantile/bin that factor value belongs to,
and (optionally) the group the asset belongs to.
long_short : bool
Should this computation happen on a long short portfolio? if so, then
factor values will be demeaned across the factor universe when factor
weighting the portfolio. Additionally, mean quantile returns will be
demeaned across the factor universe.
by_group : bool
If True, perform calcuations, and display graphs separately for
each group.
verbose : bool
Whether return data.
"""
# daily factor return for different periods
factor_returns = perf.factor_returns(factor_data, long_short=long_short)
# mean quantile return
mean_ret_quantile, std_quantile = \
perf.mean_return_by_quantile(factor_data, by_group=False, demeaned=long_short)
mean_compret_quantile = mean_ret_quantile.apply(utils.rate_of_return, axis=0)
# daily quantile return
mean_ret_quant_daily, std_quant_daily = \
perf.mean_return_by_quantile(factor_data, by_date=True, by_group=False, demeaned=long_short)
mean_compret_quant_daily = mean_ret_quant_daily.apply(utils.rate_of_return, axis=0)
compstd_quant_daily = std_quant_daily.apply(utils.std_conversion, axis=0)
# alpha beta
alpha_beta = perf.factor_alpha_beta(factor_data, long_short)
# quantile return spread
mean_ret_spread_quant, std_spread_quant = \
perf.compute_mean_returns_spread(mean_compret_quant_daily,
factor_data['factor_quantile'].max(),
factor_data['factor_quantile'].min(),
std_err=compstd_quant_daily)
# start plotting
if output_format:
print "\n\nFactor Returns Tear Sheet (long short = {}): ".format(long_short)
fr_cols = len(factor_returns.columns)
vertical_sections = 2 + fr_cols * 3
gf = GridFigure(rows=vertical_sections, cols=1)
plotting.plot_returns_table(alpha_beta,
mean_compret_quantile,
mean_ret_spread_quant)
ax_qrb = plotting.plot_quantile_returns_bar(mean_compret_quantile,
by_group=False,
ylim_percentiles=None,
ax=gf.next_row())
ax_qrv = plotting.plot_quantile_returns_violin(mean_compret_quant_daily,
ylim_percentiles=(1, 99),
ax=gf.next_row())
for p in factor_returns:
ax_cr = plotting.plot_cumulative_returns(factor_returns[p],
period=p,
ax=gf.next_row())
ax_qrbq = plotting.plot_cumulative_returns_by_quantile(mean_ret_quant_daily[p],
period=p,
ax=gf.next_row())
ax_mean_quantile_returns_spread_ts = [gf.next_row()
for x in range(fr_cols)]
ax_mqrsts = plotting.plot_mean_quantile_returns_spread_time_series(
mean_ret_spread_quant,
std_err=std_spread_quant,
bandwidth=0.5,
ax=ax_mean_quantile_returns_spread_ts
)
if by_group:
mean_return_quantile_group, mean_return_quantile_group_std_err = \
perf.mean_return_by_quantile(factor_data,
by_date=False,
by_group=by_group,
demeaned=True)
mean_compret_quantile_group = \
mean_return_quantile_group.apply(utils.rate_of_return, axis=0)
if output_format:
num_groups = len(mean_compret_quantile_group.index
.get_level_values('group')
.unique())
vertical_sections = 1 + (((num_groups - 1) // 2) + 1)
gf = GridFigure(rows=vertical_sections, cols=2)
ax_quantile_returns_bar_by_group = [gf.next_cell()
for _ in range(num_groups)]
plotting.plot_quantile_returns_bar(mean_compret_quantile_group,
by_group=True,
ylim_percentiles=(5, 95),
ax=ax_quantile_returns_bar_by_group)
if output_format == 'plot':
plt.show()
# gf.fig.show()
elif output_format == 'pdf':
from quantos.util import fileio
gf.fig.savefig(fileio.join_relative_path('..', 'output', 'returns_tear_sheet.pdf'))
elif output_format == '':
pass
if verbose:
mean_ret_quant_daily_mod = mean_ret_quant_daily.unstack(level=0)
mean_ret_quant_daily_mod.columns = mean_ret_quant_daily_mod.columns.droplevel(level=0)
tuples = mean_ret_quant_daily_mod.iteritems()
tuples = [(str(a), b) for a, b in tuples]
res = dict(tuples)
return {'mean_quantile_return_daily': res}