assert p.shape == (5, 3)
assert type(p) == pd.DataFrame
assets = symbols(['SPY'])
p = prices(assets, start, end)
assert p.shape == (5, )
assert type(p) == pd.Series
assets = [symbol('SPY')]
p = prices(assets, start, end)
print(p)
assert p.shape == (5, )
assert type(p) == pd.Series
assets = symbols(['IBM', 'F', 'AAPL'])
p = returns(assets, start, end)
print(p)
assert p.shape == (5, 3)
assert type(p) == pd.DataFrame
assets = symbols(['SPY'])
p = returns(assets, start, end)
print(p)
assert p.shape == (5, )
assert type(p) == pd.Series
assets = [symbol('SPY')]
p = returns(assets, start, end)
assert p.shape == (5, )
assert type(p) == pd.Series
import pandas as pd
from sharadar.pipeline.engine import symbol, returns
import matplotlib.pylab as plt
import numpy as np
def create_log_rets(rets, benchmark_rets):
cum_log_returns = np.log1p(rets).cumsum()
cum_log_benchmark_rets = np.log1p(benchmark_rets).cumsum()
fig, ax = plt.subplots()
cum_log_returns.plot(ax=ax, figsize=(20, 10))
cum_log_benchmark_rets.plot(ax=ax)
ax.grid(True)
ax.axhline(y=0, linestyle='--', color='black')
ax.legend(['Backtest', 'Benchmark'])
plt.title("Log returns")
return plt
if __name__ == "__main__":
results = pd.read_pickle(
'../algo/haugen20/haugen20_202006300552_perf.dump')
rets, positions, transactions = pf.utils.extract_rets_pos_txn_from_zipline(
results)
benchmark_rets = returns(symbol('SPY'), rets.index[0], rets.index[0])
plt = create_log_rets(rets, benchmark_rets)
plt.show()
def create_report(perf,
filename,
now,
doc=None,
duration=None,
param=None,
info=None,
show_image=True):
if not hasattr(perf, 'returns'):
perf['returns'] = perf['pnl'] / (perf['portfolio_value'] - perf['pnl'])
perf['returns'] = perf['returns'].replace([np.nan, np.inf, -np.inf],
0.0)
tot_positions = sum([len(x) for x in perf.positions])
if tot_positions == 0:
log.warn("No positions available")
return
rets, positions, transactions = pf.utils.extract_rets_pos_txn_from_zipline(
perf)
date_rows = OrderedDict()
if len(rets.index) > 0:
date_rows['Start date'] = rets.index[0].strftime('%Y-%m-%d')
date_rows['End date'] = rets.index[-1].strftime('%Y-%m-%d')
date_rows['Total months'] = int(len(rets) / 21)
perf_stats_series = pf.timeseries.perf_stats(rets,
positions=positions,
transactions=transactions)
benchmark_rets = returns(symbol('SPY'), rets.index[0], rets.index[-1])
benchmark_perf_stats = pf.timeseries.perf_stats(benchmark_rets)
perf_stats_df = pd.DataFrame(perf_stats_series, columns=['Backtest'])
perf_stats_df['Benchmark'] = benchmark_perf_stats
perf_stats_df[
'Spread'] = perf_stats_df['Backtest'] - perf_stats_df['Benchmark']
format_perf_stats(perf_stats_df)
drawdown_df = pf.timeseries.gen_drawdown_table(rets, top=5)
rets_interesting = pf.timeseries.extract_interesting_date_ranges(rets)
positions = utils.check_intraday('infer', rets, positions, transactions)
transactions_closed = rt.add_closing_transactions(positions, transactions)
trades = rt.extract_round_trips(
transactions_closed,
portfolio_value=positions.sum(axis='columns') / (1 + rets))
if show_image:
fig0 = None
fig1 = None
fig2 = None
fig3 = None
fig4 = None
fig5 = None
try:
fig0 = create_log_returns_chart(rets, benchmark_rets)
except Exception as e:
log.warn(e)
try:
fig1 = pf.create_returns_tear_sheet(rets,
positions,
transactions,
benchmark_rets=benchmark_rets,
return_fig=True)
except Exception as e:
log.warn(e)
try:
fig2 = pf.create_position_tear_sheet(rets,
positions,
return_fig=True)
except Exception as e:
log.warn(e)
try:
fig3 = pf.create_txn_tear_sheet(rets,
positions,
transactions,
return_fig=True)
except Exception as e:
log.warn(e)
try:
fig4 = pf.create_interesting_times_tear_sheet(rets,
return_fig=True)
except Exception as e:
log.warn(e)
try:
fig5 = pf.create_round_trip_tear_sheet(rets,
positions,
transactions,
return_fig=True)
except Exception as e:
log.warn(e)
report_suffix = "_%s_%.2f_report.htm" % (
now.strftime(DATETIME_FMT), 100. * perf_stats_series['Annual return'])
reportfile = change_extension(filename, report_suffix)
with open(reportfile, 'w') as f:
print("""<!DOCTYPE html>
<html>
<head>
<title>Performance Report</title>
<style >
body {
font-family: Arial, Helvetica, sans-serif;
}
table {
border-collapse: collapse;
}
tbody tr:nth-child(odd) {
background-color: lightgrey;
}
tbody tr:nth-child(even) {
background-color: white;
}
tr th {
border: none;
text-align: right;
padding: 2px 5px 2px;
}
tr td {
border: none;
text-align: right;
padding: 2px 5px 2px;
}
</style>
<script type="text/javascript">
function showElement() {
element = document.getElementById('code');
element.style.visibility = 'visible';
}
function hideElement() {
element = document.getElementById('code');
element.style.visibility = 'hidden';
}
</script>
</head>
<body>""",
file=f)
print("<h1>Performance report for " + os.path.basename(filename) +
"</h1>",
file=f)
print("<p>Created on %s</p>" % (now), file=f)
if duration is not None:
print("<p>Backtest executed in %s</p>" %
(time.strftime("%H:%M:%S", time.gmtime(duration))),
file=f)
if doc is not None:
print('<h3>Description</h3>', file=f)
print('<p style="white-space: pre">%s</p>' % doc.strip(), file=f)
if param is not None and len(param) > 0:
print('<h3>Parameters</h3>', file=f)
print('<pre>%s</pre><br/>' % str(param), file=f)
if info is not None and len(info) > 0:
print('<h3>Info</h3>', file=f)
print('<pre>%s</pre><br/>' % str(info), file=f)
print(to_html_table(perf_stats_df,
float_format='{0:.2f}'.format,
header_rows=date_rows),
file=f)
print("<br/>", file=f)
if show_image:
if fig0 is not None:
print("<h3>Log Returns</h3>", file=f)
print(_to_img(fig0), file=f)
print("<br/>", file=f)
print(to_html_table(
drawdown_df.sort_values('Net drawdown in %', ascending=False),
name='Worst drawdown periods',
float_format='{0:.2f}'.format,
),
file=f)
print("<br/>", file=f)
print(to_html_table(pd.DataFrame(rets_interesting).describe().
transpose().loc[:, ['mean', 'min', 'max']] * 100,
name='Stress Events',
float_format='{0:.2f}%'.format),
file=f)
print("<br/>", file=f)
if len(trades) >= 5:
stats = rt.gen_round_trip_stats(trades)
print(to_html_table(stats['summary'],
float_format='{:.2f}'.format,
name='Summary stats'),
file=f)
print("<br/>", file=f)
print(to_html_table(stats['pnl'],
float_format='${:.2f}'.format,
name='PnL stats'),
file=f)
print("<br/>", file=f)
print(to_html_table(stats['duration'],
float_format='{:.2f}'.format,
name='Duration stats'),
file=f)
print("<br/>", file=f)
print(to_html_table(stats['returns'] * 100,
float_format='{:.2f}%'.format,
name='Return stats'),
file=f)
print("<br/>", file=f)
stats['symbols'].columns = stats['symbols'].columns.map(
format_asset)
print(to_html_table(stats['symbols'] * 100,
float_format='{:.2f}%'.format,
name='Symbol stats'),
file=f)
if show_image:
if fig1 is not None:
print("<h3>Returns</h3>", file=f)
print(_to_img(fig1), file=f)
if fig2 is not None:
print("<h3>Positions</h3>", file=f)
print(_to_img(fig2), file=f)
if fig3 is not None:
print("<h3>Transactions</h3>", file=f)
print(_to_img(fig3), file=f)
if fig4 is not None:
print("<h3>Interesting Times</h3>", file=f)
print(_to_img(fig4), file=f)
if fig5 is not None:
print("<h3>Trades</h3>", file=f)
print(_to_img(fig5), file=f)
print('<br/>', file=f)
print(
'<button onclick="showElement()">Show Code</button> <button onclick="hideElement()">Hide Code</button>',
file=f)
print('<pre id="code" style="visibility: hidden">', file=f)
print(open(filename, "r").read(), file=f)
print('</pre>', file=f)
print("</body>\n</html>", file=f)
screen=universe,
)
stocks = engine.run_pipeline(pipe, start, end, hooks=[])
print(stocks)
expected = [[499.30, 2163847100000.00, 503.43, -0.01],
[506.09, 2137988000000.00, 499.30, 0.01],
[500.04, 2134533300000.00, 506.09, -0.01],
[124.81, 2206911200000.00, 125.01, -0.00],
[129.04, 2294818300000.00, 124.81, 0.03],
[134.18, 2247273200000.00, 129.04, 0.04]]
assert np.sum(abs(stocks.values - expected)) <= 1e-1
bundle = load_sharadar_bundle()
adjustments = bundle.adjustment_reader.get_adjustments_for_sid(
'splits', 199059)
print(adjustments)
print(returns(symbols(['AAPL', 'IBM', 'F']), start, end))
universe = StaticAssets(symbols(['AAPL', 'IBM', 'F']))
pipe = Pipeline(
columns={
'close': Latest([EquityPricing.close], mask=universe),
'mkt_cap': MarketCap(mask=universe)
},
screen=universe,
)
stocks = engine.run_pipeline(pipe, '2021-03-17', '2021-03-24', hooks=[])
print(stocks)
from sharadar.pipeline.engine import load_sharadar_bundle, symbol, symbols, prices, returns, trading_date
import pandas as pd
df_rets = returns(
assets=symbol('AAPL'),
start='2013-01-01',
end='2014-01-01'
)
assets = symbols(['IBM', 'F', 'AAPL'])
start = pd.to_datetime('2020-02-03', utc=True)
end = pd.to_datetime('2020-02-07', utc=True)
print(prices(assets, start, end))
print(prices(assets, '2020-02-03', '2020-02-05'))
print(returns(assets, start, end))
import pyfolio as pf
import pandas as pd
import numpy as np
from sharadar.pipeline.engine import symbol, returns
import empyrical as ep
results = pd.read_pickle('../../../algo/haugen20/haugen20_perf.dump')
rets, pos, trax = pf.utils.extract_rets_pos_txn_from_zipline(results)
start_date = rets.index[0]
end_date = rets.index[-1]
print(start_date, end_date)
benchmark_rets = returns(symbol('SPY'), start_date, end_date)
cum_rets = ep.cum_returns(rets, 1.0)
cum_benchmark_rets = ep.cum_returns(benchmark_rets, 1.0)
cum_log_returns = np.log1p(rets).cumsum()
cum_log_benchmark_rets = np.log1p(benchmark_rets).cumsum()
fig1 = pf.create_returns_tear_sheet(rets,
pos,
trax,
benchmark_rets=benchmark_rets,
return_fig=True)
def compute(self, today, assets, out):
end_dt = self.bar_reader().trading_calendar.sessions_window(
today, self.window_length)[-1]
out[:] = returns(assets, today, end_dt)
def _run(handle_data, initialize, before_trading_start, analyze, algofile,
algotext, defines, data_frequency, capital_base, bundle,
bundle_timestamp, start, end, output, trading_calendar, print_algo,
metrics_set, local_namespace, environ, blotter, benchmark_symbol,
broker, state_filename):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`zipline.run_algo`.
additions useful for live trading:
broker - wrapper to connect to a real broker
state_filename - saving the context of the algo to be able to restart
"""
log.info("Using bundle '%s'." % bundle)
if trading_calendar is None:
trading_calendar = get_calendar('XNYS')
bundle_data = load_sharadar_bundle(bundle)
now = pd.Timestamp.utcnow()
if start is None:
start = bundle_data.equity_daily_bar_reader.first_trading_day if not broker else now
if not trading_calendar.is_session(start.date()):
start = trading_calendar.next_open(start)
if end is None:
end = bundle_data.equity_daily_bar_reader.last_available_dt if not broker else start
# date parameter validation
if trading_calendar.session_distance(start, end) < 0:
raise _RunAlgoError(
'There are no trading days between %s and %s' % (
start.date(),
end.date(),
), )
if broker:
log.info("Live Trading on %s." % start.date())
# No benchmark needed for live trading
benchmark_symbol = None
else:
log.info("Backtest from %s to %s." % (start.date(), end.date()))
if benchmark_symbol:
benchmark = symbol(benchmark_symbol)
benchmark_sid = benchmark.sid
benchmark_returns = returns([benchmark], start, end)
else:
benchmark_sid = None
benchmark_returns = pd.Series(index=pd.date_range(start, end,
tz='utc'),
data=0.0)
# emission_rate is a string representing the smallest frequency at which metrics should be reported.
# emission_rate will be either minute or daily. When emission_rate is daily, end_of_bar will not be called at all.
emission_rate = 'daily'
if algotext is not None:
if local_namespace:
# noinspection PyUnresolvedReferences
ip = get_ipython() # noqa
namespace = ip.user_ns
else:
namespace = {}
for assign in defines:
try:
name, value = assign.split('=', 2)
except ValueError:
raise ValueError(
'invalid define %r, should be of the form name=value' %
assign, )
try:
# evaluate in the same namespace so names may refer to
# eachother
namespace[name] = eval(value, namespace)
except Exception as e:
raise ValueError(
'failed to execute definition for name %r: %s' %
(name, e), )
elif defines:
raise _RunAlgoError(
'cannot pass define without `algotext`',
"cannot pass '-D' / '--define' without '-t' / '--algotext'",
)
else:
namespace = {}
if algofile is not None:
algotext = algofile.read()
if print_algo:
if PYGMENTS:
highlight(
algotext,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout,
)
else:
click.echo(algotext)
first_trading_day = \
bundle_data.equity_daily_bar_reader.first_trading_day
if isinstance(metrics_set, six.string_types):
try:
metrics_set = metrics.load(metrics_set)
except ValueError as e:
raise _RunAlgoError(str(e))
if isinstance(blotter, six.string_types):
try:
blotter = load(Blotter, blotter)
except ValueError as e:
raise _RunAlgoError(str(e))
# Special defaults for live trading
if broker:
data_frequency = 'minute'
# No benchmark
benchmark_sid = None
benchmark_returns = pd.Series(index=pd.date_range(start, end,
tz='utc'),
data=0.0)
broker.daily_bar_reader = bundle_data.equity_daily_bar_reader
if start.date() < now.date():
backtest_start = start
backtest_end = bundle_data.equity_daily_bar_reader.last_available_dt
if not os.path.exists(state_filename):
log.info("Backtest from %s to %s." %
(backtest_start.date(), backtest_end.date()))
backtest_data = DataPortal(
bundle_data.asset_finder,
trading_calendar=trading_calendar,
first_trading_day=first_trading_day,
equity_minute_reader=bundle_data.equity_minute_bar_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
backtest = create_algo_class(
TradingAlgorithm, backtest_start, backtest_end, algofile,
algotext, analyze, before_trading_start, benchmark_returns,
benchmark_sid, blotter, bundle_data, capital_base,
backtest_data, 'daily', emission_rate, handle_data,
initialize, metrics_set, namespace, trading_calendar)
ctx_blacklist = ['trading_client']
ctx_whitelist = ['perf_tracker']
ctx_excludes = ctx_blacklist + [
e
for e in backtest.__dict__.keys() if e not in ctx_whitelist
]
backtest.run()
checksum = getattr(algofile, 'name', '<algorithm>')
store_context(state_filename,
context=backtest,
checksum=checksum,
exclude_list=ctx_excludes)
else:
log.warn("State file already exists. Do not run the backtest.")
# Set start and end to now for live trading
start = pd.Timestamp.utcnow()
if not trading_calendar.is_session(start.date()):
start = trading_calendar.next_open(start)
end = start
# usare store_context prima di passare da TradingAlgorithm a LiveTradingAlgorithm
TradingAlgorithmClass = (partial(
LiveTradingAlgorithm, broker=broker, state_filename=state_filename)
if broker else TradingAlgorithm)
DataPortalClass = (partial(DataPortalLive, broker)
if broker else DataPortal)
data = DataPortalClass(
bundle_data.asset_finder,
trading_calendar=trading_calendar,
first_trading_day=first_trading_day,
equity_minute_reader=bundle_data.equity_minute_bar_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
algo = create_algo_class(TradingAlgorithmClass, start, end, algofile,
algotext, analyze, before_trading_start,
benchmark_returns, benchmark_sid, blotter,
bundle_data, capital_base, data, data_frequency,
emission_rate, handle_data, initialize,
metrics_set, namespace, trading_calendar)
perf = algo.run()
if output == '-':
click.echo(str(perf))
elif output != os.devnull: # make the zipline magic not write any data
perf.to_pickle(output)
return perf