def __init__(self,
start_session,
end_session,
trading_calendar,
capital_base=DEFAULT_CAPITAL_BASE,
emission_rate='daily',
data_frequency='daily',
arena='backtest'):
assert type(start_session) == pd.Timestamp
assert type(end_session) == pd.Timestamp
assert trading_calendar is not None, \
"Must pass in trading calendar!"
assert start_session <= end_session, \
"Period start falls after period end."
assert start_session <= trading_calendar.last_trading_session, \
"Period start falls after the last known trading day."
assert end_session >= trading_calendar.first_trading_session, \
"Period end falls before the first known trading day."
# chop off any minutes or hours on the given start and end dates,
# as we only support session labels here (and we represent session
# labels as midnight UTC).
self._start_session = normalize_date(start_session)
self._end_session = normalize_date(end_session)
self._capital_base = capital_base
self._emission_rate = emission_rate
self._data_frequency = data_frequency
# copied to algorithm's environment for runtime access
self._arena = arena
self._trading_calendar = trading_calendar
if not trading_calendar.is_session(self._start_session):
# if the start date is not a valid session in this calendar,
# push it forward to the first valid session
self._start_session = trading_calendar.minute_to_session_label(
self._start_session)
if not trading_calendar.is_session(self._end_session):
# if the end date is not a valid session in this calendar,
# pull it backward to the last valid session before the given
# end date.
self._end_session = trading_calendar.minute_to_session_label(
self._end_session, direction="previous")
self._first_open = trading_calendar.open_and_close_for_session(
self._start_session)[0]
self._last_close = trading_calendar.open_and_close_for_session(
self._end_session)[1]
def _process_orders(self, df_ord):
# assign to df and reduce the number of fields
df = df_ord.copy()
fields = [
'created_at', 'average_price', 'cumulative_quantity', 'fees',
'symbol', 'side'
]
df = df[fields]
# convert types
for field in ['average_price', 'cumulative_quantity', 'fees']:
df[field] = pd.to_numeric(df[field])
for field in ['created_at']:
df[field] = pd.to_datetime(df[field])
# add days
df['date'] = df['created_at'].apply(lambda x: normalize_date(x))
# rename columns for consistency
df.rename(columns={'cumulative_quantity': 'current_size'},
inplace=True)
# quantity accounting for side of transaction for cumsum later
df['signed_size'] = np.where(df.side == 'buy', df['current_size'],
-df['current_size'])
df['signed_size'] = df['signed_size'].astype(np.int64)
return df
def handle_minute_close(self, dt, data_portal):
"""
Handles the close of the given minute in minute emission.
Parameters
__________
dt : Timestamp
The minute that is ending
Returns
_______
A minute perf packet.
"""
self.position_tracker.sync_last_sale_prices(dt, False, data_portal)
self.update_performance()
todays_date = normalize_date(dt)
account = self.get_account(False)
bench_returns = self.all_benchmark_returns.loc[todays_date:dt]
# cumulative returns
bench_since_open = (1. + bench_returns).prod() - 1
self.cumulative_risk_metrics.update(todays_date,
self.todays_performance.returns,
bench_since_open, account.leverage)
minute_packet = self.to_dict(emission_type='minute')
return minute_packet
def _process_dividends(self, df_div):
df = df_div.copy()
# convert types
for field in ['amount', 'position', 'rate']:
df[field] = pd.to_numeric(df[field])
for field in ['paid_at', 'payable_date']:
df[field] = pd.to_datetime(df[field])
# add days
df['date'] = df['paid_at'].apply(lambda x: normalize_date(x))
return df
def handle_market_close_daily(self):
"""
Function called after handle_data when running with daily emission
rate.
"""
self.update_performance()
completed_date = normalize_date(self.market_close)
# add the return results from today to the returns series
self.returns[completed_date] = self.todays_performance.returns
# update risk metrics for cumulative performance
self.cumulative_risk_metrics.update(
completed_date, self.todays_performance.returns,
self.all_benchmark_returns[completed_date])
# increment the day counter before we move markers forward.
self.day_count += 1.0
# Take a snapshot of our current performance to return to the
# browser.
daily_update = self.to_dict()
# On the last day of the test, don't create tomorrow's performance
# period. We may not be able to find the next trading day if we're at
# the end of our historical data
if self.market_close >= self.last_close:
return daily_update
# move the market day markers forward
self.market_open, self.market_close = \
trading.environment.next_open_and_close(self.market_open)
# Roll over positions to current day.
self.todays_performance.rollover()
self.todays_performance.period_open = self.market_open
self.todays_performance.period_close = self.market_close
self.check_upcoming_dividends(completed_date)
return daily_update
def handle_minute_close(self, dt):
self.update_performance()
todays_date = normalize_date(dt)
minute_returns = self.minute_performance.returns
self.minute_performance.rollover()
# the intraday risk is calculated on top of minute performance
# returns for the bench and the algo
self.intraday_risk_metrics.update(dt, minute_returns,
self.all_benchmark_returns[dt])
bench_since_open = \
self.intraday_risk_metrics.benchmark_cumulative_returns[dt]
self.cumulative_risk_metrics.update(todays_date,
self.todays_performance.returns,
bench_since_open)
# if this is the close, save the returns objects for cumulative risk
# calculations and update dividends for the next day.
if dt == self.market_close:
self.check_upcoming_dividends(todays_date)
self.returns[todays_date] = self.todays_performance.returns
def __init__(self, sim_params,
returns_frequency=None,
create_first_day_stats=False):
"""
- @returns_frequency allows for configuration of the whether
the benchmark and algorithm returns are in units of minutes or days,
if `None` defaults to the `emission_rate` in `sim_params`.
"""
self.treasury_curves = trading.environment.treasury_curves
self.start_date = sim_params.period_start.replace(
hour=0, minute=0, second=0, microsecond=0
)
self.end_date = sim_params.period_end.replace(
hour=0, minute=0, second=0, microsecond=0
)
self.trading_days = trading.environment.days_in_range(
self.start_date,
self.end_date)
# Hold on to the trading day before the start,
# used for index of the zero return value when forcing returns
# on the first day.
self.day_before_start = self.start_date - \
trading.environment.trading_days.freq
last_day = normalize_date(sim_params.period_end)
if last_day not in self.trading_days:
last_day = pd.DatetimeIndex(
[last_day]
)
self.trading_days = self.trading_days.append(last_day)
self.sim_params = sim_params
self.create_first_day_stats = create_first_day_stats
if returns_frequency is None:
returns_frequency = self.sim_params.emission_rate
self.returns_frequency = returns_frequency
if returns_frequency == 'daily':
cont_index = self.get_daily_index()
elif returns_frequency == 'minute':
cont_index = self.get_minute_index(sim_params)
self.cont_index = cont_index
self.algorithm_returns_cont = pd.Series(index=cont_index)
self.benchmark_returns_cont = pd.Series(index=cont_index)
self.mean_returns_cont = pd.Series(index=cont_index)
self.annualized_mean_returns_cont = pd.Series(index=cont_index)
self.mean_benchmark_returns_cont = pd.Series(index=cont_index)
self.annualized_mean_benchmark_returns_cont = pd.Series(
index=cont_index)
# The returns at a given time are read and reset from the respective
# returns container.
self.algorithm_returns = None
self.benchmark_returns = None
self.mean_returns = None
self.annualized_mean_returns = None
self.mean_benchmark_returns = None
self.annualized_mean_benchmark_returns = None
self.algorithm_cumulative_returns = pd.Series(index=cont_index)
self.benchmark_cumulative_returns = pd.Series(index=cont_index)
self.excess_returns = pd.Series(index=cont_index)
self.latest_dt = cont_index[0]
self.metrics = pd.DataFrame(index=cont_index,
columns=self.METRIC_NAMES)
self.drawdowns = pd.Series(index=cont_index)
self.max_drawdowns = pd.Series(index=cont_index)
self.max_drawdown = 0
self.current_max = -np.inf
self.daily_treasury = pd.Series(index=self.trading_days)
self.treasury_period_return = np.nan
self.num_trading_days = 0
def transform(self):
"""
Main generator work loop.
"""
algo = self.algo
perf_tracker = algo.perf_tracker
emission_rate = perf_tracker.emission_rate
def every_bar(dt_to_use,
current_data=self.current_data,
handle_data=algo.event_manager.handle_data):
# called every tick (minute or day).
algo.on_dt_changed(dt_to_use)
for capital_change in calculate_minute_capital_changes(dt_to_use):
yield capital_change
self.simulation_dt = dt_to_use
blotter = algo.blotter
# handle any transactions and commissions coming out new orders
# placed in the last bar
new_transactions, new_commissions, closed_orders = \
blotter.get_transactions(current_data)
blotter.prune_orders(closed_orders)
for transaction in new_transactions:
perf_tracker.process_transaction(transaction)
# since this order was modified, record it
order = blotter.orders[transaction.order_id]
perf_tracker.process_order(order)
if new_commissions:
for commission in new_commissions:
perf_tracker.process_commission(commission)
handle_data(algo, current_data, dt_to_use)
# grab any new orders from the blotter, then clear the list.
# this includes cancelled orders.
new_orders = blotter.new_orders
blotter.new_orders = []
# if we have any new orders, record them so that we know
# in what perf period they were placed.
if new_orders:
for new_order in new_orders:
perf_tracker.process_order(new_order)
algo.portfolio_needs_update = True
algo.account_needs_update = True
algo.performance_needs_update = True
def once_a_day(midnight_dt,
current_data=self.current_data,
data_portal=self.data_portal):
# set all the timestamps
self.simulation_dt = midnight_dt
algo.on_dt_changed(midnight_dt)
# process any capital changes that came overnight
for capital_change in algo.calculate_capital_changes(
midnight_dt, emission_rate=emission_rate,
is_interday=True):
yield capital_change
# handle any splits that impact any positions or any open orders.
assets_we_care_about = \
viewkeys(perf_tracker.position_tracker.positions) | \
viewkeys(algo.blotter.open_orders)
if assets_we_care_about:
splits = data_portal.get_splits(assets_we_care_about,
midnight_dt)
if splits:
algo.blotter.process_splits(splits)
perf_tracker.position_tracker.handle_splits(splits)
def handle_benchmark(date, benchmark_source=self.benchmark_source):
perf_tracker.all_benchmark_returns[
date] = benchmark_source.get_value(date)
def on_exit():
# Remove references to algo, data portal, et al to break cycles
# and ensure deterministic cleanup of these objects when the
# simulation finishes.
self.algo = None
self.benchmark_source = self.current_data = self.data_portal = None
with ExitStack() as stack:
stack.callback(on_exit)
stack.enter_context(self.processor)
stack.enter_context(ZiplineAPI(self.algo))
if algo.data_frequency == 'minute':
def execute_order_cancellation_policy():
algo.blotter.execute_cancel_policy(SESSION_END)
def calculate_minute_capital_changes(dt):
# process any capital changes that came between the last
# and current minutes
return algo.calculate_capital_changes(
dt, emission_rate=emission_rate, is_interday=False)
else:
def execute_order_cancellation_policy():
pass
def calculate_minute_capital_changes(dt):
return []
for dt, action in self.clock:
if action == BAR:
for capital_change_packet in every_bar(dt):
yield capital_change_packet
elif action == SESSION_START:
for capital_change_packet in once_a_day(dt):
yield capital_change_packet
elif action == SESSION_END:
# End of the session.
positions = perf_tracker.position_tracker.positions
position_assets = algo.asset_finder.retrieve_all(positions)
self._cleanup_expired_assets(dt, position_assets)
if emission_rate == 'daily':
handle_benchmark(normalize_date(dt))
else:
# If the emission rate is minutely then the performance
# update already happened by this point, so if any
# equities were just auto closed do another update.
perf_tracker.update_performance()
execute_order_cancellation_policy()
algo.validate_account_controls()
yield self._get_daily_message(dt, algo, perf_tracker)
elif action == BEFORE_TRADING_START_BAR:
self.simulation_dt = dt
algo.on_dt_changed(dt)
algo.before_trading_start(self.current_data)
elif action == MINUTE_END:
handle_benchmark(dt)
minute_msg = \
self._get_minute_message(dt, algo, perf_tracker)
yield minute_msg
risk_message = perf_tracker.handle_simulation_end()
yield risk_message
def _get_adjustments_in_range(self, asset, dts, field):
"""
Get the Float64Multiply objects to pass to an AdjustedArrayWindow.
For the use of AdjustedArrayWindow in the loader, which looks back
from current simulation time back to a window of data the dictionary is
structured with:
- the key into the dictionary for adjustments is the location of the
day from which the window is being viewed.
- the start of all multiply objects is always 0 (in each window all
adjustments are overlapping)
- the end of the multiply object is the location before the calendar
location of the adjustment action, making all days before the event
adjusted.
Parameters
----------
asset : Asset
The assets for which to get adjustments.
dts : iterable of datetime64-like
The dts for which adjustment data is needed.
field : str
OHLCV field for which to get the adjustments.
Returns
-------
out : dict[loc -> Float64Multiply]
The adjustments as a dict of loc -> Float64Multiply
"""
sid = int(asset)
start = normalize_date(dts[0])
end = normalize_date(dts[-1])
adjs = {}
#if field != 'volume':
# # 限定调整字段
if field in ('open', 'high', 'low', 'close'):
mergers = self._adjustments_reader.get_adjustments_for_sid(
'mergers', sid)
for m in mergers:
dt = m[0]
if start < dt <= end:
end_loc = dts.searchsorted(dt)
adj_loc = end_loc
mult = Float64Multiply(0, end_loc - 1, 0, 0, m[1])
try:
adjs[adj_loc].append(mult)
except KeyError:
adjs[adj_loc] = [mult]
divs = self._adjustments_reader.get_adjustments_for_sid(
'dividends', sid)
for d in divs:
dt = d[0]
if start < dt <= end:
end_loc = dts.searchsorted(dt)
adj_loc = end_loc
mult = Float64Multiply(0, end_loc - 1, 0, 0, d[1])
try:
adjs[adj_loc].append(mult)
except KeyError:
adjs[adj_loc] = [mult]
splits = self._adjustments_reader.get_adjustments_for_sid(
'splits', sid)
for s in splits:
dt = s[0]
if start < dt <= end:
if field == 'volume':
ratio = 1.0 / s[1]
# # else -> elif field in ('open', 'high', 'low', 'close')
elif field in ('open', 'high', 'low', 'close'):
ratio = s[1]
# # 使用原值 1
else:
ratio = 1
end_loc = dts.searchsorted(dt)
adj_loc = end_loc
mult = Float64Multiply(0, end_loc - 1, 0, 0, ratio)
try:
adjs[adj_loc].append(mult)
except KeyError:
adjs[adj_loc] = [mult]
return adjs
def __init__(self,
sim_params,
trading_calendar,
create_first_day_stats=False):
self.trading_calendar = trading_calendar
self.start_session = sim_params.start_session
self.end_session = sim_params.end_session
self.sessions = trading_calendar.sessions_in_range(
self.start_session, self.end_session)
# Hold on to the trading day before the start,
# used for index of the zero return value when forcing returns
# on the first day.
self.day_before_start = self.start_session - self.sessions.freq
last_day = normalize_date(sim_params.end_session)
if last_day not in self.sessions:
last_day = pd.tseries.index.DatetimeIndex([last_day])
self.sessions = self.sessions.append(last_day)
self.sim_params = sim_params
self.create_first_day_stats = create_first_day_stats
cont_index = self.sessions
self.cont_index = cont_index
self.cont_len = len(self.cont_index)
empty_cont = np.full(self.cont_len, np.nan)
self.algorithm_returns_cont = empty_cont.copy()
self.benchmark_returns_cont = empty_cont.copy()
self.algorithm_cumulative_leverages_cont = empty_cont.copy()
self.mean_returns_cont = empty_cont.copy()
self.annualized_mean_returns_cont = empty_cont.copy()
self.mean_benchmark_returns_cont = empty_cont.copy()
self.annualized_mean_benchmark_returns_cont = empty_cont.copy()
# The returns at a given time are read and reset from the respective
# returns container.
self.algorithm_returns = None
self.benchmark_returns = None
self.mean_returns = None
self.annualized_mean_returns = None
self.mean_benchmark_returns = None
self.annualized_mean_benchmark_returns = None
self.algorithm_cumulative_returns = empty_cont.copy()
self.benchmark_cumulative_returns = empty_cont.copy()
self.algorithm_cumulative_leverages = empty_cont.copy()
self.excess_returns = empty_cont.copy()
self.latest_dt_loc = 0
self.latest_dt = cont_index[0]
self.benchmark_volatility = empty_cont.copy()
self.algorithm_volatility = empty_cont.copy()
self.beta = empty_cont.copy()
self.alpha = empty_cont.copy()
self.sharpe = empty_cont.copy()
self.downside_risk = empty_cont.copy()
self.sortino = empty_cont.copy()
self.drawdowns = empty_cont.copy()
self.max_drawdowns = empty_cont.copy()
self.max_drawdown = 0
self.max_leverages = empty_cont.copy()
self.max_leverage = 0
self.current_max = -np.inf
self.daily_treasury = pd.Series(index=self.sessions)
self.treasury_period_return = np.nan
self.num_trading_days = 0
def __init__(self, sim_params):
self.sim_params = sim_params
self.period_start = self.sim_params.period_start
self.period_end = self.sim_params.period_end
self.last_close = self.sim_params.last_close
first_day = self.sim_params.first_open
self.market_open, self.market_close = \
trading.environment.get_open_and_close(first_day)
self.total_days = self.sim_params.days_in_period
self.capital_base = self.sim_params.capital_base
self.emission_rate = sim_params.emission_rate
all_trading_days = trading.environment.trading_days
mask = ((all_trading_days >= normalize_date(self.period_start)) &
(all_trading_days <= normalize_date(self.period_end)))
self.trading_days = all_trading_days[mask]
self.dividend_frame = pd.DataFrame()
self._dividend_count = 0
self.perf_periods = []
if self.emission_rate == 'daily':
self.all_benchmark_returns = pd.Series(index=self.trading_days)
self.intraday_risk_metrics = None
self.cumulative_risk_metrics = \
risk.RiskMetricsCumulative(self.sim_params)
elif self.emission_rate == 'minute':
self.all_benchmark_returns = pd.Series(
index=pd.date_range(self.sim_params.first_open,
self.sim_params.last_close,
freq='Min'))
self.intraday_risk_metrics = \
risk.RiskMetricsCumulative(self.sim_params)
self.cumulative_risk_metrics = \
risk.RiskMetricsCumulative(self.sim_params,
returns_frequency='daily',
create_first_day_stats=True)
self.minute_performance = PerformancePeriod(
# initial cash is your capital base.
self.capital_base,
# the cumulative period will be calculated over the
# entire test.
self.period_start,
self.period_end,
# don't save the transactions for the cumulative
# period
keep_transactions=False,
keep_orders=False,
# don't serialize positions for cumualtive period
serialize_positions=False)
self.perf_periods.append(self.minute_performance)
# this performance period will span the entire simulation from
# inception.
self.cumulative_performance = PerformancePeriod(
# initial cash is your capital base.
self.capital_base,
# the cumulative period will be calculated over the entire test.
self.period_start,
self.period_end,
# don't save the transactions for the cumulative
# period
keep_transactions=False,
keep_orders=False,
# don't serialize positions for cumualtive period
serialize_positions=False)
self.perf_periods.append(self.cumulative_performance)
# this performance period will span just the current market day
self.todays_performance = PerformancePeriod(
# initial cash is your capital base.
self.capital_base,
# the daily period will be calculated for the market day
self.market_open,
self.market_close,
keep_transactions=True,
keep_orders=True,
serialize_positions=True)
self.perf_periods.append(self.todays_performance)
self.saved_dt = self.period_start
self.returns = pd.Series(index=self.trading_days)
# one indexed so that we reach 100%
self.day_count = 0.0
self.txn_count = 0
self.event_count = 0