class AlgorithmSimulator(object):
EMISSION_TO_PERF_KEY_MAP = {'minute': 'minute_perf', 'daily': 'daily_perf'}
def get_hash(self):
"""
There should only ever be one TSC in the system, so
we don't bother passing args into the hash.
"""
return self.__class__.__name__ + hash_args()
def __init__(self, algo, sim_params):
# ==============
# Simulation
# Param Setup
# ==============
self.sim_params = sim_params
# ==============
# Algo Setup
# ==============
self.algo = algo
self.algo_start = normalize_date(self.sim_params.first_open)
# ==============
# Snapshot Setup
# ==============
# The algorithm's data as of our most recent event.
# We want an object that will have empty objects as default
# values on missing keys.
self.current_data = BarData()
# We don't have a datetime for the current snapshot until we
# receive a message.
self.simulation_dt = None
# =============
# Logging Setup
# =============
# Processor function for injecting the algo_dt into
# user prints/logs.
def inject_algo_dt(record):
if 'algo_dt' not in record.extra:
record.extra['algo_dt'] = self.simulation_dt
self.processor = Processor(inject_algo_dt)
@property
def perf_key(self):
return self.EMISSION_TO_PERF_KEY_MAP[
self.algo.perf_tracker.emission_rate]
def process_event(self, event):
process_trade = self.algo.blotter.process_trade
for txn, order in process_trade(event):
self.algo.perf_tracker.process_event(txn)
self.algo.perf_tracker.process_event(order)
self.algo.perf_tracker.process_event(event)
def transform(self, stream_in):
"""
Main generator work loop.
"""
# Initialize the mkt_close
mkt_open = self.algo.perf_tracker.market_open
mkt_close = self.algo.perf_tracker.market_close
# inject the current algo
# snapshot time to any log record generated.
with self.processor.threadbound():
data_frequency = self.sim_params.data_frequency
self._call_before_trading_start(mkt_open)
for date, snapshot in stream_in:
self.simulation_dt = date
self.on_dt_changed(date)
# If we're still in the warmup period. Use the event to
# update our universe, but don't yield any perf messages,
# and don't send a snapshot to handle_data.
if date < self.algo_start:
for event in snapshot:
if event.type == DATASOURCE_TYPE.SPLIT:
self.algo.blotter.process_split(event)
elif event.type in (DATASOURCE_TYPE.TRADE,
DATASOURCE_TYPE.CUSTOM):
self.update_universe(event)
self.algo.perf_tracker.process_event(event)
else:
message = self._process_snapshot(
date,
snapshot,
self.algo.instant_fill,
)
# Perf messages are only emitted if the snapshot contained
# a benchmark event.
if message is not None:
yield message
# When emitting minutely, we re-iterate the day as a
# packet with the entire days performance rolled up.
if date == mkt_close:
if self.algo.perf_tracker.emission_rate == 'minute':
daily_rollup = self.algo.perf_tracker.to_dict(
emission_type='daily')
daily_rollup['daily_perf']['recorded_vars'] = \
self.algo.recorded_vars
yield daily_rollup
tp = self.algo.perf_tracker.todays_performance
tp.rollover()
if mkt_close <= self.algo.perf_tracker.last_close:
before_last_close = \
mkt_close < self.algo.perf_tracker.last_close
try:
mkt_open, mkt_close = \
trading.environment \
.next_open_and_close(mkt_close)
except trading.NoFurtherDataError:
# If at the end of backtest history,
# skip advancing market close.
pass
if (self.algo.perf_tracker.emission_rate ==
'minute'):
self.algo.perf_tracker\
.handle_intraday_market_close(
mkt_open,
mkt_close)
if before_last_close:
self._call_before_trading_start(mkt_open)
elif data_frequency == 'daily':
next_day = trading.environment.next_trading_day(date)
if (next_day is not None and
next_day < self.algo.perf_tracker.last_close):
self._call_before_trading_start(next_day)
self.algo.portfolio_needs_update = True
self.algo.account_needs_update = True
self.algo.performance_needs_update = True
risk_message = self.algo.perf_tracker.handle_simulation_end()
yield risk_message
def _process_snapshot(self, dt, snapshot, instant_fill):
"""
Process a stream of events corresponding to a single datetime, possibly
returning a perf message to be yielded.
If @instant_fill = True, we delay processing of events until after the
user's call to handle_data, and we process the user's placed orders
before the snapshot's events. Note that this introduces a lookahead
bias, since the user effectively is effectively placing orders that are
filled based on trades that happened prior to the call the handle_data.
If @instant_fill = False, we process Trade events before calling
handle_data. This means that orders are filled based on trades
occurring in the next snapshot. This is the more conservative model,
and as such it is the default behavior in TradingAlgorithm.
"""
# Flags indicating whether we saw any events of type TRADE and type
# BENCHMARK. Respectively, these control whether or not handle_data is
# called for this snapshot and whether we emit a perf message for this
# snapshot.
any_trade_occurred = False
benchmark_event_occurred = False
if instant_fill:
events_to_be_processed = []
for event in snapshot:
if event.type == DATASOURCE_TYPE.TRADE:
self.update_universe(event)
any_trade_occurred = True
elif event.type == DATASOURCE_TYPE.BENCHMARK:
benchmark_event_occurred = True
elif event.type == DATASOURCE_TYPE.CUSTOM:
self.update_universe(event)
elif event.type == DATASOURCE_TYPE.SPLIT:
self.algo.blotter.process_split(event)
if not instant_fill:
self.process_event(event)
else:
events_to_be_processed.append(event)
if any_trade_occurred:
new_orders = self._call_handle_data()
for order in new_orders:
self.algo.perf_tracker.process_event(order)
if instant_fill:
# Now that handle_data has been called and orders have been placed,
# process the event stream to fill user orders based on the events
# from this snapshot.
for event in events_to_be_processed:
self.process_event(event)
if benchmark_event_occurred:
return self.get_message(dt)
else:
return None
def _call_handle_data(self):
"""
Call the user's handle_data, returning any orders placed by the algo
during the call.
"""
self.algo.event_manager.handle_data(
self.algo,
self.current_data,
self.simulation_dt,
)
orders = self.algo.blotter.new_orders
self.algo.blotter.new_orders = []
return orders
def _call_before_trading_start(self, dt):
dt = normalize_date(dt)
self.simulation_dt = dt
self.on_dt_changed(dt)
self.algo.before_trading_start()
def on_dt_changed(self, dt):
if self.algo.datetime != dt:
self.algo.on_dt_changed(dt)
def get_message(self, dt):
"""
Get a perf message for the given datetime.
"""
# Ensure that updated_portfolio has been called at least once for this
# dt before we emit a perf message. This is a no-op if
# updated_portfolio has already been called this dt.
self.algo.updated_portfolio()
self.algo.updated_account()
rvars = self.algo.recorded_vars
if self.algo.perf_tracker.emission_rate == 'daily':
perf_message = \
self.algo.perf_tracker.handle_market_close_daily()
perf_message['daily_perf']['recorded_vars'] = rvars
return perf_message
elif self.algo.perf_tracker.emission_rate == 'minute':
self.algo.perf_tracker.handle_minute_close(dt)
perf_message = self.algo.perf_tracker.to_dict()
perf_message['minute_perf']['recorded_vars'] = rvars
return perf_message
def update_universe(self, event):
"""
Update the universe with new event information.
"""
# Update our knowledge of this event's sid
# rather than use if event.sid in ..., just trying
# and handling the exception is significantly faster
try:
sid_data = self.current_data[event.sid]
except KeyError:
sid_data = self.current_data[event.sid] = SIDData(event.sid)
sid_data.__dict__.update(event.__dict__)
class AlgorithmSimulator(object):
def __init__(self,
order_book,
perf_tracker,
algo,
algo_start):
# ==========
# Algo Setup
# ==========
# We extract the order book from the txn client so that
# the algo can place new orders.
self.order_book = order_book
self.perf_tracker = perf_tracker
self.algo = algo
self.algo_start = algo_start.replace(hour=0, minute=0,
second=0,
microsecond=0)
# Monkey patch the user algorithm to place orders in the
# TransactionSimulator's order book and use our logger.
self.algo.set_order(self.order)
# ==============
# Snapshot Setup
# ==============
# The algorithm's universe as of our most recent event.
# We want an ndict that will have empty objects as default
# values on missing keys.
self.universe = ndict(internal=defaultdict(SIDData))
# We don't have a datetime for the current snapshot until we
# receive a message.
self.simulation_dt = None
self.snapshot_dt = None
# =============
# Logging Setup
# =============
# Processor function for injecting the algo_dt into
# user prints/logs.
def inject_algo_dt(record):
if not 'algo_dt' in record.extra:
record.extra['algo_dt'] = self.snapshot_dt
self.processor = Processor(inject_algo_dt)
def order(self, sid, amount):
"""
Closure to pass into the user's algo to allow placing orders
into the transaction simulator's dict of open orders.
"""
order = Order({
'dt': self.simulation_dt,
'sid': sid,
'amount': int(amount),
'filled': 0
})
# Tell the user if they try to buy 0 shares of something.
if order.amount == 0:
zero_message = "Requested to trade zero shares of {sid}".format(
sid=order.sid
)
log.debug(zero_message)
# Don't bother placing orders for 0 shares.
return
# Add non-zero orders to the order book.
# !!!IMPORTANT SIDE-EFFECT!!!
# This modifies the internal state of the transaction
# simulator so that it can fill the placed order when it
# receives its next message.
self.order_book.place_order(order)
def transform(self, stream_in):
"""
Main generator work loop.
"""
# inject the current algo
# snapshot time to any log record generated.
with self.processor.threadbound():
for date, snapshot in stream_in:
# Set the simulation date to be the first event we see.
# This should only occur once, at the start of the test.
if self.simulation_dt is None:
self.simulation_dt = date
# We're still in the warmup period. Use the event to
# update our universe, but don't yield any perf messages,
# and don't send a snapshot to handle_data.
if date < self.algo_start:
for event in snapshot:
del event['perf_messages']
self.update_universe(event)
# Regular snapshot. Update the universe and send a snapshot
# to handle data.
else:
for event in snapshot:
for perf_message in event.perf_messages:
yield perf_message
del event['perf_messages']
self.update_universe(event)
# Send the current state of the universe
# to the user's algo.
self.simulate_snapshot(date)
perf_messages, risk_message = \
self.perf_tracker.handle_simulation_end()
for message in perf_messages:
yield message
yield risk_message
def update_universe(self, event):
"""
Update the universe with new event information.
"""
# Update our portfolio.
self.algo.set_portfolio(event.portfolio)
# Update our knowledge of this event's sid
sid_data = self.universe[event.sid]
sid_data.__dict__.update(event.__dict__)
def simulate_snapshot(self, date):
"""
Run the user's algo against our current snapshot and update
the algo's simulated time.
"""
# Needs to be set so that we inject the proper date into algo
# log/print lines.
self.snapshot_dt = date
self.algo.set_datetime(self.snapshot_dt)
self.algo.handle_data(self.universe)
# Update the simulation time.
self.simulation_dt = date
class AlgorithmSimulator(object):
EMISSION_TO_PERF_KEY_MAP = {'minute': 'minute_perf', 'daily': 'daily_perf'}
def get_hash(self):
"""
There should only ever be one TSC in the system, so
we don't bother passing args into the hash.
"""
return self.__class__.__name__ + hash_args()
def __init__(self, algo, sim_params):
# ==============
# Simulation
# Param Setup
# ==============
self.sim_params = sim_params
# ==============
# Algo Setup
# ==============
self.algo = algo
self.algo_start = self.sim_params.first_open
self.algo_start = self.algo_start.replace(hour=0,
minute=0,
second=0,
microsecond=0)
# ==============
# Snapshot Setup
# ==============
# The algorithm's data as of our most recent event.
# We want an object that will have empty objects as default
# values on missing keys.
self.current_data = BarData()
# We don't have a datetime for the current snapshot until we
# receive a message.
self.simulation_dt = None
# =============
# Logging Setup
# =============
# Processor function for injecting the algo_dt into
# user prints/logs.
def inject_algo_dt(record):
if not 'algo_dt' in record.extra:
record.extra['algo_dt'] = self.simulation_dt
self.processor = Processor(inject_algo_dt)
@property
def perf_key(self):
return self.EMISSION_TO_PERF_KEY_MAP[
self.algo.perf_tracker.emission_rate]
def transform(self, stream_in):
"""
Main generator work loop.
"""
# Initialize the mkt_close
mkt_close = self.algo.perf_tracker.market_close
# inject the current algo
# snapshot time to any log record generated.
with self.processor.threadbound():
updated = False
bm_updated = False
for date, snapshot in stream_in:
self.algo.set_datetime(date)
self.simulation_dt = date
self.algo.perf_tracker.set_date(date)
self.algo.blotter.set_date(date)
# If we're still in the warmup period. Use the event to
# update our universe, but don't yield any perf messages,
# and don't send a snapshot to handle_data.
if date < self.algo_start:
for event in snapshot:
if event.type in (DATASOURCE_TYPE.TRADE,
DATASOURCE_TYPE.CUSTOM):
self.update_universe(event)
self.algo.perf_tracker.process_event(event)
else:
events = []
for event in snapshot:
if event.type in (DATASOURCE_TYPE.TRADE,
DATASOURCE_TYPE.CUSTOM):
self.update_universe(event)
updated = True
if event.type == DATASOURCE_TYPE.BENCHMARK:
self.algo.set_datetime(event.dt)
bm_updated = True
# Save events to stream through blotter below.
events.append(event)
# Update our portfolio.
self.algo.set_portfolio(
self.algo.perf_tracker.get_portfolio())
# Send the current state of the universe
# to the user's algo.
if updated:
self.algo.handle_data(self.current_data)
updated = False
# run orders placed in the algorithm call
# above through perf tracker before emitting
# the perf packet, so that the perf includes
# placed orders
for order in self.algo.blotter.new_orders:
self.algo.perf_tracker.process_event(order)
self.algo.blotter.new_orders = []
# Fill orders
for event in events:
process_trade = self.algo.blotter.process_trade
for txn, order in process_trade(event):
self.algo.perf_tracker.process_event(txn)
self.algo.perf_tracker.process_event(order)
self.algo.perf_tracker.process_event(event)
# The benchmark is our internal clock. When it
# updates, we need to emit a performance message.
if bm_updated:
bm_updated = False
yield self.get_message(date)
# When emitting minutely, we re-iterate the day as a
# packet with the entire days performance rolled up.
if self.algo.perf_tracker.emission_rate == 'minute':
if date == mkt_close:
daily_rollup = self.algo.perf_tracker.to_dict(
emission_type='daily')
daily_rollup['daily_perf']['recorded_vars'] = \
self.algo.recorded_vars
yield daily_rollup
tp = self.algo.perf_tracker.todays_performance
tp.rollover()
if mkt_close < self.algo.perf_tracker.last_close:
_, mkt_close = \
trading.environment.next_open_and_close(
mkt_close
)
self.algo.perf_tracker.handle_intraday_close()
risk_message = self.algo.perf_tracker.handle_simulation_end()
yield risk_message
def get_message(self, date):
rvars = self.algo.recorded_vars
if self.algo.perf_tracker.emission_rate == 'daily':
perf_message = \
self.algo.perf_tracker.handle_market_close()
perf_message['daily_perf']['recorded_vars'] = rvars
return perf_message
elif self.algo.perf_tracker.emission_rate == 'minute':
self.algo.perf_tracker.handle_minute_close(date)
perf_message = self.algo.perf_tracker.to_dict()
perf_message['minute_perf']['recorded_vars'] = rvars
return perf_message
def update_universe(self, event):
"""
Update the universe with new event information.
"""
# Update our knowledge of this event's sid
sid_data = self.current_data[event.sid]
sid_data.__dict__.update(event.__dict__)
class AlgorithmSimulator(object):
EMISSION_TO_PERF_KEY_MAP = {"minute": "minute_perf", "daily": "daily_perf"}
def get_hash(self):
"""
There should only ever be one TSC in the system, so
we don't bother passing args into the hash.
"""
return self.__class__.__name__ + hash_args()
def __init__(self, algo, sim_params):
# ==============
# Simulation
# Param Setup
# ==============
self.sim_params = sim_params
# ==============
# Algo Setup
# ==============
self.algo = algo
self.algo_start = self.sim_params.first_open
self.algo_start = self.algo_start.replace(hour=0, minute=0, second=0, microsecond=0)
# ==============
# Snapshot Setup
# ==============
# The algorithm's data as of our most recent event.
# We want an object that will have empty objects as default
# values on missing keys.
self.current_data = BarData()
# We don't have a datetime for the current snapshot until we
# receive a message.
self.simulation_dt = None
# =============
# Logging Setup
# =============
# Processor function for injecting the algo_dt into
# user prints/logs.
def inject_algo_dt(record):
if not "algo_dt" in record.extra:
record.extra["algo_dt"] = self.simulation_dt
self.processor = Processor(inject_algo_dt)
@property
def perf_key(self):
return self.EMISSION_TO_PERF_KEY_MAP[self.algo.perf_tracker.emission_rate]
def transform(self, stream_in):
"""
Main generator work loop.
"""
# Initialize the mkt_close
mkt_close = self.algo.perf_tracker.market_close
# inject the current algo
# snapshot time to any log record generated.
with self.processor.threadbound():
updated = False
bm_updated = False
for date, snapshot in stream_in:
self.algo.set_datetime(date)
self.simulation_dt = date
self.algo.perf_tracker.set_date(date)
self.algo.blotter.set_date(date)
# If we're still in the warmup period. Use the event to
# update our universe, but don't yield any perf messages,
# and don't send a snapshot to handle_data.
if date < self.algo_start:
for event in snapshot:
if event.type in (DATASOURCE_TYPE.TRADE, DATASOURCE_TYPE.CUSTOM):
self.update_universe(event)
self.algo.perf_tracker.process_event(event)
else:
for event in snapshot:
if event.type in (DATASOURCE_TYPE.TRADE, DATASOURCE_TYPE.CUSTOM):
self.update_universe(event)
updated = True
if event.type == DATASOURCE_TYPE.BENCHMARK:
self.algo.set_datetime(event.dt)
bm_updated = True
txns, orders = self.algo.blotter.process_trade(event)
for data in chain(txns, orders, [event]):
self.algo.perf_tracker.process_event(data)
# Update our portfolio.
self.algo.set_portfolio(self.algo.perf_tracker.get_portfolio())
# Send the current state of the universe
# to the user's algo.
if updated:
self.algo.handle_data(self.current_data)
updated = False
# run orders placed in the algorithm call
# above through perf tracker before emitting
# the perf packet, so that the perf includes
# placed orders
for order in self.algo.blotter.new_orders:
self.algo.perf_tracker.process_event(order)
self.algo.blotter.new_orders = []
# The benchmark is our internal clock. When it
# updates, we need to emit a performance message.
if bm_updated:
bm_updated = False
yield self.get_message(date)
# When emitting minutely, we re-iterate the day as a
# packet with the entire days performance rolled up.
if self.algo.perf_tracker.emission_rate == "minute":
if date == mkt_close:
daily_rollup = self.algo.perf_tracker.to_dict(emission_type="daily")
daily_rollup["daily_perf"]["recorded_vars"] = self.algo.recorded_vars
yield daily_rollup
tp = self.algo.perf_tracker.todays_performance
tp.rollover()
if mkt_close < self.algo.perf_tracker.last_close:
mkt_close = self.get_next_close(mkt_close)
self.algo.perf_tracker.handle_intraday_close()
risk_message = self.algo.perf_tracker.handle_simulation_end()
yield risk_message
def get_message(self, date):
rvars = self.algo.recorded_vars
if self.algo.perf_tracker.emission_rate == "daily":
perf_message = self.algo.perf_tracker.handle_market_close()
perf_message["daily_perf"]["recorded_vars"] = rvars
return perf_message
elif self.algo.perf_tracker.emission_rate == "minute":
self.algo.perf_tracker.handle_minute_close(date)
perf_message = self.algo.perf_tracker.to_dict()
perf_message["minute_perf"]["recorded_vars"] = rvars
return perf_message
def get_next_close(self, mkt_close):
if mkt_close >= trading.environment.last_trading_day:
return self.sim_params.last_close
else:
return trading.environment.next_open_and_close(mkt_close)[1]
def update_universe(self, event):
"""
Update the universe with new event information.
"""
# Update our knowledge of this event's sid
sid_data = self.current_data[event.sid]
sid_data.__dict__.update(event.__dict__)
class AlgorithmSimulator(object):
EMISSION_TO_PERF_KEY_MAP = {
'minute': 'minute_perf',
'daily': 'daily_perf'
}
def get_hash(self):
"""
There should only ever be one TSC in the system, so
we don't bother passing args into the hash.
"""
return self.__class__.__name__ + hash_args()
def __init__(self, algo, sim_params):
# ==============
# Simulation
# Param Setup
# ==============
self.sim_params = sim_params
# ==============
# Algo Setup
# ==============
self.algo = algo
self.algo_start = self.sim_params.first_open
self.algo_start = self.algo_start.replace(hour=0, minute=0,
second=0,
microsecond=0)
# ==============
# Snapshot Setup
# ==============
# The algorithm's data as of our most recent event.
# We want an object that will have empty objects as default
# values on missing keys.
self.current_data = BarData()
# We don't have a datetime for the current snapshot until we
# receive a message.
self.simulation_dt = None
# =============
# Logging Setup
# =============
# Processor function for injecting the algo_dt into
# user prints/logs.
def inject_algo_dt(record):
if 'algo_dt' not in record.extra:
record.extra['algo_dt'] = self.simulation_dt
self.processor = Processor(inject_algo_dt)
@property
def perf_key(self):
return self.EMISSION_TO_PERF_KEY_MAP[
self.algo.perf_tracker.emission_rate]
def process_event(self, event):
process_trade = self.algo.blotter.process_trade
for txn, order in process_trade(event):
self.algo.perf_tracker.process_event(txn)
self.algo.perf_tracker.process_event(order)
self.algo.perf_tracker.process_event(event)
def transform(self, stream_in):
"""
Main generator work loop.
"""
# Initialize the mkt_close
mkt_open = self.algo.perf_tracker.market_open
mkt_close = self.algo.perf_tracker.market_close
# inject the current algo
# snapshot time to any log record generated.
with self.processor.threadbound():
for date, snapshot in stream_in:
self.simulation_dt = date
self.algo.on_dt_changed(date)
# If we're still in the warmup period. Use the event to
# update our universe, but don't yield any perf messages,
# and don't send a snapshot to handle_data.
if date < self.algo_start:
for event in snapshot:
if event.type == DATASOURCE_TYPE.SPLIT:
self.algo.blotter.process_split(event)
if event.type in (DATASOURCE_TYPE.TRADE,
DATASOURCE_TYPE.CUSTOM):
self.update_universe(event)
self.algo.perf_tracker.process_event(event)
else:
message = self._process_snapshot(
date,
snapshot,
self.algo.instant_fill,
)
# Perf messages are only emitted if the snapshot contained
# a benchmark event.
if message is not None:
yield message
# When emitting minutely, we re-iterate the day as a
# packet with the entire days performance rolled up.
if self.algo.perf_tracker.emission_rate == 'minute':
if date == mkt_close:
daily_rollup = self.algo.perf_tracker.to_dict(
emission_type='daily'
)
daily_rollup['daily_perf']['recorded_vars'] = \
self.algo.recorded_vars
yield daily_rollup
tp = self.algo.perf_tracker.todays_performance
tp.rollover()
if mkt_close <= self.algo.perf_tracker.last_close:
try:
mkt_open, mkt_close = \
trading.environment \
.next_open_and_close(mkt_close)
except trading.NoFurtherDataError:
# If at the end of backtest history,
# skip advancing market close.
pass
self.algo.perf_tracker\
.handle_intraday_market_close(
mkt_open,
mkt_close)
self.algo.portfolio_needs_update = True
risk_message = self.algo.perf_tracker.handle_simulation_end()
yield risk_message
def _process_snapshot(self, dt, snapshot, instant_fill):
"""
Process a stream of events corresponding to a single datetime, possibly
returning a perf message to be yielded.
If @instant_fill = True, we delay processing of events until after the
user's call to handle_data, and we process the user's placed orders
before the snapshot's events. Note that this introduces a lookahead
bias, since the user effectively is effectively placing orders that are
filled based on trades that happened prior to the call the handle_data.
If @instant_fill = False, we process Trade events before calling
handle_data. This means that orders are filled based on trades
occurring in the next snapshot. This is the more conservative model,
and as such it is the default behavior in TradingAlgorithm.
"""
# Flags indicating whether we saw any events of type TRADE and type
# BENCHMARK. Respectively, these control whether or not handle_data is
# called for this snapshot and whether we emit a perf message for this
# snapshot.
any_trade_occurred = False
benchmark_event_occurred = False
if instant_fill:
events_to_be_processed = []
for event in snapshot:
if event.type == DATASOURCE_TYPE.TRADE:
self.update_universe(event)
any_trade_occurred = True
elif event.type == DATASOURCE_TYPE.BENCHMARK:
benchmark_event_occurred = True
elif event.type == DATASOURCE_TYPE.CUSTOM:
self.update_universe(event)
elif event.type == DATASOURCE_TYPE.SPLIT:
self.algo.blotter.process_split(event)
if not self.algo.instant_fill:
self.process_event(event)
else:
events_to_be_processed.append(event)
if any_trade_occurred:
new_orders = self._call_handle_data()
for order in new_orders:
self.algo.perf_tracker.process_event(order)
if instant_fill:
# Now that handle_data has been called and orders have been placed,
# process the event stream to fill user orders based on the events
# from this snapshot.
for event in events_to_be_processed:
self.process_event(event)
if benchmark_event_occurred:
self.algo.updated_portfolio()
return self.get_message(dt)
else:
return None
def _call_handle_data(self):
"""
Call the user's handle_data, returning any orders placed by the algo
during the call.
"""
self.algo.handle_data(self.current_data)
orders = self.algo.blotter.new_orders
self.algo.blotter.new_orders = []
return orders
def get_message(self, dt):
"""
Get a perf message for the given datetime.
"""
rvars = self.algo.recorded_vars
if self.algo.perf_tracker.emission_rate == 'daily':
perf_message = \
self.algo.perf_tracker.handle_market_close_daily()
perf_message['daily_perf']['recorded_vars'] = rvars
return perf_message
elif self.algo.perf_tracker.emission_rate == 'minute':
self.algo.perf_tracker.handle_minute_close(dt)
perf_message = self.algo.perf_tracker.to_dict()
perf_message['minute_perf']['recorded_vars'] = rvars
return perf_message
def update_universe(self, event):
"""
Update the universe with new event information.
"""
# Update our knowledge of this event's sid
# rather than use if event.sid in ..., just trying
# and handling the exception is significantly faster
try:
sid_data = self.current_data[event.sid]
except KeyError:
sid_data = self.current_data[event.sid] = SIDData()
sid_data.__dict__.update(event.__dict__)
class AlgorithmSimulator(object):
EMISSION_TO_PERF_KEY_MAP = {
'minute': 'minute_perf',
'daily': 'daily_perf'
}
def __init__(self, algo, sim_params):
# ==============
# Simulation
# Param Setup
# ==============
self.sim_params = sim_params
# ==============
# Algo Setup
# ==============
self.algo = algo
self.algo_start = normalize_date(self.sim_params.first_open)
# ==============
# Snapshot Setup
# ==============
# The algorithm's data as of our most recent event.
# We want an object that will have empty objects as default
# values on missing keys.
self.current_data = BarData()
# We don't have a datetime for the current snapshot until we
# receive a message.
self.simulation_dt = None
# =============
# Logging Setup
# =============
# Processor function for injecting the algo_dt into
# user prints/logs.
def inject_algo_dt(record):
if 'algo_dt' not in record.extra:
record.extra['algo_dt'] = self.simulation_dt
self.processor = Processor(inject_algo_dt)
def transform(self, stream_in):
"""
Main generator work loop.
"""
# Initialize the mkt_close
mkt_open = self.algo.perf_tracker.market_open
mkt_close = self.algo.perf_tracker.market_close
# inject the current algo
# snapshot time to any log record generated.
with ExitStack() as stack:
stack.enter_context(self.processor.threadbound())
stack.enter_context(ZiplineAPI(self.algo))
data_frequency = self.sim_params.data_frequency
self._call_before_trading_start(mkt_open)
for date, snapshot in stream_in:
self.simulation_dt = date
self.on_dt_changed(date)
# If we're still in the warmup period. Use the event to
# update our universe, but don't yield any perf messages,
# and don't send a snapshot to handle_data.
if date < self.algo_start:
for event in snapshot:
if event.type == DATASOURCE_TYPE.SPLIT:
self.algo.blotter.process_split(event)
elif event.type == DATASOURCE_TYPE.TRADE:
self.update_universe(event)
self.algo.perf_tracker.process_trade(event)
elif event.type == DATASOURCE_TYPE.CUSTOM:
self.update_universe(event)
else:
messages = self._process_snapshot(
date,
snapshot,
self.algo.instant_fill,
)
# Perf messages are only emitted if the snapshot contained
# a benchmark event.
for message in messages:
yield message
# When emitting minutely, we need to call
# before_trading_start before the next trading day begins
if date == mkt_close:
if mkt_close <= self.algo.perf_tracker.last_close:
before_last_close = \
mkt_close < self.algo.perf_tracker.last_close
try:
mkt_open, mkt_close = \
trading.environment \
.next_open_and_close(mkt_close)
except trading.NoFurtherDataError:
# If at the end of backtest history,
# skip advancing market close.
pass
if before_last_close:
self._call_before_trading_start(mkt_open)
elif data_frequency == 'daily':
next_day = trading.environment.next_trading_day(date)
if next_day is not None and \
next_day < self.algo.perf_tracker.last_close:
self._call_before_trading_start(next_day)
self.algo.portfolio_needs_update = True
self.algo.account_needs_update = True
self.algo.performance_needs_update = True
risk_message = self.algo.perf_tracker.handle_simulation_end()
yield risk_message
def _process_snapshot(self, dt, snapshot, instant_fill):
"""
Process a stream of events corresponding to a single datetime, possibly
returning a perf message to be yielded.
If @instant_fill = True, we delay processing of events until after the
user's call to handle_data, and we process the user's placed orders
before the snapshot's events. Note that this introduces a lookahead
bias, since the user effectively is effectively placing orders that are
filled based on trades that happened prior to the call the handle_data.
If @instant_fill = False, we process Trade events before calling
handle_data. This means that orders are filled based on trades
occurring in the next snapshot. This is the more conservative model,
and as such it is the default behavior in TradingAlgorithm.
"""
# Flags indicating whether we saw any events of type TRADE and type
# BENCHMARK. Respectively, these control whether or not handle_data is
# called for this snapshot and whether we emit a perf message for this
# snapshot.
any_trade_occurred = False
benchmark_event_occurred = False
if instant_fill:
events_to_be_processed = []
# Assign process events to variables to avoid attribute access in
# innermost loops.
#
# Done here, to allow for perf_tracker or blotter to be swapped out
# or changed in between snapshots.
perf_process_trade = self.algo.perf_tracker.process_trade
perf_process_transaction = self.algo.perf_tracker.process_transaction
perf_process_order = self.algo.perf_tracker.process_order
perf_process_benchmark = self.algo.perf_tracker.process_benchmark
perf_process_split = self.algo.perf_tracker.process_split
perf_process_dividend = self.algo.perf_tracker.process_dividend
perf_process_commission = self.algo.perf_tracker.process_commission
perf_process_close_position = \
self.algo.perf_tracker.process_close_position
blotter_process_trade = self.algo.blotter.process_trade
blotter_process_benchmark = self.algo.blotter.process_benchmark
# Containers for the snapshotted events, so that the events are
# processed in a predictable order, without relying on the sorted order
# of the individual sources.
# There is only one benchmark per snapshot, will be set to the current
# benchmark iff it occurs.
benchmark = None
# trades and customs are initialized as a list since process_snapshot
# is most often called on market bars, which could contain trades or
# custom events.
trades = []
customs = []
closes = []
# splits and dividends are processed once a day.
#
# The avoidance of creating the list every time this is called is more
# to attempt to show that this is the infrequent case of the method,
# since the performance benefit from deferring the list allocation is
# marginal. splits list will be allocated when a split occurs in the
# snapshot.
splits = None
# dividends list will be allocated when a dividend occurs in the
# snapshot.
dividends = None
for event in snapshot:
if event.type == DATASOURCE_TYPE.TRADE:
trades.append(event)
elif event.type == DATASOURCE_TYPE.BENCHMARK:
benchmark = event
elif event.type == DATASOURCE_TYPE.SPLIT:
if splits is None:
splits = []
splits.append(event)
elif event.type == DATASOURCE_TYPE.CUSTOM:
customs.append(event)
elif event.type == DATASOURCE_TYPE.DIVIDEND:
if dividends is None:
dividends = []
dividends.append(event)
elif event.type == DATASOURCE_TYPE.CLOSE_POSITION:
closes.append(event)
else:
raise log.warn("Unrecognized event=%s".format(event))
# Handle benchmark first.
#
# Internal broker implementation depends on the benchmark being
# processed first so that transactions and commissions reported from
# the broker can be injected.
if benchmark is not None:
benchmark_event_occurred = True
perf_process_benchmark(benchmark)
for txn, order in blotter_process_benchmark(benchmark):
if txn.type == DATASOURCE_TYPE.TRANSACTION:
perf_process_transaction(txn)
elif txn.type == DATASOURCE_TYPE.COMMISSION:
perf_process_commission(txn)
perf_process_order(order)
for trade in trades:
self.update_universe(trade)
any_trade_occurred = True
if instant_fill:
events_to_be_processed.append(trade)
else:
for txn, order in blotter_process_trade(trade):
if txn.type == DATASOURCE_TYPE.TRANSACTION:
perf_process_transaction(txn)
elif txn.type == DATASOURCE_TYPE.COMMISSION:
perf_process_commission(txn)
perf_process_order(order)
perf_process_trade(trade)
for custom in customs:
self.update_universe(custom)
for close in closes:
self.update_universe(close)
perf_process_close_position(close)
if splits is not None:
for split in splits:
# process_split is not assigned to a variable since it is
# called rarely compared to the other event processors.
self.algo.blotter.process_split(split)
perf_process_split(split)
if dividends is not None:
for dividend in dividends:
perf_process_dividend(dividend)
if any_trade_occurred:
new_orders = self._call_handle_data()
for order in new_orders:
perf_process_order(order)
if instant_fill:
# Now that handle_data has been called and orders have been placed,
# process the event stream to fill user orders based on the events
# from this snapshot.
for trade in events_to_be_processed:
for txn, order in blotter_process_trade(trade):
if txn is not None:
perf_process_transaction(txn)
if order is not None:
perf_process_order(order)
perf_process_trade(trade)
if benchmark_event_occurred:
return self.generate_messages(dt)
else:
return ()
def _call_handle_data(self):
"""
Call the user's handle_data, returning any orders placed by the algo
during the call.
"""
self.algo.event_manager.handle_data(
self.algo,
self.current_data,
self.simulation_dt,
)
orders = self.algo.blotter.new_orders
self.algo.blotter.new_orders = []
return orders
def _call_before_trading_start(self, dt):
dt = normalize_date(dt)
self.simulation_dt = dt
self.on_dt_changed(dt)
self.algo.before_trading_start()
def on_dt_changed(self, dt):
if self.algo.datetime != dt:
self.algo.on_dt_changed(dt)
def generate_messages(self, dt):
"""
Generator that yields perf messages for the given datetime.
"""
# Ensure that updated_portfolio has been called at least once for this
# dt before we emit a perf message. This is a no-op if
# updated_portfolio has already been called this dt.
self.algo.updated_portfolio()
self.algo.updated_account()
rvars = self.algo.recorded_vars
if self.algo.perf_tracker.emission_rate == 'daily':
perf_message = \
self.algo.perf_tracker.handle_market_close_daily()
perf_message['daily_perf']['recorded_vars'] = rvars
yield perf_message
elif self.algo.perf_tracker.emission_rate == 'minute':
# close the minute in the tracker, and collect the daily message if
# the minute is the close of the trading day
minute_message, daily_message = \
self.algo.perf_tracker.handle_minute_close(dt)
# collect and yield the minute's perf message
minute_message['minute_perf']['recorded_vars'] = rvars
yield minute_message
# if there was a daily perf message, collect and yield it
if daily_message:
daily_message['daily_perf']['recorded_vars'] = rvars
yield daily_message
def update_universe(self, event):
"""
Update the universe with new event information.
"""
# Update our knowledge of this event's sid
# rather than use if event.sid in ..., just trying
# and handling the exception is significantly faster
try:
sid_data = self.current_data[event.sid]
except KeyError:
sid_data = self.current_data[event.sid] = SIDData(event.sid)
sid_data.__dict__.update(event.__dict__)
class AlgorithmSimulator(object):
EMISSION_TO_PERF_KEY_MAP = {
'minute': 'minute_perf',
'daily': 'daily_perf'
}
def get_hash(self):
"""
There should only ever be one TSC in the system, so
we don't bother passing args into the hash.
"""
return self.__class__.__name__ + hash_args()
def __init__(self, algo, sim_params):
# ==============
# Simulation
# Param Setup
# ==============
self.sim_params = sim_params
# ==============
# Algo Setup
# ==============
self.algo = algo
self.algo_start = self.sim_params.first_open
self.algo_start = self.algo_start.replace(hour=0, minute=0,
second=0,
microsecond=0)
# ==============
# Snapshot Setup
# ==============
# The algorithm's data as of our most recent event.
# We want an object that will have empty objects as default
# values on missing keys.
self.current_data = BarData()
# We don't have a datetime for the current snapshot until we
# receive a message.
self.simulation_dt = None
# =============
# Logging Setup
# =============
# Processor function for injecting the algo_dt into
# user prints/logs.
def inject_algo_dt(record):
if not 'algo_dt' in record.extra:
record.extra['algo_dt'] = self.simulation_dt
self.processor = Processor(inject_algo_dt)
@property
def perf_key(self):
return self.EMISSION_TO_PERF_KEY_MAP[
self.algo.perf_tracker.emission_rate]
def process_event(self, event):
process_trade = self.algo.blotter.process_trade
for txn, order in process_trade(event):
self.algo.perf_tracker.process_event(txn)
self.algo.perf_tracker.process_event(order)
self.algo.perf_tracker.process_event(event)
def transform(self, stream_in):
"""
Main generator work loop.
"""
# Initialize the mkt_close
mkt_close = self.algo.perf_tracker.market_close
# inject the current algo
# snapshot time to any log record generated.
with self.processor.threadbound():
updated = False
bm_updated = False
for date, snapshot in stream_in:
self.algo.set_datetime(date)
self.simulation_dt = date
self.algo.perf_tracker.set_date(date)
self.algo.blotter.set_date(date)
# If we're still in the warmup period. Use the event to
# update our universe, but don't yield any perf messages,
# and don't send a snapshot to handle_data.
if date < self.algo_start:
for event in snapshot:
if event.type == DATASOURCE_TYPE.SPLIT:
self.algo.blotter.process_split(event)
if event.type in (DATASOURCE_TYPE.TRADE,
DATASOURCE_TYPE.CUSTOM):
self.update_universe(event)
self.algo.perf_tracker.process_event(event)
else:
if self.algo.instant_fill:
events = []
for event in snapshot:
if event.type == DATASOURCE_TYPE.TRADE:
self.update_universe(event)
updated = True
elif event.type == DATASOURCE_TYPE.BENCHMARK:
self.algo.set_datetime(event.dt)
bm_updated = True
elif event.type == DATASOURCE_TYPE.CUSTOM:
self.update_universe(event)
updated = True
elif event.type == DATASOURCE_TYPE.SPLIT:
self.algo.blotter.process_split(event)
# If we are instantly filling orders we process
# them after handle_data().
if not self.algo.instant_fill:
self.process_event(event)
else:
events.append(event)
# Send the current state of the universe
# to the user's algo.
if updated:
self.algo.handle_data(self.current_data)
updated = False
# run orders placed in the algorithm call
# above through perf tracker before emitting
# the perf packet, so that the perf includes
# placed orders
for order in self.algo.blotter.new_orders:
self.algo.perf_tracker.process_event(order)
self.algo.blotter.new_orders = []
# If we are instantly filling we execute orders
# in this iteration rather than the next.
if self.algo.instant_fill:
for event in events:
self.process_event(event)
# The benchmark is our internal clock. When it
# updates, we need to emit a performance message.
if bm_updated:
bm_updated = False
self.algo.updated_portfolio()
yield self.get_message(date)
# When emitting minutely, we re-iterate the day as a
# packet with the entire days performance rolled up.
if self.algo.perf_tracker.emission_rate == 'minute':
if date == mkt_close:
daily_rollup = self.algo.perf_tracker.to_dict(
emission_type='daily'
)
daily_rollup['daily_perf']['recorded_vars'] = \
self.algo.recorded_vars
yield daily_rollup
tp = self.algo.perf_tracker.todays_performance
tp.rollover()
if mkt_close <= self.algo.perf_tracker.last_close:
_, mkt_close = \
trading.environment.next_open_and_close(
mkt_close
)
self.algo.perf_tracker.handle_intraday_close()
self.algo.portfolio_needs_update = True
risk_message = self.algo.perf_tracker.handle_simulation_end()
yield risk_message
def get_message(self, date):
rvars = self.algo.recorded_vars
if self.algo.perf_tracker.emission_rate == 'daily':
perf_message = \
self.algo.perf_tracker.handle_market_close()
perf_message['daily_perf']['recorded_vars'] = rvars
return perf_message
elif self.algo.perf_tracker.emission_rate == 'minute':
self.algo.perf_tracker.handle_minute_close(date)
perf_message = self.algo.perf_tracker.to_dict()
perf_message['minute_perf']['recorded_vars'] = rvars
return perf_message
def update_universe(self, event):
"""
Update the universe with new event information.
"""
# Update our knowledge of this event's sid
# rather than use if event.sid in ..., just trying
# and handling the exception is significantly faster
try:
sid_data = self.current_data[event.sid]
except KeyError:
sid_data = self.current_data[event.sid] = SIDData()
sid_data.__dict__.update(event.__dict__)
class AlgorithmSimulator(object):
def __init__(self,
order_book,
perf_tracker,
algo,
algo_start):
# ==========
# Algo Setup
# ==========
# We extract the order book from the txn client so that
# the algo can place new orders.
self.order_book = order_book
self.perf_tracker = perf_tracker
self.algo = algo
self.algo_start = algo_start.replace(hour=0, minute=0,
second=0,
microsecond=0)
# Monkey patch the user algorithm to place orders in the
# TransactionSimulator's order book and use our logger.
self.algo.set_order(self.order)
# ==============
# Snapshot Setup
# ==============
# The algorithm's universe as of our most recent event.
# We want an ndict that will have empty objects as default
# values on missing keys.
self.universe = ndict(internal=defaultdict(SIDData))
# We don't have a datetime for the current snapshot until we
# receive a message.
self.simulation_dt = None
self.snapshot_dt = None
# =============
# Logging Setup
# =============
# Processor function for injecting the algo_dt into
# user prints/logs.
def inject_algo_dt(record):
if not 'algo_dt' in record.extra:
record.extra['algo_dt'] = self.snapshot_dt
self.processor = Processor(inject_algo_dt)
def order(self, sid, amount, **kwargs ):
# something could be done with amount to further divide
# between buy by share count OR buy shares up to a dollar amount
# numeric == share count AND "$dollar.cents" == cost amount
"""
amount > 0 :: Buy/Cover
amount < 0 :: Sell/Short
Market order: order(sid,amount)
Limit order: order(sid,amount, limit=price)
Stop order: order(sid,amount, stop=price)
StopLimit order: order(sid,amount, limit=price, stop=price)
"""
# just validates amount and passes rest on to TransactionSimulator
# Tell the user if they try to buy 0 shares of something.
if amount == 0:
zero_message = "Requested to trade zero shares of {0}".format(sid)
log.debug(zero_message)
# Don't bother placing orders for 0 shares.
return
stop_price,limit_price = None,None
for name, value in kwargs.items():
if name == "limit":
limit_price = value
if name == "stop":
stop_price = value
order = Order({
'dt': self.simulation_dt,
'sid': sid,
'amount': int(amount),
'filled': 0,
'stop': stop_price,
'limit': limit_price,
'orig_stop': stop_price,
'orig_limit': limit_price
})
# Add non-zero orders to the order book.
# !!!IMPORTANT SIDE-EFFECT!!!
# This modifies the internal state of the transaction
# simulator so that it can fill the placed order when it
# receives its next message.
err_str = self.order_book.place_order(order)
if err_str != None and len(err_str) > 0:
# error, trade was not placed, log it out
log.debug(err_str)
def transform(self, stream_in):
"""
Main generator work loop.
"""
# inject the current algo
# snapshot time to any log record generated.
with self.processor.threadbound():
for date, snapshot in stream_in:
# Set the simulation date to be the first event we see.
# This should only occur once, at the start of the test.
if self.simulation_dt is None:
self.simulation_dt = date
# We're still in the warmup period. Use the event to
# update our universe, but don't yield any perf messages,
# and don't send a snapshot to handle_data.
if date < self.algo_start:
for event in snapshot:
del event['perf_messages']
self.update_universe(event)
# Regular snapshot. Update the universe and send a snapshot
# to handle data.
else:
for event in snapshot:
for perf_message in event.perf_messages:
yield perf_message
del event['perf_messages']
self.update_universe(event)
# Send the current state of the universe
# to the user's algo.
self.simulate_snapshot(date)
perf_messages, risk_message = \
self.perf_tracker.handle_simulation_end()
for message in perf_messages:
yield message
yield risk_message
def update_universe(self, event):
"""
Update the universe with new event information.
"""
# Update our portfolio.
self.algo.set_portfolio(event.portfolio)
# Update our knowledge of this event's sid
sid_data = self.universe[event.sid]
sid_data.__dict__.update(event.__dict__)
def simulate_snapshot(self, date):
"""
Run the user's algo against our current snapshot and update
the algo's simulated time.
"""
# Needs to be set so that we inject the proper date into algo
# log/print lines.
self.snapshot_dt = date
self.algo.set_datetime(self.snapshot_dt)
self.algo.handle_data(self.universe)
# Update the simulation time.
self.simulation_dt = date
class AlgorithmSimulator(object):
EMISSION_TO_PERF_KEY_MAP = {'minute': 'minute_perf', 'daily': 'daily_perf'}
def __init__(self, algo, sim_params):
# ==============
# Simulation
# Param Setup
# ==============
self.sim_params = sim_params
# ==============
# Algo Setup
# ==============
self.algo = algo
self.algo_start = normalize_date(self.sim_params.first_open)
# ==============
# Snapshot Setup
# ==============
# The algorithm's data as of our most recent event.
# We want an object that will have empty objects as default
# values on missing keys.
self.current_data = BarData()
# We don't have a datetime for the current snapshot until we
# receive a message.
self.simulation_dt = None
# =============
# Logging Setup
# =============
# Processor function for injecting the algo_dt into
# user prints/logs.
def inject_algo_dt(record):
if 'algo_dt' not in record.extra:
record.extra['algo_dt'] = self.simulation_dt
self.processor = Processor(inject_algo_dt)
def transform(self, stream_in):
"""
Main generator work loop.
"""
# Initialize the mkt_close
mkt_open = self.algo.perf_tracker.market_open
mkt_close = self.algo.perf_tracker.market_close
# inject the current algo
# snapshot time to any log record generated.
with self.processor.threadbound():
data_frequency = self.sim_params.data_frequency
self._call_before_trading_start(mkt_open)
for date, snapshot in stream_in:
self.simulation_dt = date
self.on_dt_changed(date)
# If we're still in the warmup period. Use the event to
# update our universe, but don't yield any perf messages,
# and don't send a snapshot to handle_data.
if date < self.algo_start:
for event in snapshot:
if event.type == DATASOURCE_TYPE.SPLIT:
self.algo.blotter.process_split(event)
elif event.type == DATASOURCE_TYPE.TRADE:
self.update_universe(event)
self.algo.perf_tracker.process_trade(event)
elif event.type == DATASOURCE_TYPE.CUSTOM:
self.update_universe(event)
else:
message = self._process_snapshot(
date,
snapshot,
self.algo.instant_fill,
)
# Perf messages are only emitted if the snapshot contained
# a benchmark event.
if message is not None:
yield message
# When emitting minutely, we re-iterate the day as a
# packet with the entire days performance rolled up.
if date == mkt_close:
if self.algo.perf_tracker.emission_rate == 'minute':
daily_rollup = self.algo.perf_tracker.to_dict(
emission_type='daily')
daily_rollup['daily_perf']['recorded_vars'] = \
self.algo.recorded_vars
yield daily_rollup
tp = self.algo.perf_tracker.todays_performance
tp.rollover()
if mkt_close <= self.algo.perf_tracker.last_close:
before_last_close = \
mkt_close < self.algo.perf_tracker.last_close
try:
mkt_open, mkt_close = \
trading.environment \
.next_open_and_close(mkt_close)
except trading.NoFurtherDataError:
# If at the end of backtest history,
# skip advancing market close.
pass
if self.algo.perf_tracker.emission_rate == \
'minute':
self.algo.perf_tracker\
.handle_intraday_market_close(
mkt_open,
mkt_close)
if before_last_close:
self._call_before_trading_start(mkt_open)
elif data_frequency == 'daily':
next_day = trading.environment.next_trading_day(date)
if next_day is not None and \
next_day < self.algo.perf_tracker.last_close:
self._call_before_trading_start(next_day)
self.algo.portfolio_needs_update = True
self.algo.account_needs_update = True
self.algo.performance_needs_update = True
risk_message = self.algo.perf_tracker.handle_simulation_end()
yield risk_message
def _process_snapshot(self, dt, snapshot, instant_fill):
"""
Process a stream of events corresponding to a single datetime, possibly
returning a perf message to be yielded.
If @instant_fill = True, we delay processing of events until after the
user's call to handle_data, and we process the user's placed orders
before the snapshot's events. Note that this introduces a lookahead
bias, since the user effectively is effectively placing orders that are
filled based on trades that happened prior to the call the handle_data.
If @instant_fill = False, we process Trade events before calling
handle_data. This means that orders are filled based on trades
occurring in the next snapshot. This is the more conservative model,
and as such it is the default behavior in TradingAlgorithm.
"""
# Flags indicating whether we saw any events of type TRADE and type
# BENCHMARK. Respectively, these control whether or not handle_data is
# called for this snapshot and whether we emit a perf message for this
# snapshot.
any_trade_occurred = False
benchmark_event_occurred = False
if instant_fill:
events_to_be_processed = []
# Assign process events to variables to avoid attribute access in
# innermost loops.
#
# Done here, to allow for perf_tracker or blotter to be swapped out
# or changed in between snapshots.
perf_process_trade = self.algo.perf_tracker.process_trade
perf_process_transaction = self.algo.perf_tracker.process_transaction
perf_process_order = self.algo.perf_tracker.process_order
perf_process_benchmark = self.algo.perf_tracker.process_benchmark
perf_process_split = self.algo.perf_tracker.process_split
perf_process_dividend = self.algo.perf_tracker.process_dividend
perf_process_commission = self.algo.perf_tracker.process_commission
blotter_process_trade = self.algo.blotter.process_trade
blotter_process_benchmark = self.algo.blotter.process_benchmark
# Containers for the snapshotted events, so that the events are
# processed in a predictable order, without relying on the sorted order
# of the individual sources.
# There is only one benchmark per snapshot, will be set to the current
# benchmark iff it occurs.
benchmark = None
# trades and customs are initialized as a list since process_snapshot
# is most often called on market bars, which could contain trades or
# custom events.
trades = []
customs = []
# splits and dividends are processed once a day.
#
# The avoidance of creating the list every time this is called is more
# to attempt to show that this is the infrequent case of the method,
# since the performance benefit from deferring the list allocation is
# marginal. splits list will be allocated when a split occurs in the
# snapshot.
splits = None
# dividends list will be allocated when a dividend occurs in the
# snapshot.
dividends = None
for event in snapshot:
if event.type == DATASOURCE_TYPE.TRADE:
trades.append(event)
elif event.type == DATASOURCE_TYPE.BENCHMARK:
benchmark = event
elif event.type == DATASOURCE_TYPE.SPLIT:
if splits is None:
splits = []
splits.append(event)
elif event.type == DATASOURCE_TYPE.CUSTOM:
customs.append(event)
elif event.type == DATASOURCE_TYPE.DIVIDEND:
if dividends is None:
dividends = []
dividends.append(event)
else:
raise log.warn("Unrecognized event=%s".format(event))
# Handle benchmark first.
#
# Internal broker implementation depends on the benchmark being
# processed first so that transactions and commissions reported from
# the broker can be injected.
if benchmark is not None:
benchmark_event_occurred = True
perf_process_benchmark(benchmark)
for txn, order in blotter_process_benchmark(benchmark):
if txn.type == DATASOURCE_TYPE.TRANSACTION:
perf_process_transaction(txn)
elif txn.type == DATASOURCE_TYPE.COMMISSION:
perf_process_commission(txn)
perf_process_order(order)
for trade in trades:
self.update_universe(trade)
any_trade_occurred = True
if instant_fill:
events_to_be_processed.append(trade)
else:
for txn, order in blotter_process_trade(trade):
if txn.type == DATASOURCE_TYPE.TRANSACTION:
perf_process_transaction(txn)
elif txn.type == DATASOURCE_TYPE.COMMISSION:
perf_process_commission(txn)
perf_process_order(order)
perf_process_trade(trade)
for custom in customs:
self.update_universe(custom)
if splits is not None:
for split in splits:
# process_split is not assigned to a variable since it is
# called rarely compared to the other event processors.
self.algo.blotter.process_split(split)
perf_process_split(split)
if dividends is not None:
for dividend in dividends:
perf_process_dividend(dividend)
if any_trade_occurred:
new_orders = self._call_handle_data()
for order in new_orders:
perf_process_order(order)
if instant_fill:
# Now that handle_data has been called and orders have been placed,
# process the event stream to fill user orders based on the events
# from this snapshot.
for trade in events_to_be_processed:
for txn, order in blotter_process_trade(trade):
if txn is not None:
perf_process_transaction(txn)
if order is not None:
perf_process_order(order)
perf_process_trade(trade)
if benchmark_event_occurred:
return self.get_message(dt)
else:
return None
def _call_handle_data(self):
"""
Call the user's handle_data, returning any orders placed by the algo
during the call.
"""
self.algo.event_manager.handle_data(
self.algo,
self.current_data,
self.simulation_dt,
)
orders = self.algo.blotter.new_orders
self.algo.blotter.new_orders = []
return orders
def _call_before_trading_start(self, dt):
dt = normalize_date(dt)
self.simulation_dt = dt
self.on_dt_changed(dt)
self.algo.before_trading_start()
def on_dt_changed(self, dt):
if self.algo.datetime != dt:
self.algo.on_dt_changed(dt)
def get_message(self, dt):
"""
Get a perf message for the given datetime.
"""
# Ensure that updated_portfolio has been called at least once for this
# dt before we emit a perf message. This is a no-op if
# updated_portfolio has already been called this dt.
self.algo.updated_portfolio()
self.algo.updated_account()
rvars = self.algo.recorded_vars
if self.algo.perf_tracker.emission_rate == 'daily':
perf_message = \
self.algo.perf_tracker.handle_market_close_daily()
perf_message['daily_perf']['recorded_vars'] = rvars
return perf_message
elif self.algo.perf_tracker.emission_rate == 'minute':
self.algo.perf_tracker.handle_minute_close(dt)
perf_message = self.algo.perf_tracker.to_dict()
perf_message['minute_perf']['recorded_vars'] = rvars
return perf_message
def update_universe(self, event):
"""
Update the universe with new event information.
"""
# Update our knowledge of this event's sid
# rather than use if event.sid in ..., just trying
# and handling the exception is significantly faster
try:
sid_data = self.current_data[event.sid]
except KeyError:
sid_data = self.current_data[event.sid] = SIDData(event.sid)
sid_data.__dict__.update(event.__dict__)
class AlgorithmSimulator(object):
def __init__(self, order_book, algo, algo_start):
# ==========
# Algo Setup
# ==========
# We extract the order book from the txn client so that
# the algo can place new orders.
self.order_book = order_book
self.algo = algo
self.algo_start = algo_start.replace(hour=0,
minute=0,
second=0,
microsecond=0)
# Monkey patch the user algorithm to place orders in the
# TransactionSimulator's order book and use our logger.
self.algo.set_order(self.order)
# ==============
# Snapshot Setup
# ==============
# The algorithm's universe as of our most recent event.
# We want an ndict that will have empty ndicts as default
# values on missing keys.
self.universe = ndict(internal=defaultdict(ndict))
# We don't have a datetime for the current snapshot until we
# receive a message.
self.simulation_dt = None
self.snapshot_dt = None
# =============
# Logging Setup
# =============
# Processor function for injecting the algo_dt into
# user prints/logs.
def inject_algo_dt(record):
if not 'algo_dt' in record.extra:
record.extra['algo_dt'] = self.snapshot_dt
self.processor = Processor(inject_algo_dt)
def order(self, sid, amount):
"""
Closure to pass into the user's algo to allow placing orders
into the transaction simulator's dict of open orders.
"""
order = ndict({
'dt': self.simulation_dt,
'sid': sid,
'amount': int(amount),
'filled': 0
})
# Tell the user if they try to buy 0 shares of something.
if order.amount == 0:
zero_message = "Requested to trade zero shares of {sid}".format(
sid=order.sid)
log.debug(zero_message)
# Don't bother placing orders for 0 shares.
return
# Add non-zero orders to the order book.
# !!!IMPORTANT SIDE-EFFECT!!!
# This modifies the internal state of the transaction
# simulator so that it can fill the placed order when it
# receives its next message.
self.order_book.place_order(order)
def transform(self, stream_in):
"""
Main generator work loop.
"""
# inject the current algo
# snapshot time to any log record generated.
with self.processor.threadbound():
for date, snapshot in stream_in:
# Set the simulation date to be the first event we see.
# This should only occur once, at the start of the test.
if self.simulation_dt is None:
self.simulation_dt = date
# Done message has the risk report, so we yield before exiting.
if date == 'DONE':
for event in snapshot:
yield event.perf_message
raise StopIteration
# We're still in the warmup period. Use the event to
# update our universe, but don't yield any perf messages,
# and don't send a snapshot to handle_data.
elif date < self.algo_start:
for event in snapshot:
del event['perf_message']
self.update_universe(event)
# The algo has taken so long to process events that
# its simulated time is later than the event time.
# Update the universe and yield any perf messages
# encountered, but don't call handle_data.
elif date < self.simulation_dt:
for event in snapshot:
# Only yield if we have something interesting to say.
if event.perf_message is not None:
yield event.perf_message
# Delete the message before updating,
# so we don't send it to the user.
del event['perf_message']
self.update_universe(event)
# Regular snapshot. Update the universe and send a snapshot
# to handle data.
else:
for event in snapshot:
# Only yield if we have something interesting to say.
if event.perf_message is not None:
yield event.perf_message
del event['perf_message']
self.update_universe(event)
# Send the current state of the universe
# to the user's algo.
self.simulate_snapshot(date)
def update_universe(self, event):
"""
Update the universe with new event information.
"""
# Update our portfolio.
self.algo.set_portfolio(event.portfolio)
# Update our knowledge of this event's sid
for field in event.keys():
self.universe[event.sid][field] = event[field]
def simulate_snapshot(self, date):
"""
Run the user's algo against our current snapshot and update
the algo's simulated time.
"""
# Needs to be set so that we inject the proper date into algo
# log/print lines.
self.snapshot_dt = date
self.algo.set_datetime(self.snapshot_dt)
start_tic = datetime.now()
self.algo.handle_data(self.universe)
stop_tic = datetime.now()
# How long did you take?
delta = stop_tic - start_tic
# Update the simulation time.
self.simulation_dt = date + delta
class AlgorithmSimulator(object):
def __init__(self, order_book, perf_tracker, algo, algo_start):
# ==========
# Algo Setup
# ==========
# We extract the order book from the txn client so that
# the algo can place new orders.
self.order_book = order_book
self.perf_tracker = perf_tracker
self.algo = algo
self.algo_start = algo_start.replace(hour=0,
minute=0,
second=0,
microsecond=0)
# Monkey patch the user algorithm to place orders in the
# TransactionSimulator's order book and use our logger.
self.algo.set_order(self.order)
# ==============
# Snapshot Setup
# ==============
# The algorithm's universe as of our most recent event.
# We want an ndict that will have empty objects as default
# values on missing keys.
self.universe = ndict(internal=defaultdict(SIDData))
# We don't have a datetime for the current snapshot until we
# receive a message.
self.simulation_dt = None
self.snapshot_dt = None
# =============
# Logging Setup
# =============
# Processor function for injecting the algo_dt into
# user prints/logs.
def inject_algo_dt(record):
if not 'algo_dt' in record.extra:
record.extra['algo_dt'] = self.snapshot_dt
self.processor = Processor(inject_algo_dt)
def order(self, sid, amount, limit_price=None, stop_price=None):
# something could be done with amount to further divide
# between buy by share count OR buy shares up to a dollar amount
# numeric == share count AND "$dollar.cents" == cost amount
"""
amount > 0 :: Buy/Cover
amount < 0 :: Sell/Short
Market order: order(sid,amount)
Limit order: order(sid,amount, limit_price)
Stop order: order(sid,amount, None, stop_price)
StopLimit order: order(sid,amount, limit_price, stop_price)
"""
# just validates amount and passes rest on to TransactionSimulator
# Tell the user if they try to buy 0 shares of something.
if amount == 0:
zero_message = "Requested to trade zero shares of {psid}".format(
psid=sid)
log.debug(zero_message)
# Don't bother placing orders for 0 shares.
return
order = Order(
**{
'dt': self.simulation_dt,
'sid': sid,
'amount': int(amount),
'filled': 0,
'stop': stop_price,
'limit': limit_price
})
# Add non-zero orders to the order book.
# !!!IMPORTANT SIDE-EFFECT!!!
# This modifies the internal state of the transaction
# simulator so that it can fill the placed order when it
# receives its next message.
err_str = self.order_book.place_order(order)
if err_str is not None and len(err_str) > 0:
# error, trade was not placed, log it out
log.debug(err_str)
def transform(self, stream_in):
"""
Main generator work loop.
"""
# Set the simulation date to be the first event we see.
peek_date, peek_snapshot = next(stream_in)
self.simulation_dt = peek_date
# Stitch back together the generator by placing the peeked
# event back in front
stream = itertools.chain([(peek_date, peek_snapshot)], stream_in)
# inject the current algo
# snapshot time to any log record generated.
with self.processor.threadbound():
for date, snapshot in stream:
# We're still in the warmup period. Use the event to
# update our universe, but don't yield any perf messages,
# and don't send a snapshot to handle_data.
if date < self.algo_start:
for event in snapshot:
del event['perf_messages']
self.update_universe(event)
# Regular snapshot. Update the universe and send a snapshot
# to handle data.
else:
for event in snapshot:
for perf_message in event.perf_messages:
# append current values of recorded vars
# to emitted message
perf_message['daily_perf']['recorded_vars'] =\
self.algo.recorded_vars
yield perf_message
del event['perf_messages']
self.update_universe(event)
# Send the current state of the universe
# to the user's algo.
self.simulate_snapshot(date)
perf_messages, risk_message = \
self.perf_tracker.handle_simulation_end()
for message in perf_messages:
message['daily_perf']['recorded_vars'] =\
self.algo.recorded_vars
yield message
yield risk_message
def update_universe(self, event):
"""
Update the universe with new event information.
"""
# Update our portfolio.
self.algo.set_portfolio(event.portfolio)
# the portfolio is modified by each event passed into the
# performance tracker (prices and amounts can change).
# Performance tracker sends back an up-to-date portfolio
# with each event. However, we provide the portfolio to
# the algorithm via a setter method, rather than as part
# of the event data sent to handle_data. To avoid
# confusion, we remove it from the event here.
del event.portfolio
# Update our knowledge of this event's sid
sid_data = self.universe[event.sid]
sid_data.__dict__.update(event.__dict__)
def simulate_snapshot(self, date):
"""
Run the user's algo against our current snapshot and update
the algo's simulated time.
"""
# Needs to be set so that we inject the proper date into algo
# log/print lines.
self.snapshot_dt = date
self.algo.set_datetime(self.snapshot_dt)
self.algo.handle_data(self.universe)
# Update the simulation time.
self.simulation_dt = date
class AlgorithmSimulator(object):
EMISSION_TO_PERF_KEY_MAP = {
'minute': 'intraday_perf',
'daily': 'daily_perf'
}
def get_hash(self):
"""
There should only ever be one TSC in the system, so
we don't bother passing args into the hash.
"""
return self.__class__.__name__ + hash_args()
def __init__(self, algo, sim_params):
# ==============
# Simulation
# Param Setup
# ==============
self.sim_params = sim_params
# ==============
# Perf Tracker
# Setup
# ==============
self.perf_tracker = PerformanceTracker(self.sim_params)
self.perf_key = self.EMISSION_TO_PERF_KEY_MAP[
self.perf_tracker.emission_rate]
# ==============
# Algo Setup
# ==============
self.algo = algo
self.algo_start = self.sim_params.first_open
self.algo_start = self.algo_start.replace(hour=0, minute=0,
second=0,
microsecond=0)
# ==============
# Snapshot Setup
# ==============
# The algorithm's data as of our most recent event.
# We want an object that will have empty objects as default
# values on missing keys.
self.current_data = BarData()
# We don't have a datetime for the current snapshot until we
# receive a message.
self.simulation_dt = None
self.snapshot_dt = None
# =============
# Logging Setup
# =============
# Processor function for injecting the algo_dt into
# user prints/logs.
def inject_algo_dt(record):
if not 'algo_dt' in record.extra:
record.extra['algo_dt'] = self.snapshot_dt
self.processor = Processor(inject_algo_dt)
def transform(self, stream_in):
"""
Main generator work loop.
"""
# Set the simulation date to be the first event we see.
peek_date, peek_snapshot = next(stream_in)
self.simulation_dt = peek_date
# Stitch back together the generator by placing the peeked
# event back in front
stream = itertools.chain([(peek_date, peek_snapshot)],
stream_in)
# inject the current algo
# snapshot time to any log record generated.
with self.processor.threadbound():
updated = False
bm_updated = False
for date, snapshot in stream:
self.perf_tracker.set_date(date)
self.algo.blotter.set_date(date)
# If we're still in the warmup period. Use the event to
# update our universe, but don't yield any perf messages,
# and don't send a snapshot to handle_data.
if date < self.algo_start:
for event in snapshot:
if event.type in (DATASOURCE_TYPE.TRADE,
DATASOURCE_TYPE.CUSTOM):
self.update_universe(event)
self.perf_tracker.process_event(event)
else:
for event in snapshot:
if event.type in (DATASOURCE_TYPE.TRADE,
DATASOURCE_TYPE.CUSTOM):
self.update_universe(event)
updated = True
if event.type == DATASOURCE_TYPE.BENCHMARK:
bm_updated = True
txns, orders = self.algo.blotter.process_trade(event)
for data in chain([event], txns, orders):
self.perf_tracker.process_event(data)
# Update our portfolio.
self.algo.set_portfolio(self.perf_tracker.get_portfolio())
# Send the current state of the universe
# to the user's algo.
if updated:
self.simulate_snapshot(date)
updated = False
# run orders placed in the algorithm call
# above through perf tracker before emitting
# the perf packet, so that the perf includes
# placed orders
for order in self.algo.blotter.new_orders:
self.perf_tracker.process_event(order)
self.algo.blotter.new_orders = []
# The benchmark is our internal clock. When it
# updates, we need to emit a performance message.
if bm_updated:
bm_updated = False
yield self.get_message(date)
risk_message = self.perf_tracker.handle_simulation_end()
# When emitting minutely, it is still useful to have a final
# packet with the entire days performance rolled up.
if self.perf_tracker.emission_rate == 'minute':
daily_rollup = self.perf_tracker.to_dict(
emission_type='daily'
)
daily_rollup['daily_perf']['recorded_vars'] = \
self.algo.recorded_vars
yield daily_rollup
yield risk_message
def get_message(self, date):
rvars = self.algo.recorded_vars
if self.perf_tracker.emission_rate == 'daily':
perf_message = \
self.perf_tracker.handle_market_close()
perf_message['daily_perf']['recorded_vars'] = rvars
return perf_message
elif self.perf_tracker.emission_rate == 'minute':
self.perf_tracker.handle_minute_close(date)
perf_message = self.perf_tracker.to_dict()
perf_message['intraday_perf']['recorded_vars'] = rvars
return perf_message
def update_universe(self, event):
"""
Update the universe with new event information.
"""
# Update our knowledge of this event's sid
sid_data = self.current_data[event.sid]
sid_data.__dict__.update(event.__dict__)
def simulate_snapshot(self, date):
"""
Run the user's algo against our current snapshot and update
the algo's simulated time.
"""
# Needs to be set so that we inject the proper date into algo
# log/print lines.
self.snapshot_dt = date
self.algo.set_datetime(self.snapshot_dt)
# Update the simulation time.
self.simulation_dt = date
self.algo.handle_data(self.current_data)
class AlgorithmSimulator(object):
EMISSION_TO_PERF_KEY_MAP = {
'minute': 'intraday_perf',
'daily': 'daily_perf'
}
def __init__(self,
blotter,
perf_tracker,
algo,
algo_start):
# ==========
# Algo Setup
# ==========
# We extract the order book from the txn client so that
# the algo can place new orders.
self.blotter = blotter
self.perf_tracker = perf_tracker
self.perf_key = self.EMISSION_TO_PERF_KEY_MAP[
perf_tracker.emission_rate]
self.algo = algo
self.algo_start = algo_start.replace(hour=0, minute=0,
second=0,
microsecond=0)
# Monkey patch the user algorithm to place orders in the
# TransactionSimulator's order book and use our logger.
self.algo.set_order(self.order)
# ==============
# Snapshot Setup
# ==============
# The algorithm's universe as of our most recent event.
# We want an ndict that will have empty objects as default
# values on missing keys.
self.universe = ndict(internal=defaultdict(SIDData))
# We don't have a datetime for the current snapshot until we
# receive a message.
self.simulation_dt = None
self.snapshot_dt = None
# =============
# Logging Setup
# =============
# Processor function for injecting the algo_dt into
# user prints/logs.
def inject_algo_dt(record):
if not 'algo_dt' in record.extra:
record.extra['algo_dt'] = self.snapshot_dt
self.processor = Processor(inject_algo_dt)
def order(self, sid, amount, limit_price=None, stop_price=None):
# something could be done with amount to further divide
# between buy by share count OR buy shares up to a dollar amount
# numeric == share count AND "$dollar.cents" == cost amount
"""
amount > 0 :: Buy/Cover
amount < 0 :: Sell/Short
Market order: order(sid,amount)
Limit order: order(sid,amount, limit_price)
Stop order: order(sid,amount, None, stop_price)
StopLimit order: order(sid,amount, limit_price, stop_price)
"""
# just validates amount and passes rest on to TransactionSimulator
# Tell the user if they try to buy 0 shares of something.
if amount == 0:
zero_message = "Requested to trade zero shares of {psid}".format(
psid=sid
)
log.debug(zero_message)
# Don't bother placing orders for 0 shares.
return
order = Order(**{
'dt': self.simulation_dt,
'sid': sid,
'amount': int(amount),
'filled': 0,
'stop': stop_price,
'limit': limit_price
})
# Add non-zero orders to the order book.
# !!!IMPORTANT SIDE-EFFECT!!!
# This modifies the internal state of the blotter
# so that it can fill the placed order when it
# receives its next message.
self.blotter.place_order(order)
def transform(self, stream_in):
"""
Main generator work loop.
"""
# Set the simulation date to be the first event we see.
peek_date, peek_snapshot = next(stream_in)
self.simulation_dt = peek_date
# Stitch back together the generator by placing the peeked
# event back in front
stream = itertools.chain([(peek_date, peek_snapshot)],
stream_in)
# inject the current algo
# snapshot time to any log record generated.
with self.processor.threadbound():
for date, snapshot in stream:
# We're still in the warmup period. Use the event to
# update our universe, but don't yield any perf messages,
# and don't send a snapshot to handle_data.
if date < self.algo_start:
for event in snapshot:
del event['perf_messages']
self.update_universe(event)
# Regular snapshot. Update the universe and send a snapshot
# to handle data.
else:
for event in snapshot:
for perf_message in event.perf_messages:
# append current values of recorded vars
# to emitted message
perf_message[self.perf_key]['recorded_vars'] =\
self.algo.recorded_vars
yield perf_message
del event['perf_messages']
self.update_universe(event)
# Send the current state of the universe
# to the user's algo.
self.simulate_snapshot(date)
perf_messages, risk_message = \
self.perf_tracker.handle_simulation_end()
if self.perf_tracker.emission_rate == 'daily':
for message in perf_messages:
message[self.perf_key]['recorded_vars'] =\
self.algo.recorded_vars
yield message
# When emitting minutely, it is still useful to have a final
# packet with the entire days performance rolled up.
if self.perf_tracker.emission_rate == 'minute':
daily_rollup = self.perf_tracker.to_dict(
emission_type='daily'
)
daily_rollup['daily_perf']['recorded_vars'] = \
self.algo.recorded_vars
yield daily_rollup
yield risk_message
def update_universe(self, event):
"""
Update the universe with new event information.
"""
# Update our portfolio.
self.algo.set_portfolio(event.portfolio)
# the portfolio is modified by each event passed into the
# performance tracker (prices and amounts can change).
# Performance tracker sends back an up-to-date portfolio
# with each event. However, we provide the portfolio to
# the algorithm via a setter method, rather than as part
# of the event data sent to handle_data. To avoid
# confusion, we remove it from the event here.
del event.portfolio
# Update our knowledge of this event's sid
sid_data = self.universe[event.sid]
sid_data.__dict__.update(event.__dict__)
def simulate_snapshot(self, date):
"""
Run the user's algo against our current snapshot and update
the algo's simulated time.
"""
# Needs to be set so that we inject the proper date into algo
# log/print lines.
self.snapshot_dt = date
self.algo.set_datetime(self.snapshot_dt)
self.algo.handle_data(self.universe)
# Update the simulation time.
self.simulation_dt = date
class AlgorithmSimulator(object):
def __init__(self, order_book, algo, algo_start):
# ==========
# Algo Setup
# ==========
# We extract the order book from the txn client so that
# the algo can place new orders.
self.order_book = order_book
self.algo = algo
self.algo_start = algo_start.replace(hour=0, minute=0, second=0, microsecond=0)
# Monkey patch the user algorithm to place orders in the
# TransactionSimulator's order book and use our logger.
self.algo.set_order(self.order)
# ==============
# Snapshot Setup
# ==============
# The algorithm's universe as of our most recent event.
# We want an ndict that will have empty ndicts as default
# values on missing keys.
self.universe = ndict(internal=defaultdict(ndict))
# We don't have a datetime for the current snapshot until we
# receive a message.
self.simulation_dt = None
self.snapshot_dt = None
# =============
# Logging Setup
# =============
# Processor function for injecting the algo_dt into
# user prints/logs.
def inject_algo_dt(record):
record.extra["algo_dt"] = self.snapshot_dt
self.processor = Processor(inject_algo_dt)
def order(self, sid, amount):
"""
Closure to pass into the user's algo to allow placing orders
into the transaction simulator's dict of open orders.
"""
order = ndict({"dt": self.simulation_dt, "sid": sid, "amount": int(amount), "filled": 0})
# Tell the user if they try to buy 0 shares of something.
if order.amount == 0:
zero_message = "Requested to trade zero shares of {sid}".format(sid=order.sid)
log.debug(zero_message)
# Don't bother placing orders for 0 shares.
return
# Add non-zero orders to the order book.
# !!!IMPORTANT SIDE-EFFECT!!!
# This modifies the internal state of the transaction
# simulator so that it can fill the placed order when it
# receives its next message.
self.order_book.place_order(order)
def transform(self, stream_in):
"""
Main generator work loop.
"""
# inject the current algo
# snapshot time to any log record generated.
with self.processor.threadbound():
# Group together events with the same dt field. This depends on the
# events already being sorted.
for date, snapshot in groupby(stream_in, attrgetter("dt")):
# Set the simulation date to be the first event we see.
# This should only occur once, at the start of the test.
if self.simulation_dt is None:
self.simulation_dt = date
# Done message has the risk report, so we yield before exiting.
if date == "DONE":
for event in snapshot:
yield event.perf_message
raise StopIteration
# We're still in the warmup period. Use the event to
# update our universe, but don't yield any perf messages,
# and don't send a snapshot to handle_data.
elif date < self.algo_start:
for event in snapshot:
del event["perf_message"]
self.update_universe(event)
# The algo has taken so long to process events that
# its simulated time is later than the event time.
# Update the universe and yield any perf messages
# encountered, but don't call handle_data.
elif date < self.simulation_dt:
for event in snapshot:
# Only yield if we have something interesting to say.
if event.perf_message is not None:
yield event.perf_message
# Delete the message before updating,
# so we don't send it to the user.
del event["perf_message"]
self.update_universe(event)
# Regular snapshot. Update the universe and send a snapshot
# to handle data.
else:
for event in snapshot:
# Only yield if we have something interesting to say.
if event.perf_message is not None:
yield event.perf_message
del event["perf_message"]
self.update_universe(event)
# Send the current state of the universe
# to the user's algo.
self.simulate_snapshot(date)
def update_universe(self, event):
"""
Update the universe with new event information.
"""
# Update our portfolio.
self.algo.set_portfolio(event.portfolio)
# Update our knowledge of this event's sid
for field in event.keys():
self.universe[event.sid][field] = event[field]
def simulate_snapshot(self, date):
"""
Run the user's algo against our current snapshot and update
the algo's simulated time.
"""
# Needs to be set so that we inject the proper date into algo
# log/print lines.
self.snapshot_dt = date
self.algo.set_datetime(self.snapshot_dt)
start_tic = datetime.now()
self.algo.handle_data(self.universe)
stop_tic = datetime.now()
# How long did you take?
delta = stop_tic - start_tic
# Update the simulation time.
self.simulation_dt = date + delta
