def test_batch_order_matches_multiple_orders(self): """ Ensure the effect of order_batch is the same as multiple calls to order. """ blotter1 = Blotter(self.sim_params.data_frequency) blotter2 = Blotter(self.sim_params.data_frequency) for i in range(1, 4): order_arg_lists = [ (self.asset_24, i * 100, MarketOrder()), (self.asset_25, i * 100, LimitOrder(i * 100 + 1)), ] order_batch_ids = blotter1.batch_order(order_arg_lists) order_ids = [] for order_args in order_arg_lists: order_ids.append(blotter2.order(*order_args)) self.assertEqual(len(order_batch_ids), len(order_ids)) self.assertEqual(len(blotter1.open_orders), len(blotter2.open_orders)) for (asset, _, _), order_batch_id, order_id in zip(order_arg_lists, order_batch_ids, order_ids): self.assertEqual(len(blotter1.open_orders[asset]), len(blotter2.open_orders[asset])) self.assertEqual(order_batch_id, blotter1.open_orders[asset][i - 1].id) self.assertEqual(order_id, blotter2.open_orders[asset][i - 1].id)
def test_order_rejection(self): blotter = Blotter() # Reject a nonexistent order -> no order appears in new_order, # no exceptions raised out blotter.reject(56) self.assertEqual(blotter.new_orders, []) # Basic tests of open order behavior open_order_id = blotter.order(24, 100, MarketOrder()) second_order_id = blotter.order(24, 50, MarketOrder()) self.assertEqual(len(blotter.open_orders[24]), 2) open_order = blotter.open_orders[24][0] self.assertEqual(open_order.status, ORDER_STATUS.OPEN) self.assertEqual(open_order.id, open_order_id) self.assertIn(open_order, blotter.new_orders) # Reject that order immediately (same bar, i.e. still in new_orders) blotter.reject(open_order_id) self.assertEqual(len(blotter.new_orders), 2) self.assertEqual(len(blotter.open_orders[24]), 1) still_open_order = blotter.new_orders[0] self.assertEqual(still_open_order.id, second_order_id) self.assertEqual(still_open_order.status, ORDER_STATUS.OPEN) rejected_order = blotter.new_orders[1] self.assertEqual(rejected_order.status, ORDER_STATUS.REJECTED) self.assertEqual(rejected_order.reason, '') # Do it again, but reject it at a later time (after tradesimulation # pulls it from new_orders) blotter = Blotter() new_open_id = blotter.order(24, 10, MarketOrder()) new_open_order = blotter.open_orders[24][0] self.assertEqual(new_open_id, new_open_order.id) # Pretend that the trade simulation did this. blotter.new_orders = [] rejection_reason = "Not enough cash on hand." blotter.reject(new_open_id, reason=rejection_reason) rejected_order = blotter.new_orders[0] self.assertEqual(rejected_order.id, new_open_id) self.assertEqual(rejected_order.status, ORDER_STATUS.REJECTED) self.assertEqual(rejected_order.reason, rejection_reason) # You can't reject a filled order. blotter = Blotter() # Reset for paranoia blotter.current_dt = datetime.datetime.now() filled_id = blotter.order(24, 100, MarketOrder()) aapl_trade = create_trade(24, 50.0, 400, datetime.datetime.now()) filled_order = None for txn, updated_order in blotter.process_trade(aapl_trade): filled_order = updated_order self.assertEqual(filled_order.id, filled_id) self.assertIn(filled_order, blotter.new_orders) self.assertEqual(filled_order.status, ORDER_STATUS.FILLED) self.assertNotIn(filled_order, blotter.open_orders[24]) blotter.reject(filled_id) updated_order = blotter.orders[filled_id] self.assertEqual(updated_order.status, ORDER_STATUS.FILLED)
def test_order_hold(self): """ Held orders act almost identically to open orders, except for the status indication. When a fill happens, the order should switch status to OPEN/FILLED as necessary """ blotter = Blotter(self.sim_params.data_frequency) # Nothing happens on held of a non-existent order blotter.hold(56) self.assertEqual(blotter.new_orders, []) open_id = blotter.order(self.asset_24, 100, MarketOrder()) open_order = blotter.open_orders[self.asset_24][0] self.assertEqual(open_order.id, open_id) blotter.hold(open_id) self.assertEqual(len(blotter.new_orders), 1) self.assertEqual(len(blotter.open_orders[self.asset_24]), 1) held_order = blotter.new_orders[0] self.assertEqual(held_order.status, ORDER_STATUS.HELD) self.assertEqual(held_order.reason, '') blotter.cancel(held_order.id) self.assertEqual(len(blotter.new_orders), 1) self.assertEqual(len(blotter.open_orders[self.asset_24]), 0) cancelled_order = blotter.new_orders[0] self.assertEqual(cancelled_order.id, held_order.id) self.assertEqual(cancelled_order.status, ORDER_STATUS.CANCELLED) for data in ([100, self.sim_params.sessions[0]], [400, self.sim_params.sessions[1]]): # Verify that incoming fills will change the order status. trade_amt = data[0] dt = data[1] order_size = 100 expected_filled = int(trade_amt * DEFAULT_EQUITY_VOLUME_SLIPPAGE_BAR_LIMIT) expected_open = order_size - expected_filled expected_status = ORDER_STATUS.OPEN if expected_open else \ ORDER_STATUS.FILLED blotter = Blotter(self.sim_params.data_frequency) open_id = blotter.order(self.asset_24, order_size, MarketOrder()) open_order = blotter.open_orders[self.asset_24][0] self.assertEqual(open_id, open_order.id) blotter.hold(open_id) held_order = blotter.new_orders[0] filled_order = None blotter.current_dt = dt bar_data = self.create_bardata(simulation_dt_func=lambda: dt, ) txns, _, _ = blotter.get_transactions(bar_data) for txn in txns: filled_order = blotter.orders[txn.order_id] self.assertEqual(filled_order.id, held_order.id) self.assertEqual(filled_order.status, expected_status) self.assertEqual(filled_order.filled, expected_filled) self.assertEqual(filled_order.open_amount, expected_open)
def test_order_hold(self): """ Held orders act almost identically to open orders, except for the status indication. When a fill happens, the order should switch status to OPEN/FILLED as necessary """ blotter = Blotter() # Nothing happens on held of a non-existent order blotter.hold(56) self.assertEqual(blotter.new_orders, []) open_id = blotter.order(24, 100, MarketOrder()) open_order = blotter.open_orders[24][0] self.assertEqual(open_order.id, open_id) blotter.hold(open_id) self.assertEqual(len(blotter.new_orders), 1) self.assertEqual(len(blotter.open_orders[24]), 1) held_order = blotter.new_orders[0] self.assertEqual(held_order.status, ORDER_STATUS.HELD) self.assertEqual(held_order.reason, '') blotter.cancel(held_order.id) self.assertEqual(len(blotter.new_orders), 1) self.assertEqual(len(blotter.open_orders[24]), 0) cancelled_order = blotter.new_orders[0] self.assertEqual(cancelled_order.id, held_order.id) self.assertEqual(cancelled_order.status, ORDER_STATUS.CANCELLED) for trade_amt in (100, 400): # Verify that incoming fills will change the order status. order_size = 100 expected_filled = trade_amt * 0.25 expected_open = order_size - expected_filled expected_status = ORDER_STATUS.OPEN if expected_open else \ ORDER_STATUS.FILLED blotter = Blotter() blotter.current_dt = datetime.datetime.now() open_id = blotter.order(24, order_size, MarketOrder()) open_order = blotter.open_orders[24][0] self.assertEqual(open_id, open_order.id) blotter.hold(open_id) held_order = blotter.new_orders[0] aapl_trade = create_trade(24, 50.0, trade_amt, datetime.datetime.now()) filled_order = None for txn, updated_order in blotter.process_trade(aapl_trade): filled_order = updated_order self.assertEqual(filled_order.id, held_order.id) self.assertEqual(filled_order.status, expected_status) self.assertEqual(filled_order.filled, expected_filled) self.assertEqual(filled_order.open_amount, expected_open)
def test_blotter_processes_splits(self): blotter = Blotter('daily', equity_slippage=FixedSlippage()) # set up two open limit orders with very low limit prices, # one for sid 1 and one for sid 2 asset1 = self.asset_finder.retrieve_asset(1) asset2 = self.asset_finder.retrieve_asset(2) asset133 = self.asset_finder.retrieve_asset(133) blotter.order(asset1, 100, LimitOrder(10)) blotter.order(asset2, 100, LimitOrder(10)) # send in splits for assets 133 and 2. We have no open orders for # asset 133 so it should be ignored. blotter.process_splits([(asset133, 0.5), (asset2, 0.3333)]) for asset in [asset1, asset2]: order_lists = blotter.open_orders[asset] self.assertIsNotNone(order_lists) self.assertEqual(1, len(order_lists)) asset1_order = blotter.open_orders[1][0] asset2_order = blotter.open_orders[2][0] # make sure the asset1 order didn't change self.assertEqual(100, asset1_order.amount) self.assertEqual(10, asset1_order.limit) self.assertEqual(1, asset1_order.asset) # make sure the asset2 order did change # to 300 shares at 3.33 self.assertEqual(300, asset2_order.amount) self.assertEqual(3.33, asset2_order.limit) self.assertEqual(2, asset2_order.asset)
def test_blotter_processes_splits(self): sim_params = factory.create_simulation_parameters() blotter = Blotter() blotter.set_date(sim_params.period_start) # set up two open limit orders with very low limit prices, # one for sid 1 and one for sid 2 blotter.order(1, 100, LimitOrder(10)) blotter.order(2, 100, LimitOrder(10)) # send in a split for sid 2 split_event = factory.create_split( 2, 0.33333, sim_params.period_start + timedelta(days=1)) blotter.process_split(split_event) for sid in [1, 2]: order_lists = blotter.open_orders[sid] self.assertIsNotNone(order_lists) self.assertEqual(1, len(order_lists)) aapl_order = blotter.open_orders[1][0].to_dict() fls_order = blotter.open_orders[2][0].to_dict() # make sure the aapl order didn't change self.assertEqual(100, aapl_order['amount']) self.assertEqual(10, aapl_order['limit']) self.assertEqual(1, aapl_order['sid']) # make sure the fls order did change # to 300 shares at 3.33 self.assertEqual(300, fls_order['amount']) self.assertEqual(3.33, fls_order['limit']) self.assertEqual(2, fls_order['sid'])
def test_prune_orders(self): blotter = Blotter(self.sim_params.data_frequency, self.env.asset_finder) asset_24 = blotter.asset_finder.retrieve_asset(24) asset_25 = blotter.asset_finder.retrieve_asset(25) blotter.order(asset_24, 100, MarketOrder()) open_order = blotter.open_orders[asset_24][0] blotter.prune_orders([]) self.assertEqual(1, len(blotter.open_orders[asset_24])) blotter.prune_orders([open_order]) self.assertEqual(0, len(blotter.open_orders[asset_24])) # prune an order that isn't in our our open orders list, make sure # nothing blows up other_order = Order( dt=blotter.current_dt, sid=asset_25, amount=1 ) blotter.prune_orders([other_order])
def test_blotter_processes_splits(self): blotter = Blotter('daily', self.env.asset_finder, slippage_func=FixedSlippage()) # set up two open limit orders with very low limit prices, # one for sid 1 and one for sid 2 blotter.order(blotter.asset_finder.retrieve_asset(1), 100, LimitOrder(10)) blotter.order(blotter.asset_finder.retrieve_asset(2), 100, LimitOrder(10)) # send in a split for sid 2 blotter.process_splits([(2, 0.3333)]) for sid in [1, 2]: order_lists = blotter.open_orders[sid] self.assertIsNotNone(order_lists) self.assertEqual(1, len(order_lists)) aapl_order = blotter.open_orders[1][0].to_dict() fls_order = blotter.open_orders[2][0].to_dict() # make sure the aapl order didn't change self.assertEqual(100, aapl_order['amount']) self.assertEqual(10, aapl_order['limit']) self.assertEqual(1, aapl_order['sid']) # make sure the fls order did change # to 300 shares at 3.33 self.assertEqual(300, fls_order['amount']) self.assertEqual(3.33, fls_order['limit']) self.assertEqual(2, fls_order['sid'])
def test_slippage_and_commission_dispatching(self): blotter = Blotter( self.sim_params.data_frequency, equity_slippage=FixedSlippage(spread=0.0), future_slippage=FixedSlippage(spread=2.0), equity_commission=PerTrade(cost=1.0), future_commission=PerTrade(cost=2.0), ) blotter.order(self.asset_24, 1, MarketOrder()) blotter.order(self.future_cl, 1, MarketOrder()) bar_data = self.create_bardata( simulation_dt_func=lambda: self.sim_params.sessions[-1], ) txns, commissions, _ = blotter.get_transactions(bar_data) # The equity transaction should have the same price as its current # price because the slippage spread is zero. Its commission should be # $1.00. equity_txn = txns[0] self.assertEqual( equity_txn.price, bar_data.current(equity_txn.asset, 'price'), ) self.assertEqual(commissions[0]['cost'], 1.0) # The future transaction price should be 1.0 more than its current # price because half of the 'future_slippage' spread is added. Its # commission should be $2.00. future_txn = txns[1] self.assertEqual( future_txn.price, bar_data.current(future_txn.asset, 'price') + 1.0, ) self.assertEqual(commissions[1]['cost'], 2.0)
def test_cancel(self): blotter = Blotter('daily') oid_1 = blotter.order(self.asset_24, 100, MarketOrder()) oid_2 = blotter.order(self.asset_24, 200, MarketOrder()) oid_3 = blotter.order(self.asset_24, 300, MarketOrder()) # Create an order for another asset to verify that we don't remove it # when we do cancel_all on 24. blotter.order(self.asset_25, 150, MarketOrder()) self.assertEqual(len(blotter.open_orders), 2) self.assertEqual(len(blotter.open_orders[self.asset_24]), 3) self.assertEqual( [o.amount for o in blotter.open_orders[self.asset_24]], [100, 200, 300], ) blotter.cancel(oid_2) self.assertEqual(len(blotter.open_orders), 2) self.assertEqual(len(blotter.open_orders[self.asset_24]), 2) self.assertEqual( [o.amount for o in blotter.open_orders[self.asset_24]], [100, 300], ) self.assertEqual( [o.id for o in blotter.open_orders[self.asset_24]], [oid_1, oid_3], ) blotter.cancel_all_orders_for_asset(self.asset_24) self.assertEqual(len(blotter.open_orders), 1) self.assertEqual(list(blotter.open_orders), [self.asset_25])
def test_blotter_eod_cancellation(self): blotter = Blotter('minute', self.env.asset_finder, cancel_policy=EODCancel()) asset_24 = blotter.asset_finder.retrieve_asset(24) # Make two orders for the same sid, so we can test that we are not # mutating the orders list as we are cancelling orders blotter.order(asset_24, 100, MarketOrder()) blotter.order(asset_24, -100, MarketOrder()) self.assertEqual(len(blotter.new_orders), 2) order_ids = [order.id for order in blotter.open_orders[asset_24]] self.assertEqual(blotter.new_orders[0].status, ORDER_STATUS.OPEN) self.assertEqual(blotter.new_orders[1].status, ORDER_STATUS.OPEN) blotter.execute_cancel_policy(BAR) self.assertEqual(blotter.new_orders[0].status, ORDER_STATUS.OPEN) self.assertEqual(blotter.new_orders[1].status, ORDER_STATUS.OPEN) blotter.execute_cancel_policy(DAY_END) for order_id in order_ids: order = blotter.orders[order_id] self.assertEqual(order.status, ORDER_STATUS.CANCELLED)
def __init__(self, *args, **kwargs): """Initialize sids and other state variables. :Arguments: data_frequency : str (daily, hourly or minutely) The duration of the bars. annualizer : int <optional> Which constant to use for annualizing risk metrics. If not provided, will extract from data_frequency. capital_base : float <default: 1.0e5> How much capital to start with. """ self._portfolio = None self.datetime = None self.registered_transforms = {} self.transforms = [] self.sources = [] self._recorded_vars = {} self.logger = None self.benchmark_return_source = None # default components for transact self.slippage = VolumeShareSlippage() self.commission = PerShare() if 'data_frequency' in kwargs: self.set_data_frequency(kwargs.pop('data_frequency')) else: self.data_frequency = None self.instant_fill = kwargs.pop('instant_fill', False) # Override annualizer if set if 'annualizer' in kwargs: self.annualizer = kwargs['annualizer'] # set the capital base self.capital_base = kwargs.pop('capital_base', DEFAULT_CAPITAL_BASE) self.sim_params = kwargs.pop('sim_params', None) if self.sim_params: self.sim_params.data_frequency = self.data_frequency self.blotter = kwargs.pop('blotter', None) if not self.blotter: self.blotter = Blotter() # an algorithm subclass needs to set initialized to True when # it is fully initialized. self.initialized = False # call to user-defined constructor method self.initialize(*args, **kwargs)
def test_blotter_order_types(self, style_obj, expected_lmt, expected_stp): blotter = Blotter('daily') blotter.order(self.asset_24, 100, style_obj) result = blotter.open_orders[self.asset_24][0] self.assertEqual(result.limit, expected_lmt) self.assertEqual(result.stop, expected_stp)
def test_blotter_order_types(self, style_obj, expected_lmt, expected_stp): blotter = Blotter('daily', self.env.asset_finder) asset_24 = blotter.asset_finder.retrieve_asset(24) blotter.order(asset_24, 100, style_obj) result = blotter.open_orders[asset_24][0] self.assertEqual(result.limit, expected_lmt) self.assertEqual(result.stop, expected_stp)
def test_blotter_never_cancel(self): blotter = Blotter('minute', cancel_policy=NeverCancel()) blotter.order(self.asset_24, 100, MarketOrder()) self.assertEqual(len(blotter.new_orders), 1) self.assertEqual(blotter.new_orders[0].status, ORDER_STATUS.OPEN) blotter.execute_cancel_policy(BAR) self.assertEqual(blotter.new_orders[0].status, ORDER_STATUS.OPEN) blotter.execute_cancel_policy(SESSION_END) self.assertEqual(blotter.new_orders[0].status, ORDER_STATUS.OPEN)
def test_blotter_never_cancel(self): blotter = Blotter('minute', self.env.asset_finder, cancel_policy=NeverCancel()) blotter.order(blotter.asset_finder.retrieve_asset(24), 100, MarketOrder()) self.assertEqual(len(blotter.new_orders), 1) self.assertEqual(blotter.new_orders[0].status, ORDER_STATUS.OPEN) blotter.execute_cancel_policy(BAR) self.assertEqual(blotter.new_orders[0].status, ORDER_STATUS.OPEN) blotter.execute_cancel_policy(DAY_END) self.assertEqual(blotter.new_orders[0].status, ORDER_STATUS.OPEN)
def test_prune_orders(self): blotter = Blotter(self.sim_params.data_frequency) blotter.order(self.asset_24, 100, MarketOrder()) open_order = blotter.open_orders[self.asset_24][0] blotter.prune_orders([]) self.assertEqual(1, len(blotter.open_orders[self.asset_24])) blotter.prune_orders([open_order]) self.assertEqual(0, len(blotter.open_orders[self.asset_24])) # prune an order that isn't in our our open orders list, make sure # nothing blows up other_order = Order(dt=blotter.current_dt, asset=self.asset_25, amount=1) blotter.prune_orders([other_order])
def transaction_sim(self, **params): """ This is a utility method that asserts expected results for conversion of orders to transactions given a trade history""" tempdir = TempDirectory() try: trade_count = params['trade_count'] trade_interval = params['trade_interval'] order_count = params['order_count'] order_amount = params['order_amount'] order_interval = params['order_interval'] expected_txn_count = params['expected_txn_count'] expected_txn_volume = params['expected_txn_volume'] # optional parameters # --------------------- # if present, alternate between long and short sales alternate = params.get('alternate') # if present, expect transaction amounts to match orders exactly. complete_fill = params.get('complete_fill') env = TradingEnvironment() sid = 1 if trade_interval < timedelta(days=1): sim_params = factory.create_simulation_parameters( data_frequency="minute") minutes = env.market_minute_window( sim_params.first_open, int((trade_interval.total_seconds() / 60) * trade_count) + 100) price_data = np.array([10.1] * len(minutes)) assets = { sid: pd.DataFrame({ "open": price_data, "high": price_data, "low": price_data, "close": price_data, "volume": np.array([100] * len(minutes)), "dt": minutes }).set_index("dt") } write_bcolz_minute_data( env, env.days_in_range(minutes[0], minutes[-1]), tempdir.path, assets) equity_minute_reader = BcolzMinuteBarReader(tempdir.path) data_portal = DataPortal( env, equity_minute_reader=equity_minute_reader, ) else: sim_params = factory.create_simulation_parameters( data_frequency="daily") days = sim_params.trading_days assets = { 1: pd.DataFrame( { "open": [10.1] * len(days), "high": [10.1] * len(days), "low": [10.1] * len(days), "close": [10.1] * len(days), "volume": [100] * len(days), "day": [day.value for day in days] }, index=days) } path = os.path.join(tempdir.path, "testdata.bcolz") DailyBarWriterFromDataFrames(assets).write(path, days, assets) equity_daily_reader = BcolzDailyBarReader(path) data_portal = DataPortal( env, equity_daily_reader=equity_daily_reader, ) if "default_slippage" not in params or \ not params["default_slippage"]: slippage_func = FixedSlippage() else: slippage_func = None blotter = Blotter(sim_params.data_frequency, self.env.asset_finder, slippage_func) env.write_data( equities_data={ sid: { "start_date": sim_params.trading_days[0], "end_date": sim_params.trading_days[-1] } }) start_date = sim_params.first_open if alternate: alternator = -1 else: alternator = 1 tracker = PerformanceTracker(sim_params, self.env) # replicate what tradesim does by going through every minute or day # of the simulation and processing open orders each time if sim_params.data_frequency == "minute": ticks = minutes else: ticks = days transactions = [] order_list = [] order_date = start_date for tick in ticks: blotter.current_dt = tick if tick >= order_date and len(order_list) < order_count: # place an order direction = alternator**len(order_list) order_id = blotter.order( blotter.asset_finder.retrieve_asset(sid), order_amount * direction, MarketOrder()) order_list.append(blotter.orders[order_id]) order_date = order_date + order_interval # move after market orders to just after market next # market open. if order_date.hour >= 21: if order_date.minute >= 00: order_date = order_date + timedelta(days=1) order_date = order_date.replace(hour=14, minute=30) else: bar_data = BarData(data_portal, lambda: tick, sim_params.data_frequency) txns, _ = blotter.get_transactions(bar_data) for txn in txns: tracker.process_transaction(txn) transactions.append(txn) for i in range(order_count): order = order_list[i] self.assertEqual(order.sid, sid) self.assertEqual(order.amount, order_amount * alternator**i) if complete_fill: self.assertEqual(len(transactions), len(order_list)) total_volume = 0 for i in range(len(transactions)): txn = transactions[i] total_volume += txn.amount if complete_fill: order = order_list[i] self.assertEqual(order.amount, txn.amount) self.assertEqual(total_volume, expected_txn_volume) self.assertEqual(len(transactions), expected_txn_count) cumulative_pos = tracker.position_tracker.positions[sid] if total_volume == 0: self.assertIsNone(cumulative_pos) else: self.assertEqual(total_volume, cumulative_pos.amount) # the open orders should not contain sid. oo = blotter.open_orders self.assertNotIn(sid, oo, "Entry is removed when no open orders") finally: tempdir.cleanup()
def __init__(self, *args, **kwargs): """Initialize sids and other state variables. :Arguments: :Optional: initialize : function Function that is called with a single argument at the begninning of the simulation. handle_data : function Function that is called with 2 arguments (context and data) on every bar. script : str Algoscript that contains initialize and handle_data function definition. data_frequency : str (daily, hourly or minutely) The duration of the bars. capital_base : float <default: 1.0e5> How much capital to start with. instant_fill : bool <default: False> Whether to fill orders immediately or on next bar. """ self.datetime = None self.registered_transforms = {} self.transforms = [] self.sources = [] # List of trading controls to be used to validate orders. self.trading_controls = [] self._recorded_vars = {} self.namespace = kwargs.get('namespace', {}) self.logger = None self.benchmark_return_source = None # default components for transact self.slippage = VolumeShareSlippage() self.commission = PerShare() self.instant_fill = kwargs.pop('instant_fill', False) # set the capital base self.capital_base = kwargs.pop('capital_base', DEFAULT_CAPITAL_BASE) self.sim_params = kwargs.pop('sim_params', None) if self.sim_params is None: self.sim_params = create_simulation_parameters( capital_base=self.capital_base ) self.perf_tracker = PerformanceTracker(self.sim_params) self.blotter = kwargs.pop('blotter', None) if not self.blotter: self.blotter = Blotter() self.portfolio_needs_update = True self._portfolio = None self.history_container = None self.history_specs = {} # If string is passed in, execute and get reference to # functions. self.algoscript = kwargs.pop('script', None) self._initialize = None self._analyze = None if self.algoscript is not None: exec_(self.algoscript, self.namespace) self._initialize = self.namespace.get('initialize', None) if 'handle_data' not in self.namespace: raise ValueError('You must define a handle_data function.') else: self._handle_data = self.namespace['handle_data'] # Optional analyze function, gets called after run self._analyze = self.namespace.get('analyze', None) elif kwargs.get('initialize', False) and kwargs.get('handle_data'): if self.algoscript is not None: raise ValueError('You can not set script and \ initialize/handle_data.') self._initialize = kwargs.pop('initialize') self._handle_data = kwargs.pop('handle_data') # If method not defined, NOOP if self._initialize is None: self._initialize = lambda x: None # Alternative way of setting data_frequency for backwards # compatibility. if 'data_frequency' in kwargs: self.data_frequency = kwargs.pop('data_frequency') # Subclasses that override initialize should only worry about # setting self.initialized = True if AUTO_INITIALIZE is # is manually set to False. self.initialized = False self.initialize(*args, **kwargs) if self.AUTO_INITIALIZE: self.initialized = True
def test_order_rejection(self): blotter = Blotter(self.sim_params.data_frequency, self.env.asset_finder) asset_24 = blotter.asset_finder.retrieve_asset(24) # Reject a nonexistent order -> no order appears in new_order, # no exceptions raised out blotter.reject(56) self.assertEqual(blotter.new_orders, []) # Basic tests of open order behavior open_order_id = blotter.order(asset_24, 100, MarketOrder()) second_order_id = blotter.order(asset_24, 50, MarketOrder()) self.assertEqual(len(blotter.open_orders[asset_24]), 2) open_order = blotter.open_orders[asset_24][0] self.assertEqual(open_order.status, ORDER_STATUS.OPEN) self.assertEqual(open_order.id, open_order_id) self.assertIn(open_order, blotter.new_orders) # Reject that order immediately (same bar, i.e. still in new_orders) blotter.reject(open_order_id) self.assertEqual(len(blotter.new_orders), 2) self.assertEqual(len(blotter.open_orders[asset_24]), 1) still_open_order = blotter.new_orders[0] self.assertEqual(still_open_order.id, second_order_id) self.assertEqual(still_open_order.status, ORDER_STATUS.OPEN) rejected_order = blotter.new_orders[1] self.assertEqual(rejected_order.status, ORDER_STATUS.REJECTED) self.assertEqual(rejected_order.reason, '') # Do it again, but reject it at a later time (after tradesimulation # pulls it from new_orders) blotter = Blotter(self.sim_params.data_frequency, self.env.asset_finder) new_open_id = blotter.order(asset_24, 10, MarketOrder()) new_open_order = blotter.open_orders[asset_24][0] self.assertEqual(new_open_id, new_open_order.id) # Pretend that the trade simulation did this. blotter.new_orders = [] rejection_reason = "Not enough cash on hand." blotter.reject(new_open_id, reason=rejection_reason) rejected_order = blotter.new_orders[0] self.assertEqual(rejected_order.id, new_open_id) self.assertEqual(rejected_order.status, ORDER_STATUS.REJECTED) self.assertEqual(rejected_order.reason, rejection_reason) # You can't reject a filled order. # Reset for paranoia blotter = Blotter(self.sim_params.data_frequency, self.env.asset_finder) blotter.slippage_func = FixedSlippage() filled_id = blotter.order(asset_24, 100, MarketOrder()) filled_order = None blotter.current_dt = self.sim_params.trading_days[-1] bar_data = BarData( self.data_portal, lambda: self.sim_params.trading_days[-1], self.sim_params.data_frequency, ) txns, _ = blotter.get_transactions(bar_data) for txn in txns: filled_order = blotter.orders[txn.order_id] self.assertEqual(filled_order.id, filled_id) self.assertIn(filled_order, blotter.new_orders) self.assertEqual(filled_order.status, ORDER_STATUS.FILLED) self.assertNotIn(filled_order, blotter.open_orders[asset_24]) blotter.reject(filled_id) updated_order = blotter.orders[filled_id] self.assertEqual(updated_order.status, ORDER_STATUS.FILLED)
def __init__(self, *args, **kwargs): """Initialize sids and other state variables. :Arguments: :Optional: initialize : function Function that is called with a single argument at the begninning of the simulation. handle_data : function Function that is called with 2 arguments (context and data) on every bar. script : str Algoscript that contains initialize and handle_data function definition. data_frequency : {'daily', 'minute'} The duration of the bars. capital_base : float <default: 1.0e5> How much capital to start with. instant_fill : bool <default: False> Whether to fill orders immediately or on next bar. asset_finder : An AssetFinder object A new AssetFinder object to be used in this TradingEnvironment asset_metadata: can be either: - dict - pandas.DataFrame - object with 'read' property If dict is provided, it must have the following structure: * keys are the identifiers * values are dicts containing the metadata, with the metadata field name as the key If pandas.DataFrame is provided, it must have the following structure: * column names must be the metadata fields * index must be the different asset identifiers * array contents should be the metadata value If an object with a 'read' property is provided, 'read' must return rows containing at least one of 'sid' or 'symbol' along with the other metadata fields. identifiers : List Any asset identifiers that are not provided in the asset_metadata, but will be traded by this TradingAlgorithm """ self.sources = [] # List of trading controls to be used to validate orders. self.trading_controls = [] # List of account controls to be checked on each bar. self.account_controls = [] self._recorded_vars = {} self.namespace = kwargs.get('namespace', {}) self._platform = kwargs.pop('platform', 'zipline') self.logger = None self.benchmark_return_source = None # default components for transact self.slippage = VolumeShareSlippage() self.commission = PerShare() self.instant_fill = kwargs.pop('instant_fill', False) # set the capital base self.capital_base = kwargs.pop('capital_base', DEFAULT_CAPITAL_BASE) self.sim_params = kwargs.pop('sim_params', None) if self.sim_params is None: self.sim_params = create_simulation_parameters( capital_base=self.capital_base, start=kwargs.pop('start', None), end=kwargs.pop('end', None)) self.perf_tracker = PerformanceTracker(self.sim_params) # Update the TradingEnvironment with the provided asset metadata self.trading_environment = kwargs.pop('env', TradingEnvironment.instance()) self.trading_environment.update_asset_finder( asset_finder=kwargs.pop('asset_finder', None), asset_metadata=kwargs.pop('asset_metadata', None), identifiers=kwargs.pop('identifiers', None)) # Pull in the environment's new AssetFinder for quick reference self.asset_finder = self.trading_environment.asset_finder self.init_engine(kwargs.pop('ffc_loader', None)) # Maps from name to Term self._filters = {} self._factors = {} self._classifiers = {} self.blotter = kwargs.pop('blotter', None) if not self.blotter: self.blotter = Blotter() # Set the dt initally to the period start by forcing it to change self.on_dt_changed(self.sim_params.period_start) self.portfolio_needs_update = True self.account_needs_update = True self.performance_needs_update = True self._portfolio = None self._account = None self.history_container_class = kwargs.pop( 'history_container_class', HistoryContainer, ) self.history_container = None self.history_specs = {} # If string is passed in, execute and get reference to # functions. self.algoscript = kwargs.pop('script', None) self._initialize = None self._before_trading_start = None self._analyze = None self.event_manager = EventManager() if self.algoscript is not None: filename = kwargs.pop('algo_filename', None) if filename is None: filename = '<string>' code = compile(self.algoscript, filename, 'exec') exec_(code, self.namespace) self._initialize = self.namespace.get('initialize') if 'handle_data' not in self.namespace: raise ValueError('You must define a handle_data function.') else: self._handle_data = self.namespace['handle_data'] self._before_trading_start = \ self.namespace.get('before_trading_start') # Optional analyze function, gets called after run self._analyze = self.namespace.get('analyze') elif kwargs.get('initialize') and kwargs.get('handle_data'): if self.algoscript is not None: raise ValueError('You can not set script and \ initialize/handle_data.') self._initialize = kwargs.pop('initialize') self._handle_data = kwargs.pop('handle_data') self._before_trading_start = kwargs.pop('before_trading_start', None) self.event_manager.add_event( zipline.utils.events.Event( zipline.utils.events.Always(), # We pass handle_data.__func__ to get the unbound method. # We will explicitly pass the algorithm to bind it again. self.handle_data.__func__, ), prepend=True, ) # If method not defined, NOOP if self._initialize is None: self._initialize = lambda x: None # Alternative way of setting data_frequency for backwards # compatibility. if 'data_frequency' in kwargs: self.data_frequency = kwargs.pop('data_frequency') self._most_recent_data = None # Prepare the algo for initialization self.initialized = False self.initialize_args = args self.initialize_kwargs = kwargs
def transaction_sim(self, **params): """ This is a utility method that asserts expected results for conversion of orders to transactions given a trade history""" trade_count = params['trade_count'] trade_interval = params['trade_interval'] order_count = params['order_count'] order_amount = params['order_amount'] order_interval = params['order_interval'] expected_txn_count = params['expected_txn_count'] expected_txn_volume = params['expected_txn_volume'] # optional parameters # --------------------- # if present, alternate between long and short sales alternate = params.get('alternate') # if present, expect transaction amounts to match orders exactly. complete_fill = params.get('complete_fill') sid = 1 sim_params = factory.create_simulation_parameters() blotter = Blotter() price = [10.1] * trade_count volume = [100] * trade_count start_date = sim_params.first_open generated_trades = factory.create_trade_history( sid, price, volume, trade_interval, sim_params) if alternate: alternator = -1 else: alternator = 1 order_date = start_date for i in range(order_count): blotter.set_date(order_date) blotter.order(sid, order_amount * alternator**i, MarketOrder()) order_date = order_date + order_interval # move after market orders to just after market next # market open. if order_date.hour >= 21: if order_date.minute >= 00: order_date = order_date + timedelta(days=1) order_date = order_date.replace(hour=14, minute=30) # there should now be one open order list stored under the sid oo = blotter.open_orders self.assertEqual(len(oo), 1) self.assertTrue(sid in oo) order_list = oo[sid][:] # make copy self.assertEqual(order_count, len(order_list)) for i in range(order_count): order = order_list[i] self.assertEqual(order.sid, sid) self.assertEqual(order.amount, order_amount * alternator**i) tracker = PerformanceTracker(sim_params) benchmark_returns = [ Event({ 'dt': dt, 'returns': ret, 'type': zipline.protocol.DATASOURCE_TYPE.BENCHMARK, 'source_id': 'benchmarks' }) for dt, ret in trading.environment.benchmark_returns.iteritems() if dt.date() >= sim_params.period_start.date() and dt.date() <= sim_params.period_end.date() ] generated_events = date_sorted_sources(generated_trades, benchmark_returns) # this approximates the loop inside TradingSimulationClient transactions = [] for dt, events in itertools.groupby(generated_events, operator.attrgetter('dt')): for event in events: if event.type == DATASOURCE_TYPE.TRADE: for txn, order in blotter.process_trade(event): transactions.append(txn) tracker.process_transaction(txn) elif event.type == DATASOURCE_TYPE.BENCHMARK: tracker.process_benchmark(event) elif event.type == DATASOURCE_TYPE.TRADE: tracker.process_trade(event) if complete_fill: self.assertEqual(len(transactions), len(order_list)) total_volume = 0 for i in range(len(transactions)): txn = transactions[i] total_volume += txn.amount if complete_fill: order = order_list[i] self.assertEqual(order.amount, txn.amount) self.assertEqual(total_volume, expected_txn_volume) self.assertEqual(len(transactions), expected_txn_count) cumulative_pos = tracker.cumulative_performance.positions[sid] self.assertEqual(total_volume, cumulative_pos.amount) # the open orders should not contain sid. oo = blotter.open_orders self.assertNotIn(sid, oo, "Entry is removed when no open orders")
def __init__(self, *args, **kwargs): """Initialize sids and other state variables. :Arguments: :Optional: initialize : function Function that is called with a single argument at the begninning of the simulation. handle_data : function Function that is called with 2 arguments (context and data) on every bar. script : str Algoscript that contains initialize and handle_data function definition. data_frequency : str (daily, hourly or minutely) The duration of the bars. capital_base : float <default: 1.0e5> How much capital to start with. instant_fill : bool <default: False> Whether to fill orders immediately or on next bar. environment : str <default: 'zipline'> The environment that this algorithm is running in. """ self.datetime = None self.sources = [] # List of trading controls to be used to validate orders. self.trading_controls = [] # List of account controls to be checked on each bar. self.account_controls = [] self._recorded_vars = {} self.namespace = kwargs.get('namespace', {}) self._platform = kwargs.pop('platform', 'zipline') self.logger = None self.benchmark_return_source = None # default components for transact self.slippage = VolumeShareSlippage() self.commission = PerShare() self.instant_fill = kwargs.pop('instant_fill', False) # set the capital base self.capital_base = kwargs.pop('capital_base', DEFAULT_CAPITAL_BASE) self.sim_params = kwargs.pop('sim_params', None) if self.sim_params is None: self.sim_params = create_simulation_parameters( capital_base=self.capital_base ) self.perf_tracker = PerformanceTracker(self.sim_params) self.blotter = kwargs.pop('blotter', None) if not self.blotter: self.blotter = Blotter() self.portfolio_needs_update = True self.account_needs_update = True self.performance_needs_update = True self._portfolio = None self._account = None self.history_container_class = kwargs.pop( 'history_container_class', HistoryContainer, ) self.history_container = None self.history_specs = {} # If string is passed in, execute and get reference to # functions. self.algoscript = kwargs.pop('script', None) self._initialize = None self._before_trading_start = None self._analyze = None self.event_manager = EventManager() if self.algoscript is not None: filename = kwargs.pop('algo_filename', None) if filename is None: filename = '<string>' code = compile(self.algoscript, filename, 'exec') exec_(code, self.namespace) self._initialize = self.namespace.get('initialize') if 'handle_data' not in self.namespace: raise ValueError('You must define a handle_data function.') else: self._handle_data = self.namespace['handle_data'] self._before_trading_start = \ self.namespace.get('before_trading_start') # Optional analyze function, gets called after run self._analyze = self.namespace.get('analyze') elif kwargs.get('initialize') and kwargs.get('handle_data'): if self.algoscript is not None: raise ValueError('You can not set script and \ initialize/handle_data.') self._initialize = kwargs.pop('initialize') self._handle_data = kwargs.pop('handle_data') self._before_trading_start = kwargs.pop('before_trading_start', None) self.event_manager.add_event( zipline.utils.events.Event( zipline.utils.events.Always(), # We pass handle_data.__func__ to get the unbound method. # We will explicitly pass the algorithm to bind it again. self.handle_data.__func__, ), prepend=True, ) # If method not defined, NOOP if self._initialize is None: self._initialize = lambda x: None # Alternative way of setting data_frequency for backwards # compatibility. if 'data_frequency' in kwargs: self.data_frequency = kwargs.pop('data_frequency') self._most_recent_data = None # Subclasses that override initialize should only worry about # setting self.initialized = True if AUTO_INITIALIZE is # is manually set to False. self.initialized = False self.initialize(*args, **kwargs) if self.AUTO_INITIALIZE: self.initialized = True
def transaction_sim(self, **params): """This is a utility method that asserts expected results for conversion of orders to transactions given a trade history """ trade_count = params['trade_count'] trade_interval = params['trade_interval'] order_count = params['order_count'] order_amount = params['order_amount'] order_interval = params['order_interval'] expected_txn_count = params['expected_txn_count'] expected_txn_volume = params['expected_txn_volume'] # optional parameters # --------------------- # if present, alternate between long and short sales alternate = params.get('alternate') # if present, expect transaction amounts to match orders exactly. complete_fill = params.get('complete_fill') asset1 = self.asset_finder.retrieve_asset(1) metadata = make_simple_equity_info([asset1.sid], self.start, self.end) with TempDirectory() as tempdir, \ tmp_trading_env(equities=metadata, load=self.make_load_function()) as env: if trade_interval < timedelta(days=1): sim_params = factory.create_simulation_parameters( start=self.start, end=self.end, data_frequency="minute") minutes = self.trading_calendar.minutes_window( sim_params.first_open, int((trade_interval.total_seconds() / 60) * trade_count) + 100) price_data = np.array([10.1] * len(minutes)) assets = { asset1.sid: pd.DataFrame({ "open": price_data, "high": price_data, "low": price_data, "close": price_data, "volume": np.array([100] * len(minutes)), "dt": minutes }).set_index("dt") } write_bcolz_minute_data( self.trading_calendar, self.trading_calendar.sessions_in_range( self.trading_calendar.minute_to_session_label( minutes[0]), self.trading_calendar.minute_to_session_label( minutes[-1])), tempdir.path, iteritems(assets), ) equity_minute_reader = BcolzMinuteBarReader(tempdir.path) data_portal = DataPortal( env.asset_finder, self.trading_calendar, first_trading_day=equity_minute_reader.first_trading_day, equity_minute_reader=equity_minute_reader, ) else: sim_params = factory.create_simulation_parameters( data_frequency="daily") days = sim_params.sessions assets = { 1: pd.DataFrame( { "open": [10.1] * len(days), "high": [10.1] * len(days), "low": [10.1] * len(days), "close": [10.1] * len(days), "volume": [100] * len(days), "day": [day.value for day in days] }, index=days) } path = os.path.join(tempdir.path, "testdata.bcolz") BcolzDailyBarWriter(path, self.trading_calendar, days[0], days[-1]).write(assets.items()) equity_daily_reader = BcolzDailyBarReader(path) data_portal = DataPortal( env.asset_finder, self.trading_calendar, first_trading_day=equity_daily_reader.first_trading_day, equity_daily_reader=equity_daily_reader, ) if "default_slippage" not in params or \ not params["default_slippage"]: slippage_func = FixedBasisPointsSlippage() else: slippage_func = None blotter = Blotter(sim_params.data_frequency, slippage_func) start_date = sim_params.first_open if alternate: alternator = -1 else: alternator = 1 tracker = MetricsTracker( trading_calendar=self.trading_calendar, first_session=sim_params.start_session, last_session=sim_params.end_session, capital_base=sim_params.capital_base, emission_rate=sim_params.emission_rate, data_frequency=sim_params.data_frequency, asset_finder=self.asset_finder, metrics=load_metrics_set('none'), ) # replicate what tradesim does by going through every minute or day # of the simulation and processing open orders each time if sim_params.data_frequency == "minute": ticks = minutes else: ticks = days transactions = [] order_list = [] order_date = start_date for tick in ticks: blotter.current_dt = tick if tick >= order_date and len(order_list) < order_count: # place an order direction = alternator**len(order_list) order_id = blotter.order( asset1, order_amount * direction, MarketOrder(), ) order_list.append(blotter.orders[order_id]) order_date = order_date + order_interval # move after market orders to just after market next # market open. if order_date.hour >= 21: if order_date.minute >= 00: order_date = order_date + timedelta(days=1) order_date = order_date.replace(hour=14, minute=30) else: bar_data = BarData( data_portal=data_portal, simulation_dt_func=lambda: tick, data_frequency=sim_params.data_frequency, trading_calendar=self.trading_calendar, restrictions=NoRestrictions(), ) txns, _, closed_orders = blotter.get_transactions(bar_data) for txn in txns: tracker.process_transaction(txn) transactions.append(txn) blotter.prune_orders(closed_orders) for i in range(order_count): order = order_list[i] self.assertEqual(order.asset, asset1) self.assertEqual(order.amount, order_amount * alternator**i) if complete_fill: self.assertEqual(len(transactions), len(order_list)) total_volume = 0 for i in range(len(transactions)): txn = transactions[i] total_volume += txn.amount if complete_fill: order = order_list[i] self.assertEqual(order.amount, txn.amount) self.assertEqual(total_volume, expected_txn_volume) self.assertEqual(len(transactions), expected_txn_count) if total_volume == 0: self.assertRaises(KeyError, lambda: tracker.positions[asset1]) else: cumulative_pos = tracker.positions[asset1] self.assertEqual(total_volume, cumulative_pos.amount) # the open orders should not contain the asset. oo = blotter.open_orders self.assertNotIn(asset1, oo, "Entry is removed when no open orders")
def __init__(self, *args, **kwargs): """Initialize sids and other state variables. :Arguments: :Optional: initialize : function Function that is called with a single argument at the begninning of the simulation. handle_data : function Function that is called with 2 arguments (context and data) on every bar. script : str Algoscript that contains initialize and handle_data function definition. data_frequency : str (daily, hourly or minutely) The duration of the bars. annualizer : int <optional> Which constant to use for annualizing risk metrics. If not provided, will extract from data_frequency. capital_base : float <default: 1.0e5> How much capital to start with. instant_fill : bool <default: False> Whether to fill orders immediately or on next bar. """ self.datetime = None self.registered_transforms = {} self.transforms = [] self.sources = [] self._recorded_vars = {} self.logger = None self.benchmark_return_source = None self.perf_tracker = None # default components for transact self.slippage = VolumeShareSlippage() self.commission = PerShare() if 'data_frequency' in kwargs: self.set_data_frequency(kwargs.pop('data_frequency')) else: self.data_frequency = None self.instant_fill = kwargs.pop('instant_fill', False) # Override annualizer if set if 'annualizer' in kwargs: self.annualizer = kwargs['annualizer'] # set the capital base self.capital_base = kwargs.pop('capital_base', DEFAULT_CAPITAL_BASE) self.sim_params = kwargs.pop('sim_params', None) if self.sim_params: if self.data_frequency is None: self.data_frequency = self.sim_params.data_frequency else: self.sim_params.data_frequency = self.data_frequency self.perf_tracker = PerformanceTracker(self.sim_params) self.blotter = kwargs.pop('blotter', None) if not self.blotter: self.blotter = Blotter() self.portfolio_needs_update = True self._portfolio = None # If string is passed in, execute and get reference to # functions. self.algoscript = kwargs.pop('script', None) self._initialize = None if self.algoscript is not None: self.ns = {} exec_(self.algoscript, self.ns) if 'initialize' not in self.ns: raise ValueError('You must define an initialze function.') if 'handle_data' not in self.ns: raise ValueError('You must define a handle_data function.') self._initialize = self.ns['initialize'] self._handle_data = self.ns['handle_data'] # If two functions are passed in assume initialize and # handle_data are passed in. elif kwargs.get('initialize', False) and kwargs.get('handle_data'): if self.algoscript is not None: raise ValueError('You can not set script and \ initialize/handle_data.') self._initialize = kwargs.pop('initialize') self._handle_data = kwargs.pop('handle_data') if self._initialize is None: self._initialize = lambda x: None # an algorithm subclass needs to set initialized to True when # it is fully initialized. self.initialized = False self.initialize(*args, **kwargs)