def test_volume_limit(self, name, first_order_amount, second_order_amount,
first_order_fill_amount, second_order_fill_amount):
slippage_model = FixedBasisPointsSlippage(basis_points=5,
volume_limit=0.1)
open_orders = [
Order(dt=datetime.datetime(2006, 1, 5, 14, 30, tzinfo=pytz.utc),
amount=order_amount,
filled=0,
asset=self.ASSET133)
for order_amount in [first_order_amount, second_order_amount]
]
bar_data = self.create_bardata(
simulation_dt_func=lambda: self.first_minute, )
orders_txns = list(
slippage_model.simulate(
bar_data,
self.ASSET133,
open_orders,
))
self.assertEquals(len(orders_txns), 2)
_, first_txn = orders_txns[0]
_, second_txn = orders_txns[1]
self.assertEquals(first_txn['amount'], first_order_fill_amount)
self.assertEquals(second_txn['amount'], second_order_fill_amount)
def test_volume_limit(self, name,
first_order_amount,
second_order_amount,
first_order_fill_amount,
second_order_fill_amount):
slippage_model = FixedBasisPointsSlippage(basis_points=5,
volume_limit=0.1)
open_orders = [
Order(
dt=datetime.datetime(2006, 1, 5, 14, 30, tzinfo=pytz.utc),
amount=order_amount,
filled=0,
asset=self.ASSET133
)
for order_amount in [first_order_amount, second_order_amount]
]
bar_data = self.create_bardata(
simulation_dt_func=lambda: self.first_minute,
)
orders_txns = list(slippage_model.simulate(
bar_data,
self.ASSET133,
open_orders,
))
self.assertEquals(len(orders_txns), 2)
_, first_txn = orders_txns[0]
_, second_txn = orders_txns[1]
self.assertEquals(first_txn['amount'], first_order_fill_amount)
self.assertEquals(second_txn['amount'], second_order_fill_amount)
def test_broken_constructions(self):
err_msg = (
"FixedBasisPointsSlippage() expected a value greater than "
"or equal to 0 for argument 'basis_points', but got -1 instead.")
with pytest.raises(ValueError, match=re.escape(err_msg)):
FixedBasisPointsSlippage(basis_points=-1)
err_msg = (
"FixedBasisPointsSlippage() expected a value strictly "
"greater than 0 for argument 'volume_limit', but got 0 instead.")
with pytest.raises(ValueError, match=re.escape(err_msg)):
FixedBasisPointsSlippage(volume_limit=0)
def __init__(self, data_frequency, equity_slippage=None,
future_slippage=None, equity_commission=None,
future_commission=None, cancel_policy=None, stock_exchange='NYSE'):
# these orders are aggregated by asset
self.open_orders = defaultdict(list)
# keep a dict of orders by their own id
self.orders = {}
# holding orders that have come in since the last event.
self.new_orders = []
self.current_dt = None
self.max_shares = int(1e+11)
self.slippage_models = {
Equity: equity_slippage or FixedBasisPointsSlippage(),
Future: future_slippage or VolatilityVolumeShare(
volume_limit=DEFAULT_FUTURE_VOLUME_SLIPPAGE_BAR_LIMIT,
),
}
self.commission_models = {
Equity: equity_commission or PerShare(),
Future: future_commission or PerContract(
cost=DEFAULT_PER_CONTRACT_COST,
exchange_fee=FUTURE_EXCHANGE_FEES_BY_SYMBOL,
),
}
self.data_frequency = data_frequency
self.cancel_policy = cancel_policy if cancel_policy else NeverCancel()
def test_broken_constructions(self):
with self.assertRaises(ValueError) as e:
FixedBasisPointsSlippage(basis_points=-1)
self.assertEqual(
str(e.exception),
"FixedBasisPointsSlippage() expected a value greater than "
"or equal to 0 for argument 'basis_points', but got -1 instead.")
with self.assertRaises(ValueError) as e:
FixedBasisPointsSlippage(volume_limit=0)
self.assertEqual(
str(e.exception),
"FixedBasisPointsSlippage() expected a value strictly "
"greater than 0 for argument 'volume_limit', but got 0 instead.")
def __init__(self, data_frequency, broker):
self.broker = broker
self._processed_closed_orders = []
self._processed_transactions = []
self.data_frequency = data_frequency
self.new_orders = []
self.max_shares = int(1e+11)
self.slippage_models = {
Equity:
FixedBasisPointsSlippage(),
Future:
VolatilityVolumeShare(
volume_limit=DEFAULT_FUTURE_VOLUME_SLIPPAGE_BAR_LIMIT, ),
}
self.commission_models = {
Equity:
PerShare(),
Future:
PerContract(
cost=DEFAULT_PER_CONTRACT_COST,
exchange_fee=FUTURE_EXCHANGE_FEES_BY_SYMBOL,
),
}
log.info('Initialized blotter_live')
def test_fixed_bps_slippage(self,
name,
basis_points,
volume_limit,
order_amount,
expected_price,
expected_amount):
slippage_model = FixedBasisPointsSlippage(basis_points=basis_points,
volume_limit=volume_limit)
open_orders = [
Order(
dt=datetime.datetime(2006, 1, 5, 14, 30, tzinfo=pytz.utc),
amount=order_amount,
filled=0,
asset=self.ASSET133
)
]
bar_data = self.create_bardata(
simulation_dt_func=lambda: self.first_minute
)
orders_txns = list(slippage_model.simulate(
bar_data,
self.ASSET133,
open_orders,
))
self.assertEquals(len(orders_txns), 1)
_, txn = orders_txns[0]
expected_txn = {
'price': expected_price,
'dt': datetime.datetime(
2006, 1, 5, 14, 31, tzinfo=pytz.utc),
'amount': expected_amount,
'asset': self.ASSET133,
'commission': None,
'type': DATASOURCE_TYPE.TRANSACTION,
'order_id': open_orders[0].id
}
self.assertIsNotNone(txn)
self.assertEquals(expected_txn, txn.__dict__)
def test_fill_zero_shares(self):
slippage_model = FixedBasisPointsSlippage(basis_points=5,
volume_limit=0.1)
# since the volume limit for the bar is 20, the first order will be
# filled and there will be a transaction for it, and the second order
# will order zero shares so there should not be a transaction for it.
open_orders = [
Order(
dt=datetime.datetime(2006, 1, 5, 14, 30, tzinfo=pytz.utc),
amount=20,
filled=0,
asset=self.ASSET133,
)
] * 2
bar_data = self.create_bardata(
simulation_dt_func=lambda: self.first_minute)
orders_txns = list(
slippage_model.simulate(
bar_data,
self.ASSET133,
open_orders,
))
assert 1 == len(orders_txns)
# ordering zero shares should result in zero transactions
open_orders = [
Order(
dt=datetime.datetime(2006, 1, 5, 14, 30, tzinfo=pytz.utc),
amount=0,
filled=0,
asset=self.ASSET133,
)
]
orders_txns = list(
slippage_model.simulate(
bar_data,
self.ASSET133,
open_orders,
))
assert 0 == len(orders_txns)
def test_fixed_bps_slippage(
self,
name,
basis_points,
volume_limit,
order_amount,
expected_price,
expected_amount,
):
slippage_model = FixedBasisPointsSlippage(basis_points=basis_points,
volume_limit=volume_limit)
open_orders = [
Order(
dt=datetime.datetime(2006, 1, 5, 14, 30, tzinfo=pytz.utc),
amount=order_amount,
filled=0,
asset=self.ASSET133,
)
]
bar_data = self.create_bardata(
simulation_dt_func=lambda: self.first_minute)
orders_txns = list(
slippage_model.simulate(
bar_data,
self.ASSET133,
open_orders,
))
assert len(orders_txns) == 1
_, txn = orders_txns[0]
expected_txn = {
"price": expected_price,
"dt": datetime.datetime(2006, 1, 5, 14, 31, tzinfo=pytz.utc),
"amount": expected_amount,
"asset": self.ASSET133,
"type": DATASOURCE_TYPE.TRANSACTION,
"order_id": open_orders[0].id,
}
assert txn is not None
assert expected_txn == txn.__dict__
def test_fill_zero_shares(self):
slippage_model = FixedBasisPointsSlippage(basis_points=5,
volume_limit=0.1)
# since the volume limit for the bar is 20, the first order will be
# filled and there will be a transaction for it, and the second order
# will order zero shares so there should not be a transaction for it.
open_orders = [
Order(
dt=datetime.datetime(2006, 1, 5, 14, 30, tzinfo=pytz.utc),
amount=20,
filled=0,
asset=self.ASSET133
)
] * 2
bar_data = self.create_bardata(
simulation_dt_func=lambda: self.first_minute
)
orders_txns = list(slippage_model.simulate(
bar_data,
self.ASSET133,
open_orders,
))
self.assertEqual(1, len(orders_txns))
# ordering zero shares should result in zero transactions
open_orders = [
Order(
dt=datetime.datetime(2006, 1, 5, 14, 30, tzinfo=pytz.utc),
amount=0,
filled=0,
asset=self.ASSET133
)
]
orders_txns = list(slippage_model.simulate(
bar_data,
self.ASSET133,
open_orders,
))
self.assertEqual(0, len(orders_txns))
def test_fixed_bps_slippage(self,
name,
basis_points,
volume_limit,
order_amount,
expected_price,
expected_amount):
slippage_model = FixedBasisPointsSlippage(basis_points=basis_points,
volume_limit=volume_limit)
open_orders = [
Order(
dt=datetime.datetime(2006, 1, 5, 14, 30, tzinfo=pytz.utc),
amount=order_amount,
filled=0,
asset=self.ASSET133
)
]
bar_data = self.create_bardata(
simulation_dt_func=lambda: self.first_minute
)
orders_txns = list(slippage_model.simulate(
bar_data,
self.ASSET133,
open_orders,
))
self.assertEquals(len(orders_txns), 1)
_, txn = orders_txns[0]
expected_txn = {
'price': expected_price,
'dt': datetime.datetime(2006, 1, 5, 14, 31, tzinfo=pytz.utc),
'amount': expected_amount,
'asset': self.ASSET133,
'type': DATASOURCE_TYPE.TRANSACTION,
'order_id': open_orders[0].id
}
self.assertIsNotNone(txn)
self.assertEquals(expected_txn, txn.__dict__)
def __init__(
self,
equity_slippage=None,
future_slippage=None,
option_slippage=None,
equity_commission=None,
future_commission=None,
option_commission=None,
cancel_policy=None,
):
super(SimulationBlotter, self).__init__(cancel_policy=cancel_policy)
# these orders are aggregated by asset
self.open_orders = defaultdict(list)
# keep a dict of orders by their own id
self.orders = {}
# holding orders that have come in since the last event.
self.new_orders = []
self.max_shares = int(1e11)
self.slippage_models = {
Equity:
equity_slippage or FixedBasisPointsSlippage(),
Future:
future_slippage or VolatilityVolumeShare(
volume_limit=DEFAULT_FUTURE_VOLUME_SLIPPAGE_BAR_LIMIT),
Option:
option_slippage or CoverTheSpread(),
}
self.commission_models = {
Equity:
equity_commission or PerShare(),
Future:
future_commission or PerContract(
cost=DEFAULT_PER_CONTRACT_COST,
exchange_fee=FUTURE_EXCHANGE_FEES_BY_SYMBOL,
),
Option:
option_commission
or PerOptionContract(cost=DEFAULT_PER_OPTION_CONTRACT_COST,
exchange_fee=0.01),
}
def transaction_sim(self, **params):
"""This is a utility method that asserts expected
results for conversion of orders to transactions given a
trade history
"""
trade_count = params["trade_count"]
trade_interval = params["trade_interval"]
order_count = params["order_count"]
order_amount = params["order_amount"]
order_interval = params["order_interval"]
expected_txn_count = params["expected_txn_count"]
expected_txn_volume = params["expected_txn_volume"]
# optional parameters
# ---------------------
# if present, alternate between long and short sales
alternate = params.get("alternate")
# if present, expect transaction amounts to match orders exactly.
complete_fill = params.get("complete_fill")
asset1 = self.asset_finder.retrieve_asset(1)
with TempDirectory() as tempdir:
if trade_interval < timedelta(days=1):
sim_params = factory.create_simulation_parameters(
start=self.start, end=self.end, data_frequency="minute")
minutes = self.trading_calendar.minutes_window(
sim_params.first_open,
int((trade_interval.total_seconds() / 60) * trade_count) +
100,
)
price_data = np.array([10.1] * len(minutes))
assets = {
asset1.sid:
pd.DataFrame({
"open": price_data,
"high": price_data,
"low": price_data,
"close": price_data,
"volume": np.array([100] * len(minutes)),
"dt": minutes,
}).set_index("dt")
}
write_bcolz_minute_data(
self.trading_calendar,
self.trading_calendar.sessions_in_range(
self.trading_calendar.minute_to_session_label(
minutes[0]),
self.trading_calendar.minute_to_session_label(
minutes[-1]),
),
tempdir.path,
assets.items(),
)
equity_minute_reader = BcolzMinuteBarReader(tempdir.path)
data_portal = DataPortal(
self.asset_finder,
self.trading_calendar,
first_trading_day=equity_minute_reader.first_trading_day,
equity_minute_reader=equity_minute_reader,
)
else:
sim_params = factory.create_simulation_parameters(
data_frequency="daily")
days = sim_params.sessions
assets = {
1:
pd.DataFrame(
{
"open": [10.1] * len(days),
"high": [10.1] * len(days),
"low": [10.1] * len(days),
"close": [10.1] * len(days),
"volume": [100] * len(days),
"day": [day.value for day in days],
},
index=days,
)
}
path = os.path.join(tempdir.path, "testdata.bcolz")
BcolzDailyBarWriter(path, self.trading_calendar, days[0],
days[-1]).write(assets.items())
equity_daily_reader = BcolzDailyBarReader(path)
data_portal = DataPortal(
self.asset_finder,
self.trading_calendar,
first_trading_day=equity_daily_reader.first_trading_day,
equity_daily_reader=equity_daily_reader,
)
if "default_slippage" not in params or not params[
"default_slippage"]:
slippage_func = FixedBasisPointsSlippage()
else:
slippage_func = None
blotter = SimulationBlotter(slippage_func)
start_date = sim_params.first_open
if alternate:
alternator = -1
else:
alternator = 1
tracker = MetricsTracker(
trading_calendar=self.trading_calendar,
first_session=sim_params.start_session,
last_session=sim_params.end_session,
capital_base=sim_params.capital_base,
emission_rate=sim_params.emission_rate,
data_frequency=sim_params.data_frequency,
asset_finder=self.asset_finder,
metrics=load_metrics_set("none"),
)
# replicate what tradesim does by going through every minute or day
# of the simulation and processing open orders each time
if sim_params.data_frequency == "minute":
ticks = minutes
else:
ticks = days
transactions = []
order_list = []
order_date = start_date
for tick in ticks:
blotter.current_dt = tick
if tick >= order_date and len(order_list) < order_count:
# place an order
direction = alternator**len(order_list)
order_id = blotter.order(
asset1,
order_amount * direction,
MarketOrder(),
)
order_list.append(blotter.orders[order_id])
order_date = order_date + order_interval
# move after market orders to just after market next
# market open.
if order_date.hour >= 21:
if order_date.minute >= 00:
order_date = order_date + timedelta(days=1)
order_date = order_date.replace(hour=14, minute=30)
else:
bar_data = BarData(
data_portal=data_portal,
simulation_dt_func=lambda: tick,
data_frequency=sim_params.data_frequency,
trading_calendar=self.trading_calendar,
restrictions=NoRestrictions(),
)
txns, _, closed_orders = blotter.get_transactions(bar_data)
for txn in txns:
tracker.process_transaction(txn)
transactions.append(txn)
blotter.prune_orders(closed_orders)
for i in range(order_count):
order = order_list[i]
assert order.asset == asset1
assert order.amount == order_amount * alternator**i
if complete_fill:
assert len(transactions) == len(order_list)
total_volume = 0
for i in range(len(transactions)):
txn = transactions[i]
total_volume += txn.amount
if complete_fill:
order = order_list[i]
assert order.amount == txn.amount
assert total_volume == expected_txn_volume
assert len(transactions) == expected_txn_count
if total_volume == 0:
with pytest.raises(KeyError):
tracker.positions[asset1]
else:
cumulative_pos = tracker.positions[asset1]
assert total_volume == cumulative_pos.amount
# the open orders should not contain the asset.
oo = blotter.open_orders
assert asset1 not in oo, "Entry is removed when no open orders"
def transaction_sim(self, **params):
"""This is a utility method that asserts expected
results for conversion of orders to transactions given a
trade history
"""
trade_count = params['trade_count']
trade_interval = params['trade_interval']
order_count = params['order_count']
order_amount = params['order_amount']
order_interval = params['order_interval']
expected_txn_count = params['expected_txn_count']
expected_txn_volume = params['expected_txn_volume']
# optional parameters
# ---------------------
# if present, alternate between long and short sales
alternate = params.get('alternate')
# if present, expect transaction amounts to match orders exactly.
complete_fill = params.get('complete_fill')
asset1 = self.asset_finder.retrieve_asset(1)
metadata = make_simple_equity_info([asset1.sid], self.start, self.end)
with TempDirectory() as tempdir, \
tmp_trading_env(equities=metadata,
load=self.make_load_function()) as env:
if trade_interval < timedelta(days=1):
sim_params = factory.create_simulation_parameters(
start=self.start, end=self.end, data_frequency="minute")
minutes = self.trading_calendar.minutes_window(
sim_params.first_open,
int((trade_interval.total_seconds() / 60) * trade_count) +
100)
price_data = np.array([10.1] * len(minutes))
assets = {
asset1.sid:
pd.DataFrame({
"open": price_data,
"high": price_data,
"low": price_data,
"close": price_data,
"volume": np.array([100] * len(minutes)),
"dt": minutes
}).set_index("dt")
}
write_bcolz_minute_data(
self.trading_calendar,
self.trading_calendar.sessions_in_range(
self.trading_calendar.minute_to_session_label(
minutes[0]),
self.trading_calendar.minute_to_session_label(
minutes[-1])),
tempdir.path,
iteritems(assets),
)
equity_minute_reader = BcolzMinuteBarReader(tempdir.path)
data_portal = DataPortal(
env.asset_finder,
self.trading_calendar,
first_trading_day=equity_minute_reader.first_trading_day,
equity_minute_reader=equity_minute_reader,
)
else:
sim_params = factory.create_simulation_parameters(
data_frequency="daily")
days = sim_params.sessions
assets = {
1:
pd.DataFrame(
{
"open": [10.1] * len(days),
"high": [10.1] * len(days),
"low": [10.1] * len(days),
"close": [10.1] * len(days),
"volume": [100] * len(days),
"day": [day.value for day in days]
},
index=days)
}
path = os.path.join(tempdir.path, "testdata.bcolz")
BcolzDailyBarWriter(path, self.trading_calendar, days[0],
days[-1]).write(assets.items())
equity_daily_reader = BcolzDailyBarReader(path)
data_portal = DataPortal(
env.asset_finder,
self.trading_calendar,
first_trading_day=equity_daily_reader.first_trading_day,
equity_daily_reader=equity_daily_reader,
)
if "default_slippage" not in params or \
not params["default_slippage"]:
slippage_func = FixedBasisPointsSlippage()
else:
slippage_func = None
blotter = Blotter(sim_params.data_frequency, slippage_func)
start_date = sim_params.first_open
if alternate:
alternator = -1
else:
alternator = 1
tracker = PerformanceTracker(sim_params, self.trading_calendar,
self.env)
# replicate what tradesim does by going through every minute or day
# of the simulation and processing open orders each time
if sim_params.data_frequency == "minute":
ticks = minutes
else:
ticks = days
transactions = []
order_list = []
order_date = start_date
for tick in ticks:
blotter.current_dt = tick
if tick >= order_date and len(order_list) < order_count:
# place an order
direction = alternator**len(order_list)
order_id = blotter.order(asset1, order_amount * direction,
MarketOrder())
order_list.append(blotter.orders[order_id])
order_date = order_date + order_interval
# move after market orders to just after market next
# market open.
if order_date.hour >= 21:
if order_date.minute >= 00:
order_date = order_date + timedelta(days=1)
order_date = order_date.replace(hour=14, minute=30)
else:
bar_data = BarData(
data_portal=data_portal,
simulation_dt_func=lambda: tick,
data_frequency=sim_params.data_frequency,
trading_calendar=self.trading_calendar,
restrictions=NoRestrictions(),
)
txns, _, closed_orders = blotter.get_transactions(bar_data)
for txn in txns:
tracker.process_transaction(txn)
transactions.append(txn)
blotter.prune_orders(closed_orders)
for i in range(order_count):
order = order_list[i]
self.assertEqual(order.asset, asset1)
self.assertEqual(order.amount, order_amount * alternator**i)
if complete_fill:
self.assertEqual(len(transactions), len(order_list))
total_volume = 0
for i in range(len(transactions)):
txn = transactions[i]
total_volume += txn.amount
if complete_fill:
order = order_list[i]
self.assertEqual(order.amount, txn.amount)
self.assertEqual(total_volume, expected_txn_volume)
self.assertEqual(len(transactions), expected_txn_count)
cumulative_pos = tracker.position_tracker.positions[asset1]
if total_volume == 0:
self.assertIsNone(cumulative_pos)
else:
self.assertEqual(total_volume, cumulative_pos.amount)
# the open orders should not contain the asset.
oo = blotter.open_orders
self.assertNotIn(asset1, oo,
"Entry is removed when no open orders")