def test_union_restrictions(self):
"""
Test that we appropriately union restrictions together, including
eliminating redundancy (ignoring NoRestrictions) and flattening out
the underlying sub-restrictions of _UnionRestrictions
"""
no_restrictions_rl = NoRestrictions()
st_restrict_asset1 = StaticRestrictions([self.ASSET1])
st_restrict_asset2 = StaticRestrictions([self.ASSET2])
st_restricted_assets = [self.ASSET1, self.ASSET2]
before_frozen_dt = str_to_ts('2011-01-05')
freeze_dt_1 = str_to_ts('2011-01-06')
unfreeze_dt = str_to_ts('2011-01-06 16:00')
hist_restrict_asset3_1 = HistoricalRestrictions([
Restriction(self.ASSET3, freeze_dt_1, FROZEN),
Restriction(self.ASSET3, unfreeze_dt, ALLOWED)
])
freeze_dt_2 = str_to_ts('2011-01-07')
hist_restrict_asset3_2 = HistoricalRestrictions([
Restriction(self.ASSET3, freeze_dt_2, FROZEN)
])
# A union of a NoRestrictions with a non-trivial restriction should
# yield the original restriction
trivial_union_restrictions = no_restrictions_rl | st_restrict_asset1
self.assertIsInstance(trivial_union_restrictions, StaticRestrictions)
# A union of two non-trivial restrictions should yield a
# UnionRestrictions
st_union_restrictions = st_restrict_asset1 | st_restrict_asset2
self.assertIsInstance(st_union_restrictions, _UnionRestrictions)
arb_dt = str_to_ts('2011-01-04')
self.assert_is_restricted(st_restrict_asset1, self.ASSET1, arb_dt)
self.assert_not_restricted(st_restrict_asset1, self.ASSET2, arb_dt)
self.assert_not_restricted(st_restrict_asset2, self.ASSET1, arb_dt)
self.assert_is_restricted(st_restrict_asset2, self.ASSET2, arb_dt)
self.assert_is_restricted(st_union_restrictions, self.ASSET1, arb_dt)
self.assert_is_restricted(st_union_restrictions, self.ASSET2, arb_dt)
self.assert_many_restrictions(
st_restrict_asset1,
st_restricted_assets,
[True, False],
arb_dt
)
self.assert_many_restrictions(
st_restrict_asset2,
st_restricted_assets,
[False, True],
arb_dt
)
self.assert_many_restrictions(
st_union_restrictions,
st_restricted_assets,
[True, True],
arb_dt
)
# A union of a 2-sub-restriction UnionRestrictions and a
# non-trivial restrictions should yield a UnionRestrictions with
# 3 sub restrictions. Works with UnionRestrictions on both the left
# side or right side
for r1, r2 in [
(st_union_restrictions, hist_restrict_asset3_1),
(hist_restrict_asset3_1, st_union_restrictions)
]:
union_or_hist_restrictions = r1 | r2
self.assertIsInstance(
union_or_hist_restrictions, _UnionRestrictions)
self.assertEqual(
len(union_or_hist_restrictions.sub_restrictions), 3)
# Includes the two static restrictions on ASSET1 and ASSET2,
# and the historical restriction on ASSET3 starting on freeze_dt_1
# and ending on unfreeze_dt
self.assert_all_restrictions(
union_or_hist_restrictions,
[True, True, False],
before_frozen_dt
)
self.assert_all_restrictions(
union_or_hist_restrictions,
[True, True, True],
freeze_dt_1
)
self.assert_all_restrictions(
union_or_hist_restrictions,
[True, True, False],
unfreeze_dt
)
self.assert_all_restrictions(
union_or_hist_restrictions,
[True, True, False],
freeze_dt_2
)
# A union of two 2-sub-restrictions UnionRestrictions should yield a
# UnionRestrictions with 4 sub restrictions.
hist_union_restrictions = \
hist_restrict_asset3_1 | hist_restrict_asset3_2
multi_union_restrictions = \
st_union_restrictions | hist_union_restrictions
self.assertIsInstance(multi_union_restrictions, _UnionRestrictions)
self.assertEqual(len(multi_union_restrictions.sub_restrictions), 4)
# Includes the two static restrictions on ASSET1 and ASSET2, the
# first historical restriction on ASSET3 starting on freeze_dt_1 and
# ending on unfreeze_dt, and the second historical restriction on
# ASSET3 starting on freeze_dt_2
self.assert_all_restrictions(
multi_union_restrictions,
[True, True, False],
before_frozen_dt
)
self.assert_all_restrictions(
multi_union_restrictions,
[True, True, True],
freeze_dt_1
)
self.assert_all_restrictions(
multi_union_restrictions,
[True, True, False],
unfreeze_dt
)
self.assert_all_restrictions(
multi_union_restrictions,
[True, True, True],
freeze_dt_2
)
def transaction_sim(self, **params):
"""This is a utility method that asserts expected
results for conversion of orders to transactions given a
trade history
"""
trade_count = params['trade_count']
trade_interval = params['trade_interval']
order_count = params['order_count']
order_amount = params['order_amount']
order_interval = params['order_interval']
expected_txn_count = params['expected_txn_count']
expected_txn_volume = params['expected_txn_volume']
# optional parameters
# ---------------------
# if present, alternate between long and short sales
alternate = params.get('alternate')
# if present, expect transaction amounts to match orders exactly.
complete_fill = params.get('complete_fill')
asset1 = self.asset_finder.retrieve_asset(1)
with TempDirectory() as tempdir:
if trade_interval < timedelta(days=1):
sim_params = factory.create_simulation_parameters(
start=self.start, end=self.end, data_frequency="minute")
minutes = self.trading_calendar.minutes_window(
sim_params.first_open,
int((trade_interval.total_seconds() / 60) * trade_count) +
100)
price_data = np.array([10.1] * len(minutes))
assets = {
asset1.sid:
pd.DataFrame({
"open": price_data,
"high": price_data,
"low": price_data,
"close": price_data,
"volume": np.array([100] * len(minutes)),
"dt": minutes
}).set_index("dt")
}
write_bcolz_minute_data(
self.trading_calendar,
self.trading_calendar.sessions_in_range(
self.trading_calendar.minute_to_session_label(
minutes[0]),
self.trading_calendar.minute_to_session_label(
minutes[-1])),
tempdir.path,
iteritems(assets),
)
equity_minute_reader = BcolzMinuteBarReader(tempdir.path)
data_portal = DataPortal(
self.asset_finder,
self.trading_calendar,
first_trading_day=equity_minute_reader.first_trading_day,
equity_minute_reader=equity_minute_reader,
)
else:
sim_params = factory.create_simulation_parameters(
data_frequency="daily")
days = sim_params.sessions
assets = {
1:
pd.DataFrame(
{
"open": [10.1] * len(days),
"high": [10.1] * len(days),
"low": [10.1] * len(days),
"close": [10.1] * len(days),
"volume": [100] * len(days),
"day": [day.value for day in days]
},
index=days)
}
path = os.path.join(tempdir.path, "testdata.bcolz")
BcolzDailyBarWriter(path, self.trading_calendar, days[0],
days[-1]).write(assets.items())
equity_daily_reader = BcolzDailyBarReader(path)
data_portal = DataPortal(
self.asset_finder,
self.trading_calendar,
first_trading_day=equity_daily_reader.first_trading_day,
equity_daily_reader=equity_daily_reader,
)
if "default_slippage" not in params or \
not params["default_slippage"]:
slippage_func = FixedBasisPointsSlippage()
else:
slippage_func = None
blotter = SimulationBlotter(slippage_func)
start_date = sim_params.first_open
if alternate:
alternator = -1
else:
alternator = 1
tracker = MetricsTracker(
trading_calendar=self.trading_calendar,
first_session=sim_params.start_session,
last_session=sim_params.end_session,
capital_base=sim_params.capital_base,
emission_rate=sim_params.emission_rate,
data_frequency=sim_params.data_frequency,
asset_finder=self.asset_finder,
metrics=load_metrics_set('none'),
)
# replicate what tradesim does by going through every minute or day
# of the simulation and processing open orders each time
if sim_params.data_frequency == "minute":
ticks = minutes
else:
ticks = days
transactions = []
order_list = []
order_date = start_date
for tick in ticks:
blotter.current_dt = tick
if tick >= order_date and len(order_list) < order_count:
# place an order
direction = alternator**len(order_list)
order_id = blotter.order(
asset1,
order_amount * direction,
MarketOrder(),
)
order_list.append(blotter.orders[order_id])
order_date = order_date + order_interval
# move after market orders to just after market next
# market open.
if order_date.hour >= 21:
if order_date.minute >= 00:
order_date = order_date + timedelta(days=1)
order_date = order_date.replace(hour=14, minute=30)
else:
bar_data = BarData(
data_portal=data_portal,
simulation_dt_func=lambda: tick,
data_frequency=sim_params.data_frequency,
trading_calendar=self.trading_calendar,
restrictions=NoRestrictions(),
)
txns, _, closed_orders = blotter.get_transactions(bar_data)
for txn in txns:
tracker.process_transaction(txn)
transactions.append(txn)
blotter.prune_orders(closed_orders)
for i in range(order_count):
order = order_list[i]
self.assertEqual(order.asset, asset1)
self.assertEqual(order.amount, order_amount * alternator**i)
if complete_fill:
self.assertEqual(len(transactions), len(order_list))
total_volume = 0
for i in range(len(transactions)):
txn = transactions[i]
total_volume += txn.amount
if complete_fill:
order = order_list[i]
self.assertEqual(order.amount, txn.amount)
self.assertEqual(total_volume, expected_txn_volume)
self.assertEqual(len(transactions), expected_txn_count)
if total_volume == 0:
self.assertRaises(KeyError, lambda: tracker.positions[asset1])
else:
cumulative_pos = tracker.positions[asset1]
self.assertEqual(total_volume, cumulative_pos.amount)
# the open orders should not contain the asset.
oo = blotter.open_orders
self.assertNotIn(asset1, oo,
"Entry is removed when no open orders")
def init_instance_fixtures(self):
super().init_instance_fixtures()
self.bar_data = BarData(self.data_portal, lambda: self.minute,
"minute", self.trading_calendar,
NoRestrictions())
def get_data(dt, bundle=None, data_frequency=None):
"""
Return a zipline.protocol.BarData object for the specified bundle (or default bundle)
as of the specified datetime. This is the same object that is passed
as the `data` parameter to `handle_data` and other backtest functions.
Parameters
----------
dt : str (YYYY-MM-DD[ HH:MM:SS]), required
The datetime (for minute data) or date (for daily data) which the
data object should be anchored to.
bundle : str, optional
the bundle code. If omitted, the default bundle will be used (and
must be set).
data_frequency : str, optional
the data frequency. Possible choices: daily, minute. The default is
"daily" for daily bundles and "minute" for minute bundles. Minute
bundles also support "daily".
Returns
-------
data : zipline.protocol.BarData
Examples
--------
Get the data object for July 7, 2020 at 11 AM for the usstock minute
bundle:
>>> data = get_data('2020-07-07 11:00:00', bundle="usstock-1min")
Get the data object for July 7, 2020 for a daily bundle:
>>> data = get_data('2020-07-07', bundle="xjpx-1d-bundle")
"""
if not bundle:
bundle = get_default_bundle()
if not bundle:
raise ValidationError(
"you must specify a bundle or set a default bundle")
bundle = bundle["default_bundle"]
load_extensions(code=bundle)
bundle_data = bundles.load(
bundle,
os.environ,
pd.Timestamp.utcnow(),
)
if not data_frequency:
config = get_bundle_config(bundle)
data_frequency = config["data_frequency"]
calendar_name = bundles.bundles[bundle].calendar_name
trading_calendar = get_calendar(calendar_name)
session_minute = pd.Timestamp(dt, tz=trading_calendar.tz)
session = session_minute.normalize().tz_localize(None).tz_localize("UTC")
first_session = max(bundles.bundles[bundle].start_session,
trading_calendar.first_session)
if session < first_session:
raise ValidationError(
f"date cannot be earlier than {first_session.date().isoformat()} for this bundle"
)
if not trading_calendar.is_session(session):
raise ValidationError(
f"requested date {session.date().isoformat()} is not in {calendar_name} calendar"
)
if data_frequency == "minute" and not trading_calendar.is_open_on_minute(
session_minute):
raise ValidationError(
f"requested time {session_minute.isoformat()} is not in {calendar_name} calendar"
)
if data_frequency == "minute":
equity_minute_reader = future_minute_reader = bundle_data.equity_minute_bar_reader
else:
equity_minute_reader = future_minute_reader = None
asset_finder = asset_finder_cache.get(bundle, bundle_data.asset_finder)
asset_finder_cache[bundle] = asset_finder
data_portal = DataPortal(
asset_finder,
trading_calendar=trading_calendar,
first_trading_day=bundle_data.equity_minute_bar_reader.
first_trading_day,
equity_minute_reader=equity_minute_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
future_minute_reader=future_minute_reader,
future_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader)
data = BarData(
data_portal=data_portal,
simulation_dt_func=lambda: session_minute,
data_frequency=data_frequency,
trading_calendar=trading_calendar,
restrictions=NoRestrictions(),
universe_func=lambda: asset_finder.retrieve_all(asset_finder.sids))
return data
def create_bardata(self, simulation_dt_func, restrictions=None):
return BarData(self.data_portal, simulation_dt_func,
self.CREATE_BARDATA_DATA_FREQUENCY,
self.trading_calendar, restrictions or NoRestrictions())