def test_minute_after_assets_stopped(self):
minutes = self.env.market_minutes_for_day(
self.env.next_trading_day(self.days[-1])
)
last_trading_minute = \
self.env.market_minutes_for_day(self.days[-1])[-1]
# this entire day is after both assets have stopped trading
for idx, minute in enumerate(minutes):
bar_data = BarData(self.data_portal, lambda: minute, "minute")
self.assertFalse(bar_data.can_trade(self.ASSET1))
self.assertFalse(bar_data.can_trade(self.ASSET2))
self.assertFalse(bar_data.is_stale(self.ASSET1))
self.assertFalse(bar_data.is_stale(self.ASSET2))
self.check_internal_consistency(bar_data)
for field in ALL_FIELDS:
for asset in self.ASSETS:
asset_value = bar_data.current(asset, field)
if field in OHLCP:
self.assertTrue(np.isnan(asset_value))
elif field == "volume":
self.assertEqual(0, asset_value)
elif field == "last_traded":
self.assertEqual(last_trading_minute, asset_value)
def test_spot_price_at_midnight(self):
# make sure that if we try to get a minute price at a non-market
# minute, we use the previous market close's timestamp
day = self.days[1]
eight_fortyfive_am_eastern = \
pd.Timestamp("{0}-{1}-{2} 8:45".format(
day.year, day.month, day.day),
tz='US/Eastern'
)
bar_data = BarData(self.data_portal, lambda: day, "minute")
bar_data2 = BarData(self.data_portal,
lambda: eight_fortyfive_am_eastern, "minute")
with handle_non_market_minutes(bar_data), \
handle_non_market_minutes(bar_data2):
for bd in [bar_data, bar_data2]:
for field in ["close", "price"]:
self.assertEqual(390, bd.current(self.ASSET1, field))
# make sure that if the asset didn't trade at the previous
# close, we properly ffill (or not ffill)
self.assertEqual(
350, bd.current(self.HILARIOUSLY_ILLIQUID_ASSET, "price"))
self.assertTrue(
np.isnan(
bd.current(self.HILARIOUSLY_ILLIQUID_ASSET, "high")))
self.assertEqual(
0, bd.current(self.HILARIOUSLY_ILLIQUID_ASSET, "volume"))
def test_day_before_assets_trading(self):
# use the day before self.equity_daily_bar_days[0]
day = self.trading_schedule.previous_execution_day(
self.equity_daily_bar_days[0]
)
bar_data = BarData(self.data_portal, lambda: day, "daily")
self.check_internal_consistency(bar_data)
self.assertFalse(bar_data.can_trade(self.ASSET1))
self.assertFalse(bar_data.can_trade(self.ASSET2))
self.assertFalse(bar_data.is_stale(self.ASSET1))
self.assertFalse(bar_data.is_stale(self.ASSET2))
for field in ALL_FIELDS:
for asset in self.ASSETS:
asset_value = bar_data.current(asset, field)
if field in OHLCP:
self.assertTrue(np.isnan(asset_value))
elif field == "volume":
self.assertEqual(0, asset_value)
elif field == "last_traded":
self.assertTrue(asset_value is pd.NaT)
def test_overnight_adjustments(self):
# verify there is a split for SPLIT_ASSET
splits = self.adjustments_reader.get_adjustments_for_sid(
"splits", self.SPLIT_ASSET.sid)
self.assertEqual(1, len(splits))
split = splits[0]
self.assertEqual(split[0], pd.Timestamp("2016-01-06", tz='UTC'))
# Current day is 1/06/16
day = self.days[1]
eight_fortyfive_am_eastern = \
pd.Timestamp("{0}-{1}-{2} 8:45".format(
day.year, day.month, day.day),
tz='US/Eastern'
)
bar_data = BarData(self.data_portal,
lambda: eight_fortyfive_am_eastern, "minute")
expected = {
'open': 391 / 2.0,
'high': 392 / 2.0,
'low': 389 / 2.0,
'close': 390 / 2.0,
'volume': 39000 * 2.0,
'price': 390 / 2.0,
}
with handle_non_market_minutes(bar_data):
for field in OHLCP + ['volume']:
value = bar_data.current(self.SPLIT_ASSET, field)
# Assert the price is adjusted for the overnight split
self.assertEqual(value, expected[field])
def test_fully_active_day(self):
bar_data = BarData(
self.data_portal,
lambda: self.get_last_minute_of_session(
self.equity_daily_bar_days[1]
),
"daily",
self.trading_calendar
)
self.check_internal_consistency(bar_data)
# on self.equity_daily_bar_days[1], both assets have data
for asset in self.ASSETS:
self.assertTrue(bar_data.can_trade(asset))
self.assertFalse(bar_data.is_stale(asset))
self.assertEqual(4, bar_data.current(asset, "open"))
self.assertEqual(5, bar_data.current(asset, "high"))
self.assertEqual(2, bar_data.current(asset, "low"))
self.assertEqual(3, bar_data.current(asset, "close"))
self.assertEqual(300, bar_data.current(asset, "volume"))
self.assertEqual(3, bar_data.current(asset, "price"))
self.assertEqual(
self.equity_daily_bar_days[1],
bar_data.current(asset, "last_traded")
)
def test_get_value_is_unadjusted(self):
# verify there is a split for SPLIT_ASSET
splits = self.adjustment_reader.get_adjustments_for_sid(
"splits",
self.SPLIT_ASSET.sid
)
self.assertEqual(1, len(splits))
split = splits[0]
self.assertEqual(
split[0],
pd.Timestamp("2016-01-06", tz='UTC')
)
# ... but that's it's not applied when using spot value
minutes = self.trading_calendar.minutes_for_sessions_in_range(
self.equity_minute_bar_days[0],
self.equity_minute_bar_days[1]
)
for idx, minute in enumerate(minutes):
bar_data = BarData(self.data_portal, lambda: minute, "minute",
self.trading_calendar)
self.assertEqual(
idx + 1,
bar_data.current(self.SPLIT_ASSET, "price")
)
def test_minute_after_assets_stopped(self):
minutes = self.env.market_minutes_for_day(
self.env.next_trading_day(self.days[-1]))
last_trading_minute = \
self.env.market_minutes_for_day(self.days[-1])[-1]
# this entire day is after both assets have stopped trading
for idx, minute in enumerate(minutes):
bar_data = BarData(self.data_portal, lambda: minute, "minute")
self.assertFalse(bar_data.can_trade(self.ASSET1))
self.assertFalse(bar_data.can_trade(self.ASSET2))
self.assertFalse(bar_data.is_stale(self.ASSET1))
self.assertFalse(bar_data.is_stale(self.ASSET2))
self.check_internal_consistency(bar_data)
for field in ALL_FIELDS:
for asset in self.ASSETS:
asset_value = bar_data.current(asset, field)
if field in OHLCP:
self.assertTrue(np.isnan(asset_value))
elif field == "volume":
self.assertEqual(0, asset_value)
elif field == "last_traded":
self.assertEqual(last_trading_minute, asset_value)
def test_can_trade_equity_same_cal_no_last_price(self):
# self.HILARIOUSLY_ILLIQUID_ASSET's first trade is at
# 2016-01-05 15:20:00+00:00. Make sure that can_trade returns false
# for all minutes in that session before the first trade, and true
# for all minutes afterwards.
cal = get_calendar(self.ASSET1.exchange)
minutes_in_session = cal.minutes_for_session(self.ASSET1.start_date)
for minute in minutes_in_session[0:49]:
bar_data = BarData(
self.data_portal, lambda: minute, "minute", cal
)
self.assertFalse(bar_data.can_trade(
self.HILARIOUSLY_ILLIQUID_ASSET)
)
for minute in minutes_in_session[50:]:
bar_data = BarData(
self.data_portal, lambda: minute, "minute", cal
)
self.assertTrue(bar_data.can_trade(
self.HILARIOUSLY_ILLIQUID_ASSET)
)
def test_spot_price_is_unadjusted(self):
# verify there is a split for SPLIT_ASSET
splits = self.adjustments_reader.get_adjustments_for_sid(
"splits",
self.SPLIT_ASSET.sid
)
self.assertEqual(1, len(splits))
split = splits[0]
self.assertEqual(
split[0],
pd.Timestamp("2016-01-06", tz='UTC')
)
# ... but that's it's not applied when using spot value
minutes = self.env.minutes_for_days_in_range(
start=self.days[0], end=self.days[1]
)
for idx, minute in enumerate(minutes):
bar_data = BarData(self.data_portal, lambda: minute, "minute")
self.assertEqual(
idx + 1,
bar_data.current(self.SPLIT_ASSET, "price")
)
def test_minute_before_assets_trading(self):
# grab minutes that include the day before the asset start
minutes = self.env.market_minutes_for_day(
self.env.previous_trading_day(self.days[0])
)
# this entire day is before either asset has started trading
for idx, minute in enumerate(minutes):
bar_data = BarData(self.data_portal, lambda: minute, "minute")
self.check_internal_consistency(bar_data)
self.assertFalse(bar_data.can_trade(self.ASSET1))
self.assertFalse(bar_data.can_trade(self.ASSET2))
self.assertFalse(bar_data.is_stale(self.ASSET1))
self.assertFalse(bar_data.is_stale(self.ASSET2))
for field in ALL_FIELDS:
for asset in self.ASSETS:
asset_value = bar_data.current(asset, field)
if field in OHLCP:
self.assertTrue(np.isnan(asset_value))
elif field == "volume":
self.assertEqual(0, asset_value)
elif field == "last_traded":
self.assertTrue(asset_value is pd.NaT)
def test_day_before_assets_trading(self):
# use the day before self.bcolz_daily_bar_days[0]
minute = self.get_last_minute_of_session(
self.trading_calendar.previous_session_label(
self.equity_daily_bar_days[0]
)
)
bar_data = BarData(self.data_portal, lambda: minute, "daily",
self.trading_calendar)
self.check_internal_consistency(bar_data)
self.assertFalse(bar_data.can_trade(self.ASSET1))
self.assertFalse(bar_data.can_trade(self.ASSET2))
self.assertFalse(bar_data.is_stale(self.ASSET1))
self.assertFalse(bar_data.is_stale(self.ASSET2))
for field in ALL_FIELDS:
for asset in self.ASSETS:
asset_value = bar_data.current(asset, field)
if field in OHLCP:
self.assertTrue(np.isnan(asset_value))
elif field == "volume":
self.assertEqual(0, asset_value)
elif field == "last_traded":
self.assertTrue(asset_value is pd.NaT)
def test_minute_before_assets_trading(self):
# grab minutes that include the day before the asset start
minutes = self.env.market_minutes_for_day(
self.env.previous_trading_day(self.days[0]))
# this entire day is before either asset has started trading
for idx, minute in enumerate(minutes):
bar_data = BarData(self.data_portal, lambda: minute, "minute")
self.check_internal_consistency(bar_data)
self.assertFalse(bar_data.can_trade(self.ASSET1))
self.assertFalse(bar_data.can_trade(self.ASSET2))
self.assertFalse(bar_data.is_stale(self.ASSET1))
self.assertFalse(bar_data.is_stale(self.ASSET2))
for field in ALL_FIELDS:
for asset in self.ASSETS:
asset_value = bar_data.current(asset, field)
if field in OHLCP:
self.assertTrue(np.isnan(asset_value))
elif field == "volume":
self.assertEqual(0, asset_value)
elif field == "last_traded":
self.assertTrue(asset_value is pd.NaT)
def test_is_stale_at_midnight(self):
bar_data = BarData(
self.data_portal,
lambda: self.bcolz_minute_bar_days[1],
"minute",
)
with handle_non_market_minutes(bar_data):
self.assertTrue(bar_data.is_stale(self.HILARIOUSLY_ILLIQUID_ASSET))
def test_is_stale_during_non_market_hours(self):
bar_data = BarData(
self.data_portal,
lambda: self.equity_minute_bar_days[1],
"minute",
)
with handle_non_market_minutes(bar_data):
self.assertTrue(bar_data.is_stale(self.HILARIOUSLY_ILLIQUID_ASSET))
def test_is_stale_during_non_market_hours(self):
bar_data = BarData(
self.data_portal,
lambda: self.equity_minute_bar_days[1],
"minute",
)
with handle_non_market_minutes(bar_data):
self.assertTrue(bar_data.is_stale(self.HILARIOUSLY_ILLIQUID_ASSET))
def test_is_stale_at_midnight(self):
bar_data = BarData(
self.data_portal,
lambda: self.bcolz_minute_bar_days[1],
"minute",
)
with handle_non_market_minutes(bar_data):
self.assertTrue(bar_data.is_stale(self.HILARIOUSLY_ILLIQUID_ASSET))
def test_minute_of_last_day(self):
minutes = self.env.market_minutes_for_day(self.days[-1])
# this is the last day the assets exist
for idx, minute in enumerate(minutes):
bar_data = BarData(self.data_portal, lambda: minute, "minute")
self.assertTrue(bar_data.can_trade(self.ASSET1))
self.assertTrue(bar_data.can_trade(self.ASSET2))
def test_minute_of_last_day(self):
minutes = self.env.market_minutes_for_day(self.days[-1])
# this is the last day the assets exist
for idx, minute in enumerate(minutes):
bar_data = BarData(self.data_portal, lambda: minute, "minute")
self.assertTrue(bar_data.can_trade(self.ASSET1))
self.assertTrue(bar_data.can_trade(self.ASSET2))
def test_minute_of_last_day(self):
minutes = self.trading_schedule.execution_minutes_for_day(
self.bcolz_daily_bar_days[-1], )
# this is the last day the assets exist
for idx, minute in enumerate(minutes):
bar_data = BarData(self.data_portal, lambda: minute, "minute")
self.assertTrue(bar_data.can_trade(self.ASSET1))
self.assertTrue(bar_data.can_trade(self.ASSET2))
def test_minute_of_last_day(self):
minutes = self.trading_calendar.minutes_for_session(
self.equity_daily_bar_days[-1], )
# this is the last day the assets exist
for idx, minute in enumerate(minutes):
bar_data = BarData(self.data_portal, lambda: minute, "minute")
self.assertTrue(bar_data.can_trade(self.ASSET1))
self.assertTrue(bar_data.can_trade(self.ASSET2))
def test_spot_price_is_adjusted_if_needed(self):
# on cls.days[1], the first 9 minutes of ILLIQUID_SPLIT_ASSET are
# missing. let's get them.
day0_minutes = self.trading_calendar.minutes_for_session(
self.equity_minute_bar_days[0])
day1_minutes = self.trading_calendar.minutes_for_session(
self.equity_minute_bar_days[1])
for idx, minute in enumerate(day0_minutes[-10:-1]):
bar_data = BarData(self.data_portal, lambda: minute, "minute")
self.assertEqual(
380, bar_data.current(self.ILLIQUID_SPLIT_ASSET, "price"))
bar_data = BarData(self.data_portal, lambda: day0_minutes[-1],
"minute")
self.assertEqual(390,
bar_data.current(self.ILLIQUID_SPLIT_ASSET, "price"))
for idx, minute in enumerate(day1_minutes[0:9]):
bar_data = BarData(self.data_portal, lambda: minute, "minute")
# should be half of 390, due to the split
self.assertEqual(
195, bar_data.current(self.ILLIQUID_SPLIT_ASSET, "price"))
def test_minute_of_last_day(self):
minutes = self.trading_schedule.execution_minutes_for_day(
self.equity_daily_bar_days[-1],
)
# this is the last day the assets exist
for idx, minute in enumerate(minutes):
bar_data = BarData(self.data_portal, lambda: minute, "minute")
self.assertTrue(bar_data.can_trade(self.ASSET1))
self.assertTrue(bar_data.can_trade(self.ASSET2))
def test_can_trade_during_non_market_hours(self):
# make sure that if we use `can_trade` at midnight, we don't pretend
# we're in the previous day's last minute
the_day_after = self.trading_calendar.next_session_label(
self.equity_minute_bar_days[-1])
bar_data = BarData(self.data_portal, lambda: the_day_after, "minute")
for asset in [self.ASSET1, self.HILARIOUSLY_ILLIQUID_ASSET]:
self.assertFalse(bar_data.can_trade(asset))
with handle_non_market_minutes(bar_data):
self.assertFalse(bar_data.can_trade(asset))
# NYSE is closed at midnight, so even if the asset is alive, can_trade
# should return False
bar_data2 = BarData(
self.data_portal,
lambda: self.equity_minute_bar_days[1],
"minute",
)
for asset in [self.ASSET1, self.HILARIOUSLY_ILLIQUID_ASSET]:
self.assertFalse(bar_data2.can_trade(asset))
with handle_non_market_minutes(bar_data2):
self.assertFalse(bar_data2.can_trade(asset))
def test_can_trade_at_midnight(self):
# make sure that if we use `can_trade` at midnight, we don't pretend
# we're in the previous day's last minute
the_day_after = self.trading_schedule.next_execution_day(
self.bcolz_minute_bar_days[-1])
bar_data = BarData(self.data_portal, lambda: the_day_after, "minute")
for asset in [self.ASSET1, self.HILARIOUSLY_ILLIQUID_ASSET]:
self.assertFalse(bar_data.can_trade(asset))
with handle_non_market_minutes(bar_data):
self.assertFalse(bar_data.can_trade(asset))
# but make sure it works when the assets are alive
bar_data2 = BarData(
self.data_portal,
lambda: self.bcolz_minute_bar_days[1],
"minute",
)
for asset in [self.ASSET1, self.HILARIOUSLY_ILLIQUID_ASSET]:
self.assertTrue(bar_data2.can_trade(asset))
with handle_non_market_minutes(bar_data2):
self.assertTrue(bar_data2.can_trade(asset))
def test_can_trade_multiple_exchange_closed(self):
nyse_asset = self.asset_finder.retrieve_asset(1)
ice_asset = self.asset_finder.retrieve_asset(6)
# minutes we're going to check (to verify that that the same bardata
# can check multiple exchange calendars, all times Eastern):
# 2016-01-05:
# 20:00 (minute before ICE opens)
# 20:01 (first minute of ICE session)
# 20:02 (second minute of ICE session)
# 00:00 (Cinderella's ride becomes a pumpkin)
# 2016-01-06:
# 9:30 (minute before NYSE opens)
# 9:31 (first minute of NYSE session)
# 9:32 (second minute of NYSE session)
# 15:59 (second-to-last minute of NYSE session)
# 16:00 (last minute of NYSE session)
# 16:01 (minute after NYSE closed)
# 17:59 (second-to-last minute of ICE session)
# 18:00 (last minute of ICE session)
# 18:01 (minute after ICE closed)
# each row is dt, whether-nyse-is-open, whether-ice-is-open
minutes_to_check = [
(pd.Timestamp("2016-01-05 20:00", tz="US/Eastern"), False, False),
(pd.Timestamp("2016-01-05 20:01", tz="US/Eastern"), False, True),
(pd.Timestamp("2016-01-05 20:02", tz="US/Eastern"), False, True),
(pd.Timestamp("2016-01-06 00:00", tz="US/Eastern"), False, True),
(pd.Timestamp("2016-01-06 9:30", tz="US/Eastern"), False, True),
(pd.Timestamp("2016-01-06 9:31", tz="US/Eastern"), True, True),
(pd.Timestamp("2016-01-06 9:32", tz="US/Eastern"), True, True),
(pd.Timestamp("2016-01-06 15:59", tz="US/Eastern"), True, True),
(pd.Timestamp("2016-01-06 16:00", tz="US/Eastern"), True, True),
(pd.Timestamp("2016-01-06 16:01", tz="US/Eastern"), False, True),
(pd.Timestamp("2016-01-06 17:59", tz="US/Eastern"), False, True),
(pd.Timestamp("2016-01-06 18:00", tz="US/Eastern"), False, True),
(pd.Timestamp("2016-01-06 18:01", tz="US/Eastern"), False, False),
]
for info in minutes_to_check:
# use the CME calendar, which covers 24 hours
bar_data = BarData(self.data_portal, lambda: info[0], "minute",
trading_calendar=get_calendar("CME"))
series = bar_data.can_trade([nyse_asset, ice_asset])
self.assertEqual(info[1], series.loc[nyse_asset])
self.assertEqual(info[2], series.loc[ice_asset])
def validateSectorUniverse(candidate_sector_universe: Universe,
zipline_data: BarData):
"""Function to validate a candidate sector universe. Ensures that
Zipline can look up all tickers.
Arguments:
candidate_sector_universe {Universe} -- Candidate sector universe.
zipline_data {BarData} -- Instance zipline data bundle.
Raises:
SymbolNotFound -- Raised when a symbol is not found.
"""
for ticker in candidate_sector_universe.getUniqueTickers():
try:
symbol(ticker)
if not zipline_data.can_trade(symbol(ticker)):
raise NoDataForSid
except (SymbolNotFound, NoDataForSid):
# Updating invalid ticker in the universe
candidate_sector_universe.removeInvalidTicker(
invalid_ticker=ticker)
logging.info(
'Ticker {0} in universe not in Zipline; removing'.format(
ticker))
# Return 'clean' sector universe
return candidate_sector_universe
def _create_bar_data(self, universe_func):
return BarData(
data_portal=self.data_portal,
simulation_dt_func=self.get_simulation_dt,
data_frequency=self.sim_params.data_frequency,
universe_func=universe_func
)
def test_orders_stop(self, name, order_data, event_data, expected):
tempdir = TempDirectory()
try:
data = order_data
data['sid'] = self.ASSET133
order = Order(**data)
assets = {
133: pd.DataFrame({
"open": [event_data["open"]],
"high": [event_data["high"]],
"low": [event_data["low"]],
"close": [event_data["close"]],
"volume": [event_data["volume"]],
"dt": [pd.Timestamp('2006-01-05 14:31', tz='UTC')]
}).set_index("dt")
}
write_bcolz_minute_data(
self.env,
pd.date_range(
start=normalize_date(self.minutes[0]),
end=normalize_date(self.minutes[-1])
),
tempdir.path,
assets
)
equity_minute_reader = BcolzMinuteBarReader(tempdir.path)
data_portal = DataPortal(
self.env,
equity_minute_reader=equity_minute_reader,
)
slippage_model = VolumeShareSlippage()
try:
dt = pd.Timestamp('2006-01-05 14:31', tz='UTC')
bar_data = BarData(data_portal,
lambda: dt,
'minute')
_, txn = next(slippage_model.simulate(
bar_data,
self.ASSET133,
[order],
))
except StopIteration:
txn = None
if expected['transaction'] is None:
self.assertIsNone(txn)
else:
self.assertIsNotNone(txn)
for key, value in expected['transaction'].items():
self.assertEquals(value, txn[key])
finally:
tempdir.cleanup()
def getCurrentPrice(self,
zipline_data: BarData,
alloc_weights: np.array = None) -> float:
"""Compute and return the current asset price (using stored
component asset allocation weights) or supplied override allocation
weights.
Arguments:
zipline_data {BarData} -- Instance zipline data bundle.
Keyword Arguments:
alloc_weights {np.array} -- Alternate allocation weights to be used
instead of stored allocation weights
(default: {None}).
Returns:
float -- Synthetic ETF price at the current time step.
"""
# Getting current component asset prices
current_asset_prices = np.array(
zipline_data.current(symbols(*self.tickers), 'price'))
if alloc_weights is None:
alloc_weights = self.alloc_weights
# Comuting current price (dot product b/w current prices and alloc)
return np.dot(current_asset_prices, alloc_weights)
def test_orders_stop(self, name, order_data, event_data, expected):
data = order_data
data['asset'] = self.ASSET133
order = Order(**data)
if expected['transaction']:
expected['transaction']['asset'] = self.ASSET133
event_data['asset'] = self.ASSET133
assets = (
(133, pd.DataFrame(
{
'open': [event_data['open']],
'high': [event_data['high']],
'low': [event_data['low']],
'close': [event_data['close']],
'volume': [event_data['volume']],
},
index=[pd.Timestamp('2006-01-05 14:31', tz='UTC')],
)),
)
days = pd.date_range(
start=normalize_date(self.minutes[0]),
end=normalize_date(self.minutes[-1])
)
with tmp_bcolz_equity_minute_bar_reader(
self.trading_calendar, days, assets) as reader:
data_portal = DataPortal(
self.env.asset_finder, self.trading_calendar,
first_trading_day=reader.first_trading_day,
equity_minute_reader=reader,
)
slippage_model = VolumeShareSlippage()
try:
dt = pd.Timestamp('2006-01-05 14:31', tz='UTC')
bar_data = BarData(
data_portal,
lambda: dt,
self.sim_params.data_frequency,
self.trading_calendar,
NoRestrictions(),
)
_, txn = next(slippage_model.simulate(
bar_data,
self.ASSET133,
[order],
))
except StopIteration:
txn = None
if expected['transaction'] is None:
self.assertIsNone(txn)
else:
self.assertIsNotNone(txn)
for key, value in expected['transaction'].items():
self.assertEquals(value, txn[key])
def test_history_grow_length(self,
freq,
field,
data_frequency,
construct_digest):
bar_count = 2 if construct_digest else 1
spec = history.HistorySpec(
bar_count=bar_count,
frequency=freq,
field=field,
ffill=True,
data_frequency=data_frequency,
)
specs = {spec.key_str: spec}
initial_sids = [1]
initial_dt = pd.Timestamp(
'2013-06-28 13:31'
if data_frequency == 'minute'
else '2013-06-28 12:00AM',
tz='UTC',
)
container = HistoryContainer(
specs, initial_sids, initial_dt, data_frequency,
)
if construct_digest:
self.assertEqual(
container.digest_panels[spec.frequency].window_length, 1,
)
bar_data = BarData()
container.update(bar_data, initial_dt)
to_add = (
history.HistorySpec(
bar_count=bar_count + 1,
frequency=freq,
field=field,
ffill=True,
data_frequency=data_frequency,
),
history.HistorySpec(
bar_count=bar_count + 2,
frequency=freq,
field=field,
ffill=True,
data_frequency=data_frequency,
),
)
for spec in to_add:
container.ensure_spec(spec, initial_dt, bar_data)
self.assertEqual(
container.digest_panels[spec.frequency].window_length,
spec.bar_count - 1,
)
self.assert_history(container, spec, initial_dt)
def _create_bar_data(self, universe_func):
return BarData(data_portal=self.data_portal,
simulation_dt_func=self.get_simulation_dt,
data_frequency=self.sim_params.data_frequency,
trading_calendar=self.algo.trading_calendar,
restrictions=self.restrictions,
universe_func=universe_func)
def create_bardata(self, simulation_dt_func, restrictions=None):
return BarData(
self.data_portal,
simulation_dt_func,
self.CREATE_BARDATA_DATA_FREQUENCY,
self.trading_calendar,
restrictions or NoRestrictions()
)
def test_last_active_day(self):
bar_data = BarData(self.data_portal, lambda: self.days[-1], "daily")
self.check_internal_consistency(bar_data)
for asset in self.ASSETS:
self.assertTrue(bar_data.can_trade(asset))
self.assertFalse(bar_data.is_stale(asset))
self.assertEqual(6, bar_data.current(asset, "open"))
self.assertEqual(7, bar_data.current(asset, "high"))
self.assertEqual(4, bar_data.current(asset, "low"))
self.assertEqual(5, bar_data.current(asset, "close"))
self.assertEqual(500, bar_data.current(asset, "volume"))
self.assertEqual(5, bar_data.current(asset, "price"))
def test_after_assets_dead(self):
# both assets end on self.day[-1], so let's try the next day
minute = self.get_last_minute_of_session(
self.trading_calendar.next_session_label(
self.equity_daily_bar_days[-1]))
bar_data = BarData(self.data_portal, lambda: minute, "daily")
self.check_internal_consistency(bar_data)
for asset in self.ASSETS:
self.assertFalse(bar_data.can_trade(asset))
self.assertFalse(bar_data.is_stale(asset))
for field in OHLCP:
self.assertTrue(np.isnan(bar_data.current(asset, field)))
self.assertEqual(0, bar_data.current(asset, "volume"))
last_traded_dt = bar_data.current(asset, "last_traded")
if asset == self.ASSET1:
self.assertEqual(self.equity_daily_bar_days[-2],
last_traded_dt)
else:
self.assertEqual(self.equity_daily_bar_days[1], last_traded_dt)
def test_get_value_during_non_market_hours(self):
# make sure that if we try to get the OHLCV values of ASSET1 during
# non-market hours, we don't get the previous market minute's values
futures_cal = get_calendar("us_futures")
data_portal = DataPortal(
self.env.asset_finder,
futures_cal,
first_trading_day=self.DATA_PORTAL_FIRST_TRADING_DAY,
equity_minute_reader=self.bcolz_equity_minute_bar_reader,
)
bar_data = BarData(
data_portal,
lambda: pd.Timestamp("2016-01-06 3:15", tz="US/Eastern"),
"minute",
futures_cal
)
self.assertTrue(np.isnan(bar_data.current(self.ASSET1, "open")))
self.assertTrue(np.isnan(bar_data.current(self.ASSET1, "high")))
self.assertTrue(np.isnan(bar_data.current(self.ASSET1, "low")))
self.assertTrue(np.isnan(bar_data.current(self.ASSET1, "close")))
self.assertEqual(0, bar_data.current(self.ASSET1, "volume"))
# price should still forward fill
self.assertEqual(390, bar_data.current(self.ASSET1, "price"))
def test_multiple_specs_on_same_bar(self):
"""
Test that a ffill and non ffill spec both get
the correct results when called on the same tick
"""
spec = history.HistorySpec(
bar_count=3,
frequency='1m',
field='price',
ffill=True,
data_frequency='minute',
env=self.env,
)
no_fill_spec = history.HistorySpec(
bar_count=3,
frequency='1m',
field='price',
ffill=False,
data_frequency='minute',
env=self.env,
)
specs = {spec.key_str: spec, no_fill_spec.key_str: no_fill_spec}
initial_sids = [
1,
]
initial_dt = pd.Timestamp('2013-06-28 9:31AM',
tz='US/Eastern').tz_convert('UTC')
container = HistoryContainer(
specs,
initial_sids,
initial_dt,
'minute',
env=self.env,
)
bar_data = BarData()
container.update(bar_data, initial_dt)
# Add data on bar two of first day.
second_bar_dt = pd.Timestamp('2013-06-28 9:32AM',
tz='US/Eastern').tz_convert('UTC')
bar_data[1] = {'price': 10, 'dt': second_bar_dt}
container.update(bar_data, second_bar_dt)
third_bar_dt = pd.Timestamp('2013-06-28 9:33AM',
tz='US/Eastern').tz_convert('UTC')
del bar_data[1]
# add nan for 3rd bar
container.update(bar_data, third_bar_dt)
prices = container.get_history(spec, third_bar_dt)
no_fill_prices = container.get_history(no_fill_spec, third_bar_dt)
self.assertEqual(prices.values[-1], 10)
self.assertTrue(np.isnan(no_fill_prices.values[-1]),
"Last price should be np.nan")
def test_can_trade_equity_same_cal_exchange_closed(self):
cal = get_calendar(self.ASSET1.exchange)
# verify that can_trade returns true for minutes that are
# outside the asset's calendar (assuming the asset is alive and
# there is a last price), because the asset is alive on the
# next market minute.
minutes = cal.minutes_for_sessions_in_range(
self.ASSET1.start_date,
self.ASSET1.end_date
)
for minute in minutes:
bar_data = BarData(
self.data_portal, lambda: minute, "minute", cal
)
self.assertTrue(bar_data.can_trade(self.ASSET1))
def test_can_trade_equity_same_cal_outside_lifetime(self):
cal = get_calendar(self.ASSET1.exchange)
# verify that can_trade returns False for the session before the
# asset's first session
session_before_asset1_start = cal.previous_session_label(
self.ASSET1.start_date
)
minutes_for_session = cal.minutes_for_session(
session_before_asset1_start
)
# for good measure, check the minute before the session too
minutes_to_check = chain(
[minutes_for_session[0] - pd.Timedelta(minutes=1)],
minutes_for_session
)
for minute in minutes_to_check:
bar_data = BarData(
self.data_portal, lambda: minute, "minute", cal
)
self.assertFalse(bar_data.can_trade(self.ASSET1))
# after asset lifetime
session_after_asset1_end = cal.next_session_label(
self.ASSET1.end_date
)
bts_after_asset1_end = session_after_asset1_end.replace(
hour=8, minute=45
).tz_convert(None).tz_localize("US/Eastern")
minutes_to_check = chain(
cal.minutes_for_session(session_after_asset1_end),
[bts_after_asset1_end]
)
for minute in minutes_to_check:
bar_data = BarData(
self.data_portal, lambda: minute, "minute", cal
)
self.assertFalse(bar_data.can_trade(self.ASSET1))
def test_history_add_field(self, bar_count, freq, pair, data_frequency):
first, second = pair
spec = history.HistorySpec(
bar_count=bar_count,
frequency=freq,
field=first,
ffill=True,
data_frequency=data_frequency,
env=self.env,
)
specs = {spec.key_str: spec}
initial_sids = [1]
initial_dt = pd.Timestamp(
'2013-06-28 13:31'
if data_frequency == 'minute' else '2013-06-28 12:00AM',
tz='UTC',
)
container = HistoryContainer(specs,
initial_sids,
initial_dt,
data_frequency,
env=self.env)
if bar_count > 1:
self.assertEqual(
container.digest_panels[spec.frequency].window_length,
1,
)
bar_data = BarData()
container.update(bar_data, initial_dt)
new_spec = history.HistorySpec(
bar_count,
frequency=freq,
field=second,
ffill=True,
data_frequency=data_frequency,
env=self.env,
)
container.ensure_spec(new_spec, initial_dt, bar_data)
if bar_count > 1:
digest_panel = container.digest_panels[new_spec.frequency]
self.assertEqual(digest_panel.window_length, bar_count - 1)
self.assertIn(second, digest_panel.items)
else:
self.assertNotIn(new_spec.frequency, container.digest_panels)
with warnings.catch_warnings():
warnings.simplefilter('ignore')
self.assert_history(container, new_spec, initial_dt)
def test_overnight_adjustments(self):
# verify there is a split for SPLIT_ASSET
splits = self.adjustment_reader.get_adjustments_for_sid(
"splits",
self.SPLIT_ASSET.sid
)
self.assertEqual(1, len(splits))
split = splits[0]
self.assertEqual(
split[0],
pd.Timestamp("2016-01-06", tz='UTC')
)
# Current day is 1/06/16
day = self.equity_daily_bar_days[1]
eight_fortyfive_am_eastern = \
pd.Timestamp("{0}-{1}-{2} 8:45".format(
day.year, day.month, day.day),
tz='US/Eastern'
)
bar_data = BarData(self.data_portal,
lambda: eight_fortyfive_am_eastern,
"minute",
self.trading_calendar)
expected = {
'open': 391 / 2.0,
'high': 392 / 2.0,
'low': 389 / 2.0,
'close': 390 / 2.0,
'volume': 39000 * 2.0,
'price': 390 / 2.0,
}
with handle_non_market_minutes(bar_data):
for field in OHLCP + ['volume']:
value = bar_data.current(self.SPLIT_ASSET, field)
# Assert the price is adjusted for the overnight split
self.assertEqual(value, expected[field])
def test_spot_price_adjustments(self,
adjustment_type,
liquid_day_0_price,
liquid_day_1_price,
illiquid_day_0_price,
illiquid_day_1_price_adjusted):
"""Test the behaviour of spot prices during adjustments."""
table_name = adjustment_type + 's'
liquid_asset = getattr(self, (adjustment_type.upper() + "_ASSET"))
illiquid_asset = getattr(
self,
("ILLIQUID_" + adjustment_type.upper() + "_ASSET")
)
# verify there is an adjustment for liquid_asset
adjustments = self.adjustments_reader.get_adjustments_for_sid(
table_name,
liquid_asset.sid
)
self.assertEqual(1, len(adjustments))
adjustment = adjustments[0]
self.assertEqual(
adjustment[0],
pd.Timestamp("2016-01-06", tz='UTC')
)
# ... but that's it's not applied when using spot value
bar_data = BarData(self.data_portal, lambda: self.days[0], "daily")
self.assertEqual(
liquid_day_0_price,
bar_data.current(liquid_asset, "price")
)
bar_data = BarData(self.data_portal, lambda: self.days[1], "daily")
self.assertEqual(
liquid_day_1_price,
bar_data.current(liquid_asset, "price")
)
# ... except when we have to forward fill across a day boundary
# ILLIQUID_ASSET has no data on days 0 and 2, and a split on day 2
bar_data = BarData(self.data_portal, lambda: self.days[1], "daily")
self.assertEqual(
illiquid_day_0_price, bar_data.current(illiquid_asset, "price")
)
bar_data = BarData(self.data_portal, lambda: self.days[2], "daily")
# 3 (price from previous day) * 0.5 (split ratio)
self.assertAlmostEqual(
illiquid_day_1_price_adjusted,
bar_data.current(illiquid_asset, "price")
)
def test_last_active_day(self):
bar_data = BarData(self.data_portal, lambda: self.days[-1], "daily")
self.check_internal_consistency(bar_data)
for asset in self.ASSETS:
self.assertTrue(bar_data.can_trade(asset))
self.assertFalse(bar_data.is_stale(asset))
self.assertEqual(6, bar_data.current(asset, "open"))
self.assertEqual(7, bar_data.current(asset, "high"))
self.assertEqual(4, bar_data.current(asset, "low"))
self.assertEqual(5, bar_data.current(asset, "close"))
self.assertEqual(500, bar_data.current(asset, "volume"))
self.assertEqual(5, bar_data.current(asset, "price"))
def test_spot_price_is_adjusted_if_needed(self):
# on cls.days[1], the first 9 minutes of ILLIQUID_SPLIT_ASSET are
# missing. let's get them.
day0_minutes = self.env.market_minutes_for_day(self.days[0])
day1_minutes = self.env.market_minutes_for_day(self.days[1])
for idx, minute in enumerate(day0_minutes[-10:-1]):
bar_data = BarData(self.data_portal, lambda: minute, "minute")
self.assertEqual(
380,
bar_data.current(self.ILLIQUID_SPLIT_ASSET, "price")
)
bar_data = BarData(
self.data_portal, lambda: day0_minutes[-1], "minute"
)
self.assertEqual(
390,
bar_data.current(self.ILLIQUID_SPLIT_ASSET, "price")
)
for idx, minute in enumerate(day1_minutes[0:9]):
bar_data = BarData(self.data_portal, lambda: minute, "minute")
# should be half of 390, due to the split
self.assertEqual(
195,
bar_data.current(self.ILLIQUID_SPLIT_ASSET, "price")
)
def test_after_assets_dead(self):
# both assets end on self.day[-1], so let's try the next day
next_day = self.trading_schedule.next_execution_day(
self.equity_daily_bar_days[-1]
)
bar_data = BarData(self.data_portal, lambda: next_day, "daily")
self.check_internal_consistency(bar_data)
for asset in self.ASSETS:
self.assertFalse(bar_data.can_trade(asset))
self.assertFalse(bar_data.is_stale(asset))
for field in OHLCP:
self.assertTrue(np.isnan(bar_data.current(asset, field)))
self.assertEqual(0, bar_data.current(asset, "volume"))
last_traded_dt = bar_data.current(asset, "last_traded")
if asset == self.ASSET1:
self.assertEqual(self.equity_daily_bar_days[-2],
last_traded_dt)
else:
self.assertEqual(self.equity_daily_bar_days[1], last_traded_dt)
def test_can_trade_at_midnight(self):
# make sure that if we use `can_trade` at midnight, we don't pretend
# we're in the previous day's last minute
the_day_after = self.env.next_trading_day(
self.bcolz_minute_bar_days[-1],
)
bar_data = BarData(self.data_portal, lambda: the_day_after, "minute")
for asset in [self.ASSET1, self.HILARIOUSLY_ILLIQUID_ASSET]:
self.assertFalse(bar_data.can_trade(asset))
with handle_non_market_minutes(bar_data):
self.assertFalse(bar_data.can_trade(asset))
# but make sure it works when the assets are alive
bar_data2 = BarData(
self.data_portal,
lambda: self.bcolz_minute_bar_days[1],
"minute",
)
for asset in [self.ASSET1, self.HILARIOUSLY_ILLIQUID_ASSET]:
self.assertTrue(bar_data2.can_trade(asset))
with handle_non_market_minutes(bar_data2):
self.assertTrue(bar_data2.can_trade(asset))
def test_after_assets_dead(self):
session = self.END_DATE
bar_data = BarData(self.data_portal, lambda: session, "daily",
self.trading_calendar)
self.check_internal_consistency(bar_data)
for asset in self.ASSETS:
self.assertFalse(bar_data.can_trade(asset))
self.assertFalse(bar_data.is_stale(asset))
for field in OHLCP:
self.assertTrue(np.isnan(bar_data.current(asset, field)))
self.assertEqual(0, bar_data.current(asset, "volume"))
last_traded_dt = bar_data.current(asset, "last_traded")
if asset in (self.ASSET1, self.ASSET2):
self.assertEqual(self.equity_daily_bar_days[3],
last_traded_dt)
def test_semi_active_day(self):
# on self.days[0], only asset1 has data
bar_data = BarData(self.data_portal, lambda: self.days[0], "daily")
self.check_internal_consistency(bar_data)
self.assertTrue(bar_data.can_trade(self.ASSET1))
self.assertFalse(bar_data.can_trade(self.ASSET2))
# because there is real data
self.assertFalse(bar_data.is_stale(self.ASSET1))
# because there has never been a trade bar yet
self.assertFalse(bar_data.is_stale(self.ASSET2))
self.assertEqual(3, bar_data.current(self.ASSET1, "open"))
self.assertEqual(4, bar_data.current(self.ASSET1, "high"))
self.assertEqual(1, bar_data.current(self.ASSET1, "low"))
self.assertEqual(2, bar_data.current(self.ASSET1, "close"))
self.assertEqual(200, bar_data.current(self.ASSET1, "volume"))
self.assertEqual(2, bar_data.current(self.ASSET1, "price"))
self.assertEqual(self.days[0],
bar_data.current(self.ASSET1, "last_traded"))
for field in OHLCP:
self.assertTrue(np.isnan(bar_data.current(self.ASSET2, field)),
field)
self.assertEqual(0, bar_data.current(self.ASSET2, "volume"))
self.assertTrue(
bar_data.current(self.ASSET2, "last_traded") is pd.NaT
)
def test_last_active_day(self):
bar_data = BarData(
self.data_portal,
lambda: self.get_last_minute_of_session(
self.equity_daily_bar_days[-1]
),
"daily",
self.trading_calendar
)
self.check_internal_consistency(bar_data)
for asset in self.ASSETS:
if asset in (1, 2):
self.assertFalse(bar_data.can_trade(asset))
else:
self.assertTrue(bar_data.can_trade(asset))
self.assertFalse(bar_data.is_stale(asset))
if asset in (1, 2):
assert_almost_equal(nan, bar_data.current(asset, "open"))
assert_almost_equal(nan, bar_data.current(asset, "high"))
assert_almost_equal(nan, bar_data.current(asset, "low"))
assert_almost_equal(nan, bar_data.current(asset, "close"))
assert_almost_equal(0, bar_data.current(asset, "volume"))
assert_almost_equal(nan, bar_data.current(asset, "price"))
else:
self.assertEqual(6, bar_data.current(asset, "open"))
self.assertEqual(7, bar_data.current(asset, "high"))
self.assertEqual(4, bar_data.current(asset, "low"))
self.assertEqual(5, bar_data.current(asset, "close"))
self.assertEqual(500, bar_data.current(asset, "volume"))
self.assertEqual(5, bar_data.current(asset, "price"))
def test_regular_minute(self):
minutes = self.env.market_minutes_for_day(self.days[0])
for idx, minute in enumerate(minutes):
# day2 has prices
# (every minute for asset1, every 10 minutes for asset2)
# asset1:
# opens: 2-391
# high: 3-392
# low: 0-389
# close: 1-390
# volume: 100-3900 (by 100)
# asset2 is the same thing, but with only every 10th minute
# populated.
# this test covers the "IPO morning" case, because asset2 only
# has data starting on the 10th minute.
bar_data = BarData(self.data_portal, lambda: minute, "minute")
self.check_internal_consistency(bar_data)
asset2_has_data = (((idx + 1) % 10) == 0)
self.assertTrue(bar_data.can_trade(self.ASSET1))
self.assertFalse(bar_data.is_stale(self.ASSET1))
if idx < 9:
self.assertFalse(bar_data.can_trade(self.ASSET2))
self.assertFalse(bar_data.is_stale(self.ASSET2))
else:
self.assertTrue(bar_data.can_trade(self.ASSET2))
if asset2_has_data:
self.assertFalse(bar_data.is_stale(self.ASSET2))
else:
self.assertTrue(bar_data.is_stale(self.ASSET2))
for field in ALL_FIELDS:
asset1_value = bar_data.current(self.ASSET1, field)
asset2_value = bar_data.current(self.ASSET2, field)
# now check the actual values
if idx == 0 and field == "low":
# first low value is 0, which is interpreted as NaN
self.assertTrue(np.isnan(asset1_value))
else:
if field in OHLC:
self.assertEqual(
idx + 1 + field_info[field],
asset1_value
)
if asset2_has_data:
self.assertEqual(
idx + 1 + field_info[field],
asset2_value
)
else:
self.assertTrue(np.isnan(asset2_value))
elif field == "volume":
self.assertEqual((idx + 1) * 100, asset1_value)
if asset2_has_data:
self.assertEqual((idx + 1) * 100, asset2_value)
else:
self.assertEqual(0, asset2_value)
elif field == "price":
self.assertEqual(idx + 1, asset1_value)
if asset2_has_data:
self.assertEqual(idx + 1, asset2_value)
elif idx < 9:
# no price to forward fill from
self.assertTrue(np.isnan(asset2_value))
else:
# forward-filled price
self.assertEqual((idx // 10) * 10, asset2_value)
elif field == "last_traded":
self.assertEqual(minute, asset1_value)
if idx < 9:
self.assertTrue(asset2_value is pd.NaT)
elif asset2_has_data:
self.assertEqual(minute, asset2_value)
else:
last_traded_minute = minutes[(idx // 10) * 10]
self.assertEqual(last_traded_minute - 1,
asset2_value)
def test_old_new_data_api_paths(self):
"""
Test that the new and old data APIs hit the same code paths.
We want to ensure that the old data API(data[sid(N)].field and
similar) and the new data API(data.current(sid(N), field) and
similar) hit the same code paths on the DataPortal.
"""
test_start_minute = self.env.market_minutes_for_day(
self.sim_params.trading_days[0]
)[1]
test_end_minute = self.env.market_minutes_for_day(
self.sim_params.trading_days[0]
)[-1]
bar_data = BarData(
self.data_portal,
lambda: test_end_minute, "minute"
)
ohlcvp_fields = [
"open",
"high",
"low"
"close",
"volume",
"price",
]
spot_value_meth = 'zipline.data.data_portal.DataPortal.get_spot_value'
def assert_get_spot_value_called(fun, field):
"""
Assert that get_spot_value was called during the execution of fun.
Takes in a function fun and a string field.
"""
with patch(spot_value_meth) as gsv:
fun()
gsv.assert_called_with(
self.asset1,
field,
test_end_minute,
'minute'
)
# Ensure that data.current(sid(n), field) has the same behaviour as
# data[sid(n)].field.
for field in ohlcvp_fields:
assert_get_spot_value_called(
lambda: getattr(bar_data[self.asset1], field),
field,
)
assert_get_spot_value_called(
lambda: bar_data.current(self.asset1, field),
field,
)
history_meth = 'zipline.data.data_portal.DataPortal.get_history_window'
def assert_get_history_window_called(fun, is_legacy):
"""
Assert that get_history_window was called during fun().
Takes in a function fun and a boolean is_legacy.
"""
with patch(history_meth) as ghw:
fun()
# Slightly hacky, but done to get around the fact that
# history( explicitly passes an ffill param as the last arg,
# while data.history doesn't.
if is_legacy:
ghw.assert_called_with(
[self.asset1, self.asset2, self.asset3],
test_end_minute,
5,
"1m",
"volume",
True
)
else:
ghw.assert_called_with(
[self.asset1, self.asset2, self.asset3],
test_end_minute,
5,
"1m",
"volume",
)
test_sim_params = SimulationParameters(
period_start=test_start_minute,
period_end=test_end_minute,
data_frequency="minute",
env=self.env
)
history_algorithm = self.create_algo(
history_algo,
sim_params=test_sim_params
)
assert_get_history_window_called(
lambda: history_algorithm.run(self.data_portal),
is_legacy=True
)
assert_get_history_window_called(
lambda: bar_data.history(
[self.asset1, self.asset2, self.asset3],
"volume",
5,
"1m"
),
is_legacy=False
)