def test_minute_after_assets_stopped(self): minutes = self.env.market_minutes_for_day( self.env.next_trading_day(self.days[-1]) ) last_trading_minute = \ self.env.market_minutes_for_day(self.days[-1])[-1] # this entire day is after both assets have stopped trading for idx, minute in enumerate(minutes): bar_data = BarData(self.data_portal, lambda: minute, "minute") self.assertFalse(bar_data.can_trade(self.ASSET1)) self.assertFalse(bar_data.can_trade(self.ASSET2)) self.assertFalse(bar_data.is_stale(self.ASSET1)) self.assertFalse(bar_data.is_stale(self.ASSET2)) self.check_internal_consistency(bar_data) for field in ALL_FIELDS: for asset in self.ASSETS: asset_value = bar_data.current(asset, field) if field in OHLCP: self.assertTrue(np.isnan(asset_value)) elif field == "volume": self.assertEqual(0, asset_value) elif field == "last_traded": self.assertEqual(last_trading_minute, asset_value)
def test_spot_price_at_midnight(self): # make sure that if we try to get a minute price at a non-market # minute, we use the previous market close's timestamp day = self.days[1] eight_fortyfive_am_eastern = \ pd.Timestamp("{0}-{1}-{2} 8:45".format( day.year, day.month, day.day), tz='US/Eastern' ) bar_data = BarData(self.data_portal, lambda: day, "minute") bar_data2 = BarData(self.data_portal, lambda: eight_fortyfive_am_eastern, "minute") with handle_non_market_minutes(bar_data), \ handle_non_market_minutes(bar_data2): for bd in [bar_data, bar_data2]: for field in ["close", "price"]: self.assertEqual(390, bd.current(self.ASSET1, field)) # make sure that if the asset didn't trade at the previous # close, we properly ffill (or not ffill) self.assertEqual( 350, bd.current(self.HILARIOUSLY_ILLIQUID_ASSET, "price")) self.assertTrue( np.isnan( bd.current(self.HILARIOUSLY_ILLIQUID_ASSET, "high"))) self.assertEqual( 0, bd.current(self.HILARIOUSLY_ILLIQUID_ASSET, "volume"))
def test_day_before_assets_trading(self): # use the day before self.equity_daily_bar_days[0] day = self.trading_schedule.previous_execution_day( self.equity_daily_bar_days[0] ) bar_data = BarData(self.data_portal, lambda: day, "daily") self.check_internal_consistency(bar_data) self.assertFalse(bar_data.can_trade(self.ASSET1)) self.assertFalse(bar_data.can_trade(self.ASSET2)) self.assertFalse(bar_data.is_stale(self.ASSET1)) self.assertFalse(bar_data.is_stale(self.ASSET2)) for field in ALL_FIELDS: for asset in self.ASSETS: asset_value = bar_data.current(asset, field) if field in OHLCP: self.assertTrue(np.isnan(asset_value)) elif field == "volume": self.assertEqual(0, asset_value) elif field == "last_traded": self.assertTrue(asset_value is pd.NaT)
def test_overnight_adjustments(self): # verify there is a split for SPLIT_ASSET splits = self.adjustments_reader.get_adjustments_for_sid( "splits", self.SPLIT_ASSET.sid) self.assertEqual(1, len(splits)) split = splits[0] self.assertEqual(split[0], pd.Timestamp("2016-01-06", tz='UTC')) # Current day is 1/06/16 day = self.days[1] eight_fortyfive_am_eastern = \ pd.Timestamp("{0}-{1}-{2} 8:45".format( day.year, day.month, day.day), tz='US/Eastern' ) bar_data = BarData(self.data_portal, lambda: eight_fortyfive_am_eastern, "minute") expected = { 'open': 391 / 2.0, 'high': 392 / 2.0, 'low': 389 / 2.0, 'close': 390 / 2.0, 'volume': 39000 * 2.0, 'price': 390 / 2.0, } with handle_non_market_minutes(bar_data): for field in OHLCP + ['volume']: value = bar_data.current(self.SPLIT_ASSET, field) # Assert the price is adjusted for the overnight split self.assertEqual(value, expected[field])
def test_fully_active_day(self): bar_data = BarData( self.data_portal, lambda: self.get_last_minute_of_session( self.equity_daily_bar_days[1] ), "daily", self.trading_calendar ) self.check_internal_consistency(bar_data) # on self.equity_daily_bar_days[1], both assets have data for asset in self.ASSETS: self.assertTrue(bar_data.can_trade(asset)) self.assertFalse(bar_data.is_stale(asset)) self.assertEqual(4, bar_data.current(asset, "open")) self.assertEqual(5, bar_data.current(asset, "high")) self.assertEqual(2, bar_data.current(asset, "low")) self.assertEqual(3, bar_data.current(asset, "close")) self.assertEqual(300, bar_data.current(asset, "volume")) self.assertEqual(3, bar_data.current(asset, "price")) self.assertEqual( self.equity_daily_bar_days[1], bar_data.current(asset, "last_traded") )
def test_get_value_is_unadjusted(self): # verify there is a split for SPLIT_ASSET splits = self.adjustment_reader.get_adjustments_for_sid( "splits", self.SPLIT_ASSET.sid ) self.assertEqual(1, len(splits)) split = splits[0] self.assertEqual( split[0], pd.Timestamp("2016-01-06", tz='UTC') ) # ... but that's it's not applied when using spot value minutes = self.trading_calendar.minutes_for_sessions_in_range( self.equity_minute_bar_days[0], self.equity_minute_bar_days[1] ) for idx, minute in enumerate(minutes): bar_data = BarData(self.data_portal, lambda: minute, "minute", self.trading_calendar) self.assertEqual( idx + 1, bar_data.current(self.SPLIT_ASSET, "price") )
def test_minute_after_assets_stopped(self): minutes = self.env.market_minutes_for_day( self.env.next_trading_day(self.days[-1])) last_trading_minute = \ self.env.market_minutes_for_day(self.days[-1])[-1] # this entire day is after both assets have stopped trading for idx, minute in enumerate(minutes): bar_data = BarData(self.data_portal, lambda: minute, "minute") self.assertFalse(bar_data.can_trade(self.ASSET1)) self.assertFalse(bar_data.can_trade(self.ASSET2)) self.assertFalse(bar_data.is_stale(self.ASSET1)) self.assertFalse(bar_data.is_stale(self.ASSET2)) self.check_internal_consistency(bar_data) for field in ALL_FIELDS: for asset in self.ASSETS: asset_value = bar_data.current(asset, field) if field in OHLCP: self.assertTrue(np.isnan(asset_value)) elif field == "volume": self.assertEqual(0, asset_value) elif field == "last_traded": self.assertEqual(last_trading_minute, asset_value)
def test_can_trade_equity_same_cal_no_last_price(self): # self.HILARIOUSLY_ILLIQUID_ASSET's first trade is at # 2016-01-05 15:20:00+00:00. Make sure that can_trade returns false # for all minutes in that session before the first trade, and true # for all minutes afterwards. cal = get_calendar(self.ASSET1.exchange) minutes_in_session = cal.minutes_for_session(self.ASSET1.start_date) for minute in minutes_in_session[0:49]: bar_data = BarData( self.data_portal, lambda: minute, "minute", cal ) self.assertFalse(bar_data.can_trade( self.HILARIOUSLY_ILLIQUID_ASSET) ) for minute in minutes_in_session[50:]: bar_data = BarData( self.data_portal, lambda: minute, "minute", cal ) self.assertTrue(bar_data.can_trade( self.HILARIOUSLY_ILLIQUID_ASSET) )
def test_spot_price_is_unadjusted(self): # verify there is a split for SPLIT_ASSET splits = self.adjustments_reader.get_adjustments_for_sid( "splits", self.SPLIT_ASSET.sid ) self.assertEqual(1, len(splits)) split = splits[0] self.assertEqual( split[0], pd.Timestamp("2016-01-06", tz='UTC') ) # ... but that's it's not applied when using spot value minutes = self.env.minutes_for_days_in_range( start=self.days[0], end=self.days[1] ) for idx, minute in enumerate(minutes): bar_data = BarData(self.data_portal, lambda: minute, "minute") self.assertEqual( idx + 1, bar_data.current(self.SPLIT_ASSET, "price") )
def test_minute_before_assets_trading(self): # grab minutes that include the day before the asset start minutes = self.env.market_minutes_for_day( self.env.previous_trading_day(self.days[0]) ) # this entire day is before either asset has started trading for idx, minute in enumerate(minutes): bar_data = BarData(self.data_portal, lambda: minute, "minute") self.check_internal_consistency(bar_data) self.assertFalse(bar_data.can_trade(self.ASSET1)) self.assertFalse(bar_data.can_trade(self.ASSET2)) self.assertFalse(bar_data.is_stale(self.ASSET1)) self.assertFalse(bar_data.is_stale(self.ASSET2)) for field in ALL_FIELDS: for asset in self.ASSETS: asset_value = bar_data.current(asset, field) if field in OHLCP: self.assertTrue(np.isnan(asset_value)) elif field == "volume": self.assertEqual(0, asset_value) elif field == "last_traded": self.assertTrue(asset_value is pd.NaT)
def test_day_before_assets_trading(self): # use the day before self.bcolz_daily_bar_days[0] minute = self.get_last_minute_of_session( self.trading_calendar.previous_session_label( self.equity_daily_bar_days[0] ) ) bar_data = BarData(self.data_portal, lambda: minute, "daily", self.trading_calendar) self.check_internal_consistency(bar_data) self.assertFalse(bar_data.can_trade(self.ASSET1)) self.assertFalse(bar_data.can_trade(self.ASSET2)) self.assertFalse(bar_data.is_stale(self.ASSET1)) self.assertFalse(bar_data.is_stale(self.ASSET2)) for field in ALL_FIELDS: for asset in self.ASSETS: asset_value = bar_data.current(asset, field) if field in OHLCP: self.assertTrue(np.isnan(asset_value)) elif field == "volume": self.assertEqual(0, asset_value) elif field == "last_traded": self.assertTrue(asset_value is pd.NaT)
def test_minute_before_assets_trading(self): # grab minutes that include the day before the asset start minutes = self.env.market_minutes_for_day( self.env.previous_trading_day(self.days[0])) # this entire day is before either asset has started trading for idx, minute in enumerate(minutes): bar_data = BarData(self.data_portal, lambda: minute, "minute") self.check_internal_consistency(bar_data) self.assertFalse(bar_data.can_trade(self.ASSET1)) self.assertFalse(bar_data.can_trade(self.ASSET2)) self.assertFalse(bar_data.is_stale(self.ASSET1)) self.assertFalse(bar_data.is_stale(self.ASSET2)) for field in ALL_FIELDS: for asset in self.ASSETS: asset_value = bar_data.current(asset, field) if field in OHLCP: self.assertTrue(np.isnan(asset_value)) elif field == "volume": self.assertEqual(0, asset_value) elif field == "last_traded": self.assertTrue(asset_value is pd.NaT)
def test_is_stale_at_midnight(self): bar_data = BarData( self.data_portal, lambda: self.bcolz_minute_bar_days[1], "minute", ) with handle_non_market_minutes(bar_data): self.assertTrue(bar_data.is_stale(self.HILARIOUSLY_ILLIQUID_ASSET))
def test_is_stale_during_non_market_hours(self): bar_data = BarData( self.data_portal, lambda: self.equity_minute_bar_days[1], "minute", ) with handle_non_market_minutes(bar_data): self.assertTrue(bar_data.is_stale(self.HILARIOUSLY_ILLIQUID_ASSET))
def test_minute_of_last_day(self): minutes = self.env.market_minutes_for_day(self.days[-1]) # this is the last day the assets exist for idx, minute in enumerate(minutes): bar_data = BarData(self.data_portal, lambda: minute, "minute") self.assertTrue(bar_data.can_trade(self.ASSET1)) self.assertTrue(bar_data.can_trade(self.ASSET2))
def test_minute_of_last_day(self): minutes = self.trading_schedule.execution_minutes_for_day( self.bcolz_daily_bar_days[-1], ) # this is the last day the assets exist for idx, minute in enumerate(minutes): bar_data = BarData(self.data_portal, lambda: minute, "minute") self.assertTrue(bar_data.can_trade(self.ASSET1)) self.assertTrue(bar_data.can_trade(self.ASSET2))
def test_minute_of_last_day(self): minutes = self.trading_calendar.minutes_for_session( self.equity_daily_bar_days[-1], ) # this is the last day the assets exist for idx, minute in enumerate(minutes): bar_data = BarData(self.data_portal, lambda: minute, "minute") self.assertTrue(bar_data.can_trade(self.ASSET1)) self.assertTrue(bar_data.can_trade(self.ASSET2))
def test_spot_price_is_adjusted_if_needed(self): # on cls.days[1], the first 9 minutes of ILLIQUID_SPLIT_ASSET are # missing. let's get them. day0_minutes = self.trading_calendar.minutes_for_session( self.equity_minute_bar_days[0]) day1_minutes = self.trading_calendar.minutes_for_session( self.equity_minute_bar_days[1]) for idx, minute in enumerate(day0_minutes[-10:-1]): bar_data = BarData(self.data_portal, lambda: minute, "minute") self.assertEqual( 380, bar_data.current(self.ILLIQUID_SPLIT_ASSET, "price")) bar_data = BarData(self.data_portal, lambda: day0_minutes[-1], "minute") self.assertEqual(390, bar_data.current(self.ILLIQUID_SPLIT_ASSET, "price")) for idx, minute in enumerate(day1_minutes[0:9]): bar_data = BarData(self.data_portal, lambda: minute, "minute") # should be half of 390, due to the split self.assertEqual( 195, bar_data.current(self.ILLIQUID_SPLIT_ASSET, "price"))
def test_minute_of_last_day(self): minutes = self.trading_schedule.execution_minutes_for_day( self.equity_daily_bar_days[-1], ) # this is the last day the assets exist for idx, minute in enumerate(minutes): bar_data = BarData(self.data_portal, lambda: minute, "minute") self.assertTrue(bar_data.can_trade(self.ASSET1)) self.assertTrue(bar_data.can_trade(self.ASSET2))
def test_can_trade_during_non_market_hours(self): # make sure that if we use `can_trade` at midnight, we don't pretend # we're in the previous day's last minute the_day_after = self.trading_calendar.next_session_label( self.equity_minute_bar_days[-1]) bar_data = BarData(self.data_portal, lambda: the_day_after, "minute") for asset in [self.ASSET1, self.HILARIOUSLY_ILLIQUID_ASSET]: self.assertFalse(bar_data.can_trade(asset)) with handle_non_market_minutes(bar_data): self.assertFalse(bar_data.can_trade(asset)) # NYSE is closed at midnight, so even if the asset is alive, can_trade # should return False bar_data2 = BarData( self.data_portal, lambda: self.equity_minute_bar_days[1], "minute", ) for asset in [self.ASSET1, self.HILARIOUSLY_ILLIQUID_ASSET]: self.assertFalse(bar_data2.can_trade(asset)) with handle_non_market_minutes(bar_data2): self.assertFalse(bar_data2.can_trade(asset))
def test_can_trade_at_midnight(self): # make sure that if we use `can_trade` at midnight, we don't pretend # we're in the previous day's last minute the_day_after = self.trading_schedule.next_execution_day( self.bcolz_minute_bar_days[-1]) bar_data = BarData(self.data_portal, lambda: the_day_after, "minute") for asset in [self.ASSET1, self.HILARIOUSLY_ILLIQUID_ASSET]: self.assertFalse(bar_data.can_trade(asset)) with handle_non_market_minutes(bar_data): self.assertFalse(bar_data.can_trade(asset)) # but make sure it works when the assets are alive bar_data2 = BarData( self.data_portal, lambda: self.bcolz_minute_bar_days[1], "minute", ) for asset in [self.ASSET1, self.HILARIOUSLY_ILLIQUID_ASSET]: self.assertTrue(bar_data2.can_trade(asset)) with handle_non_market_minutes(bar_data2): self.assertTrue(bar_data2.can_trade(asset))
def test_can_trade_multiple_exchange_closed(self): nyse_asset = self.asset_finder.retrieve_asset(1) ice_asset = self.asset_finder.retrieve_asset(6) # minutes we're going to check (to verify that that the same bardata # can check multiple exchange calendars, all times Eastern): # 2016-01-05: # 20:00 (minute before ICE opens) # 20:01 (first minute of ICE session) # 20:02 (second minute of ICE session) # 00:00 (Cinderella's ride becomes a pumpkin) # 2016-01-06: # 9:30 (minute before NYSE opens) # 9:31 (first minute of NYSE session) # 9:32 (second minute of NYSE session) # 15:59 (second-to-last minute of NYSE session) # 16:00 (last minute of NYSE session) # 16:01 (minute after NYSE closed) # 17:59 (second-to-last minute of ICE session) # 18:00 (last minute of ICE session) # 18:01 (minute after ICE closed) # each row is dt, whether-nyse-is-open, whether-ice-is-open minutes_to_check = [ (pd.Timestamp("2016-01-05 20:00", tz="US/Eastern"), False, False), (pd.Timestamp("2016-01-05 20:01", tz="US/Eastern"), False, True), (pd.Timestamp("2016-01-05 20:02", tz="US/Eastern"), False, True), (pd.Timestamp("2016-01-06 00:00", tz="US/Eastern"), False, True), (pd.Timestamp("2016-01-06 9:30", tz="US/Eastern"), False, True), (pd.Timestamp("2016-01-06 9:31", tz="US/Eastern"), True, True), (pd.Timestamp("2016-01-06 9:32", tz="US/Eastern"), True, True), (pd.Timestamp("2016-01-06 15:59", tz="US/Eastern"), True, True), (pd.Timestamp("2016-01-06 16:00", tz="US/Eastern"), True, True), (pd.Timestamp("2016-01-06 16:01", tz="US/Eastern"), False, True), (pd.Timestamp("2016-01-06 17:59", tz="US/Eastern"), False, True), (pd.Timestamp("2016-01-06 18:00", tz="US/Eastern"), False, True), (pd.Timestamp("2016-01-06 18:01", tz="US/Eastern"), False, False), ] for info in minutes_to_check: # use the CME calendar, which covers 24 hours bar_data = BarData(self.data_portal, lambda: info[0], "minute", trading_calendar=get_calendar("CME")) series = bar_data.can_trade([nyse_asset, ice_asset]) self.assertEqual(info[1], series.loc[nyse_asset]) self.assertEqual(info[2], series.loc[ice_asset])
def validateSectorUniverse(candidate_sector_universe: Universe, zipline_data: BarData): """Function to validate a candidate sector universe. Ensures that Zipline can look up all tickers. Arguments: candidate_sector_universe {Universe} -- Candidate sector universe. zipline_data {BarData} -- Instance zipline data bundle. Raises: SymbolNotFound -- Raised when a symbol is not found. """ for ticker in candidate_sector_universe.getUniqueTickers(): try: symbol(ticker) if not zipline_data.can_trade(symbol(ticker)): raise NoDataForSid except (SymbolNotFound, NoDataForSid): # Updating invalid ticker in the universe candidate_sector_universe.removeInvalidTicker( invalid_ticker=ticker) logging.info( 'Ticker {0} in universe not in Zipline; removing'.format( ticker)) # Return 'clean' sector universe return candidate_sector_universe
def _create_bar_data(self, universe_func): return BarData( data_portal=self.data_portal, simulation_dt_func=self.get_simulation_dt, data_frequency=self.sim_params.data_frequency, universe_func=universe_func )
def test_orders_stop(self, name, order_data, event_data, expected): tempdir = TempDirectory() try: data = order_data data['sid'] = self.ASSET133 order = Order(**data) assets = { 133: pd.DataFrame({ "open": [event_data["open"]], "high": [event_data["high"]], "low": [event_data["low"]], "close": [event_data["close"]], "volume": [event_data["volume"]], "dt": [pd.Timestamp('2006-01-05 14:31', tz='UTC')] }).set_index("dt") } write_bcolz_minute_data( self.env, pd.date_range( start=normalize_date(self.minutes[0]), end=normalize_date(self.minutes[-1]) ), tempdir.path, assets ) equity_minute_reader = BcolzMinuteBarReader(tempdir.path) data_portal = DataPortal( self.env, equity_minute_reader=equity_minute_reader, ) slippage_model = VolumeShareSlippage() try: dt = pd.Timestamp('2006-01-05 14:31', tz='UTC') bar_data = BarData(data_portal, lambda: dt, 'minute') _, txn = next(slippage_model.simulate( bar_data, self.ASSET133, [order], )) except StopIteration: txn = None if expected['transaction'] is None: self.assertIsNone(txn) else: self.assertIsNotNone(txn) for key, value in expected['transaction'].items(): self.assertEquals(value, txn[key]) finally: tempdir.cleanup()
def getCurrentPrice(self, zipline_data: BarData, alloc_weights: np.array = None) -> float: """Compute and return the current asset price (using stored component asset allocation weights) or supplied override allocation weights. Arguments: zipline_data {BarData} -- Instance zipline data bundle. Keyword Arguments: alloc_weights {np.array} -- Alternate allocation weights to be used instead of stored allocation weights (default: {None}). Returns: float -- Synthetic ETF price at the current time step. """ # Getting current component asset prices current_asset_prices = np.array( zipline_data.current(symbols(*self.tickers), 'price')) if alloc_weights is None: alloc_weights = self.alloc_weights # Comuting current price (dot product b/w current prices and alloc) return np.dot(current_asset_prices, alloc_weights)
def test_orders_stop(self, name, order_data, event_data, expected): data = order_data data['asset'] = self.ASSET133 order = Order(**data) if expected['transaction']: expected['transaction']['asset'] = self.ASSET133 event_data['asset'] = self.ASSET133 assets = ( (133, pd.DataFrame( { 'open': [event_data['open']], 'high': [event_data['high']], 'low': [event_data['low']], 'close': [event_data['close']], 'volume': [event_data['volume']], }, index=[pd.Timestamp('2006-01-05 14:31', tz='UTC')], )), ) days = pd.date_range( start=normalize_date(self.minutes[0]), end=normalize_date(self.minutes[-1]) ) with tmp_bcolz_equity_minute_bar_reader( self.trading_calendar, days, assets) as reader: data_portal = DataPortal( self.env.asset_finder, self.trading_calendar, first_trading_day=reader.first_trading_day, equity_minute_reader=reader, ) slippage_model = VolumeShareSlippage() try: dt = pd.Timestamp('2006-01-05 14:31', tz='UTC') bar_data = BarData( data_portal, lambda: dt, self.sim_params.data_frequency, self.trading_calendar, NoRestrictions(), ) _, txn = next(slippage_model.simulate( bar_data, self.ASSET133, [order], )) except StopIteration: txn = None if expected['transaction'] is None: self.assertIsNone(txn) else: self.assertIsNotNone(txn) for key, value in expected['transaction'].items(): self.assertEquals(value, txn[key])
def test_history_grow_length(self, freq, field, data_frequency, construct_digest): bar_count = 2 if construct_digest else 1 spec = history.HistorySpec( bar_count=bar_count, frequency=freq, field=field, ffill=True, data_frequency=data_frequency, ) specs = {spec.key_str: spec} initial_sids = [1] initial_dt = pd.Timestamp( '2013-06-28 13:31' if data_frequency == 'minute' else '2013-06-28 12:00AM', tz='UTC', ) container = HistoryContainer( specs, initial_sids, initial_dt, data_frequency, ) if construct_digest: self.assertEqual( container.digest_panels[spec.frequency].window_length, 1, ) bar_data = BarData() container.update(bar_data, initial_dt) to_add = ( history.HistorySpec( bar_count=bar_count + 1, frequency=freq, field=field, ffill=True, data_frequency=data_frequency, ), history.HistorySpec( bar_count=bar_count + 2, frequency=freq, field=field, ffill=True, data_frequency=data_frequency, ), ) for spec in to_add: container.ensure_spec(spec, initial_dt, bar_data) self.assertEqual( container.digest_panels[spec.frequency].window_length, spec.bar_count - 1, ) self.assert_history(container, spec, initial_dt)
def _create_bar_data(self, universe_func): return BarData(data_portal=self.data_portal, simulation_dt_func=self.get_simulation_dt, data_frequency=self.sim_params.data_frequency, trading_calendar=self.algo.trading_calendar, restrictions=self.restrictions, universe_func=universe_func)
def create_bardata(self, simulation_dt_func, restrictions=None): return BarData( self.data_portal, simulation_dt_func, self.CREATE_BARDATA_DATA_FREQUENCY, self.trading_calendar, restrictions or NoRestrictions() )
def test_last_active_day(self): bar_data = BarData(self.data_portal, lambda: self.days[-1], "daily") self.check_internal_consistency(bar_data) for asset in self.ASSETS: self.assertTrue(bar_data.can_trade(asset)) self.assertFalse(bar_data.is_stale(asset)) self.assertEqual(6, bar_data.current(asset, "open")) self.assertEqual(7, bar_data.current(asset, "high")) self.assertEqual(4, bar_data.current(asset, "low")) self.assertEqual(5, bar_data.current(asset, "close")) self.assertEqual(500, bar_data.current(asset, "volume")) self.assertEqual(5, bar_data.current(asset, "price"))
def test_after_assets_dead(self): # both assets end on self.day[-1], so let's try the next day minute = self.get_last_minute_of_session( self.trading_calendar.next_session_label( self.equity_daily_bar_days[-1])) bar_data = BarData(self.data_portal, lambda: minute, "daily") self.check_internal_consistency(bar_data) for asset in self.ASSETS: self.assertFalse(bar_data.can_trade(asset)) self.assertFalse(bar_data.is_stale(asset)) for field in OHLCP: self.assertTrue(np.isnan(bar_data.current(asset, field))) self.assertEqual(0, bar_data.current(asset, "volume")) last_traded_dt = bar_data.current(asset, "last_traded") if asset == self.ASSET1: self.assertEqual(self.equity_daily_bar_days[-2], last_traded_dt) else: self.assertEqual(self.equity_daily_bar_days[1], last_traded_dt)
def test_get_value_during_non_market_hours(self): # make sure that if we try to get the OHLCV values of ASSET1 during # non-market hours, we don't get the previous market minute's values futures_cal = get_calendar("us_futures") data_portal = DataPortal( self.env.asset_finder, futures_cal, first_trading_day=self.DATA_PORTAL_FIRST_TRADING_DAY, equity_minute_reader=self.bcolz_equity_minute_bar_reader, ) bar_data = BarData( data_portal, lambda: pd.Timestamp("2016-01-06 3:15", tz="US/Eastern"), "minute", futures_cal ) self.assertTrue(np.isnan(bar_data.current(self.ASSET1, "open"))) self.assertTrue(np.isnan(bar_data.current(self.ASSET1, "high"))) self.assertTrue(np.isnan(bar_data.current(self.ASSET1, "low"))) self.assertTrue(np.isnan(bar_data.current(self.ASSET1, "close"))) self.assertEqual(0, bar_data.current(self.ASSET1, "volume")) # price should still forward fill self.assertEqual(390, bar_data.current(self.ASSET1, "price"))
def test_multiple_specs_on_same_bar(self): """ Test that a ffill and non ffill spec both get the correct results when called on the same tick """ spec = history.HistorySpec( bar_count=3, frequency='1m', field='price', ffill=True, data_frequency='minute', env=self.env, ) no_fill_spec = history.HistorySpec( bar_count=3, frequency='1m', field='price', ffill=False, data_frequency='minute', env=self.env, ) specs = {spec.key_str: spec, no_fill_spec.key_str: no_fill_spec} initial_sids = [ 1, ] initial_dt = pd.Timestamp('2013-06-28 9:31AM', tz='US/Eastern').tz_convert('UTC') container = HistoryContainer( specs, initial_sids, initial_dt, 'minute', env=self.env, ) bar_data = BarData() container.update(bar_data, initial_dt) # Add data on bar two of first day. second_bar_dt = pd.Timestamp('2013-06-28 9:32AM', tz='US/Eastern').tz_convert('UTC') bar_data[1] = {'price': 10, 'dt': second_bar_dt} container.update(bar_data, second_bar_dt) third_bar_dt = pd.Timestamp('2013-06-28 9:33AM', tz='US/Eastern').tz_convert('UTC') del bar_data[1] # add nan for 3rd bar container.update(bar_data, third_bar_dt) prices = container.get_history(spec, third_bar_dt) no_fill_prices = container.get_history(no_fill_spec, third_bar_dt) self.assertEqual(prices.values[-1], 10) self.assertTrue(np.isnan(no_fill_prices.values[-1]), "Last price should be np.nan")
def test_can_trade_equity_same_cal_exchange_closed(self): cal = get_calendar(self.ASSET1.exchange) # verify that can_trade returns true for minutes that are # outside the asset's calendar (assuming the asset is alive and # there is a last price), because the asset is alive on the # next market minute. minutes = cal.minutes_for_sessions_in_range( self.ASSET1.start_date, self.ASSET1.end_date ) for minute in minutes: bar_data = BarData( self.data_portal, lambda: minute, "minute", cal ) self.assertTrue(bar_data.can_trade(self.ASSET1))
def test_can_trade_equity_same_cal_outside_lifetime(self): cal = get_calendar(self.ASSET1.exchange) # verify that can_trade returns False for the session before the # asset's first session session_before_asset1_start = cal.previous_session_label( self.ASSET1.start_date ) minutes_for_session = cal.minutes_for_session( session_before_asset1_start ) # for good measure, check the minute before the session too minutes_to_check = chain( [minutes_for_session[0] - pd.Timedelta(minutes=1)], minutes_for_session ) for minute in minutes_to_check: bar_data = BarData( self.data_portal, lambda: minute, "minute", cal ) self.assertFalse(bar_data.can_trade(self.ASSET1)) # after asset lifetime session_after_asset1_end = cal.next_session_label( self.ASSET1.end_date ) bts_after_asset1_end = session_after_asset1_end.replace( hour=8, minute=45 ).tz_convert(None).tz_localize("US/Eastern") minutes_to_check = chain( cal.minutes_for_session(session_after_asset1_end), [bts_after_asset1_end] ) for minute in minutes_to_check: bar_data = BarData( self.data_portal, lambda: minute, "minute", cal ) self.assertFalse(bar_data.can_trade(self.ASSET1))
def test_history_add_field(self, bar_count, freq, pair, data_frequency): first, second = pair spec = history.HistorySpec( bar_count=bar_count, frequency=freq, field=first, ffill=True, data_frequency=data_frequency, env=self.env, ) specs = {spec.key_str: spec} initial_sids = [1] initial_dt = pd.Timestamp( '2013-06-28 13:31' if data_frequency == 'minute' else '2013-06-28 12:00AM', tz='UTC', ) container = HistoryContainer(specs, initial_sids, initial_dt, data_frequency, env=self.env) if bar_count > 1: self.assertEqual( container.digest_panels[spec.frequency].window_length, 1, ) bar_data = BarData() container.update(bar_data, initial_dt) new_spec = history.HistorySpec( bar_count, frequency=freq, field=second, ffill=True, data_frequency=data_frequency, env=self.env, ) container.ensure_spec(new_spec, initial_dt, bar_data) if bar_count > 1: digest_panel = container.digest_panels[new_spec.frequency] self.assertEqual(digest_panel.window_length, bar_count - 1) self.assertIn(second, digest_panel.items) else: self.assertNotIn(new_spec.frequency, container.digest_panels) with warnings.catch_warnings(): warnings.simplefilter('ignore') self.assert_history(container, new_spec, initial_dt)
def test_overnight_adjustments(self): # verify there is a split for SPLIT_ASSET splits = self.adjustment_reader.get_adjustments_for_sid( "splits", self.SPLIT_ASSET.sid ) self.assertEqual(1, len(splits)) split = splits[0] self.assertEqual( split[0], pd.Timestamp("2016-01-06", tz='UTC') ) # Current day is 1/06/16 day = self.equity_daily_bar_days[1] eight_fortyfive_am_eastern = \ pd.Timestamp("{0}-{1}-{2} 8:45".format( day.year, day.month, day.day), tz='US/Eastern' ) bar_data = BarData(self.data_portal, lambda: eight_fortyfive_am_eastern, "minute", self.trading_calendar) expected = { 'open': 391 / 2.0, 'high': 392 / 2.0, 'low': 389 / 2.0, 'close': 390 / 2.0, 'volume': 39000 * 2.0, 'price': 390 / 2.0, } with handle_non_market_minutes(bar_data): for field in OHLCP + ['volume']: value = bar_data.current(self.SPLIT_ASSET, field) # Assert the price is adjusted for the overnight split self.assertEqual(value, expected[field])
def test_spot_price_adjustments(self, adjustment_type, liquid_day_0_price, liquid_day_1_price, illiquid_day_0_price, illiquid_day_1_price_adjusted): """Test the behaviour of spot prices during adjustments.""" table_name = adjustment_type + 's' liquid_asset = getattr(self, (adjustment_type.upper() + "_ASSET")) illiquid_asset = getattr( self, ("ILLIQUID_" + adjustment_type.upper() + "_ASSET") ) # verify there is an adjustment for liquid_asset adjustments = self.adjustments_reader.get_adjustments_for_sid( table_name, liquid_asset.sid ) self.assertEqual(1, len(adjustments)) adjustment = adjustments[0] self.assertEqual( adjustment[0], pd.Timestamp("2016-01-06", tz='UTC') ) # ... but that's it's not applied when using spot value bar_data = BarData(self.data_portal, lambda: self.days[0], "daily") self.assertEqual( liquid_day_0_price, bar_data.current(liquid_asset, "price") ) bar_data = BarData(self.data_portal, lambda: self.days[1], "daily") self.assertEqual( liquid_day_1_price, bar_data.current(liquid_asset, "price") ) # ... except when we have to forward fill across a day boundary # ILLIQUID_ASSET has no data on days 0 and 2, and a split on day 2 bar_data = BarData(self.data_portal, lambda: self.days[1], "daily") self.assertEqual( illiquid_day_0_price, bar_data.current(illiquid_asset, "price") ) bar_data = BarData(self.data_portal, lambda: self.days[2], "daily") # 3 (price from previous day) * 0.5 (split ratio) self.assertAlmostEqual( illiquid_day_1_price_adjusted, bar_data.current(illiquid_asset, "price") )
def test_spot_price_is_adjusted_if_needed(self): # on cls.days[1], the first 9 minutes of ILLIQUID_SPLIT_ASSET are # missing. let's get them. day0_minutes = self.env.market_minutes_for_day(self.days[0]) day1_minutes = self.env.market_minutes_for_day(self.days[1]) for idx, minute in enumerate(day0_minutes[-10:-1]): bar_data = BarData(self.data_portal, lambda: minute, "minute") self.assertEqual( 380, bar_data.current(self.ILLIQUID_SPLIT_ASSET, "price") ) bar_data = BarData( self.data_portal, lambda: day0_minutes[-1], "minute" ) self.assertEqual( 390, bar_data.current(self.ILLIQUID_SPLIT_ASSET, "price") ) for idx, minute in enumerate(day1_minutes[0:9]): bar_data = BarData(self.data_portal, lambda: minute, "minute") # should be half of 390, due to the split self.assertEqual( 195, bar_data.current(self.ILLIQUID_SPLIT_ASSET, "price") )
def test_after_assets_dead(self): # both assets end on self.day[-1], so let's try the next day next_day = self.trading_schedule.next_execution_day( self.equity_daily_bar_days[-1] ) bar_data = BarData(self.data_portal, lambda: next_day, "daily") self.check_internal_consistency(bar_data) for asset in self.ASSETS: self.assertFalse(bar_data.can_trade(asset)) self.assertFalse(bar_data.is_stale(asset)) for field in OHLCP: self.assertTrue(np.isnan(bar_data.current(asset, field))) self.assertEqual(0, bar_data.current(asset, "volume")) last_traded_dt = bar_data.current(asset, "last_traded") if asset == self.ASSET1: self.assertEqual(self.equity_daily_bar_days[-2], last_traded_dt) else: self.assertEqual(self.equity_daily_bar_days[1], last_traded_dt)
def test_can_trade_at_midnight(self): # make sure that if we use `can_trade` at midnight, we don't pretend # we're in the previous day's last minute the_day_after = self.env.next_trading_day( self.bcolz_minute_bar_days[-1], ) bar_data = BarData(self.data_portal, lambda: the_day_after, "minute") for asset in [self.ASSET1, self.HILARIOUSLY_ILLIQUID_ASSET]: self.assertFalse(bar_data.can_trade(asset)) with handle_non_market_minutes(bar_data): self.assertFalse(bar_data.can_trade(asset)) # but make sure it works when the assets are alive bar_data2 = BarData( self.data_portal, lambda: self.bcolz_minute_bar_days[1], "minute", ) for asset in [self.ASSET1, self.HILARIOUSLY_ILLIQUID_ASSET]: self.assertTrue(bar_data2.can_trade(asset)) with handle_non_market_minutes(bar_data2): self.assertTrue(bar_data2.can_trade(asset))
def test_after_assets_dead(self): session = self.END_DATE bar_data = BarData(self.data_portal, lambda: session, "daily", self.trading_calendar) self.check_internal_consistency(bar_data) for asset in self.ASSETS: self.assertFalse(bar_data.can_trade(asset)) self.assertFalse(bar_data.is_stale(asset)) for field in OHLCP: self.assertTrue(np.isnan(bar_data.current(asset, field))) self.assertEqual(0, bar_data.current(asset, "volume")) last_traded_dt = bar_data.current(asset, "last_traded") if asset in (self.ASSET1, self.ASSET2): self.assertEqual(self.equity_daily_bar_days[3], last_traded_dt)
def test_semi_active_day(self): # on self.days[0], only asset1 has data bar_data = BarData(self.data_portal, lambda: self.days[0], "daily") self.check_internal_consistency(bar_data) self.assertTrue(bar_data.can_trade(self.ASSET1)) self.assertFalse(bar_data.can_trade(self.ASSET2)) # because there is real data self.assertFalse(bar_data.is_stale(self.ASSET1)) # because there has never been a trade bar yet self.assertFalse(bar_data.is_stale(self.ASSET2)) self.assertEqual(3, bar_data.current(self.ASSET1, "open")) self.assertEqual(4, bar_data.current(self.ASSET1, "high")) self.assertEqual(1, bar_data.current(self.ASSET1, "low")) self.assertEqual(2, bar_data.current(self.ASSET1, "close")) self.assertEqual(200, bar_data.current(self.ASSET1, "volume")) self.assertEqual(2, bar_data.current(self.ASSET1, "price")) self.assertEqual(self.days[0], bar_data.current(self.ASSET1, "last_traded")) for field in OHLCP: self.assertTrue(np.isnan(bar_data.current(self.ASSET2, field)), field) self.assertEqual(0, bar_data.current(self.ASSET2, "volume")) self.assertTrue( bar_data.current(self.ASSET2, "last_traded") is pd.NaT )
def test_last_active_day(self): bar_data = BarData( self.data_portal, lambda: self.get_last_minute_of_session( self.equity_daily_bar_days[-1] ), "daily", self.trading_calendar ) self.check_internal_consistency(bar_data) for asset in self.ASSETS: if asset in (1, 2): self.assertFalse(bar_data.can_trade(asset)) else: self.assertTrue(bar_data.can_trade(asset)) self.assertFalse(bar_data.is_stale(asset)) if asset in (1, 2): assert_almost_equal(nan, bar_data.current(asset, "open")) assert_almost_equal(nan, bar_data.current(asset, "high")) assert_almost_equal(nan, bar_data.current(asset, "low")) assert_almost_equal(nan, bar_data.current(asset, "close")) assert_almost_equal(0, bar_data.current(asset, "volume")) assert_almost_equal(nan, bar_data.current(asset, "price")) else: self.assertEqual(6, bar_data.current(asset, "open")) self.assertEqual(7, bar_data.current(asset, "high")) self.assertEqual(4, bar_data.current(asset, "low")) self.assertEqual(5, bar_data.current(asset, "close")) self.assertEqual(500, bar_data.current(asset, "volume")) self.assertEqual(5, bar_data.current(asset, "price"))
def test_regular_minute(self): minutes = self.env.market_minutes_for_day(self.days[0]) for idx, minute in enumerate(minutes): # day2 has prices # (every minute for asset1, every 10 minutes for asset2) # asset1: # opens: 2-391 # high: 3-392 # low: 0-389 # close: 1-390 # volume: 100-3900 (by 100) # asset2 is the same thing, but with only every 10th minute # populated. # this test covers the "IPO morning" case, because asset2 only # has data starting on the 10th minute. bar_data = BarData(self.data_portal, lambda: minute, "minute") self.check_internal_consistency(bar_data) asset2_has_data = (((idx + 1) % 10) == 0) self.assertTrue(bar_data.can_trade(self.ASSET1)) self.assertFalse(bar_data.is_stale(self.ASSET1)) if idx < 9: self.assertFalse(bar_data.can_trade(self.ASSET2)) self.assertFalse(bar_data.is_stale(self.ASSET2)) else: self.assertTrue(bar_data.can_trade(self.ASSET2)) if asset2_has_data: self.assertFalse(bar_data.is_stale(self.ASSET2)) else: self.assertTrue(bar_data.is_stale(self.ASSET2)) for field in ALL_FIELDS: asset1_value = bar_data.current(self.ASSET1, field) asset2_value = bar_data.current(self.ASSET2, field) # now check the actual values if idx == 0 and field == "low": # first low value is 0, which is interpreted as NaN self.assertTrue(np.isnan(asset1_value)) else: if field in OHLC: self.assertEqual( idx + 1 + field_info[field], asset1_value ) if asset2_has_data: self.assertEqual( idx + 1 + field_info[field], asset2_value ) else: self.assertTrue(np.isnan(asset2_value)) elif field == "volume": self.assertEqual((idx + 1) * 100, asset1_value) if asset2_has_data: self.assertEqual((idx + 1) * 100, asset2_value) else: self.assertEqual(0, asset2_value) elif field == "price": self.assertEqual(idx + 1, asset1_value) if asset2_has_data: self.assertEqual(idx + 1, asset2_value) elif idx < 9: # no price to forward fill from self.assertTrue(np.isnan(asset2_value)) else: # forward-filled price self.assertEqual((idx // 10) * 10, asset2_value) elif field == "last_traded": self.assertEqual(minute, asset1_value) if idx < 9: self.assertTrue(asset2_value is pd.NaT) elif asset2_has_data: self.assertEqual(minute, asset2_value) else: last_traded_minute = minutes[(idx // 10) * 10] self.assertEqual(last_traded_minute - 1, asset2_value)
def test_old_new_data_api_paths(self): """ Test that the new and old data APIs hit the same code paths. We want to ensure that the old data API(data[sid(N)].field and similar) and the new data API(data.current(sid(N), field) and similar) hit the same code paths on the DataPortal. """ test_start_minute = self.env.market_minutes_for_day( self.sim_params.trading_days[0] )[1] test_end_minute = self.env.market_minutes_for_day( self.sim_params.trading_days[0] )[-1] bar_data = BarData( self.data_portal, lambda: test_end_minute, "minute" ) ohlcvp_fields = [ "open", "high", "low" "close", "volume", "price", ] spot_value_meth = 'zipline.data.data_portal.DataPortal.get_spot_value' def assert_get_spot_value_called(fun, field): """ Assert that get_spot_value was called during the execution of fun. Takes in a function fun and a string field. """ with patch(spot_value_meth) as gsv: fun() gsv.assert_called_with( self.asset1, field, test_end_minute, 'minute' ) # Ensure that data.current(sid(n), field) has the same behaviour as # data[sid(n)].field. for field in ohlcvp_fields: assert_get_spot_value_called( lambda: getattr(bar_data[self.asset1], field), field, ) assert_get_spot_value_called( lambda: bar_data.current(self.asset1, field), field, ) history_meth = 'zipline.data.data_portal.DataPortal.get_history_window' def assert_get_history_window_called(fun, is_legacy): """ Assert that get_history_window was called during fun(). Takes in a function fun and a boolean is_legacy. """ with patch(history_meth) as ghw: fun() # Slightly hacky, but done to get around the fact that # history( explicitly passes an ffill param as the last arg, # while data.history doesn't. if is_legacy: ghw.assert_called_with( [self.asset1, self.asset2, self.asset3], test_end_minute, 5, "1m", "volume", True ) else: ghw.assert_called_with( [self.asset1, self.asset2, self.asset3], test_end_minute, 5, "1m", "volume", ) test_sim_params = SimulationParameters( period_start=test_start_minute, period_end=test_end_minute, data_frequency="minute", env=self.env ) history_algorithm = self.create_algo( history_algo, sim_params=test_sim_params ) assert_get_history_window_called( lambda: history_algorithm.run(self.data_portal), is_legacy=True ) assert_get_history_window_called( lambda: bar_data.history( [self.asset1, self.asset2, self.asset3], "volume", 5, "1m" ), is_legacy=False )