def test_expiring_cache(self): expiry_1 = Timestamp('2014') before_1 = expiry_1 - Timedelta('1 minute') after_1 = expiry_1 + Timedelta('1 minute') expiry_2 = Timestamp('2015') after_2 = expiry_1 + Timedelta('1 minute') expiry_3 = Timestamp('2016') cache = ExpiringCache() cache.set('foo', 1, expiry_1) cache.set('bar', 2, expiry_2) self.assertEqual(cache.get('foo', before_1), 1) # Unwrap on expiry is allowed. self.assertEqual(cache.get('foo', expiry_1), 1) with self.assertRaises(KeyError) as e: self.assertEqual(cache.get('foo', after_1)) self.assertEqual(e.exception.args, ('foo',)) # Should raise same KeyError after deletion. with self.assertRaises(KeyError) as e: self.assertEqual(cache.get('foo', before_1)) self.assertEqual(e.exception.args, ('foo',)) # Second value should still exist. self.assertEqual(cache.get('bar', after_2), 2) # Should raise similar KeyError on non-existent key. with self.assertRaises(KeyError) as e: self.assertEqual(cache.get('baz', expiry_3)) self.assertEqual(e.exception.args, ('baz',))
def test_expiring_cache(self): expiry_1 = Timestamp("2014") before_1 = expiry_1 - Timedelta("1 minute") after_1 = expiry_1 + Timedelta("1 minute") expiry_2 = Timestamp("2015") after_2 = expiry_1 + Timedelta("1 minute") expiry_3 = Timestamp("2016") cache = ExpiringCache() cache.set("foo", 1, expiry_1) cache.set("bar", 2, expiry_2) self.assertEqual(cache.get("foo", before_1), 1) # Unwrap on expiry is allowed. self.assertEqual(cache.get("foo", expiry_1), 1) with self.assertRaises(KeyError) as e: self.assertEqual(cache.get("foo", after_1)) self.assertEqual(e.exception.args, ("foo", )) # Should raise same KeyError after deletion. with self.assertRaises(KeyError) as e: self.assertEqual(cache.get("foo", before_1)) self.assertEqual(e.exception.args, ("foo", )) # Second value should still exist. self.assertEqual(cache.get("bar", after_2), 2) # Should raise similar KeyError on non-existent key. with self.assertRaises(KeyError) as e: self.assertEqual(cache.get("baz", expiry_3)) self.assertEqual(e.exception.args, ("baz", ))
def __init__(self, trading_calendar, reader, equity_adjustment_reader, asset_finder, roll_finders=None, sid_cache_size=1000, prefetch_length=0): self.trading_calendar = trading_calendar self._asset_finder = asset_finder self._reader = reader self._adjustment_readers = {} if equity_adjustment_reader is not None: self._adjustment_readers[Equity] = \ HistoryCompatibleUSEquityAdjustmentReader( equity_adjustment_reader) if roll_finders: self._adjustment_readers[ContinuousFuture] =\ ContinuousFutureAdjustmentReader(trading_calendar, asset_finder, reader, roll_finders, self._frequency) self._window_blocks = { field: ExpiringCache(LRU(sid_cache_size)) for field in self.FIELDS } self._prefetch_length = prefetch_length
def __init__(self, env, reader, adjustment_reader, sid_cache_size=1000): self.env = env self._reader = reader self._adjustments_reader = adjustment_reader self._window_blocks = { field: ExpiringCache(LRUCache(maxsize=sid_cache_size)) for field in self.FIELDS }
def __init__(self, trading_schedule, reader, adjustment_reader, sid_cache_size=1000): self.trading_schedule = trading_schedule self._reader = reader self._adjustments_reader = adjustment_reader self._window_blocks = { field: ExpiringCache(LRUCache(maxsize=sid_cache_size)) for field in self.FIELDS }
def __init__(self, trading_calendar, reader, adjustment_reader, sid_cache_size=1000): self.trading_calendar = trading_calendar self._reader = reader self._adjustments_reader = adjustment_reader self._window_blocks = { field: ExpiringCache(LRU(sid_cache_size)) for field in self.FIELDS }
def __init__(self, trading_calendar, reader, adjustment_reader, sid_cache_size=1000): self.trading_calendar = trading_calendar self._reader = reader if adjustment_reader is not None: self._adjustments_reader = \ HistoryCompatibleUSEquityAdjustmentReader(adjustment_reader) else: self._adjustments_reader = None self._window_blocks = { field: ExpiringCache(LRU(sid_cache_size)) for field in self.FIELDS }
def __init__(self): super(MarketImpactBase, self).__init__() self._window_data_cache = ExpiringCache()
class MarketImpactBase(SlippageModel): """ Base class for slippage models which compute a simulated price impact according to a history lookback. """ NO_DATA_VOLATILITY_SLIPPAGE_IMPACT = 10.0 / 10000 def __init__(self): super(MarketImpactBase, self).__init__() self._window_data_cache = ExpiringCache() @abstractmethod def get_txn_volume(self, data, order): """ Return the number of shares we would like to order in this minute. Parameters ---------- data : BarData order : Order Return ------ int : the number of shares """ raise NotImplementedError('get_txn_volume') @abstractmethod def get_simulated_impact(self, order, current_price, current_volume, txn_volume, mean_volume, volatility): """ Calculate simulated price impact. Parameters ---------- order : The order being processed. current_price : Current price of the asset being ordered. current_volume : Volume of the asset being ordered for the current bar. txn_volume : Number of shares/contracts being ordered. mean_volume : Trailing ADV of the asset. volatility : Annualized daily volatility of volume. Return ------ int : impact on the current price. """ raise NotImplementedError('get_simulated_impact') def process_order(self, data, order): if order.open_amount == 0: return None, None minute_data = data.current(order.asset, ['volume', 'high', 'low']) mean_volume, volatility = self._get_window_data(data, order.asset, 20) # Price to use is the average of the minute bar's open and close. price = np.mean([minute_data['high'], minute_data['low']]) volume = minute_data['volume'] if not volume: return None, None txn_volume = int( min(self.get_txn_volume(data, order), abs(order.open_amount)) ) # If the computed transaction volume is zero or a decimal value, 'int' # will round it down to zero. In that case just bail. if txn_volume == 0: return None, None if mean_volume == 0 or np.isnan(volatility): # If this is the first day the contract exists or there is no # volume history, default to a conservative estimate of impact. simulated_impact = price * self.NO_DATA_VOLATILITY_SLIPPAGE_IMPACT else: simulated_impact = self.get_simulated_impact( order=order, current_price=price, current_volume=volume, txn_volume=txn_volume, mean_volume=mean_volume, volatility=volatility, ) impacted_price = \ price + math.copysign(simulated_impact, order.direction) if fill_price_worse_than_limit_price(impacted_price, order): return None, None return impacted_price, math.copysign(txn_volume, order.direction) def _get_window_data(self, data, asset, window_length): """ Internal utility method to return the trailing mean volume over the past 'window_length' days, and volatility of close prices for a specific asset. Parameters ---------- data : The BarData from which to fetch the daily windows. asset : The Asset whose data we are fetching. window_length : Number of days of history used to calculate the mean volume and close price volatility. Returns ------- (mean volume, volatility) """ try: values = self._window_data_cache.get(asset, data.current_session) except KeyError: try: # Add a day because we want 'window_length' complete days, # excluding the current day. volume_history = data.history( asset, 'volume', window_length + 1, '1d', ) close_history = data.history( asset, 'close', window_length + 1, '1d', ) except HistoryWindowStartsBeforeData: # If there is not enough data to do a full history call, return # values as if there was no data. return 0, np.NaN # Exclude the first value of the percent change array because it is # always just NaN. close_volatility = close_history[:-1].pct_change()[1:].std( skipna=False, ) values = { 'volume': volume_history[:-1].mean(), 'close': close_volatility * SQRT_252, } self._window_data_cache.set(asset, values, data.current_session) return values['volume'], values['close']
class MarketImpactBase(SlippageModel): """ Base class for slippage models which compute a simulated price impact according to a history lookback. """ NO_DATA_VOLATILITY_SLIPPAGE_IMPACT = 10.0 / 10000 def __init__(self): super(MarketImpactBase, self).__init__() self._window_data_cache = ExpiringCache() @abstractmethod def get_txn_volume(self, data, order): """ Return the number of shares we would like to order in this minute. Parameters ---------- data : BarData order : Order Return ------ int : the number of shares """ raise NotImplementedError("get_txn_volume") @abstractmethod def get_simulated_impact(self, order, current_price, current_volume, txn_volume, mean_volume, volatility): """ Calculate simulated price impact. Parameters ---------- order : The order being processed. current_price : Current price of the asset being ordered. current_volume : Volume of the asset being ordered for the current bar. txn_volume : Number of shares/contracts being ordered. mean_volume : Trailing ADV of the asset. volatility : Annualized daily volatility of returns. Return ------ int : impact on the current price. """ raise NotImplementedError("get_simulated_impact") def process_order(self, data, order): if order.open_amount == 0: return None, None minute_data = data.current(order.asset, ["volume", "high", "low"]) mean_volume, volatility = self._get_window_data(data, order.asset, 20) # Price to use is the average of the minute bar's open and close. price = np.mean([minute_data["high"], minute_data["low"]]) volume = minute_data["volume"] if not volume: return None, None txn_volume = int( min(self.get_txn_volume(data, order), abs(order.open_amount))) # If the computed transaction volume is zero or a decimal value, 'int' # will round it down to zero. In that case just bail. if txn_volume == 0: return None, None if mean_volume == 0 or np.isnan(volatility): # If this is the first day the contract exists or there is no # volume history, default to a conservative estimate of impact. simulated_impact = price * self.NO_DATA_VOLATILITY_SLIPPAGE_IMPACT else: simulated_impact = self.get_simulated_impact( order=order, current_price=price, current_volume=volume, txn_volume=txn_volume, mean_volume=mean_volume, volatility=volatility, ) impacted_price = price + math.copysign(simulated_impact, order.direction) if fill_price_worse_than_limit_price(impacted_price, order): return None, None return impacted_price, math.copysign(txn_volume, order.direction) def _get_window_data(self, data, asset, window_length): """ Internal utility method to return the trailing mean volume over the past 'window_length' days, and volatility of close prices for a specific asset. Parameters ---------- data : The BarData from which to fetch the daily windows. asset : The Asset whose data we are fetching. window_length : Number of days of history used to calculate the mean volume and close price volatility. Returns ------- (mean volume, volatility) """ try: values = self._window_data_cache.get(asset, data.current_session) except KeyError: try: # Add a day because we want 'window_length' complete days, # excluding the current day. volume_history = data.history(asset, "volume", window_length + 1, "1d") close_history = data.history(asset, "close", window_length + 1, "1d") except HistoryWindowStartsBeforeData: # If there is not enough data to do a full history call, return # values as if there was no data. return 0, np.NaN # Exclude the first value of the percent change array because it is # always just NaN. close_volatility = close_history[:-1].pct_change()[1:].std( skipna=False) values = { "volume": volume_history[:-1].mean(), "close": close_volatility * SQRT_252, } self._window_data_cache.set(asset, values, data.current_session) return values["volume"], values["close"]
def __init__(self): super().__init__() self._window_data_cache = ExpiringCache()
def test_expiring_cache(self): expiry_1 = pd.Timestamp("2014") before_1 = expiry_1 - pd.Timedelta("1 minute") after_1 = expiry_1 + pd.Timedelta("1 minute") expiry_2 = pd.Timestamp("2015") after_2 = expiry_1 + pd.Timedelta("1 minute") expiry_3 = pd.Timestamp("2016") cache = ExpiringCache() cache.set("foo", 1, expiry_1) cache.set("bar", 2, expiry_2) assert cache.get("foo", before_1) == 1 # Unwrap on expiry is allowed. assert cache.get("foo", expiry_1) == 1 with pytest.raises(KeyError, match="foo"): cache.get("foo", after_1) # Should raise same KeyError after deletion. with pytest.raises(KeyError, match="foo"): cache.get("foo", before_1) # Second value should still exist. assert cache.get("bar", after_2) == 2 # Should raise similar KeyError on non-existent key. with pytest.raises(KeyError, match="baz"): cache.get("baz", expiry_3)
def __init__(self, env, reader, adjustment_reader): self.env = env self._reader = reader self._adjustments_reader = adjustment_reader # TODO: Split cache into 'by column' and 'by sid'. self._window_blocks = ExpiringCache(LRUCache(maxsize=5))
class USEquityHistoryLoader(with_metaclass(ABCMeta)): """ Loader for sliding history windows of adjusted US Equity Pricing data. Parameters ---------- reader : DailyBarReader, MinuteBarReader Reader for pricing bars. adjustment_reader : SQLiteAdjustmentReader Reader for adjustment data. """ def __init__(self, env, reader, adjustment_reader): self.env = env self._reader = reader self._adjustments_reader = adjustment_reader # TODO: Split cache into 'by column' and 'by sid'. self._window_blocks = ExpiringCache(LRUCache(maxsize=5)) @abstractproperty def _prefetch_length(self): pass @abstractproperty def _calendar(self): pass @abstractmethod def _array(self, start, end, assets, field): pass def _get_adjustments_in_range(self, assets, dts, field): """ Get the Float64Multiply objects to pass to an AdjustedArrayWindow. For the use of AdjustedArrayWindow in the loader, which looks back from current simulation time back to a window of data the dictionary is structured with: - the key into the dictionary for adjustments is the location of the day from which the window is being viewed. - the start of all multiply objects is always 0 (in each window all adjustments are overlapping) - the end of the multiply object is the location before the calendar location of the adjustment action, making all days before the event adjusted. Parameters ---------- assets : iterable of Asset The assets for which to get adjustments. days : iterable of datetime64-like The days for which adjustment data is needed. field : str OHLCV field for which to get the adjustments. Returns ------- out : The adjustments as a dict of loc -> Float64Multiply """ sids = {int(asset): i for i, asset in enumerate(assets)} start = normalize_date(dts[0]) end = normalize_date(dts[-1]) adjs = {} for sid, i in iteritems(sids): if field != 'volume': mergers = self._adjustments_reader.get_adjustments_for_sid( 'mergers', sid) for m in mergers: dt = m[0] if start < dt <= end: end_loc = dts.searchsorted(dt) mult = Float64Multiply(0, end_loc - 1, i, i, m[1]) try: adjs[end_loc].append(mult) except KeyError: adjs[end_loc] = [mult] divs = self._adjustments_reader.get_adjustments_for_sid( 'dividends', sid) for d in divs: dt = d[0] if start < dt <= end: end_loc = dts.searchsorted(dt) mult = Float64Multiply(0, end_loc - 1, i, i, d[1]) try: adjs[end_loc].append(mult) except KeyError: adjs[end_loc] = [mult] splits = self._adjustments_reader.get_adjustments_for_sid( 'splits', sid) for s in splits: dt = s[0] if field == 'volume': ratio = 1.0 / s[1] else: ratio = s[1] if start < dt <= end: end_loc = dts.searchsorted(dt) mult = Float64Multiply(0, end_loc - 1, i, i, ratio) try: adjs[end_loc].append(mult) except KeyError: adjs[end_loc] = [mult] return adjs def _ensure_sliding_window( self, assets, dts, field): """ Ensure that there is a Float64Multiply window that can provide data for the given parameters. If the corresponding window for the (assets, len(dts), field) does not exist, then create a new one. If a corresponding window does exist for (assets, len(dts), field), but can not provide data for the current dts range, then create a new one and replace the expired window. WARNING: A simulation with a high variance of assets, may cause unbounded growth of floating windows stored in `_window_blocks`. There should be some regular clean up of the cache, if stale windows prevent simulations from completing because of memory constraints. Parameters ---------- assets : iterable of Assets The assets in the window dts : iterable of datetime64-like The datetimes for which to fetch data. Makes an assumption that all dts are present and contiguous, in the calendar. field : str The OHLCV field for which to retrieve data. Returns ------- out : Float64Window with sufficient data so that the window can provide `get` for the index corresponding with the last value in `dts` """ end = dts[-1] size = len(dts) assets_key = frozenset(assets) try: return self._window_blocks.get((assets_key, field, size), end) except KeyError: pass start = dts[0] offset = 0 start_ix = self._calendar.get_loc(start) end_ix = self._calendar.get_loc(end) cal = self._calendar prefetch_end_ix = min(end_ix + self._prefetch_length, len(cal) - 1) prefetch_end = cal[prefetch_end_ix] prefetch_dts = cal[start_ix:prefetch_end_ix + 1] array = self._array(prefetch_dts, assets, field) if self._adjustments_reader: adjs = self._get_adjustments_in_range(assets, prefetch_dts, field) else: adjs = {} if field == 'volume': array = array.astype('float64') dtype_ = dtype('float64') window = Float64Window( array, dtype_, adjs, offset, size ) block = SlidingWindow(window, size, start_ix, offset) self._window_blocks.set((assets_key, field, size), block, prefetch_end) return block def history(self, assets, dts, field): """ A window of pricing data with adjustments applied assuming that the end of the window is the day before the current simulation time. Parameters ---------- assets : iterable of Assets The assets in the window. dts : iterable of datetime64-like The datetimes for which to fetch data. Makes an assumption that all dts are present and contiguous, in the calendar. field : str The OHLCV field for which to retrieve data. Returns ------- out : np.ndarray with shape(len(days between start, end), len(assets)) """ block = self._ensure_sliding_window(assets, dts, field) end_ix = self._calendar.get_loc(dts[-1]) return block.get(end_ix)