def test_expiring_cache(self):
expiry_1 = Timestamp('2014')
before_1 = expiry_1 - Timedelta('1 minute')
after_1 = expiry_1 + Timedelta('1 minute')
expiry_2 = Timestamp('2015')
after_2 = expiry_1 + Timedelta('1 minute')
expiry_3 = Timestamp('2016')
cache = ExpiringCache()
cache.set('foo', 1, expiry_1)
cache.set('bar', 2, expiry_2)
self.assertEqual(cache.get('foo', before_1), 1)
# Unwrap on expiry is allowed.
self.assertEqual(cache.get('foo', expiry_1), 1)
with self.assertRaises(KeyError) as e:
self.assertEqual(cache.get('foo', after_1))
self.assertEqual(e.exception.args, ('foo',))
# Should raise same KeyError after deletion.
with self.assertRaises(KeyError) as e:
self.assertEqual(cache.get('foo', before_1))
self.assertEqual(e.exception.args, ('foo',))
# Second value should still exist.
self.assertEqual(cache.get('bar', after_2), 2)
# Should raise similar KeyError on non-existent key.
with self.assertRaises(KeyError) as e:
self.assertEqual(cache.get('baz', expiry_3))
self.assertEqual(e.exception.args, ('baz',))
def test_expiring_cache(self):
expiry_1 = Timestamp("2014")
before_1 = expiry_1 - Timedelta("1 minute")
after_1 = expiry_1 + Timedelta("1 minute")
expiry_2 = Timestamp("2015")
after_2 = expiry_1 + Timedelta("1 minute")
expiry_3 = Timestamp("2016")
cache = ExpiringCache()
cache.set("foo", 1, expiry_1)
cache.set("bar", 2, expiry_2)
self.assertEqual(cache.get("foo", before_1), 1)
# Unwrap on expiry is allowed.
self.assertEqual(cache.get("foo", expiry_1), 1)
with self.assertRaises(KeyError) as e:
self.assertEqual(cache.get("foo", after_1))
self.assertEqual(e.exception.args, ("foo", ))
# Should raise same KeyError after deletion.
with self.assertRaises(KeyError) as e:
self.assertEqual(cache.get("foo", before_1))
self.assertEqual(e.exception.args, ("foo", ))
# Second value should still exist.
self.assertEqual(cache.get("bar", after_2), 2)
# Should raise similar KeyError on non-existent key.
with self.assertRaises(KeyError) as e:
self.assertEqual(cache.get("baz", expiry_3))
self.assertEqual(e.exception.args, ("baz", ))
def __init__(self,
trading_calendar,
reader,
equity_adjustment_reader,
asset_finder,
roll_finders=None,
sid_cache_size=1000,
prefetch_length=0):
self.trading_calendar = trading_calendar
self._asset_finder = asset_finder
self._reader = reader
self._adjustment_readers = {}
if equity_adjustment_reader is not None:
self._adjustment_readers[Equity] = \
HistoryCompatibleUSEquityAdjustmentReader(
equity_adjustment_reader)
if roll_finders:
self._adjustment_readers[ContinuousFuture] =\
ContinuousFutureAdjustmentReader(trading_calendar,
asset_finder,
reader,
roll_finders,
self._frequency)
self._window_blocks = {
field: ExpiringCache(LRU(sid_cache_size))
for field in self.FIELDS
}
self._prefetch_length = prefetch_length
def __init__(self, env, reader, adjustment_reader, sid_cache_size=1000):
self.env = env
self._reader = reader
self._adjustments_reader = adjustment_reader
self._window_blocks = {
field: ExpiringCache(LRUCache(maxsize=sid_cache_size))
for field in self.FIELDS
}
def __init__(self, trading_schedule, reader, adjustment_reader,
sid_cache_size=1000):
self.trading_schedule = trading_schedule
self._reader = reader
self._adjustments_reader = adjustment_reader
self._window_blocks = {
field: ExpiringCache(LRUCache(maxsize=sid_cache_size))
for field in self.FIELDS
}
def __init__(self, trading_calendar, reader, adjustment_reader,
sid_cache_size=1000):
self.trading_calendar = trading_calendar
self._reader = reader
self._adjustments_reader = adjustment_reader
self._window_blocks = {
field: ExpiringCache(LRU(sid_cache_size))
for field in self.FIELDS
}
def __init__(self, trading_calendar, reader, adjustment_reader,
sid_cache_size=1000):
self.trading_calendar = trading_calendar
self._reader = reader
if adjustment_reader is not None:
self._adjustments_reader = \
HistoryCompatibleUSEquityAdjustmentReader(adjustment_reader)
else:
self._adjustments_reader = None
self._window_blocks = {
field: ExpiringCache(LRU(sid_cache_size))
for field in self.FIELDS
}
def __init__(self):
super(MarketImpactBase, self).__init__()
self._window_data_cache = ExpiringCache()
class MarketImpactBase(SlippageModel):
"""
Base class for slippage models which compute a simulated price impact
according to a history lookback.
"""
NO_DATA_VOLATILITY_SLIPPAGE_IMPACT = 10.0 / 10000
def __init__(self):
super(MarketImpactBase, self).__init__()
self._window_data_cache = ExpiringCache()
@abstractmethod
def get_txn_volume(self, data, order):
"""
Return the number of shares we would like to order in this minute.
Parameters
----------
data : BarData
order : Order
Return
------
int : the number of shares
"""
raise NotImplementedError('get_txn_volume')
@abstractmethod
def get_simulated_impact(self,
order,
current_price,
current_volume,
txn_volume,
mean_volume,
volatility):
"""
Calculate simulated price impact.
Parameters
----------
order : The order being processed.
current_price : Current price of the asset being ordered.
current_volume : Volume of the asset being ordered for the current bar.
txn_volume : Number of shares/contracts being ordered.
mean_volume : Trailing ADV of the asset.
volatility : Annualized daily volatility of volume.
Return
------
int : impact on the current price.
"""
raise NotImplementedError('get_simulated_impact')
def process_order(self, data, order):
if order.open_amount == 0:
return None, None
minute_data = data.current(order.asset, ['volume', 'high', 'low'])
mean_volume, volatility = self._get_window_data(data, order.asset, 20)
# Price to use is the average of the minute bar's open and close.
price = np.mean([minute_data['high'], minute_data['low']])
volume = minute_data['volume']
if not volume:
return None, None
txn_volume = int(
min(self.get_txn_volume(data, order), abs(order.open_amount))
)
# If the computed transaction volume is zero or a decimal value, 'int'
# will round it down to zero. In that case just bail.
if txn_volume == 0:
return None, None
if mean_volume == 0 or np.isnan(volatility):
# If this is the first day the contract exists or there is no
# volume history, default to a conservative estimate of impact.
simulated_impact = price * self.NO_DATA_VOLATILITY_SLIPPAGE_IMPACT
else:
simulated_impact = self.get_simulated_impact(
order=order,
current_price=price,
current_volume=volume,
txn_volume=txn_volume,
mean_volume=mean_volume,
volatility=volatility,
)
impacted_price = \
price + math.copysign(simulated_impact, order.direction)
if fill_price_worse_than_limit_price(impacted_price, order):
return None, None
return impacted_price, math.copysign(txn_volume, order.direction)
def _get_window_data(self, data, asset, window_length):
"""
Internal utility method to return the trailing mean volume over the
past 'window_length' days, and volatility of close prices for a
specific asset.
Parameters
----------
data : The BarData from which to fetch the daily windows.
asset : The Asset whose data we are fetching.
window_length : Number of days of history used to calculate the mean
volume and close price volatility.
Returns
-------
(mean volume, volatility)
"""
try:
values = self._window_data_cache.get(asset, data.current_session)
except KeyError:
try:
# Add a day because we want 'window_length' complete days,
# excluding the current day.
volume_history = data.history(
asset, 'volume', window_length + 1, '1d',
)
close_history = data.history(
asset, 'close', window_length + 1, '1d',
)
except HistoryWindowStartsBeforeData:
# If there is not enough data to do a full history call, return
# values as if there was no data.
return 0, np.NaN
# Exclude the first value of the percent change array because it is
# always just NaN.
close_volatility = close_history[:-1].pct_change()[1:].std(
skipna=False,
)
values = {
'volume': volume_history[:-1].mean(),
'close': close_volatility * SQRT_252,
}
self._window_data_cache.set(asset, values, data.current_session)
return values['volume'], values['close']
def __init__(self):
super(MarketImpactBase, self).__init__()
self._window_data_cache = ExpiringCache()
class MarketImpactBase(SlippageModel):
"""
Base class for slippage models which compute a simulated price impact
according to a history lookback.
"""
NO_DATA_VOLATILITY_SLIPPAGE_IMPACT = 10.0 / 10000
def __init__(self):
super(MarketImpactBase, self).__init__()
self._window_data_cache = ExpiringCache()
@abstractmethod
def get_txn_volume(self, data, order):
"""
Return the number of shares we would like to order in this minute.
Parameters
----------
data : BarData
order : Order
Return
------
int : the number of shares
"""
raise NotImplementedError("get_txn_volume")
@abstractmethod
def get_simulated_impact(self, order, current_price, current_volume,
txn_volume, mean_volume, volatility):
"""
Calculate simulated price impact.
Parameters
----------
order : The order being processed.
current_price : Current price of the asset being ordered.
current_volume : Volume of the asset being ordered for the current bar.
txn_volume : Number of shares/contracts being ordered.
mean_volume : Trailing ADV of the asset.
volatility : Annualized daily volatility of returns.
Return
------
int : impact on the current price.
"""
raise NotImplementedError("get_simulated_impact")
def process_order(self, data, order):
if order.open_amount == 0:
return None, None
minute_data = data.current(order.asset, ["volume", "high", "low"])
mean_volume, volatility = self._get_window_data(data, order.asset, 20)
# Price to use is the average of the minute bar's open and close.
price = np.mean([minute_data["high"], minute_data["low"]])
volume = minute_data["volume"]
if not volume:
return None, None
txn_volume = int(
min(self.get_txn_volume(data, order), abs(order.open_amount)))
# If the computed transaction volume is zero or a decimal value, 'int'
# will round it down to zero. In that case just bail.
if txn_volume == 0:
return None, None
if mean_volume == 0 or np.isnan(volatility):
# If this is the first day the contract exists or there is no
# volume history, default to a conservative estimate of impact.
simulated_impact = price * self.NO_DATA_VOLATILITY_SLIPPAGE_IMPACT
else:
simulated_impact = self.get_simulated_impact(
order=order,
current_price=price,
current_volume=volume,
txn_volume=txn_volume,
mean_volume=mean_volume,
volatility=volatility,
)
impacted_price = price + math.copysign(simulated_impact,
order.direction)
if fill_price_worse_than_limit_price(impacted_price, order):
return None, None
return impacted_price, math.copysign(txn_volume, order.direction)
def _get_window_data(self, data, asset, window_length):
"""
Internal utility method to return the trailing mean volume over the
past 'window_length' days, and volatility of close prices for a
specific asset.
Parameters
----------
data : The BarData from which to fetch the daily windows.
asset : The Asset whose data we are fetching.
window_length : Number of days of history used to calculate the mean
volume and close price volatility.
Returns
-------
(mean volume, volatility)
"""
try:
values = self._window_data_cache.get(asset, data.current_session)
except KeyError:
try:
# Add a day because we want 'window_length' complete days,
# excluding the current day.
volume_history = data.history(asset, "volume",
window_length + 1, "1d")
close_history = data.history(asset, "close", window_length + 1,
"1d")
except HistoryWindowStartsBeforeData:
# If there is not enough data to do a full history call, return
# values as if there was no data.
return 0, np.NaN
# Exclude the first value of the percent change array because it is
# always just NaN.
close_volatility = close_history[:-1].pct_change()[1:].std(
skipna=False)
values = {
"volume": volume_history[:-1].mean(),
"close": close_volatility * SQRT_252,
}
self._window_data_cache.set(asset, values, data.current_session)
return values["volume"], values["close"]
def __init__(self):
super().__init__()
self._window_data_cache = ExpiringCache()
def test_expiring_cache(self):
expiry_1 = pd.Timestamp("2014")
before_1 = expiry_1 - pd.Timedelta("1 minute")
after_1 = expiry_1 + pd.Timedelta("1 minute")
expiry_2 = pd.Timestamp("2015")
after_2 = expiry_1 + pd.Timedelta("1 minute")
expiry_3 = pd.Timestamp("2016")
cache = ExpiringCache()
cache.set("foo", 1, expiry_1)
cache.set("bar", 2, expiry_2)
assert cache.get("foo", before_1) == 1 # Unwrap on expiry is allowed.
assert cache.get("foo", expiry_1) == 1
with pytest.raises(KeyError, match="foo"):
cache.get("foo", after_1)
# Should raise same KeyError after deletion.
with pytest.raises(KeyError, match="foo"):
cache.get("foo", before_1)
# Second value should still exist.
assert cache.get("bar", after_2) == 2
# Should raise similar KeyError on non-existent key.
with pytest.raises(KeyError, match="baz"):
cache.get("baz", expiry_3)
def __init__(self, env, reader, adjustment_reader):
self.env = env
self._reader = reader
self._adjustments_reader = adjustment_reader
# TODO: Split cache into 'by column' and 'by sid'.
self._window_blocks = ExpiringCache(LRUCache(maxsize=5))
class USEquityHistoryLoader(with_metaclass(ABCMeta)):
"""
Loader for sliding history windows of adjusted US Equity Pricing data.
Parameters
----------
reader : DailyBarReader, MinuteBarReader
Reader for pricing bars.
adjustment_reader : SQLiteAdjustmentReader
Reader for adjustment data.
"""
def __init__(self, env, reader, adjustment_reader):
self.env = env
self._reader = reader
self._adjustments_reader = adjustment_reader
# TODO: Split cache into 'by column' and 'by sid'.
self._window_blocks = ExpiringCache(LRUCache(maxsize=5))
@abstractproperty
def _prefetch_length(self):
pass
@abstractproperty
def _calendar(self):
pass
@abstractmethod
def _array(self, start, end, assets, field):
pass
def _get_adjustments_in_range(self, assets, dts, field):
"""
Get the Float64Multiply objects to pass to an AdjustedArrayWindow.
For the use of AdjustedArrayWindow in the loader, which looks back
from current simulation time back to a window of data the dictionary is
structured with:
- the key into the dictionary for adjustments is the location of the
day from which the window is being viewed.
- the start of all multiply objects is always 0 (in each window all
adjustments are overlapping)
- the end of the multiply object is the location before the calendar
location of the adjustment action, making all days before the event
adjusted.
Parameters
----------
assets : iterable of Asset
The assets for which to get adjustments.
days : iterable of datetime64-like
The days for which adjustment data is needed.
field : str
OHLCV field for which to get the adjustments.
Returns
-------
out : The adjustments as a dict of loc -> Float64Multiply
"""
sids = {int(asset): i for i, asset in enumerate(assets)}
start = normalize_date(dts[0])
end = normalize_date(dts[-1])
adjs = {}
for sid, i in iteritems(sids):
if field != 'volume':
mergers = self._adjustments_reader.get_adjustments_for_sid(
'mergers', sid)
for m in mergers:
dt = m[0]
if start < dt <= end:
end_loc = dts.searchsorted(dt)
mult = Float64Multiply(0,
end_loc - 1,
i,
i,
m[1])
try:
adjs[end_loc].append(mult)
except KeyError:
adjs[end_loc] = [mult]
divs = self._adjustments_reader.get_adjustments_for_sid(
'dividends', sid)
for d in divs:
dt = d[0]
if start < dt <= end:
end_loc = dts.searchsorted(dt)
mult = Float64Multiply(0,
end_loc - 1,
i,
i,
d[1])
try:
adjs[end_loc].append(mult)
except KeyError:
adjs[end_loc] = [mult]
splits = self._adjustments_reader.get_adjustments_for_sid(
'splits', sid)
for s in splits:
dt = s[0]
if field == 'volume':
ratio = 1.0 / s[1]
else:
ratio = s[1]
if start < dt <= end:
end_loc = dts.searchsorted(dt)
mult = Float64Multiply(0,
end_loc - 1,
i,
i,
ratio)
try:
adjs[end_loc].append(mult)
except KeyError:
adjs[end_loc] = [mult]
return adjs
def _ensure_sliding_window(
self, assets, dts, field):
"""
Ensure that there is a Float64Multiply window that can provide data
for the given parameters.
If the corresponding window for the (assets, len(dts), field) does not
exist, then create a new one.
If a corresponding window does exist for (assets, len(dts), field), but
can not provide data for the current dts range, then create a new
one and replace the expired window.
WARNING: A simulation with a high variance of assets, may cause
unbounded growth of floating windows stored in `_window_blocks`.
There should be some regular clean up of the cache, if stale windows
prevent simulations from completing because of memory constraints.
Parameters
----------
assets : iterable of Assets
The assets in the window
dts : iterable of datetime64-like
The datetimes for which to fetch data.
Makes an assumption that all dts are present and contiguous,
in the calendar.
field : str
The OHLCV field for which to retrieve data.
Returns
-------
out : Float64Window with sufficient data so that the window can
provide `get` for the index corresponding with the last value in `dts`
"""
end = dts[-1]
size = len(dts)
assets_key = frozenset(assets)
try:
return self._window_blocks.get((assets_key, field, size), end)
except KeyError:
pass
start = dts[0]
offset = 0
start_ix = self._calendar.get_loc(start)
end_ix = self._calendar.get_loc(end)
cal = self._calendar
prefetch_end_ix = min(end_ix + self._prefetch_length, len(cal) - 1)
prefetch_end = cal[prefetch_end_ix]
prefetch_dts = cal[start_ix:prefetch_end_ix + 1]
array = self._array(prefetch_dts, assets, field)
if self._adjustments_reader:
adjs = self._get_adjustments_in_range(assets, prefetch_dts, field)
else:
adjs = {}
if field == 'volume':
array = array.astype('float64')
dtype_ = dtype('float64')
window = Float64Window(
array,
dtype_,
adjs,
offset,
size
)
block = SlidingWindow(window, size, start_ix, offset)
self._window_blocks.set((assets_key, field, size),
block,
prefetch_end)
return block
def history(self, assets, dts, field):
"""
A window of pricing data with adjustments applied assuming that the
end of the window is the day before the current simulation time.
Parameters
----------
assets : iterable of Assets
The assets in the window.
dts : iterable of datetime64-like
The datetimes for which to fetch data.
Makes an assumption that all dts are present and contiguous,
in the calendar.
field : str
The OHLCV field for which to retrieve data.
Returns
-------
out : np.ndarray with shape(len(days between start, end), len(assets))
"""
block = self._ensure_sliding_window(assets, dts, field)
end_ix = self._calendar.get_loc(dts[-1])
return block.get(end_ix)
