def lookup_future_by_expiration(self, root_symbol, as_of_date, ref_date):
""" Find a specific contract in the futures chain by expiration
date.
Parameters
----------
root_symbol : str
Root symbol of the desired future.
as_of_date : pd.Timestamp
Date at the time of the lookup.
ref_date : pd.Timestamp
Reference point for expiration dates.
Returns
-------
Future
The valid contract the has the closest expiration date
after ref_date. If none exists, returns None.
"""
root_symbol.upper()
as_of_date = normalize_date(as_of_date)
ref_date = normalize_date(ref_date)
valid_contracts = self._valid_contracts(root_symbol, as_of_date)
contracts_after_date = (c for c in valid_contracts
if c.expiration_date > ref_date)
return next(contracts_after_date, None)
def _get_range_edges(axis, offset, closed='left', base=0):
if isinstance(offset, basestring):
offset = to_offset(offset)
if isinstance(offset, Tick):
day_nanos = _delta_to_nanoseconds(timedelta(1))
# #1165
if (day_nanos % offset.nanos) == 0:
return _adjust_dates_anchored(axis[0], axis[-1], offset,
closed=closed, base=base)
first, last = axis[0], axis[-1]
if not isinstance(offset, Tick): # and first.time() != last.time():
# hack!
first = tools.normalize_date(first)
last = tools.normalize_date(last)
if closed == 'left':
first = Timestamp(offset.rollback(first))
else:
first = Timestamp(first - offset)
last = Timestamp(last + offset)
return first, last
def _get_range_edges(first, last, offset, closed='left', base=0):
if isinstance(offset, compat.string_types):
offset = to_offset(offset)
if isinstance(offset, Tick):
is_day = isinstance(offset, Day)
day_nanos = _delta_to_nanoseconds(timedelta(1))
# #1165
if (is_day and day_nanos % offset.nanos == 0) or not is_day:
return _adjust_dates_anchored(first, last, offset,
closed=closed, base=base)
if not isinstance(offset, Tick): # and first.time() != last.time():
# hack!
first = tools.normalize_date(first)
last = tools.normalize_date(last)
if closed == 'left':
first = Timestamp(offset.rollback(first))
else:
first = Timestamp(first - offset)
last = Timestamp(last + offset)
return first, last
def test_normalize_date():
value = date(2012, 9, 7)
result = normalize_date(value)
assert (result == datetime(2012, 9, 7))
value = datetime(2012, 9, 7, 12)
result = normalize_date(value)
assert (result == datetime(2012, 9, 7))
def _generate(cls, start, end, periods, name, offset,
tz=None, normalize=False):
_normalized = True
if start is not None:
start = Timestamp(start)
if not isinstance(start, Timestamp):
raise ValueError('Failed to convert %s to timestamp'
% start)
if normalize:
start = normalize_date(start)
_normalized = True
else:
_normalized = _normalized and start.time() == _midnight
if end is not None:
end = Timestamp(end)
if not isinstance(end, Timestamp):
raise ValueError('Failed to convert %s to timestamp'
% end)
if normalize:
end = normalize_date(end)
_normalized = True
else:
_normalized = _normalized and end.time() == _midnight
start, end, tz = tools._figure_out_timezone(start, end, tz)
if (offset._should_cache() and
not (offset._normalize_cache and not _normalized) and
_naive_in_cache_range(start, end)):
index = cls._cached_range(start, end, periods=periods,
offset=offset, name=name)
else:
index = _generate_regular_range(start, end, periods, offset)
if tz is not None:
# Convert local to UTC
ints = index.view('i8')
lib.tz_localize_check(ints, tz)
index = lib.tz_convert(ints, tz, _utc())
index = index.view('M8[us]')
index = index.view(cls)
index.name = name
index.offset = offset
index.tz = tz
return index
def lookup_symbol_resolve_multiple(self, symbol, as_of_date=None):
"""
Return matching Asset of name symbol in database.
If multiple Assets are found and as_of_date is not set,
raises MultipleSymbolsFound.
If no Asset was active at as_of_date, and allow_expired is False
raises SymbolNotFound.
"""
if as_of_date is not None:
as_of_date = normalize_date(as_of_date)
if symbol not in self.sym_cache:
raise SymbolNotFound(symbol=symbol)
infos = self.sym_cache[symbol]
if as_of_date is None:
if len(infos) == 1:
return infos[0]
else:
raise MultipleSymbolsFound(symbol=symbol,
options=str(infos))
# Try to find symbol matching as_of_date
asset, _ = self._lookup_symbol_in_infos(infos, as_of_date)
if asset is None:
raise SymbolNotFound(symbol=symbol)
return asset
def factors(self):
algo = get_algo_instance()
today = normalize_date(algo.get_datetime())
if today > self._factor_matrix_expires:
self._factor_matrix, self._factor_matrix_expires = \
algo.compute_factor_matrix(today)
return self._factor_matrix.loc[today]
def lookup_future_in_chain(self, root_symbol, as_of_date, contract_num=0):
""" Find a specific contract in the futures chain for a given
root symbol.
Parameters
----------
root_symbol : str
Root symbol of the desired future.
as_of_date : pd.Timestamp
Date at the time of the lookup.
contract_num : int
1 for the primary contract, 2 for the secondary, etc.,
relative to as_of_date.
Returns
-------
Future
The (contract_num)th contract in the futures chain. If none
exits, returns None.
"""
root_symbol.upper()
as_of_date = normalize_date(as_of_date)
valid_contracts = self._valid_contracts(root_symbol, as_of_date)
if valid_contracts and contract_num >= 0:
try:
return valid_contracts[contract_num]
except IndexError:
pass
return None
def handle_minute_close(self, dt, data_portal):
"""
Handles the close of the given minute in minute emission.
Parameters
__________
dt : Timestamp
The minute that is ending
Returns
_______
A minute perf packet.
"""
self.position_tracker.sync_last_sale_prices(dt, False, data_portal)
self.update_performance()
todays_date = normalize_date(dt)
account = self.get_account(False)
bench_returns = self.all_benchmark_returns.loc[todays_date:dt]
# cumulative returns
bench_since_open = (1. + bench_returns).prod() - 1
self.cumulative_risk_metrics.update(todays_date,
self.todays_performance.returns,
bench_since_open,
account.leverage)
minute_packet = self.to_dict(emission_type='minute')
return minute_packet
def handle_minute_close(self, dt):
todays_date = normalize_date(dt)
minute_returns = self.minute_performance.returns
self.minute_performance.rollover()
# the intraday risk is calculated on top of minute performance
# returns for the bench and the algo
self.intraday_risk_metrics.update(dt,
minute_returns,
self.all_benchmark_returns[dt])
bench_since_open = \
self.intraday_risk_metrics.benchmark_period_returns[dt]
# if we've reached market close, check on dividends
if dt == self.market_close:
for perf_period in self.perf_periods:
perf_period.update_dividends(todays_date)
self.cumulative_risk_metrics.update(todays_date,
self.todays_performance.returns,
bench_since_open)
# if this is the close, save the returns objects for cumulative
# risk calculations
if dt == self.market_close:
self.returns[todays_date] = self.todays_performance.returns
def __init__(self, sim_params, returns_frequency=None, create_first_day_stats=False):
"""
- @returns_frequency allows for configuration of the whether
the benchmark and algorithm returns are in units of minutes or days,
if `None` defaults to the `emission_rate` in `sim_params`.
"""
self.treasury_curves = trading.environment.treasury_curves
self.start_date = sim_params.period_start.replace(hour=0, minute=0, second=0, microsecond=0)
self.end_date = sim_params.period_end.replace(hour=0, minute=0, second=0, microsecond=0)
self.trading_days = trading.environment.days_in_range(self.start_date, self.end_date)
last_day = normalize_date(sim_params.period_end)
if last_day not in self.trading_days:
last_day = pd.tseries.index.DatetimeIndex([last_day])
self.trading_days = self.trading_days.append(last_day)
self.sim_params = sim_params
self.create_first_day_stats = create_first_day_stats
if returns_frequency is None:
returns_frequency = self.sim_params.emission_rate
self.returns_frequency = returns_frequency
if returns_frequency == "daily":
cont_index = self.get_daily_index()
elif returns_frequency == "minute":
cont_index = self.get_minute_index(sim_params)
self.cont_index = cont_index
self.algorithm_returns_cont = pd.Series(index=cont_index)
self.benchmark_returns_cont = pd.Series(index=cont_index)
# The returns at a given time are read and reset from the respective
# returns container.
self.algorithm_returns = None
self.benchmark_returns = None
self.mean_returns = None
self.annualized_mean_returns = None
self.mean_benchmark_returns = None
self.annualized_benchmark_returns = None
self.algorithm_cumulative_returns = pd.Series(index=cont_index)
self.benchmark_cumulative_returns = pd.Series(index=cont_index)
self.excess_returns = pd.Series(index=cont_index)
self.latest_dt = cont_index[0]
self.metrics = pd.DataFrame(index=cont_index, columns=self.METRIC_NAMES)
self.drawdowns = pd.Series(index=cont_index)
self.max_drawdowns = pd.Series(index=cont_index)
self.max_drawdown = 0
self.current_max = -np.inf
self.daily_treasury = pd.Series(index=self.trading_days)
def _generate(cls, start, end, periods, name, offset,
tz=None, normalize=False):
_normalized = True
if start is not None:
start = Timestamp(start)
if normalize:
start = normalize_date(start)
_normalized = True
else:
_normalized = _normalized and start.time() == _midnight
if end is not None:
end = Timestamp(end)
if normalize:
end = normalize_date(end)
_normalized = True
else:
_normalized = _normalized and end.time() == _midnight
start, end, tz = tools._figure_out_timezone(start, end, tz)
if com._count_not_none(start, end, periods) < 2:
raise ValueError('Must specify two of start, end, or periods')
if (offset._should_cache() and
not (offset._normalize_cache and not _normalized) and
_naive_in_cache_range(start, end)):
index = cls._cached_range(start, end, periods=periods,
offset=offset, name=name)
else:
index = _generate_regular_range(start, end, periods, offset)
if tz is not None:
# Convert local to UTC
ints = index.view('i8', type=np.ndarray)
index = lib.tz_localize_to_utc(ints, tz)
index = index.view(_NS_DTYPE)
index = index.view(cls)
index.name = name
index.offset = offset
index.tz = tz
return index
def _get_datetime(self):
"""
Returns the normalized simulation datetime.
Returns
-------
pandas.Timestamp
The normalized datetime of FutureChain's TradingAlgorithm.
"""
return normalize_date(Timestamp(self._algorithm_get_datetime(), tz="UTC"))
def __init__(self,
metadata=None,
allow_sid_assignment=True,
fuzzy_char=None,
db_path=':memory:',
create_table=True):
self.fuzzy_char = fuzzy_char
# This flag controls if the AssetFinder is allowed to generate its own
# sids. If False, metadata that does not contain a sid will raise an
# exception when building assets.
self.allow_sid_assignment = allow_sid_assignment
if allow_sid_assignment:
self.end_date_to_assign = normalize_date(
pd.Timestamp('now', tz='UTC'))
self.conn = sqlite3.connect(db_path)
self.conn.text_factory = str
self.cursor = self.conn.cursor()
# The AssetFinder also holds a nested-dict of all metadata for
# reference when building Assets
self.metadata_cache = {}
# Create table and read in metadata.
# Should we use flags like 'r', 'w', instead?
# What we need to support is:
# - A 'throwaway' mode where the metadata is read each run.
# - A 'write' mode where the data is written to the provided db_path
# - A 'read' mode where the asset finder uses a prexisting db.
if create_table:
self.create_db_tables()
if metadata is not None:
self.consume_metadata(metadata)
# Cache for lookup of assets by sid, the objects in the asset lookp may
# be shared with the results from equity and future lookup caches.
#
# The top level cache exists to minimize lookups on the asset type
# routing.
#
# The caches are read through, i.e. accessing an asset through
# retrieve_asset, _retrieve_equity etc. will populate the cache on
# first retrieval.
self._asset_cache = {}
self._equity_cache = {}
self._future_cache = {}
self._asset_type_cache = {}
# Populated on first call to `lifetimes`.
self._asset_lifetimes = None
def _adjust_dates_anchored(first, last, offset, closed='right', base=0):
from pandas.tseries.tools import normalize_date
start_day_nanos = Timestamp(normalize_date(first)).value
last_day_nanos = Timestamp(normalize_date(last)).value
base_nanos = (base % offset.n) * offset.nanos // offset.n
start_day_nanos += base_nanos
last_day_nanos += base_nanos
foffset = (first.value - start_day_nanos) % offset.nanos
loffset = (last.value - last_day_nanos) % offset.nanos
if closed == 'right':
if foffset > 0:
# roll back
fresult = first.value - foffset
else:
fresult = first.value - offset.nanos
if loffset > 0:
# roll forward
lresult = last.value + (offset.nanos - loffset)
else:
# already the end of the road
lresult = last.value
else: # closed == 'left'
if foffset > 0:
fresult = first.value - foffset
else:
# start of the road
fresult = first.value
if loffset > 0:
# roll forward
lresult = last.value + (offset.nanos - loffset)
else:
lresult = last.value + offset.nanos
return (Timestamp(fresult, tz=first.tz),
Timestamp(lresult, tz=last.tz))
def lookup_symbol(self, symbol, as_of_date, fuzzy=False):
"""
If a fuzzy string is provided, then we try various symbols based on
the provided symbol. This is to facilitate mapping from a broker's
symbol to ours in cases where mapping to the broker's symbol loses
information. For example, if we have CMCS_A, but a broker has CMCSA,
when the broker provides CMCSA, it can also provide fuzzy='_',
so we can find a match by inserting an underscore.
"""
symbol = symbol.upper()
ad_value = pd.Timestamp(normalize_date(as_of_date)).value
if not fuzzy:
try:
return self.lookup_symbol_resolve_multiple(symbol, as_of_date)
except SymbolNotFound:
return None
fuzzy = symbol.replace(self.fuzzy_char, "")
equities_cols = self.equities.c
candidates = (
sa.select((equities_cols.sid,))
.where(
(equities_cols.fuzzy == fuzzy)
& (equities_cols.start_date <= ad_value)
& (equities_cols.end_date >= ad_value)
)
.execute()
.fetchall()
)
# If one SID exists for symbol, return that symbol
if len(candidates) == 1:
return self._retrieve_equity(candidates[0]["sid"])
# If multiple SIDs exist for symbol, return latest start_date with
# end_date as a tie-breaker
elif candidates:
sid = (
sa.select((equities_cols.sid,))
.where((equities_cols.symbol == symbol) & (equities_cols.start_date <= ad_value))
.order_by(equities_cols.start_date.desc(), equities_cols.end_date.desc())
.scalar()
)
if sid:
return self._retrieve_equity(sid)
raise SymbolNotFound(symbol=symbol)
def factors(self):
algo = get_algo_instance()
today = normalize_date(algo.get_datetime())
if today > self._factor_matrix_expires:
self._factor_matrix, self._factor_matrix_expires = \
algo.compute_factor_matrix(today)
try:
return self._factor_matrix.loc[today]
except KeyError:
# This happens if no assets passed our filters on a given day.
return pd.DataFrame(
index=[],
columns=self._factor_matrix.columns,
)
def wrapper(self, other):
if other is tslib.NaT:
return tslib.NaT
elif isinstance(other, (timedelta, Tick, DateOffset)):
# timedelta path
return func(self, other)
elif isinstance(other, (np.datetime64, datetime, date)):
other = as_timestamp(other)
tz = getattr(other, 'tzinfo', None)
nano = getattr(other, 'nanosecond', 0)
try:
if self._adjust_dst and isinstance(other, Timestamp):
other = other.tz_localize(None)
result = func(self, other)
if self._adjust_dst:
result = tslib._localize_pydatetime(result, tz)
result = Timestamp(result)
if self.normalize:
result = result.normalize()
# nanosecond may be deleted depending on offset process
if not self.normalize and nano != 0:
if not isinstance(self, Nano) and result.nanosecond != nano:
if result.tz is not None:
# convert to UTC
value = tslib.tz_convert_single(
result.value, 'UTC', result.tz)
else:
value = result.value
result = Timestamp(value + nano)
if tz is not None and result.tzinfo is None:
result = tslib._localize_pydatetime(result, tz)
except OutOfBoundsDatetime:
result = func(self, as_datetime(other))
if self.normalize:
# normalize_date returns normal datetime
result = normalize_date(result)
if tz is not None and result.tzinfo is None:
result = tslib._localize_pydatetime(result, tz)
return result
def handle_market_close(self):
self.update_performance()
# add the return results from today to the returns series
todays_date = normalize_date(self.market_close)
self.cumulative_performance.update_dividends(todays_date)
self.todays_performance.update_dividends(todays_date)
self.returns[todays_date] = self.todays_performance.returns
# update risk metrics for cumulative performance
self.cumulative_risk_metrics.update(
todays_date,
self.todays_performance.returns,
self.all_benchmark_returns[todays_date])
# increment the day counter before we move markers forward.
self.day_count += 1.0
# Take a snapshot of our current performance to return to the
# browser.
daily_update = self.to_dict()
# On the last day of the test, don't create tomorrow's performance
# period. We may not be able to find the next trading day if we're
# at the end of our historical data
if self.market_close >= self.last_close:
return daily_update
# move the market day markers forward
self.market_open, self.market_close = \
trading.environment.next_open_and_close(self.market_open)
# Roll over positions to current day.
self.todays_performance.rollover()
self.todays_performance.period_open = self.market_open
self.todays_performance.period_close = self.market_close
# The dividend calculation for the daily needs to be made
# after the rollover. midnight_between is the last midnight
# hour between the close of markets and the next open. To
# make sure midnight_between matches identically with
# dividend data dates, it is in UTC.
midnight_between = self.market_open.replace(hour=0, minute=0, second=0,
microsecond=0)
self.cumulative_performance.update_dividends(midnight_between)
self.todays_performance.update_dividends(midnight_between)
return daily_update
def lookup_future_chain(self, root_symbol, as_of_date):
""" Return the futures chain for a given root symbol.
Parameters
----------
root_symbol : str
Root symbol of the desired future.
as_of_date : pd.Timestamp
Date at the time of the lookup.
Returns
-------
[Future]
"""
root_symbol.upper()
as_of_date = normalize_date(as_of_date)
return self._valid_contracts(root_symbol, as_of_date)
def test_sid_assignment(self):
# This metadata does not contain SIDs
metadata = ["PLAY", "MSFT"]
today = normalize_date(pd.Timestamp("2015-07-09", tz="UTC"))
# Write data with sid assignment
self.env.write_data(equities_identifiers=metadata, allow_sid_assignment=True)
# Verify that Assets were built and different sids were assigned
finder = AssetFinder(self.env.engine)
play = finder.lookup_symbol("PLAY", today)
msft = finder.lookup_symbol("MSFT", today)
self.assertEqual("PLAY", play.symbol)
self.assertIsNotNone(play.sid)
self.assertNotEqual(play.sid, msft.sid)
def lookup_symbol(self, symbol, as_of_date, fuzzy=False):
"""
If a fuzzy string is provided, then we try various symbols based on
the provided symbol. This is to facilitate mapping from a broker's
symbol to ours in cases where mapping to the broker's symbol loses
information. For example, if we have CMCS_A, but a broker has CMCSA,
when the broker provides CMCSA, it can also provide fuzzy='_',
so we can find a match by inserting an underscore.
"""
symbol = symbol.upper()
as_of_date = normalize_date(as_of_date)
if not fuzzy:
try:
return self.lookup_symbol_resolve_multiple(symbol, as_of_date)
except SymbolNotFound:
return None
else:
c = self.conn.cursor()
fuzzy = symbol.replace(self.fuzzy_char, '')
t = (fuzzy, as_of_date.value, as_of_date.value)
query = ("select sid from equities "
"where fuzzy=? " +
"and start_date<=? " +
"and end_date>=?")
c.execute(query, t)
candidates = c.fetchall()
# If one SID exists for symbol, return that symbol
if len(candidates) == 1:
return self._retrieve_equity(candidates[0][0])
# If multiple SIDs exist for symbol, return latest start_date with
# end_date as a tie-breaker
if len(candidates) > 1:
t = (symbol, as_of_date.value)
query = ("select sid from equities "
"where symbol=? " +
"and start_date<=? " +
"order by start_date desc, end_date desc" +
"limit 1")
c.execute(query, t)
data = c.fetchone()
if data:
return self._retrieve_equity(data[0])
def _adjust_dates_anchored(first, last, offset, closed='right', base=0):
from pandas.tseries.tools import normalize_date
# First and last offsets should be calculated from the start day to fix an
# error cause by resampling across multiple days when a one day period is
# not a multiple of the frequency.
#
# See https://github.com/pydata/pandas/issues/8683
start_day_nanos = Timestamp(normalize_date(first)).value
base_nanos = (base % offset.n) * offset.nanos // offset.n
start_day_nanos += base_nanos
foffset = (first.value - start_day_nanos) % offset.nanos
loffset = (last.value - start_day_nanos) % offset.nanos
if closed == 'right':
if foffset > 0:
# roll back
fresult = first.value - foffset
else:
fresult = first.value - offset.nanos
if loffset > 0:
# roll forward
lresult = last.value + (offset.nanos - loffset)
else:
# already the end of the road
lresult = last.value
else: # closed == 'left'
if foffset > 0:
fresult = first.value - foffset
else:
# start of the road
fresult = first.value
if loffset > 0:
# roll forward
lresult = last.value + (offset.nanos - loffset)
else:
lresult = last.value + offset.nanos
return (Timestamp(fresult, tz=first.tz),
Timestamp(lresult, tz=last.tz))
def test_sid_assignment(self):
# This metadata does not contain SIDs
metadata = {'PLAY': {'symbol': 'PLAY'},
'MSFT': {'symbol': 'MSFT'}}
today = normalize_date(pd.Timestamp('2015-07-09', tz='UTC'))
# Build a finder that is allowed to assign sids
finder = AssetFinder(metadata=metadata,
allow_sid_assignment=True)
# Verify that Assets were built and different sids were assigned
play = finder.lookup_symbol('PLAY', today)
msft = finder.lookup_symbol('MSFT', today)
self.assertEqual('PLAY', play.symbol)
self.assertIsNotNone(play.sid)
self.assertNotEqual(play.sid, msft.sid)
def test_sid_assignment(self):
# This metadata does not contain SIDs
metadata = ['PLAY', 'MSFT']
today = normalize_date(pd.Timestamp('2015-07-09', tz='UTC'))
# Write data with sid assignment
self.env.write_data(equities_identifiers=metadata,
allow_sid_assignment=True)
# Verify that Assets were built and different sids were assigned
finder = self.asset_finder_type(self.env.engine)
play = finder.lookup_symbol('PLAY', today)
msft = finder.lookup_symbol('MSFT', today)
self.assertEqual('PLAY', play.symbol)
self.assertIsNotNone(play.sid)
self.assertNotEqual(play.sid, msft.sid)
def handle_data(context, data):
today = normalize_date(get_datetime())
results = pipeline_output("test")
expect_over_300 = {AAPL: today < self.AAPL_split_date, MSFT: False, BRK_A: True}
for asset in assets:
should_pass_filter = expect_over_300[asset]
if set_screen and not should_pass_filter:
self.assertNotIn(asset, results.index)
continue
asset_results = results.loc[asset]
self.assertEqual(asset_results["filter"], should_pass_filter)
for length in vwaps:
computed = results.loc[asset, vwap_key(length)]
expected = vwaps[length][asset].loc[today]
# Only having two places of precision here is a bit
# unfortunate.
assert_almost_equal(computed, expected, decimal=2)
def handle_market_close_daily(self):
"""
Function called after handle_data when running with daily emission
rate.
"""
self.update_performance()
completed_date = normalize_date(self.market_close)
account = self.get_account(True)
# update risk metrics for cumulative performance
self.cumulative_risk_metrics.update(
completed_date,
self.todays_performance.returns,
self.all_benchmark_returns[completed_date],
account)
# increment the day counter before we move markers forward.
self.day_count += 1.0
# Take a snapshot of our current performance to return to the
# browser.
daily_update = self.to_dict()
# On the last day of the test, don't create tomorrow's performance
# period. We may not be able to find the next trading day if we're at
# the end of our historical data
if self.market_close >= self.last_close:
return daily_update
# move the market day markers forward
self.market_open, self.market_close = \
trading.environment.next_open_and_close(self.market_open)
# Roll over positions to current day.
self.todays_performance.rollover()
self.todays_performance.period_open = self.market_open
self.todays_performance.period_close = self.market_close
self.check_upcoming_dividends(completed_date)
self.account_needs_update = True
return daily_update
def handle_minute_close(self, dt):
self.update_performance()
todays_date = normalize_date(dt)
account = self.get_account(False)
self.minute_performance.rollover()
bench_returns = self.all_benchmark_returns.loc[todays_date:dt]
# cumulative returns
bench_since_open = (1. + bench_returns).prod() - 1
self.cumulative_risk_metrics.update(todays_date,
self.todays_performance.returns,
bench_since_open,
account)
# if this is the close, update dividends for the next day.
if dt == self.market_close:
self.check_upcoming_dividends(todays_date)
def handle_minute_close(self, dt, data_portal):
"""
Handles the close of the given minute. This includes handling
market-close functions if the given minute is the end of the market
day.
Parameters
__________
dt : Timestamp
The minute that is ending
Returns
_______
(dict, dict/None)
A tuple of the minute perf packet and daily perf packet.
If the market day has not ended, the daily perf packet is None.
"""
self.position_tracker.sync_last_sale_prices(dt, False, data_portal)
self.update_performance()
todays_date = normalize_date(dt)
account = self.get_account(False)
bench_returns = self.all_benchmark_returns.loc[todays_date:dt]
# cumulative returns
bench_since_open = (1. + bench_returns).prod() - 1
self.cumulative_risk_metrics.update(todays_date,
self.todays_performance.returns,
bench_since_open,
account.leverage)
minute_packet = self.to_dict(emission_type='minute')
# if this is the close, update dividends for the next day.
# Return the performance tuple
if dt == self.market_close:
return minute_packet, self._handle_market_close(
todays_date, data_portal._adjustment_reader,
)
else:
return minute_packet, None
def lookup_symbol(self, symbol, as_of_date, fuzzy=None):
"""
If a fuzzy string is provided, then we try various symbols based on
the provided symbol. This is to facilitate mapping from a broker's
symbol to ours in cases where mapping to the broker's symbol loses
information. For example, if we have CMCS_A, but a broker has CMCSA,
when the broker provides CMCSA, it can also provide fuzzy='_',
so we can find a match by inserting an underscore.
"""
symbol = symbol.upper()
as_of_date = normalize_date(as_of_date)
if not fuzzy:
try:
return self.lookup_symbol_resolve_multiple(symbol, as_of_date)
except SymbolNotFound:
return None
else:
try:
return self.fuzzy_match[(symbol, fuzzy, as_of_date)]
except KeyError:
# if symbol is CMCSA and fuzzy is '_', then
# try CMCSA, then CMCS_A, then CMC_SA, etc.
for fuzzy_symbol in chain(
(symbol,),
(symbol[:i] + fuzzy + symbol[i:]
for i in range(len(symbol) - 1, 0, -1))):
infos = self.sym_cache.get(fuzzy_symbol)
if infos:
info, date_match = self._lookup_symbol_in_infos(
infos,
as_of_date,
)
if info is not None and date_match:
self.fuzzy_match[(symbol, fuzzy, as_of_date)] = \
info
return info
else:
self.fuzzy_match[(symbol, fuzzy, as_of_date)] = None
def __init__(self, sim_params):
self.sim_params = sim_params
self.period_start = self.sim_params.period_start
self.period_end = self.sim_params.period_end
self.last_close = self.sim_params.last_close
first_day = self.sim_params.first_open
self.market_open, self.market_close = \
trading.environment.get_open_and_close(first_day)
self.total_days = self.sim_params.days_in_period
self.capital_base = self.sim_params.capital_base
self.emission_rate = sim_params.emission_rate
all_trading_days = trading.environment.trading_days
mask = ((all_trading_days >= normalize_date(self.period_start)) &
(all_trading_days <= normalize_date(self.period_end)))
self.trading_days = all_trading_days[mask]
self.perf_periods = []
if self.emission_rate == 'daily':
self.all_benchmark_returns = pd.Series(
index=self.trading_days)
self.intraday_risk_metrics = None
self.cumulative_risk_metrics = \
risk.RiskMetricsCumulative(self.sim_params)
elif self.emission_rate == 'minute':
self.all_benchmark_returns = pd.Series(index=pd.date_range(
self.sim_params.first_open, self.sim_params.last_close,
freq='Min'))
self.intraday_risk_metrics = \
risk.RiskMetricsCumulative(self.sim_params)
self.cumulative_risk_metrics = \
risk.RiskMetricsCumulative(self.sim_params,
returns_frequency='daily',
create_first_day_stats=True)
self.minute_performance = PerformancePeriod(
# initial cash is your capital base.
self.capital_base,
# the cumulative period will be calculated over the
# entire test.
self.period_start,
self.period_end,
# don't save the transactions for the cumulative
# period
keep_transactions=False,
keep_orders=False,
# don't serialize positions for cumualtive period
serialize_positions=False
)
self.perf_periods.append(self.minute_performance)
# this performance period will span the entire simulation from
# inception.
self.cumulative_performance = PerformancePeriod(
# initial cash is your capital base.
self.capital_base,
# the cumulative period will be calculated over the entire test.
self.period_start,
self.period_end,
# don't save the transactions for the cumulative
# period
keep_transactions=False,
keep_orders=False,
# don't serialize positions for cumualtive period
serialize_positions=False
)
self.perf_periods.append(self.cumulative_performance)
# this performance period will span just the current market day
self.todays_performance = PerformancePeriod(
# initial cash is your capital base.
self.capital_base,
# the daily period will be calculated for the market day
self.market_open,
self.market_close,
keep_transactions=True,
keep_orders=True,
serialize_positions=True
)
self.perf_periods.append(self.todays_performance)
self.saved_dt = self.period_start
self.returns = pd.Series(index=self.trading_days)
# one indexed so that we reach 100%
self.day_count = 0.0
self.txn_count = 0
self.event_count = 0
def test_history_in_bts_volume_days(self, data_freq):
"""
Test calling history() in before_trading_start()
with daily volume bars.
"""
algo_text = """
from zipline.api import history
def initialize(context):
context.first_bts_call = True
def before_trading_start(context, data):
if not context.first_bts_call:
volume_bts = history(bar_count=2, frequency='1d', field='volume')
context.volume_bts = volume_bts
context.first_bts_call = False
def handle_data(context, data):
volume_hd = history(bar_count=2, frequency='1d', field='volume')
context.volume_hd = volume_hd
""".strip()
# March 2006
# Su Mo Tu We Th Fr Sa
# 1 2 3 4
# 5 6 7 8 9 10 11
# 12 13 14 15 16 17 18
# 19 20 21 22 23 24 25
# 26 27 28 29 30 31
start = pd.Timestamp('2006-03-20', tz='UTC')
end = pd.Timestamp('2006-03-22', tz='UTC')
sim_params = factory.create_simulation_parameters(
start=start, end=end, data_frequency=data_freq)
test_algo = TradingAlgorithm(
script=algo_text,
data_frequency=data_freq,
sim_params=sim_params,
env=TestHistoryAlgo.env,
)
source = RandomWalkSource(start=start, end=end, freq=data_freq)
output = test_algo.run(source)
self.assertIsNotNone(output)
# Get the volume recorded by history() within handle_data()
volume_hd_0 = test_algo.volume_hd[0]
volume_hd_1 = test_algo.volume_hd[1]
# Get the volume recorded by history() within BTS
volume_bts_0 = test_algo.volume_bts[0]
volume_bts_1 = test_algo.volume_bts[1]
penultimate_hd_dt = pd.Timestamp(
'2006-03-21 4:00 PM', tz='US/Eastern').tz_convert('UTC')
# Midnight of the day on which BTS is invoked.
newest_bts_dt = normalize_date(pd.Timestamp(
'2006-03-22 04:00 PM', tz='US/Eastern').tz_convert('UTC'))
if data_freq == 'daily':
# If we're dealing with daily data, then we record
# canonicalized timestamps, so make conversion here:
penultimate_hd_dt = normalize_date(penultimate_hd_dt)
# When history() is called in BTS, its 'current' volume value
# should equal the sum of the previous day.
self.assertEquals(volume_hd_0[penultimate_hd_dt],
volume_bts_0[newest_bts_dt])
self.assertEquals(volume_hd_1[penultimate_hd_dt],
volume_bts_1[newest_bts_dt])
def __init__(self, sim_params):
self.sim_params = sim_params
env = TradingEnvironment.instance()
self.period_start = self.sim_params.period_start
self.period_end = self.sim_params.period_end
self.last_close = self.sim_params.last_close
first_open = self.sim_params.first_open.tz_convert(env.exchange_tz)
self.day = pd.Timestamp(datetime(first_open.year, first_open.month,
first_open.day),
tz='UTC')
self.market_open, self.market_close = env.get_open_and_close(self.day)
self.total_days = self.sim_params.days_in_period
self.capital_base = self.sim_params.capital_base
self.emission_rate = sim_params.emission_rate
all_trading_days = env.trading_days
mask = ((all_trading_days >= normalize_date(self.period_start)) &
(all_trading_days <= normalize_date(self.period_end)))
self.trading_days = all_trading_days[mask]
self.dividend_frame = pd.DataFrame()
self._dividend_count = 0
self.position_tracker = PositionTracker()
self.perf_periods = []
if self.emission_rate == 'daily':
self.all_benchmark_returns = pd.Series(index=self.trading_days)
self.cumulative_risk_metrics = \
risk.RiskMetricsCumulative(self.sim_params)
elif self.emission_rate == 'minute':
self.all_benchmark_returns = pd.Series(
index=pd.date_range(self.sim_params.first_open,
self.sim_params.last_close,
freq='Min'))
self.cumulative_risk_metrics = \
risk.RiskMetricsCumulative(self.sim_params,
returns_frequency='daily',
create_first_day_stats=True)
self.minute_performance = PerformancePeriod(
# initial cash is your capital base.
self.capital_base,
# the cumulative period will be calculated over the
# entire test.
self.period_start,
self.period_end,
# don't save the transactions for the cumulative
# period
keep_transactions=False,
keep_orders=False,
# don't serialize positions for cumualtive period
serialize_positions=False)
self.minute_performance.position_tracker = self.position_tracker
self.perf_periods.append(self.minute_performance)
# this performance period will span the entire simulation from
# inception.
self.cumulative_performance = PerformancePeriod(
# initial cash is your capital base.
self.capital_base,
# the cumulative period will be calculated over the entire test.
self.period_start,
self.period_end,
# don't save the transactions for the cumulative
# period
keep_transactions=False,
keep_orders=False,
# don't serialize positions for cumualtive period
serialize_positions=False,
)
self.cumulative_performance.position_tracker = self.position_tracker
self.perf_periods.append(self.cumulative_performance)
# this performance period will span just the current market day
self.todays_performance = PerformancePeriod(
# initial cash is your capital base.
self.capital_base,
# the daily period will be calculated for the market day
self.market_open,
self.market_close,
keep_transactions=True,
keep_orders=True,
serialize_positions=True,
)
self.todays_performance.position_tracker = self.position_tracker
self.perf_periods.append(self.todays_performance)
self.saved_dt = self.period_start
# one indexed so that we reach 100%
self.day_count = 0.0
self.txn_count = 0
self.account_needs_update = True
self._account = None
def __init__(self, sim_params,
returns_frequency=None,
create_first_day_stats=False,
account=None):
"""
- @returns_frequency allows for configuration of the whether
the benchmark and algorithm returns are in units of minutes or days,
if `None` defaults to the `emission_rate` in `sim_params`.
"""
self.treasury_curves = trading.environment.treasury_curves
self.start_date = sim_params.period_start.replace(
hour=0, minute=0, second=0, microsecond=0
)
self.end_date = sim_params.period_end.replace(
hour=0, minute=0, second=0, microsecond=0
)
self.trading_days = trading.environment.days_in_range(
self.start_date,
self.end_date)
# Hold on to the trading day before the start,
# used for index of the zero return value when forcing returns
# on the first day.
self.day_before_start = self.start_date - \
trading.environment.trading_days.freq
last_day = normalize_date(sim_params.period_end)
if last_day not in self.trading_days:
last_day = pd.tseries.index.DatetimeIndex(
[last_day]
)
self.trading_days = self.trading_days.append(last_day)
self.sim_params = sim_params
self.create_first_day_stats = create_first_day_stats
if returns_frequency is None:
returns_frequency = self.sim_params.emission_rate
self.returns_frequency = returns_frequency
if returns_frequency == 'daily':
cont_index = self.get_daily_index()
elif returns_frequency == 'minute':
cont_index = self.get_minute_index(sim_params)
self.cont_index = cont_index
self.cont_len = len(self.cont_index)
empty_cont = np.full(self.cont_len, np.nan)
self.algorithm_returns_cont = empty_cont.copy()
self.benchmark_returns_cont = empty_cont.copy()
self.algorithm_cumulative_leverages_cont = empty_cont.copy()
self.mean_returns_cont = empty_cont.copy()
self.annualized_mean_returns_cont = empty_cont.copy()
self.mean_benchmark_returns_cont = empty_cont.copy()
self.annualized_mean_benchmark_returns_cont = empty_cont.copy()
# The returns at a given time are read and reset from the respective
# returns container.
self.algorithm_returns = None
self.benchmark_returns = None
self.mean_returns = None
self.annualized_mean_returns = None
self.mean_benchmark_returns = None
self.annualized_mean_benchmark_returns = None
self.algorithm_cumulative_returns = empty_cont.copy()
self.benchmark_cumulative_returns = empty_cont.copy()
self.algorithm_cumulative_leverages = empty_cont.copy()
self.excess_returns = empty_cont.copy()
self.latest_dt_loc = 0
self.latest_dt = cont_index[0]
self.metrics = pd.DataFrame(index=cont_index,
columns=self.METRIC_NAMES,
dtype=float)
self.drawdowns = empty_cont.copy()
self.max_drawdowns = empty_cont.copy()
self.max_drawdown = 0
self.max_leverages = empty_cont.copy()
self.max_leverage = 0
self.current_max = -np.inf
self.daily_treasury = pd.Series(index=self.trading_days)
self.treasury_period_return = np.nan
self.num_trading_days = 0
def lookup_symbol(self, symbol, as_of_date, fuzzy=False):
"""
Return matching Equity of name symbol in database.
If multiple Equities are found and as_of_date is not set,
raises MultipleSymbolsFound.
If no Equity was active at as_of_date raises SymbolNotFound.
"""
# Format inputs
if as_of_date is not None:
as_of_date = pd.Timestamp(normalize_date(as_of_date))
company_symbol, share_class_symbol, fuzzy_symbol = \
split_delimited_symbol(symbol)
equities_cols = self.equities.c
if as_of_date:
ad_value = as_of_date.value
if fuzzy:
# Search for a single exact match on the fuzzy column
fuzzy_candidates = sa.select((equities_cols.sid, )).where(
(equities_cols.fuzzy_symbol == fuzzy_symbol) &
(equities_cols.start_date <= ad_value) &
(equities_cols.end_date >=
ad_value), ).execute().fetchall()
# If exactly one SID exists for fuzzy_symbol, return that sid
if len(fuzzy_candidates) == 1:
return self._retrieve_equity(fuzzy_candidates[0]['sid'])
# Search for exact matches of the split-up company_symbol and
# share_class_symbol
candidates = sa.select((equities_cols.sid, )).where(
(equities_cols.company_symbol == company_symbol) &
(equities_cols.share_class_symbol == share_class_symbol) &
(equities_cols.start_date <= ad_value) &
(equities_cols.end_date >= ad_value), ).execute().fetchall()
# If exactly one SID exists for symbol, return that symbol
if len(candidates) == 1:
return self._retrieve_equity(candidates[0]['sid'])
# If no SID exists for symbol, return SID with the
# highest-but-not-over end_date
elif not candidates:
sid = sa.select((equities_cols.sid, )).where(
(equities_cols.company_symbol == company_symbol) &
(equities_cols.share_class_symbol == share_class_symbol) &
(equities_cols.start_date <= ad_value), ).order_by(
equities_cols.end_date.desc(), ).scalar()
if sid is not None:
return self._retrieve_equity(sid)
# If multiple SIDs exist for symbol, return latest start_date with
# end_date as a tie-breaker
elif len(candidates) > 1:
sid = sa.select((equities_cols.sid, )).where(
(equities_cols.company_symbol == company_symbol) &
(equities_cols.share_class_symbol == share_class_symbol) &
(equities_cols.start_date <= ad_value), ).order_by(
equities_cols.start_date.desc(),
equities_cols.end_date.desc(),
).scalar()
if sid is not None:
return self._retrieve_equity(sid)
raise SymbolNotFound(symbol=symbol)
else:
# If this is a fuzzy look-up, check if there is exactly one match
# for the fuzzy symbol
if fuzzy:
fuzzy_sids = sa.select((equities_cols.sid, )).where(
(equities_cols.fuzzy_symbol == fuzzy_symbol
)).execute().fetchall()
if len(fuzzy_sids) == 1:
return self._retrieve_equity(fuzzy_sids[0]['sid'])
sids = sa.select((equities_cols.sid, )).where(
(equities_cols.company_symbol == company_symbol)
& (equities_cols.share_class_symbol == share_class_symbol)
).execute().fetchall()
if len(sids) == 1:
return self._retrieve_equity(sids[0]['sid'])
elif not sids:
raise SymbolNotFound(symbol=symbol)
else:
raise MultipleSymbolsFound(symbol=symbol,
options=list(
map(
compose(
self._retrieve_equity,
itemgetter('sid')),
sids,
)))
def __init__(self,
sim_params,
returns_frequency=None,
create_first_day_stats=False):
"""
- @returns_frequency allows for configuration of the whether
the benchmark and algorithm returns are in units of minutes or days,
if `None` defaults to the `emission_rate` in `sim_params`.
"""
self.treasury_curves = trading.environment.treasury_curves
self.start_date = sim_params.period_start.replace(hour=0,
minute=0,
second=0,
microsecond=0)
self.end_date = sim_params.period_end.replace(hour=0,
minute=0,
second=0,
microsecond=0)
self.trading_days = trading.environment.days_in_range(
self.start_date, self.end_date)
last_day = normalize_date(sim_params.period_end)
if last_day not in self.trading_days:
last_day = pd.tseries.index.DatetimeIndex([last_day])
self.trading_days = self.trading_days.append(last_day)
self.sim_params = sim_params
self.create_first_day_stats = create_first_day_stats
if returns_frequency is None:
returns_frequency = self.sim_params.emission_rate
self.returns_frequency = returns_frequency
if returns_frequency == 'daily':
cont_index = self.get_daily_index()
elif returns_frequency == 'minute':
cont_index = self.get_minute_index(sim_params)
self.cont_index = cont_index
self.algorithm_returns_cont = pd.Series(index=cont_index)
self.benchmark_returns_cont = pd.Series(index=cont_index)
# The returns at a given time are read and reset from the respective
# returns container.
self.algorithm_returns = None
self.benchmark_returns = None
self.mean_returns = None
self.annualized_mean_returns = None
self.mean_benchmark_returns = None
self.annualized_benchmark_returns = None
self.compounded_log_returns = pd.Series(index=cont_index)
self.algorithm_period_returns = pd.Series(index=cont_index)
self.benchmark_period_returns = pd.Series(index=cont_index)
self.excess_returns = pd.Series(index=cont_index)
self.latest_dt = cont_index[0]
self.metrics = pd.DataFrame(index=cont_index,
columns=self.METRIC_NAMES)
self.max_drawdown = 0
self.current_max = -np.inf
self.daily_treasury = pd.Series(index=self.trading_days)
def __init__(self, sim_params, env, data_portal):
self.sim_params = sim_params
self.env = env
self.period_start = self.sim_params.period_start
self.period_end = self.sim_params.period_end
self.last_close = self.sim_params.last_close
first_open = self.sim_params.first_open.tz_convert(
self.env.exchange_tz)
self.day = pd.Timestamp(datetime(first_open.year, first_open.month,
first_open.day),
tz='UTC')
self.market_open, self.market_close = env.get_open_and_close(self.day)
self.total_days = self.sim_params.days_in_period
self.capital_base = self.sim_params.capital_base
self.emission_rate = sim_params.emission_rate
all_trading_days = env.trading_days
mask = ((all_trading_days >= normalize_date(self.period_start)) &
(all_trading_days <= normalize_date(self.period_end)))
self.trading_days = all_trading_days[mask]
self._data_portal = data_portal
if data_portal is not None:
self._adjustment_reader = data_portal._adjustment_reader
else:
self._adjustment_reader = None
self.position_tracker = PositionTracker(
asset_finder=env.asset_finder,
data_portal=data_portal,
data_frequency=self.sim_params.data_frequency)
if self.emission_rate == 'daily':
self.all_benchmark_returns = pd.Series(index=self.trading_days)
self.cumulative_risk_metrics = \
risk.RiskMetricsCumulative(self.sim_params, self.env)
elif self.emission_rate == 'minute':
self.all_benchmark_returns = pd.Series(
index=pd.date_range(self.sim_params.first_open,
self.sim_params.last_close,
freq='Min'))
self.cumulative_risk_metrics = \
risk.RiskMetricsCumulative(self.sim_params, self.env,
create_first_day_stats=True)
# this performance period will span the entire simulation from
# inception.
self.cumulative_performance = PerformancePeriod(
# initial cash is your capital base.
starting_cash=self.capital_base,
data_frequency=self.sim_params.data_frequency,
data_portal=data_portal,
# the cumulative period will be calculated over the entire test.
period_open=self.period_start,
period_close=self.period_end,
# don't save the transactions for the cumulative
# period
keep_transactions=False,
keep_orders=False,
# don't serialize positions for cumulative period
serialize_positions=False,
asset_finder=self.env.asset_finder,
name="Cumulative")
self.cumulative_performance.position_tracker = self.position_tracker
# this performance period will span just the current market day
self.todays_performance = PerformancePeriod(
# initial cash is your capital base.
starting_cash=self.capital_base,
data_frequency=self.sim_params.data_frequency,
data_portal=data_portal,
# the daily period will be calculated for the market day
period_open=self.market_open,
period_close=self.market_close,
keep_transactions=True,
keep_orders=True,
serialize_positions=True,
asset_finder=self.env.asset_finder,
name="Daily")
self.todays_performance.position_tracker = self.position_tracker
self.saved_dt = self.period_start
# one indexed so that we reach 100%
self.day_count = 0.0
self.txn_count = 0
self.account_needs_update = True
self._account = None
def test_history_in_bts_price_days(self, data_freq):
"""
Test calling history() in before_trading_start()
with daily price bars.
"""
algo_text = """
from zipline.api import history
def initialize(context):
context.first_bts_call = True
def before_trading_start(context, data):
if not context.first_bts_call:
prices_bts = history(bar_count=3, frequency='1d', field='price')
context.prices_bts = prices_bts
context.first_bts_call = False
def handle_data(context, data):
prices_hd = history(bar_count=3, frequency='1d', field='price')
context.prices_hd = prices_hd
""".strip()
# March 2006
# Su Mo Tu We Th Fr Sa
# 1 2 3 4
# 5 6 7 8 9 10 11
# 12 13 14 15 16 17 18
# 19 20 21 22 23 24 25
# 26 27 28 29 30 31
start = pd.Timestamp('2006-03-20', tz='UTC')
end = pd.Timestamp('2006-03-22', tz='UTC')
sim_params = factory.create_simulation_parameters(
start=start, end=end, data_frequency=data_freq)
test_algo = TradingAlgorithm(
script=algo_text,
data_frequency=data_freq,
sim_params=sim_params,
env=TestHistoryAlgo.env,
)
source = RandomWalkSource(start=start, end=end, freq=data_freq)
output = test_algo.run(source)
self.assertIsNotNone(output)
# Get the prices recorded by history() within handle_data()
prices_hd = test_algo.prices_hd[0]
# Get the prices recorded by history() within BTS
prices_bts = test_algo.prices_bts[0]
# before_trading_start() is timestamp'd to midnight prior to
# the day's trading. Since no equity trades occur at midnight,
# the price recorded for this time is forward filled from the
# last trade - typically ~4pm the previous day. This results
# in the OHLCV data recorded by history() in BTS lagging
# that recorded by history in handle_data().
# The trace of the pricing data from history() called within
# handle_data() vs. BTS in the above algo is as follows:
# When called within handle_data()
# ---------------------------------
# 2006-03-20 21:00:00 139.369469
# 2006-03-21 21:00:00 180.156620
# 2006-03-22 21:00:00 221.344654
# When called within BTS
# ---------------------------------
# 2006-03-17 21:00:00 NaN
# 2006-03-20 21:00:00 139.369469
# 2006-03-22 00:00:00 180.156620
# Get relevant Timestamps for the history() call within handle_data()
oldest_hd_dt = pd.Timestamp(
'2006-03-20 4:00 PM', tz='US/Eastern').tz_convert('UTC')
penultimate_hd_dt = pd.Timestamp(
'2006-03-21 4:00 PM', tz='US/Eastern').tz_convert('UTC')
# Get relevant Timestamps for the history() call within BTS
penultimate_bts_dt = pd.Timestamp(
'2006-03-20 4:00 PM', tz='US/Eastern').tz_convert('UTC')
newest_bts_dt = normalize_date(pd.Timestamp(
'2006-03-22 04:00 PM', tz='US/Eastern').tz_convert('UTC'))
if data_freq == 'daily':
# If we're dealing with daily data, then we record
# canonicalized timestamps, so make conversion here:
oldest_hd_dt = normalize_date(oldest_hd_dt)
penultimate_hd_dt = normalize_date(penultimate_hd_dt)
penultimate_bts_dt = normalize_date(penultimate_bts_dt)
self.assertEquals(prices_hd[oldest_hd_dt],
prices_bts[penultimate_bts_dt])
self.assertEquals(prices_hd[penultimate_hd_dt],
prices_bts[newest_bts_dt])
def lookup_symbol_resolve_multiple(self, symbol, as_of_date=None):
"""
Return matching Asset of name symbol in database.
If multiple Assets are found and as_of_date is not set,
raises MultipleSymbolsFound.
If no Asset was active at as_of_date, and allow_expired is False
raises SymbolNotFound.
"""
if as_of_date is not None:
as_of_date = pd.Timestamp(normalize_date(as_of_date))
c = self.conn.cursor()
if as_of_date:
# If one SID exists for symbol, return that symbol
t = (symbol, as_of_date.value, as_of_date.value)
query = ("select sid from equities "
"where symbol=? "
"and start_date<=? "
"and end_date>=?")
c.execute(query, t)
candidates = c.fetchall()
if len(candidates) == 1:
return self._retrieve_equity(candidates[0][0])
# If no SID exists for symbol, return SID with the
# highest-but-not-over end_date
if len(candidates) == 0:
t = (symbol, as_of_date.value)
query = ("select sid from equities "
"where symbol=? "
"and start_date<=? "
"order by end_date desc "
"limit 1")
c.execute(query, t)
data = c.fetchone()
if data:
return self._retrieve_equity(data[0])
# If multiple SIDs exist for symbol, return latest start_date with
# end_date as a tie-breaker
if len(candidates) > 1:
t = (symbol, as_of_date.value)
query = ("select sid from equities "
"where symbol=? " + "and start_date<=? " +
"order by start_date desc, end_date desc " +
"limit 1")
c.execute(query, t)
data = c.fetchone()
if data:
return self._retrieve_equity(data[0])
raise SymbolNotFound(symbol=symbol)
else:
t = (symbol, )
query = ("select sid from equities where symbol=?")
c.execute(query, t)
data = c.fetchall()
if len(data) == 1:
return self._retrieve_equity(data[0][0])
elif not data:
raise SymbolNotFound(symbol=symbol)
else:
options = []
for row in data:
sid = row[0]
asset = self._retrieve_equity(sid)
options.append(asset)
raise MultipleSymbolsFound(symbol=symbol, options=options)
def __init__(self,
sim_params,
env,
create_first_day_stats=False,
account=None):
self.treasury_curves = env.treasury_curves
self.start_date = sim_params.period_start.replace(hour=0,
minute=0,
second=0,
microsecond=0)
self.end_date = sim_params.period_end.replace(hour=0,
minute=0,
second=0,
microsecond=0)
self.trading_days = env.days_in_range(self.start_date, self.end_date)
# Hold on to the trading day before the start,
# used for index of the zero return value when forcing returns
# on the first day.
self.day_before_start = self.start_date - env.trading_days.freq
last_day = normalize_date(sim_params.period_end)
if last_day not in self.trading_days:
last_day = pd.tseries.index.DatetimeIndex([last_day])
self.trading_days = self.trading_days.append(last_day)
self.sim_params = sim_params
self.env = env
self.create_first_day_stats = create_first_day_stats
cont_index = self.trading_days
self.cont_index = cont_index
self.cont_len = len(self.cont_index)
empty_cont = np.full(self.cont_len, np.nan)
self.algorithm_returns_cont = empty_cont.copy()
self.benchmark_returns_cont = empty_cont.copy()
self.algorithm_cumulative_leverages_cont = empty_cont.copy()
self.mean_returns_cont = empty_cont.copy()
self.annualized_mean_returns_cont = empty_cont.copy()
self.mean_benchmark_returns_cont = empty_cont.copy()
self.annualized_mean_benchmark_returns_cont = empty_cont.copy()
# The returns at a given time are read and reset from the respective
# returns container.
self.algorithm_returns = None
self.benchmark_returns = None
self.mean_returns = None
self.annualized_mean_returns = None
self.mean_benchmark_returns = None
self.annualized_mean_benchmark_returns = None
self.algorithm_cumulative_returns = empty_cont.copy()
self.benchmark_cumulative_returns = empty_cont.copy()
self.algorithm_cumulative_leverages = empty_cont.copy()
self.excess_returns = empty_cont.copy()
self.latest_dt_loc = 0
self.latest_dt = cont_index[0]
self.benchmark_volatility = empty_cont.copy()
self.algorithm_volatility = empty_cont.copy()
self.beta = empty_cont.copy()
self.alpha = empty_cont.copy()
self.sharpe = empty_cont.copy()
self.downside_risk = empty_cont.copy()
self.sortino = empty_cont.copy()
self.information = empty_cont.copy()
self.drawdowns = empty_cont.copy()
self.max_drawdowns = empty_cont.copy()
self.max_drawdown = 0
self.max_leverages = empty_cont.copy()
self.max_leverage = 0
self.current_max = -np.inf
self.daily_treasury = pd.Series(index=self.trading_days)
self.treasury_period_return = np.nan
self.num_trading_days = 0
def _generate(cls,
start,
end,
periods,
name,
offset,
tz=None,
normalize=False):
if com._count_not_none(start, end, periods) < 2:
raise ValueError('Must specify two of start, end, or periods')
_normalized = True
if start is not None:
start = Timestamp(start)
if end is not None:
end = Timestamp(end)
inferred_tz = tools._infer_tzinfo(start, end)
if tz is not None and inferred_tz is not None:
assert (inferred_tz == tz)
elif inferred_tz is not None:
tz = inferred_tz
tz = tools._maybe_get_tz(tz)
if start is not None:
if normalize:
start = normalize_date(start)
_normalized = True
else:
_normalized = _normalized and start.time() == _midnight
if end is not None:
if normalize:
end = normalize_date(end)
_normalized = True
else:
_normalized = _normalized and end.time() == _midnight
if hasattr(offset, 'delta') and offset != offsets.Day():
if inferred_tz is None and tz is not None:
# naive dates
if start is not None and start.tz is None:
start = start.tz_localize(tz)
if end is not None and end.tz is None:
end = end.tz_localize(tz)
if start and end:
if start.tz is None and end.tz is not None:
start = start.tz_localize(end.tz)
if end.tz is None and start.tz is not None:
end = end.tz_localize(start.tz)
if (offset._should_cache()
and not (offset._normalize_cache and not _normalized)
and _naive_in_cache_range(start, end)):
index = cls._cached_range(start,
end,
periods=periods,
offset=offset,
name=name)
else:
index = _generate_regular_range(start, end, periods, offset)
else:
if inferred_tz is None and tz is not None:
# naive dates
if start is not None and start.tz is not None:
start = start.replace(tzinfo=None)
if end is not None and end.tz is not None:
end = end.replace(tzinfo=None)
if start and end:
if start.tz is None and end.tz is not None:
end = end.replace(tzinfo=None)
if end.tz is None and start.tz is not None:
start = start.replace(tzinfo=None)
if (offset._should_cache()
and not (offset._normalize_cache and not _normalized)
and _naive_in_cache_range(start, end)):
index = cls._cached_range(start,
end,
periods=periods,
offset=offset,
name=name)
else:
index = _generate_regular_range(start, end, periods, offset)
if tz is not None and getattr(index, 'tz', None) is None:
index = lib.tz_localize_to_utc(com._ensure_int64(index), tz)
index = index.view(_NS_DTYPE)
index = index.view(cls)
index.name = name
index.offset = offset
index.tz = tz
return index
