def test_force_registration(self):
register_calendar("DMY", self.dummy_cal_type())
first_dummy = get_calendar("DMY")
# force-register a new instance
register_calendar("DMY", self.dummy_cal_type(), force=True)
second_dummy = get_calendar("DMY")
self.assertNotEqual(first_dummy, second_dummy)
def read(cls, rootdir):
path = cls.metadata_path(rootdir)
with open(path) as fp:
raw_data = json.load(fp)
try:
version = raw_data['version']
except KeyError:
# Version was first written with version 1, assume 0,
# if version does not match.
version = 0
default_ohlc_ratio = raw_data['ohlc_ratio']
if version >= 1:
minutes_per_day = raw_data['minutes_per_day']
else:
# version 0 always assumed US equities.
minutes_per_day = US_EQUITIES_MINUTES_PER_DAY
if version >= 2:
calendar = get_calendar(raw_data['calendar_name'])
start_session = pd.Timestamp(raw_data['start_session'],
tz='UTC')
end_session = pd.Timestamp(raw_data['end_session'], tz='UTC')
else:
# No calendar info included in older versions, so
# default to NYSE.
calendar = get_calendar('NYSE')
start_session = pd.Timestamp(raw_data['first_trading_day'],
tz='UTC')
end_session = calendar.minute_to_session_label(
pd.Timestamp(raw_data['market_closes'][-1],
unit='m',
tz='UTC'))
if version >= 3:
ohlc_ratios_per_sid = raw_data['ohlc_ratios_per_sid']
if ohlc_ratios_per_sid is not None:
ohlc_ratios_per_sid = keymap(int, ohlc_ratios_per_sid)
else:
ohlc_ratios_per_sid = None
return cls(
default_ohlc_ratio,
ohlc_ratios_per_sid,
calendar,
start_session,
end_session,
minutes_per_day,
version=version,
)
def test_register_calendar(self):
# Build a fake calendar
dummy_cal = self.dummy_cal_type()
# Try to register and retrieve the calendar
register_calendar('DMY', dummy_cal)
retr_cal = get_calendar('DMY')
self.assertEqual(dummy_cal, retr_cal)
# Try to register again, expecting a name collision
with self.assertRaises(CalendarNameCollision):
register_calendar('DMY', dummy_cal)
# Deregister the calendar and ensure that it is removed
deregister_calendar('DMY')
with self.assertRaises(InvalidCalendarName):
get_calendar('DMY')
def __init__(self, raw_price_loader, adjustments_loader):
self.raw_price_loader = raw_price_loader
self.adjustments_loader = adjustments_loader
cal = self.raw_price_loader.trading_calendar or \
get_calendar("NYSE")
self._all_sessions = cal.all_sessions
def __init__(self, env, trading_calendar=None,
first_trading_day=None):
if trading_calendar is None:
trading_calendar = get_calendar("NYSE")
super(FakeDataPortal, self).__init__(env.asset_finder,
trading_calendar,
first_trading_day)
def setUpClass(cls):
cls.nyse_calendar = get_calendar("NYSE")
# july 15 is friday, so there are 3 sessions in this range (15, 18, 19)
cls.sessions = cls.nyse_calendar.sessions_in_range(
pd.Timestamp("2016-07-15"), pd.Timestamp("2016-07-19"))
trading_o_and_c = cls.nyse_calendar.schedule.ix[cls.sessions]
cls.opens = trading_o_and_c['market_open']
cls.closes = trading_o_and_c['market_close']
def gen_calendars(start, stop, critical_dates):
"""
Generate calendars to use as inputs.
"""
all_dates = pd.date_range(start, stop, tz='utc')
for to_drop in map(list, powerset(critical_dates)):
# Have to yield tuples.
yield (all_dates.drop(to_drop),)
# Also test with the trading calendar.
trading_days = get_calendar("NYSE").all_days
yield (trading_days[trading_days.slice_indexer(start, stop)],)
def setUp(self):
self.trading_day = get_calendar("NYSE").day
self.nsids = 5
self.ndates = 20
self.sids = Int64Index(range(self.nsids))
self.dates = DatetimeIndex(
start='2014-01-02',
freq=self.trading_day,
periods=self.ndates,
)
self.mask = ones((len(self.dates), len(self.sids)), dtype=bool)
def sessions(self):
if 'calendar' in self._table.attrs.attrs:
# backwards compatibility with old formats, will remove
return DatetimeIndex(self._table.attrs['calendar'], tz='UTC')
else:
cal = get_calendar(self._table.attrs['calendar_name'])
start_session_ns = self._table.attrs['start_session_ns']
start_session = Timestamp(start_session_ns, tz='UTC')
end_session_ns = self._table.attrs['end_session_ns']
end_session = Timestamp(end_session_ns, tz='UTC')
sessions = cal.sessions_in_range(start_session, end_session)
return sessions
def setUpClass(cls):
# On the AfterOpen and BeforeClose tests, we want ensure that the
# functions are pure, and that running them with the same input will
# provide the same output, regardless of whether the function is run 1
# or N times. (For performance reasons, we cache some internal state
# in AfterOpen and BeforeClose, but we don't want it to affect
# purity). Hence, we use the same before_close and after_open across
# subtests.
cls.before_close = BeforeClose(hours=1, minutes=5)
cls.after_open = AfterOpen(hours=1, minutes=5)
cls.class_ = None # Mark that this is the base class.
cal = get_calendar(cls.CALENDAR_STRING)
cls.before_close.cal = cal
cls.after_open.cal = cal
def run_example(example_name, environ):
"""
Run an example module from gateway.examples.
"""
mod = EXAMPLE_MODULES[example_name]
register_calendar("YAHOO", get_calendar("NYSE"), force=True)
return run_algorithm(
initialize=getattr(mod, 'initialize', None),
handle_data=getattr(mod, 'handle_data', None),
before_trading_start=getattr(mod, 'before_trading_start', None),
analyze=getattr(mod, 'analyze', None),
bundle='test',
environ=environ,
# Provide a default capital base, but allow the test to override.
**merge({'capital_base': 1e7}, mod._test_args()))
def setUpClass(cls):
super(StatelessRulesTests, cls).setUpClass()
cls.class_ = StatelessRule
cls.cal = get_calendar(cls.CALENDAR_STRING)
# First day of 09/2014 is closed whereas that for 10/2014 is open
cls.sept_sessions = cls.cal.sessions_in_range(
pd.Timestamp('2014-09-01', tz='UTC'),
pd.Timestamp('2014-09-30', tz='UTC'),
)
cls.oct_sessions = cls.cal.sessions_in_range(
pd.Timestamp('2014-10-01', tz='UTC'),
pd.Timestamp('2014-10-31', tz='UTC'),
)
cls.sept_week = cls.cal.minutes_for_sessions_in_range(
pd.Timestamp("2014-09-22", tz='UTC'),
pd.Timestamp("2014-09-26", tz='UTC'))
cls.HALF_SESSION = None
cls.FULL_SESSION = None
def __init__(
self,
load=None,
bm_symbol='SPY',
exchange_tz="US/Eastern",
trading_calendar=None,
asset_db_path=':memory:',
future_chain_predicates=CHAIN_PREDICATES,
environ=None,
):
self.bm_symbol = bm_symbol
if not load:
load = partial(load_market_data, environ=environ)
if not trading_calendar:
trading_calendar = get_calendar("NYSE")
self.benchmark_returns, self.treasury_curves = load(
trading_calendar.day,
trading_calendar.schedule.index,
self.bm_symbol,
)
self.exchange_tz = exchange_tz
if isinstance(asset_db_path, string_types):
asset_db_path = 'sqlite:///' + asset_db_path
self.engine = engine = create_engine(asset_db_path)
else:
self.engine = engine = asset_db_path
if engine is not None:
AssetDBWriter(engine).init_db()
self.asset_finder = AssetFinder(
engine, future_chain_predicates=future_chain_predicates)
else:
self.asset_finder = None
def test_write_attrs(self):
result = self.bcolz_daily_bar_ctable
expected_first_row = {
'1': 0,
'2': 5, # Asset 1 has 5 trading days.
'3': 12, # Asset 2 has 7 trading days.
'4': 33, # Asset 3 has 21 trading days.
'5': 44, # Asset 4 has 11 trading days.
'6': 49, # Asset 5 has 5 trading days.
}
expected_last_row = {
'1': 4,
'2': 11,
'3': 32,
'4': 43,
'5': 48,
'6': 57, # Asset 6 has 9 trading days.
}
expected_calendar_offset = {
'1': 0, # Starts on 6-01, 1st trading day of month.
'2': 15, # Starts on 6-22, 16th trading day of month.
'3': 1, # Starts on 6-02, 2nd trading day of month.
'4': 0, # Starts on 6-01, 1st trading day of month.
'5': 9, # Starts on 6-12, 10th trading day of month.
'6': 10, # Starts on 6-15, 11th trading day of month.
}
self.assertEqual(result.attrs['first_row'], expected_first_row)
self.assertEqual(result.attrs['last_row'], expected_last_row)
self.assertEqual(
result.attrs['calendar_offset'],
expected_calendar_offset,
)
cal = get_calendar(result.attrs['calendar_name'])
first_session = Timestamp(result.attrs['start_session_ns'], tz='UTC')
end_session = Timestamp(result.attrs['end_session_ns'], tz='UTC')
sessions = cal.sessions_in_range(first_session, end_session)
assert_index_equal(self.sessions, sessions)
def create_simulation_parameters(year=2006,
start=None,
end=None,
capital_base=float("1.0e5"),
num_days=None,
data_frequency='daily',
emission_rate='daily',
trading_calendar=None):
if not trading_calendar:
trading_calendar = get_calendar("NYSE")
if start is None:
start = pd.Timestamp("{0}-01-01".format(year), tz='UTC')
elif type(start) == datetime:
start = pd.Timestamp(start)
if end is None:
if num_days:
start_index = trading_calendar.all_sessions.searchsorted(start)
end = trading_calendar.all_sessions[start_index + num_days - 1]
else:
end = pd.Timestamp("{0}-12-31".format(year), tz='UTC')
elif type(end) == datetime:
end = pd.Timestamp(end)
sim_params = SimulationParameters(
start_session=start,
end_session=end,
capital_base=capital_base,
data_frequency=data_frequency,
emission_rate=emission_rate,
trading_calendar=trading_calendar,
)
return sim_params
def init_class_fixtures(cls):
super(WithPanelBarReader, cls).init_class_fixtures()
finder = cls.asset_finder
trading_calendar = get_calendar('NYSE')
items = finder.retrieve_all(finder.sids)
major_axis = (trading_calendar.sessions_in_range
if cls.FREQUENCY == 'daily' else
trading_calendar.minutes_for_sessions_in_range)(
cls.START_DATE, cls.END_DATE)
minor_axis = ['open', 'high', 'low', 'close', 'volume']
shape = tuple(map(len, [items, major_axis, minor_axis]))
raw_data = np.arange(shape[0] * shape[1] * shape[2]).reshape(shape)
cls.panel = pd.Panel(
raw_data,
items=items,
major_axis=major_axis,
minor_axis=minor_axis,
)
cls.reader = PanelBarReader(trading_calendar, cls.panel, cls.FREQUENCY)
class ClosesAndVolumes(WithDataPortal, GatewayTestCase):
sids = 1, 2, 3
START_DATE = pd.Timestamp('2014-01-01', tz='utc')
END_DATE = pd.Timestamp('2014-02-01', tz='utc')
dates = date_range(START_DATE,
END_DATE,
freq=get_calendar("NYSE").day,
tz='utc')
@classmethod
def make_equity_info(cls):
cls.equity_info = ret = DataFrame.from_records([
{
'sid': 1,
'symbol': 'A',
'start_date': cls.dates[10],
'end_date': cls.dates[13],
'exchange': 'TEST',
},
{
'sid': 2,
'symbol': 'B',
'start_date': cls.dates[11],
'end_date': cls.dates[14],
'exchange': 'TEST',
},
{
'sid': 3,
'symbol': 'C',
'start_date': cls.dates[12],
'end_date': cls.dates[15],
'exchange': 'TEST',
},
])
return ret
@classmethod
def make_equity_daily_bar_data(cls):
cls.closes = DataFrame(
{sid: arange(1,
len(cls.dates) + 1) * sid
for sid in cls.sids},
index=cls.dates,
dtype=float,
)
cls.volumes = cls.closes * 1000
for sid in cls.sids:
yield sid, DataFrame(
{
'open': cls.closes[sid].values,
'high': cls.closes[sid].values,
'low': cls.closes[sid].values,
'close': cls.closes[sid].values,
'volume': cls.volumes[sid].values,
},
index=cls.dates,
)
@classmethod
def init_class_fixtures(cls):
super(ClosesAndVolumes, cls).init_class_fixtures()
cls.first_asset_start = min(cls.equity_info.start_date)
cls.last_asset_end = max(cls.equity_info.end_date)
cls.assets = cls.asset_finder.retrieve_all(cls.sids)
cls.trading_day = cls.trading_calendar.day
# Add a split for 'A' on its second date.
cls.split_asset = cls.assets[0]
cls.split_date = cls.split_asset.start_date + cls.trading_day
cls.split_ratio = 0.5
cls.adjustments = DataFrame.from_records([{
'sid':
cls.split_asset.sid,
'value':
cls.split_ratio,
'kind':
MULTIPLY,
'start_date':
Timestamp('NaT'),
'end_date':
cls.split_date,
'apply_date':
cls.split_date,
}])
def init_instance_fixtures(self):
super(ClosesAndVolumes, self).init_instance_fixtures()
# View of the data on/after the split.
self.adj_closes = adj_closes = self.closes.copy()
adj_closes.ix[:self.split_date, self.split_asset] *= self.split_ratio
self.adj_volumes = adj_volumes = self.volumes.copy()
adj_volumes.ix[:self.split_date, self.split_asset] *= self.split_ratio
self.pipeline_close_loader = DataFrameLoader(
column=USEquityPricing.close,
baseline=self.closes,
adjustments=self.adjustments,
)
self.pipeline_volume_loader = DataFrameLoader(
column=USEquityPricing.volume,
baseline=self.volumes,
adjustments=self.adjustments,
)
def expected_close(self, date, asset):
if date < self.split_date:
lookup = self.closes
else:
lookup = self.adj_closes
return lookup.loc[date, asset]
def expected_volume(self, date, asset):
if date < self.split_date:
lookup = self.volumes
else:
lookup = self.adj_volumes
return lookup.loc[date, asset]
def exists(self, date, asset):
return asset.start_date <= date <= asset.end_date
def test_attach_pipeline_after_initialize(self):
"""
Assert that calling attach_pipeline after initialize raises correctly.
"""
def initialize(context):
pass
def late_attach(context, data):
attach_pipeline(Pipeline(), 'test')
raise AssertionError("Shouldn't make it past attach_pipeline!")
algo = TradingAlgorithm(
initialize=initialize,
handle_data=late_attach,
data_frequency='daily',
get_pipeline_loader=lambda column: self.pipeline_close_loader,
start=self.first_asset_start - self.trading_day,
end=self.last_asset_end + self.trading_day,
env=self.env,
)
with self.assertRaises(AttachPipelineAfterInitialize):
algo.run(self.data_portal)
def barf(context, data):
raise AssertionError("Shouldn't make it past before_trading_start")
algo = TradingAlgorithm(
initialize=initialize,
before_trading_start=late_attach,
handle_data=barf,
data_frequency='daily',
get_pipeline_loader=lambda column: self.pipeline_close_loader,
start=self.first_asset_start - self.trading_day,
end=self.last_asset_end + self.trading_day,
env=self.env,
)
with self.assertRaises(AttachPipelineAfterInitialize):
algo.run(self.data_portal)
def test_pipeline_output_after_initialize(self):
"""
Assert that calling pipeline_output after initialize raises correctly.
"""
def initialize(context):
attach_pipeline(Pipeline(), 'test')
pipeline_output('test')
raise AssertionError("Shouldn't make it past pipeline_output()")
def handle_data(context, data):
raise AssertionError("Shouldn't make it past initialize!")
def before_trading_start(context, data):
raise AssertionError("Shouldn't make it past initialize!")
algo = TradingAlgorithm(
initialize=initialize,
handle_data=handle_data,
before_trading_start=before_trading_start,
data_frequency='daily',
get_pipeline_loader=lambda column: self.pipeline_close_loader,
start=self.first_asset_start - self.trading_day,
end=self.last_asset_end + self.trading_day,
env=self.env,
)
with self.assertRaises(PipelineOutputDuringInitialize):
algo.run(self.data_portal)
def test_get_output_nonexistent_pipeline(self):
"""
Assert that calling add_pipeline after initialize raises appropriately.
"""
def initialize(context):
attach_pipeline(Pipeline(), 'test')
def handle_data(context, data):
raise AssertionError("Shouldn't make it past before_trading_start")
def before_trading_start(context, data):
pipeline_output('not_test')
raise AssertionError("Shouldn't make it past pipeline_output!")
algo = TradingAlgorithm(
initialize=initialize,
handle_data=handle_data,
before_trading_start=before_trading_start,
data_frequency='daily',
get_pipeline_loader=lambda column: self.pipeline_close_loader,
start=self.first_asset_start - self.trading_day,
end=self.last_asset_end + self.trading_day,
env=self.env,
)
with self.assertRaises(NoSuchPipeline):
algo.run(self.data_portal)
@parameterized.expand([('default', None), ('day', 1), ('week', 5),
('year', 252),
('all_but_one_day', 'all_but_one_day'),
('custom_iter', 'custom_iter')])
def test_assets_appear_on_correct_days(self, test_name, chunks):
"""
Assert that assets appear at correct times during a backtest, with
correctly-adjusted close price values.
"""
if chunks == 'all_but_one_day':
chunks = (self.dates.get_loc(self.last_asset_end) -
self.dates.get_loc(self.first_asset_start)) - 1
elif chunks == 'custom_iter':
chunks = []
st = np.random.RandomState(12345)
remaining = (self.dates.get_loc(self.last_asset_end) -
self.dates.get_loc(self.first_asset_start))
while remaining > 0:
chunk = st.randint(3)
chunks.append(chunk)
remaining -= chunk
def initialize(context):
p = attach_pipeline(Pipeline(), 'test', chunks=chunks)
p.add(USEquityPricing.close.latest, 'close')
def handle_data(context, data):
results = pipeline_output('test')
date = get_datetime().normalize()
for asset in self.assets:
# Assets should appear iff they exist today and yesterday.
exists_today = self.exists(date, asset)
existed_yesterday = self.exists(date - self.trading_day, asset)
if exists_today and existed_yesterday:
latest = results.loc[asset, 'close']
self.assertEqual(latest, self.expected_close(date, asset))
else:
self.assertNotIn(asset, results.index)
before_trading_start = handle_data
algo = TradingAlgorithm(
initialize=initialize,
handle_data=handle_data,
before_trading_start=before_trading_start,
data_frequency='daily',
get_pipeline_loader=lambda column: self.pipeline_close_loader,
start=self.first_asset_start,
end=self.last_asset_end,
env=self.env,
)
# Run for a week in the middle of our data.
algo.run(self.data_portal)
def test_multiple_pipelines(self):
"""
Test that we can attach multiple pipelines and access the correct
output based on the pipeline name.
"""
def initialize(context):
pipeline_close = attach_pipeline(Pipeline(), 'test_close')
pipeline_volume = attach_pipeline(Pipeline(), 'test_volume')
pipeline_close.add(USEquityPricing.close.latest, 'close')
pipeline_volume.add(USEquityPricing.volume.latest, 'volume')
def handle_data(context, data):
closes = pipeline_output('test_close')
volumes = pipeline_output('test_volume')
date = get_datetime().normalize()
for asset in self.assets:
# Assets should appear iff they exist today and yesterday.
exists_today = self.exists(date, asset)
existed_yesterday = self.exists(date - self.trading_day, asset)
if exists_today and existed_yesterday:
self.assertEqual(closes.loc[asset, 'close'],
self.expected_close(date, asset))
self.assertEqual(volumes.loc[asset, 'volume'],
self.expected_volume(date, asset))
else:
self.assertNotIn(asset, closes.index)
self.assertNotIn(asset, volumes.index)
column_to_loader = {
USEquityPricing.close: self.pipeline_close_loader,
USEquityPricing.volume: self.pipeline_volume_loader,
}
algo = TradingAlgorithm(
initialize=initialize,
handle_data=handle_data,
data_frequency='daily',
get_pipeline_loader=lambda column: column_to_loader[column],
start=self.first_asset_start,
end=self.last_asset_end,
env=self.env,
)
algo.run(self.data_portal)
def test_duplicate_pipeline_names(self):
"""
Test that we raise an error when we try to attach a pipeline with a
name that already exists for another attached pipeline.
"""
def initialize(context):
attach_pipeline(Pipeline(), 'test')
attach_pipeline(Pipeline(), 'test')
algo = TradingAlgorithm(
initialize=initialize,
data_frequency='daily',
get_pipeline_loader=lambda column: self.pipeline_close_loader,
start=self.first_asset_start,
end=self.last_asset_end,
env=self.env,
)
with self.assertRaises(DuplicatePipelineName):
algo.run(self.data_portal)
def ingest(name,
environ=os.environ,
timestamp=None,
assets_versions=(),
show_progress=False):
"""Ingest data for a given bundle.
Parameters
----------
name : str
The name of the bundle.
environ : mapping, optional
The environment variables. By default this is os.environ.
timestamp : datetime, optional
The timestamp to use for the load.
By default this is the current time.
assets_versions : Iterable[int], optional
Versions of the assets db to which to downgrade.
show_progress : bool, optional
Tell the ingest function to display the progress where possible.
"""
try:
bundle = bundles[name]
except KeyError:
raise UnknownBundle(name)
calendar = get_calendar(bundle.calendar_name)
start_session = bundle.start_session
end_session = bundle.end_session
if start_session is None or start_session < calendar.first_session:
start_session = calendar.first_session
if end_session is None or end_session > calendar.last_session:
end_session = calendar.last_session
if timestamp is None:
timestamp = pd.Timestamp.utcnow()
timestamp = timestamp.tz_convert('utc').tz_localize(None)
timestr = to_bundle_ingest_dirname(timestamp)
cachepath = cache_path(name, environ=environ)
pth.ensure_directory(pth.data_path([name, timestr], environ=environ))
pth.ensure_directory(cachepath)
with dataframe_cache(cachepath, clean_on_failure=False) as cache, \
ExitStack() as stack:
# we use `cleanup_on_failure=False` so that we don't purge the
# cache directory if the load fails in the middle
if bundle.create_writers:
wd = stack.enter_context(working_dir(
pth.data_path([], environ=environ))
)
daily_bars_path = wd.ensure_dir(
*daily_equity_relative(
name, timestr, environ=environ,
)
)
daily_bar_writer = BcolzDailyBarWriter(
daily_bars_path,
calendar,
start_session,
end_session,
)
# Do an empty write to ensure that the daily ctables exist
# when we create the SQLiteAdjustmentWriter below. The
# SQLiteAdjustmentWriter needs to open the daily ctables so
# that it can compute the adjustment ratios for the dividends.
daily_bar_writer.write(())
minute_bar_writer = BcolzMinuteBarWriter(
wd.ensure_dir(*minute_equity_relative(
name, timestr, environ=environ)
),
calendar,
start_session,
end_session,
minutes_per_day=bundle.minutes_per_day,
)
assets_db_path = wd.getpath(*asset_db_relative(
name, timestr, environ=environ,
))
asset_db_writer = AssetDBWriter(assets_db_path)
adjustment_db_writer = stack.enter_context(
SQLiteAdjustmentWriter(
wd.getpath(*adjustment_db_relative(
name, timestr, environ=environ)),
BcolzDailyBarReader(daily_bars_path),
calendar.all_sessions,
overwrite=True,
)
)
else:
daily_bar_writer = None
minute_bar_writer = None
asset_db_writer = None
adjustment_db_writer = None
if assets_versions:
raise ValueError('Need to ingest a bundle that creates '
'writers in order to downgrade the assets'
' db.')
bundle.ingest(
environ,
asset_db_writer,
minute_bar_writer,
daily_bar_writer,
adjustment_db_writer,
calendar,
start_session,
end_session,
cache,
show_progress,
pth.data_path([name, timestr], environ=environ),
)
for version in sorted(set(assets_versions), reverse=True):
version_path = wd.getpath(*asset_db_relative(
name, timestr, environ=environ, db_version=version,
))
with working_file(version_path) as wf:
shutil.copy2(assets_db_path, wf.path)
downgrade(wf.path, version)
def init_class_fixtures(cls):
super(TestMinuteBarDataFuturesCalendar, cls).init_class_fixtures()
cls.trading_calendar = get_calendar('CME')
def test_register_calendar_type(self):
register_calendar_type("DMY", self.dummy_cal_type)
retr_cal = get_calendar("DMY")
self.assertEqual(self.dummy_cal_type, type(retr_cal))
def trading_calendar(self):
if 'calendar_name' in self._table.attrs.attrs:
return get_calendar(self._table.attrs['calendar_name'])
else:
return None
def load_market_data(trading_day=None, trading_days=None, bm_symbol='SPY',
environ=None):
"""
Load benchmark returns and treasury yield curves for the given calendar and
benchmark symbol.
Benchmarks are downloaded as a Series from IEX Trading. Treasury curves
are US Treasury Bond rates and are downloaded from 'www.federalreserve.gov'
by default. For Canadian exchanges, a loader for Canadian bonds from the
Bank of Canada is also available.
Results downloaded from the internet are cached in
~/.gateway/data. Subsequent loads will attempt to read from the cached
files before falling back to redownload.
Parameters
----------
trading_day : pandas.CustomBusinessDay, optional
A trading_day used to determine the latest day for which we
expect to have data. Defaults to an NYSE trading day.
trading_days : pd.DatetimeIndex, optional
A calendar of trading days. Also used for determining what cached
dates we should expect to have cached. Defaults to the NYSE calendar.
bm_symbol : str, optional
Symbol for the benchmark index to load. Defaults to 'SPY', the ticker
for the S&P 500, provided by IEX Trading.
Returns
-------
(benchmark_returns, treasury_curves) : (pd.Series, pd.DataFrame)
Notes
-----
Both return values are DatetimeIndexed with values dated to midnight in UTC
of each stored date. The columns of `treasury_curves` are:
'1month', '3month', '6month',
'1year','2year','3year','5year','7year','10year','20year','30year'
"""
if trading_day is None:
trading_day = get_calendar('NYSE').trading_day
if trading_days is None:
trading_days = get_calendar('NYSE').all_sessions
first_date = trading_days[0]
now = pd.Timestamp.utcnow()
# we will fill missing benchmark data through latest trading date
last_date = trading_days[trading_days.get_loc(now, method='ffill')]
br = ensure_benchmark_data(
bm_symbol,
first_date,
last_date,
now,
# We need the trading_day to figure out the close prior to the first
# date so that we can compute returns for the first date.
trading_day,
environ,
)
tc = ensure_treasury_data(
bm_symbol,
first_date,
last_date,
now,
environ,
)
# combine dt indices and reindex using ffill then bfill
all_dt = br.index.union(tc.index)
br = br.reindex(all_dt, method='ffill').fillna(method='bfill')
tc = tc.reindex(all_dt, method='ffill').fillna(method='bfill')
benchmark_returns = br[br.index.slice_indexer(first_date, last_date)]
treasury_curves = tc[tc.index.slice_indexer(first_date, last_date)]
return benchmark_returns, treasury_curves
def _run(handle_data, initialize, before_trading_start, analyze, algofile,
algotext, defines, data_frequency, capital_base, data, bundle,
bundle_timestamp, start, end, output, trading_calendar, print_algo,
local_namespace, environ):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`gateway.run_algo`.
"""
if algotext is not None:
if local_namespace:
ip = get_ipython() # noqa
namespace = ip.user_ns
else:
namespace = {}
for assign in defines:
try:
name, value = assign.split('=', 2)
except ValueError:
raise ValueError(
'invalid define %r, should be of the form name=value' %
assign, )
try:
# evaluate in the same namespace so names may refer to
# eachother
namespace[name] = eval(value, namespace)
except Exception as e:
raise ValueError(
'failed to execute definition for name %r: %s' %
(name, e), )
elif defines:
raise _RunAlgoError(
'cannot pass define without `algotext`',
"cannot pass '-D' / '--define' without '-t' / '--algotext'",
)
else:
namespace = {}
if algofile is not None:
algotext = algofile.read()
if print_algo:
if PYGMENTS:
highlight(
algotext,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout,
)
else:
click.echo(algotext)
if trading_calendar is None:
trading_calendar = get_calendar('NYSE')
if bundle is not None:
bundle_data = load(
bundle,
environ,
bundle_timestamp,
)
prefix, connstr = re.split(
r'sqlite:///',
str(bundle_data.asset_finder.engine.url),
maxsplit=1,
)
if prefix:
raise ValueError(
"invalid url %r, must begin with 'sqlite:///'" %
str(bundle_data.asset_finder.engine.url), )
env = TradingEnvironment(asset_db_path=connstr, environ=environ)
first_trading_day =\
bundle_data.equity_minute_bar_reader.first_trading_day
data = DataPortal(
env.asset_finder,
trading_calendar=trading_calendar,
first_trading_day=first_trading_day,
equity_minute_reader=bundle_data.equity_minute_bar_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
pipeline_loader = USEquityPricingLoader(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader,
)
def choose_loader(column):
if column in USEquityPricing.columns:
return pipeline_loader
raise ValueError("No PipelineLoader registered for column %s." %
column)
else:
env = TradingEnvironment(environ=environ)
choose_loader = None
perf = TradingAlgorithm(
namespace=namespace,
env=env,
get_pipeline_loader=choose_loader,
trading_calendar=trading_calendar,
sim_params=create_simulation_parameters(
start=start,
end=end,
capital_base=capital_base,
data_frequency=data_frequency,
trading_calendar=trading_calendar,
),
**{
'initialize': initialize,
'handle_data': handle_data,
'before_trading_start': before_trading_start,
'analyze': analyze,
} if algotext is None else {
'algo_filename': getattr(algofile, 'name', '<algorithm>'),
'script': algotext,
}).run(
data,
overwrite_sim_params=False,
)
if output == '-':
click.echo(str(perf))
elif output != os.devnull: # make the gateway magic not write any data
perf.to_pickle(output)
return perf
def create_test_gateway(**config):
"""
:param config: A configuration object that is a dict with:
- sid - an integer, which will be used as the asset ID.
- order_count - the number of orders the test algo will place,
defaults to 100
- order_amount - the number of shares per order, defaults to 100
- trade_count - the number of trades to simulate, defaults to 101
to ensure all orders are processed.
- algorithm - optional parameter providing an algorithm. defaults
to :py:class:`gateway.test.algorithms.TestAlgorithm`
- trade_source - optional parameter to specify trades, if present.
If not present :py:class:`gateway.sources.SpecificEquityTrades`
is the source, with daily frequency in trades.
- slippage: optional parameter that configures the
:py:class:`gateway.gens.tradingsimulation.TransactionSimulator`.
Expects an object with a simulate mehod, such as
:py:class:`gateway.gens.tradingsimulation.FixedSlippage`.
:py:mod:`gateway.finance.trading`
"""
assert isinstance(config, dict)
try:
sid_list = config['sid_list']
except KeyError:
try:
sid_list = [config['sid']]
except KeyError:
raise Exception("simfactory create_test_gateway() requires "
"argument 'sid_list' or 'sid'")
concurrent_trades = config.get('concurrent_trades', False)
order_count = config.get('order_count', 100)
order_amount = config.get('order_amount', 100)
trading_calendar = config.get('trading_calendar', get_calendar("NYSE"))
# -------------------
# Create the Algo
# -------------------
if 'algorithm' in config:
test_algo = config['algorithm']
else:
test_algo = TestAlgorithm(sid_list[0],
order_amount,
order_count,
sim_params=config.get(
'sim_params',
factory.create_simulation_parameters()),
trading_calendar=trading_calendar,
slippage=config.get('slippage'),
identifiers=sid_list)
# -------------------
# Trade Source
# -------------------
if 'skip_data' not in config:
if 'trade_source' in config:
trade_source = config['trade_source']
else:
trade_source = factory.create_daily_trade_source(
sid_list,
test_algo.sim_params,
test_algo.trading_environment,
trading_calendar,
concurrent=concurrent_trades,
)
trades_by_sid = {}
for trade in trade_source:
if trade.sid not in trades_by_sid:
trades_by_sid[trade.sid] = []
trades_by_sid[trade.sid].append(trade)
data_portal = create_data_portal_from_trade_history(
config['env'].asset_finder, trading_calendar, config['tempdir'],
config['sim_params'], trades_by_sid)
test_algo.data_portal = data_portal
# -------------------
# Benchmark source
# -------------------
test_algo.benchmark_return_source = config.get('benchmark_source', None)
# ------------------
# generator/simulator
sim = test_algo.get_generator()
return sim
from gateway.utils.calendars import get_calendar, register_calendar
from .exchange_calendar_shsz import SHSZExchangeCalendar
register_calendar("SHSZ", SHSZExchangeCalendar(), force=True)
#singleton in python
shsz_calendar = get_calendar("SHSZ")
def test_ingest(self):
calendar = get_calendar('NYSE')
sessions = calendar.sessions_in_range(self.START_DATE, self.END_DATE)
minutes = calendar.minutes_for_sessions_in_range(
self.START_DATE,
self.END_DATE,
)
sids = tuple(range(3))
equities = make_simple_equity_info(
sids,
self.START_DATE,
self.END_DATE,
)
daily_bar_data = make_bar_data(equities, sessions)
minute_bar_data = make_bar_data(equities, minutes)
first_split_ratio = 0.5
second_split_ratio = 0.1
splits = pd.DataFrame.from_records([
{
'effective_date': str_to_seconds('2014-01-08'),
'ratio': first_split_ratio,
'sid': 0,
},
{
'effective_date': str_to_seconds('2014-01-09'),
'ratio': second_split_ratio,
'sid': 1,
},
])
@self.register(
'bundle',
calendar_name='NYSE',
start_session=self.START_DATE,
end_session=self.END_DATE,
)
def bundle_ingest(environ, asset_db_writer, minute_bar_writer,
daily_bar_writer, adjustment_writer, calendar,
start_session, end_session, cache, show_progress,
output_dir):
assert_is(environ, self.environ)
asset_db_writer.write(equities=equities)
minute_bar_writer.write(minute_bar_data)
daily_bar_writer.write(daily_bar_data)
adjustment_writer.write(splits=splits)
assert_is_instance(calendar, TradingCalendar)
assert_is_instance(cache, dataframe_cache)
assert_is_instance(show_progress, bool)
self.ingest('bundle', environ=self.environ)
bundle = self.load('bundle', environ=self.environ)
assert_equal(set(bundle.asset_finder.sids), set(sids))
columns = 'open', 'high', 'low', 'close', 'volume'
actual = bundle.equity_minute_bar_reader.load_raw_arrays(
columns,
minutes[0],
minutes[-1],
sids,
)
for actual_column, colname in zip(actual, columns):
assert_equal(
actual_column,
expected_bar_values_2d(minutes, equities, colname),
msg=colname,
)
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
columns,
self.START_DATE,
self.END_DATE,
sids,
)
for actual_column, colname in zip(actual, columns):
assert_equal(
actual_column,
expected_bar_values_2d(sessions, equities, colname),
msg=colname,
)
adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
columns,
sessions,
pd.Index(sids),
)
for column, adjustments in zip(columns, adjustments_for_cols[:-1]):
# iterate over all the adjustments but `volume`
assert_equal(
adjustments,
{
2: [
Float64Multiply(
first_row=0,
last_row=2,
first_col=0,
last_col=0,
value=first_split_ratio,
)
],
3: [
Float64Multiply(
first_row=0,
last_row=3,
first_col=1,
last_col=1,
value=second_split_ratio,
)
],
},
msg=column,
)
# check the volume, the value should be 1/ratio
assert_equal(
adjustments_for_cols[-1],
{
2: [
Float64Multiply(
first_row=0,
last_row=2,
first_col=0,
last_col=0,
value=1 / first_split_ratio,
)
],
3: [
Float64Multiply(
first_row=0,
last_row=3,
first_col=1,
last_col=1,
value=1 / second_split_ratio,
)
],
},
msg='volume',
)
def init_class_fixtures(cls):
super(TestDateUtils, cls).init_class_fixtures()
cls.calendar = get_calendar('NYSE')
class CSVDIRBundleTestCase(GatewayTestCase):
symbols = 'AAPL', 'IBM', 'KO', 'MSFT'
asset_start = pd.Timestamp('2012-01-03', tz='utc')
asset_end = pd.Timestamp('2014-12-31', tz='utc')
bundle = bundles['csvdir']
calendar = get_calendar(bundle.calendar_name)
start_date = calendar.first_session
end_date = calendar.last_session
api_key = 'ayylmao'
columns = 'open', 'high', 'low', 'close', 'volume'
def _expected_data(self, asset_finder):
sids = {
symbol: asset_finder.lookup_symbol(
symbol,
self.asset_start,
).sid
for symbol in self.symbols
}
def per_symbol(symbol):
df = pd.read_csv(
test_resource_path('csvdir_samples', 'csvdir', 'daily',
symbol + '.csv.gz'),
parse_dates=['date'],
index_col='date',
usecols=[
'open',
'high',
'low',
'close',
'volume',
'date',
'dividend',
'split',
],
na_values=['NA'],
)
df['sid'] = sids[symbol]
return df
all_ = pd.concat(map(per_symbol, self.symbols)).set_index(
'sid',
append=True,
).unstack()
# fancy list comprehension with statements
@list
@apply
def pricing():
for column in self.columns:
vs = all_[column].values
if column == 'volume':
vs = np.nan_to_num(vs)
yield vs
adjustments = [[
5572, 5576, 5595, 5634, 5639, 5659, 5698, 5699, 5701, 5702, 5722,
5760, 5764, 5774, 5821, 5822, 5829, 5845, 5884, 5885, 5888, 5908,
5947, 5948, 5951, 5972, 6011, 6020, 6026, 6073, 6080, 6096, 6135,
6136, 6139, 6157, 6160, 6198, 6199, 6207, 6223, 6263, 6271, 6277
],
[
5572, 5576, 5595, 5634, 5639, 5659, 5698, 5699,
5701, 5702, 5722, 5760, 5764, 5774, 5821, 5822,
5829, 5845, 5884, 5885, 5888, 5908, 5947, 5948,
5951, 5972, 6011, 6020, 6026, 6073, 6080, 6096,
6135, 6136, 6139, 6157, 6160, 6198, 6199, 6207,
6223, 6263, 6271, 6277
],
[
5572, 5576, 5595, 5634, 5639, 5659, 5698, 5699,
5701, 5702, 5722, 5760, 5764, 5774, 5821, 5822,
5829, 5845, 5884, 5885, 5888, 5908, 5947, 5948,
5951, 5972, 6011, 6020, 6026, 6073, 6080, 6096,
6135, 6136, 6139, 6157, 6160, 6198, 6199, 6207,
6223, 6263, 6271, 6277
],
[
5572, 5576, 5595, 5634, 5639, 5659, 5698, 5699,
5701, 5702, 5722, 5760, 5764, 5774, 5821, 5822,
5829, 5845, 5884, 5885, 5888, 5908, 5947, 5948,
5951, 5972, 6011, 6020, 6026, 6073, 6080, 6096,
6135, 6136, 6139, 6157, 6160, 6198, 6199, 6207,
6223, 6263, 6271, 6277
], [5701, 6157]]
return pricing, adjustments
def test_bundle(self):
environ = {'CSVDIR': test_resource_path('csvdir_samples', 'csvdir')}
ingest('csvdir', environ=environ)
bundle = load('csvdir', environ=environ)
sids = 0, 1, 2, 3
assert_equal(set(bundle.asset_finder.sids), set(sids))
for equity in bundle.asset_finder.retrieve_all(sids):
assert_equal(equity.start_date, self.asset_start, msg=equity)
assert_equal(equity.end_date, self.asset_end, msg=equity)
sessions = self.calendar.all_sessions
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
sessions[sessions.get_loc(self.asset_start, 'bfill')],
sessions[sessions.get_loc(self.asset_end, 'ffill')],
sids,
)
expected_pricing, expected_adjustments = self._expected_data(
bundle.asset_finder, )
assert_equal(actual, expected_pricing, array_decimal=2)
adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
self.columns,
sessions,
pd.Index(sids),
)
assert_equal([sorted(adj.keys()) for adj in adjustments_for_cols],
expected_adjustments)
def setUpClass(cls):
super(StatefulRulesTests, cls).setUpClass()
cls.class_ = StatefulRule
cls.cal = get_calendar(cls.CALENDAR_STRING)
def test_default_calendars(self):
for name in concat(
[_default_calendar_factories, _default_calendar_aliases]):
self.assertIsNotNone(get_calendar(name),
"get_calendar(%r) returned None" % name)
