def init_class_fixtures(cls):
super(SecurityListTestCase, cls).init_class_fixtures()
# this is ugly, but we need to create two different
# TradingEnvironment/DataPortal pairs
cls.start = pd.Timestamp(list(LEVERAGED_ETFS.keys())[0])
end = pd.Timestamp('2015-02-17', tz='utc')
cls.extra_knowledge_date = pd.Timestamp('2015-01-27', tz='utc')
cls.trading_day_before_first_kd = pd.Timestamp('2015-01-23', tz='utc')
symbols = ['AAPL', 'GOOG', 'BZQ', 'URTY', 'JFT']
cls.env = cls.enter_class_context(tmp_trading_env(
equities=pd.DataFrame.from_records([{
'start_date': cls.start,
'end_date': end,
'symbol': symbol,
'exchange': "TEST",
} for symbol in symbols]),
load=cls.make_load_function(),
))
cls.sim_params = factory.create_simulation_parameters(
start=cls.start,
num_days=4,
trading_calendar=cls.trading_calendar
)
cls.sim_params2 = sp2 = factory.create_simulation_parameters(
start=cls.trading_day_before_first_kd, num_days=4
)
cls.env2 = cls.enter_class_context(tmp_trading_env(
equities=pd.DataFrame.from_records([{
'start_date': sp2.start_session,
'end_date': sp2.end_session,
'symbol': symbol,
'exchange': "TEST",
} for symbol in symbols]),
load=cls.make_load_function(),
))
cls.tempdir = cls.enter_class_context(tmp_dir())
cls.tempdir2 = cls.enter_class_context(tmp_dir())
cls.data_portal = create_data_portal(
asset_finder=cls.env.asset_finder,
tempdir=cls.tempdir,
sim_params=cls.sim_params,
sids=range(0, 5),
trading_calendar=cls.trading_calendar,
)
cls.data_portal2 = create_data_portal(
asset_finder=cls.env2.asset_finder,
tempdir=cls.tempdir2,
sim_params=cls.sim_params2,
sids=range(0, 5),
trading_calendar=cls.trading_calendar,
)
def init_class_fixtures(cls):
super(SecurityListTestCase, cls).init_class_fixtures()
# this is ugly, but we need to create two different
# TradingEnvironment/DataPortal pairs
cls.start = pd.Timestamp(list(LEVERAGED_ETFS.keys())[0])
end = pd.Timestamp('2015-02-17', tz='utc')
cls.extra_knowledge_date = pd.Timestamp('2015-01-27', tz='utc')
cls.trading_day_before_first_kd = pd.Timestamp('2015-01-23', tz='utc')
symbols = ['AAPL', 'GOOG', 'BZQ', 'URTY', 'JFT']
cls.env = cls.enter_class_context(
tmp_trading_env(equities=pd.DataFrame.from_records([{
'start_date':
cls.start,
'end_date':
end,
'symbol':
symbol,
'exchange':
"TEST",
} for symbol in symbols]), ))
cls.sim_params = factory.create_simulation_parameters(
start=cls.start, num_days=4, trading_calendar=cls.trading_calendar)
cls.sim_params2 = sp2 = factory.create_simulation_parameters(
start=cls.trading_day_before_first_kd, num_days=4)
cls.env2 = cls.enter_class_context(
tmp_trading_env(equities=pd.DataFrame.from_records([{
'start_date':
sp2.start_session,
'end_date':
sp2.end_session,
'symbol':
symbol,
'exchange':
"TEST",
} for symbol in symbols]), ))
cls.tempdir = cls.enter_class_context(tmp_dir())
cls.tempdir2 = cls.enter_class_context(tmp_dir())
cls.data_portal = create_data_portal(
asset_finder=cls.env.asset_finder,
tempdir=cls.tempdir,
sim_params=cls.sim_params,
sids=range(0, 5),
trading_calendar=cls.trading_calendar,
)
cls.data_portal2 = create_data_portal(
asset_finder=cls.env2.asset_finder,
tempdir=cls.tempdir2,
sim_params=cls.sim_params2,
sids=range(0, 5),
trading_calendar=cls.trading_calendar,
)
def test_bundle(self):
with open(test_resource_path(
'quandl_samples',
'QUANDL_ARCHIVE.zip'), 'rb') as quandl_response:
self.responses.add(
self.responses.GET,
'https://file_url.mock.quandl',
body=quandl_response.read(),
content_type='application/zip',
status=200,
)
url_map = {
format_metadata_url(self.api_key): test_resource_path(
'quandl_samples',
'metadata.csv.gz',
)
}
zipline_root = self.enter_instance_context(tmp_dir()).path
environ = {
'ZIPLINE_ROOT': zipline_root,
'QUANDL_API_KEY': self.api_key,
}
with patch_read_csv(url_map):
ingest('quandl', environ=environ)
bundle = load('quandl', environ=environ)
sids = 0, 1, 2, 3
assert_equal(set(bundle.asset_finder.sids), set(sids))
sessions = self.calendar.all_sessions
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
sessions[sessions.get_loc(self.start_date, 'bfill')],
sessions[sessions.get_loc(self.end_date, 'ffill')],
sids,
)
expected_pricing, expected_adjustments = self._expected_data(
bundle.asset_finder,
)
assert_equal(actual, expected_pricing, array_decimal=2)
adjs_for_cols = bundle.adjustment_reader.load_pricing_adjustments(
self.columns,
sessions,
pd.Index(sids),
)
for column, adjustments, expected in zip(self.columns,
adjs_for_cols,
expected_adjustments):
assert_equal(
adjustments,
expected,
msg=column,
)
def test_bundle(self):
with open(test_resource_path(
'quandl_samples',
'QUANDL_ARCHIVE.zip'), 'rb') as quandl_response:
self.responses.add(
self.responses.GET,
'https://file_url.mock.quandl',
body=quandl_response.read(),
content_type='application/zip',
status=200,
)
url_map = {
format_metadata_url(self.api_key): test_resource_path(
'quandl_samples',
'metadata.csv.gz',
)
}
zipline_root = self.enter_instance_context(tmp_dir()).path
environ = {
'ZIPLINE_ROOT': zipline_root,
'QUANDL_API_KEY': self.api_key,
}
with patch_read_csv(url_map):
ingest('quandl', environ=environ)
bundle = load('quandl', environ=environ)
sids = 0, 1, 2, 3
assert_equal(set(bundle.asset_finder.sids), set(sids))
sessions = self.calendar.all_sessions
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
sessions[sessions.get_loc(self.start_date, 'bfill')],
sessions[sessions.get_loc(self.end_date, 'ffill')],
sids,
)
expected_pricing, expected_adjustments = self._expected_data(
bundle.asset_finder,
)
assert_equal(actual, expected_pricing, array_decimal=2)
adjs_for_cols = bundle.adjustment_reader.load_pricing_adjustments(
self.columns,
sessions,
pd.Index(sids),
)
for column, adjustments, expected in zip(self.columns,
adjs_for_cols,
expected_adjustments):
assert_equal(
adjustments,
expected,
msg=column,
)
def test_bundle(self):
with open(
join(TEST_RESOURCE_PATH, "quandl_samples",
"QUANDL_ARCHIVE.zip"),
"rb",
) as quandl_response:
self.responses.add(
self.responses.GET,
"https://file_url.mock.quandl",
body=quandl_response.read(),
content_type="application/zip",
status=200,
)
url_map = {
format_metadata_url(self.api_key):
join(
TEST_RESOURCE_PATH,
"quandl_samples",
"metadata.csv.gz",
)
}
zipline_root = self.enter_instance_context(tmp_dir()).path
environ = {
"ZIPLINE_ROOT": zipline_root,
"QUANDL_API_KEY": self.api_key,
}
with patch_read_csv(url_map):
ingest("quandl", environ=environ)
bundle = load("quandl", environ=environ)
sids = 0, 1, 2, 3
assert set(bundle.asset_finder.sids) == set(sids)
sessions = self.calendar.all_sessions
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
sessions[sessions.get_loc(self.start_date, "bfill")],
sessions[sessions.get_loc(self.end_date, "ffill")],
sids,
)
expected_pricing, expected_adjustments = self._expected_data(
bundle.asset_finder, )
np.testing.assert_array_almost_equal(actual,
expected_pricing,
decimal=2)
adjs_for_cols = bundle.adjustment_reader.load_pricing_adjustments(
self.columns,
sessions,
pd.Index(sids),
)
for column, adjustments, expected in zip(self.columns, adjs_for_cols,
expected_adjustments):
assert adjustments == expected, column
def test_bundle(self):
zipline_root = self.enter_instance_context(tmp_dir()).path
environ = {
'ZIPLINE_ROOT': zipline_root,
'QUANDL_API_KEY': self.api_key,
}
# custom bundles need to be registered before use or they will not
# be recognized
register(
'ZacksQuandl',
from_zacks_dump(
test_resource_path('zacks_samples', 'fictitious.csv')))
ingest('ZacksQuandl', environ=environ)
# load bundle now that it has been ingested
bundle = load('ZacksQuandl', environ=environ)
sids = 0, 1, 2
# check sids match
assert_equal(set(bundle.asset_finder.sids), set(sids))
# check asset_{start, end} is the same as {start, end}_date
for equity in bundle.asset_finder.retrieve_all(sids):
assert_equal(equity.start_date, self.asset_start, msg=equity)
assert_equal(equity.end_date, self.asset_end, msg=equity)
# get daily OHLCV data from bundle
sessions = self.calendar.all_sessions
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
sessions[sessions.get_loc(self.asset_start, 'bfill')],
sessions[sessions.get_loc(self.asset_end, 'ffill')],
sids,
)
# get expected data from csv
expected_pricing, expected_adjustments = self._expected_data(
bundle.asset_finder, )
# check OHLCV data matches
assert_equal(actual, expected_pricing, array_decimal=2)
adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
self.columns,
sessions,
pd.Index(sids),
)
for column, adjustments, expected in zip(self.columns,
adjustments_for_cols,
expected_adjustments):
assert_equal(
adjustments,
expected,
msg=column,
)
def test_bundle(self):
url_map = merge(
{
format_wiki_url(
self.api_key,
symbol,
self.start_date,
self.end_date,
): test_resource_path('quandl_samples', symbol + '.csv.gz')
for symbol in self.symbols
},
{
format_metadata_url(self.api_key, n): test_resource_path(
'quandl_samples',
'metadata-%d.csv.gz' % n,
)
for n in (1, 2)
},
)
zipline_root = self.enter_instance_context(tmp_dir()).path
environ = {
'ZIPLINE_ROOT': zipline_root,
'QUANDL_API_KEY': self.api_key,
}
with patch_read_csv(url_map, strict=True):
ingest('quandl', environ=environ)
bundle = load('quandl', environ=environ)
sids = 0, 1, 2, 3
assert_equal(set(bundle.asset_finder.sids), set(sids))
for equity in bundle.asset_finder.retrieve_all(sids):
assert_equal(equity.start_date, self.asset_start, msg=equity)
assert_equal(equity.end_date, self.asset_end, msg=equity)
sessions = self.calendar.all_sessions
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
sessions[sessions.get_loc(self.asset_start, 'bfill')],
sessions[sessions.get_loc(self.asset_end, 'ffill')],
sids,
)
expected_pricing, expected_adjustments = self._expected_data(
bundle.asset_finder,
)
assert_equal(actual, expected_pricing, array_decimal=2)
adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
self.columns,
sessions,
pd.Index(sids),
)
for column, adjustments, expected in zip(self.columns,
adjustments_for_cols,
expected_adjustments):
assert_equal(
adjustments,
expected,
msg=column,
)
def test_bundle(self):
def get_symbol_from_url(url):
params = parse_qs(urlparse(url).query)
symbol, = params['s']
return symbol
def pricing_callback(request):
headers = {
'content-encoding': 'gzip',
'content-type': 'text/csv',
}
path = test_resource_path(
'yahoo_samples',
get_symbol_from_url(request.url) + '.csv.gz',
)
with open(path, 'rb') as f:
return (
200,
headers,
f.read(),
)
for _ in range(3):
self.responses.add_callback(
self.responses.GET,
'http://ichart.finance.yahoo.com/table.csv',
pricing_callback,
)
def adjustments_callback(request):
path = test_resource_path(
'yahoo_samples',
get_symbol_from_url(request.url) + '.adjustments.gz',
)
return 200, {}, read_compressed(path)
for _ in range(3):
self.responses.add_callback(
self.responses.GET,
'http://ichart.finance.yahoo.com/x',
adjustments_callback,
)
cal = self.calendar
self.register(
'bundle',
yahoo_equities(self.symbols),
calendar=cal,
)
zipline_root = self.enter_instance_context(tmp_dir()).path
environ = {
'ZIPLINE_ROOT': zipline_root,
}
self.ingest('bundle', environ=environ)
bundle = load('bundle', environ=environ)
sids = 0, 1, 2
equities = bundle.asset_finder.retrieve_all(sids)
for equity, expected_symbol in zip(equities, self.symbols):
assert_equal(equity.symbol, expected_symbol)
for equity in bundle.asset_finder.retrieve_all(sids):
assert_equal(equity.start_date, self.asset_start, msg=equity)
assert_equal(equity.end_date, self.asset_end, msg=equity)
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
cal[cal.get_loc(self.asset_start, 'bfill')],
cal[cal.get_loc(self.asset_end, 'ffill')],
sids,
)
expected_pricing, expected_adjustments = self._expected_data()
assert_equal(actual, expected_pricing, array_decimal=2)
adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
self.columns,
cal,
pd.Index(sids),
)
for column, adjustments, expected in zip(self.columns,
adjustments_for_cols,
expected_adjustments):
assert_equal(
adjustments,
expected,
msg=column,
)
def test_bundle(self):
url_map = merge(
{
format_wiki_url(
self.api_key,
symbol,
self.start_date,
self.end_date,
): test_resource_path('quandl_samples', symbol + '.csv.gz')
for symbol in self.symbols
},
{
format_metadata_url(self.api_key, n): test_resource_path(
'quandl_samples',
'metadata-%d.csv.gz' % n,
)
for n in (1, 2)
},
)
zipline_root = self.enter_instance_context(tmp_dir()).path
environ = {
'ZIPLINE_ROOT': zipline_root,
'QUANDL_API_KEY': self.api_key,
}
with patch_read_csv(url_map, strict=True):
ingest('quandl', environ=environ)
bundle = load('quandl', environ=environ)
sids = 0, 1, 2, 3
assert_equal(set(bundle.asset_finder.sids), set(sids))
for equity in bundle.asset_finder.retrieve_all(sids):
assert_equal(equity.start_date, self.asset_start, msg=equity)
assert_equal(equity.end_date, self.asset_end, msg=equity)
cal = self.calendar
actual = bundle.daily_bar_reader.load_raw_arrays(
self.columns,
cal[cal.get_loc(self.asset_start, 'bfill')],
cal[cal.get_loc(self.asset_end, 'ffill')],
sids,
)
expected_pricing, expected_adjustments = self._expected_data(
bundle.asset_finder,
)
assert_equal(actual, expected_pricing, array_decimal=2)
adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
self.columns,
cal,
pd.Index(sids),
)
for column, adjustments, expected in zip(self.columns,
adjustments_for_cols,
expected_adjustments):
assert_equal(
adjustments,
expected,
msg=column,
)
def test_bundle(self):
def get_symbol_from_url(url):
params = parse_qs(urlparse(url).query)
symbol, = params['s']
return symbol
def pricing_callback(request):
headers = {
'content-encoding': 'gzip',
'content-type': 'text/csv',
}
path = test_resource_path(
'yahoo_samples',
get_symbol_from_url(request.url) + '.csv.gz',
)
with open(path, 'rb') as f:
return (
200,
headers,
f.read(),
)
for _ in range(3):
self.responses.add_callback(
self.responses.GET,
'http://ichart.finance.yahoo.com/table.csv',
pricing_callback,
)
def adjustments_callback(request):
path = test_resource_path(
'yahoo_samples',
get_symbol_from_url(request.url) + '.adjustments.gz',
)
return 200, {}, read_compressed(path)
for _ in range(3):
self.responses.add_callback(
self.responses.GET,
'http://ichart.finance.yahoo.com/x',
adjustments_callback,
)
self.register(
'bundle',
yahoo_equities(self.symbols),
calendar_name='NYSE',
start_session=self.asset_start,
end_session=self.asset_end,
)
zipline_root = self.enter_instance_context(tmp_dir()).path
environ = {
'ZIPLINE_ROOT': zipline_root,
}
self.ingest('bundle', environ=environ, show_progress=False)
bundle = self.load('bundle', environ=environ)
sids = 0, 1, 2
equities = bundle.asset_finder.retrieve_all(sids)
for equity, expected_symbol in zip(equities, self.symbols):
assert_equal(equity.symbol, expected_symbol)
for equity in bundle.asset_finder.retrieve_all(sids):
assert_equal(equity.start_date, self.asset_start, msg=equity)
assert_equal(equity.end_date, self.asset_end, msg=equity)
sessions = self.sessions
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
self.columns,
sessions[sessions.get_loc(self.asset_start, 'bfill')],
sessions[sessions.get_loc(self.asset_end, 'ffill')],
sids,
)
expected_pricing, expected_adjustments = self._expected_data()
assert_equal(actual, expected_pricing, array_decimal=2)
adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
self.columns,
self.sessions,
pd.Index(sids),
)
for column, adjustments, expected in zip(self.columns,
adjustments_for_cols,
expected_adjustments):
assert_equal(
adjustments,
expected,
msg=column,
decimal=4,
)
def test_ingest(self):
zipline_root = self.enter_instance_context(tmp_dir()).path
env = self.enter_instance_context(tmp_trading_env())
start = pd.Timestamp('2014-01-06', tz='utc')
end = pd.Timestamp('2014-01-10', tz='utc')
calendar = trading_days[trading_days.slice_indexer(start, end)]
minutes = env.minutes_for_days_in_range(calendar[0], calendar[-1])
outer_environ = {
'ZIPLINE_ROOT': zipline_root,
}
sids = tuple(range(3))
equities = make_simple_equity_info(
sids,
calendar[0],
calendar[-1],
)
daily_bar_data = make_bar_data(equities, calendar)
minute_bar_data = make_bar_data(equities, minutes)
first_split_ratio = 0.5
second_split_ratio = 0.1
splits = pd.DataFrame.from_records([
{
'effective_date': str_to_seconds('2014-01-08'),
'ratio': first_split_ratio,
'sid': 0,
},
{
'effective_date': str_to_seconds('2014-01-09'),
'ratio': second_split_ratio,
'sid': 1,
},
])
@self.register('bundle',
calendar=calendar,
opens=env.opens_in_range(calendar[0], calendar[-1]),
closes=env.closes_in_range(calendar[0], calendar[-1]))
def bundle_ingest(environ,
asset_db_writer,
minute_bar_writer,
daily_bar_writer,
adjustment_writer,
calendar,
cache,
show_progress):
assert_is(environ, outer_environ)
asset_db_writer.write(equities=equities)
minute_bar_writer.write(minute_bar_data)
daily_bar_writer.write(daily_bar_data)
adjustment_writer.write(splits=splits)
assert_is_instance(calendar, pd.DatetimeIndex)
assert_is_instance(cache, dataframe_cache)
assert_is_instance(show_progress, bool)
self.ingest('bundle', environ=outer_environ)
bundle = load('bundle', environ=outer_environ)
assert_equal(set(bundle.asset_finder.sids), set(sids))
columns = 'open', 'high', 'low', 'close', 'volume'
actual = bundle.minute_bar_reader.load_raw_arrays(
columns,
minutes[0],
minutes[-1],
sids,
)
for actual_column, colname in zip(actual, columns):
assert_equal(
actual_column,
expected_bar_values_2d(minutes, equities, colname),
msg=colname,
)
actual = bundle.daily_bar_reader.load_raw_arrays(
columns,
calendar[0],
calendar[-1],
sids,
)
for actual_column, colname in zip(actual, columns):
assert_equal(
actual_column,
expected_bar_values_2d(calendar, equities, colname),
msg=colname,
)
adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
columns,
calendar,
pd.Index(sids),
)
for column, adjustments in zip(columns, adjustments_for_cols[:-1]):
# iterate over all the adjustments but `volume`
assert_equal(
adjustments,
{
2: [Float64Multiply(
first_row=0,
last_row=2,
first_col=0,
last_col=0,
value=first_split_ratio,
)],
3: [Float64Multiply(
first_row=0,
last_row=3,
first_col=1,
last_col=1,
value=second_split_ratio,
)],
},
msg=column,
)
# check the volume, the value should be 1/ratio
assert_equal(
adjustments_for_cols[-1],
{
2: [Float64Multiply(
first_row=0,
last_row=2,
first_col=0,
last_col=0,
value=1 / first_split_ratio,
)],
3: [Float64Multiply(
first_row=0,
last_row=3,
first_col=1,
last_col=1,
value=1 / second_split_ratio,
)],
},
msg='volume',
)
def main(ctx, rebuild_input):
"""Rebuild the perf data for test_examples"""
example_path = test_resource_path("example_data.tar.gz")
with tmp_dir() as d:
with tarfile.open(example_path) as tar:
tar.extractall(d.path)
# The environ here should be the same (modulo the tempdir location)
# as we use in test_examples.py.
environ = {"ZIPLINE_ROOT": d.getpath("example_data/root")}
if rebuild_input:
raise NotImplementedError(
"We cannot rebuild input for Yahoo because of "
"changes Yahoo made to their API, so we cannot "
"use Yahoo data bundles anymore. This will be fixed in "
"a future release",
)
# we need to register the bundle; it is already ingested and saved in
# the example_data.tar.gz file
@register("test")
def nop_ingest(*args, **kwargs):
raise NotImplementedError("we cannot rebuild the test buindle")
new_perf_path = d.getpath(
"example_data/new_perf/%s" % pd.__version__.replace(".", "-"),
)
c = dataframe_cache(
new_perf_path,
serialization="pickle:2",
)
with c:
for name in EXAMPLE_MODULES:
c[name] = examples.run_example(
EXAMPLE_MODULES,
name,
environ=environ,
benchmark_returns=read_checked_in_benchmark_data(),
)
correct_called = [False]
console = None
def _exit(*args, **kwargs):
console.raw_input = eof
def correct():
correct_called[0] = True
_exit()
expected_perf_path = d.getpath(
"example_data/expected_perf/%s"
% pd.__version__.replace(".", "-"),
)
# allow users to run some analysis to make sure that the new
# results check out
console = InteractiveConsole(
{
"correct": correct,
"exit": _exit,
"incorrect": _exit,
"new": c,
"np": np,
"old": dataframe_cache(
expected_perf_path,
serialization="pickle",
),
"pd": pd,
"cols_to_check": examples._cols_to_check,
"changed_results": changed_results,
}
)
console.interact(banner)
if not correct_called[0]:
ctx.fail(
"`correct()` was not called! This means that the new"
" results will not be written",
)
# move the new results to the expected path
shutil.rmtree(expected_perf_path)
shutil.copytree(new_perf_path, expected_perf_path)
# Clear out all the temporary new perf so it doesn't get added to the
# tarball.
shutil.rmtree(d.getpath("example_data/new_perf/"))
with tarfile.open(example_path, "w|gz") as tar:
tar.add(d.getpath("example_data"), "example_data")