def init_class_fixtures(cls):
super(ExamplesTests, cls).init_class_fixtures()
register('test', lambda *args: None)
cls.add_class_callback(partial(unregister, 'test'))
with tarfile.open(test_resource_path('example_data.tar.gz')) as tar:
tar.extractall(cls.tmpdir.path)
cls.expected_perf = dataframe_cache(
cls.tmpdir.getpath(
'example_data/expected_perf/%s' %
pd.__version__.replace('.', '-'), ),
serialization='pickle',
)
cls.benchmark_returns = read_checked_in_benchmark_data()
class TestsExamplesTests:
# some columns contain values with unique ids that will not be the same
@pytest.mark.filterwarnings("ignore: Matplotlib is currently using agg")
@pytest.mark.parametrize("benchmark_returns",
[read_checked_in_benchmark_data(), None])
@pytest.mark.parametrize("example_name", sorted(EXAMPLE_MODULES))
@pytest.mark.parametrize("_df_cache", STORED_DATA_VERSIONS, indirect=True)
def test_example(self, example_name, benchmark_returns):
actual_perf = examples.run_example(
EXAMPLE_MODULES,
example_name,
# This should match the invocation in
# zipline/tests/resources/rebuild_example_data
environ={
"ZIPLINE_ROOT": join(self.tmpdir, "example_data", "root"),
},
benchmark_returns=benchmark_returns,
)
if benchmark_returns is not None:
expected_perf = self.expected_perf[example_name]
else:
expected_perf = self.no_benchmark_expected_perf[example_name]
# Exclude positions column as the positions do not always have the
# same order
columns = [
column for column in examples._cols_to_check
if column != "positions"
]
assert_equal(
actual_perf[columns],
expected_perf[columns],
# There is a difference in the datetime columns in pandas
# 0.16 and 0.17 because in 16 they are object and in 17 they are
# datetime[ns, UTC]. We will just ignore the dtypes for now.
# check_dtype=False,
)
# Sort positions by SID before comparing
assert_equal(
expected_perf["positions"].apply(sorted, key=itemgetter("sid")),
actual_perf["positions"].apply(sorted, key=itemgetter("sid")),
)
class ExamplesTests(WithTmpDir, ZiplineTestCase):
# some columns contain values with unique ids that will not be the same
if platform.system() != 'Windows':
EXAMPLE_MODULES = examples.load_example_modules()
else:
EXAMPLE_MODULES = {}
@classmethod
def init_class_fixtures(cls):
super(ExamplesTests, cls).init_class_fixtures()
register('test', lambda *args: None)
cls.add_class_callback(partial(unregister, 'test'))
with tarfile.open(test_resource_path('example_data.tar.gz')) as tar:
tar.extractall(cls.tmpdir.path)
cls.expected_perf = dataframe_cache(
cls.tmpdir.getpath(
'example_data/expected_perf/%s' %
pd.__version__.replace('.', '-'),
),
serialization='pickle',
)
cls.no_benchmark_expected_perf = {
example_name: cls._no_benchmark_expectations_applied(
expected_perf.copy()
)
for example_name, expected_perf in cls.expected_perf.items()
}
@staticmethod
def _no_benchmark_expectations_applied(expected_perf):
# With no benchmark, expect zero results for these metrics:
expected_perf[['alpha', 'beta']] = None
for col in ['benchmark_period_return', 'benchmark_volatility']:
expected_perf.loc[
~pd.isnull(expected_perf[col]),
col,
] = 0.0
return expected_perf
@unittest.skipIf(platform.system() == 'Windows', "Don't run test on windows")
@parameter_space(
example_name=sorted(EXAMPLE_MODULES),
benchmark_returns=[read_checked_in_benchmark_data(), None]
)
def test_example(self, example_name, benchmark_returns):
actual_perf = examples.run_example(
self.EXAMPLE_MODULES,
example_name,
# This should match the invocation in
# zipline/tests/resources/rebuild_example_data
environ={
'ZIPLINE_ROOT': self.tmpdir.getpath('example_data/root'),
},
benchmark_returns=benchmark_returns,
)
if benchmark_returns is not None:
expected_perf = self.expected_perf[example_name]
else:
expected_perf = self.no_benchmark_expected_perf[example_name]
# Exclude positions column as the positions do not always have the
# same order
columns = [column for column in examples._cols_to_check
if column != 'positions']
assert_equal(
actual_perf[columns],
expected_perf[columns],
# There is a difference in the datetime columns in pandas
# 0.16 and 0.17 because in 16 they are object and in 17 they are
# datetime[ns, UTC]. We will just ignore the dtypes for now.
check_dtype=False,
)
# Sort positions by SID before comparing
assert_equal(
expected_perf['positions'].apply(sorted, key=itemgetter('sid')),
actual_perf['positions'].apply(sorted, key=itemgetter('sid')),
)
def main(ctx, rebuild_input):
"""Rebuild the perf data for test_examples"""
example_path = test_resource_path("example_data.tar.gz")
with tmp_dir() as d:
with tarfile.open(example_path) as tar:
tar.extractall(d.path)
# The environ here should be the same (modulo the tempdir location)
# as we use in test_examples.py.
environ = {"ZIPLINE_ROOT": d.getpath("example_data/root")}
if rebuild_input:
raise NotImplementedError(
"We cannot rebuild input for Yahoo because of "
"changes Yahoo made to their API, so we cannot "
"use Yahoo data bundles anymore. This will be fixed in "
"a future release",
)
# we need to register the bundle; it is already ingested and saved in
# the example_data.tar.gz file
@register("test")
def nop_ingest(*args, **kwargs):
raise NotImplementedError("we cannot rebuild the test buindle")
new_perf_path = d.getpath(
"example_data/new_perf/%s" % pd.__version__.replace(".", "-"),
)
c = dataframe_cache(
new_perf_path,
serialization="pickle:2",
)
with c:
for name in EXAMPLE_MODULES:
c[name] = examples.run_example(
EXAMPLE_MODULES,
name,
environ=environ,
benchmark_returns=read_checked_in_benchmark_data(),
)
correct_called = [False]
console = None
def _exit(*args, **kwargs):
console.raw_input = eof
def correct():
correct_called[0] = True
_exit()
expected_perf_path = d.getpath(
"example_data/expected_perf/%s"
% pd.__version__.replace(".", "-"),
)
# allow users to run some analysis to make sure that the new
# results check out
console = InteractiveConsole(
{
"correct": correct,
"exit": _exit,
"incorrect": _exit,
"new": c,
"np": np,
"old": dataframe_cache(
expected_perf_path,
serialization="pickle",
),
"pd": pd,
"cols_to_check": examples._cols_to_check,
"changed_results": changed_results,
}
)
console.interact(banner)
if not correct_called[0]:
ctx.fail(
"`correct()` was not called! This means that the new"
" results will not be written",
)
# move the new results to the expected path
shutil.rmtree(expected_perf_path)
shutil.copytree(new_perf_path, expected_perf_path)
# Clear out all the temporary new perf so it doesn't get added to the
# tarball.
shutil.rmtree(d.getpath("example_data/new_perf/"))
with tarfile.open(example_path, "w|gz") as tar:
tar.add(d.getpath("example_data"), "example_data")