def init_class_fixtures(cls): super(ExamplesTests, cls).init_class_fixtures() register('test', lambda *args: None) cls.add_class_callback(partial(unregister, 'test')) with tarfile.open(test_resource_path('example_data.tar.gz')) as tar: tar.extractall(cls.tmpdir.path) cls.expected_perf = dataframe_cache( cls.tmpdir.getpath( 'example_data/expected_perf/%s' % pd.__version__.replace('.', '-'), ), serialization='pickle', ) cls.benchmark_returns = read_checked_in_benchmark_data()
class TestsExamplesTests: # some columns contain values with unique ids that will not be the same @pytest.mark.filterwarnings("ignore: Matplotlib is currently using agg") @pytest.mark.parametrize("benchmark_returns", [read_checked_in_benchmark_data(), None]) @pytest.mark.parametrize("example_name", sorted(EXAMPLE_MODULES)) @pytest.mark.parametrize("_df_cache", STORED_DATA_VERSIONS, indirect=True) def test_example(self, example_name, benchmark_returns): actual_perf = examples.run_example( EXAMPLE_MODULES, example_name, # This should match the invocation in # zipline/tests/resources/rebuild_example_data environ={ "ZIPLINE_ROOT": join(self.tmpdir, "example_data", "root"), }, benchmark_returns=benchmark_returns, ) if benchmark_returns is not None: expected_perf = self.expected_perf[example_name] else: expected_perf = self.no_benchmark_expected_perf[example_name] # Exclude positions column as the positions do not always have the # same order columns = [ column for column in examples._cols_to_check if column != "positions" ] assert_equal( actual_perf[columns], expected_perf[columns], # There is a difference in the datetime columns in pandas # 0.16 and 0.17 because in 16 they are object and in 17 they are # datetime[ns, UTC]. We will just ignore the dtypes for now. # check_dtype=False, ) # Sort positions by SID before comparing assert_equal( expected_perf["positions"].apply(sorted, key=itemgetter("sid")), actual_perf["positions"].apply(sorted, key=itemgetter("sid")), )
class ExamplesTests(WithTmpDir, ZiplineTestCase): # some columns contain values with unique ids that will not be the same if platform.system() != 'Windows': EXAMPLE_MODULES = examples.load_example_modules() else: EXAMPLE_MODULES = {} @classmethod def init_class_fixtures(cls): super(ExamplesTests, cls).init_class_fixtures() register('test', lambda *args: None) cls.add_class_callback(partial(unregister, 'test')) with tarfile.open(test_resource_path('example_data.tar.gz')) as tar: tar.extractall(cls.tmpdir.path) cls.expected_perf = dataframe_cache( cls.tmpdir.getpath( 'example_data/expected_perf/%s' % pd.__version__.replace('.', '-'), ), serialization='pickle', ) cls.no_benchmark_expected_perf = { example_name: cls._no_benchmark_expectations_applied( expected_perf.copy() ) for example_name, expected_perf in cls.expected_perf.items() } @staticmethod def _no_benchmark_expectations_applied(expected_perf): # With no benchmark, expect zero results for these metrics: expected_perf[['alpha', 'beta']] = None for col in ['benchmark_period_return', 'benchmark_volatility']: expected_perf.loc[ ~pd.isnull(expected_perf[col]), col, ] = 0.0 return expected_perf @unittest.skipIf(platform.system() == 'Windows', "Don't run test on windows") @parameter_space( example_name=sorted(EXAMPLE_MODULES), benchmark_returns=[read_checked_in_benchmark_data(), None] ) def test_example(self, example_name, benchmark_returns): actual_perf = examples.run_example( self.EXAMPLE_MODULES, example_name, # This should match the invocation in # zipline/tests/resources/rebuild_example_data environ={ 'ZIPLINE_ROOT': self.tmpdir.getpath('example_data/root'), }, benchmark_returns=benchmark_returns, ) if benchmark_returns is not None: expected_perf = self.expected_perf[example_name] else: expected_perf = self.no_benchmark_expected_perf[example_name] # Exclude positions column as the positions do not always have the # same order columns = [column for column in examples._cols_to_check if column != 'positions'] assert_equal( actual_perf[columns], expected_perf[columns], # There is a difference in the datetime columns in pandas # 0.16 and 0.17 because in 16 they are object and in 17 they are # datetime[ns, UTC]. We will just ignore the dtypes for now. check_dtype=False, ) # Sort positions by SID before comparing assert_equal( expected_perf['positions'].apply(sorted, key=itemgetter('sid')), actual_perf['positions'].apply(sorted, key=itemgetter('sid')), )
def main(ctx, rebuild_input): """Rebuild the perf data for test_examples""" example_path = test_resource_path("example_data.tar.gz") with tmp_dir() as d: with tarfile.open(example_path) as tar: tar.extractall(d.path) # The environ here should be the same (modulo the tempdir location) # as we use in test_examples.py. environ = {"ZIPLINE_ROOT": d.getpath("example_data/root")} if rebuild_input: raise NotImplementedError( "We cannot rebuild input for Yahoo because of " "changes Yahoo made to their API, so we cannot " "use Yahoo data bundles anymore. This will be fixed in " "a future release", ) # we need to register the bundle; it is already ingested and saved in # the example_data.tar.gz file @register("test") def nop_ingest(*args, **kwargs): raise NotImplementedError("we cannot rebuild the test buindle") new_perf_path = d.getpath( "example_data/new_perf/%s" % pd.__version__.replace(".", "-"), ) c = dataframe_cache( new_perf_path, serialization="pickle:2", ) with c: for name in EXAMPLE_MODULES: c[name] = examples.run_example( EXAMPLE_MODULES, name, environ=environ, benchmark_returns=read_checked_in_benchmark_data(), ) correct_called = [False] console = None def _exit(*args, **kwargs): console.raw_input = eof def correct(): correct_called[0] = True _exit() expected_perf_path = d.getpath( "example_data/expected_perf/%s" % pd.__version__.replace(".", "-"), ) # allow users to run some analysis to make sure that the new # results check out console = InteractiveConsole( { "correct": correct, "exit": _exit, "incorrect": _exit, "new": c, "np": np, "old": dataframe_cache( expected_perf_path, serialization="pickle", ), "pd": pd, "cols_to_check": examples._cols_to_check, "changed_results": changed_results, } ) console.interact(banner) if not correct_called[0]: ctx.fail( "`correct()` was not called! This means that the new" " results will not be written", ) # move the new results to the expected path shutil.rmtree(expected_perf_path) shutil.copytree(new_perf_path, expected_perf_path) # Clear out all the temporary new perf so it doesn't get added to the # tarball. shutil.rmtree(d.getpath("example_data/new_perf/")) with tarfile.open(example_path, "w|gz") as tar: tar.add(d.getpath("example_data"), "example_data")