def __init__(self, engine): if isinstance(engine, str): engine = sa.create_engine('sqlite:///' + engine) self.engine = engine metadata = sa.MetaData(bind=engine) metadata.reflect(only=asset_db_table_names) for table_name in asset_db_table_names: setattr(self, table_name, metadata.tables[table_name]) # Check the version info of the db for compatibility check_version_info(self.version_info, ASSET_DB_VERSION) # Cache for lookup of assets by sid, the objects in the asset lookup # may be shared with the results from equity and future lookup caches. # # The top level cache exists to minimize lookups on the asset type # routing. # # The caches are read through, i.e. accessing an asset through # retrieve_asset will populate the cache on first retrieval. self._caches = (self._asset_cache, self._asset_type_cache) = {}, {} # Populated on first call to `lifetimes`. self._asset_lifetimes = None
def test_downgrade(self): # Attempt to downgrade a current assets db all the way down to v0 conn = self.engine.connect() # first downgrade to v3 downgrade(self.engine, 3) metadata = sa.MetaData(conn) metadata.reflect() check_version_info(conn, metadata.tables['version_info'], 3) self.assertFalse('exchange_full' in metadata.tables) # now go all the way to v0 downgrade(self.engine, 0) # Verify that the db version is now 0 metadata = sa.MetaData(conn) metadata.reflect() version_table = metadata.tables['version_info'] check_version_info(conn, version_table, 0) # Check some of the v1-to-v0 downgrades self.assertTrue('futures_contracts' in metadata.tables) self.assertTrue('version_info' in metadata.tables) self.assertFalse('tick_size' in metadata.tables['futures_contracts'].columns) self.assertTrue('contract_multiplier' in metadata.tables['futures_contracts'].columns)
def test_ingest_assets_versions(self): versions = (1, 2) called = [False] @self.register('bundle', create_writers=False) def bundle_ingest_no_create_writers(*args, **kwargs): called[0] = True now = pd.Timestamp.utcnow() with self.assertRaisesRegex(ValueError, "ingest .* creates writers .* downgrade"): self.ingest('bundle', self.environ, assets_versions=versions, timestamp=now - pd.Timedelta(seconds=1)) assert_false(called[0]) assert_equal(len(ingestions_for_bundle('bundle', self.environ)), 1) @self.register('bundle', create_writers=True) def bundle_ingest_create_writers(environ, asset_db_writer, minute_bar_writer, daily_bar_writer, adjustment_writer, calendar, start_session, end_session, cache, show_progress, output_dir): self.assertIsNotNone(asset_db_writer) self.assertIsNotNone(minute_bar_writer) self.assertIsNotNone(daily_bar_writer) self.assertIsNotNone(adjustment_writer) equities = make_simple_equity_info( tuple(range(3)), self.START_DATE, self.END_DATE, ) asset_db_writer.write(equities=equities) called[0] = True # Explicitly use different timestamp; otherwise, test could run so fast # that first ingestion is re-used. self.ingest('bundle', self.environ, assets_versions=versions, timestamp=now) assert_true(called[0]) ingestions = ingestions_for_bundle('bundle', self.environ) assert_equal(len(ingestions), 2) for version in sorted(set(versions) | {ASSET_DB_VERSION}): eng = sa.create_engine('sqlite:///' + asset_db_path( 'bundle', to_bundle_ingest_dirname(ingestions[0]), # most recent self.environ, version, )) metadata = sa.MetaData() metadata.reflect(eng) version_table = metadata.tables['version_info'] check_version_info(eng, version_table, version)
def test_ingest_assets_versions(self): versions = (1, 2) called = [False] @self.register("bundle", create_writers=False) def bundle_ingest_no_create_writers(*args, **kwargs): called[0] = True now = pd.Timestamp.utcnow() with self.assertRaisesRegexp(ValueError, "ingest .* creates writers .* downgrade"): self.ingest("bundle", self.environ, assets_versions=versions, timestamp=now - pd.Timedelta(seconds=1)) assert_false(called[0]) assert_equal(len(ingestions_for_bundle("bundle", self.environ)), 1) @self.register("bundle", create_writers=True) def bundle_ingest_create_writers( environ, asset_db_writer, minute_bar_writer, daily_bar_writer, adjustment_writer, calendar, start_session, end_session, cache, show_progress, output_dir, ): self.assertIsNotNone(asset_db_writer) self.assertIsNotNone(minute_bar_writer) self.assertIsNotNone(daily_bar_writer) self.assertIsNotNone(adjustment_writer) equities = make_simple_equity_info(tuple(range(3)), self.START_DATE, self.END_DATE) asset_db_writer.write(equities=equities) called[0] = True # Explicitly use different timestamp; otherwise, test could run so fast # that first ingestion is re-used. self.ingest("bundle", self.environ, assets_versions=versions, timestamp=now) assert_true(called[0]) ingestions = ingestions_for_bundle("bundle", self.environ) assert_equal(len(ingestions), 2) for version in sorted(set(versions) | {ASSET_DB_VERSION}): eng = sa.create_engine( "sqlite:///" + asset_db_path("bundle", to_bundle_ingest_dirname(ingestions[0]), self.environ, version) # most recent ) metadata = sa.MetaData() metadata.reflect(eng) version_table = metadata.tables["version_info"] check_version_info(eng, version_table, version)
def test_finder_checks_version(self): version_table = self.metadata.tables['version_info'] version_table.delete().execute() write_version_info(self.engine, version_table, -2) check_version_info(self.engine, version_table, -2) # Assert that trying to build a finder with a bad db raises an error with self.assertRaises(AssetDBVersionError): AssetFinder(engine=self.engine) # Change the version number of the db to the correct version version_table.delete().execute() write_version_info(self.engine, version_table, ASSET_DB_VERSION) check_version_info(self.engine, version_table, ASSET_DB_VERSION) # Now that the versions match, this Finder should succeed AssetFinder(engine=self.engine)
def test_finder_checks_version(self): version_table = self.metadata.tables['version_info'] version_table.delete().execute() write_version_info(version_table, -2) check_version_info(version_table, -2) # Assert that trying to build a finder with a bad db raises an error with self.assertRaises(AssetDBVersionError): AssetFinder(engine=self.engine) # Change the version number of the db to the correct version version_table.delete().execute() write_version_info(version_table, ASSET_DB_VERSION) check_version_info(version_table, ASSET_DB_VERSION) # Now that the versions match, this Finder should succeed AssetFinder(engine=self.engine)
def test_finder_checks_version(self): # Create an env and give it a bogus version number env = TradingEnvironment(load=noop_load) metadata = sa.MetaData(bind=env.engine) version_table = _version_table_schema(metadata) version_table.delete().execute() write_version_info(version_table, -2) check_version_info(version_table, -2) # Assert that trying to build a finder with a bad db raises an error with self.assertRaises(AssetDBVersionError): AssetFinder(engine=env.engine) # Change the version number of the db to the correct version version_table.delete().execute() write_version_info(version_table, ASSET_DB_VERSION) check_version_info(version_table, ASSET_DB_VERSION) # Now that the versions match, this Finder should succeed AssetFinder(engine=env.engine)
def test_write_version(self): version_table = self.metadata.tables['version_info'] version_table.delete().execute() # Assert that the version is not present in the table self.assertIsNone(sa.select((version_table.c.version,)).scalar()) # This should fail because the table has no version info and is, # therefore, consdered v0 with self.assertRaises(AssetDBVersionError): check_version_info(version_table, -2) # This should not raise an error because the version has been written write_version_info(version_table, -2) check_version_info(version_table, -2) # Assert that the version is in the table and correct self.assertEqual(sa.select((version_table.c.version,)).scalar(), -2) # Assert that trying to overwrite the version fails with self.assertRaises(sa.exc.IntegrityError): write_version_info(version_table, -3)
def test_write_version(self): version_table = self.metadata.tables['version_info'] version_table.delete().execute() # Assert that the version is not present in the table self.assertIsNone(sa.select((version_table.c.version, )).scalar()) # This should fail because the table has no version info and is, # therefore, consdered v0 with self.assertRaises(AssetDBVersionError): check_version_info(self.engine, version_table, -2) # This should not raise an error because the version has been written write_version_info(self.engine, version_table, -2) check_version_info(self.engine, version_table, -2) # Assert that the version is in the table and correct self.assertEqual(sa.select((version_table.c.version, )).scalar(), -2) # Assert that trying to overwrite the version fails with self.assertRaises(sa.exc.IntegrityError): write_version_info(self.engine, version_table, -3)
def test_write_version(self): env = TradingEnvironment(load=noop_load) metadata = sa.MetaData(bind=env.engine) version_table = _version_table_schema(metadata) version_table.delete().execute() # Assert that the version is not present in the table self.assertIsNone(sa.select((version_table.c.version,)).scalar()) # This should fail because the table has no version info and is, # therefore, consdered v0 with self.assertRaises(AssetDBVersionError): check_version_info(version_table, -2) # This should not raise an error because the version has been written write_version_info(version_table, -2) check_version_info(version_table, -2) # Assert that the version is in the table and correct self.assertEqual(sa.select((version_table.c.version,)).scalar(), -2) # Assert that trying to overwrite the version fails with self.assertRaises(sa.exc.IntegrityError): write_version_info(version_table, -3)
def test_write_version(self): env = TradingEnvironment(load=noop_load) metadata = sa.MetaData(bind=env.engine) version_table = _version_table_schema(metadata) version_table.delete().execute() # Assert that the version is not present in the table self.assertIsNone(sa.select((version_table.c.version, )).scalar()) # This should fail because the table has no version info and is, # therefore, consdered v0 with self.assertRaises(AssetDBVersionError): check_version_info(version_table, -2) # This should not raise an error because the version has been written write_version_info(version_table, -2) check_version_info(version_table, -2) # Assert that the version is in the table and correct self.assertEqual(sa.select((version_table.c.version, )).scalar(), -2) # Assert that trying to overwrite the version fails with self.assertRaises(sa.exc.IntegrityError): write_version_info(version_table, -3)
def test_check_version(self): version_table = self.metadata.tables['version_info'] # This should not raise an error check_version_info(version_table, ASSET_DB_VERSION) # This should fail because the version is too low with self.assertRaises(AssetDBVersionError): check_version_info(version_table, ASSET_DB_VERSION - 1) # This should fail because the version is too high with self.assertRaises(AssetDBVersionError): check_version_info(version_table, ASSET_DB_VERSION + 1)
def test_check_version(self): env = TradingEnvironment(load=noop_load) version_table = env.asset_finder.version_info # This should not raise an error check_version_info(version_table, ASSET_DB_VERSION) # This should fail because the version is too low with self.assertRaises(AssetDBVersionError): check_version_info(version_table, ASSET_DB_VERSION - 1) # This should fail because the version is too high with self.assertRaises(AssetDBVersionError): check_version_info(version_table, ASSET_DB_VERSION + 1)
def test_ingest_assets_versions(self): versions = (1, 2) called = [False] @self.register("bundle", create_writers=False) def bundle_ingest_no_create_writers(*args, **kwargs): called[0] = True now = pd.Timestamp.utcnow() with pytest.raises(ValueError, match="ingest .* creates writers .* downgrade"): self.ingest( "bundle", self.environ, assets_versions=versions, timestamp=now - pd.Timedelta(seconds=1), ) assert not called[0] assert len(ingestions_for_bundle("bundle", self.environ)) == 1 @self.register("bundle", create_writers=True) def bundle_ingest_create_writers( environ, asset_db_writer, minute_bar_writer, daily_bar_writer, adjustment_writer, calendar, start_session, end_session, cache, show_progress, output_dir, ): assert asset_db_writer is not None assert minute_bar_writer is not None assert daily_bar_writer is not None assert adjustment_writer is not None equities = make_simple_equity_info( tuple(range(3)), self.START_DATE, self.END_DATE, ) asset_db_writer.write(equities=equities) called[0] = True # Explicitly use different timestamp; otherwise, test could run so fast # that first ingestion is re-used. self.ingest("bundle", self.environ, assets_versions=versions, timestamp=now) assert called[0] ingestions = ingestions_for_bundle("bundle", self.environ) assert len(ingestions) == 2 for version in sorted(set(versions) | {ASSET_DB_VERSION}): eng = sa.create_engine("sqlite:///" + asset_db_path( "bundle", to_bundle_ingest_dirname(ingestions[0]), # most recent self.environ, version, )) metadata = sa.MetaData() metadata.reflect(eng) version_table = metadata.tables["version_info"] check_version_info(eng, version_table, version)