def __init__(self, engine):
if isinstance(engine, str):
engine = sa.create_engine('sqlite:///' + engine)
self.engine = engine
metadata = sa.MetaData(bind=engine)
metadata.reflect(only=asset_db_table_names)
for table_name in asset_db_table_names:
setattr(self, table_name, metadata.tables[table_name])
# Check the version info of the db for compatibility
check_version_info(self.version_info, ASSET_DB_VERSION)
# Cache for lookup of assets by sid, the objects in the asset lookup
# may be shared with the results from equity and future lookup caches.
#
# The top level cache exists to minimize lookups on the asset type
# routing.
#
# The caches are read through, i.e. accessing an asset through
# retrieve_asset will populate the cache on first retrieval.
self._caches = (self._asset_cache, self._asset_type_cache) = {}, {}
# Populated on first call to `lifetimes`.
self._asset_lifetimes = None
def test_downgrade(self):
# Attempt to downgrade a current assets db all the way down to v0
conn = self.engine.connect()
# first downgrade to v3
downgrade(self.engine, 3)
metadata = sa.MetaData(conn)
metadata.reflect()
check_version_info(conn, metadata.tables['version_info'], 3)
self.assertFalse('exchange_full' in metadata.tables)
# now go all the way to v0
downgrade(self.engine, 0)
# Verify that the db version is now 0
metadata = sa.MetaData(conn)
metadata.reflect()
version_table = metadata.tables['version_info']
check_version_info(conn, version_table, 0)
# Check some of the v1-to-v0 downgrades
self.assertTrue('futures_contracts' in metadata.tables)
self.assertTrue('version_info' in metadata.tables)
self.assertFalse('tick_size' in
metadata.tables['futures_contracts'].columns)
self.assertTrue('contract_multiplier' in
metadata.tables['futures_contracts'].columns)
def test_ingest_assets_versions(self):
versions = (1, 2)
called = [False]
@self.register('bundle', create_writers=False)
def bundle_ingest_no_create_writers(*args, **kwargs):
called[0] = True
now = pd.Timestamp.utcnow()
with self.assertRaisesRegex(ValueError,
"ingest .* creates writers .* downgrade"):
self.ingest('bundle',
self.environ,
assets_versions=versions,
timestamp=now - pd.Timedelta(seconds=1))
assert_false(called[0])
assert_equal(len(ingestions_for_bundle('bundle', self.environ)), 1)
@self.register('bundle', create_writers=True)
def bundle_ingest_create_writers(environ, asset_db_writer,
minute_bar_writer, daily_bar_writer,
adjustment_writer, calendar,
start_session, end_session, cache,
show_progress, output_dir):
self.assertIsNotNone(asset_db_writer)
self.assertIsNotNone(minute_bar_writer)
self.assertIsNotNone(daily_bar_writer)
self.assertIsNotNone(adjustment_writer)
equities = make_simple_equity_info(
tuple(range(3)),
self.START_DATE,
self.END_DATE,
)
asset_db_writer.write(equities=equities)
called[0] = True
# Explicitly use different timestamp; otherwise, test could run so fast
# that first ingestion is re-used.
self.ingest('bundle',
self.environ,
assets_versions=versions,
timestamp=now)
assert_true(called[0])
ingestions = ingestions_for_bundle('bundle', self.environ)
assert_equal(len(ingestions), 2)
for version in sorted(set(versions) | {ASSET_DB_VERSION}):
eng = sa.create_engine('sqlite:///' + asset_db_path(
'bundle',
to_bundle_ingest_dirname(ingestions[0]), # most recent
self.environ,
version,
))
metadata = sa.MetaData()
metadata.reflect(eng)
version_table = metadata.tables['version_info']
check_version_info(eng, version_table, version)
def test_ingest_assets_versions(self):
versions = (1, 2)
called = [False]
@self.register("bundle", create_writers=False)
def bundle_ingest_no_create_writers(*args, **kwargs):
called[0] = True
now = pd.Timestamp.utcnow()
with self.assertRaisesRegexp(ValueError, "ingest .* creates writers .* downgrade"):
self.ingest("bundle", self.environ, assets_versions=versions, timestamp=now - pd.Timedelta(seconds=1))
assert_false(called[0])
assert_equal(len(ingestions_for_bundle("bundle", self.environ)), 1)
@self.register("bundle", create_writers=True)
def bundle_ingest_create_writers(
environ,
asset_db_writer,
minute_bar_writer,
daily_bar_writer,
adjustment_writer,
calendar,
start_session,
end_session,
cache,
show_progress,
output_dir,
):
self.assertIsNotNone(asset_db_writer)
self.assertIsNotNone(minute_bar_writer)
self.assertIsNotNone(daily_bar_writer)
self.assertIsNotNone(adjustment_writer)
equities = make_simple_equity_info(tuple(range(3)), self.START_DATE, self.END_DATE)
asset_db_writer.write(equities=equities)
called[0] = True
# Explicitly use different timestamp; otherwise, test could run so fast
# that first ingestion is re-used.
self.ingest("bundle", self.environ, assets_versions=versions, timestamp=now)
assert_true(called[0])
ingestions = ingestions_for_bundle("bundle", self.environ)
assert_equal(len(ingestions), 2)
for version in sorted(set(versions) | {ASSET_DB_VERSION}):
eng = sa.create_engine(
"sqlite:///"
+ asset_db_path("bundle", to_bundle_ingest_dirname(ingestions[0]), self.environ, version) # most recent
)
metadata = sa.MetaData()
metadata.reflect(eng)
version_table = metadata.tables["version_info"]
check_version_info(eng, version_table, version)
def test_finder_checks_version(self):
version_table = self.metadata.tables['version_info']
version_table.delete().execute()
write_version_info(self.engine, version_table, -2)
check_version_info(self.engine, version_table, -2)
# Assert that trying to build a finder with a bad db raises an error
with self.assertRaises(AssetDBVersionError):
AssetFinder(engine=self.engine)
# Change the version number of the db to the correct version
version_table.delete().execute()
write_version_info(self.engine, version_table, ASSET_DB_VERSION)
check_version_info(self.engine, version_table, ASSET_DB_VERSION)
# Now that the versions match, this Finder should succeed
AssetFinder(engine=self.engine)
def test_finder_checks_version(self):
version_table = self.metadata.tables['version_info']
version_table.delete().execute()
write_version_info(version_table, -2)
check_version_info(version_table, -2)
# Assert that trying to build a finder with a bad db raises an error
with self.assertRaises(AssetDBVersionError):
AssetFinder(engine=self.engine)
# Change the version number of the db to the correct version
version_table.delete().execute()
write_version_info(version_table, ASSET_DB_VERSION)
check_version_info(version_table, ASSET_DB_VERSION)
# Now that the versions match, this Finder should succeed
AssetFinder(engine=self.engine)
def test_finder_checks_version(self):
# Create an env and give it a bogus version number
env = TradingEnvironment(load=noop_load)
metadata = sa.MetaData(bind=env.engine)
version_table = _version_table_schema(metadata)
version_table.delete().execute()
write_version_info(version_table, -2)
check_version_info(version_table, -2)
# Assert that trying to build a finder with a bad db raises an error
with self.assertRaises(AssetDBVersionError):
AssetFinder(engine=env.engine)
# Change the version number of the db to the correct version
version_table.delete().execute()
write_version_info(version_table, ASSET_DB_VERSION)
check_version_info(version_table, ASSET_DB_VERSION)
# Now that the versions match, this Finder should succeed
AssetFinder(engine=env.engine)
def test_finder_checks_version(self):
# Create an env and give it a bogus version number
env = TradingEnvironment(load=noop_load)
metadata = sa.MetaData(bind=env.engine)
version_table = _version_table_schema(metadata)
version_table.delete().execute()
write_version_info(version_table, -2)
check_version_info(version_table, -2)
# Assert that trying to build a finder with a bad db raises an error
with self.assertRaises(AssetDBVersionError):
AssetFinder(engine=env.engine)
# Change the version number of the db to the correct version
version_table.delete().execute()
write_version_info(version_table, ASSET_DB_VERSION)
check_version_info(version_table, ASSET_DB_VERSION)
# Now that the versions match, this Finder should succeed
AssetFinder(engine=env.engine)
def test_write_version(self):
version_table = self.metadata.tables['version_info']
version_table.delete().execute()
# Assert that the version is not present in the table
self.assertIsNone(sa.select((version_table.c.version,)).scalar())
# This should fail because the table has no version info and is,
# therefore, consdered v0
with self.assertRaises(AssetDBVersionError):
check_version_info(version_table, -2)
# This should not raise an error because the version has been written
write_version_info(version_table, -2)
check_version_info(version_table, -2)
# Assert that the version is in the table and correct
self.assertEqual(sa.select((version_table.c.version,)).scalar(), -2)
# Assert that trying to overwrite the version fails
with self.assertRaises(sa.exc.IntegrityError):
write_version_info(version_table, -3)
def test_write_version(self):
version_table = self.metadata.tables['version_info']
version_table.delete().execute()
# Assert that the version is not present in the table
self.assertIsNone(sa.select((version_table.c.version, )).scalar())
# This should fail because the table has no version info and is,
# therefore, consdered v0
with self.assertRaises(AssetDBVersionError):
check_version_info(self.engine, version_table, -2)
# This should not raise an error because the version has been written
write_version_info(self.engine, version_table, -2)
check_version_info(self.engine, version_table, -2)
# Assert that the version is in the table and correct
self.assertEqual(sa.select((version_table.c.version, )).scalar(), -2)
# Assert that trying to overwrite the version fails
with self.assertRaises(sa.exc.IntegrityError):
write_version_info(self.engine, version_table, -3)
def test_write_version(self):
env = TradingEnvironment(load=noop_load)
metadata = sa.MetaData(bind=env.engine)
version_table = _version_table_schema(metadata)
version_table.delete().execute()
# Assert that the version is not present in the table
self.assertIsNone(sa.select((version_table.c.version,)).scalar())
# This should fail because the table has no version info and is,
# therefore, consdered v0
with self.assertRaises(AssetDBVersionError):
check_version_info(version_table, -2)
# This should not raise an error because the version has been written
write_version_info(version_table, -2)
check_version_info(version_table, -2)
# Assert that the version is in the table and correct
self.assertEqual(sa.select((version_table.c.version,)).scalar(), -2)
# Assert that trying to overwrite the version fails
with self.assertRaises(sa.exc.IntegrityError):
write_version_info(version_table, -3)
def test_write_version(self):
env = TradingEnvironment(load=noop_load)
metadata = sa.MetaData(bind=env.engine)
version_table = _version_table_schema(metadata)
version_table.delete().execute()
# Assert that the version is not present in the table
self.assertIsNone(sa.select((version_table.c.version, )).scalar())
# This should fail because the table has no version info and is,
# therefore, consdered v0
with self.assertRaises(AssetDBVersionError):
check_version_info(version_table, -2)
# This should not raise an error because the version has been written
write_version_info(version_table, -2)
check_version_info(version_table, -2)
# Assert that the version is in the table and correct
self.assertEqual(sa.select((version_table.c.version, )).scalar(), -2)
# Assert that trying to overwrite the version fails
with self.assertRaises(sa.exc.IntegrityError):
write_version_info(version_table, -3)
def test_check_version(self):
version_table = self.metadata.tables['version_info']
# This should not raise an error
check_version_info(version_table, ASSET_DB_VERSION)
# This should fail because the version is too low
with self.assertRaises(AssetDBVersionError):
check_version_info(version_table, ASSET_DB_VERSION - 1)
# This should fail because the version is too high
with self.assertRaises(AssetDBVersionError):
check_version_info(version_table, ASSET_DB_VERSION + 1)
def test_check_version(self):
version_table = self.metadata.tables['version_info']
# This should not raise an error
check_version_info(version_table, ASSET_DB_VERSION)
# This should fail because the version is too low
with self.assertRaises(AssetDBVersionError):
check_version_info(version_table, ASSET_DB_VERSION - 1)
# This should fail because the version is too high
with self.assertRaises(AssetDBVersionError):
check_version_info(version_table, ASSET_DB_VERSION + 1)
def test_check_version(self):
env = TradingEnvironment(load=noop_load)
version_table = env.asset_finder.version_info
# This should not raise an error
check_version_info(version_table, ASSET_DB_VERSION)
# This should fail because the version is too low
with self.assertRaises(AssetDBVersionError):
check_version_info(version_table, ASSET_DB_VERSION - 1)
# This should fail because the version is too high
with self.assertRaises(AssetDBVersionError):
check_version_info(version_table, ASSET_DB_VERSION + 1)
def test_check_version(self):
env = TradingEnvironment(load=noop_load)
version_table = env.asset_finder.version_info
# This should not raise an error
check_version_info(version_table, ASSET_DB_VERSION)
# This should fail because the version is too low
with self.assertRaises(AssetDBVersionError):
check_version_info(version_table, ASSET_DB_VERSION - 1)
# This should fail because the version is too high
with self.assertRaises(AssetDBVersionError):
check_version_info(version_table, ASSET_DB_VERSION + 1)
def test_ingest_assets_versions(self):
versions = (1, 2)
called = [False]
@self.register("bundle", create_writers=False)
def bundle_ingest_no_create_writers(*args, **kwargs):
called[0] = True
now = pd.Timestamp.utcnow()
with pytest.raises(ValueError,
match="ingest .* creates writers .* downgrade"):
self.ingest(
"bundle",
self.environ,
assets_versions=versions,
timestamp=now - pd.Timedelta(seconds=1),
)
assert not called[0]
assert len(ingestions_for_bundle("bundle", self.environ)) == 1
@self.register("bundle", create_writers=True)
def bundle_ingest_create_writers(
environ,
asset_db_writer,
minute_bar_writer,
daily_bar_writer,
adjustment_writer,
calendar,
start_session,
end_session,
cache,
show_progress,
output_dir,
):
assert asset_db_writer is not None
assert minute_bar_writer is not None
assert daily_bar_writer is not None
assert adjustment_writer is not None
equities = make_simple_equity_info(
tuple(range(3)),
self.START_DATE,
self.END_DATE,
)
asset_db_writer.write(equities=equities)
called[0] = True
# Explicitly use different timestamp; otherwise, test could run so fast
# that first ingestion is re-used.
self.ingest("bundle",
self.environ,
assets_versions=versions,
timestamp=now)
assert called[0]
ingestions = ingestions_for_bundle("bundle", self.environ)
assert len(ingestions) == 2
for version in sorted(set(versions) | {ASSET_DB_VERSION}):
eng = sa.create_engine("sqlite:///" + asset_db_path(
"bundle",
to_bundle_ingest_dirname(ingestions[0]), # most recent
self.environ,
version,
))
metadata = sa.MetaData()
metadata.reflect(eng)
version_table = metadata.tables["version_info"]
check_version_info(eng, version_table, version)
