def test_ingest_assets_versions(self):
versions = (1, 2)
called = [False]
@self.register('bundle', create_writers=False)
def bundle_ingest_no_create_writers(*args, **kwargs):
called[0] = True
now = pd.Timestamp.utcnow()
with self.assertRaisesRegex(ValueError,
"ingest .* creates writers .* downgrade"):
self.ingest('bundle',
self.environ,
assets_versions=versions,
timestamp=now - pd.Timedelta(seconds=1))
assert_false(called[0])
assert_equal(len(ingestions_for_bundle('bundle', self.environ)), 1)
@self.register('bundle', create_writers=True)
def bundle_ingest_create_writers(environ, asset_db_writer,
minute_bar_writer, daily_bar_writer,
adjustment_writer, calendar,
start_session, end_session, cache,
show_progress, output_dir):
self.assertIsNotNone(asset_db_writer)
self.assertIsNotNone(minute_bar_writer)
self.assertIsNotNone(daily_bar_writer)
self.assertIsNotNone(adjustment_writer)
equities = make_simple_equity_info(
tuple(range(3)),
self.START_DATE,
self.END_DATE,
)
asset_db_writer.write(equities=equities)
called[0] = True
# Explicitly use different timestamp; otherwise, test could run so fast
# that first ingestion is re-used.
self.ingest('bundle',
self.environ,
assets_versions=versions,
timestamp=now)
assert_true(called[0])
ingestions = ingestions_for_bundle('bundle', self.environ)
assert_equal(len(ingestions), 2)
for version in sorted(set(versions) | {ASSET_DB_VERSION}):
eng = sa.create_engine('sqlite:///' + asset_db_path(
'bundle',
to_bundle_ingest_dirname(ingestions[0]), # most recent
self.environ,
version,
))
metadata = sa.MetaData()
metadata.reflect(eng)
version_table = metadata.tables['version_info']
check_version_info(eng, version_table, version)
def test_ingest_assets_versions(self):
versions = (1, 2)
called = [False]
@self.register("bundle", create_writers=False)
def bundle_ingest_no_create_writers(*args, **kwargs):
called[0] = True
now = pd.Timestamp.utcnow()
with self.assertRaisesRegexp(ValueError, "ingest .* creates writers .* downgrade"):
self.ingest("bundle", self.environ, assets_versions=versions, timestamp=now - pd.Timedelta(seconds=1))
assert_false(called[0])
assert_equal(len(ingestions_for_bundle("bundle", self.environ)), 1)
@self.register("bundle", create_writers=True)
def bundle_ingest_create_writers(
environ,
asset_db_writer,
minute_bar_writer,
daily_bar_writer,
adjustment_writer,
calendar,
start_session,
end_session,
cache,
show_progress,
output_dir,
):
self.assertIsNotNone(asset_db_writer)
self.assertIsNotNone(minute_bar_writer)
self.assertIsNotNone(daily_bar_writer)
self.assertIsNotNone(adjustment_writer)
equities = make_simple_equity_info(tuple(range(3)), self.START_DATE, self.END_DATE)
asset_db_writer.write(equities=equities)
called[0] = True
# Explicitly use different timestamp; otherwise, test could run so fast
# that first ingestion is re-used.
self.ingest("bundle", self.environ, assets_versions=versions, timestamp=now)
assert_true(called[0])
ingestions = ingestions_for_bundle("bundle", self.environ)
assert_equal(len(ingestions), 2)
for version in sorted(set(versions) | {ASSET_DB_VERSION}):
eng = sa.create_engine(
"sqlite:///"
+ asset_db_path("bundle", to_bundle_ingest_dirname(ingestions[0]), self.environ, version) # most recent
)
metadata = sa.MetaData()
metadata.reflect(eng)
version_table = metadata.tables["version_info"]
check_version_info(eng, version_table, version)
def bundles():
"""List all of the available data bundles."""
for bundle in sorted(bundles_module.bundles.keys()):
if bundle.startswith("."):
# hide the test data
continue
try:
ingestions = list(map(str, bundles_module.ingestions_for_bundle(bundle)))
except OSError as e:
if e.errno != errno.ENOENT:
raise
ingestions = []
# If we got no ingestions, either because the directory didn't exist or
# because there were no entries, print a single message indicating that
# no ingestions have yet been made.
for timestamp in ingestions or ["<no ingestions>"]:
click.echo("%s %s" % (bundle, timestamp))
def bundles():
"""List all of the available data bundles.
"""
for bundle in sorted(bundles_module.bundles.keys()):
if bundle.startswith('.'):
# hide the test data
continue
try:
ingestions = list(
map(text_type, bundles_module.ingestions_for_bundle(bundle))
)
except OSError as e:
if e.errno != errno.ENOENT:
raise
ingestions = []
# If we got no ingestions, either because the directory didn't exist or
# because there were no entries, print a single message indicating that
# no ingestions have yet been made.
for timestamp in ingestions or ["<no ingestions>"]:
click.echo("%s %s" % (bundle, timestamp))
def test_ingest_assets_versions(self):
versions = (1, 2)
called = [False]
@self.register("bundle", create_writers=False)
def bundle_ingest_no_create_writers(*args, **kwargs):
called[0] = True
now = pd.Timestamp.utcnow()
with pytest.raises(ValueError,
match="ingest .* creates writers .* downgrade"):
self.ingest(
"bundle",
self.environ,
assets_versions=versions,
timestamp=now - pd.Timedelta(seconds=1),
)
assert not called[0]
assert len(ingestions_for_bundle("bundle", self.environ)) == 1
@self.register("bundle", create_writers=True)
def bundle_ingest_create_writers(
environ,
asset_db_writer,
minute_bar_writer,
daily_bar_writer,
adjustment_writer,
calendar,
start_session,
end_session,
cache,
show_progress,
output_dir,
):
assert asset_db_writer is not None
assert minute_bar_writer is not None
assert daily_bar_writer is not None
assert adjustment_writer is not None
equities = make_simple_equity_info(
tuple(range(3)),
self.START_DATE,
self.END_DATE,
)
asset_db_writer.write(equities=equities)
called[0] = True
# Explicitly use different timestamp; otherwise, test could run so fast
# that first ingestion is re-used.
self.ingest("bundle",
self.environ,
assets_versions=versions,
timestamp=now)
assert called[0]
ingestions = ingestions_for_bundle("bundle", self.environ)
assert len(ingestions) == 2
for version in sorted(set(versions) | {ASSET_DB_VERSION}):
eng = sa.create_engine("sqlite:///" + asset_db_path(
"bundle",
to_bundle_ingest_dirname(ingestions[0]), # most recent
self.environ,
version,
))
metadata = sa.MetaData()
metadata.reflect(eng)
version_table = metadata.tables["version_info"]
check_version_info(eng, version_table, version)