def test_ingest_assets_versions(self):
versions = (1, 2)
called = [False]
@self.register('bundle', create_writers=False)
def bundle_ingest_no_create_writers(*args, **kwargs):
called[0] = True
now = pd.Timestamp.utcnow()
with self.assertRaisesRegexp(ValueError,
"ingest .* creates writers .* downgrade"):
self.ingest('bundle',
self.environ,
assets_versions=versions,
timestamp=now - pd.Timedelta(seconds=1))
assert_false(called[0])
assert_equal(len(ingestions_for_bundle('bundle', self.environ)), 1)
@self.register('bundle', create_writers=True)
def bundle_ingest_create_writers(environ, asset_db_writer,
minute_bar_writer, daily_bar_writer,
adjustment_writer, calendar,
start_session, end_session, cache,
show_progress, output_dir):
self.assertIsNotNone(asset_db_writer)
self.assertIsNotNone(minute_bar_writer)
self.assertIsNotNone(daily_bar_writer)
self.assertIsNotNone(adjustment_writer)
equities = make_simple_equity_info(
tuple(range(3)),
self.START_DATE,
self.END_DATE,
)
asset_db_writer.write(equities=equities)
called[0] = True
# Explicitly use different timestamp; otherwise, test could run so fast
# that first ingestion is re-used.
self.ingest('bundle',
self.environ,
assets_versions=versions,
timestamp=now)
assert_true(called[0])
ingestions = ingestions_for_bundle('bundle', self.environ)
assert_equal(len(ingestions), 2)
for version in sorted(set(versions) | {ASSET_DB_VERSION}):
eng = sa.create_engine('sqlite:///' + asset_db_path(
'bundle',
to_bundle_ingest_dirname(ingestions[0]), # most recent
self.environ,
version,
))
metadata = sa.MetaData()
metadata.reflect(eng)
version_table = metadata.tables['version_info']
check_version_info(eng, version_table, version)
def bundles():
"""List all of the available data bundles.
"""
for bundle in sorted(bundles_module.bundles.keys()):
if bundle.startswith('.'):
# hide the test data
continue
try:
ingestions = list(
map(text_type, bundles_module.ingestions_for_bundle(bundle)))
except OSError as e:
if e.errno != errno.ENOENT:
raise
ingestions = []
# If we got no ingestions, either because the directory didn't exist or
# because there were no entries, print a single message indicating that
# no ingestions have yet been made.
for timestamp in ingestions or ["<no ingestions>"]:
click.echo("%s %s" % (bundle, timestamp))
def bundles():
"""List all of the available data bundles.
"""
for bundle in sorted(bundles_module.bundles.keys()):
if bundle.startswith('.'):
# hide the test data
continue
try:
ingestions = list(
map(text_type, bundles_module.ingestions_for_bundle(bundle))
)
except OSError as e:
if e.errno != errno.ENOENT:
raise
ingestions = []
# If we got no ingestions, either because the directory didn't exist or
# because there were no entries, print a single message indicating that
# no ingestions have yet been made.
for timestamp in ingestions or ["<no ingestions>"]:
click.echo("%s %s" % (bundle, timestamp), sys.stdout)
def test_ingest_assets_versions(self):
versions = (1, 2)
called = [False]
@self.register('bundle', create_writers=False)
def bundle_ingest_no_create_writers(*args, **kwargs):
called[0] = True
now = pd.Timestamp.utcnow()
with self.assertRaisesRegexp(ValueError,
"ingest .* creates writers .* downgrade"):
self.ingest('bundle', self.environ, assets_versions=versions,
timestamp=now - pd.Timedelta(seconds=1))
assert_false(called[0])
assert_equal(len(ingestions_for_bundle('bundle', self.environ)), 1)
@self.register('bundle', create_writers=True)
def bundle_ingest_create_writers(
environ,
asset_db_writer,
minute_bar_writer,
daily_bar_writer,
adjustment_writer,
calendar,
start_session,
end_session,
cache,
show_progress,
output_dir):
self.assertIsNotNone(asset_db_writer)
self.assertIsNotNone(minute_bar_writer)
self.assertIsNotNone(daily_bar_writer)
self.assertIsNotNone(adjustment_writer)
equities = make_simple_equity_info(
tuple(range(3)),
self.START_DATE,
self.END_DATE,
)
asset_db_writer.write(equities=equities)
called[0] = True
# Explicitly use different timestamp; otherwise, test could run so fast
# that first ingestion is re-used.
self.ingest('bundle', self.environ, assets_versions=versions,
timestamp=now)
assert_true(called[0])
ingestions = ingestions_for_bundle('bundle', self.environ)
assert_equal(len(ingestions), 2)
for version in sorted(set(versions) | {ASSET_DB_VERSION}):
eng = sa.create_engine(
'sqlite:///' +
asset_db_path(
'bundle',
to_bundle_ingest_dirname(ingestions[0]), # most recent
self.environ,
version,
)
)
metadata = sa.MetaData()
metadata.reflect(eng)
version_table = metadata.tables['version_info']
check_version_info(eng, version_table, version)
