def test_ingest_assets_versions(self): versions = (1, 2) called = [False] @self.register('bundle', create_writers=False) def bundle_ingest_no_create_writers(*args, **kwargs): called[0] = True now = pd.Timestamp.utcnow() with self.assertRaisesRegex(ValueError, "ingest .* creates writers .* downgrade"): self.ingest('bundle', self.environ, assets_versions=versions, timestamp=now - pd.Timedelta(seconds=1)) assert_false(called[0]) assert_equal(len(ingestions_for_bundle('bundle', self.environ)), 1) @self.register('bundle', create_writers=True) def bundle_ingest_create_writers(environ, asset_db_writer, minute_bar_writer, daily_bar_writer, adjustment_writer, calendar, start_session, end_session, cache, show_progress, output_dir): self.assertIsNotNone(asset_db_writer) self.assertIsNotNone(minute_bar_writer) self.assertIsNotNone(daily_bar_writer) self.assertIsNotNone(adjustment_writer) equities = make_simple_equity_info( tuple(range(3)), self.START_DATE, self.END_DATE, ) asset_db_writer.write(equities=equities) called[0] = True # Explicitly use different timestamp; otherwise, test could run so fast # that first ingestion is re-used. self.ingest('bundle', self.environ, assets_versions=versions, timestamp=now) assert_true(called[0]) ingestions = ingestions_for_bundle('bundle', self.environ) assert_equal(len(ingestions), 2) for version in sorted(set(versions) | {ASSET_DB_VERSION}): eng = sa.create_engine('sqlite:///' + asset_db_path( 'bundle', to_bundle_ingest_dirname(ingestions[0]), # most recent self.environ, version, )) metadata = sa.MetaData() metadata.reflect(eng) version_table = metadata.tables['version_info'] check_version_info(eng, version_table, version)
def test_ingest_assets_versions(self): versions = (1, 2) called = [False] @self.register("bundle", create_writers=False) def bundle_ingest_no_create_writers(*args, **kwargs): called[0] = True now = pd.Timestamp.utcnow() with self.assertRaisesRegexp(ValueError, "ingest .* creates writers .* downgrade"): self.ingest("bundle", self.environ, assets_versions=versions, timestamp=now - pd.Timedelta(seconds=1)) assert_false(called[0]) assert_equal(len(ingestions_for_bundle("bundle", self.environ)), 1) @self.register("bundle", create_writers=True) def bundle_ingest_create_writers( environ, asset_db_writer, minute_bar_writer, daily_bar_writer, adjustment_writer, calendar, start_session, end_session, cache, show_progress, output_dir, ): self.assertIsNotNone(asset_db_writer) self.assertIsNotNone(minute_bar_writer) self.assertIsNotNone(daily_bar_writer) self.assertIsNotNone(adjustment_writer) equities = make_simple_equity_info(tuple(range(3)), self.START_DATE, self.END_DATE) asset_db_writer.write(equities=equities) called[0] = True # Explicitly use different timestamp; otherwise, test could run so fast # that first ingestion is re-used. self.ingest("bundle", self.environ, assets_versions=versions, timestamp=now) assert_true(called[0]) ingestions = ingestions_for_bundle("bundle", self.environ) assert_equal(len(ingestions), 2) for version in sorted(set(versions) | {ASSET_DB_VERSION}): eng = sa.create_engine( "sqlite:///" + asset_db_path("bundle", to_bundle_ingest_dirname(ingestions[0]), self.environ, version) # most recent ) metadata = sa.MetaData() metadata.reflect(eng) version_table = metadata.tables["version_info"] check_version_info(eng, version_table, version)
def load_sharadar_bundle(name=SHARADAR_BUNDLE_NAME, timestr=SHARADAR_BUNDLE_DIR, environ=os.environ): return BundleData( asset_finder=SQLiteAssetFinder( asset_db_path(name, timestr, environ=environ), ), equity_minute_bar_reader=None, equity_daily_bar_reader=SQLiteDailyBarReader( daily_equity_path(name, timestr, environ=environ), ), adjustment_reader=SQLiteAdjustmentReader( adjustment_db_path(name, timestr, environ=environ), ), )
def _asset_finder(name=SHARADAR_BUNDLE_NAME, timestr=SHARADAR_BUNDLE_DIR, environ=os.environ): return SQLiteAssetFinder(asset_db_path(name, timestr, environ=environ))
def test_ingest_assets_versions(self): versions = (1, 2) called = [False] @self.register("bundle", create_writers=False) def bundle_ingest_no_create_writers(*args, **kwargs): called[0] = True now = pd.Timestamp.utcnow() with pytest.raises(ValueError, match="ingest .* creates writers .* downgrade"): self.ingest( "bundle", self.environ, assets_versions=versions, timestamp=now - pd.Timedelta(seconds=1), ) assert not called[0] assert len(ingestions_for_bundle("bundle", self.environ)) == 1 @self.register("bundle", create_writers=True) def bundle_ingest_create_writers( environ, asset_db_writer, minute_bar_writer, daily_bar_writer, adjustment_writer, calendar, start_session, end_session, cache, show_progress, output_dir, ): assert asset_db_writer is not None assert minute_bar_writer is not None assert daily_bar_writer is not None assert adjustment_writer is not None equities = make_simple_equity_info( tuple(range(3)), self.START_DATE, self.END_DATE, ) asset_db_writer.write(equities=equities) called[0] = True # Explicitly use different timestamp; otherwise, test could run so fast # that first ingestion is re-used. self.ingest("bundle", self.environ, assets_versions=versions, timestamp=now) assert called[0] ingestions = ingestions_for_bundle("bundle", self.environ) assert len(ingestions) == 2 for version in sorted(set(versions) | {ASSET_DB_VERSION}): eng = sa.create_engine("sqlite:///" + asset_db_path( "bundle", to_bundle_ingest_dirname(ingestions[0]), # most recent self.environ, version, )) metadata = sa.MetaData() metadata.reflect(eng) version_table = metadata.tables["version_info"] check_version_info(eng, version_table, version)