def test_asset_secondary_index(asset_aware_context):
with asset_aware_context() as ctx:
instance, event_log_storage = ctx
execute_pipeline(pipeline_one, instance=instance)
asset_keys = event_log_storage.get_all_asset_keys()
assert len(asset_keys) == 1
migrate_asset_key_data(event_log_storage)
two = execute_pipeline(pipeline_two, instance=instance)
two_two = execute_pipeline(pipeline_two, instance=instance)
asset_keys = event_log_storage.get_all_asset_keys()
assert len(asset_keys) == 3
event_log_storage.delete_events(two.run_id)
asset_keys = event_log_storage.get_all_asset_keys()
assert len(asset_keys) == 3
event_log_storage.delete_events(two_two.run_id)
asset_keys = event_log_storage.get_all_asset_keys()
assert len(asset_keys) == 1
def test_secondary_index_asset_keys(conn_string):
event_log_storage = PostgresEventLogStorage.create_clean_storage(
conn_string)
asset_key_one = AssetKey(["one"])
asset_key_two = AssetKey(["two"])
@solid
def materialize_one(_):
yield AssetMaterialization(asset_key=asset_key_one)
yield Output(1)
@solid
def materialize_two(_):
yield AssetMaterialization(asset_key=asset_key_two)
yield Output(1)
def _one():
materialize_one()
def _two():
materialize_two()
events_one, _ = synthesize_events(_one)
for event in events_one:
event_log_storage.store_event(event)
asset_keys = event_log_storage.get_all_asset_keys()
assert len(asset_keys) == 1
assert asset_key_one in set(asset_keys)
migrate_asset_key_data(event_log_storage)
asset_keys = event_log_storage.get_all_asset_keys()
assert len(asset_keys) == 1
assert asset_key_one in set(asset_keys)
events_two, _ = synthesize_events(_two)
for event in events_two:
event_log_storage.store_event(event)
asset_keys = event_log_storage.get_all_asset_keys()
assert len(asset_keys) == 2
assert asset_key_one in set(asset_keys)
assert asset_key_two in set(asset_keys)
def test_secondary_index_asset_keys(self, storage):
asset_key_one = AssetKey(["one"])
asset_key_two = AssetKey(["two"])
@solid
def materialize_one(_):
yield AssetMaterialization(asset_key=asset_key_one)
yield Output(1)
@solid
def materialize_two(_):
yield AssetMaterialization(asset_key=asset_key_two)
yield Output(1)
def _one():
materialize_one()
def _two():
materialize_two()
events_one, _ = _synthesize_events(_one)
for event in events_one:
storage.store_event(event)
asset_keys = storage.all_asset_keys()
assert len(asset_keys) == 1
assert asset_key_one in set(asset_keys)
migrate_asset_key_data(storage)
asset_keys = storage.all_asset_keys()
assert len(asset_keys) == 1
assert asset_key_one in set(asset_keys)
events_two, _ = _synthesize_events(_two)
for event in events_two:
storage.store_event(event)
asset_keys = storage.all_asset_keys()
assert len(asset_keys) == 2
assert asset_key_one in set(asset_keys)
assert asset_key_two in set(asset_keys)
