def test_event_log_step_key_migration():
src_dir = file_relative_path(__file__, "snapshot_0_7_6_pre_event_log_migration/sqlite")
with copy_directory(src_dir) as test_dir:
instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir))
# Make sure the schema is migrated
instance.upgrade()
runs = instance.get_runs()
assert len(runs) == 1
run_ids = instance._event_storage.get_all_run_ids()
assert run_ids == ["6405c4a0-3ccc-4600-af81-b5ee197f8528"]
assert isinstance(instance._event_storage, SqlEventLogStorage)
events_by_id = instance._event_storage.get_logs_for_run_by_log_id(
"6405c4a0-3ccc-4600-af81-b5ee197f8528"
)
assert len(events_by_id) == 40
step_key_records = []
for record_id, _event in events_by_id.items():
row_data = instance._event_storage.get_event_log_table_data(
"6405c4a0-3ccc-4600-af81-b5ee197f8528", record_id
)
if row_data.step_key is not None:
step_key_records.append(row_data)
assert len(step_key_records) == 0
# run the event_log backfill migration
migrate_event_log_data(instance=instance)
step_key_records = []
for record_id, _event in events_by_id.items():
row_data = instance._event_storage.get_event_log_table_data(
"6405c4a0-3ccc-4600-af81-b5ee197f8528", record_id
)
if row_data.step_key is not None:
step_key_records.append(row_data)
assert len(step_key_records) > 0
def test_0_7_6_postgres_pre_event_log_migration(hostname, conn_string):
engine = create_engine(conn_string)
engine.execute("drop schema public cascade;")
engine.execute("create schema public;")
env = os.environ.copy()
env["PGPASSWORD"] = "******"
subprocess.check_call(
[
"psql",
"-h",
hostname,
"-p",
"5432",
"-U",
"test",
"-f",
file_relative_path(
__file__,
"snapshot_0_7_6_pre_event_log_migration/postgres/pg_dump.txt"),
],
env=env,
)
run_id = "ca7f1e33-526d-4f75-9bc5-3e98da41ab97"
with seven.TemporaryDirectory() as tempdir:
with open(file_relative_path(__file__, "dagster.yaml"),
"r") as template_fd:
with open(os.path.join(tempdir, "dagster.yaml"), "w") as target_fd:
template = template_fd.read().format(hostname=hostname)
target_fd.write(template)
instance = DagsterInstance.from_config(tempdir)
# Runs will appear in DB, but event logs need migration
runs = instance.get_runs()
assert len(runs) == 1
assert instance.get_run_by_id(run_id)
# Make sure the schema is migrated
instance.upgrade()
assert isinstance(instance._event_storage, SqlEventLogStorage)
events_by_id = instance._event_storage.get_logs_for_run_by_log_id(
run_id)
assert len(events_by_id) == 40
step_key_records = []
for record_id, _event in events_by_id.items():
row_data = instance._event_storage.get_event_log_table_data(
run_id, record_id)
if row_data.step_key is not None:
step_key_records.append(row_data)
assert len(step_key_records) == 0
# run the event_log data migration
migrate_event_log_data(instance=instance)
step_key_records = []
for record_id, _event in events_by_id.items():
row_data = instance._event_storage.get_event_log_table_data(
run_id, record_id)
if row_data.step_key is not None:
step_key_records.append(row_data)
assert len(step_key_records) > 0
