def test_event_log_step_key_migration(): src_dir = file_relative_path(__file__, "snapshot_0_7_6_pre_event_log_migration/sqlite") with copy_directory(src_dir) as test_dir: instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) # Make sure the schema is migrated instance.upgrade() runs = instance.get_runs() assert len(runs) == 1 run_ids = instance._event_storage.get_all_run_ids() assert run_ids == ["6405c4a0-3ccc-4600-af81-b5ee197f8528"] assert isinstance(instance._event_storage, SqlEventLogStorage) events_by_id = instance._event_storage.get_logs_for_run_by_log_id( "6405c4a0-3ccc-4600-af81-b5ee197f8528" ) assert len(events_by_id) == 40 step_key_records = [] for record_id, _event in events_by_id.items(): row_data = instance._event_storage.get_event_log_table_data( "6405c4a0-3ccc-4600-af81-b5ee197f8528", record_id ) if row_data.step_key is not None: step_key_records.append(row_data) assert len(step_key_records) == 0 # run the event_log backfill migration migrate_event_log_data(instance=instance) step_key_records = [] for record_id, _event in events_by_id.items(): row_data = instance._event_storage.get_event_log_table_data( "6405c4a0-3ccc-4600-af81-b5ee197f8528", record_id ) if row_data.step_key is not None: step_key_records.append(row_data) assert len(step_key_records) > 0
def test_0_7_6_postgres_pre_event_log_migration(hostname, conn_string): engine = create_engine(conn_string) engine.execute("drop schema public cascade;") engine.execute("create schema public;") env = os.environ.copy() env["PGPASSWORD"] = "******" subprocess.check_call( [ "psql", "-h", hostname, "-p", "5432", "-U", "test", "-f", file_relative_path( __file__, "snapshot_0_7_6_pre_event_log_migration/postgres/pg_dump.txt"), ], env=env, ) run_id = "ca7f1e33-526d-4f75-9bc5-3e98da41ab97" with seven.TemporaryDirectory() as tempdir: with open(file_relative_path(__file__, "dagster.yaml"), "r") as template_fd: with open(os.path.join(tempdir, "dagster.yaml"), "w") as target_fd: template = template_fd.read().format(hostname=hostname) target_fd.write(template) instance = DagsterInstance.from_config(tempdir) # Runs will appear in DB, but event logs need migration runs = instance.get_runs() assert len(runs) == 1 assert instance.get_run_by_id(run_id) # Make sure the schema is migrated instance.upgrade() assert isinstance(instance._event_storage, SqlEventLogStorage) events_by_id = instance._event_storage.get_logs_for_run_by_log_id( run_id) assert len(events_by_id) == 40 step_key_records = [] for record_id, _event in events_by_id.items(): row_data = instance._event_storage.get_event_log_table_data( run_id, record_id) if row_data.step_key is not None: step_key_records.append(row_data) assert len(step_key_records) == 0 # run the event_log data migration migrate_event_log_data(instance=instance) step_key_records = [] for record_id, _event in events_by_id.items(): row_data = instance._event_storage.get_event_log_table_data( run_id, record_id) if row_data.step_key is not None: step_key_records.append(row_data) assert len(step_key_records) > 0