def test_run_created_in_0_7_9_snapshot_id_change(): test_dir = file_relative_path( __file__, 'snapshot_0_7_9_shapshot_id_creation_change/sqlite') with restore_directory(test_dir): instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) # run_id = 'e297fa70-49e8-43f8-abfe-1634f02644f6' old_pipeline_snapshot_id = '88528edde2ed64da3c39cca0da8ba2f7586c1a5d' old_execution_plan_snapshot_id = '2246f8e5a10d21e15fbfa3773d7b2d0bc1fa9d3d' with pytest.warns( UserWarning, match=re.escape( '"input_hydration_schema_key" is deprecated and will be removed in 0.10.0, use ' '"loader_schema_key" instead.'), ): historical_pipeline = instance.get_historical_pipeline( old_pipeline_snapshot_id) pipeline_snapshot = historical_pipeline.pipeline_snapshot ep_snapshot = instance.get_execution_plan_snapshot( old_execution_plan_snapshot_id) # It is the pipeline snapshot that changed # Verify that snapshot ids are not equal. This changed in 0.7.10 assert create_pipeline_snapshot_id( pipeline_snapshot) != old_pipeline_snapshot_id # We also changed execution plan schema in 0.7.11.post1 assert create_execution_plan_snapshot_id( ep_snapshot) != old_execution_plan_snapshot_id # This previously failed with a check error assert ExternalExecutionPlan(ep_snapshot, historical_pipeline)
def test_snapshot_0_7_6_pre_add_pipeline_snapshot(): run_id = "fb0b3905-068b-4444-8f00-76fcbaef7e8b" test_dir = file_relative_path( __file__, "snapshot_0_7_6_pre_add_pipeline_snapshot/sqlite") with restore_directory(test_dir): # invariant check to make sure migration has not been run yet db_path = os.path.join(test_dir, "history", "runs.db") assert get_current_alembic_version(db_path) == "9fe9e746268c" assert "snapshots" not in get_sqlite3_tables(db_path) instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) @solid def noop_solid(_): pass @pipeline def noop_pipeline(): noop_solid() with pytest.raises( DagsterInstanceMigrationRequired, match=_run_storage_migration_regex( current_revision="9fe9e746268c"), ): execute_pipeline(noop_pipeline, instance=instance) assert len(instance.get_runs()) == 1 # Make sure the schema is migrated instance.upgrade() assert get_current_alembic_version(db_path) == "c63a27054f08" assert "snapshots" in get_sqlite3_tables(db_path) assert {"id", "snapshot_id", "snapshot_body", "snapshot_type" } == set(get_sqlite3_columns(db_path, "snapshots")) assert len(instance.get_runs()) == 1 run = instance.get_run_by_id(run_id) assert run.run_id == run_id assert run.pipeline_snapshot_id is None result = execute_pipeline(noop_pipeline, instance=instance) assert result.success runs = instance.get_runs() assert len(runs) == 2 new_run_id = result.run_id new_run = instance.get_run_by_id(new_run_id) assert new_run.pipeline_snapshot_id
def test_0_6_6_sqlite_migrate(): test_dir = file_relative_path(__file__, 'snapshot_0_6_6/sqlite') assert os.path.exists( file_relative_path(__file__, 'snapshot_0_6_6/sqlite/runs.db')) assert not os.path.exists( file_relative_path(__file__, 'snapshot_0_6_6/sqlite/history/runs.db')) with restore_directory(test_dir): instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) instance.upgrade() runs = instance.get_runs() assert len(runs) == 1 run_ids = instance._event_storage.get_all_run_ids() assert run_ids == ['89296095-892d-4a15-aa0d-9018d1580945'] instance._event_storage.get_logs_for_run( '89296095-892d-4a15-aa0d-9018d1580945') assert not os.path.exists( file_relative_path(__file__, 'snapshot_0_6_6/sqlite/runs.db')) assert os.path.exists( file_relative_path(__file__, 'snapshot_0_6_6/sqlite/history/runs.db'))
def test_run_created_in_0_7_9_snapshot_id_change(): test_dir = file_relative_path( __file__, 'snapshot_0_7_9_shapshot_id_creation_change/sqlite') with restore_directory(test_dir): instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) # run_id = 'e297fa70-49e8-43f8-abfe-1634f02644f6' old_pipeline_snapshot_id = '88528edde2ed64da3c39cca0da8ba2f7586c1a5d' old_execution_plan_snapshot_id = '2246f8e5a10d21e15fbfa3773d7b2d0bc1fa9d3d' pipeline_snapshot = instance.get_pipeline_snapshot( old_pipeline_snapshot_id) ep_snapshot = instance.get_execution_plan_snapshot( old_execution_plan_snapshot_id) # It is the pipeline snapshot that changed # Verify that snapshot ids are not equal. This changed in 0.7.10 assert create_pipeline_snapshot_id( pipeline_snapshot) != old_pipeline_snapshot_id # We also changed execution plan schema in 0.7.11.post1 assert create_execution_plan_snapshot_id( ep_snapshot) != old_execution_plan_snapshot_id # This previously failed with a check error assert ExecutionPlanIndex(ep_snapshot, PipelineIndex(pipeline_snapshot))
def test_0_6_4(): test_dir = file_relative_path(__file__, 'snapshot_0_6_4') with restore_directory(test_dir): instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) runs = instance.get_runs() with pytest.raises( DagsterInstanceMigrationRequired, match=_event_log_migration_regex( run_id='c7a6c4d7-6c88-46d0-8baa-d4937c3cefe5', current_revision=None ), ): for run in runs: instance.all_logs(run.run_id)
def test_event_log_asset_key_migration(): test_dir = file_relative_path(__file__, 'snapshot_0_7_8_pre_asset_key_migration/sqlite') with restore_directory(test_dir): db_path = os.path.join( test_dir, 'history', 'runs', '722183e4-119f-4a00-853f-e1257be82ddb.db' ) assert get_current_alembic_version(db_path) == '3b1e175a2be3' assert 'asset_key' not in set(get_sqlite3_columns(db_path, 'event_log')) # Make sure the schema is migrated instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) instance.upgrade() assert 'asset_key' in set(get_sqlite3_columns(db_path, 'event_logs'))
def test_0_6_4(): test_dir = file_relative_path(__file__, 'snapshot_0_6_4') with restore_directory(test_dir): instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) runs = instance.get_runs() with pytest.raises( DagsterInstanceMigrationRequired, match=re.escape( 'Instance is out of date and must be migrated (SqliteEventLogStorage for run ' 'c7a6c4d7-6c88-46d0-8baa-d4937c3cefe5). Database is at revision None, head is ' '567bc23fd1ac. Please run `dagster instance migrate`.'), ): for run in runs: instance.all_logs(run.run_id)
def test_event_log_asset_key_migration(): test_dir = file_relative_path( __file__, "snapshot_0_7_8_pre_asset_key_migration/sqlite") with restore_directory(test_dir): db_path = os.path.join(test_dir, "history", "runs", "722183e4-119f-4a00-853f-e1257be82ddb.db") assert get_current_alembic_version(db_path) == "3b1e175a2be3" assert "asset_key" not in set(get_sqlite3_columns( db_path, "event_log")) # Make sure the schema is migrated instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) instance.upgrade() assert "asset_key" in set(get_sqlite3_columns(db_path, "event_logs"))
def test_downgrade_and_upgrade(): test_dir = file_relative_path(__file__, 'snapshot_0_7_6_pre_add_pipeline_snapshot/sqlite') with restore_directory(test_dir): # invariant check to make sure migration has not been run yet db_path = os.path.join(test_dir, 'history', 'runs.db') assert get_current_alembic_version(db_path) == '9fe9e746268c' assert 'snapshots' not in get_sqlite3_tables(db_path) instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) assert len(instance.get_runs()) == 1 # Make sure the schema is migrated instance.upgrade() assert get_current_alembic_version(db_path) == 'c63a27054f08' assert 'snapshots' in get_sqlite3_tables(db_path) assert {'id', 'snapshot_id', 'snapshot_body', 'snapshot_type'} == set( get_sqlite3_columns(db_path, 'snapshots') ) assert len(instance.get_runs()) == 1 instance._run_storage._alembic_downgrade(rev='9fe9e746268c') assert get_current_alembic_version(db_path) == '9fe9e746268c' assert 'snapshots' not in get_sqlite3_tables(db_path) instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) assert len(instance.get_runs()) == 1 instance.upgrade() assert get_current_alembic_version(db_path) == 'c63a27054f08' assert 'snapshots' in get_sqlite3_tables(db_path) assert {'id', 'snapshot_id', 'snapshot_body', 'snapshot_type'} == set( get_sqlite3_columns(db_path, 'snapshots') ) assert len(instance.get_runs()) == 1
def test_downgrade_and_upgrade(): test_dir = file_relative_path( __file__, "snapshot_0_7_6_pre_add_pipeline_snapshot/sqlite") with restore_directory(test_dir): # invariant check to make sure migration has not been run yet db_path = os.path.join(test_dir, "history", "runs.db") assert get_current_alembic_version(db_path) == "9fe9e746268c" assert "snapshots" not in get_sqlite3_tables(db_path) instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) assert len(instance.get_runs()) == 1 # Make sure the schema is migrated instance.upgrade() assert get_current_alembic_version(db_path) == "c63a27054f08" assert "snapshots" in get_sqlite3_tables(db_path) assert {"id", "snapshot_id", "snapshot_body", "snapshot_type" } == set(get_sqlite3_columns(db_path, "snapshots")) assert len(instance.get_runs()) == 1 instance._run_storage._alembic_downgrade(rev="9fe9e746268c") assert get_current_alembic_version(db_path) == "9fe9e746268c" assert "snapshots" not in get_sqlite3_tables(db_path) instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) assert len(instance.get_runs()) == 1 instance.upgrade() assert get_current_alembic_version(db_path) == "c63a27054f08" assert "snapshots" in get_sqlite3_tables(db_path) assert {"id", "snapshot_id", "snapshot_body", "snapshot_type" } == set(get_sqlite3_columns(db_path, "snapshots")) assert len(instance.get_runs()) == 1
def test_0_8_0_scheduler_migration(): test_dir = file_relative_path(__file__, 'snapshot_0_8_0_scheduler_change') with restore_directory(test_dir): instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) with pytest.raises( DagsterInstanceMigrationRequired, match=_schedule_storage_migration_regex(current_revision='da7cd32b690d'), ): instance.all_stored_schedule_state() instance.upgrade() # upgrade just drops tables, and user upgrade flow is cli entry - so # emulate by new-ing up instance which will create new tables instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) instance.all_stored_schedule_state()
def test_0_6_6_sqlite_exc(): test_dir = file_relative_path(__file__, 'snapshot_0_6_6/sqlite') with restore_directory(test_dir): instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) runs = instance.get_runs() # Note that this is a deliberate choice -- old runs are simply invisible, and their # presence won't raise DagsterInstanceMigrationRequired. This is a reasonable choice since # the runs.db has moved and otherwise we would have to do a check for the existence of an # old runs.db every time we accessed the runs. Instead, we'll do this only in the upgrade # method. assert len(runs) == 0 run_ids = instance._event_storage.get_all_run_ids() assert run_ids == ['89296095-892d-4a15-aa0d-9018d1580945'] with pytest.raises( DagsterInstanceMigrationRequired, match=_event_log_migration_regex( run_id='89296095-892d-4a15-aa0d-9018d1580945', current_revision=None ), ): instance._event_storage.get_logs_for_run('89296095-892d-4a15-aa0d-9018d1580945')
def test_event_log_step_key_migration(): test_dir = file_relative_path(__file__, 'snapshot_0_7_6_pre_event_log_migration/sqlite') with restore_directory(test_dir): instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) # Make sure the schema is migrated instance.upgrade() runs = instance.get_runs() assert len(runs) == 1 run_ids = instance._event_storage.get_all_run_ids() assert run_ids == ['6405c4a0-3ccc-4600-af81-b5ee197f8528'] assert isinstance(instance._event_storage, SqlEventLogStorage) events_by_id = instance._event_storage.get_logs_for_run_by_log_id( '6405c4a0-3ccc-4600-af81-b5ee197f8528' ) assert len(events_by_id) == 40 step_key_records = [] for record_id, _event in events_by_id.items(): row_data = instance._event_storage.get_event_log_table_data( '6405c4a0-3ccc-4600-af81-b5ee197f8528', record_id ) if row_data.step_key is not None: step_key_records.append(row_data) assert len(step_key_records) == 0 # run the event_log backfill migration migrate_event_log_data(instance=instance) step_key_records = [] for record_id, _event in events_by_id.items(): row_data = instance._event_storage.get_event_log_table_data( '6405c4a0-3ccc-4600-af81-b5ee197f8528', record_id ) if row_data.step_key is not None: step_key_records.append(row_data) assert len(step_key_records) > 0
def test_0_6_6_sqlite_exc(): test_dir = file_relative_path(__file__, 'snapshot_0_6_6/sqlite') with restore_directory(test_dir): instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) runs = instance.get_runs() # Note that this is a deliberate choice -- old runs are simply invisible, and their # presence won't raise DagsterInstanceMigrationRequired. This is a reasonable choice since # the runs.db has moved and otherwise we would have to do a check for the existence of an # old runs.db every time we accessed the runs. Instead, we'll do this only in the upgrade # method. assert len(runs) == 0 run_ids = instance._event_storage.get_all_run_ids() assert run_ids == ['89296095-892d-4a15-aa0d-9018d1580945'] with pytest.raises( DagsterInstanceMigrationRequired, match=re.escape( 'Instance is out of date and must be migrated (SqliteEventLogStorage for run ' '89296095-892d-4a15-aa0d-9018d1580945). Database is at revision None, head is ' '567bc23fd1ac. Please run `dagster instance migrate`.'), ): instance._event_storage.get_logs_for_run( '89296095-892d-4a15-aa0d-9018d1580945')