def test_single_write_read_with_snapshot(self, storage):
run_with_snapshot_id = "lkasjdflkjasdf"
pipeline_def = PipelineDefinition(name="some_pipeline", solid_defs=[])
pipeline_snapshot = pipeline_def.get_pipeline_snapshot()
pipeline_snapshot_id = create_pipeline_snapshot_id(pipeline_snapshot)
run_with_snapshot = PipelineRun(
run_id=run_with_snapshot_id,
pipeline_name=pipeline_def.name,
pipeline_snapshot_id=pipeline_snapshot_id,
)
assert not storage.has_pipeline_snapshot(pipeline_snapshot_id)
assert storage.add_pipeline_snapshot(
pipeline_snapshot) == pipeline_snapshot_id
assert serialize_pp(storage.get_pipeline_snapshot(
pipeline_snapshot_id)) == serialize_pp(pipeline_snapshot)
storage.add_run(run_with_snapshot)
assert storage.get_run_by_id(run_with_snapshot_id) == run_with_snapshot
storage.wipe()
assert not storage.has_pipeline_snapshot(pipeline_snapshot_id)
assert not storage.has_run(run_with_snapshot_id)
def test_debug_snapshot_import(self, storage):
from dagster.core.execution.api import create_execution_plan
from dagster.core.snap import (
snapshot_from_execution_plan,
create_execution_plan_snapshot_id,
)
run_id = make_new_run_id()
run_to_add = TestRunStorage.build_run(pipeline_name="pipeline_name",
run_id=run_id)
storage.add_run(run_to_add)
pipeline_def = PipelineDefinition(name="some_pipeline", solid_defs=[])
pipeline_snapshot = pipeline_def.get_pipeline_snapshot()
pipeline_snapshot_id = create_pipeline_snapshot_id(pipeline_snapshot)
new_pipeline_snapshot_id = f"{pipeline_snapshot_id}-new-snapshot"
storage.add_snapshot(pipeline_snapshot,
snapshot_id=new_pipeline_snapshot_id)
assert not storage.has_snapshot(pipeline_snapshot_id)
assert storage.has_snapshot(new_pipeline_snapshot_id)
execution_plan = create_execution_plan(pipeline_def)
ep_snapshot = snapshot_from_execution_plan(execution_plan,
new_pipeline_snapshot_id)
ep_snapshot_id = create_execution_plan_snapshot_id(ep_snapshot)
new_ep_snapshot_id = f"{ep_snapshot_id}-new-snapshot"
storage.add_snapshot(ep_snapshot, snapshot_id=new_ep_snapshot_id)
assert not storage.has_snapshot(ep_snapshot_id)
assert storage.has_snapshot(new_ep_snapshot_id)
def test_single_write_read_with_snapshot(self, storage):
if not isinstance(storage, InMemoryRunStorage):
pytest.skip()
run_with_snapshot_id = 'lkasjdflkjasdf'
pipeline_def = PipelineDefinition(name='some_pipeline', solid_defs=[])
pipeline_snapshot = pipeline_def.get_pipeline_snapshot()
pipeline_snapshot_id = create_pipeline_snapshot_id(pipeline_snapshot)
run_with_snapshot = PipelineRun.create_empty_run(
run_id=run_with_snapshot_id,
pipeline_name=pipeline_def.name,
pipeline_snapshot_id=pipeline_snapshot_id,
)
assert not storage.has_pipeline_snapshot(pipeline_snapshot_id)
assert storage.add_pipeline_snapshot(pipeline_snapshot) == pipeline_snapshot_id
assert storage.get_pipeline_snapshot(pipeline_snapshot_id) == pipeline_snapshot
storage.add_run(run_with_snapshot)
assert storage.get_run_by_id(run_with_snapshot_id) == run_with_snapshot
storage.wipe()
assert not storage.has_pipeline_snapshot(pipeline_snapshot_id)
assert not storage.has_run(run_with_snapshot_id)
def test_fetch_by_snapshot_id(self, storage):
assert storage
pipeline_def_a = PipelineDefinition(name="some_pipeline",
solid_defs=[])
pipeline_def_b = PipelineDefinition(name="some_other_pipeline",
solid_defs=[])
pipeline_snapshot_a = pipeline_def_a.get_pipeline_snapshot()
pipeline_snapshot_b = pipeline_def_b.get_pipeline_snapshot()
pipeline_snapshot_a_id = create_pipeline_snapshot_id(
pipeline_snapshot_a)
pipeline_snapshot_b_id = create_pipeline_snapshot_id(
pipeline_snapshot_b)
assert storage.add_pipeline_snapshot(
pipeline_snapshot_a) == pipeline_snapshot_a_id
assert storage.add_pipeline_snapshot(
pipeline_snapshot_b) == pipeline_snapshot_b_id
one = make_new_run_id()
two = make_new_run_id()
storage.add_run(
TestRunStorage.build_run(
run_id=one,
pipeline_name="some_pipeline",
pipeline_snapshot_id=pipeline_snapshot_a_id,
))
storage.add_run(
TestRunStorage.build_run(
run_id=two,
pipeline_name="some_other_pipeline",
pipeline_snapshot_id=pipeline_snapshot_b_id,
))
assert len(storage.get_runs()) == 2
runs_a = storage.get_runs(
PipelineRunsFilter(snapshot_id=pipeline_snapshot_a_id))
assert len(runs_a) == 1
assert runs_a[0].run_id == one
runs_b = storage.get_runs(
PipelineRunsFilter(snapshot_id=pipeline_snapshot_b_id))
assert len(runs_b) == 1
assert runs_b[0].run_id == two
def external_pipeline_data_from_def(
pipeline_def: PipelineDefinition) -> ExternalPipelineData:
check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition)
return ExternalPipelineData(
name=pipeline_def.name,
pipeline_snapshot=pipeline_def.get_pipeline_snapshot(),
parent_pipeline_snapshot=pipeline_def.get_parent_pipeline_snapshot(),
active_presets=sorted(
list(map(external_preset_data_from_def, pipeline_def.preset_defs)),
key=lambda pd: pd.name,
),
is_job=isinstance(pipeline_def, JobDefinition),
)
def test_add_get_snapshot(self, storage):
pipeline_def = PipelineDefinition(name="some_pipeline", solid_defs=[])
pipeline_snapshot = pipeline_def.get_pipeline_snapshot()
pipeline_snapshot_id = create_pipeline_snapshot_id(pipeline_snapshot)
assert storage.add_pipeline_snapshot(pipeline_snapshot) == pipeline_snapshot_id
fetched_pipeline_snapshot = storage.get_pipeline_snapshot(pipeline_snapshot_id)
assert fetched_pipeline_snapshot
assert serialize_pp(fetched_pipeline_snapshot) == serialize_pp(pipeline_snapshot)
assert storage.has_pipeline_snapshot(pipeline_snapshot_id)
assert not storage.has_pipeline_snapshot("nope")
storage.wipe()
assert not storage.has_pipeline_snapshot(pipeline_snapshot_id)
