def __init__(self, pipeline_snapshot):
self.pipeline_snapshot = check.inst_param(pipeline_snapshot,
'pipeline_snapshot',
PipelineSnapshot)
self._solid_defs_snaps_index = {
sd.name: sd
for sd in
pipeline_snapshot.solid_definitions_snapshot.solid_def_snaps +
pipeline_snapshot.solid_definitions_snapshot.
composite_solid_def_snaps
}
self._dagster_type_snaps_by_name_index = {
dagster_type_snap.name: dagster_type_snap
for dagster_type_snap in
pipeline_snapshot.dagster_type_namespace_snapshot.
all_dagster_type_snaps_by_key.values() if dagster_type_snap.name
}
self.dep_structure_index = DependencyStructureIndex(
pipeline_snapshot.dep_structure_snapshot)
self._comp_dep_structures = {
comp_snap.name:
DependencyStructureIndex(comp_snap.dep_structure_snapshot)
for comp_snap in pipeline_snapshot.solid_definitions_snapshot.
composite_solid_def_snaps
}
self.pipeline_snapshot_id = create_pipeline_snapshot_id(
pipeline_snapshot)
def test_basic_comp_solid_definition():
@solid
def return_one(_):
return 1
@solid
def take_one(_, one):
return one
@composite_solid
def comp_solid():
take_one(return_one())
comp_solid_meta = build_composite_solid_def_snap(comp_solid)
assert isinstance(comp_solid_meta, CompositeSolidDefSnap)
assert (
deserialize_json_to_dagster_namedtuple(serialize_dagster_namedtuple(comp_solid_meta))
== comp_solid_meta
)
index = DependencyStructureIndex(comp_solid_meta.dep_structure_snapshot)
assert index.get_invocation('return_one')
assert index.get_invocation('take_one')
assert index.get_upstream_output('take_one', 'one').solid_name == 'return_one'
assert index.get_upstream_output('take_one', 'one').output_name == 'result'
def __init__(self, pipeline_snapshot_with_id,
parent_pipeline_snapshot_with_id):
self.pipeline_snapshot_with_id = check.inst_param(
pipeline_snapshot_with_id, 'pipeline_snapshot_with_id',
PipelineSnapshotWithID)
self.parent_pipeline_snapshot_with_id = check.opt_inst_param(
parent_pipeline_snapshot_with_id,
'parent_pipeline_snapshot_with_id',
PipelineSnapshotWithID,
)
pipeline_snapshot = pipeline_snapshot_with_id.pipeline_snapshot
parent_pipeline_snapshot = (
parent_pipeline_snapshot_with_id.pipeline_snapshot
if parent_pipeline_snapshot_with_id else None)
if pipeline_snapshot.lineage_snapshot:
check.invariant(
parent_pipeline_snapshot is not None,
'Can not create PipelineIndex for pipeline_snapshot with '
'lineage without parent_pipeline_snapshot',
)
check.invariant(
pipeline_snapshot.lineage_snapshot.parent_snapshot_id ==
parent_pipeline_snapshot_with_id.pipeline_snapshot_id,
'Mismatch in IDs between pipeline_snapshot lineage and parent_pipeline_snapshot',
)
self._solid_defs_snaps_index = {
sd.name: sd
for sd in
pipeline_snapshot.solid_definitions_snapshot.solid_def_snaps +
pipeline_snapshot.solid_definitions_snapshot.
composite_solid_def_snaps
}
all_dagster_type_snaps = (
pipeline_snapshot.dagster_type_namespace_snapshot.
all_dagster_type_snaps_by_key).values()
self._dagster_type_snaps_by_name_index = {
dagster_type_snap.name: dagster_type_snap
for dagster_type_snap in all_dagster_type_snaps
if dagster_type_snap.name
}
self.dep_structure_index = DependencyStructureIndex(
pipeline_snapshot.dep_structure_snapshot)
self._comp_dep_structures = {
comp_snap.name:
DependencyStructureIndex(comp_snap.dep_structure_snapshot)
for comp_snap in pipeline_snapshot.solid_definitions_snapshot.
composite_solid_def_snaps
}
def __init__(self, pipeline_snapshot, parent_pipeline_snapshot):
self.pipeline_snapshot = check.inst_param(pipeline_snapshot,
"pipeline_snapshot",
PipelineSnapshot)
self.parent_pipeline_snapshot = check.opt_inst_param(
parent_pipeline_snapshot, "parent_pipeline_snapshot",
PipelineSnapshot)
if self.pipeline_snapshot.lineage_snapshot:
check.invariant(
self.parent_pipeline_snapshot is not None,
"Can not create PipelineIndex for pipeline_snapshot with lineage without parent_pipeline_snapshot",
)
parent_id = create_pipeline_snapshot_id(
self.parent_pipeline_snapshot)
check.invariant(
pipeline_snapshot.lineage_snapshot.parent_snapshot_id ==
parent_id,
"Mismatch in IDs between pipeline_snapshot lineage and parent_pipeline_snapshot",
)
self._solid_defs_snaps_index = {
sd.name: sd
for sd in
pipeline_snapshot.solid_definitions_snapshot.solid_def_snaps +
pipeline_snapshot.solid_definitions_snapshot.
composite_solid_def_snaps
}
self._dagster_type_snaps_by_name_index = {
dagster_type_snap.name: dagster_type_snap
for dagster_type_snap in
pipeline_snapshot.dagster_type_namespace_snapshot.
all_dagster_type_snaps_by_key.values() if dagster_type_snap.name
}
self.dep_structure_index = DependencyStructureIndex(
pipeline_snapshot.dep_structure_snapshot)
self._comp_dep_structures = {
comp_snap.name:
DependencyStructureIndex(comp_snap.dep_structure_snapshot)
for comp_snap in pipeline_snapshot.solid_definitions_snapshot.
composite_solid_def_snaps
}
self.pipeline_snapshot_id = create_pipeline_snapshot_id(
pipeline_snapshot)
def __init__(self, pipeline_snapshot, parent_pipeline_snapshot, is_historical=False):
self.pipeline_snapshot = check.inst_param(
pipeline_snapshot, "pipeline_snapshot", PipelineSnapshot
)
self.parent_pipeline_snapshot = check.opt_inst_param(
parent_pipeline_snapshot, "parent_pipeline_snapshot", PipelineSnapshot
)
if self.pipeline_snapshot.lineage_snapshot:
check.invariant(
self.parent_pipeline_snapshot is not None,
"Can not create PipelineIndex for pipeline_snapshot with lineage without parent_pipeline_snapshot",
)
self._node_defs_snaps_index = {
sd.name: sd
for sd in pipeline_snapshot.solid_definitions_snapshot.solid_def_snaps
+ pipeline_snapshot.solid_definitions_snapshot.composite_solid_def_snaps
}
self._dagster_type_snaps_by_name_index = {
dagster_type_snap.name: dagster_type_snap
for dagster_type_snap in pipeline_snapshot.dagster_type_namespace_snapshot.all_dagster_type_snaps_by_key.values()
if dagster_type_snap.name
}
self.dep_structure_index = DependencyStructureIndex(
pipeline_snapshot.dep_structure_snapshot
)
self._comp_dep_structures = {
comp_snap.name: DependencyStructureIndex(comp_snap.dep_structure_snapshot)
for comp_snap in pipeline_snapshot.solid_definitions_snapshot.composite_solid_def_snaps
}
if is_historical:
# defer the pipeline snapshot calculation for historical pipelines. This tends to be an
# expensive operation, so we want to avoid it unless we need it. Also, because this is
# a historical pipeline, we already have an identifying pipeline snapshot id (which may
# or may not be the same as this calculated snapshot id). The identifying pipeline
# snapshot id is the calculated snapshot id at the time that the run was created.
self._pipeline_snapshot_id = None
else:
self._pipeline_snapshot_id = create_pipeline_snapshot_id(pipeline_snapshot)
def test_two_invocations_deps_snap(snapshot):
@solid
def noop_solid(_):
pass
@pipeline
def two_solid_pipeline():
noop_solid.alias('one')()
noop_solid.alias('two')()
index = DependencyStructureIndex(
build_dep_structure_snapshot_from_icontains_solids(two_solid_pipeline))
assert index.get_invocation('one')
assert index.get_invocation('two')
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(two_solid_pipeline)
assert pipeline_snapshot == serialize_rt(pipeline_snapshot)
snapshot.assert_match(serialize_pp(pipeline_snapshot))
snapshot.assert_match(create_pipeline_snapshot_id(pipeline_snapshot))
def test_complex_comp_solid_definition():
@solid
def return_one(_):
return 1
@solid
def take_many(_, items):
return items
@composite_solid
def comp_solid(this_number):
take_many(
[return_one(), this_number,
return_one.alias("return_one_also")()])
comp_solid_meta = build_composite_solid_def_snap(comp_solid)
assert isinstance(comp_solid_meta, CompositeSolidDefSnap)
assert (deserialize_json_to_dagster_namedtuple(
serialize_dagster_namedtuple(comp_solid_meta)) == comp_solid_meta)
index = DependencyStructureIndex(comp_solid_meta.dep_structure_snapshot)
assert index.get_invocation("return_one")
assert index.get_invocation("take_many")
assert index.get_upstream_outputs("take_many",
"items")[0].solid_name == "return_one"
assert index.get_upstream_outputs(
"take_many", "items")[1].solid_name == "return_one_also"
def test_basic_fan_in(snapshot):
@solid(output_defs=[OutputDefinition(Nothing)])
def return_nothing(_):
return None
@solid(input_defs=[InputDefinition('nothing', Nothing)])
def take_nothings(_):
return None
@pipeline
def fan_in_test():
take_nothings([
return_nothing.alias('nothing_one')(),
return_nothing.alias('nothing_two')()
])
dep_structure_snapshot = build_dep_structure_snapshot_from_icontains_solids(
fan_in_test)
index = DependencyStructureIndex(dep_structure_snapshot)
assert index.get_invocation('nothing_one')
assert index.get_invocation('take_nothings')
assert index.get_upstream_outputs('take_nothings', 'nothing') == [
OutputHandleSnap('nothing_one', 'result'),
OutputHandleSnap('nothing_two', 'result'),
]
assert (deserialize_json_to_dagster_namedtuple(
serialize_dagster_namedtuple(dep_structure_snapshot)) ==
dep_structure_snapshot)
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(fan_in_test)
assert pipeline_snapshot == serialize_rt(pipeline_snapshot)
snapshot.assert_match(serialize_pp(pipeline_snapshot))
snapshot.assert_match(create_pipeline_snapshot_id(pipeline_snapshot))
def test_basic_dep():
@solid
def return_one(_):
return 1
@solid(input_defs=[InputDefinition('value', int)])
def passthrough(_, value):
return value
@pipeline
def single_dep_pipeline():
passthrough(return_one())
index = DependencyStructureIndex(
build_dep_structure_snapshot_from_icontains_solids(single_dep_pipeline)
)
assert index.get_invocation('return_one')
assert index.get_invocation('passthrough')
outputs = index.get_upstream_outputs('passthrough', 'value')
assert len(outputs) == 1
assert outputs[0].solid_name == 'return_one'
assert outputs[0].output_name == 'result'
def test_multiple_non_argument_deps():
@asset
def foo():
pass
@asset(namespace="namespace")
def bar():
pass
@asset
def baz():
return 1
@asset(non_argument_deps={AssetKey("foo"), AssetKey(["namespace", "bar"])})
def qux(baz):
return baz
job = build_assets_job("a", [foo, bar, baz, qux])
dep_structure_snapshot = build_dep_structure_snapshot_from_icontains_solids(
job.graph)
index = DependencyStructureIndex(dep_structure_snapshot)
assert index.get_invocation("foo")
assert index.get_invocation("namespace__bar")
assert index.get_invocation("baz")
assert index.get_upstream_outputs("qux", "foo") == [
OutputHandleSnap("foo", "result"),
]
assert index.get_upstream_outputs("qux", "namespace_bar") == [
OutputHandleSnap("namespace__bar", "result")
]
assert index.get_upstream_outputs(
"qux", "baz") == [OutputHandleSnap("baz", "result")]
result = job.execute_in_process()
assert result.success
assert result.output_for_node("qux") == 1
def test_basic_dep_fan_out(snapshot):
@solid
def return_one(_):
return 1
@solid(input_defs=[InputDefinition('value', int)])
def passthrough(_, value):
return value
@pipeline
def single_dep_pipeline():
return_one_result = return_one()
passthrough.alias('passone')(return_one_result)
passthrough.alias('passtwo')(return_one_result)
dep_structure_snapshot = build_dep_structure_snapshot_from_icontains_solids(
single_dep_pipeline)
index = DependencyStructureIndex(dep_structure_snapshot)
assert index.get_invocation('return_one')
assert index.get_invocation('passone')
assert index.get_invocation('passtwo')
assert index.get_upstream_output('passone', 'value') == OutputHandleSnap(
'return_one', 'result')
assert index.get_upstream_output('passtwo', 'value') == OutputHandleSnap(
'return_one', 'result')
assert set(index.get_downstream_inputs('return_one', 'result')) == set([
InputHandle('passthrough', 'passone', 'value'),
InputHandle('passthrough', 'passtwo', 'value'),
])
assert (deserialize_json_to_dagster_namedtuple(
serialize_dagster_namedtuple(dep_structure_snapshot)) ==
dep_structure_snapshot)
pipeline_snapshot = PipelineSnapshot.from_pipeline_def(single_dep_pipeline)
assert pipeline_snapshot == serialize_rt(pipeline_snapshot)
snapshot.assert_match(serialize_pp(pipeline_snapshot))
snapshot.assert_match(create_pipeline_snapshot_id(pipeline_snapshot))
class PipelineIndex:
def __init__(self, pipeline_snapshot, parent_pipeline_snapshot):
self.pipeline_snapshot = check.inst_param(pipeline_snapshot,
"pipeline_snapshot",
PipelineSnapshot)
self.parent_pipeline_snapshot = check.opt_inst_param(
parent_pipeline_snapshot, "parent_pipeline_snapshot",
PipelineSnapshot)
if self.pipeline_snapshot.lineage_snapshot:
check.invariant(
self.parent_pipeline_snapshot is not None,
"Can not create PipelineIndex for pipeline_snapshot with lineage without parent_pipeline_snapshot",
)
self._node_defs_snaps_index = {
sd.name: sd
for sd in
pipeline_snapshot.solid_definitions_snapshot.solid_def_snaps +
pipeline_snapshot.solid_definitions_snapshot.
composite_solid_def_snaps
}
self._dagster_type_snaps_by_name_index = {
dagster_type_snap.name: dagster_type_snap
for dagster_type_snap in
pipeline_snapshot.dagster_type_namespace_snapshot.
all_dagster_type_snaps_by_key.values() if dagster_type_snap.name
}
self.dep_structure_index = DependencyStructureIndex(
pipeline_snapshot.dep_structure_snapshot)
self._comp_dep_structures = {
comp_snap.name:
DependencyStructureIndex(comp_snap.dep_structure_snapshot)
for comp_snap in pipeline_snapshot.solid_definitions_snapshot.
composite_solid_def_snaps
}
self._pipeline_snapshot_id = None
@property
def name(self):
return self.pipeline_snapshot.name
@property
def description(self):
return self.pipeline_snapshot.description
@property
def tags(self):
return self.pipeline_snapshot.tags
@property
def pipeline_snapshot_id(self):
if not self._pipeline_snapshot_id:
self._pipeline_snapshot_id = create_pipeline_snapshot_id(
self.pipeline_snapshot)
return self._pipeline_snapshot_id
def has_dagster_type_name(self, type_name):
return type_name in self._dagster_type_snaps_by_name_index
def get_dagster_type_from_name(self, type_name):
return self._dagster_type_snaps_by_name_index[type_name]
def get_node_def_snap(self, node_def_name):
check.str_param(node_def_name, "node_def_name")
return self._node_defs_snaps_index[node_def_name]
def get_dep_structure_index(self, comp_solid_def_name):
return self._comp_dep_structures[comp_solid_def_name]
def get_dagster_type_snaps(self):
dt_namespace = self.pipeline_snapshot.dagster_type_namespace_snapshot
return list(dt_namespace.all_dagster_type_snaps_by_key.values())
def has_solid_invocation(self, solid_name):
return self.dep_structure_index.has_invocation(solid_name)
def get_default_mode_name(self) -> str:
return self.pipeline_snapshot.mode_def_snaps[0].name
def has_mode_def(self, name):
check.str_param(name, "name")
for mode_def_snap in self.pipeline_snapshot.mode_def_snaps:
if mode_def_snap.name == name:
return True
return False
@property
def available_modes(self):
return [
mode_def_snap.name
for mode_def_snap in self.pipeline_snapshot.mode_def_snaps
]
def get_mode_def_snap(self, name):
check.str_param(name, "name")
for mode_def_snap in self.pipeline_snapshot.mode_def_snaps:
if mode_def_snap.name == name:
return mode_def_snap
check.failed("Mode {mode} not found".format(mode=name))
@property
def config_schema_snapshot(self):
return self.pipeline_snapshot.config_schema_snapshot
class PipelineIndex:
def __init__(self, pipeline_snapshot):
self.pipeline_snapshot = check.inst_param(pipeline_snapshot,
'pipeline_snapshot',
PipelineSnapshot)
self._solid_defs_snaps_index = {
sd.name: sd
for sd in
pipeline_snapshot.solid_definitions_snapshot.solid_def_snaps +
pipeline_snapshot.solid_definitions_snapshot.
composite_solid_def_snaps
}
self._dagster_type_snaps_by_name_index = {
dagster_type_snap.name: dagster_type_snap
for dagster_type_snap in
pipeline_snapshot.dagster_type_namespace_snapshot.
all_dagster_type_snaps_by_key.values() if dagster_type_snap.name
}
self.dep_structure_index = DependencyStructureIndex(
pipeline_snapshot.dep_structure_snapshot)
self._comp_dep_structures = {
comp_snap.name:
DependencyStructureIndex(comp_snap.dep_structure_snapshot)
for comp_snap in pipeline_snapshot.solid_definitions_snapshot.
composite_solid_def_snaps
}
self.pipeline_snapshot_id = create_pipeline_snapshot_id(
pipeline_snapshot)
@property
def name(self):
return self.pipeline_snapshot.name
@property
def description(self):
return self.pipeline_snapshot.description
@property
def tags(self):
return self.pipeline_snapshot.tags
def has_dagster_type_name(self, type_name):
return type_name in self._dagster_type_snaps_by_name_index
def get_dagster_type_from_name(self, type_name):
return self._dagster_type_snaps_by_name_index[type_name]
def get_solid_def_snap(self, solid_def_name):
check.str_param(solid_def_name, 'solid_def_name')
return self._solid_defs_snaps_index[solid_def_name]
def get_dep_structure_index(self, comp_solid_def_name):
return self._comp_dep_structures[comp_solid_def_name]
def get_dagster_type_snaps(self):
dt_namespace = self.pipeline_snapshot.dagster_type_namespace_snapshot
return list(dt_namespace.all_dagster_type_snaps_by_key.values())
def has_solid_invocation(self, solid_name):
return self.dep_structure_index.has_invocation(solid_name)
def get_default_mode_name(self):
return self.pipeline_snapshot.mode_def_snaps[0].name
def has_mode_def(self, name):
check.str_param(name, 'name')
for mode_def_snap in self.pipeline_snapshot.mode_def_snaps:
if mode_def_snap.name == name:
return True
return False
def get_mode_def_snap(self, name):
check.str_param(name, 'name')
for mode_def_snap in self.pipeline_snapshot.mode_def_snaps:
if mode_def_snap.name == name:
return mode_def_snap
check.failed('Mode {mode} not found'.format(mode=name))
@property
def config_schema_snapshot(self):
return self.pipeline_snapshot.config_schema_snapshot
class PipelineIndex:
def __init__(self, pipeline_snapshot_with_id,
parent_pipeline_snapshot_with_id):
self.pipeline_snapshot_with_id = check.inst_param(
pipeline_snapshot_with_id, 'pipeline_snapshot_with_id',
PipelineSnapshotWithID)
self.parent_pipeline_snapshot_with_id = check.opt_inst_param(
parent_pipeline_snapshot_with_id,
'parent_pipeline_snapshot_with_id',
PipelineSnapshotWithID,
)
pipeline_snapshot = pipeline_snapshot_with_id.pipeline_snapshot
parent_pipeline_snapshot = (
parent_pipeline_snapshot_with_id.pipeline_snapshot
if parent_pipeline_snapshot_with_id else None)
if pipeline_snapshot.lineage_snapshot:
check.invariant(
parent_pipeline_snapshot is not None,
'Can not create PipelineIndex for pipeline_snapshot with '
'lineage without parent_pipeline_snapshot',
)
check.invariant(
pipeline_snapshot.lineage_snapshot.parent_snapshot_id ==
parent_pipeline_snapshot_with_id.pipeline_snapshot_id,
'Mismatch in IDs between pipeline_snapshot lineage and parent_pipeline_snapshot',
)
self._solid_defs_snaps_index = {
sd.name: sd
for sd in
pipeline_snapshot.solid_definitions_snapshot.solid_def_snaps +
pipeline_snapshot.solid_definitions_snapshot.
composite_solid_def_snaps
}
all_dagster_type_snaps = (
pipeline_snapshot.dagster_type_namespace_snapshot.
all_dagster_type_snaps_by_key).values()
self._dagster_type_snaps_by_name_index = {
dagster_type_snap.name: dagster_type_snap
for dagster_type_snap in all_dagster_type_snaps
if dagster_type_snap.name
}
self.dep_structure_index = DependencyStructureIndex(
pipeline_snapshot.dep_structure_snapshot)
self._comp_dep_structures = {
comp_snap.name:
DependencyStructureIndex(comp_snap.dep_structure_snapshot)
for comp_snap in pipeline_snapshot.solid_definitions_snapshot.
composite_solid_def_snaps
}
@property
def pipeline_snapshot(self):
return self.pipeline_snapshot_with_id.pipeline_snapshot
@property
def parent_pipeline_snapshot(self):
return (self.parent_pipeline_snapshot_with_id.pipeline_snapshot
if self.parent_pipeline_snapshot_with_id else None)
@property
def pipeline_snapshot_id(self):
return self.pipeline_snapshot_with_id.pipeline_snapshot_id
@property
def name(self):
return self.pipeline_snapshot.name
@property
def description(self):
return self.pipeline_snapshot.description
@property
def tags(self):
return self.pipeline_snapshot.tags
def has_dagster_type_name(self, type_name):
return type_name in self._dagster_type_snaps_by_name_index
def get_dagster_type_from_name(self, type_name):
return self._dagster_type_snaps_by_name_index[type_name]
def get_solid_def_snap(self, solid_def_name):
check.str_param(solid_def_name, 'solid_def_name')
return self._solid_defs_snaps_index[solid_def_name]
def get_dep_structure_index(self, comp_solid_def_name):
return self._comp_dep_structures[comp_solid_def_name]
def get_dagster_type_snaps(self):
dt_namespace = self.pipeline_snapshot.dagster_type_namespace_snapshot
return list(dt_namespace.all_dagster_type_snaps_by_key.values())
def has_solid_invocation(self, solid_name):
return self.dep_structure_index.has_invocation(solid_name)
def get_default_mode_name(self):
return self.pipeline_snapshot.mode_def_snaps[0].name
def has_mode_def(self, name):
check.str_param(name, 'name')
for mode_def_snap in self.pipeline_snapshot.mode_def_snaps:
if mode_def_snap.name == name:
return True
return False
def get_mode_def_snap(self, name):
check.str_param(name, 'name')
for mode_def_snap in self.pipeline_snapshot.mode_def_snaps:
if mode_def_snap.name == name:
return mode_def_snap
check.failed('Mode {mode} not found'.format(mode=name))
@property
def config_schema_snapshot(self):
return self.pipeline_snapshot.config_schema_snapshot
class PipelineIndex:
def __init__(self, pipeline_snapshot, parent_pipeline_snapshot, is_historical=False):
self.pipeline_snapshot = check.inst_param(
pipeline_snapshot, "pipeline_snapshot", PipelineSnapshot
)
self.parent_pipeline_snapshot = check.opt_inst_param(
parent_pipeline_snapshot, "parent_pipeline_snapshot", PipelineSnapshot
)
if self.pipeline_snapshot.lineage_snapshot:
check.invariant(
self.parent_pipeline_snapshot is not None,
"Can not create PipelineIndex for pipeline_snapshot with lineage without parent_pipeline_snapshot",
)
self._node_defs_snaps_index = {
sd.name: sd
for sd in pipeline_snapshot.solid_definitions_snapshot.solid_def_snaps
+ pipeline_snapshot.solid_definitions_snapshot.composite_solid_def_snaps
}
self._dagster_type_snaps_by_name_index = {
dagster_type_snap.name: dagster_type_snap
for dagster_type_snap in pipeline_snapshot.dagster_type_namespace_snapshot.all_dagster_type_snaps_by_key.values()
if dagster_type_snap.name
}
self.dep_structure_index = DependencyStructureIndex(
pipeline_snapshot.dep_structure_snapshot
)
self._comp_dep_structures = {
comp_snap.name: DependencyStructureIndex(comp_snap.dep_structure_snapshot)
for comp_snap in pipeline_snapshot.solid_definitions_snapshot.composite_solid_def_snaps
}
if is_historical:
# defer the pipeline snapshot calculation for historical pipelines. This tends to be an
# expensive operation, so we want to avoid it unless we need it. Also, because this is
# a historical pipeline, we already have an identifying pipeline snapshot id (which may
# or may not be the same as this calculated snapshot id). The identifying pipeline
# snapshot id is the calculated snapshot id at the time that the run was created.
self._pipeline_snapshot_id = None
else:
self._pipeline_snapshot_id = create_pipeline_snapshot_id(pipeline_snapshot)
@property
def name(self):
return self.pipeline_snapshot.name
@property
def description(self):
return self.pipeline_snapshot.description
@property
def tags(self):
return self.pipeline_snapshot.tags
@property
def pipeline_snapshot_id(self):
if not self._pipeline_snapshot_id:
self._pipeline_snapshot_id = create_pipeline_snapshot_id(self.pipeline_snapshot)
return self._pipeline_snapshot_id
def has_dagster_type_name(self, type_name):
return type_name in self._dagster_type_snaps_by_name_index
def get_dagster_type_from_name(self, type_name):
return self._dagster_type_snaps_by_name_index[type_name]
def get_node_def_snap(self, node_def_name):
check.str_param(node_def_name, "node_def_name")
return self._node_defs_snaps_index[node_def_name]
def get_dep_structure_index(self, comp_solid_def_name):
return self._comp_dep_structures[comp_solid_def_name]
def get_dagster_type_snaps(self):
dt_namespace = self.pipeline_snapshot.dagster_type_namespace_snapshot
return list(dt_namespace.all_dagster_type_snaps_by_key.values())
def has_solid_invocation(self, solid_name):
return self.dep_structure_index.has_invocation(solid_name)
def get_default_mode_name(self):
return self.pipeline_snapshot.mode_def_snaps[0].name
def has_mode_def(self, name):
check.str_param(name, "name")
for mode_def_snap in self.pipeline_snapshot.mode_def_snaps:
if mode_def_snap.name == name:
return True
return False
@property
def available_modes(self):
return [mode_def_snap.name for mode_def_snap in self.pipeline_snapshot.mode_def_snaps]
def get_mode_def_snap(self, name):
check.str_param(name, "name")
for mode_def_snap in self.pipeline_snapshot.mode_def_snaps:
if mode_def_snap.name == name:
return mode_def_snap
check.failed("Mode {mode} not found".format(mode=name))
@property
def config_schema_snapshot(self):
return self.pipeline_snapshot.config_schema_snapshot
