class PipelineIndex: def __init__(self, pipeline_snapshot, parent_pipeline_snapshot): self.pipeline_snapshot = check.inst_param(pipeline_snapshot, "pipeline_snapshot", PipelineSnapshot) self.parent_pipeline_snapshot = check.opt_inst_param( parent_pipeline_snapshot, "parent_pipeline_snapshot", PipelineSnapshot) if self.pipeline_snapshot.lineage_snapshot: check.invariant( self.parent_pipeline_snapshot is not None, "Can not create PipelineIndex for pipeline_snapshot with lineage without parent_pipeline_snapshot", ) self._node_defs_snaps_index = { sd.name: sd for sd in pipeline_snapshot.solid_definitions_snapshot.solid_def_snaps + pipeline_snapshot.solid_definitions_snapshot. composite_solid_def_snaps } self._dagster_type_snaps_by_name_index = { dagster_type_snap.name: dagster_type_snap for dagster_type_snap in pipeline_snapshot.dagster_type_namespace_snapshot. all_dagster_type_snaps_by_key.values() if dagster_type_snap.name } self.dep_structure_index = DependencyStructureIndex( pipeline_snapshot.dep_structure_snapshot) self._comp_dep_structures = { comp_snap.name: DependencyStructureIndex(comp_snap.dep_structure_snapshot) for comp_snap in pipeline_snapshot.solid_definitions_snapshot. composite_solid_def_snaps } self._pipeline_snapshot_id = None @property def name(self): return self.pipeline_snapshot.name @property def description(self): return self.pipeline_snapshot.description @property def tags(self): return self.pipeline_snapshot.tags @property def pipeline_snapshot_id(self): if not self._pipeline_snapshot_id: self._pipeline_snapshot_id = create_pipeline_snapshot_id( self.pipeline_snapshot) return self._pipeline_snapshot_id def has_dagster_type_name(self, type_name): return type_name in self._dagster_type_snaps_by_name_index def get_dagster_type_from_name(self, type_name): return self._dagster_type_snaps_by_name_index[type_name] def get_node_def_snap(self, node_def_name): check.str_param(node_def_name, "node_def_name") return self._node_defs_snaps_index[node_def_name] def get_dep_structure_index(self, comp_solid_def_name): return self._comp_dep_structures[comp_solid_def_name] def get_dagster_type_snaps(self): dt_namespace = self.pipeline_snapshot.dagster_type_namespace_snapshot return list(dt_namespace.all_dagster_type_snaps_by_key.values()) def has_solid_invocation(self, solid_name): return self.dep_structure_index.has_invocation(solid_name) def get_default_mode_name(self) -> str: return self.pipeline_snapshot.mode_def_snaps[0].name def has_mode_def(self, name): check.str_param(name, "name") for mode_def_snap in self.pipeline_snapshot.mode_def_snaps: if mode_def_snap.name == name: return True return False @property def available_modes(self): return [ mode_def_snap.name for mode_def_snap in self.pipeline_snapshot.mode_def_snaps ] def get_mode_def_snap(self, name): check.str_param(name, "name") for mode_def_snap in self.pipeline_snapshot.mode_def_snaps: if mode_def_snap.name == name: return mode_def_snap check.failed("Mode {mode} not found".format(mode=name)) @property def config_schema_snapshot(self): return self.pipeline_snapshot.config_schema_snapshot
class PipelineIndex: def __init__(self, pipeline_snapshot_with_id, parent_pipeline_snapshot_with_id): self.pipeline_snapshot_with_id = check.inst_param( pipeline_snapshot_with_id, 'pipeline_snapshot_with_id', PipelineSnapshotWithID) self.parent_pipeline_snapshot_with_id = check.opt_inst_param( parent_pipeline_snapshot_with_id, 'parent_pipeline_snapshot_with_id', PipelineSnapshotWithID, ) pipeline_snapshot = pipeline_snapshot_with_id.pipeline_snapshot parent_pipeline_snapshot = ( parent_pipeline_snapshot_with_id.pipeline_snapshot if parent_pipeline_snapshot_with_id else None) if pipeline_snapshot.lineage_snapshot: check.invariant( parent_pipeline_snapshot is not None, 'Can not create PipelineIndex for pipeline_snapshot with ' 'lineage without parent_pipeline_snapshot', ) check.invariant( pipeline_snapshot.lineage_snapshot.parent_snapshot_id == parent_pipeline_snapshot_with_id.pipeline_snapshot_id, 'Mismatch in IDs between pipeline_snapshot lineage and parent_pipeline_snapshot', ) self._solid_defs_snaps_index = { sd.name: sd for sd in pipeline_snapshot.solid_definitions_snapshot.solid_def_snaps + pipeline_snapshot.solid_definitions_snapshot. composite_solid_def_snaps } all_dagster_type_snaps = ( pipeline_snapshot.dagster_type_namespace_snapshot. all_dagster_type_snaps_by_key).values() self._dagster_type_snaps_by_name_index = { dagster_type_snap.name: dagster_type_snap for dagster_type_snap in all_dagster_type_snaps if dagster_type_snap.name } self.dep_structure_index = DependencyStructureIndex( pipeline_snapshot.dep_structure_snapshot) self._comp_dep_structures = { comp_snap.name: DependencyStructureIndex(comp_snap.dep_structure_snapshot) for comp_snap in pipeline_snapshot.solid_definitions_snapshot. composite_solid_def_snaps } @property def pipeline_snapshot(self): return self.pipeline_snapshot_with_id.pipeline_snapshot @property def parent_pipeline_snapshot(self): return (self.parent_pipeline_snapshot_with_id.pipeline_snapshot if self.parent_pipeline_snapshot_with_id else None) @property def pipeline_snapshot_id(self): return self.pipeline_snapshot_with_id.pipeline_snapshot_id @property def name(self): return self.pipeline_snapshot.name @property def description(self): return self.pipeline_snapshot.description @property def tags(self): return self.pipeline_snapshot.tags def has_dagster_type_name(self, type_name): return type_name in self._dagster_type_snaps_by_name_index def get_dagster_type_from_name(self, type_name): return self._dagster_type_snaps_by_name_index[type_name] def get_solid_def_snap(self, solid_def_name): check.str_param(solid_def_name, 'solid_def_name') return self._solid_defs_snaps_index[solid_def_name] def get_dep_structure_index(self, comp_solid_def_name): return self._comp_dep_structures[comp_solid_def_name] def get_dagster_type_snaps(self): dt_namespace = self.pipeline_snapshot.dagster_type_namespace_snapshot return list(dt_namespace.all_dagster_type_snaps_by_key.values()) def has_solid_invocation(self, solid_name): return self.dep_structure_index.has_invocation(solid_name) def get_default_mode_name(self): return self.pipeline_snapshot.mode_def_snaps[0].name def has_mode_def(self, name): check.str_param(name, 'name') for mode_def_snap in self.pipeline_snapshot.mode_def_snaps: if mode_def_snap.name == name: return True return False def get_mode_def_snap(self, name): check.str_param(name, 'name') for mode_def_snap in self.pipeline_snapshot.mode_def_snaps: if mode_def_snap.name == name: return mode_def_snap check.failed('Mode {mode} not found'.format(mode=name)) @property def config_schema_snapshot(self): return self.pipeline_snapshot.config_schema_snapshot
class PipelineIndex: def __init__(self, pipeline_snapshot): self.pipeline_snapshot = check.inst_param(pipeline_snapshot, 'pipeline_snapshot', PipelineSnapshot) self._solid_defs_snaps_index = { sd.name: sd for sd in pipeline_snapshot.solid_definitions_snapshot.solid_def_snaps + pipeline_snapshot.solid_definitions_snapshot. composite_solid_def_snaps } self._dagster_type_snaps_by_name_index = { dagster_type_snap.name: dagster_type_snap for dagster_type_snap in pipeline_snapshot.dagster_type_namespace_snapshot. all_dagster_type_snaps_by_key.values() if dagster_type_snap.name } self.dep_structure_index = DependencyStructureIndex( pipeline_snapshot.dep_structure_snapshot) self._comp_dep_structures = { comp_snap.name: DependencyStructureIndex(comp_snap.dep_structure_snapshot) for comp_snap in pipeline_snapshot.solid_definitions_snapshot. composite_solid_def_snaps } self.pipeline_snapshot_id = create_pipeline_snapshot_id( pipeline_snapshot) @property def name(self): return self.pipeline_snapshot.name @property def description(self): return self.pipeline_snapshot.description @property def tags(self): return self.pipeline_snapshot.tags def has_dagster_type_name(self, type_name): return type_name in self._dagster_type_snaps_by_name_index def get_dagster_type_from_name(self, type_name): return self._dagster_type_snaps_by_name_index[type_name] def get_solid_def_snap(self, solid_def_name): check.str_param(solid_def_name, 'solid_def_name') return self._solid_defs_snaps_index[solid_def_name] def get_dep_structure_index(self, comp_solid_def_name): return self._comp_dep_structures[comp_solid_def_name] def get_dagster_type_snaps(self): dt_namespace = self.pipeline_snapshot.dagster_type_namespace_snapshot return list(dt_namespace.all_dagster_type_snaps_by_key.values()) def has_solid_invocation(self, solid_name): return self.dep_structure_index.has_invocation(solid_name) def get_default_mode_name(self): return self.pipeline_snapshot.mode_def_snaps[0].name def has_mode_def(self, name): check.str_param(name, 'name') for mode_def_snap in self.pipeline_snapshot.mode_def_snaps: if mode_def_snap.name == name: return True return False def get_mode_def_snap(self, name): check.str_param(name, 'name') for mode_def_snap in self.pipeline_snapshot.mode_def_snaps: if mode_def_snap.name == name: return mode_def_snap check.failed('Mode {mode} not found'.format(mode=name)) @property def config_schema_snapshot(self): return self.pipeline_snapshot.config_schema_snapshot
class PipelineIndex: def __init__(self, pipeline_snapshot, parent_pipeline_snapshot, is_historical=False): self.pipeline_snapshot = check.inst_param( pipeline_snapshot, "pipeline_snapshot", PipelineSnapshot ) self.parent_pipeline_snapshot = check.opt_inst_param( parent_pipeline_snapshot, "parent_pipeline_snapshot", PipelineSnapshot ) if self.pipeline_snapshot.lineage_snapshot: check.invariant( self.parent_pipeline_snapshot is not None, "Can not create PipelineIndex for pipeline_snapshot with lineage without parent_pipeline_snapshot", ) self._node_defs_snaps_index = { sd.name: sd for sd in pipeline_snapshot.solid_definitions_snapshot.solid_def_snaps + pipeline_snapshot.solid_definitions_snapshot.composite_solid_def_snaps } self._dagster_type_snaps_by_name_index = { dagster_type_snap.name: dagster_type_snap for dagster_type_snap in pipeline_snapshot.dagster_type_namespace_snapshot.all_dagster_type_snaps_by_key.values() if dagster_type_snap.name } self.dep_structure_index = DependencyStructureIndex( pipeline_snapshot.dep_structure_snapshot ) self._comp_dep_structures = { comp_snap.name: DependencyStructureIndex(comp_snap.dep_structure_snapshot) for comp_snap in pipeline_snapshot.solid_definitions_snapshot.composite_solid_def_snaps } if is_historical: # defer the pipeline snapshot calculation for historical pipelines. This tends to be an # expensive operation, so we want to avoid it unless we need it. Also, because this is # a historical pipeline, we already have an identifying pipeline snapshot id (which may # or may not be the same as this calculated snapshot id). The identifying pipeline # snapshot id is the calculated snapshot id at the time that the run was created. self._pipeline_snapshot_id = None else: self._pipeline_snapshot_id = create_pipeline_snapshot_id(pipeline_snapshot) @property def name(self): return self.pipeline_snapshot.name @property def description(self): return self.pipeline_snapshot.description @property def tags(self): return self.pipeline_snapshot.tags @property def pipeline_snapshot_id(self): if not self._pipeline_snapshot_id: self._pipeline_snapshot_id = create_pipeline_snapshot_id(self.pipeline_snapshot) return self._pipeline_snapshot_id def has_dagster_type_name(self, type_name): return type_name in self._dagster_type_snaps_by_name_index def get_dagster_type_from_name(self, type_name): return self._dagster_type_snaps_by_name_index[type_name] def get_node_def_snap(self, node_def_name): check.str_param(node_def_name, "node_def_name") return self._node_defs_snaps_index[node_def_name] def get_dep_structure_index(self, comp_solid_def_name): return self._comp_dep_structures[comp_solid_def_name] def get_dagster_type_snaps(self): dt_namespace = self.pipeline_snapshot.dagster_type_namespace_snapshot return list(dt_namespace.all_dagster_type_snaps_by_key.values()) def has_solid_invocation(self, solid_name): return self.dep_structure_index.has_invocation(solid_name) def get_default_mode_name(self): return self.pipeline_snapshot.mode_def_snaps[0].name def has_mode_def(self, name): check.str_param(name, "name") for mode_def_snap in self.pipeline_snapshot.mode_def_snaps: if mode_def_snap.name == name: return True return False @property def available_modes(self): return [mode_def_snap.name for mode_def_snap in self.pipeline_snapshot.mode_def_snaps] def get_mode_def_snap(self, name): check.str_param(name, "name") for mode_def_snap in self.pipeline_snapshot.mode_def_snaps: if mode_def_snap.name == name: return mode_def_snap check.failed("Mode {mode} not found".format(mode=name)) @property def config_schema_snapshot(self): return self.pipeline_snapshot.config_schema_snapshot