def test_parse_solid_selection_invalid(): with pytest.raises( DagsterInvalidSubsetError, match="No qualified solids to execute found for solid_selection", ): parse_solid_selection(foo_pipeline, ["some,solid"])
def test_parse_solid_selection_single(): solid_selection_single = parse_solid_selection(foo_pipeline, ["add_nums"]) assert len(solid_selection_single) == 1 assert solid_selection_single == {"add_nums"} solid_selection_star = parse_solid_selection(foo_pipeline, ["add_nums*"]) assert len(solid_selection_star) == 3 assert set(solid_selection_star) == {"add_nums", "multiply_two", "add_one"} solid_selection_both = parse_solid_selection(foo_pipeline, ["*add_nums+"]) assert len(solid_selection_both) == 4 assert set(solid_selection_both) == {"return_one", "return_two", "add_nums", "multiply_two"}
def test_parse_solid_selection_single(): solid_selection_single = parse_solid_selection(foo_pipeline, ['add_nums']) assert len(solid_selection_single) == 1 assert solid_selection_single == {'add_nums'} solid_selection_star = parse_solid_selection(foo_pipeline, ['add_nums*']) assert len(solid_selection_star) == 3 assert set(solid_selection_star) == {'add_nums', 'multiply_two', 'add_one'} solid_selection_both = parse_solid_selection(foo_pipeline, ['*add_nums+']) assert len(solid_selection_both) == 4 assert set(solid_selection_both) == { 'return_one', 'return_two', 'add_nums', 'multiply_two' }
def test_parse_solid_selection_multi(): solid_selection_multi_disjoint = parse_solid_selection( foo_pipeline, ["return_one", "add_nums+"] ) assert len(solid_selection_multi_disjoint) == 3 assert set(solid_selection_multi_disjoint) == {"return_one", "add_nums", "multiply_two"} solid_selection_multi_overlap = parse_solid_selection( foo_pipeline, ["*add_nums", "return_one+"] ) assert len(solid_selection_multi_overlap) == 3 assert set(solid_selection_multi_overlap) == {"return_one", "return_two", "add_nums"} with pytest.raises( DagsterInvalidSubsetError, match="No qualified solids to execute found for solid_selection", ): parse_solid_selection(foo_pipeline, ["*add_nums", "a"])
def test_parse_solid_selection_multi(): solid_selection_multi_disjoint = parse_solid_selection( foo_pipeline, ['return_one', 'add_nums+']) assert len(solid_selection_multi_disjoint) == 3 assert set(solid_selection_multi_disjoint) == { 'return_one', 'add_nums', 'multiply_two' } solid_selection_multi_overlap = parse_solid_selection( foo_pipeline, ['*add_nums', 'return_one+']) assert len(solid_selection_multi_overlap) == 3 assert set(solid_selection_multi_overlap) == { 'return_one', 'return_two', 'add_nums' } solid_selection_multi_with_invalid = parse_solid_selection( foo_pipeline, ['*add_nums', 'a']) assert len(solid_selection_multi_with_invalid) == 3 assert set(solid_selection_multi_with_invalid) == { 'return_one', 'return_two', 'add_nums' }
def test_parse_solid_selection_multi(): solid_selection_multi_disjoint = parse_solid_selection( foo_pipeline, ["return_one", "add_nums+"]) assert len(solid_selection_multi_disjoint) == 3 assert set(solid_selection_multi_disjoint) == { "return_one", "add_nums", "multiply_two" } solid_selection_multi_overlap = parse_solid_selection( foo_pipeline, ["*add_nums", "return_one+"]) assert len(solid_selection_multi_overlap) == 3 assert set(solid_selection_multi_overlap) == { "return_one", "return_two", "add_nums" } solid_selection_multi_with_invalid = parse_solid_selection( foo_pipeline, ["*add_nums", "a"]) assert len(solid_selection_multi_with_invalid) == 3 assert set(solid_selection_multi_with_invalid) == { "return_one", "return_two", "add_nums" }
def get_job_def_for_op_selection( self, op_selection: Optional[List[str]] = None, ) -> "JobDefinition": if not op_selection: return self op_selection = check.opt_list_param(op_selection, "op_selection", str) # TODO: https://github.com/dagster-io/dagster/issues/2115 # op selection currently still operates on PipelineSubsetDefinition. ideally we'd like to # 1) move away from creating PipelineSubsetDefinition # 2) consolidate solid selection and step selection # 3) enable subsetting nested graphs/ops which isn't working with the current setting solids_to_execute = (self._op_selection_data.resolved_op_selection if self._op_selection_data else None) if op_selection: solids_to_execute = parse_solid_selection( super(JobDefinition, self).get_pipeline_subset_def(solids_to_execute), op_selection, ) subset_pipeline_def = super( JobDefinition, self).get_pipeline_subset_def(solids_to_execute) ignored_solids = [ solid for solid in self._graph_def.solids if not subset_pipeline_def.has_solid_named(solid.name) ] return JobDefinition( name=self.name, description=self.description, mode_def=self.get_mode_definition(), preset_defs=self.preset_defs, tags=self.tags, hook_defs=self.hook_defs, op_retry_policy=self._solid_retry_policy, graph_def=subset_pipeline_def.graph, version_strategy=self.version_strategy, _op_selection_data=OpSelectionData( op_selection=op_selection, resolved_op_selection=solids_to_execute, ignored_solids=ignored_solids, parent_job_def=self, ), )
def test_parse_solid_selection_invalid(): result = parse_solid_selection(foo_pipeline, ['1+some_solid']) assert result == set() result_comma = parse_solid_selection(foo_pipeline, ['some,solid']) assert result_comma == set()