def test_parse_solid_selection_invalid():
with pytest.raises(
DagsterInvalidSubsetError,
match="No qualified solids to execute found for solid_selection",
):
parse_solid_selection(foo_pipeline, ["some,solid"])
def test_parse_solid_selection_single():
solid_selection_single = parse_solid_selection(foo_pipeline, ["add_nums"])
assert len(solid_selection_single) == 1
assert solid_selection_single == {"add_nums"}
solid_selection_star = parse_solid_selection(foo_pipeline, ["add_nums*"])
assert len(solid_selection_star) == 3
assert set(solid_selection_star) == {"add_nums", "multiply_two", "add_one"}
solid_selection_both = parse_solid_selection(foo_pipeline, ["*add_nums+"])
assert len(solid_selection_both) == 4
assert set(solid_selection_both) == {"return_one", "return_two", "add_nums", "multiply_two"}
def test_parse_solid_selection_single():
solid_selection_single = parse_solid_selection(foo_pipeline, ['add_nums'])
assert len(solid_selection_single) == 1
assert solid_selection_single == {'add_nums'}
solid_selection_star = parse_solid_selection(foo_pipeline, ['add_nums*'])
assert len(solid_selection_star) == 3
assert set(solid_selection_star) == {'add_nums', 'multiply_two', 'add_one'}
solid_selection_both = parse_solid_selection(foo_pipeline, ['*add_nums+'])
assert len(solid_selection_both) == 4
assert set(solid_selection_both) == {
'return_one', 'return_two', 'add_nums', 'multiply_two'
}
def test_parse_solid_selection_multi():
solid_selection_multi_disjoint = parse_solid_selection(
foo_pipeline, ["return_one", "add_nums+"]
)
assert len(solid_selection_multi_disjoint) == 3
assert set(solid_selection_multi_disjoint) == {"return_one", "add_nums", "multiply_two"}
solid_selection_multi_overlap = parse_solid_selection(
foo_pipeline, ["*add_nums", "return_one+"]
)
assert len(solid_selection_multi_overlap) == 3
assert set(solid_selection_multi_overlap) == {"return_one", "return_two", "add_nums"}
with pytest.raises(
DagsterInvalidSubsetError, match="No qualified solids to execute found for solid_selection",
):
parse_solid_selection(foo_pipeline, ["*add_nums", "a"])
def test_parse_solid_selection_multi():
solid_selection_multi_disjoint = parse_solid_selection(
foo_pipeline, ['return_one', 'add_nums+'])
assert len(solid_selection_multi_disjoint) == 3
assert set(solid_selection_multi_disjoint) == {
'return_one', 'add_nums', 'multiply_two'
}
solid_selection_multi_overlap = parse_solid_selection(
foo_pipeline, ['*add_nums', 'return_one+'])
assert len(solid_selection_multi_overlap) == 3
assert set(solid_selection_multi_overlap) == {
'return_one', 'return_two', 'add_nums'
}
solid_selection_multi_with_invalid = parse_solid_selection(
foo_pipeline, ['*add_nums', 'a'])
assert len(solid_selection_multi_with_invalid) == 3
assert set(solid_selection_multi_with_invalid) == {
'return_one', 'return_two', 'add_nums'
}
def test_parse_solid_selection_multi():
solid_selection_multi_disjoint = parse_solid_selection(
foo_pipeline, ["return_one", "add_nums+"])
assert len(solid_selection_multi_disjoint) == 3
assert set(solid_selection_multi_disjoint) == {
"return_one", "add_nums", "multiply_two"
}
solid_selection_multi_overlap = parse_solid_selection(
foo_pipeline, ["*add_nums", "return_one+"])
assert len(solid_selection_multi_overlap) == 3
assert set(solid_selection_multi_overlap) == {
"return_one", "return_two", "add_nums"
}
solid_selection_multi_with_invalid = parse_solid_selection(
foo_pipeline, ["*add_nums", "a"])
assert len(solid_selection_multi_with_invalid) == 3
assert set(solid_selection_multi_with_invalid) == {
"return_one", "return_two", "add_nums"
}
def get_job_def_for_op_selection(
self,
op_selection: Optional[List[str]] = None,
) -> "JobDefinition":
if not op_selection:
return self
op_selection = check.opt_list_param(op_selection, "op_selection", str)
# TODO: https://github.com/dagster-io/dagster/issues/2115
# op selection currently still operates on PipelineSubsetDefinition. ideally we'd like to
# 1) move away from creating PipelineSubsetDefinition
# 2) consolidate solid selection and step selection
# 3) enable subsetting nested graphs/ops which isn't working with the current setting
solids_to_execute = (self._op_selection_data.resolved_op_selection
if self._op_selection_data else None)
if op_selection:
solids_to_execute = parse_solid_selection(
super(JobDefinition,
self).get_pipeline_subset_def(solids_to_execute),
op_selection,
)
subset_pipeline_def = super(
JobDefinition, self).get_pipeline_subset_def(solids_to_execute)
ignored_solids = [
solid for solid in self._graph_def.solids
if not subset_pipeline_def.has_solid_named(solid.name)
]
return JobDefinition(
name=self.name,
description=self.description,
mode_def=self.get_mode_definition(),
preset_defs=self.preset_defs,
tags=self.tags,
hook_defs=self.hook_defs,
op_retry_policy=self._solid_retry_policy,
graph_def=subset_pipeline_def.graph,
version_strategy=self.version_strategy,
_op_selection_data=OpSelectionData(
op_selection=op_selection,
resolved_op_selection=solids_to_execute,
ignored_solids=ignored_solids,
parent_job_def=self,
),
)
def test_parse_solid_selection_invalid():
result = parse_solid_selection(foo_pipeline, ['1+some_solid'])
assert result == set()
result_comma = parse_solid_selection(foo_pipeline, ['some,solid'])
assert result_comma == set()
