def test_get_transcode_compatible_name_multiple_separators(self):
dataset_name = "mydata@formA@formB"
pattern = "Expected maximum 1 transcoding separator, "
pattern += "found 2 instead: 'mydata@formA@formB'"
with pytest.raises(ValueError, match=pattern):
_strip_transcoding(dataset_name)
def _validate_datasets_exist(
inputs: AbstractSet[str],
outputs: AbstractSet[str],
parameters: AbstractSet[str],
pipe: Pipeline,
) -> None:
inputs = {_strip_transcoding(k) for k in inputs}
outputs = {_strip_transcoding(k) for k in outputs}
existing = {_strip_transcoding(ds) for ds in pipe.data_sets()}
non_existent = (inputs | outputs | parameters) - existing
if non_existent:
raise ModularPipelineError(
"Failed to map datasets and/or parameters: {}".format(", ".join(
sorted(non_existent))))
def _validate_inputs_outputs(
inputs: AbstractSet[str], outputs: AbstractSet[str], pipe: Pipeline
) -> None:
"""Safeguards to ensure that:
- parameters are not specified under inputs
- inputs are only free inputs
- outputs do not contain free inputs
"""
inputs = {_strip_transcoding(k) for k in inputs}
outputs = {_strip_transcoding(k) for k in outputs}
if any(_is_parameter(i) for i in inputs):
raise ModularPipelineError(
"Parameters should be specified in the `parameters` argument"
)
free_inputs = {_strip_transcoding(i) for i in pipe.inputs()}
if not inputs <= free_inputs:
raise ModularPipelineError("Inputs should be free inputs to the pipeline")
if outputs & free_inputs:
raise ModularPipelineError("Outputs can't contain free inputs to the pipeline")
def test_get_transcode_compatible_name_no_separator(self):
dataset_name = "mydata"
assert _strip_transcoding(dataset_name) == dataset_name
def test_get_transcode_compatible_name(self):
dataset_name = "mydata@pandas"
assert _strip_transcoding(dataset_name) == "mydata"
def test_get_transcode_compatible_name(self):
assert _strip_transcoding("abc@def") == "abc"
