def test_get_transcode_compatible_name_multiple_separators(self): dataset_name = "mydata@formA@formB" pattern = "Expected maximum 1 transcoding separator, " pattern += "found 2 instead: 'mydata@formA@formB'" with pytest.raises(ValueError, match=pattern): _strip_transcoding(dataset_name)
def _validate_datasets_exist( inputs: AbstractSet[str], outputs: AbstractSet[str], parameters: AbstractSet[str], pipe: Pipeline, ) -> None: inputs = {_strip_transcoding(k) for k in inputs} outputs = {_strip_transcoding(k) for k in outputs} existing = {_strip_transcoding(ds) for ds in pipe.data_sets()} non_existent = (inputs | outputs | parameters) - existing if non_existent: raise ModularPipelineError( "Failed to map datasets and/or parameters: {}".format(", ".join( sorted(non_existent))))
def _validate_inputs_outputs( inputs: AbstractSet[str], outputs: AbstractSet[str], pipe: Pipeline ) -> None: """Safeguards to ensure that: - parameters are not specified under inputs - inputs are only free inputs - outputs do not contain free inputs """ inputs = {_strip_transcoding(k) for k in inputs} outputs = {_strip_transcoding(k) for k in outputs} if any(_is_parameter(i) for i in inputs): raise ModularPipelineError( "Parameters should be specified in the `parameters` argument" ) free_inputs = {_strip_transcoding(i) for i in pipe.inputs()} if not inputs <= free_inputs: raise ModularPipelineError("Inputs should be free inputs to the pipeline") if outputs & free_inputs: raise ModularPipelineError("Outputs can't contain free inputs to the pipeline")
def test_get_transcode_compatible_name_no_separator(self): dataset_name = "mydata" assert _strip_transcoding(dataset_name) == dataset_name
def test_get_transcode_compatible_name(self): dataset_name = "mydata@pandas" assert _strip_transcoding(dataset_name) == "mydata"
def test_get_transcode_compatible_name(self): assert _strip_transcoding("abc@def") == "abc"