def test_simple_component_spec_save_to_component_yaml(self): # tests writing old style (less verbose) and reading in new style (more verbose) original_component_spec = structures.ComponentSpec( name='component_1', implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', command=[ 'sh', '-c', 'set -ex\necho "$0" > "$1"', structures.InputValuePlaceholder(input_name='input1'), structures.OutputParameterPlaceholder( output_name='output1'), ], )), inputs={'input1': structures.InputSpec(type='String')}, outputs={'output1': structures.OutputSpec(type='String')}, ) from kfp.components import yaml_component yaml_component = yaml_component.YamlComponent( component_spec=original_component_spec) with tempfile.TemporaryDirectory() as tempdir: output_path = os.path.join(tempdir, 'component.yaml') compiler.Compiler().compile(yaml_component, output_path) # test that it can be read back correctly with open(output_path, 'r') as f: contents = f.read() new_component_spec = structures.ComponentSpec.load_from_component_yaml( contents) self.assertEqual(original_component_spec, new_component_spec)
def test_component_spec_with_placeholder_referencing_nonexisting_input_output( self): with self.assertRaisesRegex( ValueError, r'^Argument \"InputValuePlaceholder[\s\S]*\'input000\'[\s\S]*references non-existing input.' ): structures.ComponentSpec( name='component_1', implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', command=[ 'sh', '-c', 'set -ex\necho "$0" > "$1"', structures.InputValuePlaceholder( input_name='input000'), structures.OutputPathPlaceholder( output_name='output1'), ], )), inputs={'input1': structures.InputSpec(type='String')}, outputs={'output1': structures.OutputSpec(type='String')}, ) with self.assertRaisesRegex( ValueError, r'^Argument \"OutputPathPlaceholder[\s\S]*\'output000\'[\s\S]*references non-existing output.' ): structures.ComponentSpec( name='component_1', implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', command=[ 'sh', '-c', 'set -ex\necho "$0" > "$1"', structures.InputValuePlaceholder( input_name='input1'), structures.OutputPathPlaceholder( output_name='output000'), ], )), inputs={'input1': structures.InputSpec(type='String')}, outputs={'output1': structures.OutputSpec(type='String')}, )
def test_simple_component_spec_save_to_component_yaml(self): open_mock = mock.mock_open() expected_yaml = textwrap.dedent("""\ implementation: container: command: - sh - -c - 'set -ex echo "$0" > "$1"' - {inputValue: input1} - {outputPath: output1} image: alpine inputs: input1: {type: String} name: component_1 outputs: output1: {type: String} """) with mock.patch( "builtins.open", open_mock, create=True), self.assertWarnsRegex( DeprecationWarning, r"Compiling to JSON is deprecated"): structures.ComponentSpec( name='component_1', implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', command=[ 'sh', '-c', 'set -ex\necho "$0" > "$1"', structures.InputValuePlaceholder( input_name='input1'), structures.OutputPathPlaceholder( output_name='output1'), ], )), inputs={ 'input1': structures.InputSpec(type='String') }, outputs={ 'output1': structures.OutputSpec(type='String') }, ).save_to_component_yaml('test_save_file.json') open_mock.assert_called_once_with('test_save_file.json', 'w')
def test_from_container_dict_no_placeholders(self): component_spec = structures.ComponentSpec( name='test', implementation=structures.Implementation( container=structures.ContainerSpec( image='python:3.7', command=[ 'sh', '-c', '\nif ! [ -x "$(command -v pip)" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location \'kfp==2.0.0-alpha.2\' && "$0" "$@"\n', 'sh', '-ec', 'program_path=$(mktemp -d)\nprintf "%s" "$0" > "$program_path/ephemeral_component.py"\npython3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@"\n', '\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef concat_message(first: str, second: str) -> str:\n return first + second\n\n' ], args=[ '--executor_input', '{{$}}', '--function_to_execute', 'concat_message' ], env=None, resources=None), graph=None, importer=None), description=None, inputs={ 'first': structures.InputSpec(type='String', default=None), 'second': structures.InputSpec(type='String', default=None) }, outputs={'Output': structures.OutputSpec(type='String')}) container_dict = { 'args': [ '--executor_input', '{{$}}', '--function_to_execute', 'fail_op' ], 'command': [ 'sh', '-c', '\nif ! [ -x "$(command -v pip)" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location \'kfp==2.0.0-alpha.2\' && "$0" "$@"\n', 'sh', '-ec', 'program_path=$(mktemp -d)\nprintf "%s" "$0" > "$program_path/ephemeral_component.py"\npython3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@"\n', '\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef fail_op(message: str):\n """Fails."""\n import sys\n print(message)\n sys.exit(1)\n\n' ], 'image': 'python:3.7' } loaded_container_spec = structures.ContainerSpec.from_container_dict( container_dict)
def test_create_pipeline_task_valid(self): expected_component_spec = structures.ComponentSpec( name='component1', implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', command=['sh', '-c', 'echo "$0" >> "$1"'], args=[ structures.InputValuePlaceholder(input_name='input1'), structures.OutputPathPlaceholder( output_name='output1'), ], )), inputs={ 'input1': structures.InputSpec(type='String'), }, outputs={ 'output1': structures.OutputSpec(type='Artifact'), }, ) expected_task_spec = structures.TaskSpec( name='component1', inputs={'input1': 'value'}, dependent_tasks=[], component_ref='component1', ) expected_container_spec = structures.ContainerSpec( image='alpine', command=['sh', '-c', 'echo "$0" >> "$1"'], args=[ "{{$.inputs.parameters['input1']}}", "{{$.outputs.artifacts['output1'].path}}", ], ) task = pipeline_task.PipelineTask( component_spec=structures.ComponentSpec.load_from_component_yaml( V2_YAML), args={'input1': 'value'}, ) self.assertEqual(task.task_spec, expected_task_spec) self.assertEqual(task.component_spec, expected_component_spec) self.assertEqual(task.container_spec, expected_container_spec)
def test_simple_component_spec_load_from_v2_component_yaml(self): component_yaml_v2 = textwrap.dedent("""\ name: component_1 inputs: input1: type: String outputs: output1: type: String implementation: container: image: alpine command: - sh - -c - 'set -ex echo "$0" > "$1"' - inputValue: input1 - outputPath: output1 """) generated_spec = structures.ComponentSpec.load_from_component_yaml( component_yaml_v2) expected_spec = structures.ComponentSpec( name='component_1', implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', command=[ 'sh', '-c', 'set -ex\necho "$0" > "$1"', structures.InputValuePlaceholder(input_name='input1'), structures.OutputPathPlaceholder( output_name='output1'), ], )), inputs={'input1': structures.InputSpec(type='String')}, outputs={'output1': structures.OutputSpec(type='String')}) self.assertEqual(generated_spec, expected_spec)
def importer( artifact_uri: Union[pipeline_channel.PipelineParameterChannel, str], artifact_class: Type[artifact_types.Artifact], reimport: bool = False, metadata: Optional[Mapping[str, Any]] = None, ) -> pipeline_task.PipelineTask: """dsl.importer for importing an existing artifact. Only for v2 pipeline. Args: artifact_uri: The artifact uri to import from. artifact_type_schema: The user specified artifact type schema of the artifact to be imported. reimport: Whether to reimport the artifact. Defaults to False. metadata: Properties of the artifact. Returns: A PipelineTask instance. Raises: ValueError if the passed in artifact_uri is neither a PipelineParam nor a constant string value. """ component_spec = structures.ComponentSpec( name='importer', implementation=structures.Implementation( importer=structures.ImporterSpec( artifact_uri=structures.InputValuePlaceholder( INPUT_KEY).to_placeholder(), type_schema=artifact_class.TYPE_NAME, reimport=reimport, metadata=metadata)), inputs={INPUT_KEY: structures.InputSpec(type='String')}, outputs={ OUTPUT_KEY: structures.OutputSpec(type=artifact_class.__name__) }, ) importer = importer_component.ImporterComponent( component_spec=component_spec) return importer(uri=artifact_uri)
class TestComponent(base_component.BaseComponent): def execute(self, *args, **kwargs): pass component_op = TestComponent(component_spec=structures.ComponentSpec( name='component_1', implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', command=[ 'sh', '-c', 'set -ex\necho "$0" "$1" "$2" > "$3"', structures.InputValuePlaceholder(input_name='input1'), structures.InputValuePlaceholder(input_name='input2'), structures.InputValuePlaceholder(input_name='input3'), structures.OutputPathPlaceholder(output_name='output1'), ], )), inputs={ 'input1': structures.InputSpec(type='String'), 'input2': structures.InputSpec(type='Integer'), 'input3': structures.InputSpec(type='Float', default=3.14), 'input4': structures.InputSpec(type='Optional[Float]', default=None), }, outputs={ 'output1': structures.OutputSpec(type='String'), },
def test_if_placeholder(self): compiled_yaml = textwrap.dedent(""" components: comp-if: executorLabel: exec-if inputDefinitions: parameters: optional_input_1: parameterType: STRING deploymentSpec: executors: exec-if: container: args: - 'input: ' - '{{$.inputs.parameters[''optional_input_1'']}}' command: - sh - -c - echo "$0" "$1" image: alpine pipelineInfo: name: if root: dag: tasks: if: cachingOptions: enableCache: true componentRef: name: comp-if inputs: parameters: optional_input_1: componentInputParameter: optional_input_1 taskInfo: name: if inputDefinitions: parameters: optional_input_1: parameterType: STRING schemaVersion: 2.1.0 sdkVersion: kfp-2.0.0-alpha.2""") loaded_component_spec = structures.ComponentSpec.load_from_component_yaml( compiled_yaml) component_spec = structures.ComponentSpec( name='if', implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', command=['sh', '-c', 'echo "$0" "$1"'], args=[ 'input: ', structures.InputValuePlaceholder( input_name='optional_input_1') ], env=None, resources=None), graph=None, importer=None), description=None, inputs={ 'optional_input_1': structures.InputSpec(type='String', default=None) }, outputs=None) self.assertEqual(loaded_component_spec, component_spec)
def test_simple_placeholder(self): compiled_yaml = textwrap.dedent(""" components: comp-component1: executorLabel: exec-component1 inputDefinitions: parameters: input1: parameterType: STRING outputDefinitions: artifacts: output1: artifactType: schemaTitle: system.Artifact schemaVersion: 0.0.1 deploymentSpec: executors: exec-component1: container: args: - '{{$.inputs.parameters[''input1'']}}' - '{{$.outputs.artifacts[''output1''].path}}' command: - sh - -c - echo "$0" >> "$1" image: alpine pipelineInfo: name: component1 root: dag: tasks: component1: cachingOptions: enableCache: true componentRef: name: comp-component1 inputs: parameters: input1: componentInputParameter: input1 taskInfo: name: component1 inputDefinitions: parameters: input1: parameterType: STRING schemaVersion: 2.1.0 sdkVersion: kfp-2.0.0-alpha.2""") loaded_component_spec = structures.ComponentSpec.load_from_component_yaml( compiled_yaml) component_spec = structures.ComponentSpec( name='component1', implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', command=['sh', '-c', 'echo "$0" >> "$1"'], args=[ structures.InputValuePlaceholder(input_name='input1'), structures.OutputPathPlaceholder(output_name='output1') ], env=None, resources=None), graph=None, importer=None), description=None, inputs={ 'input1': structures.InputSpec(type='String', default=None) }, outputs={'output1': structures.OutputSpec(type='Artifact')}) self.assertEqual(loaded_component_spec, component_spec)
def test_outputs(self): obj = structures.ComponentSpec( name='name', implementation=structures.Implementation(container=None), outputs={}) self.assertEqual(obj.outputs, None)
def test_component_spec_load_from_v1_component_yaml(self): component_yaml_v1 = textwrap.dedent("""\ name: Component with 2 inputs and 2 outputs inputs: - {name: Input parameter, type: String} - {name: Input artifact} outputs: - {name: Output 1} - {name: Output 2} implementation: container: image: busybox command: [sh, -c, ' mkdir -p $(dirname "$2") mkdir -p $(dirname "$3") echo "$0" > "$2" cp "$1" "$3" ' ] args: - {inputValue: Input parameter} - {inputPath: Input artifact} - {outputPath: Output 1} - {outputPath: Output 2} """) generated_spec = structures.ComponentSpec.load_from_component_yaml( component_yaml_v1) expected_spec = structures.ComponentSpec( name='Component with 2 inputs and 2 outputs', implementation=structures.Implementation( container=structures.ContainerSpec( image='busybox', command=[ 'sh', '-c', (' mkdir -p $(dirname "$2") mkdir -p $(dirname "$3") ' 'echo "$0" > "$2" cp "$1" "$3" '), ], args=[ structures.InputValuePlaceholder( input_name='input_parameter'), structures.InputPathPlaceholder( input_name='input_artifact'), structures.OutputPathPlaceholder( output_name='output_1'), structures.OutputPathPlaceholder( output_name='output_2'), ], env={}, )), inputs={ 'input_parameter': structures.InputSpec(type='String'), 'input_artifact': structures.InputSpec(type='Artifact') }, outputs={ 'output_1': structures.OutputSpec(type='Artifact'), 'output_2': structures.OutputSpec(type='Artifact'), }) self.assertEqual(generated_spec, expected_spec)
def test_simple_component_spec_load_from_v2_component_yaml(self): component_yaml_v2 = textwrap.dedent("""\ components: comp-component-1: executorLabel: exec-component-1 inputDefinitions: parameters: input1: parameterType: STRING outputDefinitions: parameters: output1: parameterType: STRING deploymentSpec: executors: exec-component-1: container: command: - sh - -c - 'set -ex echo "$0" > "$1"' - '{{$.inputs.parameters[''input1'']}}' - '{{$.outputs.parameters[''output1''].output_file}}' image: alpine pipelineInfo: name: component-1 root: dag: tasks: component-1: cachingOptions: enableCache: true componentRef: name: comp-component-1 inputs: parameters: input1: componentInputParameter: input1 taskInfo: name: component-1 inputDefinitions: parameters: input1: parameterType: STRING schemaVersion: 2.1.0 sdkVersion: kfp-2.0.0-alpha.2 """) generated_spec = structures.ComponentSpec.load_from_component_yaml( component_yaml_v2) expected_spec = structures.ComponentSpec( name='component-1', implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', command=[ 'sh', '-c', 'set -ex\necho "$0" > "$1"', structures.InputValuePlaceholder(input_name='input1'), structures.OutputParameterPlaceholder( output_name='output1'), ], )), inputs={'input1': structures.InputSpec(type='String')}, outputs={'output1': structures.OutputSpec(type='String')}) self.assertEqual(generated_spec, expected_spec)
inputs: - {name: optional_input_1, optional: true, type: String} name: component_if """) COMPONENT_SPEC_IF_PLACEHOLDER = structures.ComponentSpec( name='component_if', implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', args=[ structures.IfPresentPlaceholder( if_structure=structures.IfPresentPlaceholderStructure( input_name='optional_input_1', then=[ '--arg1', structures.InputUriPlaceholder( input_name='optional_input_1'), ], otherwise=[ '--arg2', 'default', ])) ])), inputs={ 'optional_input_1': structures.InputSpec(type='String', default=None) }, ) V1_YAML_CONCAT_PLACEHOLDER = textwrap.dedent("""\ name: component_concat
def create_component_from_func(func: Callable, base_image: Optional[str] = None, target_image: Optional[str] = None, packages_to_install: List[str] = None, pip_index_urls: Optional[List[str]] = None, output_component_file: Optional[str] = None, install_kfp_package: bool = True, kfp_package_path: Optional[str] = None): """Implementation for the @component decorator. The decorator is defined under component_decorator.py. See the decorator for the canonical documentation for this function. """ packages_to_install = packages_to_install or [] if install_kfp_package and target_image is None: if kfp_package_path is None: kfp_package_path = _get_default_kfp_package_path() packages_to_install.append(kfp_package_path) packages_to_install_command = _get_packages_to_install_command( package_list=packages_to_install, pip_index_urls=pip_index_urls) command = [] args = [] if base_image is None: base_image = _DEFAULT_BASE_IMAGE component_image = base_image if target_image: component_image = target_image command, args = _get_command_and_args_for_containerized_component( function_name=func.__name__, ) else: command, args = _get_command_and_args_for_lightweight_component( func=func) component_spec = extract_component_interface(func) component_spec.implementation = structures.Implementation( container=structures.ContainerSpec( image=component_image, command=packages_to_install_command + command, args=args, )) module_path = pathlib.Path(inspect.getsourcefile(func)) module_path.resolve() component_name = _python_function_name_to_component_name(func.__name__) component_info = ComponentInfo(name=component_name, function_name=func.__name__, func=func, target_image=target_image, module_path=module_path, component_spec=component_spec, output_component_file=output_component_file, base_image=base_image) if REGISTERED_MODULES is not None: REGISTERED_MODULES[component_name] = component_info if output_component_file: component_spec.save_to_component_yaml(output_component_file) return python_component.PythonComponent(component_spec=component_spec, python_func=func)
def extract_component_interface(func: Callable) -> structures.ComponentSpec: single_output_name_const = 'Output' signature = inspect.signature(func) parameters = list(signature.parameters.values()) parsed_docstring = docstring_parser.parse(inspect.getdoc(func)) doc_dict = {p.arg_name: p.description for p in parsed_docstring.params} inputs = {} outputs = {} input_names = set() output_names = set() for parameter in parameters: parameter_type = type_annotations.maybe_strip_optional_from_annotation( parameter.annotation) passing_style = None io_name = parameter.name if type_annotations.is_artifact_annotation(parameter_type): # passing_style is either type_annotations.InputAnnotation or # type_annotations.OutputAnnotation. passing_style = type_annotations.get_io_artifact_annotation( parameter_type) # parameter_type is type_annotations.Artifact or one of its subclasses. parameter_type = type_annotations.get_io_artifact_class( parameter_type) if not issubclass(parameter_type, artifact_types.Artifact): raise ValueError( 'Input[T] and Output[T] are only supported when T is a ' 'subclass of Artifact. Found `{} with type {}`'.format( io_name, parameter_type)) if parameter.default is not inspect.Parameter.empty: raise ValueError( 'Default values for Input/Output artifacts are not supported.' ) elif isinstance( parameter_type, (type_annotations.InputPath, type_annotations.OutputPath)): passing_style = type(parameter_type) parameter_type = parameter_type.type if parameter.default is not inspect.Parameter.empty and not ( passing_style == type_annotations.InputPath and parameter.default is None): raise ValueError( 'Path inputs only support default values of None. Default' ' values for outputs are not supported.') type_struct = _annotation_to_type_struct(parameter_type) if type_struct is None: raise TypeError('Missing type annotation for argument: {}'.format( parameter.name)) if passing_style in [ type_annotations.OutputAnnotation, type_annotations.OutputPath ]: io_name = _maybe_make_unique(io_name, output_names) output_names.add(io_name) output_spec = structures.OutputSpec(type=type_struct, description=doc_dict.get( parameter.name)) outputs[io_name] = output_spec else: io_name = _maybe_make_unique(io_name, input_names) input_names.add(io_name) if parameter.default is not inspect.Parameter.empty: input_spec = structures.InputSpec( type=type_struct, description=doc_dict.get(parameter.name), default=parameter.default, ) else: input_spec = structures.InputSpec( type=type_struct, description=doc_dict.get(parameter.name), ) inputs[io_name] = input_spec #Analyzing the return type annotations. return_ann = signature.return_annotation if hasattr(return_ann, '_fields'): #NamedTuple # Getting field type annotations. # __annotations__ does not exist in python 3.5 and earlier # _field_types does not exist in python 3.9 and later field_annotations = getattr(return_ann, '__annotations__', None) or getattr( return_ann, '_field_types', None) for field_name in return_ann._fields: type_struct = None if field_annotations: type_struct = _annotation_to_type_struct( field_annotations.get(field_name, None)) output_name = _maybe_make_unique(field_name, output_names) output_names.add(output_name) output_spec = structures.OutputSpec(type=type_struct) outputs[output_name] = output_spec # Deprecated dict-based way of declaring multiple outputs. Was only used by # the @component decorator elif isinstance(return_ann, dict): warnings.warn( 'The ability to specify multiple outputs using the dict syntax' ' has been deprecated. It will be removed soon after release' ' 0.1.32. Please use typing.NamedTuple to declare multiple' ' outputs.') for output_name, output_type_annotation in return_ann.items(): output_type_struct = _annotation_to_type_struct( output_type_annotation) output_spec = structures.OutputSpec(type=output_type_struct) outputs[name] = output_spec elif signature.return_annotation is not None and signature.return_annotation != inspect.Parameter.empty: output_name = _maybe_make_unique(single_output_name_const, output_names) # Fixes exotic, but possible collision: # `def func(output_path: OutputPath()) -> str: ...` output_names.add(output_name) type_struct = _annotation_to_type_struct(signature.return_annotation) output_spec = structures.OutputSpec(type=type_struct) outputs[output_name] = output_spec # Component name and description are derived from the function's name and # docstring. The name can be overridden by setting setting func.__name__ # attribute (of the legacy func._component_human_name attribute). The # description can be overridden by setting the func.__doc__ attribute (or # the legacy func._component_description attribute). component_name = getattr(func, '_component_human_name', None) or _python_function_name_to_component_name( func.__name__) description = getattr(func, '_component_description', None) or parsed_docstring.short_description if description: description = description.strip() component_spec = structures.ComponentSpec( name=component_name, description=description, inputs=inputs if inputs else None, outputs=outputs if outputs else None, # Dummy implementation to bypass model validation. implementation=structures.Implementation(), ) return component_spec