def execute(self):
annotations = inspect.getfullargspec(self._func).annotations
# Function arguments.
func_kwargs = {}
for k, v in annotations.items():
if k == 'return':
continue
# Annotations for parameter types could be written as, for example,
# `Optional[str]`. In this case, we need to strip off the part
# `Optional[]` to get the actual parameter type.
v = type_annotations.maybe_strip_optional_from_annotation(v)
if self._is_parameter(v):
func_kwargs[k] = self._get_input_parameter_value(k, v)
if type_annotations.is_artifact_annotation(v):
if type_annotations.is_input_artifact(v):
func_kwargs[k] = self._get_input_artifact(k)
if type_annotations.is_output_artifact(v):
func_kwargs[k] = self._get_output_artifact(k)
elif isinstance(v, type_annotations.OutputPath):
if self._is_parameter(v.type):
func_kwargs[k] = self._get_output_parameter_path(k)
else:
func_kwargs[k] = self._get_output_artifact_path(k)
elif isinstance(v, type_annotations.InputPath):
func_kwargs[k] = self._get_input_artifact_path(k)
result = self._func(**func_kwargs)
self._write_executor_output(result)
def test_is_artifact_annotation(self):
self.assertTrue(type_annotations.is_artifact_annotation(Input[Model]))
self.assertTrue(type_annotations.is_artifact_annotation(Output[Model]))
self.assertTrue(
type_annotations.is_artifact_annotation(Output['MyArtifact']))
self.assertFalse(type_annotations.is_artifact_annotation(Model))
self.assertFalse(type_annotations.is_artifact_annotation(int))
self.assertFalse(type_annotations.is_artifact_annotation('Dataset'))
self.assertFalse(type_annotations.is_artifact_annotation(List[str]))
self.assertFalse(type_annotations.is_artifact_annotation(
Optional[str]))
def execute(self):
annotations = inspect.getfullargspec(self._func).annotations
# Function arguments.
func_kwargs = {}
for k, v in annotations.items():
if k == 'return':
continue
# Annotations for parameter types could be written as, for example,
# `Optional[str]`. In this case, we need to strip off the part
# `Optional[]` to get the actual parameter type.
v = type_annotations.maybe_strip_optional_from_annotation(v)
if v == task_final_status.PipelineTaskFinalStatus:
value = self._get_input_parameter_value(k)
func_kwargs[k] = task_final_status.PipelineTaskFinalStatus(
state=value.get('state'),
pipeline_job_resource_name=value.get(
'pipelineJobResourceName'),
error_code=value.get('error').get('code', None),
error_message=value.get('error').get('message', None),
)
elif self._is_parameter(v):
value = self._get_input_parameter_value(k)
if value is not None:
func_kwargs[k] = value
elif type_annotations.is_artifact_annotation(v):
if type_annotations.is_input_artifact(v):
func_kwargs[k] = self._get_input_artifact(k)
if type_annotations.is_output_artifact(v):
func_kwargs[k] = self._get_output_artifact(k)
elif isinstance(v, type_annotations.OutputPath):
if self._is_parameter(v.type):
func_kwargs[k] = self._get_output_parameter_path(k)
else:
func_kwargs[k] = self._get_output_artifact_path(k)
elif isinstance(v, type_annotations.InputPath):
func_kwargs[k] = self._get_input_artifact_path(k)
result = self._func(**func_kwargs)
self._write_executor_output(result)
def extract_component_interface(func: Callable) -> structures.ComponentSpec:
single_output_name_const = 'Output'
signature = inspect.signature(func)
parameters = list(signature.parameters.values())
parsed_docstring = docstring_parser.parse(inspect.getdoc(func))
doc_dict = {p.arg_name: p.description for p in parsed_docstring.params}
inputs = []
outputs = []
input_names = set()
output_names = set()
for parameter in parameters:
parameter_type = type_annotation_utils.maybe_strip_optional_from_annotation(
parameter.annotation)
passing_style = None
io_name = parameter.name
if type_annotations.is_artifact_annotation(parameter_type):
# passing_style is either type_annotations.InputAnnotation or
# type_annotations.OutputAnnotation.
passing_style = type_annotations.get_io_artifact_annotation(
parameter_type)
# parameter_type is type_annotations.Artifact or one of its subclasses.
parameter_type = type_annotations.get_io_artifact_class(
parameter_type)
if not issubclass(parameter_type, artifact_types.Artifact):
raise ValueError(
'Input[T] and Output[T] are only supported when T is a '
'subclass of Artifact. Found `{} with type {}`'.format(
io_name, parameter_type))
if parameter.default is not inspect.Parameter.empty:
raise ValueError(
'Default values for Input/Output artifacts are not supported.'
)
elif isinstance(parameter_type,
(v1_components.InputPath, v1_components.OutputPath)):
raise TypeError(
'In v2 components, please import the Python function'
' annotations `InputPath` and `OutputPath` from'
' package `kfp.v2.dsl` instead of `kfp.dsl`.')
elif isinstance(
parameter_type,
(type_annotations.InputPath, type_annotations.OutputPath)):
passing_style = type(parameter_type)
parameter_type = parameter_type.type
if parameter.default is not inspect.Parameter.empty and not (
passing_style == type_annotations.InputPath
and parameter.default is None):
raise ValueError(
'Path inputs only support default values of None. Default'
' values for outputs are not supported.')
type_struct = _annotation_to_type_struct(parameter_type)
if passing_style in [
type_annotations.OutputAnnotation, type_annotations.OutputPath
]:
io_name = _maybe_make_unique(io_name, output_names)
output_names.add(io_name)
output_spec = structures.OutputSpec(name=io_name,
type=type_struct,
description=doc_dict.get(
parameter.name))
output_spec._passing_style = passing_style
output_spec._parameter_name = parameter.name
outputs.append(output_spec)
else:
io_name = _maybe_make_unique(io_name, input_names)
input_names.add(io_name)
input_spec = structures.InputSpec(name=io_name,
type=type_struct,
description=doc_dict.get(
parameter.name))
if parameter.default is not inspect.Parameter.empty:
input_spec.optional = True
if parameter.default is not None:
outer_type_name = list(
type_struct.keys())[0] if isinstance(
type_struct, dict) else type_struct
try:
input_spec.default = _data_passing.serialize_value(
parameter.default, outer_type_name)
except Exception as ex:
warnings.warn(
'Could not serialize the default value of the'
' parameter "{}". {}'.format(parameter.name, ex))
input_spec._passing_style = passing_style
input_spec._parameter_name = parameter.name
inputs.append(input_spec)
#Analyzing the return type annotations.
return_ann = signature.return_annotation
if hasattr(return_ann, '_fields'): #NamedTuple
# Getting field type annotations.
# __annotations__ does not exist in python 3.5 and earlier
# _field_types does not exist in python 3.9 and later
field_annotations = getattr(return_ann,
'__annotations__', None) or getattr(
return_ann, '_field_types', None)
for field_name in return_ann._fields:
type_struct = None
if field_annotations:
type_struct = _annotation_to_type_struct(
field_annotations.get(field_name, None))
output_name = _maybe_make_unique(field_name, output_names)
output_names.add(output_name)
output_spec = structures.OutputSpec(
name=output_name,
type=type_struct,
)
output_spec._passing_style = None
output_spec._return_tuple_field_name = field_name
outputs.append(output_spec)
# Deprecated dict-based way of declaring multiple outputs. Was only used by
# the @component decorator
elif isinstance(return_ann, dict):
warnings.warn(
"The ability to specify multiple outputs using the dict syntax"
" has been deprecated. It will be removed soon after release"
" 0.1.32. Please use typing.NamedTuple to declare multiple"
" outputs.")
for output_name, output_type_annotation in return_ann.items():
output_type_struct = _annotation_to_type_struct(
output_type_annotation)
output_spec = structures.OutputSpec(
name=output_name,
type=output_type_struct,
)
outputs.append(output_spec)
elif signature.return_annotation is not None and signature.return_annotation != inspect.Parameter.empty:
output_name = _maybe_make_unique(single_output_name_const,
output_names)
# Fixes exotic, but possible collision:
# `def func(output_path: OutputPath()) -> str: ...`
output_names.add(output_name)
type_struct = _annotation_to_type_struct(signature.return_annotation)
output_spec = structures.OutputSpec(
name=output_name,
type=type_struct,
)
output_spec._passing_style = None
outputs.append(output_spec)
# Component name and description are derived from the function's name and
# docstring. The name can be overridden by setting setting func.__name__
# attribute (of the legacy func._component_human_name attribute). The
# description can be overridden by setting the func.__doc__ attribute (or
# the legacy func._component_description attribute).
component_name = getattr(func, '_component_human_name',
None) or _python_function_name_to_component_name(
func.__name__)
description = getattr(func, '_component_description',
None) or parsed_docstring.short_description
if description:
description = description.strip()
component_spec = structures.ComponentSpec(
name=component_name,
description=description,
inputs=inputs if inputs else None,
outputs=outputs if outputs else None,
)
return component_spec