def _compare_component(component_spec: ComponentSpec, compare_path: str):
"""Compare an existing specification file to a new specification.
Args:
component_spec: A `component.yaml` specification object.
compare_path: The path of the existing specification file.
"""
# Write new spec into a temporary file
temp_spec_file = NamedTemporaryFile(mode="w", delete=False)
component_spec.save(temp_spec_file.name)
ignore_image: Callable[
[str], bool] = lambda line: not line.lstrip().startswith("image:")
with open(temp_spec_file.name, mode="r") as temp_file:
with open(compare_path, mode="r") as existing_file:
temp_lines = list(filter(ignore_image, temp_file.readlines()))
existing_lines = list(
filter(ignore_image, existing_file.readlines()))
# Cast to list to read through generator
diff_results = list(
difflib.unified_diff(
temp_lines,
existing_lines,
fromfile=temp_spec_file.name,
tofile=compare_path,
))
if len(diff_results) == 0:
return False
return "\n".join(diff_results)
def _write_component(component_spec: ComponentSpec, output_path: str):
"""Write a component YAML specification to a file.
Args:
component_spec: A `component.yaml` specification object.
output_path: The path to write the specification.
"""
component_spec.save(output_path)
def component_yaml_generator(**kwargs):
input_specs = []
input_args = []
input_kwargs = {}
serialized_args = {INIT_KEY: {}, METHOD_KEY: {}}
init_kwargs = {}
method_kwargs = {}
for key, value in kwargs.items():
if key in init_arg_names:
prefix_key = INIT_KEY
init_kwargs[key] = value
signature = init_signature
else:
prefix_key = METHOD_KEY
method_kwargs[key] = value
signature = method_signature
# no need to add this argument because it's optional
# this param is validated against the signature because
# of init_kwargs, method_kwargs
if value is None:
continue
param_type = signature.parameters[key].annotation
param_type = resolve_annotation(param_type)
serializer = get_serializer(param_type)
if serializer:
param_type = str
value = serializer(value)
# TODO remove PipelineParam check when Metadata Importer component available
# if we serialize we need to include the argument as input
# perhaps, another option is to embed in yaml as json serialized list
component_param_name = component_param_name_to_mb_sdk_param_name.get(
key, key
)
if isinstance(value,
kfp.dsl._pipeline_param.PipelineParam) or serializer:
if is_mb_sdk_resource_noun_type(param_type):
metadata_type = map_resource_to_metadata_type(param_type)[1]
component_param_type = metadata_type
else:
component_param_type = 'String'
input_specs.append(
InputSpec(
name=key,
type=component_param_type,
)
)
input_args.append(f'--{prefix_key}.{component_param_name}')
if is_mb_sdk_resource_noun_type(param_type):
input_args.append(InputUriPlaceholder(input_name=key))
else:
input_args.append(InputValuePlaceholder(input_name=key))
input_kwargs[key] = value
else:
# Serialized arguments must always be strings
value = str(value)
serialized_args[prefix_key][component_param_name] = value
# validate parameters
if should_serialize_init:
init_signature.bind(**init_kwargs)
method_signature.bind(**method_kwargs)
component_spec = ComponentSpec(
name=f'{cls_name}-{method_name}',
inputs=input_specs,
outputs=output_specs,
implementation=ContainerImplementation(
container=ContainerSpec(
image=DEFAULT_CONTAINER_IMAGE,
command=[
'python3',
'-m',
'google_cloud_pipeline_components.aiplatform.remote_runner',
'--cls_name',
cls_name,
'--method_name',
method_name,
],
args=make_args(serialized_args) + output_args + input_args,
)
)
)
component_path = tempfile.mktemp()
component_spec.save(component_path)
return components.load_component_from_file(component_path)(
**input_kwargs
)