def component(func: Optional[Callable] = None,
*,
base_image: Optional[str] = None,
packages_to_install: List[str] = None,
output_component_file: Optional[str] = None):
"""Decorator for Python-function based components in v2.
Example usage:
from kfp.v2 import dsl
@dsl.component
def my_function_one(input: str, output: Output[Model]):
...
@dsl.component(
base_image='python:3.9',
output_component_file='my_function.yaml'
)
def my_function_two(input: Input[Mode])):
...
@dsl.pipeline(pipeline_root='...',
name='my-pipeline')
def pipeline():
my_function_one_task = my_function_one(input=...)
my_function_two_task = my_function_two(input=my_function_one_task.outputs..
"""
if func is None:
return functools.partial(component,
base_image=base_image,
packages_to_install=packages_to_install,
output_component_file=output_component_file)
return components.create_component_from_func_v2(
func,
base_image=base_image,
packages_to_install=packages_to_install,
output_component_file=output_component_file)
'name': 'accuracy',
'numberValue': 0.9,
'format': "PERCENTAGE",
}]
})
with open(artifact.path, 'r') as f:
artifact_contents = f.read()
model = "Model contents: " + artifact_contents
from collections import namedtuple
output = namedtuple('Outputs', ['scalar', 'metrics', 'model'])
return output(scalar, metrics, model)
concat_op = components.create_component_from_func_v2(concat_message)
add_op = components.create_component_from_func_v2(add_numbers)
output_artifact_op = components.create_component_from_func_v2(output_artifact)
output_named_tuple_op = components.create_component_from_func_v2(
output_named_tuple)
@dsl.pipeline(pipeline_root='dummy_root', name='functions-with-outputs')
def pipeline(first_message: str, second_message: str, first_number: int,
second_number: int):
concat = concat_op(first=first_message, second=second_message)
add_numbers = add_op(first=first_number, second=second_number)
output_artifact = output_artifact_op(number=add_numbers.output,
message=concat.output)
output_name_tuple = output_named_tuple_op(output_artifact.output)
with open(dataset_two.path, 'r') as input_file:
dataset_two_contents = input_file.read()
line = "dataset_one_contents: {} || dataset_two_contents: {} || message: {}\n".format(
dataset_one_contents, dataset_two_contents, message)
with open(model.path, 'w') as output_file:
for i in range(num_steps):
output_file.write("Step {}\n{}\n=====\n".format(i, line))
# Use model.get() to get a Model artifact, which has a .metadata dictionary
# to store arbitrary metadata for the output artifact.
model.get().metadata['accuracy'] = 0.9
preprocess_op = components.create_component_from_func_v2(preprocess)
train_op = components.create_component_from_func_v2(train)
@dsl.pipeline(pipeline_root='dummy_root', name='my-test-pipeline-beta')
def pipeline(message: str):
preprocess_task = preprocess_op(message=message)
train_task = train_op(
dataset_one=preprocess_task.outputs['output_dataset_one'],
dataset_two=preprocess_task.outputs['output_dataset_two'],
message=preprocess_task.outputs['output_parameter'],
num_steps=5)
if __name__ == '__main__':
compiler.Compiler().compile(pipeline_func=pipeline,
('model', Model),
]):
"""Dummy Training step."""
with open(dataset.path, 'r') as f:
data = f.read()
print('Dataset:', data)
scalar = '123'
model = 'My model trained using data: {}'.format(data)
from collections import namedtuple
output = namedtuple('Outputs', ['scalar', 'model'])
return output(scalar, model)
train_op = components.create_component_from_func_v2(train)
@dsl.pipeline(name='pipeline-with-importer', pipeline_root='dummy_root')
def my_pipeline(
dataset2: str = 'gs://ml-pipeline-playground/shakespeare2.txt'):
importer = dsl.importer(
artifact_uri='gs://ml-pipeline-playground/shakespeare1.txt',
artifact_class=Dataset,
reimport=False)
train1 = train_op(dataset=importer.output)
with dsl.Condition(train1.outputs['scalar'] == '123'):
importer2 = dsl.importer(artifact_uri=dataset2,
artifact_class=Dataset,
PROJECT_ID = os.getenv('PROJECT_ID')
REGION = os.getenv('REGION')
TRAINING_CONTAINER_IMAGE_URI = os.getenv('TRAINING_CONTAINER_IMAGE_URI')
SERVING_CONTAINER_IMAGE_URI = os.getenv('SERVING_CONTAINER_IMAGE_URI')
TRAINING_FILE_PATH = os.getenv('TRAINING_FILE_PATH')
VALIDATION_FILE_PATH = os.getenv('VALIDATION_FILE_PATH')
MAX_TRIAL_COUNT = os.getenv('MAX_TRIAL_COUNT', 5)
PARALLEL_TRIAL_COUNT = os.getenv('PARALLEL_TRIAL_COUNT', 5)
THRESHOLD = os.getenv('THRESHOLD', 0.6)
tune_hyperparameters_component = create_component_from_func_v2(
tune_hyperparameters,
base_image='python:3.8',
output_component_file='covertype_kfp_tune_hyperparameters.yaml',
packages_to_install=['google-cloud-aiplatform'],
)
train_and_deploy_component = create_component_from_func_v2(
train_and_deploy,
base_image='python:3.8',
output_component_file='covertype_kfp_train_and_deploy.yaml',
packages_to_install=['google-cloud-aiplatform'],
)
@dsl.pipeline(
name="covertype-kfp-pipeline",
description="The pipeline training and deploying the Covertype classifier",
pipeline_root=PIPELINE_ROOT,
def component(func: Optional[Callable] = None,
*,
base_image: Optional[str] = None,
packages_to_install: List[str] = None,
output_component_file: Optional[str] = None,
install_kfp_package: bool = True,
kfp_package_path: Optional[str] = None):
"""Decorator for Python-function based components in KFP v2.
A lightweight component is a self-contained Python function that includes
all necessary imports and dependencies.
Example usage:
from kfp.v2 import dsl
@dsl.component
def my_function_one(input: str, output: Output[Model]):
...
@dsl.component(
base_image='python:3.9',
output_component_file='my_function.yaml'
)
def my_function_two(input: Input[Mode])):
...
@dsl.pipeline(pipeline_root='...',
name='my-pipeline')
def pipeline():
my_function_one_task = my_function_one(input=...)
my_function_two_task = my_function_two(input=my_function_one_task.outputs..
Args:
func: The python function to create a component from. The function
should have type annotations for all its arguments, indicating how
it is intended to be used (e.g. as an input/output Artifact object,
a plain parameter, or a path to a file).
base_image: The image to use when executing |func|. It should
contain a default Python interpreter that is compatible with KFP.
packages_to_install: A list of optional packages to install before
executing |func|.
install_kfp_package: Specifies if we should add a KFP Python package to
|packages_to_install|. Lightweight Python functions always require
an installation of KFP in |base_image| to work. If you specify
a |base_image| that already contains KFP, you can set this to False.
kfp_package_path: Specifies the location from which to install KFP. By
default, this will try to install from PyPi using the same version
as that used when this component was created. KFP developers can
choose to override this to point to a Github pull request or
other pip-compatible location when testing changes to lightweight
Python functions.
Returns:
A component task factory that can be used in pipeline definitions.
"""
if func is None:
return functools.partial(component,
base_image=base_image,
packages_to_install=packages_to_install,
output_component_file=output_component_file,
install_kfp_package=install_kfp_package,
kfp_package_path=kfp_package_path)
return components.create_component_from_func_v2(
func,
base_image=base_image,
packages_to_install=packages_to_install,
output_component_file=output_component_file,
install_kfp_package=install_kfp_package,
kfp_package_path=kfp_package_path)