def component(func: Optional[Callable] = None, *, base_image: Optional[str] = None, packages_to_install: List[str] = None, output_component_file: Optional[str] = None): """Decorator for Python-function based components in v2. Example usage: from kfp.v2 import dsl @dsl.component def my_function_one(input: str, output: Output[Model]): ... @dsl.component( base_image='python:3.9', output_component_file='my_function.yaml' ) def my_function_two(input: Input[Mode])): ... @dsl.pipeline(pipeline_root='...', name='my-pipeline') def pipeline(): my_function_one_task = my_function_one(input=...) my_function_two_task = my_function_two(input=my_function_one_task.outputs.. """ if func is None: return functools.partial(component, base_image=base_image, packages_to_install=packages_to_install, output_component_file=output_component_file) return components.create_component_from_func_v2( func, base_image=base_image, packages_to_install=packages_to_install, output_component_file=output_component_file)
'name': 'accuracy', 'numberValue': 0.9, 'format': "PERCENTAGE", }] }) with open(artifact.path, 'r') as f: artifact_contents = f.read() model = "Model contents: " + artifact_contents from collections import namedtuple output = namedtuple('Outputs', ['scalar', 'metrics', 'model']) return output(scalar, metrics, model) concat_op = components.create_component_from_func_v2(concat_message) add_op = components.create_component_from_func_v2(add_numbers) output_artifact_op = components.create_component_from_func_v2(output_artifact) output_named_tuple_op = components.create_component_from_func_v2( output_named_tuple) @dsl.pipeline(pipeline_root='dummy_root', name='functions-with-outputs') def pipeline(first_message: str, second_message: str, first_number: int, second_number: int): concat = concat_op(first=first_message, second=second_message) add_numbers = add_op(first=first_number, second=second_number) output_artifact = output_artifact_op(number=add_numbers.output, message=concat.output) output_name_tuple = output_named_tuple_op(output_artifact.output)
with open(dataset_two.path, 'r') as input_file: dataset_two_contents = input_file.read() line = "dataset_one_contents: {} || dataset_two_contents: {} || message: {}\n".format( dataset_one_contents, dataset_two_contents, message) with open(model.path, 'w') as output_file: for i in range(num_steps): output_file.write("Step {}\n{}\n=====\n".format(i, line)) # Use model.get() to get a Model artifact, which has a .metadata dictionary # to store arbitrary metadata for the output artifact. model.get().metadata['accuracy'] = 0.9 preprocess_op = components.create_component_from_func_v2(preprocess) train_op = components.create_component_from_func_v2(train) @dsl.pipeline(pipeline_root='dummy_root', name='my-test-pipeline-beta') def pipeline(message: str): preprocess_task = preprocess_op(message=message) train_task = train_op( dataset_one=preprocess_task.outputs['output_dataset_one'], dataset_two=preprocess_task.outputs['output_dataset_two'], message=preprocess_task.outputs['output_parameter'], num_steps=5) if __name__ == '__main__': compiler.Compiler().compile(pipeline_func=pipeline,
('model', Model), ]): """Dummy Training step.""" with open(dataset.path, 'r') as f: data = f.read() print('Dataset:', data) scalar = '123' model = 'My model trained using data: {}'.format(data) from collections import namedtuple output = namedtuple('Outputs', ['scalar', 'model']) return output(scalar, model) train_op = components.create_component_from_func_v2(train) @dsl.pipeline(name='pipeline-with-importer', pipeline_root='dummy_root') def my_pipeline( dataset2: str = 'gs://ml-pipeline-playground/shakespeare2.txt'): importer = dsl.importer( artifact_uri='gs://ml-pipeline-playground/shakespeare1.txt', artifact_class=Dataset, reimport=False) train1 = train_op(dataset=importer.output) with dsl.Condition(train1.outputs['scalar'] == '123'): importer2 = dsl.importer(artifact_uri=dataset2, artifact_class=Dataset,
PROJECT_ID = os.getenv('PROJECT_ID') REGION = os.getenv('REGION') TRAINING_CONTAINER_IMAGE_URI = os.getenv('TRAINING_CONTAINER_IMAGE_URI') SERVING_CONTAINER_IMAGE_URI = os.getenv('SERVING_CONTAINER_IMAGE_URI') TRAINING_FILE_PATH = os.getenv('TRAINING_FILE_PATH') VALIDATION_FILE_PATH = os.getenv('VALIDATION_FILE_PATH') MAX_TRIAL_COUNT = os.getenv('MAX_TRIAL_COUNT', 5) PARALLEL_TRIAL_COUNT = os.getenv('PARALLEL_TRIAL_COUNT', 5) THRESHOLD = os.getenv('THRESHOLD', 0.6) tune_hyperparameters_component = create_component_from_func_v2( tune_hyperparameters, base_image='python:3.8', output_component_file='covertype_kfp_tune_hyperparameters.yaml', packages_to_install=['google-cloud-aiplatform'], ) train_and_deploy_component = create_component_from_func_v2( train_and_deploy, base_image='python:3.8', output_component_file='covertype_kfp_train_and_deploy.yaml', packages_to_install=['google-cloud-aiplatform'], ) @dsl.pipeline( name="covertype-kfp-pipeline", description="The pipeline training and deploying the Covertype classifier", pipeline_root=PIPELINE_ROOT,
def component(func: Optional[Callable] = None, *, base_image: Optional[str] = None, packages_to_install: List[str] = None, output_component_file: Optional[str] = None, install_kfp_package: bool = True, kfp_package_path: Optional[str] = None): """Decorator for Python-function based components in KFP v2. A lightweight component is a self-contained Python function that includes all necessary imports and dependencies. Example usage: from kfp.v2 import dsl @dsl.component def my_function_one(input: str, output: Output[Model]): ... @dsl.component( base_image='python:3.9', output_component_file='my_function.yaml' ) def my_function_two(input: Input[Mode])): ... @dsl.pipeline(pipeline_root='...', name='my-pipeline') def pipeline(): my_function_one_task = my_function_one(input=...) my_function_two_task = my_function_two(input=my_function_one_task.outputs.. Args: func: The python function to create a component from. The function should have type annotations for all its arguments, indicating how it is intended to be used (e.g. as an input/output Artifact object, a plain parameter, or a path to a file). base_image: The image to use when executing |func|. It should contain a default Python interpreter that is compatible with KFP. packages_to_install: A list of optional packages to install before executing |func|. install_kfp_package: Specifies if we should add a KFP Python package to |packages_to_install|. Lightweight Python functions always require an installation of KFP in |base_image| to work. If you specify a |base_image| that already contains KFP, you can set this to False. kfp_package_path: Specifies the location from which to install KFP. By default, this will try to install from PyPi using the same version as that used when this component was created. KFP developers can choose to override this to point to a Github pull request or other pip-compatible location when testing changes to lightweight Python functions. Returns: A component task factory that can be used in pipeline definitions. """ if func is None: return functools.partial(component, base_image=base_image, packages_to_install=packages_to_install, output_component_file=output_component_file, install_kfp_package=install_kfp_package, kfp_package_path=kfp_package_path) return components.create_component_from_func_v2( func, base_image=base_image, packages_to_install=packages_to_install, output_component_file=output_component_file, install_kfp_package=install_kfp_package, kfp_package_path=kfp_package_path)