def test_end_to_end_python_component_pipeline(self): #Defining the Python function def add(a: float, b: float) -> float: """Returns sum of two arguments.""" return a + b with tempfile.TemporaryDirectory() as temp_dir_name: add_component_file = str( Path(temp_dir_name).joinpath('add.component.yaml')) #Converting the function to a component. Instantiate it to create a pipeline task (ContaineOp instance) add_op = comp.func_to_container_op( add, base_image='python:3.5', output_component_file=add_component_file) #Checking that the component artifact is usable: add_op2 = comp.load_component_from_file(add_component_file) #Building the pipeline def calc_pipeline( a1, a2='7', a3='17', ): task_1 = add_op(a1, a2) task_2 = add_op2(a1, a2) task_3 = add_op(task_1.outputs['Output'], task_2.outputs['Output']) task_4 = add_op2(task_3.outputs['Output'], a3) #Instantiating the pipleine: calc_pipeline(42)
def test_func_to_component_file(self): func = add_two_numbers with tempfile.TemporaryDirectory() as temp_dir_name: component_path = str(Path(temp_dir_name) / 'component.yaml') comp._python_op.func_to_component_file( func, output_component_file=component_path) op = comp.load_component_from_file(component_path) self.helper_test_2_in_1_out_component_using_local_call(func, op)
def test_handle_creating_graph_component_from_pipeline_that_uses_container_components( self): test_data_dir = Path(__file__).parent / 'test_data' producer_op = comp.load_component_from_file( str(test_data_dir / 'component_with_0_inputs_and_2_outputs.component.yaml')) processor_op = comp.load_component_from_file( str(test_data_dir / 'component_with_2_inputs_and_2_outputs.component.yaml')) consumer_op = comp.load_component_from_file( str(test_data_dir / 'component_with_2_inputs_and_0_outputs.component.yaml')) def pipeline1(pipeline_param_1: int): producer_task = producer_op() processor_task = processor_op(pipeline_param_1, producer_task.outputs['Output 2']) consumer_task = consumer_op(processor_task.outputs['Output 1'], processor_task.outputs['Output 2']) return OrderedDict( [ # You can safely return normal dict in python 3.6+ ('Pipeline output 1', producer_task.outputs['Output 1']), ('Pipeline output 2', processor_task.outputs['Output 2']), ]) graph_component = create_graph_component_spec_from_pipeline_func( pipeline1) self.assertEqual(len(graph_component.inputs), 1) self.assertListEqual( [input.name for input in graph_component.inputs], ['pipeline_param_1' ]) #Relies on human name conversion function stability self.assertListEqual( [output.name for output in graph_component.outputs], ['Pipeline output 1', 'Pipeline output 2']) self.assertEqual(len(graph_component.implementation.graph.tasks), 3)
def _test_load_component_from_file(self, component_path: str): task_factory1 = comp.load_component_from_file(component_path) arg1 = 3 arg2 = 5 task1 = task_factory1(arg1, arg2) self.assertEqual(task_factory1.__name__, 'Add') self.assertEqual(task_factory1.__doc__.strip(), 'Add\nReturns sum of two arguments') self.assertEqual( task1.component_ref.spec.implementation.container.image, 'python:3.5') resolved_cmd = _resolve_command_line_and_paths(task1.component_ref.spec, task1.arguments) self.assertEqual(resolved_cmd.args[0], str(arg1)) self.assertEqual(resolved_cmd.args[1], str(arg2))
def test_end_to_end_python_component_pipeline_compilation(self): import kfp.deprecated.components as comp #Defining the Python function def add(a: float, b: float) -> float: """Returns sum of two arguments.""" return a + b with tempfile.TemporaryDirectory() as temp_dir_name: add_component_file = str( Path(temp_dir_name).joinpath('add.component.yaml')) #Converting the function to a component. Instantiate it to create a pipeline task (ContaineOp instance) add_op = comp.func_to_container_op( add, base_image='python:3.5', output_component_file=add_component_file) #Checking that the component artifact is usable: add_op2 = comp.load_component_from_file(add_component_file) #Building the pipeline @kfp.dsl.pipeline( name='Calculation pipeline', description='A pipeline that performs arithmetic calculations.') def calc_pipeline( a1, a2='7', a3='17', ): task_1 = add_op(a1, a2) task_2 = add_op2(a1, a2) task_3 = add_op(task_1.output, task_2.output) task_4 = add_op2(task_3.output, a3) #Compiling the pipleine: pipeline_filename = str( Path(temp_dir_name).joinpath(calc_pipeline.__name__ + '.pipeline.tar.gz')) kfp.compiler.Compiler().compile(calc_pipeline, pipeline_filename)
def test_load_component_from_file_fail_on_none_arg(self): with self.assertRaises(TypeError): comp.load_component_from_file(None)
from pathlib import Path import kfp.deprecated as kfp from kfp.deprecated.components import load_component_from_file, create_component_from_func from typing import NamedTuple test_data_dir = Path(__file__).parent / 'test_data' producer_op = load_component_from_file( str(test_data_dir / 'produce_2.component.yaml')) processor_op = load_component_from_file( str(test_data_dir / 'process_2_2.component.yaml')) consumer_op = load_component_from_file( str(test_data_dir / 'consume_2.component.yaml')) def metadata_and_metrics() -> NamedTuple( "Outputs", [("mlpipeline_ui_metadata", "UI_metadata"), ("mlpipeline_metrics", "Metrics" )], ): metadata = { "outputs": [{ "storage": "inline", "source": "*this should be bold*", "type": "markdown" }] } metrics = { "metrics": [ { "name": "train-accuracy",