def test_end_to_end_python_component_pipeline(self):
#Defining the Python function
def add(a: float, b: float) -> float:
"""Returns sum of two arguments."""
return a + b
with tempfile.TemporaryDirectory() as temp_dir_name:
add_component_file = str(
Path(temp_dir_name).joinpath('add.component.yaml'))
#Converting the function to a component. Instantiate it to create a pipeline task (ContaineOp instance)
add_op = comp.func_to_container_op(
add,
base_image='python:3.5',
output_component_file=add_component_file)
#Checking that the component artifact is usable:
add_op2 = comp.load_component_from_file(add_component_file)
#Building the pipeline
def calc_pipeline(
a1,
a2='7',
a3='17',
):
task_1 = add_op(a1, a2)
task_2 = add_op2(a1, a2)
task_3 = add_op(task_1.outputs['Output'],
task_2.outputs['Output'])
task_4 = add_op2(task_3.outputs['Output'], a3)
#Instantiating the pipleine:
calc_pipeline(42)
def test_func_to_component_file(self):
func = add_two_numbers
with tempfile.TemporaryDirectory() as temp_dir_name:
component_path = str(Path(temp_dir_name) / 'component.yaml')
comp._python_op.func_to_component_file(
func, output_component_file=component_path)
op = comp.load_component_from_file(component_path)
self.helper_test_2_in_1_out_component_using_local_call(func, op)
def test_handle_creating_graph_component_from_pipeline_that_uses_container_components(
self):
test_data_dir = Path(__file__).parent / 'test_data'
producer_op = comp.load_component_from_file(
str(test_data_dir /
'component_with_0_inputs_and_2_outputs.component.yaml'))
processor_op = comp.load_component_from_file(
str(test_data_dir /
'component_with_2_inputs_and_2_outputs.component.yaml'))
consumer_op = comp.load_component_from_file(
str(test_data_dir /
'component_with_2_inputs_and_0_outputs.component.yaml'))
def pipeline1(pipeline_param_1: int):
producer_task = producer_op()
processor_task = processor_op(pipeline_param_1,
producer_task.outputs['Output 2'])
consumer_task = consumer_op(processor_task.outputs['Output 1'],
processor_task.outputs['Output 2'])
return OrderedDict(
[ # You can safely return normal dict in python 3.6+
('Pipeline output 1', producer_task.outputs['Output 1']),
('Pipeline output 2', processor_task.outputs['Output 2']),
])
graph_component = create_graph_component_spec_from_pipeline_func(
pipeline1)
self.assertEqual(len(graph_component.inputs), 1)
self.assertListEqual(
[input.name for input in graph_component.inputs],
['pipeline_param_1'
]) #Relies on human name conversion function stability
self.assertListEqual(
[output.name for output in graph_component.outputs],
['Pipeline output 1', 'Pipeline output 2'])
self.assertEqual(len(graph_component.implementation.graph.tasks), 3)
def _test_load_component_from_file(self, component_path: str):
task_factory1 = comp.load_component_from_file(component_path)
arg1 = 3
arg2 = 5
task1 = task_factory1(arg1, arg2)
self.assertEqual(task_factory1.__name__, 'Add')
self.assertEqual(task_factory1.__doc__.strip(),
'Add\nReturns sum of two arguments')
self.assertEqual(
task1.component_ref.spec.implementation.container.image,
'python:3.5')
resolved_cmd = _resolve_command_line_and_paths(task1.component_ref.spec,
task1.arguments)
self.assertEqual(resolved_cmd.args[0], str(arg1))
self.assertEqual(resolved_cmd.args[1], str(arg2))
def test_end_to_end_python_component_pipeline_compilation(self):
import kfp.deprecated.components as comp
#Defining the Python function
def add(a: float, b: float) -> float:
"""Returns sum of two arguments."""
return a + b
with tempfile.TemporaryDirectory() as temp_dir_name:
add_component_file = str(
Path(temp_dir_name).joinpath('add.component.yaml'))
#Converting the function to a component. Instantiate it to create a pipeline task (ContaineOp instance)
add_op = comp.func_to_container_op(
add,
base_image='python:3.5',
output_component_file=add_component_file)
#Checking that the component artifact is usable:
add_op2 = comp.load_component_from_file(add_component_file)
#Building the pipeline
@kfp.dsl.pipeline(
name='Calculation pipeline',
description='A pipeline that performs arithmetic calculations.')
def calc_pipeline(
a1,
a2='7',
a3='17',
):
task_1 = add_op(a1, a2)
task_2 = add_op2(a1, a2)
task_3 = add_op(task_1.output, task_2.output)
task_4 = add_op2(task_3.output, a3)
#Compiling the pipleine:
pipeline_filename = str(
Path(temp_dir_name).joinpath(calc_pipeline.__name__ +
'.pipeline.tar.gz'))
kfp.compiler.Compiler().compile(calc_pipeline, pipeline_filename)
def test_load_component_from_file_fail_on_none_arg(self):
with self.assertRaises(TypeError):
comp.load_component_from_file(None)
from pathlib import Path
import kfp.deprecated as kfp
from kfp.deprecated.components import load_component_from_file, create_component_from_func
from typing import NamedTuple
test_data_dir = Path(__file__).parent / 'test_data'
producer_op = load_component_from_file(
str(test_data_dir / 'produce_2.component.yaml'))
processor_op = load_component_from_file(
str(test_data_dir / 'process_2_2.component.yaml'))
consumer_op = load_component_from_file(
str(test_data_dir / 'consume_2.component.yaml'))
def metadata_and_metrics() -> NamedTuple(
"Outputs",
[("mlpipeline_ui_metadata", "UI_metadata"), ("mlpipeline_metrics", "Metrics"
)],
):
metadata = {
"outputs": [{
"storage": "inline",
"source": "*this should be bold*",
"type": "markdown"
}]
}
metrics = {
"metrics": [
{
"name": "train-accuracy",