def test_end_to_end_python_component_pipeline(self):
#Defining the Python function
def add(a: float, b: float) -> float:
'''Returns sum of two arguments'''
return a + b
with tempfile.TemporaryDirectory() as temp_dir_name:
add_component_file = str(Path(temp_dir_name).joinpath('add.component.yaml'))
#Converting the function to a component. Instantiate it to create a pipeline task (ContaineOp instance)
add_op = comp.func_to_container_op(add, base_image='python:3.5', output_component_file=add_component_file)
#Checking that the component artifact is usable:
add_op2 = comp.load_component_from_file(add_component_file)
#Building the pipeline
def calc_pipeline(
a1,
a2='7',
a3='17',
):
task_1 = add_op(a1, a2)
task_2 = add_op2(a1, a2)
task_3 = add_op(task_1.outputs['Output'], task_2.outputs['Output'])
task_4 = add_op2(task_3.outputs['Output'], a3)
#Instantiating the pipleine:
calc_pipeline(42)
def test_func_to_component_file(self):
func = add_two_numbers
with tempfile.TemporaryDirectory() as temp_dir_name:
component_path = str(Path(temp_dir_name) / 'component.yaml')
comp._python_op.func_to_component_file(func, output_component_file=component_path)
op = comp.load_component_from_file(component_path)
self.helper_test_2_in_1_out_component_using_local_call(func, op)
def test_handle_creating_graph_component_from_pipeline_that_uses_container_components(
self):
test_data_dir = Path(__file__).parent / 'test_data'
producer_op = comp.load_component_from_file(
str(test_data_dir /
'component_with_0_inputs_and_2_outputs.component.yaml'))
processor_op = comp.load_component_from_file(
str(test_data_dir /
'component_with_2_inputs_and_2_outputs.component.yaml'))
consumer_op = comp.load_component_from_file(
str(test_data_dir /
'component_with_2_inputs_and_0_outputs.component.yaml'))
def pipeline1(pipeline_param_1: int):
producer_task = producer_op()
processor_task = processor_op(pipeline_param_1,
producer_task.outputs['Output 2'])
consumer_task = consumer_op(processor_task.outputs['Output 1'],
processor_task.outputs['Output 2'])
return OrderedDict(
[ # You can safely return normal dict in python 3.6+
('Pipeline output 1', producer_task.outputs['Output 1']),
('Pipeline output 2', processor_task.outputs['Output 2']),
])
graph_component = create_graph_component_spec_from_pipeline_func(
pipeline1)
self.assertEqual(len(graph_component.inputs), 1)
self.assertListEqual(
[input.name for input in graph_component.inputs],
['pipeline_param_1'
]) #Relies on human name conversion function stability
self.assertListEqual(
[output.name for output in graph_component.outputs],
['Pipeline output 1', 'Pipeline output 2'])
self.assertEqual(len(graph_component.implementation.graph.tasks), 3)
def _test_load_component_from_file(self, component_path: str):
task_factory1 = comp.load_component_from_file(component_path)
arg1 = 3
arg2 = 5
task1 = task_factory1(arg1, arg2)
self.assertEqual(task_factory1.__name__, 'Add')
self.assertEqual(task_factory1.__doc__.strip(),
'Add\nReturns sum of two arguments')
self.assertEqual(
task1.component_ref.spec.implementation.container.image,
'python:3.5')
resolved_cmd = _resolve_command_line_and_paths(
task1.component_ref.spec, task1.arguments)
self.assertEqual(resolved_cmd.args[0], str(arg1))
self.assertEqual(resolved_cmd.args[1], str(arg2))
def test_load_component_from_file_fail_on_none_arg(self):
with self.assertRaises(TypeError):
comp.load_component_from_file(None)