def test_func_to_container_op_output_component_file(self):
func = add_two_numbers
with tempfile.TemporaryDirectory() as temp_dir_name:
component_path = str(Path(temp_dir_name) / 'component.yaml')
comp.func_to_container_op(func,
output_component_file=component_path)
op = comp.load_component_from_file(component_path)
self.helper_test_2_in_1_out_component_using_local_call(func, op)
def test_packages_to_install_feature(self):
task_factory = comp.func_to_container_op(
dummy_in_0_out_0, packages_to_install=['six', 'pip'])
self.helper_test_component_using_local_call(task_factory,
arguments={},
expected_output_values={})
task_factory2 = comp.func_to_container_op(
dummy_in_0_out_0,
packages_to_install=[
'bad-package-0ee7cf93f396cd5072603dec154425cd53bf1c681c7c7605c60f8faf7799b901'
])
with self.assertRaises(Exception):
self.helper_test_component_using_local_call(
task_factory2, arguments={}, expected_output_values={})
def test_legacy_python_component_name_description_overrides(self):
# Deprecated feature
expected_name = 'Sum component name'
expected_description = 'Sum component description'
expected_image = 'org/image'
def add_two_numbers_decorated(
a: float,
b: float,
) -> float:
"""Returns sum of two arguments."""
return a + b
# Deprecated features
add_two_numbers_decorated._component_human_name = expected_name
add_two_numbers_decorated._component_description = expected_description
add_two_numbers_decorated._component_base_image = expected_image
func = add_two_numbers_decorated
op = comp.func_to_container_op(func)
component_spec = op.component_spec
self.assertEqual(component_spec.name, expected_name)
self.assertEqual(component_spec.description.strip(),
expected_description.strip())
self.assertEqual(component_spec.implementation.container.image,
expected_image)
def test_handling_default_value_of_none(self):
def assert_is_none(arg=None):
assert arg is None
func = assert_is_none
op = comp.func_to_container_op(func)
self.helper_test_component_using_local_call(op)
def test_end_to_end_python_component_pipeline(self):
#Defining the Python function
def add(a: float, b: float) -> float:
"""Returns sum of two arguments."""
return a + b
with tempfile.TemporaryDirectory() as temp_dir_name:
add_component_file = str(
Path(temp_dir_name).joinpath('add.component.yaml'))
#Converting the function to a component. Instantiate it to create a pipeline task (ContaineOp instance)
add_op = comp.func_to_container_op(
add,
base_image='python:3.5',
output_component_file=add_component_file)
#Checking that the component artifact is usable:
add_op2 = comp.load_component_from_file(add_component_file)
#Building the pipeline
def calc_pipeline(
a1,
a2='7',
a3='17',
):
task_1 = add_op(a1, a2)
task_2 = add_op2(a1, a2)
task_3 = add_op(task_1.outputs['Output'],
task_2.outputs['Output'])
task_4 = add_op2(task_3.outputs['Output'], a3)
#Instantiating the pipleine:
calc_pipeline(42)
def test_func_to_container_op_with_imported_func(self):
from kfp.deprecated.components_tests.test_data.module1 import \
module_func_with_deps as module1_func_with_deps
func = module1_func_with_deps
op = comp.func_to_container_op(func, use_code_pickling=True)
self.helper_test_2_in_1_out_component_using_local_call(func, op)
def test_fail_on_input_path_non_none_default(self):
def read_from_file_path(
file_path: InputPath(int) = '/tmp/something') -> str:
return file_path
with self.assertRaises(ValueError):
task_factory = comp.func_to_container_op(read_from_file_path)
def test_indented_func_to_container_op_local_call(self):
def add_two_numbers_indented(a: float, b: float) -> float:
"""Returns sum of two arguments."""
return a + b
func = add_two_numbers_indented
op = comp.func_to_container_op(func)
self.helper_test_2_in_1_out_component_using_local_call(func, op)
def test_handling_same_input_default_output_names(self):
def add_two_numbers_indented(a: float, Output: float) -> float:
"""Returns sum of two arguments."""
return a + Output
func = add_two_numbers_indented
op = comp.func_to_container_op(func)
self.helper_test_2_in_1_out_component_using_local_call(func, op)
def test_func_to_container_op_multiple_named_typed_outputs_using_list_syntax(
self):
def add_multiply_two_numbers(
a: float, b: float) -> [('sum', float), ('product', float)]:
"""Returns sum and product of two arguments."""
return (a + b, a * b)
func = add_multiply_two_numbers
op = comp.func_to_container_op(func)
self.helper_test_2_in_2_out_component_using_local_call(
func, op, output_names=['sum', 'product'])
def test_func_to_container_op_named_typed_outputs_with_spaces(self):
from typing import NamedTuple
def add_two_numbers_name3(
a: float,
b: float) -> NamedTuple('DummyName', [('Output data', float)]):
"""Returns sum of two arguments."""
return (a + b, )
func = add_two_numbers_name3
op = comp.func_to_container_op(func)
self.helper_test_2_in_1_out_component_using_local_call(func, op)
def test_func_to_container_op_with_imported_func2(self):
from kfp.deprecated.components_tests.test_data import module1, module2_which_depends_on_module1
func = module2_which_depends_on_module1.module2_func_with_deps
op = comp.func_to_container_op(
func,
use_code_pickling=True,
modules_to_capture=[
module1.__name__, # '*.components_tests.test_data.module1'
func.
__module__, # '*.components_tests.test_data.module2_which_depends_on_module1'
])
self.helper_test_2_in_1_out_component_using_local_call(func, op)
def test_handling_boolean_arguments(self):
def assert_values_are_true_false(
bool1: bool,
bool2: bool,
) -> int:
assert bool1 is True
assert bool2 is False
return 1
func = assert_values_are_true_false
op = comp.func_to_container_op(func)
self.helper_test_2_in_1_out_component_using_local_call(
func, op, arguments=[True, False])
def test_input_path(self):
def consume_file_path(number_file_path: InputPath(int)) -> int:
with open(number_file_path) as f:
string_data = f.read()
return int(string_data)
task_factory = comp.func_to_container_op(consume_file_path)
self.assertEqual(task_factory.component_spec.inputs[0].type, 'Integer')
self.helper_test_component_using_local_call(
task_factory,
arguments={'number': "42"},
expected_output_values={'Output': '42'})
def test_input_binary_file(self):
def consume_file_path(number_file: InputBinaryFile(int)) -> int:
bytes_data = number_file.read()
assert isinstance(bytes_data, bytes)
return int(bytes_data)
task_factory = comp.func_to_container_op(consume_file_path)
self.assertEqual(task_factory.component_spec.inputs[0].type, 'Integer')
self.helper_test_component_using_local_call(
task_factory,
arguments={'number': "42"},
expected_output_values={'Output': '42'})
def test_handling_same_input_output_names(self):
from typing import NamedTuple
def add_multiply_two_numbers(
a: float, b: float
) -> NamedTuple('DummyName', [('a', float), ('b', float)]):
"""Returns sum and product of two arguments."""
return (a + b, a * b)
func = add_multiply_two_numbers
op = comp.func_to_container_op(func)
self.helper_test_2_in_2_out_component_using_local_call(
func, op, output_names=['a', 'b'])
def test_handling_list_dict_arguments(self):
def assert_values_are_same(
list_param: list,
dict_param: dict,
) -> int:
import unittest
unittest.TestCase().assertEqual(
list_param,
["string", 1, 2.2, True, False, None, [3, 4], {
's': 5
}])
unittest.TestCase().assertEqual(
dict_param, {
'str': "string",
'int': 1,
'float': 2.2,
'false': False,
'true': True,
'none': None,
'list': [3, 4],
'dict': {
's': 4
}
})
return 1
# ! JSON map keys are always strings. Python converts all keys to strings without warnings
func = assert_values_are_same
op = comp.func_to_container_op(func)
self.helper_test_2_in_1_out_component_using_local_call(
func,
op,
arguments=[
["string", 1, 2.2, True, False, None, [3, 4], {
's': 5
}],
{
'str': "string",
'int': 1,
'float': 2.2,
'false': False,
'true': True,
'none': None,
'list': [3, 4],
'dict': {
's': 4
}
},
])
def test_handling_complex_default_values(self):
def assert_values_are_default(
singleton_param=None,
function_param=ascii,
dict_param: dict = {'b': [2, 3, 4]},
func_call_param='_'.join(['a', 'b', 'c']),
):
assert singleton_param is None
assert function_param is ascii
assert dict_param == {'b': [2, 3, 4]}
assert func_call_param == '_'.join(['a', 'b', 'c'])
func = assert_values_are_default
op = comp.func_to_container_op(func)
self.helper_test_component_using_local_call(op)
def test_output_binary_file(self):
def write_to_file_path(number_file: OutputBinaryFile(int)):
number_file.write(b'42')
task_factory = comp.func_to_container_op(write_to_file_path)
self.assertFalse(task_factory.component_spec.inputs)
self.assertEqual(len(task_factory.component_spec.outputs), 1)
self.assertEqual(task_factory.component_spec.outputs[0].type,
'Integer')
self.helper_test_component_using_local_call(
task_factory,
arguments={},
expected_output_values={'number': '42'})
def test_output_path(self):
def write_to_file_path(number_file_path: OutputPath(int)):
with open(number_file_path, 'w') as f:
f.write(str(42))
task_factory = comp.func_to_container_op(write_to_file_path)
self.assertFalse(task_factory.component_spec.inputs)
self.assertEqual(len(task_factory.component_spec.outputs), 1)
self.assertEqual(task_factory.component_spec.outputs[0].type,
'Integer')
self.helper_test_component_using_local_call(
task_factory,
arguments={},
expected_output_values={'number': '42'})
def test_all_data_passing_ways(self):
def write_to_file_path(
file_input1_path: InputPath(str),
file_input2_file: InputTextFile(str),
file_output1_path: OutputPath(str),
file_output2_file: OutputTextFile(str),
value_input1: str = 'foo',
value_input2: str = 'foo',
) -> NamedTuple('Outputs', [
('return_output1', str),
('return_output2', str),
]):
with open(file_input1_path, 'r') as file_input1_file:
with open(file_output1_path, 'w') as file_output1_file:
file_output1_file.write(file_input1_file.read())
file_output2_file.write(file_input2_file.read())
return (value_input1, value_input2)
task_factory = comp.func_to_container_op(write_to_file_path)
self.assertEqual(
set(input.name for input in task_factory.component_spec.inputs),
{'file_input1', 'file_input2', 'value_input1', 'value_input2'})
self.assertEqual(
set(output.name for output in task_factory.component_spec.outputs),
{
'file_output1', 'file_output2', 'return_output1',
'return_output2'
})
self.helper_test_component_using_local_call(
task_factory,
arguments={
'file_input1': 'file_input1_value',
'file_input2': 'file_input2_value',
'value_input1': 'value_input1_value',
'value_input2': 'value_input2_value',
},
expected_output_values={
'file_output1': 'file_input1_value',
'file_output2': 'file_input2_value',
'return_output1': 'value_input1_value',
'return_output2': 'value_input2_value',
},
)
def test_func_to_container_op_call_other_func(self):
extra_variable = 10
class ExtraClass:
def class_method(self, x):
return x * extra_variable
def extra_func(a: float) -> float:
return a * 5
def main_func(a: float, b: float) -> float:
return ExtraClass().class_method(a) + extra_func(b)
func = main_func
op = comp.func_to_container_op(func, use_code_pickling=True)
self.helper_test_2_in_1_out_component_using_local_call(func, op)
def test_handling_list_dict_output_values(self):
def produce_list() -> list:
return ["string", 1, 2.2, True, False, None, [3, 4], {'s': 5}]
# ! JSON map keys are always strings. Python converts all keys to strings without warnings
task_factory = comp.func_to_container_op(produce_list)
import json
expected_output = json.dumps(
["string", 1, 2.2, True, False, None, [3, 4], {
's': 5
}])
self.helper_test_component_using_local_call(
task_factory,
arguments={},
expected_output_values={'Output': expected_output})
def test_optional_input_path(self):
def consume_file_path(number_file_path: InputPath(int) = None) -> int:
result = -1
if number_file_path:
with open(number_file_path) as f:
string_data = f.read()
result = int(string_data)
return result
task_factory = comp.func_to_container_op(consume_file_path)
self.helper_test_component_using_local_call(
task_factory,
arguments={},
expected_output_values={'Output': '-1'})
self.helper_test_component_using_local_call(
task_factory,
arguments={'number': "42"},
expected_output_values={'Output': '42'})
def test_handling_base64_pickle_arguments(self):
def assert_values_are_same(
obj1: 'Base64Pickle', # noqa: F821
obj2: 'Base64Pickle', # noqa: F821
) -> int:
import unittest
unittest.TestCase().assertEqual(obj1['self'], obj1)
unittest.TestCase().assertEqual(obj2, open)
return 1
func = assert_values_are_same
op = comp.func_to_container_op(func)
recursive_obj = {}
recursive_obj['self'] = recursive_obj
self.helper_test_2_in_1_out_component_using_local_call(
func, op, arguments=[
recursive_obj,
open,
])
def test_func_to_container_op_check_nothing_extra_captured(self):
def f1():
pass
def f2():
pass
def main_func(a: float, b: float) -> float:
f1()
try:
eval('f2()')
except:
return a + b
raise AssertionError(
"f2 should not be captured, because it's not a dependency.")
expected_func = lambda a, b: a + b
op = comp.func_to_container_op(main_func, use_code_pickling=True)
self.helper_test_2_in_1_out_component_using_local_call(
expected_func, op)
def test_end_to_end_python_component_pipeline_compilation(self):
import kfp.deprecated.components as comp
#Defining the Python function
def add(a: float, b: float) -> float:
"""Returns sum of two arguments."""
return a + b
with tempfile.TemporaryDirectory() as temp_dir_name:
add_component_file = str(
Path(temp_dir_name).joinpath('add.component.yaml'))
#Converting the function to a component. Instantiate it to create a pipeline task (ContaineOp instance)
add_op = comp.func_to_container_op(
add,
base_image='python:3.5',
output_component_file=add_component_file)
#Checking that the component artifact is usable:
add_op2 = comp.load_component_from_file(add_component_file)
#Building the pipeline
@kfp.dsl.pipeline(
name='Calculation pipeline',
description='A pipeline that performs arithmetic calculations.')
def calc_pipeline(
a1,
a2='7',
a3='17',
):
task_1 = add_op(a1, a2)
task_2 = add_op2(a1, a2)
task_3 = add_op(task_1.output, task_2.output)
task_4 = add_op2(task_3.output, a3)
#Compiling the pipleine:
pipeline_filename = str(
Path(temp_dir_name).joinpath(calc_pipeline.__name__ +
'.pipeline.tar.gz'))
kfp.compiler.Compiler().compile(calc_pipeline, pipeline_filename)
def test_func_to_container_op_local_call(self):
func = add_two_numbers
op = comp.func_to_container_op(func)
self.helper_test_2_in_1_out_component_using_local_call(func, op)
def test_func_to_container_op_call_other_func_global(self):
func = module_func_with_deps
op = comp.func_to_container_op(func, use_code_pickling=True)
self.helper_test_2_in_1_out_component_using_local_call(func, op)
def helper_test_component_created_from_func_using_local_call(
self, func: Callable, arguments: dict):
task_factory = comp.func_to_container_op(func)
self.helper_test_component_against_func_using_local_call(
func, task_factory, arguments)
