def test_command_if_input_value_then(self):
component_text = '''\
inputs:
- {name: Do test, type: Boolean, optional: true}
- {name: Test data, type: Integer, optional: true}
- {name: Test parameter 1, optional: true}
implementation:
container:
image: busybox
args:
- if:
cond: {inputValue: Do test}
then: [--test-data, {inputValue: Test data}, --test-param1, {inputValue: Test parameter 1}]
'''
task_factory1 = comp.load_component(text=component_text)
task_then = task_factory1(True, 'test_data.txt', '42')
resolved_cmd_then = _resolve_command_line_and_paths(
task_then.component_ref.spec, task_then.arguments)
self.assertEqual(
resolved_cmd_then.args,
['--test-data', 'test_data.txt', '--test-param1', '42'])
task_else = task_factory1()
resolved_cmd_else = _resolve_command_line_and_paths(
task_else.component_ref.spec, task_else.arguments)
self.assertEqual(resolved_cmd_else.args, [])
def helper_test_2_in_2_out_component_using_local_call(self, func, op, output_names):
arg1 = float(3)
arg2 = float(5)
expected_tuple = func(arg1, arg2)
expected1_str = str(expected_tuple[0])
expected2_str = str(expected_tuple[1])
with tempfile.TemporaryDirectory() as temp_dir_name:
with components_local_output_dir_context(temp_dir_name):
task = op(arg1, arg2)
resolved_cmd = _resolve_command_line_and_paths(
task.component_ref.spec,
task.arguments,
)
full_command = resolved_cmd.command + resolved_cmd.args
subprocess.run(full_command, check=True)
(output_path1, output_path2) = (resolved_cmd.output_paths[output_names[0]], resolved_cmd.output_paths[output_names[1]])
actual1_str = Path(output_path1).read_text()
actual2_str = Path(output_path2).read_text()
self.assertEqual(float(actual1_str), float(expected1_str))
self.assertEqual(float(actual2_str), float(expected2_str))
def helper_test_component_using_local_call(self, component_task_factory: Callable, arguments: dict = None, expected_output_values: dict = None):
arguments = arguments or {}
expected_output_values = expected_output_values or {}
with tempfile.TemporaryDirectory() as temp_dir_name:
# Creating task from the component.
# We do it in a special context that allows us to control the output file locations.
inputs_path = Path(temp_dir_name) / 'inputs'
outputs_path = Path(temp_dir_name) / 'outputs'
with components_override_input_output_dirs_context(str(inputs_path), str(outputs_path)):
task = component_task_factory(**arguments)
resolved_cmd = _resolve_command_line_and_paths(
task.component_ref.spec,
task.arguments,
)
# Preparing input files
for input_name, input_file_path in (resolved_cmd.input_paths or {}).items():
Path(input_file_path).parent.mkdir(parents=True, exist_ok=True)
Path(input_file_path).write_text(str(arguments[input_name]))
# Constructing the full command-line from resolved command+args
full_command = resolved_cmd.command + resolved_cmd.args
# Executing the command-line locally
subprocess.run(full_command, check=True)
actual_output_values_dict = {output_name: Path(output_path).read_text() for output_name, output_path in resolved_cmd.output_paths.items()}
self.assertDictEqual(actual_output_values_dict, expected_output_values)
def test_handle_default_values_in_task_factory(self):
component_text = '''\
inputs:
- {name: Data, default: '123'}
implementation:
container:
image: busybox
args:
- {inputValue: Data}
'''
task_factory1 = comp.load_component_from_text(text=component_text)
task1 = task_factory1()
resolved_cmd1 = _resolve_command_line_and_paths(
task1.component_ref.spec, task1.arguments)
self.assertEqual(resolved_cmd1.args, ['123'])
task2 = task_factory1('456')
resolved_cmd2 = _resolve_command_line_and_paths(
task2.component_ref.spec, task2.arguments)
self.assertEqual(resolved_cmd2.args, ['456'])
def test_command_if_is_present_then_else(self):
component_text = '''\
inputs:
- {name: In, optional: true}
implementation:
container:
image: busybox
args:
- if:
cond: {isPresent: In}
then: [--in, {inputValue: In}]
else: --no-in
'''
task_factory1 = comp.load_component(text=component_text)
task_then = task_factory1('data')
resolved_cmd_then = _resolve_command_line_and_paths(
task_then.component_ref.spec, task_then.arguments)
self.assertEqual(resolved_cmd_then.args, ['--in', 'data'])
task_else = task_factory1()
resolved_cmd_else = _resolve_command_line_and_paths(
task_else.component_ref.spec, task_else.arguments)
self.assertEqual(resolved_cmd_else.args, ['--no-in'])
def test_command_if_true_string_then_else(self):
component_text = '''\
implementation:
container:
image: busybox
args:
- if:
cond: 'true'
then: --true-arg
else: --false-arg
'''
task_factory1 = comp.load_component(text=component_text)
task = task_factory1()
resolved_cmd = _resolve_command_line_and_paths(task.component_ref.spec,
task.arguments)
self.assertEqual(resolved_cmd.args, ['--true-arg'])
def test_command_concat(self):
component_text = '''\
inputs:
- {name: In1}
- {name: In2}
implementation:
container:
image: busybox
args:
- concat: [{inputValue: In1}, {inputValue: In2}]
'''
task_factory1 = comp.load_component(text=component_text)
task1 = task_factory1('some', 'data')
resolved_cmd = _resolve_command_line_and_paths(
task1.component_ref.spec, task1.arguments)
self.assertEqual(resolved_cmd.args, ['somedata'])
def test_input_value_resolving(self):
component_text = '''\
inputs:
- {name: Data}
implementation:
container:
image: busybox
args:
- --data
- inputValue: Data
'''
task_factory1 = comp.load_component(text=component_text)
task1 = task_factory1('some-data')
resolved_cmd = _resolve_command_line_and_paths(
task1.component_ref.spec, task1.arguments)
self.assertEqual(resolved_cmd.args, ['--data', 'some-data'])
def test_missing_optional_input_file_argument(self):
'''Missing optional inputs should resolve to nothing'''
component_text = '''\
inputs:
- {name: input 1, optional: true}
implementation:
container:
image: busybox
command:
- a
- {inputPath: input 1}
- z
'''
task_factory1 = comp.load_component_from_text(component_text)
task1 = task_factory1()
resolved_cmd = _resolve_command_line_and_paths(
task1.component_ref.spec, task1.arguments)
self.assertEqual(resolved_cmd.command, ['a', 'z'])
def test_input_path_placeholder_with_constant_argument(self):
component_text = '''\
inputs:
- {name: input 1}
implementation:
container:
image: busybox
command:
- --input-data
- {inputPath: input 1}
'''
task_factory1 = comp.load_component_from_text(component_text)
task1 = task_factory1('Text')
resolved_cmd = _resolve_command_line_and_paths(
task1.component_ref.spec, task1.arguments)
self.assertEqual(resolved_cmd.command,
['--input-data', resolved_cmd.input_paths['input 1']])
self.assertEqual(task1.arguments, {'input 1': 'Text'})
def test_automatic_output_resolving(self):
component_text = '''\
outputs:
- {name: Data}
implementation:
container:
image: busybox
args:
- --output-data
- {outputPath: Data}
'''
task_factory1 = comp.load_component(text=component_text)
task1 = task_factory1()
resolved_cmd = _resolve_command_line_and_paths(
task1.component_ref.spec, task1.arguments)
self.assertEqual(len(resolved_cmd.args), 2)
self.assertEqual(resolved_cmd.args[0], '--output-data')
self.assertTrue(resolved_cmd.args[1].startswith('/'))
def _test_load_component_from_file(self, component_path: str):
task_factory1 = comp.load_component_from_file(component_path)
arg1 = 3
arg2 = 5
task1 = task_factory1(arg1, arg2)
self.assertEqual(task_factory1.__name__, 'Add')
self.assertEqual(task_factory1.__doc__.strip(),
'Add\nReturns sum of two arguments')
self.assertEqual(
task1.component_ref.spec.implementation.container.image,
'python:3.5')
resolved_cmd = _resolve_command_line_and_paths(
task1.component_ref.spec, task1.arguments)
self.assertEqual(resolved_cmd.args[0], str(arg1))
self.assertEqual(resolved_cmd.args[1], str(arg2))
def helper_test_2_in_1_out_component_using_local_call(self, func, op, arguments=[3., 5.]):
expected = func(arguments[0], arguments[1])
if isinstance(expected, tuple):
expected = expected[0]
expected_str = str(expected)
with tempfile.TemporaryDirectory() as temp_dir_name:
with components_local_output_dir_context(temp_dir_name):
task = op(arguments[0], arguments[1])
resolved_cmd = _resolve_command_line_and_paths(
task.component_ref.spec,
task.arguments,
)
full_command = resolved_cmd.command + resolved_cmd.args
subprocess.run(full_command, check=True)
output_path = list(resolved_cmd.output_paths.values())[0]
actual_str = Path(output_path).read_text()
self.assertEqual(float(actual_str), float(expected_str))
def test_load_component_from_url(self):
url = 'https://raw.githubusercontent.com/kubeflow/pipelines/e54fe675432cfef1d115a7a2909f08ed95ea8933/sdk/python/tests/components/test_data/python_add.component.yaml'
import requests
resp = requests.get(url)
component_text = resp.content
component_dict = load_yaml(component_text)
task_factory1 = comp.load_component_from_url(url)
self.assertEqual(
task_factory1.__doc__,
component_dict['name'] + '\n' + component_dict['description'])
arg1 = 3
arg2 = 5
task1 = task_factory1(arg1, arg2)
self.assertEqual(
task1.component_ref.spec.implementation.container.image,
component_dict['implementation']['container']['image'])
resolved_cmd = _resolve_command_line_and_paths(
task1.component_ref.spec, task1.arguments)
self.assertEqual(resolved_cmd.args[0], str(arg1))
self.assertEqual(resolved_cmd.args[1], str(arg2))