def test_check_type_validation_of_task_spec_outputs(self):
producer_component_text = '''\
outputs:
- {name: out1, type: Integer}
- {name: out2, type: String}
implementation:
container:
image: busybox
command: [touch, {outputPath: out1}, {outputPath: out2}]
'''
consumer_component_text = '''\
inputs:
- {name: data, type: Integer}
implementation:
container:
image: busybox
command: [echo, {inputValue: data}]
'''
producer_op = comp.load_component_from_text(producer_component_text)
consumer_op = comp.load_component_from_text(consumer_component_text)
producer_task = producer_op()
consumer_op(producer_task.outputs['out1'])
consumer_op(producer_task.outputs['out2'].without_type())
consumer_op(producer_task.outputs['out2'].with_type('Integer'))
with self.assertRaises(TypeError):
consumer_op(producer_task.outputs['out2'])
def test_inputs_reordering_stability(self):
'''Tests input reordering stability. Required inputs and optional/default inputs should keep the ordering.
In python signature, optional arguments must come after the required arguments.
'''
component_text = '''\
inputs:
- {name: a1}
- {name: b1, default: val}
- {name: a2}
- {name: b2, optional: True}
- {name: a3}
- {name: b3, default: val}
- {name: a4}
- {name: b4, optional: True}
implementation:
container:
image: busybox
'''
task_factory1 = comp.load_component_from_text(component_text)
import inspect
signature = inspect.signature(task_factory1)
actual_signature = list(signature.parameters.keys())
self.assertSequenceEqual(
actual_signature, ['a1', 'a2', 'a3', 'a4', 'b1', 'b2', 'b3', 'b4'],
str)
def test_handle_output_names_with_spaces(self):
component_text = '''\
outputs:
- {name: Training data}
implementation:
container:
image: busybox
'''
task_factory1 = comp.load_component_from_text(component_text)
def test_handle_underscored_output_names(self):
component_text = '''\
outputs:
- {name: Data}
- {name: _Data}
implementation:
container:
image: busybox
'''
task_factory1 = comp.load_component_from_text(component_text)
def test_handle_duplicate_input_output_names(self):
component_text = '''\
inputs:
- {name: Data}
outputs:
- {name: Data}
implementation:
container:
image: busybox
'''
task_factory1 = comp.load_component_from_text(component_text)
def test_handle_similar_input_names(self):
component_text = '''\
inputs:
- {name: Input 1}
- {name: Input_1}
- {name: Input-1}
implementation:
container:
image: busybox
'''
task_factory1 = comp.load_component_from_text(component_text)
def test_handle_file_outputs_with_spaces(self):
component_text = '''\
outputs:
- {name: Output data}
implementation:
container:
image: busybox
fileOutputs:
Output data: /outputs/output-data
'''
task_factory1 = comp.load_component_from_text(component_text)
def test_fail_on_duplicate_output_names(self):
component_text = '''\
outputs:
- {name: Data1}
- {name: Data1}
implementation:
container:
image: busybox
'''
with self.assertRaises(ValueError):
task_factory1 = comp.load_component_from_text(component_text)
def test_type_compatibility_check_for_types_with_schema(self):
component_a = '''\
outputs:
- {name: out1, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: "^gs://.*$" } }}}
implementation:
container:
image: busybox
command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}]
'''
component_b = '''\
inputs:
- {name: in1, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: "^gs://.*$" } }}}
implementation:
container:
image: busybox
command: [echo, {inputValue: in1}]
'''
task_factory_a = comp.load_component_from_text(component_a)
task_factory_b = comp.load_component_from_text(component_b)
a_task = task_factory_a()
b_task = task_factory_b(in1=a_task.outputs['out1'])
def test_type_compatibility_check_for_types_with_parameters(self):
component_a = '''\
outputs:
- {name: out1, type: {parametrized_type: {property_a: value_a, property_b: value_b}}}
implementation:
container:
image: busybox
command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}]
'''
component_b = '''\
inputs:
- {name: in1, type: {parametrized_type: {property_a: value_a, property_b: value_b}}}
implementation:
container:
image: busybox
command: [echo, {inputValue: in1}]
'''
task_factory_a = comp.load_component_from_text(component_a)
task_factory_b = comp.load_component_from_text(component_b)
a_task = task_factory_a()
b_task = task_factory_b(in1=a_task.outputs['out1'])
def test_type_compatibility_check_not_failing_when_type_is_ignored(self):
component_a = '''\
outputs:
- {name: out1, type: type_A}
implementation:
container:
image: busybox
command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}]
'''
component_b = '''\
inputs:
- {name: in1, type: type_Z}
implementation:
container:
image: busybox
command: [echo, {inputValue: in1}]
'''
task_factory_a = comp.load_component_from_text(component_a)
task_factory_b = comp.load_component_from_text(component_b)
a_task = task_factory_a()
b_task = task_factory_b(in1=a_task.outputs['out1'].without_type())
def test_fail_on_unknown_file_output(self):
component_text = '''\
outputs:
- {name: Data}
implementation:
container:
image: busybox
fileOutputs:
Wrong: '/outputs/output.txt'
'''
with self.assertRaises(TypeError):
task_factory1 = comp.load_component_from_text(component_text)
def test_fail_on_unknown_value_argument(self):
component_text = '''\
inputs:
- {name: Data}
implementation:
container:
image: busybox
args:
- {inputValue: Wrong}
'''
with self.assertRaises(TypeError):
task_factory1 = comp.load_component_from_text(component_text)
def test_type_compatibility_check_when_using_positional_arguments(self):
"""Tests that `op2(task1.output)` works as good as `op2(in1=task1.output)`"""
component_a = '''\
outputs:
- {name: out1, type: {parametrized_type: {property_a: value_a, property_b: value_b}}}
implementation:
container:
image: busybox
command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}]
'''
component_b = '''\
inputs:
- {name: in1, type: {parametrized_type: {property_a: value_a, property_b: value_b}}}
implementation:
container:
image: busybox
command: [echo, {inputValue: in1}]
'''
task_factory_a = comp.load_component_from_text(component_a)
task_factory_b = comp.load_component_from_text(component_b)
a_task = task_factory_a()
b_task = task_factory_b(a_task.outputs['out1'])
def test_fail_type_compatibility_check_when_type_property_value_is_different(
self):
component_a = '''\
outputs:
- {name: out1, type: {parametrized_type: {property_a: value_a}}}
implementation:
container:
image: busybox
command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}]
'''
component_b = '''\
inputs:
- {name: in1, type: {parametrized_type: {property_a: DIFFERENT VALUE}}}
implementation:
container:
image: busybox
command: [echo, {inputValue: in1}]
'''
task_factory_a = comp.load_component_from_text(component_a)
task_factory_b = comp.load_component_from_text(component_b)
a_task = task_factory_a()
with self.assertRaises(TypeError):
b_task = task_factory_b(in1=a_task.outputs['out1'])
def test_check_task_spec_outputs_dictionary(self):
component_text = '''\
outputs:
- {name: out 1}
- {name: out 2}
implementation:
container:
image: busybox
command: [touch, {outputPath: out 1}, {outputPath: out 2}]
'''
op = comp.load_component_from_text(component_text)
task = op()
self.assertEqual(list(task.outputs.keys()), ['out 1', 'out 2'])
def test_accessing_component_spec_from_task_factory(self):
component_text = '''\
implementation:
container:
image: busybox
'''
task_factory1 = comp.load_component_from_text(component_text)
actual_component_spec = task_factory1.component_spec
actual_component_spec_dict = actual_component_spec.to_dict()
expected_component_spec_dict = load_yaml(component_text)
expected_component_spec = ComponentSpec.from_dict(
expected_component_spec_dict)
self.assertEqual(expected_component_spec_dict,
actual_component_spec_dict)
self.assertEqual(expected_component_spec, task_factory1.component_spec)
def test_inputs_reordering_when_inputs_have_defaults(self):
'''Tests reordering of inputs with default values.
In python signature, optional arguments must come after the required arguments.
'''
component_text = '''\
inputs:
- {name: in1}
- {name: in2, default: val}
- {name: in3}
implementation:
container:
image: busybox
'''
task_factory1 = comp.load_component_from_text(component_text)
import inspect
signature = inspect.signature(task_factory1)
actual_signature = list(signature.parameters.keys())
self.assertSequenceEqual(actual_signature, ['in1', 'in3', 'in2'], str)
def test_input_path_placeholder_with_constant_argument(self):
component_text = '''\
inputs:
- {name: input 1}
implementation:
container:
image: busybox
command:
- --input-data
- {inputPath: input 1}
'''
task_factory1 = comp.load_component_from_text(component_text)
task1 = task_factory1('Text')
resolved_cmd = _resolve_command_line_and_paths(
task1.component_ref.spec, task1.arguments)
self.assertEqual(resolved_cmd.command,
['--input-data', resolved_cmd.input_paths['input 1']])
self.assertEqual(task1.arguments, {'input 1': 'Text'})
def test_missing_optional_input_file_argument(self):
'''Missing optional inputs should resolve to nothing'''
component_text = '''\
inputs:
- {name: input 1, optional: true}
implementation:
container:
image: busybox
command:
- a
- {inputPath: input 1}
- z
'''
task_factory1 = comp.load_component_from_text(component_text)
task1 = task_factory1()
resolved_cmd = _resolve_command_line_and_paths(
task1.component_ref.spec, task1.arguments)
self.assertEqual(resolved_cmd.command, ['a', 'z'])
def test_handle_default_values_in_task_factory(self):
component_text = '''\
inputs:
- {name: Data, default: '123'}
implementation:
container:
image: busybox
args:
- {inputValue: Data}
'''
task_factory1 = comp.load_component_from_text(text=component_text)
task1 = task_factory1()
resolved_cmd1 = _resolve_command_line_and_paths(
task1.component_ref.spec, task1.arguments)
self.assertEqual(resolved_cmd1.args, ['123'])
task2 = task_factory1('456')
resolved_cmd2 = _resolve_command_line_and_paths(
task2.component_ref.spec, task2.arguments)
self.assertEqual(resolved_cmd2.args, ['456'])
def test_load_nested_graph_components(self):
component_text = '''\
inputs:
- {name: graph in 1}
- {name: graph in 2}
outputs:
- {name: graph out 1}
- {name: graph out 2}
- {name: graph out 3}
- {name: graph out 4}
implementation:
graph:
tasks:
task 1:
componentRef:
spec:
name: Component 1
inputs:
- {name: in1_1}
outputs:
- {name: out1_1}
- {name: out1_2}
implementation:
container:
image: busybox
command: [sh, -c, 'echo "$0" > $1; echo "$0" > $2', {inputValue: in1_1}, {outputPath: out1_1}, {outputPath: out1_2}]
arguments:
in1_1: '11'
task 2:
componentRef:
spec:
name: Component 2
inputs:
- {name: in2_1}
- {name: in2_2}
outputs:
- {name: out2_1}
implementation:
container:
image: busybox
command: [sh, -c, 'cat "$0" "$1" > $2', {inputValue: in2_1}, {inputValue: in2_2}, {outputPath: out2_1}]
arguments:
in2_1: '21'
in2_2: {taskOutput: {taskId: task 1, outputName: out1_1}}
task 3:
componentRef:
spec:
inputs:
- {name: in3_1}
- {name: in3_2}
outputs:
- {name: out3_1}
implementation:
graph:
tasks:
graph subtask:
componentRef:
spec:
name: Component 3
inputs:
- {name: in3_1}
- {name: in3_2}
outputs:
- {name: out3_1}
implementation:
container:
image: busybox
command: [sh, -c, 'cat "$0" "$1" > $2', {inputValue: in3_1}, {inputValue: in3_2}, {outputPath: out3_1}]
arguments:
in3_1: {graphInput: {inputName: in3_1}}
in3_2: {graphInput: {inputName: in3_1}}
outputValues:
out3_1: {taskOutput: {taskId: graph subtask, outputName: out3_1}}
arguments:
in3_1: {taskOutput: {taskId: task 2, outputName: out2_1}}
in3_2: {graphInput: {inputName: graph in 1}}
outputValues:
graph out 1: {taskOutput: {taskId: task 3, outputName: out3_1}}
graph out 2: {taskOutput: {taskId: task 1, outputName: out1_2}}
graph out 3: {graphInput: {inputName: graph in 2}}
graph out 4: '42'
'''
op = comp.load_component_from_text(component_text)
old_value = comp._components._always_expand_graph_components = True
try:
task = op('graph 1', 'graph 2')
finally:
comp._components._always_expand_graph_components = old_value
self.assertIn(
'out3_1', str(task.outputs['graph out 1'])
) # Checks that the outputs coming from tasks in nested subgraphs are properly resolved.
self.assertIn('out1_2', str(task.outputs['graph out 2']))
self.assertEqual(task.outputs['graph out 3'], 'graph 2')
self.assertEqual(task.outputs['graph out 4'], '42')
def test_load_graph_component(self):
component_text = '''\
inputs:
- {name: graph in 1}
- {name: graph in 2}
outputs:
- {name: graph out 1}
- {name: graph out 2}
- {name: graph out 3}
- {name: graph out 4}
implementation:
graph:
tasks:
task 1:
componentRef:
spec:
name: Component 1
inputs:
- {name: in1_1}
outputs:
- {name: out1_1}
- {name: out1_2}
implementation:
container:
image: busybox
command: [sh, -c, 'echo "$0" > $1; echo "$0" > $2', {inputValue: in1_1}, {outputPath: out1_1}, {outputPath: out1_2}]
arguments:
in1_1: '11'
task 2:
componentRef:
spec:
name: Component 2
inputs:
- {name: in2_1}
- {name: in2_2}
outputs:
- {name: out2_1}
implementation:
container:
image: busybox
command: [sh, -c, 'cat "$0" "$1" > $2', {inputValue: in2_1}, {inputValue: in2_2}, {outputPath: out2_1}]
arguments:
in2_1: '21'
in2_2: {taskOutput: {taskId: task 1, outputName: out1_1}}
task 3:
componentRef:
spec:
name: Component 3
inputs:
- {name: in3_1}
- {name: in3_2}
outputs:
- {name: out3_1}
implementation:
container:
image: busybox
command: [sh, -c, 'cat "$0" "$1" > $2', {inputValue: in3_1}, {inputValue: in3_2}, {outputPath: out3_1}]
arguments:
in3_1: {taskOutput: {taskId: task 2, outputName: out2_1}}
in3_2: {graphInput: {inputName: graph in 1}}
outputValues:
graph out 1: {taskOutput: {taskId: task 3, outputName: out3_1}}
graph out 2: {taskOutput: {taskId: task 1, outputName: out1_2}}
graph out 3: {graphInput: {inputName: graph in 2}}
graph out 4: '42'
'''
op = comp.load_component_from_text(component_text)
task = op('graph 1', 'graph 2')
self.assertEqual(len(task.outputs), 4)
def test_load_component_from_text_fail_on_none_arg(self):
with self.assertRaises(TypeError):
comp.load_component_from_text(None)
