def test_check_type_validation_of_task_spec_outputs(self):
producer_component_text = '''\
outputs:
- {name: out1, type: Integer}
- {name: out2, type: String}
implementation:
container:
image: busybox
command: [touch, {outputPath: out1}, {outputPath: out2}]
'''
consumer_component_text = '''\
inputs:
- {name: data, type: Integer}
implementation:
container:
image: busybox
command: [echo, {inputValue: data}]
'''
producer_op = comp.load_component_from_text(producer_component_text)
consumer_op = comp.load_component_from_text(consumer_component_text)
producer_task = producer_op()
consumer_op(producer_task.outputs['out1'])
consumer_op(producer_task.outputs['out2'].without_type())
consumer_op(producer_task.outputs['out2'].with_type('Integer'))
with self.assertRaises(TypeError):
consumer_op(producer_task.outputs['out2'])
def test_prevent_passing_unserializable_objects_as_argument(self):
component_text = textwrap.dedent('''\
inputs:
- {name: input 1}
- {name: input 2}
implementation:
container:
image: busybox
command:
- prog
- {inputValue: input 1}
- {inputPath: input 2}
''')
component = comp.load_component_from_text(component_text)
# Passing normal values to component
task1 = component(input_1="value 1", input_2="value 2")
# Passing unserializable values to component
with self.assertRaises(TypeError):
component(input_1=task1, input_2="value 2")
with self.assertRaises(TypeError):
component(input_1=open, input_2="value 2")
with self.assertRaises(TypeError):
component(input_1="value 1", input_2=task1)
with self.assertRaises(TypeError):
component(input_1="value 1", input_2=open)
def test_inputs_reordering_stability(self):
"""Tests input reordering stability.
Required inputs and optional/default inputs should keep the
ordering. In python signature, optional arguments must come
after the required arguments.
"""
component_text = '''\
inputs:
- {name: a1}
- {name: b1, default: val}
- {name: a2}
- {name: b2, optional: True}
- {name: a3}
- {name: b3, default: val}
- {name: a4}
- {name: b4, optional: True}
implementation:
container:
image: busybox
'''
task_factory1 = comp.load_component_from_text(component_text)
import inspect
signature = inspect.signature(task_factory1)
actual_signature = list(signature.parameters.keys())
self.assertSequenceEqual(
actual_signature, ['a1', 'a2', 'a3', 'a4', 'b1', 'b2', 'b3', 'b4'],
str)
def test_input_output_uri_resolving(self):
component_text = textwrap.dedent('''\
inputs:
- {name: In1}
outputs:
- {name: Out1}
implementation:
container:
image: busybox
command:
- program
- --in1-uri
- {inputUri: In1}
- --out1-uri
- {outputUri: Out1}
''')
op = load_component_from_text(text=component_text)
task = op(in1='foo')
self.assertEqual([
'program',
'--in1-uri',
'{{$.inputs.artifacts[\'In1\'].uri}}',
'--out1-uri',
'{{$.outputs.artifacts[\'Out1\'].uri}}',
], task.command)
def test_handle_output_names_with_spaces(self):
component_text = '''\
outputs:
- {name: Training data}
implementation:
container:
image: busybox
'''
task_factory1 = comp.load_component_from_text(component_text)
def test_handle_underscored_output_names(self):
component_text = '''\
outputs:
- {name: Data}
- {name: _Data}
implementation:
container:
image: busybox
'''
task_factory1 = comp.load_component_from_text(component_text)
def test_fail_on_duplicate_output_names(self):
component_text = '''\
outputs:
- {name: Data1}
- {name: Data1}
implementation:
container:
image: busybox
'''
with self.assertRaises(ValueError):
task_factory1 = comp.load_component_from_text(component_text)
def test_handle_similar_input_names(self):
component_text = '''\
inputs:
- {name: Input 1}
- {name: Input_1}
- {name: Input-1}
implementation:
container:
image: busybox
'''
task_factory1 = comp.load_component_from_text(component_text)
def test_conversion_to_container_op(self):
component_text = textwrap.dedent('''\
name: Custom component
implementation:
container:
image: busybox
''')
task_factory1 = load_component_from_text(component_text)
task1 = task_factory1()
self.assertEqual(task1.human_name, 'Custom component')
def test_handle_file_outputs_with_spaces(self):
component_text = '''\
outputs:
- {name: Output data}
implementation:
container:
image: busybox
fileOutputs:
Output data: /outputs/output-data
'''
task_factory1 = comp.load_component_from_text(component_text)
def test_handle_duplicate_input_output_names(self):
component_text = '''\
inputs:
- {name: Data}
outputs:
- {name: Data}
implementation:
container:
image: busybox
'''
task_factory1 = comp.load_component_from_text(component_text)
def test_fail_on_unknown_file_output(self):
component_text = '''\
outputs:
- {name: Data}
implementation:
container:
image: busybox
fileOutputs:
Wrong: '/outputs/output.txt'
'''
with self.assertRaises(TypeError):
task_factory1 = comp.load_component_from_text(component_text)
def test_fail_on_unknown_value_argument(self):
component_text = '''\
inputs:
- {name: Data}
implementation:
container:
image: busybox
args:
- {inputValue: Wrong}
'''
with self.assertRaises(TypeError):
task_factory1 = comp.load_component_from_text(component_text)
def test_type_compatibility_check_not_failing_when_type_is_ignored(self):
component_a = textwrap.dedent('''\
outputs:
- {name: out1, type: type_A}
implementation:
container:
image: busybox
command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}]
''')
component_b = textwrap.dedent('''\
inputs:
- {name: in1, type: type_Z}
implementation:
container:
image: busybox
command: [echo, {inputValue: in1}]
''')
task_factory_a = load_component_from_text(component_a)
task_factory_b = load_component_from_text(component_b)
a_task = task_factory_a()
b_task = task_factory_b(in1=a_task.outputs['out1'].ignore_type())
def test_type_compatibility_check_for_types_with_parameters(self):
component_a = '''\
outputs:
- {name: out1, type: {parametrized_type: {property_a: value_a, property_b: value_b}}}
implementation:
container:
image: busybox
command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}]
'''
component_b = '''\
inputs:
- {name: in1, type: {parametrized_type: {property_a: value_a, property_b: value_b}}}
implementation:
container:
image: busybox
command: [echo, {inputValue: in1}]
'''
task_factory_a = comp.load_component_from_text(component_a)
task_factory_b = comp.load_component_from_text(component_b)
a_task = task_factory_a()
b_task = task_factory_b(in1=a_task.outputs['out1'])
def test_type_compatibility_check_for_types_with_schema(self):
component_a = '''\
outputs:
- {name: out1, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: "^gs://.*$" } }}}
implementation:
container:
image: busybox
command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}]
'''
component_b = '''\
inputs:
- {name: in1, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: "^gs://.*$" } }}}
implementation:
container:
image: busybox
command: [echo, {inputValue: in1}]
'''
task_factory_a = comp.load_component_from_text(component_a)
task_factory_b = comp.load_component_from_text(component_b)
a_task = task_factory_a()
b_task = task_factory_b(in1=a_task.outputs['out1'])
def test_type_compatibility_check_when_argument_type_is_missing(self):
component_a = '''\
outputs:
- {name: out1}
implementation:
container:
image: busybox
command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}]
'''
component_b = '''\
inputs:
- {name: in1, type: custom_type}
implementation:
container:
image: busybox
command: [echo, {inputValue: in1}]
'''
task_factory_a = comp.load_component_from_text(component_a)
task_factory_b = comp.load_component_from_text(component_b)
a_task = task_factory_a()
b_task = task_factory_b(in1=a_task.outputs['out1'])
def test_digest_of_loaded_component(self):
component_text = textwrap.dedent('''\
implementation:
container:
image: busybox
''')
task_factory1 = comp.load_component_from_text(component_text)
task1 = task_factory1()
self.assertEqual(
task1.component_ref.digest,
'1ede211233e869581d098673962c2c1e8c1e4cebb7cf5d7332c2f73cb4900823')
def test_fail_executor_input_with_key(self):
test_component = textwrap.dedent("""\
inputs:
- {name: in1}
outputs:
- {name: out1}
implementation:
container:
image: busybox
command: [echo, {executorInput: a_bad_key}]
""")
with self.assertRaises(TypeError):
_ = load_component_from_text(test_component)
def test_check_task_object_no_output_attribute_when_0_outputs(self):
component_text = textwrap.dedent(
'''\
implementation:
container:
image: busybox
command: []
''',)
op = comp.load_component_from_text(component_text)
task = op()
self.assertFalse(hasattr(task, 'output'))
def test_fail_type_compatibility_check_when_type_property_value_is_different(
self):
component_a = '''\
outputs:
- {name: out1, type: {parametrized_type: {property_a: value_a}}}
implementation:
container:
image: busybox
command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}]
'''
component_b = '''\
inputs:
- {name: in1, type: {parametrized_type: {property_a: DIFFERENT VALUE}}}
implementation:
container:
image: busybox
command: [echo, {inputValue: in1}]
'''
task_factory_a = comp.load_component_from_text(component_a)
task_factory_b = comp.load_component_from_text(component_b)
a_task = task_factory_a()
with self.assertRaises(TypeError):
b_task = task_factory_b(in1=a_task.outputs['out1'])
def test_type_compatibility_check_when_using_positional_arguments(self):
"""Tests that `op2(task1.output)` works as good as
`op2(in1=task1.output)`"""
component_a = '''\
outputs:
- {name: out1, type: {parametrized_type: {property_a: value_a, property_b: value_b}}}
implementation:
container:
image: busybox
command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}]
'''
component_b = '''\
inputs:
- {name: in1, type: {parametrized_type: {property_a: value_a, property_b: value_b}}}
implementation:
container:
image: busybox
command: [echo, {inputValue: in1}]
'''
task_factory_a = comp.load_component_from_text(component_a)
task_factory_b = comp.load_component_from_text(component_b)
a_task = task_factory_a()
b_task = task_factory_b(a_task.outputs['out1'])
def test_volatile_components(self):
component_text = textwrap.dedent('''\
metadata:
annotations:
volatile_component: "true"
implementation:
container:
image: busybox
''')
task_factory1 = load_component_from_text(text=component_text)
task1 = task_factory1()
self.assertEqual(
task1.execution_options.caching_strategy.max_cache_staleness, 'P0D')
def test_nonpythonic_container_output_handled_by_graph(self):
component_a = textwrap.dedent('''\
inputs: []
outputs:
- {name: out1, type: str}
implementation:
graph:
tasks:
some-container:
arguments: {}
componentRef:
spec:
outputs:
- {name: out-1, type: str}
implementation:
container:
image: busybox
command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out-1}]
outputValues:
out1:
taskOutput:
taskId: some-container
outputName: out-1
''')
component_b = textwrap.dedent('''\
inputs:
- {name: in1, type: str}
implementation:
container:
image: busybox
command: [echo, {inputValue: in1}]
''')
task_factory_a = load_component_from_text(component_a)
task_factory_b = load_component_from_text(component_b)
a_task = task_factory_a()
b_task = task_factory_b(in1=a_task.outputs['out1'])
def test_container_component_without_command_should_warn(self):
component_a = '''\
name: component without command
inputs:
- {name: in1, type: String}
implementation:
container:
image: busybox
'''
with self.assertWarnsRegex(
FutureWarning,
'Container component must specify command to be compatible with '
'KFP v2 compatible mode and emissary executor'):
task_factory_a = comp.load_component_from_text(component_a)
def test_check_task_object_has_output_attribute_when_1_output(self):
component_text = textwrap.dedent(
'''\
outputs:
- {name: out 1}
implementation:
container:
image: busybox
command: [touch, {outputPath: out 1}]
''',)
op = comp.load_component_from_text(component_text)
task = op()
self.assertEqual(task.output.task_output.output_name, 'out 1')
def test_check_task_spec_outputs_dictionary(self):
component_text = '''\
outputs:
- {name: out 1}
- {name: out 2}
implementation:
container:
image: busybox
command: [touch, {outputPath: out 1}, {outputPath: out 2}]
'''
op = comp.load_component_from_text(component_text)
task = op()
self.assertEqual(list(task.outputs.keys()), ['out 1', 'out 2'])
def test_passing_component_metadata_to_container_op(self):
component_text = textwrap.dedent('''\
metadata:
annotations:
key1: value1
labels:
key1: value1
implementation:
container:
image: busybox
''')
task_factory1 = load_component_from_text(text=component_text)
task1 = task_factory1()
self.assertEqual(task1.pod_annotations['key1'], 'value1')
self.assertEqual(task1.pod_labels['key1'], 'value1')
def test_accessing_component_spec_from_task_factory(self):
component_text = '''\
implementation:
container:
image: busybox
'''
task_factory1 = comp.load_component_from_text(component_text)
actual_component_spec = task_factory1.component_spec
actual_component_spec_dict = actual_component_spec.to_dict()
expected_component_spec_dict = load_yaml(component_text)
expected_component_spec = ComponentSpec.from_dict(
expected_component_spec_dict)
self.assertEqual(expected_component_spec_dict,
actual_component_spec_dict)
self.assertEqual(expected_component_spec, task_factory1.component_spec)
def test_check_task_object_no_output_attribute_when_multiple_outputs(self):
component_text = textwrap.dedent(
'''\
outputs:
- {name: out 1}
- {name: out 2}
implementation:
container:
image: busybox
command: [touch, {outputPath: out 1}, {outputPath: out 2}]
''',)
op = comp.load_component_from_text(component_text)
task = op()
self.assertFalse(hasattr(task, 'output'))
