def test_check_type_validation_of_task_spec_outputs(self): producer_component_text = '''\ outputs: - {name: out1, type: Integer} - {name: out2, type: String} implementation: container: image: busybox command: [touch, {outputPath: out1}, {outputPath: out2}] ''' consumer_component_text = '''\ inputs: - {name: data, type: Integer} implementation: container: image: busybox command: [echo, {inputValue: data}] ''' producer_op = comp.load_component_from_text(producer_component_text) consumer_op = comp.load_component_from_text(consumer_component_text) producer_task = producer_op() consumer_op(producer_task.outputs['out1']) consumer_op(producer_task.outputs['out2'].without_type()) consumer_op(producer_task.outputs['out2'].with_type('Integer')) with self.assertRaises(TypeError): consumer_op(producer_task.outputs['out2'])
def test_prevent_passing_unserializable_objects_as_argument(self): component_text = textwrap.dedent('''\ inputs: - {name: input 1} - {name: input 2} implementation: container: image: busybox command: - prog - {inputValue: input 1} - {inputPath: input 2} ''') component = comp.load_component_from_text(component_text) # Passing normal values to component task1 = component(input_1="value 1", input_2="value 2") # Passing unserializable values to component with self.assertRaises(TypeError): component(input_1=task1, input_2="value 2") with self.assertRaises(TypeError): component(input_1=open, input_2="value 2") with self.assertRaises(TypeError): component(input_1="value 1", input_2=task1) with self.assertRaises(TypeError): component(input_1="value 1", input_2=open)
def test_inputs_reordering_stability(self): """Tests input reordering stability. Required inputs and optional/default inputs should keep the ordering. In python signature, optional arguments must come after the required arguments. """ component_text = '''\ inputs: - {name: a1} - {name: b1, default: val} - {name: a2} - {name: b2, optional: True} - {name: a3} - {name: b3, default: val} - {name: a4} - {name: b4, optional: True} implementation: container: image: busybox ''' task_factory1 = comp.load_component_from_text(component_text) import inspect signature = inspect.signature(task_factory1) actual_signature = list(signature.parameters.keys()) self.assertSequenceEqual( actual_signature, ['a1', 'a2', 'a3', 'a4', 'b1', 'b2', 'b3', 'b4'], str)
def test_input_output_uri_resolving(self): component_text = textwrap.dedent('''\ inputs: - {name: In1} outputs: - {name: Out1} implementation: container: image: busybox command: - program - --in1-uri - {inputUri: In1} - --out1-uri - {outputUri: Out1} ''') op = load_component_from_text(text=component_text) task = op(in1='foo') self.assertEqual([ 'program', '--in1-uri', '{{$.inputs.artifacts[\'In1\'].uri}}', '--out1-uri', '{{$.outputs.artifacts[\'Out1\'].uri}}', ], task.command)
def test_handle_output_names_with_spaces(self): component_text = '''\ outputs: - {name: Training data} implementation: container: image: busybox ''' task_factory1 = comp.load_component_from_text(component_text)
def test_handle_underscored_output_names(self): component_text = '''\ outputs: - {name: Data} - {name: _Data} implementation: container: image: busybox ''' task_factory1 = comp.load_component_from_text(component_text)
def test_fail_on_duplicate_output_names(self): component_text = '''\ outputs: - {name: Data1} - {name: Data1} implementation: container: image: busybox ''' with self.assertRaises(ValueError): task_factory1 = comp.load_component_from_text(component_text)
def test_handle_similar_input_names(self): component_text = '''\ inputs: - {name: Input 1} - {name: Input_1} - {name: Input-1} implementation: container: image: busybox ''' task_factory1 = comp.load_component_from_text(component_text)
def test_conversion_to_container_op(self): component_text = textwrap.dedent('''\ name: Custom component implementation: container: image: busybox ''') task_factory1 = load_component_from_text(component_text) task1 = task_factory1() self.assertEqual(task1.human_name, 'Custom component')
def test_handle_file_outputs_with_spaces(self): component_text = '''\ outputs: - {name: Output data} implementation: container: image: busybox fileOutputs: Output data: /outputs/output-data ''' task_factory1 = comp.load_component_from_text(component_text)
def test_handle_duplicate_input_output_names(self): component_text = '''\ inputs: - {name: Data} outputs: - {name: Data} implementation: container: image: busybox ''' task_factory1 = comp.load_component_from_text(component_text)
def test_fail_on_unknown_file_output(self): component_text = '''\ outputs: - {name: Data} implementation: container: image: busybox fileOutputs: Wrong: '/outputs/output.txt' ''' with self.assertRaises(TypeError): task_factory1 = comp.load_component_from_text(component_text)
def test_fail_on_unknown_value_argument(self): component_text = '''\ inputs: - {name: Data} implementation: container: image: busybox args: - {inputValue: Wrong} ''' with self.assertRaises(TypeError): task_factory1 = comp.load_component_from_text(component_text)
def test_type_compatibility_check_not_failing_when_type_is_ignored(self): component_a = textwrap.dedent('''\ outputs: - {name: out1, type: type_A} implementation: container: image: busybox command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] ''') component_b = textwrap.dedent('''\ inputs: - {name: in1, type: type_Z} implementation: container: image: busybox command: [echo, {inputValue: in1}] ''') task_factory_a = load_component_from_text(component_a) task_factory_b = load_component_from_text(component_b) a_task = task_factory_a() b_task = task_factory_b(in1=a_task.outputs['out1'].ignore_type())
def test_type_compatibility_check_for_types_with_parameters(self): component_a = '''\ outputs: - {name: out1, type: {parametrized_type: {property_a: value_a, property_b: value_b}}} implementation: container: image: busybox command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] ''' component_b = '''\ inputs: - {name: in1, type: {parametrized_type: {property_a: value_a, property_b: value_b}}} implementation: container: image: busybox command: [echo, {inputValue: in1}] ''' task_factory_a = comp.load_component_from_text(component_a) task_factory_b = comp.load_component_from_text(component_b) a_task = task_factory_a() b_task = task_factory_b(in1=a_task.outputs['out1'])
def test_type_compatibility_check_for_types_with_schema(self): component_a = '''\ outputs: - {name: out1, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: "^gs://.*$" } }}} implementation: container: image: busybox command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] ''' component_b = '''\ inputs: - {name: in1, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: "^gs://.*$" } }}} implementation: container: image: busybox command: [echo, {inputValue: in1}] ''' task_factory_a = comp.load_component_from_text(component_a) task_factory_b = comp.load_component_from_text(component_b) a_task = task_factory_a() b_task = task_factory_b(in1=a_task.outputs['out1'])
def test_type_compatibility_check_when_argument_type_is_missing(self): component_a = '''\ outputs: - {name: out1} implementation: container: image: busybox command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] ''' component_b = '''\ inputs: - {name: in1, type: custom_type} implementation: container: image: busybox command: [echo, {inputValue: in1}] ''' task_factory_a = comp.load_component_from_text(component_a) task_factory_b = comp.load_component_from_text(component_b) a_task = task_factory_a() b_task = task_factory_b(in1=a_task.outputs['out1'])
def test_digest_of_loaded_component(self): component_text = textwrap.dedent('''\ implementation: container: image: busybox ''') task_factory1 = comp.load_component_from_text(component_text) task1 = task_factory1() self.assertEqual( task1.component_ref.digest, '1ede211233e869581d098673962c2c1e8c1e4cebb7cf5d7332c2f73cb4900823')
def test_fail_executor_input_with_key(self): test_component = textwrap.dedent("""\ inputs: - {name: in1} outputs: - {name: out1} implementation: container: image: busybox command: [echo, {executorInput: a_bad_key}] """) with self.assertRaises(TypeError): _ = load_component_from_text(test_component)
def test_check_task_object_no_output_attribute_when_0_outputs(self): component_text = textwrap.dedent( '''\ implementation: container: image: busybox command: [] ''',) op = comp.load_component_from_text(component_text) task = op() self.assertFalse(hasattr(task, 'output'))
def test_fail_type_compatibility_check_when_type_property_value_is_different( self): component_a = '''\ outputs: - {name: out1, type: {parametrized_type: {property_a: value_a}}} implementation: container: image: busybox command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] ''' component_b = '''\ inputs: - {name: in1, type: {parametrized_type: {property_a: DIFFERENT VALUE}}} implementation: container: image: busybox command: [echo, {inputValue: in1}] ''' task_factory_a = comp.load_component_from_text(component_a) task_factory_b = comp.load_component_from_text(component_b) a_task = task_factory_a() with self.assertRaises(TypeError): b_task = task_factory_b(in1=a_task.outputs['out1'])
def test_type_compatibility_check_when_using_positional_arguments(self): """Tests that `op2(task1.output)` works as good as `op2(in1=task1.output)`""" component_a = '''\ outputs: - {name: out1, type: {parametrized_type: {property_a: value_a, property_b: value_b}}} implementation: container: image: busybox command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] ''' component_b = '''\ inputs: - {name: in1, type: {parametrized_type: {property_a: value_a, property_b: value_b}}} implementation: container: image: busybox command: [echo, {inputValue: in1}] ''' task_factory_a = comp.load_component_from_text(component_a) task_factory_b = comp.load_component_from_text(component_b) a_task = task_factory_a() b_task = task_factory_b(a_task.outputs['out1'])
def test_volatile_components(self): component_text = textwrap.dedent('''\ metadata: annotations: volatile_component: "true" implementation: container: image: busybox ''') task_factory1 = load_component_from_text(text=component_text) task1 = task_factory1() self.assertEqual( task1.execution_options.caching_strategy.max_cache_staleness, 'P0D')
def test_nonpythonic_container_output_handled_by_graph(self): component_a = textwrap.dedent('''\ inputs: [] outputs: - {name: out1, type: str} implementation: graph: tasks: some-container: arguments: {} componentRef: spec: outputs: - {name: out-1, type: str} implementation: container: image: busybox command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out-1}] outputValues: out1: taskOutput: taskId: some-container outputName: out-1 ''') component_b = textwrap.dedent('''\ inputs: - {name: in1, type: str} implementation: container: image: busybox command: [echo, {inputValue: in1}] ''') task_factory_a = load_component_from_text(component_a) task_factory_b = load_component_from_text(component_b) a_task = task_factory_a() b_task = task_factory_b(in1=a_task.outputs['out1'])
def test_container_component_without_command_should_warn(self): component_a = '''\ name: component without command inputs: - {name: in1, type: String} implementation: container: image: busybox ''' with self.assertWarnsRegex( FutureWarning, 'Container component must specify command to be compatible with ' 'KFP v2 compatible mode and emissary executor'): task_factory_a = comp.load_component_from_text(component_a)
def test_check_task_object_has_output_attribute_when_1_output(self): component_text = textwrap.dedent( '''\ outputs: - {name: out 1} implementation: container: image: busybox command: [touch, {outputPath: out 1}] ''',) op = comp.load_component_from_text(component_text) task = op() self.assertEqual(task.output.task_output.output_name, 'out 1')
def test_check_task_spec_outputs_dictionary(self): component_text = '''\ outputs: - {name: out 1} - {name: out 2} implementation: container: image: busybox command: [touch, {outputPath: out 1}, {outputPath: out 2}] ''' op = comp.load_component_from_text(component_text) task = op() self.assertEqual(list(task.outputs.keys()), ['out 1', 'out 2'])
def test_passing_component_metadata_to_container_op(self): component_text = textwrap.dedent('''\ metadata: annotations: key1: value1 labels: key1: value1 implementation: container: image: busybox ''') task_factory1 = load_component_from_text(text=component_text) task1 = task_factory1() self.assertEqual(task1.pod_annotations['key1'], 'value1') self.assertEqual(task1.pod_labels['key1'], 'value1')
def test_accessing_component_spec_from_task_factory(self): component_text = '''\ implementation: container: image: busybox ''' task_factory1 = comp.load_component_from_text(component_text) actual_component_spec = task_factory1.component_spec actual_component_spec_dict = actual_component_spec.to_dict() expected_component_spec_dict = load_yaml(component_text) expected_component_spec = ComponentSpec.from_dict( expected_component_spec_dict) self.assertEqual(expected_component_spec_dict, actual_component_spec_dict) self.assertEqual(expected_component_spec, task_factory1.component_spec)
def test_check_task_object_no_output_attribute_when_multiple_outputs(self): component_text = textwrap.dedent( '''\ outputs: - {name: out 1} - {name: out 2} implementation: container: image: busybox command: [touch, {outputPath: out 1}, {outputPath: out 2}] ''',) op = comp.load_component_from_text(component_text) task = op() self.assertFalse(hasattr(task, 'output'))