def test_reusable_component_warnings(self):
op1 = load_component_from_text('''\
implementation:
container:
image: busybox
''')
with warnings.catch_warnings(record=True) as warning_messages:
op1()
deprecation_messages = list(
str(message) for message in warning_messages
if message.category == DeprecationWarning)
self.assertListEqual(deprecation_messages, [])
with self.assertWarnsRegex(FutureWarning, expected_regex='reusable'):
kfp.dsl.ContainerOp(name='name', image='image')
def test_fail_type_compatibility_check_when_simple_type_name_is_different(
self):
component_a = '''\
outputs:
- {name: out1, type: type_A}
implementation:
container:
image: busybox
command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}]
'''
component_b = '''\
inputs:
- {name: in1, type: type_Z}
implementation:
container:
image: busybox
command: [echo, {inputValue: in1}]
'''
kfp.TYPE_CHECK = True
task_factory_a = comp.load_component_from_text(component_a)
task_factory_b = comp.load_component_from_text(component_b)
a_task = task_factory_a()
with self.assertRaises(InconsistentTypeException):
b_task = task_factory_b(in1=a_task.outputs['out1'])
def test_fail_type_compatibility_check_for_types_with_different_schemas(
self):
component_a = '''\
outputs:
- {name: out1, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: AAA } }}}
implementation:
container:
image: busybox
command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}]
'''
component_b = '''\
inputs:
- {name: in1, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: ZZZ } }}}
implementation:
container:
image: busybox
command: [echo, {inputValue: in1}]
'''
kfp.TYPE_CHECK = True
task_factory_a = comp.load_component_from_text(component_a)
task_factory_b = comp.load_component_from_text(component_b)
a_task = task_factory_a()
with self.assertRaises(InconsistentTypeException):
b_task = task_factory_b(in1=a_task.outputs['out1'])
def test_check_task_object_has_output_attribute_when_1_output(self):
component_text = textwrap.dedent(
'''\
outputs:
- {name: out 1}
implementation:
container:
image: busybox
command: [touch, {outputPath: out 1}]
''', )
op = comp.load_component_from_text(component_text)
task = op()
self.assertEqual(task.output.task_output.output_name, 'out 1')
def test_fail_type_compatibility_check_when_argument_type_has_missing_type_parameters(
self):
component_a = '''\
outputs:
- {name: out1, type: {parametrized_type: {property_a: value_a}}}
implementation:
container:
image: busybox
command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}]
'''
component_b = '''\
inputs:
- {name: in1, type: {parametrized_type: {property_a: value_a, property_b: value_b}}}
implementation:
container:
image: busybox
command: [echo, {inputValue: in1}]
'''
kfp.TYPE_CHECK = True
task_factory_a = comp.load_component_from_text(component_a)
task_factory_b = comp.load_component_from_text(component_b)
a_task = task_factory_a()
with self.assertRaises(InconsistentTypeException):
b_task = task_factory_b(in1=a_task.outputs['out1'])
def test_check_task_spec_outputs_dictionary(self):
component_text = '''\
outputs:
- {name: out 1}
- {name: out 2}
implementation:
container:
image: busybox
command: [touch, {outputPath: out 1}, {outputPath: out 2}]
'''
op = comp.load_component_from_text(component_text)
with no_task_resolving_context():
task = op()
self.assertEqual(list(task.outputs.keys()), ['out 1', 'out 2'])
def dkube_op(name, token, stage, **kwargs):
assert stage in VALID_STAGES, "Invalid value for stage, must be one of training/preprocessing/serving/custom"
component = None
path = Path(__file__).parent
with open('{}/dkube.yaml'.format(path), 'rb') as stream:
cdict = load_yaml(stream)
cdict['name'] = name
cidct['metadata']['labels']['stage'] = stage
cyaml = dump_yaml(cdict)
component = components.load_component_from_text(cyaml)
assert component != None, "Internal error, loading DKube component failed"
return component(name, token, stage, **kwargs)
def pipeline(my_pipe_param: int = 10):
loop_args = [1, 2]
# The DSL above should produce the same result and the DSL in the bottom
# with dsl.ParallelFor(loop_args, parallelism=1) as item:
# op1_template = components.load_component_from_text(op1_yaml)
# op1 = op1_template(item, my_pipe_param)
# condi_1 = tekton.CEL_ConditionOp(f"{item} == 0").output
# with dsl.Condition(condi_1 == 'true'):
# tekton.Break()
with tekton.Loop.sequential(loop_args) as item:
op1_template = components.load_component_from_text(op1_yaml)
op1 = op1_template(item, my_pipe_param)
condi_1 = tekton.CEL_ConditionOp(f"{item} == 1").output
with dsl.Condition(condi_1 == 'true'):
tekton.Break()
def test_nonpythonic_container_output_handled_by_graph(self):
component_a = textwrap.dedent('''\
inputs: []
outputs:
- {name: out1, type: str}
implementation:
graph:
tasks:
some-container:
arguments: {}
componentRef:
spec:
outputs:
- {name: out-1, type: str}
implementation:
container:
image: busybox
command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out-1}]
outputValues:
out1:
taskOutput:
taskId: some-container
outputName: out-1
''')
component_b = textwrap.dedent('''\
inputs:
- {name: in1, type: str}
implementation:
container:
image: busybox
command: [echo, {inputValue: in1}]
''')
task_factory_a = load_component_from_text(component_a)
task_factory_b = load_component_from_text(component_b)
a_task = task_factory_a()
b_task = task_factory_b(in1=a_task.outputs['out1'])
def test_handling_env(self):
component_text = '''\
implementation:
container:
image: busybox
env:
key1: value 1
key2: value 2
'''
task_factory1 = comp.load_component_from_text(component_text)
task1 = task_factory1()
actual_env = {env_var.name: env_var.value for env_var in task1.container.env}
expected_env = {'key1': 'value 1', 'key2': 'value 2'}
self.assertDictEqual(expected_env, actual_env)
def test_volatile_components(self):
component_text = textwrap.dedent('''\
metadata:
annotations:
volatile_component: "true"
implementation:
container:
image: busybox
''')
task_factory1 = load_component_from_text(text=component_text)
task1 = task_factory1()
self.assertEqual(
task1.execution_options.caching_strategy.max_cache_staleness,
'P0D')
def component_yaml_generator(**kwargs):
"""Function to create the actual component yaml for the input kwargs."""
inputs = ["inputs:"]
input_args = []
input_kwargs = {}
serialized_args = {"init": {}, "method": {}}
for key, value in kwargs.items():
prefix_key = "init" if key in init_arg_names else "method"
if isinstance(value, kfp.dsl.PipelineParam):
name = key
inputs.append("- {name: %s, type: Artifact}" % (name))
input_args.append("""
- --%s
- {inputUri: %s}
""" % (f"{prefix_key}.{key}", key))
input_kwargs[key] = value
else:
serialized_args[prefix_key][key] = value
inputs = "\n".join(inputs) if len(inputs) > 1 else ""
input_args = "\n".join(input_args) if input_args else ""
component_text = """
name: %s-%s
%s
outputs:
- {name: resource_name_output, type: Artifact}
implementation:
container:
image: gcr.io/sashaproject-1/AIPlatform_component:latest
command:
- python3
- remote_runner.py
- --cls_name=%s
- --method_name=%s
%s
args:
- --resource_name_output_uri
- {outputUri: resource_name_output}
%s
""" % (cls_name, method_name, inputs, cls_name, method_name,
make_args(serialized_args), input_args)
print(component_text)
return components.load_component_from_text(component_text)(
**input_kwargs)
def main_fn(url1: str = 'gs://ml-pipeline-playground/shakespeare1.txt'):
download1_task = gcs_download_op(url1)
printop = components.load_component_from_text("""
name: %s
description: print
inputs:
- {name: text, type: String}
implementation:
container:
image: alpine:3.6
command:
- echo
- {inputValue: text}
""" % ('print' * 10))(download1_task.outputs['Data-_123' * 100])
cel_condition = CEL_ConditionOp("'%s' == 'heads'" %
download1_task.outputs['Data-_123' * 100])
def test_passing_component_metadata_to_container_op(self):
component_text = textwrap.dedent('''\
metadata:
annotations:
key1: value1
labels:
key1: value1
implementation:
container:
image: busybox
''')
task_factory1 = load_component_from_text(text=component_text)
task1 = task_factory1()
self.assertEqual(task1.pod_annotations['key1'], 'value1')
self.assertEqual(task1.pod_labels['key1'], 'value1')
def test_convert_executor_input_and_output_metadata_placeholder(self):
test_component = textwrap.dedent("""\
inputs:
- {name: in1}
outputs:
- {name: out1}
implementation:
container:
image: busybox
command: [echo, {executorInput}, {outputMetadata}]
""")
task_factory = load_component_from_text(test_component)
task = task_factory(in1='foo')
self.assertListEqual(
['echo', '{{$}}', '/tmp/outputs/executor_output.json'],
task.command)
def test_input_path_placeholder_with_constant_argument(self):
component_text = '''\
inputs:
- {name: input 1}
implementation:
container:
image: busybox
command:
- --input-data
- {inputPath: input 1}
'''
task_factory1 = comp.load_component_from_text(component_text)
task1 = task_factory1('Text')
self.assertEqual(task1.command, ['--input-data', task1.input_artifact_paths['input 1']])
self.assertEqual(task1.artifact_arguments, {'input 1': 'Text'})
def echo_op(text: str):
return components.load_component_from_text("""
name: echo
description: print msg
inputs:
- {name: msg, type: String}
implementation:
container:
image: library/bash:4.4.23
command:
- sh
- -c
args:
- echo
- {inputValue: msg}
""")(msg=text)
def test_check_task_object_no_output_attribute_when_multiple_outputs(self):
component_text = textwrap.dedent(
'''\
outputs:
- {name: out 1}
- {name: out 2}
implementation:
container:
image: busybox
command: [touch, {outputPath: out 1}, {outputPath: out 2}]
''', )
op = comp.load_component_from_text(component_text)
task = op()
self.assertFalse(hasattr(task, 'output'))
def test_accessing_component_spec_from_task_factory(self):
component_text = '''\
implementation:
container:
image: busybox
'''
task_factory1 = comp.load_component_from_text(component_text)
actual_component_spec = task_factory1.component_spec
actual_component_spec_dict = actual_component_spec.to_dict()
expected_component_spec_dict = load_yaml(component_text)
expected_component_spec = ComponentSpec.from_dict(
expected_component_spec_dict)
self.assertEqual(expected_component_spec_dict,
actual_component_spec_dict)
self.assertEqual(expected_component_spec, task_factory1.component_spec)
def tekton_pipeline_variables():
task1 = components.load_component_from_text("""
name: task1
description: task1
implementation:
container:
image: registry.access.redhat.com/ubi8/ubi-minimal
command:
- /bin/bash
- -c
args:
- |
echo Pipeline name: $(context.pipeline.name), \
PipelineRun name: $(context.pipelineRun.name), \
PipelineRun namespace: $(context.pipelineRun.namespace), \
pipelineRun id: $(context.pipelineRun.uid)
""")()
def test_handling_env(self):
component_text = '''\
implementation:
container:
image: busybox
env:
key1: value 1
key2: value 2
'''
task_factory1 = comp.load_component_from_text(component_text)
import kfp
with kfp.dsl.Pipeline('Dummy'): #Forcing the TaskSpec conversion to ContainerOp
task1 = task_factory1()
actual_env = {env_var.name: env_var.value for env_var in task1.env_variables}
expected_env = {'key1': 'value 1', 'key2': 'value 2'}
self.assertDictEqual(expected_env, actual_env)
def test_converted_outputs(self):
component_text = textwrap.dedent('''\
outputs:
- name: Output 1
implementation:
container:
image: busybox
command:
- producer
- {outputPath: Output 1} # Outputs must be used in the implementation
''')
task_factory1 = load_component_from_text(component_text)
task1 = task_factory1()
self.assertSetEqual(set(task1.outputs.keys()),
{'Output 1', 'output_1'})
self.assertIsNotNone(task1.output)
def test_handle_default_values_in_task_factory(self):
component_text = '''\
inputs:
- {name: Data, default: '123'}
implementation:
container:
image: busybox
args:
- {inputValue: Data}
'''
task_factory1 = comp.load_component_from_text(text=component_text)
task1 = task_factory1()
self.assertEqual(task1.arguments, ['123'])
task2 = task_factory1('456')
self.assertEqual(task2.arguments, ['456'])
def test_missing_optional_input_file_argument(self):
'''Missing optional inputs should resolve to nothing'''
component_text = '''\
inputs:
- {name: input 1, optional: true}
implementation:
container:
image: busybox
command:
- a
- {inputPath: input 1}
- z
'''
task_factory1 = comp.load_component_from_text(component_text)
task1 = task_factory1()
self.assertEqual(task1.command, ['a', 'z'])
def test_inputs_reordering_when_inputs_have_defaults(self):
'''Tests reordering of inputs with default values.
In python signature, optional arguments must come after the required arguments.
'''
component_text = '''\
inputs:
- {name: in1}
- {name: in2, default: val}
- {name: in3}
implementation:
container:
image: busybox
'''
task_factory1 = comp.load_component_from_text(component_text)
import inspect
signature = inspect.signature(task_factory1)
actual_signature = list(signature.parameters.keys())
self.assertSequenceEqual(actual_signature, ['in1', 'in3', 'in2'], str)
def random_num_op(low, high):
"""Generate a random number between low and high."""
return components.load_component_from_text("""
name: Generate random number
outputs:
- {name: output, type: Integer}
implementation:
container:
image: python:alpine3.6
command:
- sh
- -c
args:
- mkdir -p "$(dirname $2)" && python -c "import random; print(random.randint($0, $1), end='')" | tee $2
- "%s"
- "%s"
- {outputPath: output}
""" % (low, high))
def test_compile_pipeline_with_importer_on_inputpath_should_raise_error(
self):
# YAML componet authoring
component_op = components.load_component_from_text("""
name: compoent with misused placeholder
inputs:
- {name: model, type: Model}
implementation:
container:
image: dummy
args:
- {inputPath: model}
""")
@dsl.pipeline(name='my-component')
def my_pipeline(model):
component_op(model=model)
with self.assertRaisesRegex(
TypeError,
'Input "model" with type "Model" is not connected to any upstream '
'output. However it is used with InputPathPlaceholder.'):
compiler.Compiler().compile(pipeline_func=my_pipeline,
pipeline_root='dummy',
output_path='output.json')
# Python function based component authoring
def my_component(datasets: components.InputPath('Datasets')):
pass
component_op = components.create_component_from_func(my_component)
@dsl.pipeline(name='my-component')
def my_pipeline(datasets):
component_op(datasets=datasets)
with self.assertRaisesRegex(
TypeError,
'Input "datasets" with type "Datasets" is not connected to any upstream '
'output. However it is used with InputPathPlaceholder.'):
compiler.Compiler().compile(pipeline_func=my_pipeline,
pipeline_root='dummy',
output_path='output.json')
def flip_coin_op():
"""Flip a coin and output heads or tails randomly."""
return components.load_component_from_text("""
name: flip-coin
description: Flip coin
outputs:
- {name: output_result, type: String}
implementation:
container:
image: python:alpine3.6
command:
- sh
- -c
args:
- |
python -c "import random; result = 'heads' if random.randint(0,1) == 0 \
else 'tails'; print(result)" | tee $0
- {outputPath: output_result}
""")()
def random_failure_op(exit_codes: str):
"""A component that fails randomly."""
return components.load_component_from_text("""
name: random-failure
description: random failure
inputs:
- {name: exitcode, type: String}
implementation:
container:
image: python:alpine3.6
command:
- python
- -c
args:
- |
import random; import sys; exit_code = random.choice([int(i) for i in sys.argv[1].split(",")]); \
print(exit_code); sys.exit(exit_code)
- {inputValue: exitcode}
""")(exitcode=exit_codes)
def test_missing_optional_input_value_argument(self):
'''Missing optional inputs should resolve to nothing'''
component_text = '''\
inputs:
- {name: input 1, optional: true}
implementation:
container:
image: busybox
command:
- a
- {inputValue: input 1}
- z
'''
task_factory1 = comp.load_component_from_text(component_text)
task1 = task_factory1()
resolved_cmd = _resolve_command_line_and_paths(
task1.component_ref.spec, task1.arguments)
self.assertEqual(resolved_cmd.command, ['a', 'z'])
