def test_compile_pipeline_with_misused_inputvalue_should_raise_error(self): upstream_op = components.load_component_from_text(""" name: upstream compoent outputs: - {name: model, type: Model} implementation: container: image: dummy args: - {outputPath: model} """) downstream_op = components.load_component_from_text(""" name: compoent with misused placeholder inputs: - {name: model, type: Model} implementation: container: image: dummy args: - {inputValue: model} """) @dsl.pipeline(name='test-pipeline', pipeline_root='dummy_root') def my_pipeline(): downstream_op(model=upstream_op().output) with self.assertRaisesRegex( TypeError, ' type "Model" cannot be paired with InputValuePlaceholder.'): compiler.Compiler().compile( pipeline_func=my_pipeline, package_path='output.json')
def test_compile_pipeline_with_dsl_condition_should_raise_error(self): flip_coin_op = components.load_component_from_text(""" name: flip coin inputs: - {name: name, type: String} outputs: - {name: result, type: String} implementation: container: image: gcr.io/my-project/my-image:tag args: - {inputValue: name} - {outputPath: result} """) print_op = components.load_component_from_text(""" name: print inputs: - {name: name, type: String} - {name: msg, type: String} implementation: container: image: gcr.io/my-project/my-image:tag args: - {inputValue: name} - {inputValue: msg} """) @dsl.pipeline() def flipcoin(): flip = flip_coin_op('flip') with dsl.Condition(flip.outputs['result'] == 'heads'): flip2 = flip_coin_op('flip-again') with dsl.Condition(flip2.outputs['result'] == 'tails'): print_op('print1', flip2.outputs['result']) with dsl.Condition(flip.outputs['result'] == 'tails'): print_op('print2', flip2.outputs['results']) with self.assertRaisesRegex( NotImplementedError, 'dsl.Condition is not yet supported in KFP v2 compiler.'): compiler.Compiler().compile( pipeline_func=flipcoin, pipeline_root='dummy_root', output_path='output.json')
def test_passing_concrete_artifact_to_input_expecting_generic_artifact( self): producer_op1 = components.load_component_from_text(""" name: producer compoent outputs: - {name: output, type: Dataset} implementation: container: image: dummy args: - {outputPath: output} """) @dsl.component def producer_op2(output: dsl.Output[dsl.Model]): pass consumer_op1 = components.load_component_from_text(""" name: consumer compoent inputs: - {name: input1, type: Artifact} implementation: container: image: dummy args: - {inputPath: input1} """) @dsl.component def consumer_op2(input1: dsl.Input[dsl.Artifact]): pass @dsl.pipeline(name='test-pipeline') def my_pipeline(): consumer_op1(input1=producer_op1().output) consumer_op1(input1=producer_op2().output) consumer_op2(input1=producer_op1().output) consumer_op2(input1=producer_op2().output) try: tmpdir = tempfile.mkdtemp() target_json_file = os.path.join(tmpdir, 'result.json') compiler.Compiler().compile( pipeline_func=my_pipeline, package_path=target_json_file) self.assertTrue(os.path.exists(target_json_file)) finally: shutil.rmtree(tmpdir)
def test_compile_simple_pipeline(self): tmpdir = tempfile.mkdtemp() try: producer_op = components.load_component_from_text(""" name: producer inputs: - {name: input_param, type: String} outputs: - {name: output_model, type: Model} - {name: output_value, type: Integer} implementation: container: image: gcr.io/my-project/my-image:tag args: - {inputValue: input_param} - {outputPath: output_model} - {outputPath: output_value} """) consumer_op = components.load_component_from_text(""" name: consumer inputs: - {name: input_model, type: Model} - {name: input_value, type: Integer} implementation: container: image: gcr.io/my-project/my-image:tag args: - {inputPath: input_model} - {inputValue: input_value} """) @dsl.pipeline(name='test-pipeline') def simple_pipeline(pipeline_input: str = 'Hello KFP!'): producer = producer_op(input_param=pipeline_input) consumer = consumer_op( input_model=producer.outputs['output_model'], input_value=producer.outputs['output_value']) target_json_file = os.path.join(tmpdir, 'result.json') compiler.Compiler().compile( pipeline_func=simple_pipeline, package_path=target_json_file) self.assertTrue(os.path.exists(target_json_file)) with open(target_json_file, 'r') as f: print(f.read()) finally: shutil.rmtree(tmpdir)
def test_compile_pipeline_with_bool(self): tmpdir = tempfile.mkdtemp() try: predict_op = components.load_component_from_text(""" name: predict inputs: - {name: generate_explanation, type: Boolean, default: False} implementation: container: image: gcr.io/my-project/my-image:tag args: - {inputValue: generate_explanation} """) @dsl.pipeline(name='test-boolean-pipeline') def simple_pipeline(): predict_op(generate_explanation=True) target_json_file = os.path.join(tmpdir, 'result.json') compiler.Compiler().compile(pipeline_func=simple_pipeline, package_path=target_json_file) self.assertTrue(os.path.exists(target_json_file)) with open(target_json_file, 'r') as f: print(f.read()) finally: shutil.rmtree(tmpdir)
def test_passing_arbitrary_artifact_to_input_expecting_concrete_artifact( self): producer_op1 = components.load_component_from_text(""" name: producer compoent outputs: - {name: output, type: SomeArbitraryType} implementation: container: image: dummy args: - {outputPath: output} """) @dsl.component def consumer_op(input1: dsl.Input[dsl.Dataset]): pass @dsl.pipeline(name='test-pipeline') def my_pipeline(): consumer_op(input1=producer_op1().output) consumer_op(input1=producer_op2().output) with self.assertRaisesRegex( type_utils.InconsistentTypeException, 'Incompatible argument passed to the input "input1" of component' ' "Consumer op": Argument type "SomeArbitraryType" is' ' incompatible with the input type "Dataset"'): compiler.Compiler().compile( pipeline_func=my_pipeline, package_path='result.json')
def test_compile_pipeline_with_dsl_exithandler_should_raise_error(self): gcs_download_op = components.load_component_from_text(""" name: GCS - Download inputs: - {name: url, type: String} outputs: - {name: result, type: String} implementation: container: image: gcr.io/my-project/my-image:tag args: - {inputValue: url} - {outputPath: result} """) echo_op = components.load_component_from_text(""" name: echo inputs: - {name: msg, type: String} implementation: container: image: gcr.io/my-project/my-image:tag args: - {inputValue: msg} """) @dsl.pipeline(name='pipeline-with-exit-handler') def download_and_print( url='gs://ml-pipeline/shakespeare/shakespeare1.txt'): """A sample pipeline showing exit handler.""" exit_task = echo_op('exit!') with dsl.ExitHandler(exit_task): download_task = gcs_download_op(url) echo_task = echo_op(download_task.outputs['result']) with self.assertRaisesRegex( NotImplementedError, 'dsl.ExitHandler is not yet supported in KFP v2 compiler.'): compiler.Compiler().compile(pipeline_func=download_and_print, pipeline_root='dummy_root', output_path='output.json')
def test_compile_pipeline_with_inputpath_should_warn(self): with self.assertWarnsRegex( UserWarning, 'Local file paths are currently unsupported for I/O.'): component_op = components.load_component_from_text(""" name: compoent use inputPath inputs: - {name: data, type: Datasets} implementation: container: image: dummy args: - {inputPath: data} """)
def test_compile_pipeline_with_importer_on_inputpath_should_raise_error(self): # YAML componet authoring component_op = components.load_component_from_text(""" name: compoent with misused placeholder inputs: - {name: model, type: Model} implementation: container: image: dummy args: - {inputPath: model} """) @dsl.pipeline(name='my-component') def my_pipeline(model): component_op(model=model) with self.assertRaisesRegex( TypeError, 'Input "model" with type "Model" is not connected to any upstream ' 'output. However it is used with InputPathPlaceholder.'): compiler.Compiler().compile( pipeline_func=my_pipeline, pipeline_root='dummy', output_path='output.json') # Python function based component authoring def my_component(datasets: components.InputPath('Datasets')): pass component_op = components.create_component_from_func(my_component) @dsl.pipeline(name='my-component') def my_pipeline(datasets): component_op(datasets=datasets) with self.assertRaisesRegex( TypeError, 'Input "datasets" with type "Datasets" is not connected to any upstream ' 'output. However it is used with InputPathPlaceholder.'): compiler.Compiler().compile( pipeline_func=my_pipeline, pipeline_root='dummy', output_path='output.json')
def random_num_op(low, high): """Generate a random number between low and high.""" return components.load_component_from_text(""" name: Generate random number outputs: - {name: output, type: Integer} implementation: container: image: python:alpine3.6 command: - sh - -c args: - mkdir -p "$(dirname $2)" && python -c "import random; print(random.randint($0, $1), end='')" | tee $2 - "%s" - "%s" - {outputPath: output} """ % (low, high))
def _create_a_container_based_component(self) -> callable: """Creates a test container based component factory.""" return components.load_component_from_text(""" name: ContainerComponent inputs: - {name: input_text, type: String, description: "Represents an input parameter."} outputs: - {name: output_value, type: String, description: "Represents an output paramter."} implementation: container: image: google/cloud-sdk:latest command: - sh - -c - | set -e -x echo "$0, this is an output parameter" - {inputValue: input_text} - {outputPath: output_value} """)
def test_compile_pipeline_with_misused_outputuri_should_raise_error(self): component_op = components.load_component_from_text(""" name: compoent with misused placeholder outputs: - {name: value, type: Integer} implementation: container: image: dummy args: - {outputUri: value} """) @dsl.pipeline(name='test-pipeline', pipeline_root='dummy_root') def my_pipeline(): component_op() with self.assertRaisesRegex( TypeError, ' type "Integer" cannot be paired with OutputUriPlaceholder.'): compiler.Compiler().compile( pipeline_func=my_pipeline, package_path='output.json')
def test_compile_pipeline_with_misused_inputuri_should_raise_error(self): component_op = components.load_component_from_text(""" name: compoent with misused placeholder inputs: - {name: value, type: Float} implementation: container: image: dummy args: - {inputUri: value} """) def my_pipeline(value): component_op(value=value) with self.assertRaisesRegex( TypeError, ' type "Float" cannot be paired with InputUriPlaceholder.'): compiler.Compiler().compile(pipeline_func=my_pipeline, pipeline_root='dummy', output_path='output.json')
def test_compile_pipeline_with_misused_inputpath_should_raise_error(self): component_op = components.load_component_from_text(""" name: compoent with misused placeholder inputs: - {name: text, type: String} implementation: container: image: dummy args: - {inputPath: text} """) @dsl.pipeline(name='test-pipeline', pipeline_root='dummy_root') def my_pipeline(text: str): component_op(text=text) with self.assertRaisesRegex( TypeError, ' type "String" cannot be paired with InputPathPlaceholder.'): compiler.Compiler().compile( pipeline_func=my_pipeline, package_path='output.json')
def test_use_task_final_status_in_non_exit_op_yaml(self): print_op = components.load_component_from_text(""" name: Print Op inputs: - {name: message, type: PipelineTaskFinalStatus} implementation: container: image: python:3.7 command: - echo - {inputValue: message} """) @dsl.pipeline(name='test-pipeline') def my_pipeline(text: bool): print_op() with self.assertRaisesRegex( ValueError, 'PipelineTaskFinalStatus can only be used in an exit task.'): compiler.Compiler().compile(pipeline_func=my_pipeline, package_path='result.json')
def test_passing_string_parameter_to_artifact_should_error(self): component_op = components.load_component_from_text(""" name: compoent inputs: - {name: some_input, type: , description: an uptyped input} implementation: container: image: dummy args: - {inputPath: some_input} """) @dsl.pipeline(name='test-pipeline', pipeline_root='gs://path') def my_pipeline(input1: str): component_op(some_input=input1) with self.assertRaisesRegex( TypeError, 'Passing PipelineParam "input1" with type "String" \(as "Parameter"\) ' 'to component input "some_input" with type "None" \(as "Artifact"\) is ' 'incompatible. Please fix the type of the component input.'): compiler.Compiler().compile(pipeline_func=my_pipeline, package_path='output.json')
def test_passing_string_parameter_to_artifact_should_error(self): component_op = components.load_component_from_text(""" name: compoent inputs: - {name: some_input, type: , description: an uptyped input} implementation: container: image: dummy args: - {inputPath: some_input} """) @dsl.pipeline(name='test-pipeline', pipeline_root='gs://path') def my_pipeline(input1: str): component_op(some_input=input1) with self.assertRaisesRegex( type_utils.InconsistentTypeException, 'Incompatible argument passed to the input "some_input" of ' 'component "compoent": Argument type "STRING" is incompatible ' 'with the input type "Artifact"'): compiler.Compiler().compile( pipeline_func=my_pipeline, package_path='output.json')
component_op_1 = components.load_component_from_text(""" name: upstream inputs: - {name: input_1, type: String} - {name: input_2, type: Float} - {name: input_3, type: String} - {name: input_4, type: String} outputs: - {name: output_1, type: Integer} - {name: output_2, type: Model} - {name: output_3} - {name: output_4, type: Model} - {name: output_5, type: Datasets} - {name: output_6, type: Some arbitrary type} - {name: output_7, type: {GcsPath: {data_type: TSV}}} - {name: output_8, type: HTML} - {name: output_9, type: google.BQMLModel} implementation: container: image: gcr.io/image args: - {inputValue: input_1} - {inputValue: input_2} - {inputValue: input_3} - {inputValue: input_4} - {outputPath: output_1} - {outputUri: output_2} - {outputPath: output_3} - {outputUri: output_4} - {outputUri: output_5} - {outputPath: output_6} - {outputPath: output_7} - {outputPath: output_8} """)
import unittest from absl.testing import parameterized from kfp.v2 import components from kfp.v2 import compiler from kfp.v2 import dsl from kfp.v2.components.types import type_utils VALID_PRODUCER_COMPONENT_SAMPLE = components.load_component_from_text(""" name: producer inputs: - {name: input_param, type: String} outputs: - {name: output_model, type: Model} - {name: output_value, type: Integer} implementation: container: image: gcr.io/my-project/my-image:tag args: - {inputValue: input_param} - {outputPath: output_model} - {outputPath: output_value} """) class CompilerTest(parameterized.TestCase): def test_compile_simple_pipeline(self): tmpdir = tempfile.mkdtemp() try:
from kfp.v2 import components from kfp.v2.dsl import component, Input, Output from kfp.v2 import compiler from kfp.v2 import dsl class VertexModel(dsl.Artifact): TYPE_NAME = 'google.VertexModel' producer_op = components.load_component_from_text(""" name: producer outputs: - {name: model, type: google.VertexModel} implementation: container: image: dummy command: - cmd args: - {outputPath: model} """) @component def consumer_op(model: Input[VertexModel]): pass @dsl.pipeline(name='pipeline-with-gcpc-types') def my_pipeline(): consumer_op(model=producer_op().outputs['model'])
- -c args: - mkdir -p "$(dirname $2)" && python -c "import random; print(random.randint($0, $1), end='')" | tee $2 - "%s" - "%s" - {outputPath: output} """ % (low, high)) flip_coin_op = components.load_component_from_text(""" name: Flip coin outputs: - {name: output, type: String} implementation: container: image: python:alpine3.6 command: - sh - -c args: - mkdir -p "$(dirname $0)" && python -c "import random; result = \'heads\' if random.randint(0,1) == 0 else \'tails\'; print(result, end='')" | tee $0 - {outputPath: output} """) print_op = components.load_component_from_text(""" name: Print inputs: - {name: msg, type: String} implementation: container: image: python:alpine3.6 command:
@component def print_env_op(): import os print(os.environ['ENV1']) print_env_2_op = components.load_component_from_text(""" name: Print env implementation: container: image: alpine commands: - sh - -c - | set -e -x echo "$ENV2" echo "$ENV3" env: ENV2: val0 """) @dsl.pipeline(name='pipeline-with-env', pipeline_root='dummy_root') def my_pipeline(): print_env_op().set_env_variable(name='ENV1', value='val1') print_env_2_op().set_env_variable( name='ENV2', value='val2').set_env_variable(name='ENV3', value='val3')
# limitations under the License. """Pipeline using ExitHandler with PipelineTaskFinalStatus (YAML).""" from kfp.v2 import components from kfp.v2 import dsl from kfp.v2 import compiler from kfp.v2.dsl import component exit_op = components.load_component_from_text(""" name: Exit Op inputs: - {name: user_input, type: String} - {name: status, type: PipelineTaskFinalStatus} implementation: container: image: python:3.7 command: - echo - "user input:" - {inputValue: user_input} - "pipeline status:" - {inputValue: status} """) print_op = components.load_component_from_text(""" name: Print Op inputs: - {name: message, type: String} implementation: container: image: python:3.7
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Two step v2-compatible pipeline with URI placeholders.""" import kfp from kfp.v2 import components, dsl write_to_gcs_op = components.load_component_from_text(""" name: write-to-gcs inputs: - {name: msg, type: String, description: 'Content to be written to GCS'} outputs: - {name: artifact, type: Artifact, description: 'GCS file path'} implementation: container: image: google/cloud-sdk:slim command: - sh - -c - | set -e -x echo "$0" | gsutil cp - "$1" - {inputValue: msg} - {outputUri: artifact} """) read_from_gcs_op = components.load_component_from_text(""" name: read-from-gcs inputs: - {name: artifact, type: Artifact, description: 'GCS file path'} implementation: container:
import pathlib from kfp.v2 import components from kfp.v2 import dsl import kfp.v2.compiler as compiler component_op_1 = components.load_component_from_text(""" name: Write to GCS inputs: - {name: text, type: String, description: 'Content to be written to GCS'} outputs: - {name: output_gcs_path, type: GCSPath, description: 'GCS file path'} implementation: container: image: google/cloud-sdk:slim command: - sh - -c - | set -e -x echo "$0" | gsutil cp - "$1" - {inputValue: text} - {outputUri: output_gcs_path} """) component_op_2 = components.load_component_from_text(""" name: Read from GCS inputs: - {name: input_gcs_path, type: GCSPath, description: 'GCS file path'} implementation: container:
component_op = components.load_component_from_text(''' name: Component with optional inputs inputs: - {name: required_input, type: String, optional: false} - {name: optional_input_1, type: String, optional: true} - {name: optional_input_2, type: String, optional: true} implementation: container: image: gcr.io/google-containers/busybox command: - echo args: - --arg0 - {inputValue: required_input} - if: cond: isPresent: optional_input_1 then: - --arg1 - {inputValue: optional_input_1} - if: cond: isPresent: optional_input_2 then: - --arg2 - {inputValue: optional_input_2} else: - --arg2 - 'default value' ''')
# Simple two-step pipeline with 'producer' and 'consumer' steps from kfp.v2 import components from kfp.v2 import compiler from kfp.v2 import dsl producer_op = components.load_component_from_text( """ name: Producer inputs: - {name: input_text, type: String, description: 'Represents an input parameter.'} outputs: - {name: output_value, type: String, description: 'Represents an output paramter.'} implementation: container: image: google/cloud-sdk:latest command: - sh - -c - | set -e -x echo "$0, this is an output parameter" | gsutil cp - "$1" - {inputValue: input_text} - {outputPath: output_value} """ ) consumer_op = components.load_component_from_text( """ name: Consumer inputs: - {name: input_value, type: String, description: 'Represents an input parameter. It connects to an upstream output parameter.'}
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from kfp.v2 import components from kfp.v2 import dsl import kfp.v2.compiler as compiler component_op = components.load_component_from_text(""" name: Component with concat placeholder inputs: - {name: input_one, type: String} - {name: input_two, type: String} implementation: container: image: gcr.io/google-containers/busybox command: - sh - -ec args: - echo "$0" > /tmp/test && [[ "$0" == 'one+two=three' ]] - concat: [{inputValue: input_one}, '+', {inputValue: input_two}, '=three'] """) @dsl.pipeline(name='one-step-pipeline-with-concat-placeholder') def pipeline_with_concat_placeholder(): component = component_op(input_one='one', input_two='two')
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from kfp.v2 import components from kfp.v2 import dsl import kfp.v2.compiler as compiler component_op = components.load_component_from_text(""" name: Print Text inputs: - {name: text, type: String} implementation: container: image: alpine command: - sh - -c - | set -e -x echo "$0" - {inputValue: text} """) @dsl.pipeline(name='pipeline-with-after') def my_pipeline(): task1 = component_op(text='1st task') task2 = component_op(text='2nd task').after(task1) task3 = component_op(text='3rd task').after(task1, task2)