def test_compile_pipeline_with_misused_inputvalue_should_raise_error(self):
upstream_op = components.load_component_from_text("""
name: upstream compoent
outputs:
- {name: model, type: Model}
implementation:
container:
image: dummy
args:
- {outputPath: model}
""")
downstream_op = components.load_component_from_text("""
name: compoent with misused placeholder
inputs:
- {name: model, type: Model}
implementation:
container:
image: dummy
args:
- {inputValue: model}
""")
@dsl.pipeline(name='test-pipeline', pipeline_root='dummy_root')
def my_pipeline():
downstream_op(model=upstream_op().output)
with self.assertRaisesRegex(
TypeError,
' type "Model" cannot be paired with InputValuePlaceholder.'):
compiler.Compiler().compile(
pipeline_func=my_pipeline, package_path='output.json')
def test_compile_pipeline_with_dsl_condition_should_raise_error(self):
flip_coin_op = components.load_component_from_text("""
name: flip coin
inputs:
- {name: name, type: String}
outputs:
- {name: result, type: String}
implementation:
container:
image: gcr.io/my-project/my-image:tag
args:
- {inputValue: name}
- {outputPath: result}
""")
print_op = components.load_component_from_text("""
name: print
inputs:
- {name: name, type: String}
- {name: msg, type: String}
implementation:
container:
image: gcr.io/my-project/my-image:tag
args:
- {inputValue: name}
- {inputValue: msg}
""")
@dsl.pipeline()
def flipcoin():
flip = flip_coin_op('flip')
with dsl.Condition(flip.outputs['result'] == 'heads'):
flip2 = flip_coin_op('flip-again')
with dsl.Condition(flip2.outputs['result'] == 'tails'):
print_op('print1', flip2.outputs['result'])
with dsl.Condition(flip.outputs['result'] == 'tails'):
print_op('print2', flip2.outputs['results'])
with self.assertRaisesRegex(
NotImplementedError,
'dsl.Condition is not yet supported in KFP v2 compiler.'):
compiler.Compiler().compile(
pipeline_func=flipcoin,
pipeline_root='dummy_root',
output_path='output.json')
def test_passing_concrete_artifact_to_input_expecting_generic_artifact(
self):
producer_op1 = components.load_component_from_text("""
name: producer compoent
outputs:
- {name: output, type: Dataset}
implementation:
container:
image: dummy
args:
- {outputPath: output}
""")
@dsl.component
def producer_op2(output: dsl.Output[dsl.Model]):
pass
consumer_op1 = components.load_component_from_text("""
name: consumer compoent
inputs:
- {name: input1, type: Artifact}
implementation:
container:
image: dummy
args:
- {inputPath: input1}
""")
@dsl.component
def consumer_op2(input1: dsl.Input[dsl.Artifact]):
pass
@dsl.pipeline(name='test-pipeline')
def my_pipeline():
consumer_op1(input1=producer_op1().output)
consumer_op1(input1=producer_op2().output)
consumer_op2(input1=producer_op1().output)
consumer_op2(input1=producer_op2().output)
try:
tmpdir = tempfile.mkdtemp()
target_json_file = os.path.join(tmpdir, 'result.json')
compiler.Compiler().compile(
pipeline_func=my_pipeline, package_path=target_json_file)
self.assertTrue(os.path.exists(target_json_file))
finally:
shutil.rmtree(tmpdir)
def test_compile_simple_pipeline(self):
tmpdir = tempfile.mkdtemp()
try:
producer_op = components.load_component_from_text("""
name: producer
inputs:
- {name: input_param, type: String}
outputs:
- {name: output_model, type: Model}
- {name: output_value, type: Integer}
implementation:
container:
image: gcr.io/my-project/my-image:tag
args:
- {inputValue: input_param}
- {outputPath: output_model}
- {outputPath: output_value}
""")
consumer_op = components.load_component_from_text("""
name: consumer
inputs:
- {name: input_model, type: Model}
- {name: input_value, type: Integer}
implementation:
container:
image: gcr.io/my-project/my-image:tag
args:
- {inputPath: input_model}
- {inputValue: input_value}
""")
@dsl.pipeline(name='test-pipeline')
def simple_pipeline(pipeline_input: str = 'Hello KFP!'):
producer = producer_op(input_param=pipeline_input)
consumer = consumer_op(
input_model=producer.outputs['output_model'],
input_value=producer.outputs['output_value'])
target_json_file = os.path.join(tmpdir, 'result.json')
compiler.Compiler().compile(
pipeline_func=simple_pipeline, package_path=target_json_file)
self.assertTrue(os.path.exists(target_json_file))
with open(target_json_file, 'r') as f:
print(f.read())
finally:
shutil.rmtree(tmpdir)
def test_compile_pipeline_with_bool(self):
tmpdir = tempfile.mkdtemp()
try:
predict_op = components.load_component_from_text("""
name: predict
inputs:
- {name: generate_explanation, type: Boolean, default: False}
implementation:
container:
image: gcr.io/my-project/my-image:tag
args:
- {inputValue: generate_explanation}
""")
@dsl.pipeline(name='test-boolean-pipeline')
def simple_pipeline():
predict_op(generate_explanation=True)
target_json_file = os.path.join(tmpdir, 'result.json')
compiler.Compiler().compile(pipeline_func=simple_pipeline,
package_path=target_json_file)
self.assertTrue(os.path.exists(target_json_file))
with open(target_json_file, 'r') as f:
print(f.read())
finally:
shutil.rmtree(tmpdir)
def test_passing_arbitrary_artifact_to_input_expecting_concrete_artifact(
self):
producer_op1 = components.load_component_from_text("""
name: producer compoent
outputs:
- {name: output, type: SomeArbitraryType}
implementation:
container:
image: dummy
args:
- {outputPath: output}
""")
@dsl.component
def consumer_op(input1: dsl.Input[dsl.Dataset]):
pass
@dsl.pipeline(name='test-pipeline')
def my_pipeline():
consumer_op(input1=producer_op1().output)
consumer_op(input1=producer_op2().output)
with self.assertRaisesRegex(
type_utils.InconsistentTypeException,
'Incompatible argument passed to the input "input1" of component'
' "Consumer op": Argument type "SomeArbitraryType" is'
' incompatible with the input type "Dataset"'):
compiler.Compiler().compile(
pipeline_func=my_pipeline, package_path='result.json')
def test_compile_pipeline_with_dsl_exithandler_should_raise_error(self):
gcs_download_op = components.load_component_from_text("""
name: GCS - Download
inputs:
- {name: url, type: String}
outputs:
- {name: result, type: String}
implementation:
container:
image: gcr.io/my-project/my-image:tag
args:
- {inputValue: url}
- {outputPath: result}
""")
echo_op = components.load_component_from_text("""
name: echo
inputs:
- {name: msg, type: String}
implementation:
container:
image: gcr.io/my-project/my-image:tag
args:
- {inputValue: msg}
""")
@dsl.pipeline(name='pipeline-with-exit-handler')
def download_and_print(
url='gs://ml-pipeline/shakespeare/shakespeare1.txt'):
"""A sample pipeline showing exit handler."""
exit_task = echo_op('exit!')
with dsl.ExitHandler(exit_task):
download_task = gcs_download_op(url)
echo_task = echo_op(download_task.outputs['result'])
with self.assertRaisesRegex(
NotImplementedError,
'dsl.ExitHandler is not yet supported in KFP v2 compiler.'):
compiler.Compiler().compile(pipeline_func=download_and_print,
pipeline_root='dummy_root',
output_path='output.json')
def test_compile_pipeline_with_inputpath_should_warn(self):
with self.assertWarnsRegex(
UserWarning, 'Local file paths are currently unsupported for I/O.'):
component_op = components.load_component_from_text("""
name: compoent use inputPath
inputs:
- {name: data, type: Datasets}
implementation:
container:
image: dummy
args:
- {inputPath: data}
""")
def test_compile_pipeline_with_importer_on_inputpath_should_raise_error(self):
# YAML componet authoring
component_op = components.load_component_from_text("""
name: compoent with misused placeholder
inputs:
- {name: model, type: Model}
implementation:
container:
image: dummy
args:
- {inputPath: model}
""")
@dsl.pipeline(name='my-component')
def my_pipeline(model):
component_op(model=model)
with self.assertRaisesRegex(
TypeError,
'Input "model" with type "Model" is not connected to any upstream '
'output. However it is used with InputPathPlaceholder.'):
compiler.Compiler().compile(
pipeline_func=my_pipeline,
pipeline_root='dummy',
output_path='output.json')
# Python function based component authoring
def my_component(datasets: components.InputPath('Datasets')):
pass
component_op = components.create_component_from_func(my_component)
@dsl.pipeline(name='my-component')
def my_pipeline(datasets):
component_op(datasets=datasets)
with self.assertRaisesRegex(
TypeError,
'Input "datasets" with type "Datasets" is not connected to any upstream '
'output. However it is used with InputPathPlaceholder.'):
compiler.Compiler().compile(
pipeline_func=my_pipeline,
pipeline_root='dummy',
output_path='output.json')
def random_num_op(low, high):
"""Generate a random number between low and high."""
return components.load_component_from_text("""
name: Generate random number
outputs:
- {name: output, type: Integer}
implementation:
container:
image: python:alpine3.6
command:
- sh
- -c
args:
- mkdir -p "$(dirname $2)" && python -c "import random; print(random.randint($0, $1), end='')" | tee $2
- "%s"
- "%s"
- {outputPath: output}
""" % (low, high))
def _create_a_container_based_component(self) -> callable:
"""Creates a test container based component factory."""
return components.load_component_from_text("""
name: ContainerComponent
inputs:
- {name: input_text, type: String, description: "Represents an input parameter."}
outputs:
- {name: output_value, type: String, description: "Represents an output paramter."}
implementation:
container:
image: google/cloud-sdk:latest
command:
- sh
- -c
- |
set -e -x
echo "$0, this is an output parameter"
- {inputValue: input_text}
- {outputPath: output_value}
""")
def test_compile_pipeline_with_misused_outputuri_should_raise_error(self):
component_op = components.load_component_from_text("""
name: compoent with misused placeholder
outputs:
- {name: value, type: Integer}
implementation:
container:
image: dummy
args:
- {outputUri: value}
""")
@dsl.pipeline(name='test-pipeline', pipeline_root='dummy_root')
def my_pipeline():
component_op()
with self.assertRaisesRegex(
TypeError,
' type "Integer" cannot be paired with OutputUriPlaceholder.'):
compiler.Compiler().compile(
pipeline_func=my_pipeline, package_path='output.json')
def test_compile_pipeline_with_misused_inputuri_should_raise_error(self):
component_op = components.load_component_from_text("""
name: compoent with misused placeholder
inputs:
- {name: value, type: Float}
implementation:
container:
image: dummy
args:
- {inputUri: value}
""")
def my_pipeline(value):
component_op(value=value)
with self.assertRaisesRegex(
TypeError,
' type "Float" cannot be paired with InputUriPlaceholder.'):
compiler.Compiler().compile(pipeline_func=my_pipeline,
pipeline_root='dummy',
output_path='output.json')
def test_compile_pipeline_with_misused_inputpath_should_raise_error(self):
component_op = components.load_component_from_text("""
name: compoent with misused placeholder
inputs:
- {name: text, type: String}
implementation:
container:
image: dummy
args:
- {inputPath: text}
""")
@dsl.pipeline(name='test-pipeline', pipeline_root='dummy_root')
def my_pipeline(text: str):
component_op(text=text)
with self.assertRaisesRegex(
TypeError,
' type "String" cannot be paired with InputPathPlaceholder.'):
compiler.Compiler().compile(
pipeline_func=my_pipeline, package_path='output.json')
def test_use_task_final_status_in_non_exit_op_yaml(self):
print_op = components.load_component_from_text("""
name: Print Op
inputs:
- {name: message, type: PipelineTaskFinalStatus}
implementation:
container:
image: python:3.7
command:
- echo
- {inputValue: message}
""")
@dsl.pipeline(name='test-pipeline')
def my_pipeline(text: bool):
print_op()
with self.assertRaisesRegex(
ValueError,
'PipelineTaskFinalStatus can only be used in an exit task.'):
compiler.Compiler().compile(pipeline_func=my_pipeline,
package_path='result.json')
def test_passing_string_parameter_to_artifact_should_error(self):
component_op = components.load_component_from_text("""
name: compoent
inputs:
- {name: some_input, type: , description: an uptyped input}
implementation:
container:
image: dummy
args:
- {inputPath: some_input}
""")
@dsl.pipeline(name='test-pipeline', pipeline_root='gs://path')
def my_pipeline(input1: str):
component_op(some_input=input1)
with self.assertRaisesRegex(
TypeError,
'Passing PipelineParam "input1" with type "String" \(as "Parameter"\) '
'to component input "some_input" with type "None" \(as "Artifact"\) is '
'incompatible. Please fix the type of the component input.'):
compiler.Compiler().compile(pipeline_func=my_pipeline,
package_path='output.json')
def test_passing_string_parameter_to_artifact_should_error(self):
component_op = components.load_component_from_text("""
name: compoent
inputs:
- {name: some_input, type: , description: an uptyped input}
implementation:
container:
image: dummy
args:
- {inputPath: some_input}
""")
@dsl.pipeline(name='test-pipeline', pipeline_root='gs://path')
def my_pipeline(input1: str):
component_op(some_input=input1)
with self.assertRaisesRegex(
type_utils.InconsistentTypeException,
'Incompatible argument passed to the input "some_input" of '
'component "compoent": Argument type "STRING" is incompatible '
'with the input type "Artifact"'):
compiler.Compiler().compile(
pipeline_func=my_pipeline, package_path='output.json')
component_op_1 = components.load_component_from_text("""
name: upstream
inputs:
- {name: input_1, type: String}
- {name: input_2, type: Float}
- {name: input_3, type: String}
- {name: input_4, type: String}
outputs:
- {name: output_1, type: Integer}
- {name: output_2, type: Model}
- {name: output_3}
- {name: output_4, type: Model}
- {name: output_5, type: Datasets}
- {name: output_6, type: Some arbitrary type}
- {name: output_7, type: {GcsPath: {data_type: TSV}}}
- {name: output_8, type: HTML}
- {name: output_9, type: google.BQMLModel}
implementation:
container:
image: gcr.io/image
args:
- {inputValue: input_1}
- {inputValue: input_2}
- {inputValue: input_3}
- {inputValue: input_4}
- {outputPath: output_1}
- {outputUri: output_2}
- {outputPath: output_3}
- {outputUri: output_4}
- {outputUri: output_5}
- {outputPath: output_6}
- {outputPath: output_7}
- {outputPath: output_8}
""")
import unittest
from absl.testing import parameterized
from kfp.v2 import components
from kfp.v2 import compiler
from kfp.v2 import dsl
from kfp.v2.components.types import type_utils
VALID_PRODUCER_COMPONENT_SAMPLE = components.load_component_from_text("""
name: producer
inputs:
- {name: input_param, type: String}
outputs:
- {name: output_model, type: Model}
- {name: output_value, type: Integer}
implementation:
container:
image: gcr.io/my-project/my-image:tag
args:
- {inputValue: input_param}
- {outputPath: output_model}
- {outputPath: output_value}
""")
class CompilerTest(parameterized.TestCase):
def test_compile_simple_pipeline(self):
tmpdir = tempfile.mkdtemp()
try:
from kfp.v2 import components
from kfp.v2.dsl import component, Input, Output
from kfp.v2 import compiler
from kfp.v2 import dsl
class VertexModel(dsl.Artifact):
TYPE_NAME = 'google.VertexModel'
producer_op = components.load_component_from_text("""
name: producer
outputs:
- {name: model, type: google.VertexModel}
implementation:
container:
image: dummy
command:
- cmd
args:
- {outputPath: model}
""")
@component
def consumer_op(model: Input[VertexModel]):
pass
@dsl.pipeline(name='pipeline-with-gcpc-types')
def my_pipeline():
consumer_op(model=producer_op().outputs['model'])
- -c
args:
- mkdir -p "$(dirname $2)" && python -c "import random; print(random.randint($0, $1), end='')" | tee $2
- "%s"
- "%s"
- {outputPath: output}
""" % (low, high))
flip_coin_op = components.load_component_from_text("""
name: Flip coin
outputs:
- {name: output, type: String}
implementation:
container:
image: python:alpine3.6
command:
- sh
- -c
args:
- mkdir -p "$(dirname $0)" && python -c "import random; result = \'heads\' if random.randint(0,1) == 0 else \'tails\'; print(result, end='')" | tee $0
- {outputPath: output}
""")
print_op = components.load_component_from_text("""
name: Print
inputs:
- {name: msg, type: String}
implementation:
container:
image: python:alpine3.6
command:
@component
def print_env_op():
import os
print(os.environ['ENV1'])
print_env_2_op = components.load_component_from_text("""
name: Print env
implementation:
container:
image: alpine
commands:
- sh
- -c
- |
set -e -x
echo "$ENV2"
echo "$ENV3"
env:
ENV2: val0
""")
@dsl.pipeline(name='pipeline-with-env', pipeline_root='dummy_root')
def my_pipeline():
print_env_op().set_env_variable(name='ENV1', value='val1')
print_env_2_op().set_env_variable(
name='ENV2', value='val2').set_env_variable(name='ENV3', value='val3')
# limitations under the License.
"""Pipeline using ExitHandler with PipelineTaskFinalStatus (YAML)."""
from kfp.v2 import components
from kfp.v2 import dsl
from kfp.v2 import compiler
from kfp.v2.dsl import component
exit_op = components.load_component_from_text("""
name: Exit Op
inputs:
- {name: user_input, type: String}
- {name: status, type: PipelineTaskFinalStatus}
implementation:
container:
image: python:3.7
command:
- echo
- "user input:"
- {inputValue: user_input}
- "pipeline status:"
- {inputValue: status}
""")
print_op = components.load_component_from_text("""
name: Print Op
inputs:
- {name: message, type: String}
implementation:
container:
image: python:3.7
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Two step v2-compatible pipeline with URI placeholders."""
import kfp
from kfp.v2 import components, dsl
write_to_gcs_op = components.load_component_from_text("""
name: write-to-gcs
inputs:
- {name: msg, type: String, description: 'Content to be written to GCS'}
outputs:
- {name: artifact, type: Artifact, description: 'GCS file path'}
implementation:
container:
image: google/cloud-sdk:slim
command:
- sh
- -c
- |
set -e -x
echo "$0" | gsutil cp - "$1"
- {inputValue: msg}
- {outputUri: artifact}
""")
read_from_gcs_op = components.load_component_from_text("""
name: read-from-gcs
inputs:
- {name: artifact, type: Artifact, description: 'GCS file path'}
implementation:
container:
import pathlib
from kfp.v2 import components
from kfp.v2 import dsl
import kfp.v2.compiler as compiler
component_op_1 = components.load_component_from_text("""
name: Write to GCS
inputs:
- {name: text, type: String, description: 'Content to be written to GCS'}
outputs:
- {name: output_gcs_path, type: GCSPath, description: 'GCS file path'}
implementation:
container:
image: google/cloud-sdk:slim
command:
- sh
- -c
- |
set -e -x
echo "$0" | gsutil cp - "$1"
- {inputValue: text}
- {outputUri: output_gcs_path}
""")
component_op_2 = components.load_component_from_text("""
name: Read from GCS
inputs:
- {name: input_gcs_path, type: GCSPath, description: 'GCS file path'}
implementation:
container:
component_op = components.load_component_from_text('''
name: Component with optional inputs
inputs:
- {name: required_input, type: String, optional: false}
- {name: optional_input_1, type: String, optional: true}
- {name: optional_input_2, type: String, optional: true}
implementation:
container:
image: gcr.io/google-containers/busybox
command:
- echo
args:
- --arg0
- {inputValue: required_input}
- if:
cond:
isPresent: optional_input_1
then:
- --arg1
- {inputValue: optional_input_1}
- if:
cond:
isPresent: optional_input_2
then:
- --arg2
- {inputValue: optional_input_2}
else:
- --arg2
- 'default value'
''')
# Simple two-step pipeline with 'producer' and 'consumer' steps
from kfp.v2 import components
from kfp.v2 import compiler
from kfp.v2 import dsl
producer_op = components.load_component_from_text(
"""
name: Producer
inputs:
- {name: input_text, type: String, description: 'Represents an input parameter.'}
outputs:
- {name: output_value, type: String, description: 'Represents an output paramter.'}
implementation:
container:
image: google/cloud-sdk:latest
command:
- sh
- -c
- |
set -e -x
echo "$0, this is an output parameter" | gsutil cp - "$1"
- {inputValue: input_text}
- {outputPath: output_value}
"""
)
consumer_op = components.load_component_from_text(
"""
name: Consumer
inputs:
- {name: input_value, type: String, description: 'Represents an input parameter. It connects to an upstream output parameter.'}
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from kfp.v2 import components
from kfp.v2 import dsl
import kfp.v2.compiler as compiler
component_op = components.load_component_from_text("""
name: Component with concat placeholder
inputs:
- {name: input_one, type: String}
- {name: input_two, type: String}
implementation:
container:
image: gcr.io/google-containers/busybox
command:
- sh
- -ec
args:
- echo "$0" > /tmp/test && [[ "$0" == 'one+two=three' ]]
- concat: [{inputValue: input_one}, '+', {inputValue: input_two}, '=three']
""")
@dsl.pipeline(name='one-step-pipeline-with-concat-placeholder')
def pipeline_with_concat_placeholder():
component = component_op(input_one='one', input_two='two')
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from kfp.v2 import components
from kfp.v2 import dsl
import kfp.v2.compiler as compiler
component_op = components.load_component_from_text("""
name: Print Text
inputs:
- {name: text, type: String}
implementation:
container:
image: alpine
command:
- sh
- -c
- |
set -e -x
echo "$0"
- {inputValue: text}
""")
@dsl.pipeline(name='pipeline-with-after')
def my_pipeline():
task1 = component_op(text='1st task')
task2 = component_op(text='2nd task').after(task1)
task3 = component_op(text='3rd task').after(task1, task2)