def test_compile_simple_pipeline(self):
tmpdir = tempfile.mkdtemp()
try:
producer_op = components.load_component_from_text("""
name: producer
inputs:
- {name: input_param, type: String}
outputs:
- {name: output_model, type: Model}
- {name: output_value, type: Integer}
implementation:
container:
image: gcr.io/my-project/my-image:tag
args:
- {inputValue: input_param}
- {outputPath: output_model}
- {outputPath: output_value}
""")
consumer_op = components.load_component_from_text("""
name: consumer
inputs:
- {name: input_model, type: Model}
- {name: input_value, type: Integer}
implementation:
container:
image: gcr.io/my-project/my-image:tag
args:
- {inputPath: input_model}
- {inputValue: input_value}
""")
@dsl.pipeline(name='test-pipeline', pipeline_root='dummy_root')
def simple_pipeline(pipeline_input:str='Hello KFP!'):
producer = producer_op(input_param=pipeline_input)
consumer = consumer_op(
input_model=producer.outputs['output_model'],
input_value=producer.outputs['output_value'])
target_json_file = os.path.join(tmpdir, 'result.json')
compiler.Compiler().compile(
pipeline_func=simple_pipeline,
package_path=target_json_file)
self.assertTrue(os.path.exists(target_json_file))
finally:
shutil.rmtree(tmpdir)
def test_compile_pipeline_with_dsl_condition_should_raise_error(self):
flip_coin_op = components.load_component_from_text("""
name: flip coin
inputs:
- {name: name, type: String}
outputs:
- {name: result, type: String}
implementation:
container:
image: gcr.io/my-project/my-image:tag
args:
- {inputValue: name}
- {outputPath: result}
""")
print_op = components.load_component_from_text("""
name: print
inputs:
- {name: name, type: String}
- {name: msg, type: String}
implementation:
container:
image: gcr.io/my-project/my-image:tag
args:
- {inputValue: name}
- {inputValue: msg}
""")
@dsl.pipeline()
def flipcoin():
flip = flip_coin_op('flip')
with dsl.Condition(flip.outputs['result'] == 'heads'):
flip2 = flip_coin_op('flip-again')
with dsl.Condition(flip2.outputs['result'] == 'tails'):
print_op('print1', flip2.outputs['result'])
with dsl.Condition(flip.outputs['result'] == 'tails'):
print_op('print2', flip2.outputs['results'])
with self.assertRaises(NotImplementedError) as cm:
compiler.Compiler().compile(flipcoin, 'output.json')
self.assertEqual(
'dsl.Condition is not yet supported in KFP v2 compiler.',
str(cm.exception))
def test_constructing_container_op_directly_should_error(self):
@dsl.pipeline(name='test-pipeline')
def my_pipeline():
dsl.ContainerOp(
name='comp1',
image='gcr.io/dummy',
command=['python', 'main.py'])
with self.assertRaisesRegex(
RuntimeError,
'Constructing ContainerOp instances directly is deprecated and not '
'supported when compiling to v2 \(using v2 compiler or v1 compiler '
'with V2_COMPATIBLE or V2_ENGINE mode\).'):
compiler.Compiler().compile(
pipeline_func=my_pipeline, package_path='result.json')
def test_passing_missing_type_annotation_on_pipeline_input_should_error(
self):
@dsl.pipeline(name='test-pipeline', pipeline_root='gs://path')
def my_pipeline(input1):
pass
with self.assertRaisesRegex(
TypeError,
'The pipeline argument \"input1\" is viewed as an artifact due '
'to its type \"None\". And we currently do not support passing '
'artifacts as pipeline inputs. Consider type annotating the '
'argument with a primitive type, such as \"str\", \"int\", '
'\"float\", \"bool\", \"dict\", and \"list\".'):
compiler.Compiler().compile(
pipeline_func=my_pipeline, package_path='output.json')
def test_video_data_pipeline_component_ops_compile(self):
@kfp.dsl.pipeline(name="training-test")
def pipeline():
dataset_create_op = VideoDatasetCreateOp(
project=self._project,
display_name=self._display_name,
gcs_source=self._gcs_source,
import_schema_uri=aiplatform.schema.dataset.ioformat.video.
classification,
)
training_job_run_op = AutoMLVideoTrainingJobRunOp(
project=self._project,
display_name=self._display_name,
model_type="CLOUD",
dataset=dataset_create_op.outputs["dataset"],
prediction_type="classification",
training_fraction_split=0.6,
test_fraction_split=0.2,
model_display_name=self._model_display_name,
)
model_deploy_op = ModelDeployOp(
model=training_job_run_op.outputs["model"])
batch_predict_op = ModelBatchPredictOp(
project=self._project,
model=training_job_run_op.outputs["model"],
job_display_name=self._display_name,
gcs_source_uris=self._gcs_source,
gcs_destination_output_uri_prefix=self._gcs_destination_prefix,
)
dataset_export_op = VideoDatasetExportDataOp(
project=self._project,
dataset=dataset_create_op.outputs["dataset"],
output_dir=self._gcs_output_dir,
)
dataset_import_op = VideoDatasetImportDataOp(
gcs_source=self._gcs_source,
dataset=dataset_create_op.outputs["dataset"],
import_schema_uri=aiplatform.schema.dataset.ioformat.video.
classification)
compiler.Compiler().compile(pipeline_func=pipeline,
package_path=self._package_path)
def test_model_deploy_op_and_model_undeploy_op_compile(self):
@kfp.dsl.pipeline(name="training-test")
def pipeline():
model_upload_op = ModelUploadOp(
project=self._project,
display_name=self._display_name,
serving_container_image_uri=self._serving_container_image_uri,
artifact_uri=self._artifact_uri)
create_endpoint_op = EndpointCreateOp(
project=self._project,
location=self._location,
display_name=self._display_name)
model_deploy_op = ModelDeployOp(
model=model_upload_op.outputs["model"],
endpoint=create_endpoint_op.outputs["endpoint"],
deployed_model_display_name="deployed_model_display_name",
traffic_split={},
dedicated_resources_machine_type="n1-standard-4",
dedicated_resources_min_replica_count=1,
dedicated_resources_max_replica_count=2,
dedicated_resources_accelerator_type="fake-accelerator",
dedicated_resources_accelerator_count=1,
automatic_resources_min_replica_count=1,
automatic_resources_max_replica_count=2,
service_account="fake-sa",
explanation_metadata={"xai_m": "bar"},
explanation_parameters={"xai_p": "foo"},
)
_ = ModelUndeployOp(
model=model_upload_op.outputs["model"],
endpoint=create_endpoint_op.outputs["endpoint"],
).after(model_deploy_op)
compiler.Compiler().compile(pipeline_func=pipeline,
package_path=self._package_path)
with open(self._package_path) as f:
executor_output_json = json.load(f, strict=False)
with open("testdata/model_deploy_and_undeploy_pipeline.json") as ef:
expected_executor_output_json = json.load(ef, strict=False)
# Ignore the kfp SDK & schema version during comparison
del executor_output_json["sdkVersion"]
del executor_output_json["schemaVersion"]
self.assertEqual(executor_output_json, expected_executor_output_json)
def test_task_input_contain_placedholder_should_fail(self):
@components.create_component_from_func
def print_op(s: str):
print(s)
@dsl.pipeline(
name='pipeline-with-placeholder-in-input-argument',
pipeline_root='dummy')
def my_pipeline(name: str = 'KFP'):
print_op('Hello {}'.format(name))
with self.assertRaisesRegex(
NotImplementedError,
'a component input can only accept either a constant value or '
'a reference to another pipeline parameter.'):
compiler.Compiler().compile(my_pipeline, 'dummy')
def test_automl_image_component_compile(self):
@kfp.dsl.pipeline(name="training-test")
def pipeline():
dataset_create_op = ImageDatasetCreateOp(
project=self._project,
display_name=self._display_name,
gcs_source=self._gcs_source,
import_schema_uri=aiplatform.schema.dataset.ioformat.image.
single_label_classification,
)
training_job_run_op = AutoMLImageTrainingJobRunOp(
project=self._project,
display_name=self._display_name,
prediction_type="classification",
model_type="CLOUD",
# dataset=dataset_create_op.outputs["dataset"],
model_display_name=self._model_display_name,
training_fraction_split=0.6,
validation_fraction_split=0.2,
test_fraction_split=0.2,
budget_milli_node_hours=8000,
)
dataset_export_op = ImageDatasetExportDataOp(
project=self._project,
dataset=dataset_create_op.outputs["dataset"],
output_dir=self._gcs_output_dir,
)
dataset_import_op = ImageDatasetImportDataOp(
project=self._project,
gcs_source=self._gcs_source,
dataset=dataset_create_op.outputs["dataset"],
import_schema_uri=aiplatform.schema.dataset.ioformat.image.
single_label_classification)
compiler.Compiler().compile(pipeline_func=pipeline,
package_path=self._package_path)
with open(self._package_path) as f:
executor_output_json = json.load(f, strict=False)
with open("testdata/automl_image_pipeline.json") as ef:
expected_executor_output_json = json.load(ef, strict=False)
# Ignore the kfp SDK version during comparison
del executor_output_json["sdkVersion"]
self.assertEqual(executor_output_json, expected_executor_output_json)
def test_tabular_data_pipeline_component_ops_compile(self):
@kfp.dsl.pipeline(name="training-test")
def pipeline():
dataset_create_op = TabularDatasetCreateOp(
project=self._project,
display_name=self._display_name,
gcs_source=self._gcs_source,
)
training_job_run_op = AutoMLTabularTrainingJobRunOp(
project=self._project,
display_name=self._display_name,
optimization_prediction_type='regression',
optimization_objective='minimize-rmse',
column_transformations=[
{
"numeric": {
"column_name": "longitude"
}
},
],
target_column="longitude",
dataset=dataset_create_op.outputs["dataset"],
)
model_deploy_op = ModelDeployOp(
project=self._project,
model=training_job_run_op.outputs["model"])
batch_predict_op = ModelBatchPredictOp(
project=self._project,
model=training_job_run_op.outputs["model"],
job_display_name=self._display_name,
gcs_source=self._gcs_source,
gcs_destination_prefix=self._gcs_destination_prefix,
)
dataset_export_op = TabularDatasetExportDataOp(
project=self._project,
dataset=dataset_create_op.outputs["dataset"],
output_dir=self._gcs_output_dir,
)
compiler.Compiler().compile(pipeline_func=pipeline,
pipeline_root=self._pipeline_root,
output_path="pipeline.json")
def test_batch_prediction_op_compile(self):
@kfp.dsl.pipeline(name="training-test")
def pipeline():
model_upload_op = ModelUploadOp(
project=self._project,
display_name=self._display_name,
serving_container_image_uri=self._serving_container_image_uri,
artifact_uri=self._artifact_uri)
batch_predict_op = ModelBatchPredictOp(
project=self._project,
location=self._location,
job_display_name=self._display_name,
model=model_upload_op.outputs["model"],
instances_format="instance_format",
gcs_source_uris=[self._gcs_source],
bigquery_source_input_uri="bigquery_source_input_uri",
model_parameters={"foo": "bar"},
predictions_format="predictions_format",
gcs_destination_output_uri_prefix=self._gcs_destination_prefix,
bigquery_destination_output_uri=
"bigquery_destination_output_uri",
machine_type="machine_type",
accelerator_type="accelerator_type",
accelerator_count=1,
starting_replica_count=2,
max_replica_count=3,
manual_batch_tuning_parameters_batch_size=4,
generate_explanation=True,
explanation_metadata={"xai_m": "bar"},
explanation_parameters={"xai_p": "foo"},
encryption_spec_key_name="some encryption_spec_key_name",
labels={"foo": "bar"})
compiler.Compiler().compile(pipeline_func=pipeline,
package_path=self._package_path)
with open(self._package_path) as f:
executor_output_json = json.load(f, strict=False)
with open("testdata/batch_prediction_pipeline.json") as ef:
expected_executor_output_json = json.load(ef, strict=False)
# Ignore the kfp SDK version during comparison
del executor_output_json["sdkVersion"]
self.assertEqual(executor_output_json, expected_executor_output_json)
def test_compile_pipeline_with_dsl_exithandler_should_raise_error(self):
gcs_download_op = components.load_component_from_text("""
name: GCS - Download
inputs:
- {name: url, type: String}
outputs:
- {name: result, type: String}
implementation:
container:
image: gcr.io/my-project/my-image:tag
args:
- {inputValue: url}
- {outputPath: result}
""")
echo_op = components.load_component_from_text("""
name: echo
inputs:
- {name: msg, type: String}
implementation:
container:
image: gcr.io/my-project/my-image:tag
args:
- {inputValue: msg}
""")
@dsl.pipeline()
def download_and_print(url='gs://ml-pipeline/shakespeare/shakespeare1.txt'):
"""A sample pipeline showing exit handler."""
exit_task = echo_op('exit!')
with dsl.ExitHandler(exit_task):
download_task = gcs_download_op(url)
echo_task = echo_op(download_task.outputs['result'])
with self.assertRaises(NotImplementedError) as cm:
compiler.Compiler().compile(
pipeline_func=download_and_print,
pipeline_root='dummy_root',
output_path='output.json')
self.assertEqual('dsl.ExitHandler is not yet supported in KFP v2 compiler.',
str(cm.exception))
def _compile_pipeline_function(pipeline_funcs: List[Callable],
function_name: Optional[str],
pipeline_root: str,
pipeline_parameters: Optional[Mapping[str,
Any]],
package_path: str, type_check: bool) -> None:
"""Compiles a pipeline function.
Args:
pipeline_funcs: A list of pipeline_functions.
function_name: The name of the pipeline function to compile if there were
multiple.
pipeline_root: The root output directory for pipeline runtime.
pipeline_parameters: The pipeline parameters as a dict of {name: value}.
package_path: The output path of the compiled result.
type_check: Whether to enable the type checking.
"""
if len(pipeline_funcs) == 0:
raise ValueError(
'A function with @dsl.pipeline decorator is required in the py file.'
)
if len(pipeline_funcs) > 1 and not function_name:
func_names = [x.__name__ for x in pipeline_funcs]
raise ValueError(
'There are multiple pipelines: %s. Please specify --function.' %
func_names)
if function_name:
pipeline_func = next(
(x for x in pipeline_funcs if x.__name__ == function_name), None)
if not pipeline_func:
raise ValueError('The function "%s" does not exist. '
'Did you forget @dsl.pipeline decoration?' %
function_name)
else:
pipeline_func = pipeline_funcs[0]
compiler.Compiler().compile(pipeline_func=pipeline_func,
pipeline_root=pipeline_root,
pipeline_parameters=pipeline_parameters,
package_path=package_path,
type_check=type_check)
def test_compile_pipeline_with_dsl_parallelfor_should_raise_error(self):
@components.create_component_from_func
def print_op(s: str):
print(s)
@dsl.pipeline()
def my_pipeline():
loop_args = [{'A_a': 1, 'B_b': 2}, {'A_a': 10, 'B_b': 20}]
with dsl.ParallelFor(loop_args, parallelism=10) as item:
print_op(item)
print_op(item.A_a)
print_op(item.B_b)
with self.assertRaisesRegex(
NotImplementedError,
'dsl.ParallelFor is not yet supported in KFP v2 compiler.'):
compiler.Compiler().compile(pipeline_func=my_pipeline,
pipeline_root='dummy_root',
output_path='output.json')
def test_compile_pipeline_with_dsl_parallelfor_should_raise_error(self):
@components.create_component_from_func
def print_op(s: str):
print(s)
@dsl.pipeline()
def my_pipeline():
loop_args = [{'A_a': 1, 'B_b': 2}, {'A_a': 10, 'B_b': 20}]
with dsl.ParallelFor(loop_args, parallelism=10) as item:
print_op(item)
print_op(item.A_a)
print_op(item.B_b)
with self.assertRaises(NotImplementedError) as cm:
compiler.Compiler().compile(my_pipeline, 'output.json')
self.assertEqual(
'dsl.ParallelFor is not yet supported in KFP v2 compiler.',
str(cm.exception))
def test_set_pipeline_root_through_pipeline_decorator(self):
tmpdir = tempfile.mkdtemp()
try:
@dsl.pipeline(name='test-pipeline', pipeline_root='gs://path')
def my_pipeline():
VALID_PRODUCER_COMPONENT_SAMPLE(input_param='input')
target_json_file = os.path.join(tmpdir, 'result.json')
compiler.Compiler().compile(
pipeline_func=my_pipeline, package_path=target_json_file)
self.assertTrue(os.path.exists(target_json_file))
with open(target_json_file) as f:
pipeline_spec = json.load(f)
self.assertEqual('gs://path', pipeline_spec['defaultPipelineRoot'])
finally:
shutil.rmtree(tmpdir)
def test_automl_forecasting_component_compile(self):
@kfp.dsl.pipeline(name="training-test")
def pipeline():
dataset_create_op = TimeSeriesDatasetCreateOp(
project=self._project,
display_name=self._display_name,
gcs_source=self._gcs_source,
)
training_job_run_op = AutoMLForecastingTrainingJobRunOp(
project=self._project,
display_name=self._display_name,
dataset=dataset_create_op.outputs["dataset"],
optimization_objective="optimization_objective",
training_fraction_split=0.6,
test_fraction_split=0.2,
model_display_name=self._model_display_name,
target_column='target_column',
time_column='time_column',
time_series_identifier_column='time_series_identifier_column',
unavailable_at_forecast_columns=[],
available_at_forecast_columns=[],
forecast_horizon=12,
data_granularity_unit='data_granularity_unit',
data_granularity_count=1,
)
dataset_export_op = TimeSeriesDatasetExportDataOp(
project=self._project,
dataset=dataset_create_op.outputs["dataset"],
output_dir=self._gcs_output_dir,
)
compiler.Compiler().compile(pipeline_func=pipeline,
package_path=self._package_path)
with open(self._package_path) as f:
executor_output_json = json.load(f, strict=False)
with open("testdata/automl_forecasting_pipeline.json") as ef:
expected_executor_output_json = json.load(ef, strict=False)
# Ignore the kfp SDK version during comparison
del executor_output_json["sdkVersion"]
self.assertEqual(executor_output_json, expected_executor_output_json)
def test_dataproc_create_pyspark_batch_op_compile(self):
"""Compile a test pipeline using the Dataproc PySparkBatch component."""
@kfp.dsl.pipeline(name='create-pyspark-batch-test')
def pipeline():
dataproc.DataprocPySparkBatchOp(
project=self._project,
location=self._location,
batch_id=self._batch_id,
labels=self._labels,
container_image=self._container_image,
runtime_config_version=self._runtime_config_version,
runtime_config_properties=self._runtime_config_properties,
service_account=self._service_account,
network_tags=self._network_tags,
kms_key=self._kms_key,
network_uri=self._network_uri,
subnetwork_uri=self._subnetwork_uri,
metastore_service=self._metastore_service,
spark_history_dataproc_cluster=self.
_spark_history_dataproc_cluster,
main_python_file_uri=self._main_python_file_uri,
python_file_uris=self._python_file_uris,
jar_file_uris=self._jar_file_uris,
file_uris=self._file_uris,
archive_uris=self._archive_uris,
args=self._batch_specific_args)
compiler.Compiler().compile(pipeline_func=pipeline,
package_path=self._package_path)
with open(self._package_path) as f:
executor_output_json = json.load(f, strict=False)
with open(
'testdata/dataproc_create_pyspark_batch_component_pipeline.json'
) as ef:
expected_executor_output_json = json.load(ef, strict=False)
# Ignore the kfp SDK & schema version during comparison
del executor_output_json['sdkVersion']
del executor_output_json['schemaVersion']
self.assertDictEqual(executor_output_json,
expected_executor_output_json)
def test_set_pipeline_root_through_pipeline_decorator(self):
tmpdir = tempfile.mkdtemp()
try:
@dsl.pipeline(name='my-pipeline', pipeline_root='gs://path')
def my_pipeline():
pass
target_json_file = os.path.join(tmpdir, 'result.json')
compiler.Compiler().compile(pipeline_func=my_pipeline,
output_path=target_json_file)
self.assertTrue(os.path.exists(target_json_file))
with open(target_json_file) as f:
job_spec = json.load(f)
self.assertEqual('gs://path',
job_spec['runtimeConfig']['gcsOutputDirectory'])
finally:
shutil.rmtree(tmpdir)
def test_missing_pipeline_root_is_allowed_but_warned(self):
tmpdir = tempfile.mkdtemp()
try:
@dsl.pipeline(name='my-pipeline')
def my_pipeline():
pass
target_json_file = os.path.join(tmpdir, 'result.json')
with self.assertWarnsRegex(UserWarning, 'pipeline_root is None or empty'):
compiler.Compiler().compile(
pipeline_func=my_pipeline, output_path=target_json_file)
self.assertTrue(os.path.exists(target_json_file))
with open(target_json_file) as f:
job_spec = json.load(f)
self.assertTrue('gcsOutputDirectory' not in job_spec['runtimeConfig'])
finally:
shutil.rmtree(tmpdir)
def test_automl_tabular_component_compile(self):
@kfp.dsl.pipeline(name="training-test")
def pipeline():
dataset_create_op = TabularDatasetCreateOp(
project=self._project,
display_name=self._display_name,
gcs_source=self._gcs_source,
)
training_job_run_op = AutoMLTabularTrainingJobRunOp(
project=self._project,
display_name=self._display_name,
optimization_prediction_type="regression",
optimization_objective="minimize-rmse",
column_transformations=[
{
"numeric": {
"column_name": "longitude"
}
},
],
target_column="longitude",
dataset=dataset_create_op.outputs["dataset"],
)
dataset_export_op = TabularDatasetExportDataOp(
project=self._project,
dataset=dataset_create_op.outputs["dataset"],
output_dir=self._gcs_output_dir,
)
compiler.Compiler().compile(pipeline_func=pipeline,
package_path=self._package_path)
with open(self._package_path) as f:
executor_output_json = json.load(f, strict=False)
with open("testdata/automl_tabular_pipeline.json") as ef:
expected_executor_output_json = json.load(ef, strict=False)
# Ignore the kfp SDK version during comparison
del executor_output_json["sdkVersion"]
self.assertEqual(executor_output_json, expected_executor_output_json)
def test_model_upload_and_model_delete_op_compile(self):
@kfp.dsl.pipeline(name="training-test")
def pipeline():
model_upload_op = ModelUploadOp(
project=self._project,
location=self._location,
display_name=self._display_name,
description="some description",
serving_container_image_uri=self._serving_container_image_uri,
serving_container_command=["command1", "command2"],
serving_container_args=["arg1", "arg2"],
serving_container_environment_variables=["env1", "env2"],
serving_container_ports=["123", "456"],
serving_container_predict_route=
"some serving_container_predict_route",
serving_container_health_route=
"some serving_container_health_route",
instance_schema_uri="some instance_schema_uri",
parameters_schema_uri="some parameters_schema_uri",
prediction_schema_uri="some prediction_schema_uri",
artifact_uri="some artifact_uri",
explanation_metadata={"xai_m": "bar"},
explanation_parameters={"xai_p": "foo"},
encryption_spec_key_name="some encryption_spec_key_name",
labels={"foo": "bar"})
_ = ModelDeleteOp(model=model_upload_op.outputs["model"], )
compiler.Compiler().compile(pipeline_func=pipeline,
package_path=self._package_path)
with open(self._package_path) as f:
executor_output_json = json.load(f, strict=False)
with open("testdata/model_upload_and_delete_pipeline.json") as ef:
expected_executor_output_json = json.load(ef, strict=False)
# Ignore the kfp SDK & schema version during comparison
del executor_output_json["sdkVersion"]
del executor_output_json["schemaVersion"]
self.assertEqual(executor_output_json, expected_executor_output_json)
def test_compile_pipeline_with_misused_outputuri_should_raise_error(self):
component_op = components.load_component_from_text("""
name: compoent with misused placeholder
outputs:
- {name: value, type: Integer}
implementation:
container:
image: dummy
args:
- {outputUri: value}
""")
@dsl.pipeline(name='test-pipeline', pipeline_root='dummy_root')
def my_pipeline():
component_op()
with self.assertRaisesRegex(
TypeError,
' type "Integer" cannot be paired with OutputUriPlaceholder.'):
compiler.Compiler().compile(
pipeline_func=my_pipeline, package_path='output.json')
def test_compile_pipeline_with_misused_inputpath_should_raise_error(self):
component_op = components.load_component_from_text("""
name: compoent with misused placeholder
inputs:
- {name: text, type: String}
implementation:
container:
image: dummy
args:
- {inputPath: text}
""")
@dsl.pipeline(name='test-pipeline', pipeline_root='dummy_root')
def my_pipeline(text: str):
component_op(text=text)
with self.assertRaisesRegex(
TypeError,
' type "String" cannot be paired with InputPathPlaceholder.'):
compiler.Compiler().compile(
pipeline_func=my_pipeline, package_path='output.json')
def test_compile_pipeline_with_misused_inputuri_should_raise_error(self):
component_op = components.load_component_from_text("""
name: compoent with misused placeholder
inputs:
- {name: value, type: Float}
implementation:
container:
image: dummy
args:
- {inputUri: value}
""")
def my_pipeline(value):
component_op(value=value)
with self.assertRaisesRegex(
TypeError,
' type "Float" cannot be paired with InputUriPlaceholder.'):
compiler.Compiler().compile(pipeline_func=my_pipeline,
pipeline_root='dummy',
output_path='output.json')
def test_compile_pipeline_with_misused_inputvalue_should_raise_error(self):
component_op = components.load_component_from_text("""
name: compoent with misused placeholder
inputs:
- {name: model, type: Model}
implementation:
container:
image: dummy
args:
- {inputValue: model}
""")
@dsl.pipeline(name='test-pipeline', pipeline_root='dummy_root')
def my_pipeline(model):
component_op(model=model)
with self.assertRaisesRegex(
TypeError,
' type "Model" cannot be paired with InputValuePlaceholder.'):
compiler.Compiler().compile(pipeline_func=my_pipeline,
package_path='output.json')
def test_invalid_data_dependency(self):
@dsl.component
def producer_op() -> str:
return 'a'
@dsl.component
def dummy_op(msg: str = ''):
pass
@dsl.pipeline(name='test-pipeline')
def my_pipeline(text: bool):
with dsl.ParallelFor(['a, b']):
producer_task = producer_op()
dummy_op(msg=producer_task.output)
with self.assertRaisesRegex(
RuntimeError,
'Task dummy-op cannot dependent on any task inside the group:'
):
compiler.Compiler().compile(pipeline_func=my_pipeline,
package_path='result.json')
def test_invalid_after_dependency(self):
@dsl.component
def producer_op() -> str:
return 'a'
@dsl.component
def dummy_op(msg: str = ''):
pass
@dsl.pipeline(name='test-pipeline')
def my_pipeline(text: str):
with dsl.Condition(text == 'a'):
producer_task = producer_op()
dummy_op().after(producer_task)
with self.assertRaisesRegex(
RuntimeError,
'Task dummy-op cannot dependent on any task inside the group:'
):
compiler.Compiler().compile(pipeline_func=my_pipeline,
package_path='result.json')
def main(pipeline_root: str = 'gs://gongyuan-test/hello_world'):
def hello_world(text: str):
print(text)
return text
components.func_to_container_op(hello_world,
output_component_file='hw.yaml')
# Create a pipeline op from the component we defined above.
hw_op = components.load_component_from_file(
'./hw.yaml') # you can also use load_component_from_url
@dsl.pipeline(name='hello-world', description='A simple intro pipeline')
def pipeline_parameter_to_consumer(text: str = 'hi there'):
'''Pipeline that passes small pipeline parameter string to consumer op'''
consume_task = hw_op(
text) # Passing pipeline parameter as argument to consumer op
pipeline_func = pipeline_parameter_to_consumer
compiler.Compiler().compile(pipeline_func=pipeline_func,
pipeline_root=pipeline_root,
output_path='hw_pipeline_job.json')
def test_create_endpoint_op_compile(self):
@kfp.dsl.pipeline(name="create-endpoint-test")
def pipeline():
create_endpoint_op = EndpointCreateOp(
project=self._project,
location=self._location,
display_name=self._display_name,
description="some description",
labels={"foo": "bar"},
network="abc",
encryption_spec_key_name='some encryption_spec_key_name')
compiler.Compiler().compile(pipeline_func=pipeline,
package_path=self._package_path)
with open(self._package_path) as f:
executor_output_json = json.load(f, strict=False)
with open('testdata/create_endpoint_pipeline.json') as ef:
expected_executor_output_json = json.load(ef, strict=False)
# Ignore the kfp SDK & schema version during comparision
del executor_output_json['pipelineSpec']['sdkVersion']
del executor_output_json['pipelineSpec']['schemaVersion']
self.assertEqual(executor_output_json, expected_executor_output_json)
def test_use_task_final_status_in_non_exit_op_yaml(self):
print_op = components.load_component_from_text("""
name: Print Op
inputs:
- {name: message, type: PipelineTaskFinalStatus}
implementation:
container:
image: python:3.7
command:
- echo
- {inputValue: message}
""")
@dsl.pipeline(name='test-pipeline')
def my_pipeline(text: bool):
print_op()
with self.assertRaisesRegex(
ValueError,
'PipelineTaskFinalStatus can only be used in an exit task.'):
compiler.Compiler().compile(pipeline_func=my_pipeline,
package_path='result.json')