def main(pipeline_root: str = 'gs://gongyuan-test/hello_world'):
def hello_world(text: str):
print(text)
return text
components.func_to_container_op(hello_world,
output_component_file='hw.yaml')
# Create a pipeline op from the component we defined above.
hw_op = components.load_component_from_file(
'./hw.yaml') # you can also use load_component_from_url
@dsl.pipeline(name='hello-world', description='A simple intro pipeline')
def pipeline_parameter_to_consumer(text: str = 'hi there'):
'''Pipeline that passes small pipeline parameter string to consumer op'''
consume_task = hw_op(
text) # Passing pipeline parameter as argument to consumer op
pipeline_func = pipeline_parameter_to_consumer
compiler.Compiler().compile(pipeline_func=pipeline_func,
pipeline_root=pipeline_root,
output_path='hw_pipeline_job.json')
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pathlib
from kfp.v2 import components
from kfp.v2 import dsl
import kfp.v2.compiler as compiler
test_data_dir = pathlib.Path(__file__).parent / 'component_yaml'
trainer_op = components.load_component_from_file(
str(test_data_dir / 'trainer_component.yaml'))
serving_op = components.load_component_from_file(
str(test_data_dir / 'serving_component.yaml'))
@dsl.pipeline(name='two-step-pipeline-with-importer',
description='A linear two-step pipeline.')
def my_pipeline(input_gcs='gs://test-bucket/pipeline_root',
optimizer: str = 'sgd',
epochs: int = 200):
trainer = trainer_op(input_location=input_gcs,
train_optimizer=optimizer,
num_epochs=epochs)
serving = serving_op(model=trainer.outputs['model_output'],
model_cfg=trainer.outputs['model_config'])
# limitations under the License.
"""Google Cloud Pipeline Experimental Forecasting Components."""
import os
from typing import Optional
from google.cloud import aiplatform as aiplatform_sdk
from google_cloud_pipeline_components.aiplatform import utils
try:
from kfp.v2.components import load_component_from_file
except ImportError:
from kfp.components import load_component_from_file
__all__ = [
'ForecastingPreprocessingOp',
'ForecastingValidationOp',
'ForecastingPrepareDataForTrainOp',
]
ForecastingPreprocessingOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'preprocess/component.yaml'))
ForecastingValidationOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'validate/component.yaml'))
ForecastingPrepareDataForTrainOp = load_component_from_file(
os.path.join(
os.path.dirname(__file__), 'prepare_data_for_train/component.yaml'))
"""Module for AutoML Tables KFP components."""
import os
try:
from kfp.v2.components import load_component_from_file
except ImportError:
from kfp.components import load_component_from_file
__all__ = [
'CvTrainerOp', 'InfraValidatorOp', 'Stage1TunerOp', 'EnsembleOp',
'StatsAndExampleGenOp', 'FeatureSelectionOp', 'TransformOp', 'FinalizerOp',
'WideAndDeepTrainerOp'
]
CvTrainerOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'cv_trainer.yaml'))
InfraValidatorOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'infra_validator.yaml'))
Stage1TunerOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'stage_1_tuner.yaml'))
EnsembleOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'ensemble.yaml'))
StatsAndExampleGenOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'stats_and_example_gen.yaml'))
FeatureSelectionOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'feature_selection.yaml'))
TransformOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'transform.yaml'))
FinalizerOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'finalizer.yaml'))
WideAndDeepTrainerOp = load_component_from_file(
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module for supporting Google Vertex AI Custom Training Job Op."""
import os
from .utils import create_custom_training_job_op_from_component
# Aliasing for better readability
create_custom_training_job_from_component = create_custom_training_job_op_from_component
try:
from kfp.v2.components import load_component_from_file
except ImportError:
from kfp.components import load_component_from_file
__all__ = [
'CustomTrainingJobOp',
'create_custom_training_job_op_from_component',
'create_custom_training_job_from_component',
]
CustomTrainingJobOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'component.yaml'))
# pytype: skip-file
# Copyright 2021 The Kubeflow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Core modules for AI Platform Pipeline Components."""
import os
try:
from kfp.v2.components import load_component_from_file
except ImportError:
from kfp.components import load_component_from_file
__all__ = [
'ModelBatchPredictOp',
]
ModelBatchPredictOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'component.yaml'))
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pathlib
from kfp.v2 import components
from kfp.v2 import dsl
from kfp.v2 import compiler
test_data_dir = pathlib.Path(__file__).parent / 'component_yaml'
component_op = components.load_component_from_file(
str(test_data_dir / 'if_placeholder_component.yaml'))
@dsl.pipeline(name='one-step-pipeline-with-if-placeholder',
pipeline_root='dummy_root')
def my_pipeline(input0: str, input1: str, input2: str):
# supply only optional_input_1 but not optional_input_2
component = component_op(required_input=input0, optional_input_1=input1)
if __name__ == '__main__':
compiler.Compiler().compile(pipeline_func=my_pipeline,
package_path=__file__.replace('.py', '.json'))
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from kfp.v2 import components
from kfp.v2 import dsl
import kfp.v2.compiler as compiler
import pathlib
test_data_dir = pathlib.Path(__file__).parent / 'component_yaml'
ingestion_op = components.load_component_from_file(
str(test_data_dir / 'ingestion_component.yaml'))
training_op = components.load_component_from_file(
str(test_data_dir / 'fancy_trainer_component.yaml'))
@dsl.pipeline(
name='two-step-pipeline-with-resource-spec',
description='A linear two-step pipeline with resource specification.')
def my_pipeline(input_location='gs://test-bucket/pipeline_root',
optimizer: str = 'sgd',
n_epochs: int = 200):
ingestor = ingestion_op(input_location=input_location)
_ = (training_op(examples=ingestor.outputs['examples'],
schema=ingestor.outputs['schema'],
optimizer=optimizer,
# Copyright 2021 The Kubeflow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Google Cloud Pipeline Model Evaluation components."""
import os
try:
from kfp.v2.components import load_component_from_file
except ImportError:
from kfp.components import load_component_from_file
__all__ = [
'ModelEvaluationOp',
]
ModelEvaluationOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'component.yaml'))
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Core modules for AI Platform Pipeline Components."""
import os
try:
from kfp.v2.components import load_component_from_file
except ImportError:
from kfp.components import load_component_from_file
__all__ = [
'EndpointCreateOp',
'ModelDeployOp',
]
EndpointCreateOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'create_endpoint/component.yaml'))
ModelDeployOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'deploy_model/component.yaml'))
import os
try:
from kfp.v2.components import load_component_from_file
except ImportError:
from kfp.components import load_component_from_file
__all__ = [
'BigqueryQueryJobOp',
'BigqueryCreateModelJobOp',
'BigqueryExportModelJobOp',
'BigqueryPredictModelJobOp',
'BigqueryEvaluateModelJobOp',
]
BigqueryQueryJobOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'query_job/component.yaml'))
BigqueryCreateModelJobOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'create_model/component.yaml'))
BigqueryExportModelJobOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'export_model/component.yaml'))
BigqueryPredictModelJobOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'predict_model/component.yaml'))
BigqueryEvaluateModelJobOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'evaluate_model/component.yaml'))
'CvTrainerOp',
'InfraValidatorOp',
'Stage1TunerOp',
'EnsembleOp',
'StatsAndExampleGenOp',
'FeatureSelectionOp',
'TransformOp',
'FinalizerOp',
'WideAndDeepTrainerOp',
'BuiltinAlgorithmHyperparameterTuningJobOp',
'TabNetTrainerOp',
'FeatureTransformEngineOp',
'TransformConfigurationPlannerOp',
]
CvTrainerOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'cv_trainer.yaml'))
InfraValidatorOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'infra_validator.yaml'))
Stage1TunerOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'stage_1_tuner.yaml'))
EnsembleOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'ensemble.yaml'))
StatsAndExampleGenOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'stats_and_example_gen.yaml'))
FeatureSelectionOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'feature_selection.yaml'))
TransformOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'transform.yaml'))
FeatureTransformEngineOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'feature_transform_engine.yaml'))
TransformConfigurationPlannerOp = load_component_from_file(
# Copyright 2022 The Kubeflow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Google Cloud Pipeline Experimental Vertex Notification Email Components."""
import os
try:
from kfp.v2.components import load_component_from_file
except ImportError:
from kfp.components import load_component_from_file
__all__ = [
'VertexNotificationEmailOp',
]
VertexNotificationEmailOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'component.yaml'))
# Copyright 2021 The Kubeflow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Google Cloud Pipeline Experimental Forecasting Components."""
import os
from typing import Optional
try:
from kfp.v2.components import load_component_from_file
except ImportError:
from kfp.components import load_component_from_file
__all__ = [
'NotebooksExecutorOp',
]
NotebooksExecutorOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'executor/component.yaml'))
# limitations under the License.
"""Google Cloud Pipeline Dataproc Batch components."""
import os
try:
from kfp.v2.components import load_component_from_file
except ImportError:
from kfp.components import load_component_from_file
__all__ = [
'DataprocPySparkBatchOp', 'DataprocSparkBatchOp', 'DataprocSparkRBatchOp',
'DataprocSparkSqlBatchOp'
]
DataprocPySparkBatchOp = load_component_from_file(
os.path.join(os.path.dirname(__file__),
'create_pyspark_batch/component.yaml'))
DataprocSparkBatchOp = load_component_from_file(
os.path.join(os.path.dirname(__file__),
'create_spark_batch/component.yaml'))
DataprocSparkRBatchOp = load_component_from_file(
os.path.join(os.path.dirname(__file__),
'create_spark_r_batch/component.yaml'))
DataprocSparkSqlBatchOp = load_component_from_file(
os.path.join(os.path.dirname(__file__),
'create_spark_sql_batch/component.yaml'))
'ModelDeployOp',
'ModelUndeployOp',
'ModelBatchPredictOp',
'ModelDeleteOp',
'ModelExportOp',
'ModelUploadOp',
'ModelImportEvaluationOp',
'EndpointCreateOp',
'EndpointDeleteOp',
'TimeSeriesDatasetCreateOp',
'TimeSeriesDatasetExportDataOp',
'AutoMLForecastingTrainingJobRunOp',
]
TimeSeriesDatasetCreateOp = load_component_from_file(
os.path.join(os.path.dirname(__file__),
'dataset/create_time_series_dataset/component.yaml'))
ImageDatasetCreateOp = load_component_from_file(
os.path.join(os.path.dirname(__file__),
'dataset/create_image_dataset/component.yaml'))
TabularDatasetCreateOp = load_component_from_file(
os.path.join(os.path.dirname(__file__),
'dataset/create_tabular_dataset/component.yaml'))
TextDatasetCreateOp = load_component_from_file(
os.path.join(os.path.dirname(__file__),
'dataset/create_text_dataset/component.yaml'))
VideoDatasetCreateOp = load_component_from_file(
# Copyright 2021 The Kubeflow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Google Cloud Pipeline Dataflow python components."""
import os
try:
from kfp.v2.components import load_component_from_file
except ImportError:
from kfp.components import load_component_from_file
__all__ = [
'DataflowPythonJobOp',
]
DataflowPythonJobOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'python_job/component.yaml'))
'TabularDatasetCreateOp',
'TextDatasetCreateOp',
'VideoDatasetCreateOp',
'ImageDatasetExportDataOp',
'TabularDatasetExportDataOp',
'TextDatasetExportDataOp',
'VideoDatasetExportDataOp',
'ImageDatasetImportDataOp',
'TextDatasetImportDataOp',
'VideoDatasetImportDataOp',
'TimeSeriesDatasetCreateOp',
'TimeSeriesDatasetExportDataOp',
]
TimeSeriesDatasetCreateOp = load_component_from_file(
os.path.join(
os.path.dirname(__file__), 'create_time_series_dataset/component.yaml'))
ImageDatasetCreateOp = load_component_from_file(
os.path.join(
os.path.dirname(__file__), 'create_image_dataset/component.yaml'))
TabularDatasetCreateOp = load_component_from_file(
os.path.join(
os.path.dirname(__file__), 'create_tabular_dataset/component.yaml'))
TextDatasetCreateOp = load_component_from_file(
os.path.join(
os.path.dirname(__file__), 'create_text_dataset/component.yaml'))
VideoDatasetCreateOp = load_component_from_file(
# Copyright 2021 The Kubeflow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Google Cloud Pipeline Wait GCP Resource Components."""
import os
from typing import Optional
try:
from kfp.v2.components import load_component_from_file
except ImportError:
from kfp.components import load_component_from_file
__all__ = [
'WaitGcpResourcesOp',
]
WaitGcpResourcesOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'component.yaml'))
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pathlib
from kfp.v2 import components
from kfp.v2 import compiler
from kfp.v2 import dsl
test_data_dir = pathlib.Path(__file__).parent / 'component_yaml'
add_op = components.load_component_from_file(
str(test_data_dir / 'add_component.yaml'))
@dsl.pipeline(name='add-pipeline', pipeline_root='dummy_root')
def my_pipeline(
a: int = 2,
b: int = 5,
):
first_add_task = add_op(a, 3)
second_add_task = add_op(first_add_task.outputs['sum'], b)
third_add_task = add_op(second_add_task.outputs['sum'], 7)
if __name__ == '__main__':
compiler.Compiler().compile(
pipeline_func=my_pipeline,
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module for supporting Google Vertex AI Hyperparameter Tuning Job Op."""
import os
from .utils import serialize_parameters, serialize_metrics
try:
from kfp.v2.components import load_component_from_file
except ImportError:
from kfp.components import load_component_from_file
__all__ = [
'HyperparameterTuningJobRunOp',
'serialize_parameters',
'serialize_metrics',
]
HyperparameterTuningJobRunOp = load_component_from_file(
os.path.join(os.path.dirname(__file__), 'component.yaml'))
aiplatform_sdk.AutoMLTabularTrainingJob,
aiplatform_sdk.AutoMLTabularTrainingJob.run,
)
AutoMLForecastingTrainingJobRunOp = utils.convert_method_to_component(
aiplatform_sdk.AutoMLForecastingTrainingJob,
aiplatform_sdk.AutoMLForecastingTrainingJob.run,
)
AutoMLVideoTrainingJobRunOp = utils.convert_method_to_component(
aiplatform_sdk.AutoMLVideoTrainingJob,
aiplatform_sdk.AutoMLVideoTrainingJob.run,
)
ModelExportOp = load_component_from_file(
os.path.join(os.path.dirname(__file__),
'model/export_model/component.yaml'))
ModelDeployOp = load_component_from_file(
os.path.join(os.path.dirname(__file__),
'endpoint/deploy_model/component.yaml'))
ModelBatchPredictOp = load_component_from_file(
os.path.join(os.path.dirname(__file__),
'batch_predict_job/component.yaml'))
ModelUploadOp = load_component_from_file(
os.path.join(os.path.dirname(__file__),
'model/upload_model/component.yaml'))
EndpointCreateOp = load_component_from_file(
