def main(pipeline_root: str = 'gs://gongyuan-test/hello_world'): def hello_world(text: str): print(text) return text components.func_to_container_op(hello_world, output_component_file='hw.yaml') # Create a pipeline op from the component we defined above. hw_op = components.load_component_from_file( './hw.yaml') # you can also use load_component_from_url @dsl.pipeline(name='hello-world', description='A simple intro pipeline') def pipeline_parameter_to_consumer(text: str = 'hi there'): '''Pipeline that passes small pipeline parameter string to consumer op''' consume_task = hw_op( text) # Passing pipeline parameter as argument to consumer op pipeline_func = pipeline_parameter_to_consumer compiler.Compiler().compile(pipeline_func=pipeline_func, pipeline_root=pipeline_root, output_path='hw_pipeline_job.json')
# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import pathlib from kfp.v2 import components from kfp.v2 import dsl import kfp.v2.compiler as compiler test_data_dir = pathlib.Path(__file__).parent / 'component_yaml' trainer_op = components.load_component_from_file( str(test_data_dir / 'trainer_component.yaml')) serving_op = components.load_component_from_file( str(test_data_dir / 'serving_component.yaml')) @dsl.pipeline(name='two-step-pipeline-with-importer', description='A linear two-step pipeline.') def my_pipeline(input_gcs='gs://test-bucket/pipeline_root', optimizer: str = 'sgd', epochs: int = 200): trainer = trainer_op(input_location=input_gcs, train_optimizer=optimizer, num_epochs=epochs) serving = serving_op(model=trainer.outputs['model_output'], model_cfg=trainer.outputs['model_config'])
# limitations under the License. """Google Cloud Pipeline Experimental Forecasting Components.""" import os from typing import Optional from google.cloud import aiplatform as aiplatform_sdk from google_cloud_pipeline_components.aiplatform import utils try: from kfp.v2.components import load_component_from_file except ImportError: from kfp.components import load_component_from_file __all__ = [ 'ForecastingPreprocessingOp', 'ForecastingValidationOp', 'ForecastingPrepareDataForTrainOp', ] ForecastingPreprocessingOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'preprocess/component.yaml')) ForecastingValidationOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'validate/component.yaml')) ForecastingPrepareDataForTrainOp = load_component_from_file( os.path.join( os.path.dirname(__file__), 'prepare_data_for_train/component.yaml'))
"""Module for AutoML Tables KFP components.""" import os try: from kfp.v2.components import load_component_from_file except ImportError: from kfp.components import load_component_from_file __all__ = [ 'CvTrainerOp', 'InfraValidatorOp', 'Stage1TunerOp', 'EnsembleOp', 'StatsAndExampleGenOp', 'FeatureSelectionOp', 'TransformOp', 'FinalizerOp', 'WideAndDeepTrainerOp' ] CvTrainerOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'cv_trainer.yaml')) InfraValidatorOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'infra_validator.yaml')) Stage1TunerOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'stage_1_tuner.yaml')) EnsembleOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'ensemble.yaml')) StatsAndExampleGenOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'stats_and_example_gen.yaml')) FeatureSelectionOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'feature_selection.yaml')) TransformOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'transform.yaml')) FinalizerOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'finalizer.yaml')) WideAndDeepTrainerOp = load_component_from_file(
# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Module for supporting Google Vertex AI Custom Training Job Op.""" import os from .utils import create_custom_training_job_op_from_component # Aliasing for better readability create_custom_training_job_from_component = create_custom_training_job_op_from_component try: from kfp.v2.components import load_component_from_file except ImportError: from kfp.components import load_component_from_file __all__ = [ 'CustomTrainingJobOp', 'create_custom_training_job_op_from_component', 'create_custom_training_job_from_component', ] CustomTrainingJobOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'component.yaml'))
# pytype: skip-file # Copyright 2021 The Kubeflow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Core modules for AI Platform Pipeline Components.""" import os try: from kfp.v2.components import load_component_from_file except ImportError: from kfp.components import load_component_from_file __all__ = [ 'ModelBatchPredictOp', ] ModelBatchPredictOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'component.yaml'))
# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import pathlib from kfp.v2 import components from kfp.v2 import dsl from kfp.v2 import compiler test_data_dir = pathlib.Path(__file__).parent / 'component_yaml' component_op = components.load_component_from_file( str(test_data_dir / 'if_placeholder_component.yaml')) @dsl.pipeline(name='one-step-pipeline-with-if-placeholder', pipeline_root='dummy_root') def my_pipeline(input0: str, input1: str, input2: str): # supply only optional_input_1 but not optional_input_2 component = component_op(required_input=input0, optional_input_1=input1) if __name__ == '__main__': compiler.Compiler().compile(pipeline_func=my_pipeline, package_path=__file__.replace('.py', '.json'))
# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from kfp.v2 import components from kfp.v2 import dsl import kfp.v2.compiler as compiler import pathlib test_data_dir = pathlib.Path(__file__).parent / 'component_yaml' ingestion_op = components.load_component_from_file( str(test_data_dir / 'ingestion_component.yaml')) training_op = components.load_component_from_file( str(test_data_dir / 'fancy_trainer_component.yaml')) @dsl.pipeline( name='two-step-pipeline-with-resource-spec', description='A linear two-step pipeline with resource specification.') def my_pipeline(input_location='gs://test-bucket/pipeline_root', optimizer: str = 'sgd', n_epochs: int = 200): ingestor = ingestion_op(input_location=input_location) _ = (training_op(examples=ingestor.outputs['examples'], schema=ingestor.outputs['schema'], optimizer=optimizer,
# Copyright 2021 The Kubeflow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Google Cloud Pipeline Model Evaluation components.""" import os try: from kfp.v2.components import load_component_from_file except ImportError: from kfp.components import load_component_from_file __all__ = [ 'ModelEvaluationOp', ] ModelEvaluationOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'component.yaml'))
# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Core modules for AI Platform Pipeline Components.""" import os try: from kfp.v2.components import load_component_from_file except ImportError: from kfp.components import load_component_from_file __all__ = [ 'EndpointCreateOp', 'ModelDeployOp', ] EndpointCreateOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'create_endpoint/component.yaml')) ModelDeployOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'deploy_model/component.yaml'))
import os try: from kfp.v2.components import load_component_from_file except ImportError: from kfp.components import load_component_from_file __all__ = [ 'BigqueryQueryJobOp', 'BigqueryCreateModelJobOp', 'BigqueryExportModelJobOp', 'BigqueryPredictModelJobOp', 'BigqueryEvaluateModelJobOp', ] BigqueryQueryJobOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'query_job/component.yaml')) BigqueryCreateModelJobOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'create_model/component.yaml')) BigqueryExportModelJobOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'export_model/component.yaml')) BigqueryPredictModelJobOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'predict_model/component.yaml')) BigqueryEvaluateModelJobOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'evaluate_model/component.yaml'))
'CvTrainerOp', 'InfraValidatorOp', 'Stage1TunerOp', 'EnsembleOp', 'StatsAndExampleGenOp', 'FeatureSelectionOp', 'TransformOp', 'FinalizerOp', 'WideAndDeepTrainerOp', 'BuiltinAlgorithmHyperparameterTuningJobOp', 'TabNetTrainerOp', 'FeatureTransformEngineOp', 'TransformConfigurationPlannerOp', ] CvTrainerOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'cv_trainer.yaml')) InfraValidatorOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'infra_validator.yaml')) Stage1TunerOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'stage_1_tuner.yaml')) EnsembleOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'ensemble.yaml')) StatsAndExampleGenOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'stats_and_example_gen.yaml')) FeatureSelectionOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'feature_selection.yaml')) TransformOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'transform.yaml')) FeatureTransformEngineOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'feature_transform_engine.yaml')) TransformConfigurationPlannerOp = load_component_from_file(
# Copyright 2022 The Kubeflow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Google Cloud Pipeline Experimental Vertex Notification Email Components.""" import os try: from kfp.v2.components import load_component_from_file except ImportError: from kfp.components import load_component_from_file __all__ = [ 'VertexNotificationEmailOp', ] VertexNotificationEmailOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'component.yaml'))
# Copyright 2021 The Kubeflow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Google Cloud Pipeline Experimental Forecasting Components.""" import os from typing import Optional try: from kfp.v2.components import load_component_from_file except ImportError: from kfp.components import load_component_from_file __all__ = [ 'NotebooksExecutorOp', ] NotebooksExecutorOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'executor/component.yaml'))
# limitations under the License. """Google Cloud Pipeline Dataproc Batch components.""" import os try: from kfp.v2.components import load_component_from_file except ImportError: from kfp.components import load_component_from_file __all__ = [ 'DataprocPySparkBatchOp', 'DataprocSparkBatchOp', 'DataprocSparkRBatchOp', 'DataprocSparkSqlBatchOp' ] DataprocPySparkBatchOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'create_pyspark_batch/component.yaml')) DataprocSparkBatchOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'create_spark_batch/component.yaml')) DataprocSparkRBatchOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'create_spark_r_batch/component.yaml')) DataprocSparkSqlBatchOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'create_spark_sql_batch/component.yaml'))
'ModelDeployOp', 'ModelUndeployOp', 'ModelBatchPredictOp', 'ModelDeleteOp', 'ModelExportOp', 'ModelUploadOp', 'ModelImportEvaluationOp', 'EndpointCreateOp', 'EndpointDeleteOp', 'TimeSeriesDatasetCreateOp', 'TimeSeriesDatasetExportDataOp', 'AutoMLForecastingTrainingJobRunOp', ] TimeSeriesDatasetCreateOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'dataset/create_time_series_dataset/component.yaml')) ImageDatasetCreateOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'dataset/create_image_dataset/component.yaml')) TabularDatasetCreateOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'dataset/create_tabular_dataset/component.yaml')) TextDatasetCreateOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'dataset/create_text_dataset/component.yaml')) VideoDatasetCreateOp = load_component_from_file(
# Copyright 2021 The Kubeflow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Google Cloud Pipeline Dataflow python components.""" import os try: from kfp.v2.components import load_component_from_file except ImportError: from kfp.components import load_component_from_file __all__ = [ 'DataflowPythonJobOp', ] DataflowPythonJobOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'python_job/component.yaml'))
'TabularDatasetCreateOp', 'TextDatasetCreateOp', 'VideoDatasetCreateOp', 'ImageDatasetExportDataOp', 'TabularDatasetExportDataOp', 'TextDatasetExportDataOp', 'VideoDatasetExportDataOp', 'ImageDatasetImportDataOp', 'TextDatasetImportDataOp', 'VideoDatasetImportDataOp', 'TimeSeriesDatasetCreateOp', 'TimeSeriesDatasetExportDataOp', ] TimeSeriesDatasetCreateOp = load_component_from_file( os.path.join( os.path.dirname(__file__), 'create_time_series_dataset/component.yaml')) ImageDatasetCreateOp = load_component_from_file( os.path.join( os.path.dirname(__file__), 'create_image_dataset/component.yaml')) TabularDatasetCreateOp = load_component_from_file( os.path.join( os.path.dirname(__file__), 'create_tabular_dataset/component.yaml')) TextDatasetCreateOp = load_component_from_file( os.path.join( os.path.dirname(__file__), 'create_text_dataset/component.yaml')) VideoDatasetCreateOp = load_component_from_file(
# Copyright 2021 The Kubeflow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Google Cloud Pipeline Wait GCP Resource Components.""" import os from typing import Optional try: from kfp.v2.components import load_component_from_file except ImportError: from kfp.components import load_component_from_file __all__ = [ 'WaitGcpResourcesOp', ] WaitGcpResourcesOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'component.yaml'))
# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import pathlib from kfp.v2 import components from kfp.v2 import compiler from kfp.v2 import dsl test_data_dir = pathlib.Path(__file__).parent / 'component_yaml' add_op = components.load_component_from_file( str(test_data_dir / 'add_component.yaml')) @dsl.pipeline(name='add-pipeline', pipeline_root='dummy_root') def my_pipeline( a: int = 2, b: int = 5, ): first_add_task = add_op(a, 3) second_add_task = add_op(first_add_task.outputs['sum'], b) third_add_task = add_op(second_add_task.outputs['sum'], 7) if __name__ == '__main__': compiler.Compiler().compile( pipeline_func=my_pipeline,
# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Module for supporting Google Vertex AI Hyperparameter Tuning Job Op.""" import os from .utils import serialize_parameters, serialize_metrics try: from kfp.v2.components import load_component_from_file except ImportError: from kfp.components import load_component_from_file __all__ = [ 'HyperparameterTuningJobRunOp', 'serialize_parameters', 'serialize_metrics', ] HyperparameterTuningJobRunOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'component.yaml'))
aiplatform_sdk.AutoMLTabularTrainingJob, aiplatform_sdk.AutoMLTabularTrainingJob.run, ) AutoMLForecastingTrainingJobRunOp = utils.convert_method_to_component( aiplatform_sdk.AutoMLForecastingTrainingJob, aiplatform_sdk.AutoMLForecastingTrainingJob.run, ) AutoMLVideoTrainingJobRunOp = utils.convert_method_to_component( aiplatform_sdk.AutoMLVideoTrainingJob, aiplatform_sdk.AutoMLVideoTrainingJob.run, ) ModelExportOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'model/export_model/component.yaml')) ModelDeployOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'endpoint/deploy_model/component.yaml')) ModelBatchPredictOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'batch_predict_job/component.yaml')) ModelUploadOp = load_component_from_file( os.path.join(os.path.dirname(__file__), 'model/upload_model/component.yaml')) EndpointCreateOp = load_component_from_file(