def test_load_component_from_url(self): component_path = Path( __file__).parent / 'test_data' / 'python_add.component.yaml' component_url = 'https://raw.githubusercontent.com/some/repo/components/component_group/python_add/component.yaml' component_bytes = component_path.read_bytes() component_dict = load_yaml(component_bytes) def mock_response_factory(url, params=None, **kwargs): if url == component_url: response = requests.Response() response.url = component_url response.status_code = 200 response._content = component_bytes return response raise RuntimeError('Unexpected URL "{}"'.format(url)) with mock.patch('requests.get', mock_response_factory): task_factory1 = comp.load_component_from_url(component_url) self.assertEqual( task_factory1.__doc__, component_dict['name'] + '\n' + component_dict['description']) arg1 = 3 arg2 = 5 task1 = task_factory1(arg1, arg2) self.assertEqual( task1.component_ref.spec.implementation.container.image, component_dict['implementation']['container']['image']) resolved_cmd = _resolve_command_line_and_paths(task1.component_ref.spec, task1.arguments) self.assertEqual(resolved_cmd.args[0], str(arg1)) self.assertEqual(resolved_cmd.args[1], str(arg2))
# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import json from kfp.deprecated.onprem import use_k8s_secret from kfp.deprecated import dsl, components from kfp.deprecated.components import OutputPath, create_component_from_func prepare_tensorboard = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/1b107eb4bb2510ecb99fd5f4fb438cbf7c96a87a/components/contrib/tensorflow/tensorboard/prepare_tensorboard/component.yaml' ) def train(minio_endpoint: str, log_bucket: str, log_dir: str): # Reference: https://www.tensorflow.org/tensorboard/get_started import tensorflow as tf mnist = tf.keras.datasets.mnist (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train, x_test = x_train / 255.0, x_test / 255.0 def create_model(): return tf.keras.models.Sequential([ tf.keras.layers.Flatten(input_shape=(28, 28)),
# you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from kfp.deprecated import dsl, components, compiler from kfp.deprecated.components import InputPath, load_component_from_url gcs_download_op = load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/961b17fa6844e1d79e5d3686bb557d830d7b5a95/components/google-cloud/storage/download_blob/component.yaml' ) @components.create_component_from_func def print_file(file_path: InputPath('Any')): """Print a file.""" with open(file_path) as f: print(f.read()) @components.create_component_from_func def echo_msg(msg: str): """Echo a message by parameter.""" print(msg)
# This sample demonstrates a common training scenario. # New models are being trained strarting from the production model (if it exists). # This sample produces two runs: # 1. The trainer will train the model from scratch and set as prod after testing it # 2. Exact same configuration, but the pipeline will discover the existing prod model (published by the 1st run) and warm-start the training from it. # GCS URI of a directory where the models and the model pointers should be be stored. model_dir_uri = 'gs://<bucket>/<path>' kfp_endpoint = None import kfp.deprecated as kfp from kfp.deprecated import components chicago_taxi_dataset_op = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/e3337b8bdcd63636934954e592d4b32c95b49129/components/datasets/Chicago%20Taxi/component.yaml' ) xgboost_train_on_csv_op = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/567c04c51ff00a1ee525b3458425b17adbe3df61/components/XGBoost/Train/component.yaml' ) xgboost_predict_on_csv_op = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/567c04c51ff00a1ee525b3458425b17adbe3df61/components/XGBoost/Predict/component.yaml' ) pandas_transform_csv_op = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/6162d55998b176b50267d351241100bb0ee715bc/components/pandas/Transform_DataFrame/in_CSV_format/component.yaml' ) drop_header_op = kfp.components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/02c9638287468c849632cf9f7885b51de4c66f86/components/tables/Remove_header/component.yaml' ) calculate_regression_metrics_from_csv_op = kfp.components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/616542ac0f789914f4eb53438da713dd3004fba4/components/ml_metrics/Calculate_regression_metrics/from_CSV/component.yaml'
import kfp.deprecated as kfp from kfp.deprecated import components chicago_taxi_dataset_op = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/e3337b8bdcd63636934954e592d4b32c95b49129/components/datasets/Chicago%20Taxi/component.yaml' ) convert_csv_to_apache_parquet_op = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/0d7d6f41c92bdc05c2825232afe2b47e5cb6c4b3/components/_converters/ApacheParquet/from_CSV/component.yaml' ) xgboost_train_on_csv_op = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/567c04c51ff00a1ee525b3458425b17adbe3df61/components/XGBoost/Train/component.yaml' ) xgboost_predict_on_csv_op = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/31939086d66d633732f75300ce69eb60e9fb0269/components/XGBoost/Predict/component.yaml' ) xgboost_train_on_parquet_op = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/0ae2f30ff24beeef1c64cc7c434f1f652c065192/components/XGBoost/Train/from_ApacheParquet/component.yaml' ) xgboost_predict_on_parquet_op = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/31939086d66d633732f75300ce69eb60e9fb0269/components/XGBoost/Predict/from_ApacheParquet/component.yaml' ) @kfp.dsl.pipeline(name='xgboost') def xgboost_pipeline(): # Based on experimentation, many steps need 1Gi memory. training_data_csv = chicago_taxi_dataset_op( where='trip_start_timestamp >= "2019-01-01" AND trip_start_timestamp < "2019-02-01"', select='tips,trip_seconds,trip_miles,pickup_community_area,dropoff_community_area,fare,tolls,extras,trip_total', limit=10000,
# you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from kfp.deprecated import dsl, components from kfp.deprecated.components import create_component_from_func prepare_tensorboard = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/1.5.0/components/tensorflow/tensorboard/prepare_tensorboard/component.yaml' ) def train(log_dir: 'URI'): # Reference: https://www.tensorflow.org/tensorboard/get_started import tensorflow as tf mnist = tf.keras.datasets.mnist (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train, x_test = x_train / 255.0, x_test / 255.0 def create_model(): return tf.keras.models.Sequential([ tf.keras.layers.Flatten(input_shape=(28, 28)),
# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json from kfp.deprecated import dsl, compiler, components import os import subprocess diagnose_me_op = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/566dddfdfc0a6a725b6e50ea85e73d8d5578bbb9/components/diagnostics/diagnose_me/component.yaml' ) confusion_matrix_op = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/1.8.0-alpha.0/components/local/confusion_matrix/component.yaml' ) roc_op = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/1.8.0-alpha.0/components/local/roc/component.yaml' ) dataproc_create_cluster_op = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/1.7.0-rc.3/components/gcp/dataproc/create_cluster/component.yaml' ) dataproc_delete_cluster_op = components.load_component_from_url(
def test_load_component_from_url_fail_on_none_arg(self): with self.assertRaises(TypeError): comp.load_component_from_url(None)