def test_load_component_from_url(self):
component_path = Path(
__file__).parent / 'test_data' / 'python_add.component.yaml'
component_url = 'https://raw.githubusercontent.com/some/repo/components/component_group/python_add/component.yaml'
component_bytes = component_path.read_bytes()
component_dict = load_yaml(component_bytes)
def mock_response_factory(url, params=None, **kwargs):
if url == component_url:
response = requests.Response()
response.url = component_url
response.status_code = 200
response._content = component_bytes
return response
raise RuntimeError('Unexpected URL "{}"'.format(url))
with mock.patch('requests.get', mock_response_factory):
task_factory1 = comp.load_component_from_url(component_url)
self.assertEqual(
task_factory1.__doc__,
component_dict['name'] + '\n' + component_dict['description'])
arg1 = 3
arg2 = 5
task1 = task_factory1(arg1, arg2)
self.assertEqual(
task1.component_ref.spec.implementation.container.image,
component_dict['implementation']['container']['image'])
resolved_cmd = _resolve_command_line_and_paths(task1.component_ref.spec,
task1.arguments)
self.assertEqual(resolved_cmd.args[0], str(arg1))
self.assertEqual(resolved_cmd.args[1], str(arg2))
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import json
from kfp.deprecated.onprem import use_k8s_secret
from kfp.deprecated import dsl, components
from kfp.deprecated.components import OutputPath, create_component_from_func
prepare_tensorboard = components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/1b107eb4bb2510ecb99fd5f4fb438cbf7c96a87a/components/contrib/tensorflow/tensorboard/prepare_tensorboard/component.yaml'
)
def train(minio_endpoint: str, log_bucket: str, log_dir: str):
# Reference: https://www.tensorflow.org/tensorboard/get_started
import tensorflow as tf
mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
def create_model():
return tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)),
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from kfp.deprecated import dsl, components, compiler
from kfp.deprecated.components import InputPath, load_component_from_url
gcs_download_op = load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/961b17fa6844e1d79e5d3686bb557d830d7b5a95/components/google-cloud/storage/download_blob/component.yaml'
)
@components.create_component_from_func
def print_file(file_path: InputPath('Any')):
"""Print a file."""
with open(file_path) as f:
print(f.read())
@components.create_component_from_func
def echo_msg(msg: str):
"""Echo a message by parameter."""
print(msg)
# This sample demonstrates a common training scenario.
# New models are being trained strarting from the production model (if it exists).
# This sample produces two runs:
# 1. The trainer will train the model from scratch and set as prod after testing it
# 2. Exact same configuration, but the pipeline will discover the existing prod model (published by the 1st run) and warm-start the training from it.
# GCS URI of a directory where the models and the model pointers should be be stored.
model_dir_uri = 'gs://<bucket>/<path>'
kfp_endpoint = None
import kfp.deprecated as kfp
from kfp.deprecated import components
chicago_taxi_dataset_op = components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/e3337b8bdcd63636934954e592d4b32c95b49129/components/datasets/Chicago%20Taxi/component.yaml'
)
xgboost_train_on_csv_op = components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/567c04c51ff00a1ee525b3458425b17adbe3df61/components/XGBoost/Train/component.yaml'
)
xgboost_predict_on_csv_op = components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/567c04c51ff00a1ee525b3458425b17adbe3df61/components/XGBoost/Predict/component.yaml'
)
pandas_transform_csv_op = components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/6162d55998b176b50267d351241100bb0ee715bc/components/pandas/Transform_DataFrame/in_CSV_format/component.yaml'
)
drop_header_op = kfp.components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/02c9638287468c849632cf9f7885b51de4c66f86/components/tables/Remove_header/component.yaml'
)
calculate_regression_metrics_from_csv_op = kfp.components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/616542ac0f789914f4eb53438da713dd3004fba4/components/ml_metrics/Calculate_regression_metrics/from_CSV/component.yaml'
import kfp.deprecated as kfp
from kfp.deprecated import components
chicago_taxi_dataset_op = components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/e3337b8bdcd63636934954e592d4b32c95b49129/components/datasets/Chicago%20Taxi/component.yaml'
)
convert_csv_to_apache_parquet_op = components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/0d7d6f41c92bdc05c2825232afe2b47e5cb6c4b3/components/_converters/ApacheParquet/from_CSV/component.yaml'
)
xgboost_train_on_csv_op = components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/567c04c51ff00a1ee525b3458425b17adbe3df61/components/XGBoost/Train/component.yaml'
)
xgboost_predict_on_csv_op = components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/31939086d66d633732f75300ce69eb60e9fb0269/components/XGBoost/Predict/component.yaml'
)
xgboost_train_on_parquet_op = components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/0ae2f30ff24beeef1c64cc7c434f1f652c065192/components/XGBoost/Train/from_ApacheParquet/component.yaml'
)
xgboost_predict_on_parquet_op = components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/31939086d66d633732f75300ce69eb60e9fb0269/components/XGBoost/Predict/from_ApacheParquet/component.yaml'
)
@kfp.dsl.pipeline(name='xgboost')
def xgboost_pipeline():
# Based on experimentation, many steps need 1Gi memory.
training_data_csv = chicago_taxi_dataset_op(
where='trip_start_timestamp >= "2019-01-01" AND trip_start_timestamp < "2019-02-01"',
select='tips,trip_seconds,trip_miles,pickup_community_area,dropoff_community_area,fare,tolls,extras,trip_total',
limit=10000,
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from kfp.deprecated import dsl, components
from kfp.deprecated.components import create_component_from_func
prepare_tensorboard = components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/1.5.0/components/tensorflow/tensorboard/prepare_tensorboard/component.yaml'
)
def train(log_dir: 'URI'):
# Reference: https://www.tensorflow.org/tensorboard/get_started
import tensorflow as tf
mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
def create_model():
return tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)),
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from kfp.deprecated import dsl, compiler, components
import os
import subprocess
diagnose_me_op = components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/566dddfdfc0a6a725b6e50ea85e73d8d5578bbb9/components/diagnostics/diagnose_me/component.yaml'
)
confusion_matrix_op = components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/1.8.0-alpha.0/components/local/confusion_matrix/component.yaml'
)
roc_op = components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/1.8.0-alpha.0/components/local/roc/component.yaml'
)
dataproc_create_cluster_op = components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/1.7.0-rc.3/components/gcp/dataproc/create_cluster/component.yaml'
)
dataproc_delete_cluster_op = components.load_component_from_url(
def test_load_component_from_url_fail_on_none_arg(self):
with self.assertRaises(TypeError):
comp.load_component_from_url(None)