def test_end_to_end_python_component_pipeline_compilation(self):
import kfp.components as comp
from kfp.components import python_op
#Defining the Python function
def add(a: float, b: float) -> float:
'''Returns sum of two arguments'''
return a + b
with tempfile.TemporaryDirectory() as temp_dir_name:
add_component_file = str(
Path(temp_dir_name).joinpath('add.component.yaml'))
subtract_component_file = str(
Path(temp_dir_name).joinpath('subtract.component.yaml'))
#Converting the function to a component. Instantiate it to create a pipeline task (ContaineOp instance)
add_op = comp.func_to_container_op(
add,
base_image='python:3.5',
output_component_file=add_component_file)
#Checking that the component artifact is usable:
add_op2 = comp.load_component_from_file(add_component_file)
#Using decorator to perform the same thing (convert function to a component):
@python_op(base_image='python:3.5',
output_component_file=subtract_component_file)
def subtract_op(a: float, b: float) -> float:
'''Returns difference between two arguments'''
return a - b
#Checking that the component artifact is usable:
subtract_op2 = comp.load_component_from_file(
subtract_component_file)
#Building the pipeline
import kfp.dsl as dsl
@dsl.pipeline(
name='Calculation pipeline',
description='A pipeline that performs arithmetic calculations.'
)
def calc_pipeline(
a1=dsl.PipelineParam('a1'),
a2=dsl.PipelineParam('a2', value='7'),
a3=dsl.PipelineParam('a3', value='17'),
):
task_11 = add_op(a1, a2)
task_12 = subtract_op(task_11.output, a3)
task_21 = add_op2(a1, a2)
task_22 = subtract_op2(task_21.output, a3)
task_3 = subtract_op(task_12.output, task_22.output)
#Compiling the pipleine:
pipeline_filename = str(
Path(temp_dir_name).joinpath(calc_pipeline.__name__ +
'.pipeline.tar.gz'))
import kfp.compiler as compiler
compiler.Compiler().compile(calc_pipeline, pipeline_filename)
def pipeline(gcs_bucket_name='<bucket where data and model will be exported>'):
bq2gcs_op = comp.load_component_from_file(BQ2GCS_YAML)
bq2gcs = bq2gcs_op(input_bucket=gcs_bucket_name, )
trainjob_op = comp.load_component_from_file(TRAINJOB_YAML)
trainjob = trainjob_op(input_bucket=gcs_bucket_name, )
deploymodel_op = comp.load_component_from_file(DEPLOYMODEL_YAML)
deploymodel = deploymodel_op(input_bucket=gcs_bucket_name, )
trainjob.after(bq2gcs)
deploymodel.after(trainjob)
def mnist_pipeline(
storage_bucket: str,
output_path: str,
):
import os
train_op = components.load_component_from_file('./train/component.yaml')
train_step = train_op(storage_bucket=storage_bucket).apply(
use_gcp_secret('user-gcp-sa'))
visualize_op = components.load_component_from_file(
'./tensorboard/component.yaml')
visualize_step = visualize_op(logdir='%s' % train_step.outputs['logdir'],
output_path=output_path).apply(
use_gcp_secret('user-gcp-sa'))
def test_end_to_end_python_component_pipeline_compilation(self):
import kfp.components as comp
#Defining the Python function
def add(a: float, b: float) -> float:
'''Returns sum of two arguments'''
return a + b
with tempfile.TemporaryDirectory() as temp_dir_name:
add_component_file = str(Path(temp_dir_name).joinpath('add.component.yaml'))
#Converting the function to a component. Instantiate it to create a pipeline task (ContaineOp instance)
add_op = comp.func_to_container_op(add, base_image='python:3.5', output_component_file=add_component_file)
#Checking that the component artifact is usable:
add_op2 = comp.load_component_from_file(add_component_file)
#Building the pipeline
@kfp.dsl.pipeline(
name='Calculation pipeline',
description='A pipeline that performs arithmetic calculations.'
)
def calc_pipeline(
a1,
a2='7',
a3='17',
):
task_1 = add_op(a1, a2)
task_2 = add_op2(a1, a2)
task_3 = add_op(task_1.output, task_2.output)
task_4 = add_op2(task_3.output, a3)
#Compiling the pipleine:
pipeline_filename = str(Path(temp_dir_name).joinpath(calc_pipeline.__name__ + '.pipeline.tar.gz'))
kfp.compiler.Compiler().compile(calc_pipeline, pipeline_filename)
def mnist_train(name: str = "mnist",
namespace: str = "kubeflow",
workerNum: int = 3,
ttlSecondsAfterFinished: int = -1,
tfjobTimeoutMinutes: int = 60,
deleteAfterDone=False):
tfjob_launcher_op = components.load_component_from_file("./component.yaml")
# tfjob_launcher_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/master/components/kubeflow/launcher/component.yaml')
chief = {
"replicas": 1,
"restartPolicy": "OnFailure",
"template": {
"spec": {
"containers": [{
"command": ["python", "/opt/model.py"],
"args": ["--tf-train-steps=60"],
"image": "liuhougangxa/tf-estimator-mnist",
"name": "tensorflow",
}]
}
}
}
worker_spec_create = worker_spec_op(workerNum)
tfjob_launcher_op(name=name,
namespace=namespace,
ttl_seconds_after_finished=ttlSecondsAfterFinished,
worker_spec=worker_spec_create.outputs['worker_spec'],
chief_spec=chief,
tfjob_timeout_minutes=tfjobTimeoutMinutes,
delete_finished_tfjob=deleteAfterDone)
def test_end_to_end_python_component_pipeline(self):
import kfp.components as comp
#Defining the Python function
def add(a: float, b: float) -> float:
'''Returns sum of two arguments'''
return a + b
with tempfile.TemporaryDirectory() as temp_dir_name:
add_component_file = str(
Path(temp_dir_name).joinpath('add.component.yaml'))
#Converting the function to a component. Instantiate it to create a pipeline task (ContaineOp instance)
add_op = comp.func_to_container_op(
add,
base_image='python:3.5',
output_component_file=add_component_file)
#Checking that the component artifact is usable:
add_op2 = comp.load_component_from_file(add_component_file)
#Building the pipeline
def calc_pipeline(
a1,
a2='7',
a3='17',
):
task_1 = add_op(a1, a2)
task_2 = add_op2(a1, a2)
task_3 = add_op(task_1.outputs['Output'],
task_2.outputs['Output'])
task_4 = add_op2(task_3.outputs['Output'], a3)
#Instantiating the pipleine:
calc_pipeline(42)
def test_func_to_component_file(self):
func = add_two_numbers
with tempfile.TemporaryDirectory() as temp_dir_name:
component_path = str(Path(temp_dir_name) / 'component.yaml')
comp._python_op.func_to_component_file(func, output_component_file=component_path)
op = comp.load_component_from_file(component_path)
self.helper_test_2_in_1_out_component_using_local_call(func, op)
def load_launcher_from_file(src_file, name_placeholder, name):
""" Load launcher's commponent yaml file and update launcher name """
dest_file = shutil.copyfile(src_file, gen_random_string(20) + ".tmp.yaml")
with fileinput.FileInput(dest_file, inplace=True) as f:
for line in f:
line = line.replace(name_placeholder, name)
print(line, end='')
from kfp import components
component = components.load_component_from_file(dest_file)
os.remove(dest_file)
return component
def _test_load_component_from_file(self, component_path: str):
task_factory1 = comp.load_component_from_file(component_path)
arg1 = 3
arg2 = 5
task1 = task_factory1(arg1, arg2)
self.assertEqual(task1.human_name, 'Add')
self.assertEqual(task_factory1.__doc__.strip(), 'Add\nReturns sum of two arguments')
self.assertEqual(task1.container.image, 'python:3.5')
self.assertEqual(task1.container.args[0], str(arg1))
self.assertEqual(task1.container.args[1], str(arg2))
def test_handle_creating_graph_component_from_pipeline_that_uses_container_components(self):
test_data_dir = Path(__file__).parent / 'test_data'
producer_op = comp.load_component_from_file(str(test_data_dir / 'component_with_0_inputs_and_2_outputs.component.yaml'))
processor_op = comp.load_component_from_file(str(test_data_dir / 'component_with_2_inputs_and_2_outputs.component.yaml'))
consumer_op = comp.load_component_from_file(str(test_data_dir / 'component_with_2_inputs_and_0_outputs.component.yaml'))
def pipeline1(pipeline_param_1: int):
producer_task = producer_op()
processor_task = processor_op(pipeline_param_1, producer_task.outputs['Output 2'])
consumer_task = consumer_op(processor_task.outputs['Output 1'], processor_task.outputs['Output 2'])
return OrderedDict([ # You can safely return normal dict in python 3.6+
('Pipeline output 1', producer_task.outputs['Output 1']),
('Pipeline output 2', processor_task.outputs['Output 2']),
])
graph_component = create_graph_component_spec_from_pipeline_func(pipeline1)
self.assertEqual(len(graph_component.inputs), 1)
self.assertListEqual([input.name for input in graph_component.inputs], ['pipeline_param_1']) #Relies on human name conversion function stability
self.assertListEqual([output.name for output in graph_component.outputs], ['Pipeline output 1', 'Pipeline output 2'])
self.assertEqual(len(graph_component.implementation.graph.tasks), 3)
def videopose_pull(
image,
update_data,
data,
):
train_env = {}
train_num_gpus = 1
train_op = components.load_component_from_file(
'components/videopose_pull.yaml')(image=image,
update_data=update_data,
data=data)
get_container(train_op, train_env, train_num_gpus)
def train_eval_epic(owner,
project,
experiment,
model,
git_rev,
pretrained_s3,
mode,
train_additional_args='',
eval_additional_args=''):
train_env = {}
train_num_gpus = 1
train_op = components.load_component_from_file('components/train.yaml')(
owner=owner,
project=project,
experiment=experiment,
model=model,
git_rev=git_rev,
pretrained_s3=pretrained_s3,
mode=mode,
additional_args=train_additional_args)
(train_op.container.set_memory_request('56Gi').set_memory_limit(
'56Gi').set_cpu_request('7.5').set_cpu_limit('7.5').set_gpu_limit(
str(train_num_gpus)).add_volume_mount(
V1VolumeMount(
name='tensorboard',
mount_path='/shared/tensorboard')).add_volume_mount(
V1VolumeMount(name='data',
mount_path='/data/')).add_volume_mount(
V1VolumeMount(
name='shm',
mount_path='/dev/shm')))
(add_env(add_ssh_volume(train_op), train_env).add_toleration(
V1Toleration(key='nvidia.com/gpu',
operator='Exists',
effect='NoSchedule')).add_node_selector_constraint(
'beta.kubernetes.io/instance-type',
f'p3.{2*train_num_gpus}xlarge').
add_volume(
V1Volume(name='tensorboard',
persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(
'tensorboard-research-kf'))
).add_volume(
V1Volume(name='data',
persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(
'dataset-epic-kitchen')))
# .add_volume(V1Volume(name='shm', host_path=V1HostPathVolumeSource(path='/dev/shm')))
.add_volume(
V1Volume(name='shm',
empty_dir=V1EmptyDirVolumeSource(medium='Memory'))))
def kaggle_houseprice(
bucket_name: str,
commit_sha: str
):
downloadDataOp = components.load_component_from_file('./download_dataset/component.yaml')
downloadDataStep = downloadDataOp(bucket_name=bucket_name).apply(use_gcp_secret('user-gcp-sa'))
visualizeTableOp = components.load_component_from_file('./visualize_table/component.yaml')
visualizeTableStep = visualizeTableOp(train_file_path='%s' % downloadDataStep.outputs['train_dataset']).apply(use_gcp_secret('user-gcp-sa'))
visualizeHTMLOp = components.load_component_from_file('./visualize_html/component.yaml')
visualizeHTMLStep = visualizeHTMLOp(train_file_path='%s' % downloadDataStep.outputs['train_dataset'],
commit_sha=commit_sha,
bucket_name=bucket_name).apply(use_gcp_secret('user-gcp-sa'))
trainModelOp = components.load_component_from_file('./train_model/component.yaml')
trainModelStep = trainModelOp(train_file='%s' % downloadDataStep.outputs['train_dataset'],
test_file='%s' % downloadDataStep.outputs['test_dataset'],
bucket_name=bucket_name).apply(use_gcp_secret('user-gcp-sa'))
submitResultOp = components.load_component_from_file('./submit_result/component.yaml')
submitResultStep = submitResultOp(result_file='%s' % trainModelStep.outputs['result'],
submit_message='submit').apply(use_gcp_secret('user-gcp-sa'))
def test_load_component_from_file(self):
component_path_obj = _test_data_dir.joinpath(
'python_add.component.yaml')
component_text = component_path_obj.read_text()
component_dict = load_yaml(component_text)
task_factory1 = comp.load_component_from_file(str(component_path_obj))
assert task_factory1.__doc__ == component_dict['description']
arg1 = 3
arg2 = 5
task1 = task_factory1(arg1, arg2)
assert task1.human_name == component_dict['name']
assert task1.image == component_dict['implementation'][
'dockerContainer']['image']
assert task1.arguments[0] == str(arg1)
assert task1.arguments[1] == str(arg2)
def _test_load_component_from_file(self, component_path: str):
task_factory1 = comp.load_component_from_file(component_path)
arg1 = 3
arg2 = 5
task1 = task_factory1(arg1, arg2)
self.assertEqual(task_factory1.__name__, 'Add')
self.assertEqual(task_factory1.__doc__.strip(),
'Add\nReturns sum of two arguments')
self.assertEqual(
task1.component_ref.spec.implementation.container.image,
'python:3.5')
resolved_cmd = _resolve_command_line_and_paths(
task1.component_ref.spec, task1.arguments)
self.assertEqual(resolved_cmd.args[0], str(arg1))
self.assertEqual(resolved_cmd.args[1], str(arg2))
def videopose_gen2(
image,
git_rev,
resnet_model,
mode,
name,
additional_args,
):
train_env = {}
train_num_gpus = 1
train_op = components.load_component_from_file(
'components/videopose_gen.yaml')(image=image,
git_rev=git_rev,
resnet_model=resnet_model,
mode=mode,
name=name,
additional_args=additional_args)
get_container_no_data(train_op, train_env, train_num_gpus)
def train_headset_overlay(
image,
git_rev,
update_data,
config,
name,
model_file,
additional_args,
):
train_env = {}
train_num_gpus = 4
train_op = components.load_component_from_file('components/train.yaml')(
image=image,
git_rev=git_rev,
update_data=update_data,
config=config,
name=name,
model_file=model_file,
additional_args=additional_args)
get_container(train_op, train_env, train_num_gpus)
def main(args):
OUT_COMPONENTS_DIR = args.output_component_dir
OUT_PIPELINE_DIR = args.output_pipeline_dir
# Write the component file of Python function
hello_component = cpt.create_component_from_func(
func=hello_kubeflow,
output_component_file=f'{OUT_COMPONENTS_DIR}/hello_kubeflow.component')
# Read the component file
hello_component = cpt.load_component_from_file(
filename=f'{OUT_COMPONENTS_DIR}/hello_kubeflow.component')
# Write a pipeline function using the Kubeflow Pipelines DSL
@dsl.pipeline(name='Hello Kubeflow Pipeline',
description='A Hello Kubeflow pipeline')
def hello_kubeflow_pipeline(name='Ivan'):
task = hello_component(name)
# Compile the pipeline to generate a compressed YAML definition of the pipeline
cmp.Compiler().compile(
pipeline_func=hello_kubeflow_pipeline,
package_path=f'{OUT_PIPELINE_DIR}/hello_kubeflow_pipeline.zip')
import kfp
from kfp import components
from kfp import dsl
from kfp.aws import use_aws_secret
sagemaker_model_op = components.load_component_from_file(
"../../model/component.yaml")
sagemaker_batch_transform_op = components.load_component_from_file(
"../../batch_transform/component.yaml")
@dsl.pipeline(
name="Batch Transform Job in SageMaker",
description="SageMaker batch transform component test",
)
def batch_transform_pipeline(
region="",
image="",
model_name="",
job_name="",
model_artifact_url="",
instance_type="",
instance_count="",
data_input="",
data_type="",
content_type="",
compression_type="",
output_location="",
max_concurrent="",
max_payload="",
batch_strategy="",
platform = 'onprem' #can be in ['GCP', 'onprem']
storage = 'minio' #all data on minio object storage, and use s3 API to access.
GITPAHT = 'https://github.com/kubeflow/pipelines.git'
GITDIR = '/tmp/pipelines'
MINIOPATH = '/mlpipeline/tfx/taxi-cab-classification/'
DATAPATH = '/tmp/pipelines/samples/tfx/taxi-cab-classification/'
AWS_ACCESS_KEY_ID='minio'
AWS_SECRET_ACCESS_KEY='minio123'
AWS_REGION='us-west-1'
S3_ENDPOINT='minio-service.kubeflow:9000'
S3_USE_HTTPS='0'
S3_VERIFY_SSL='0'
dataflow_tf_data_validation_op = components.load_component_from_file('components/dataflow/tfdv/component.yaml')
dataflow_tf_transform_op = components.load_component_from_file('components/dataflow/tft/component.yaml')
tf_train_op = components.load_component_from_file('components/kubeflow/dnntrainer/component.yaml')
dataflow_tf_model_analyze_op = components.load_component_from_file('components/dataflow/tfma/component.yaml')
dataflow_tf_predict_op = components.load_component_from_file('components/dataflow/predict/component.yaml')
confusion_matrix_op = components.load_component_from_file('components/local/confusion_matrix/component.yaml')
roc_op = components.load_component_from_file('components/local/roc/component.yaml')
kubeflow_deploy_op = components.load_component_from_file('components/kubeflow/deployer/component.yaml')
@dsl.pipeline(
name='TFX Taxi Cab Classification Pipeline Example',
description='Example pipeline that does classification with model analysis based on a public BigQuery dataset.'
)
def taxi_cab_classification(
#!/usr/bin/env python3
import kfp
import json
import copy
from kfp import components
from kfp import dsl
from kfp.aws import use_aws_secret
sagemaker_hpo_op = components.load_component_from_file(
'../../../../components/aws/sagemaker/hyperparameter_tuning/component.yaml'
)
sagemaker_train_op = components.load_component_from_file(
'../../../../components/aws/sagemaker/train/component.yaml')
sagemaker_model_op = components.load_component_from_file(
'../../../../components/aws/sagemaker/model/component.yaml')
sagemaker_deploy_op = components.load_component_from_file(
'../../../../components/aws/sagemaker/deploy/component.yaml')
sagemaker_batch_transform_op = components.load_component_from_file(
'../../../../components/aws/sagemaker/batch_transform/component.yaml')
hpoChannels = []
trainChannels = []
channelObj = {
'ChannelName': '',
'DataSource': {
'S3DataSource': {
'S3Uri': '',
'S3DataType': 'S3Prefix',
'S3DataDistributionType': 'FullyReplicated'
def test_load_component_from_file_fail_on_none_arg(self):
with self.assertRaises(TypeError):
comp.load_component_from_file(None)
from pathlib import Path
import kfp
from kfp.components import load_component_from_file, create_component_from_func
from typing import NamedTuple
test_data_dir = Path(__file__).parent / 'test_data'
producer_op = load_component_from_file(
str(test_data_dir / 'produce_2.component.yaml'))
processor_op = load_component_from_file(
str(test_data_dir / 'process_2_2.component.yaml'))
consumer_op = load_component_from_file(
str(test_data_dir / 'consume_2.component.yaml'))
def metadata_and_metrics() -> NamedTuple(
"Outputs",
[("mlpipeline_ui_metadata", "UI_metadata"),
("mlpipeline_metrics", "Metrics")],
):
metadata = {
"outputs": [{
"storage": "inline",
"source": "*this should be bold*",
"type": "markdown"
}]
}
metrics = {
"metrics": [
{
"name": "train-accuracy",
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pathlib
from kfp import components
from kfp import compiler
from kfp import dsl
test_data_dir = pathlib.Path(__file__).parent / 'component_yaml'
add_op = components.load_component_from_file(
str(test_data_dir / 'add_component.yaml'))
@dsl.pipeline(name='add-pipeline', pipeline_root='dummy_root')
def my_pipeline(
a: int = 2,
b: int = 5,
):
first_add_task = add_op(op_1=a, op2=3)
second_add_task = add_op(op_1=first_add_task.outputs['sum'], op2=b)
third_add_task = add_op(op_1=second_add_task.outputs['sum'], op2=7)
if __name__ == '__main__':
compiler.Compiler().compile(
pipeline_func=my_pipeline,
#!/usr/bin/env python3
import kfp
import json
import copy
from kfp import components
from kfp import dsl
from kfp.aws import use_aws_secret
sagemaker_workteam_op = components.load_component_from_file(
'../../../../components/aws/sagemaker/workteam/component.yaml')
sagemaker_gt_op = components.load_component_from_file(
'../../../../components/aws/sagemaker/ground_truth/component.yaml')
sagemaker_train_op = components.load_component_from_file(
'../../../../components/aws/sagemaker/train/component.yaml')
channelObjList = []
channelObj = {
'ChannelName': '',
'DataSource': {
'S3DataSource': {
'S3Uri': '',
'S3DataType': 'AugmentedManifestFile',
'S3DataDistributionType': 'FullyReplicated',
'AttributeNames': ['source-ref', 'category']
}
},
'ContentType': 'application/x-recordio',
'CompressionType': 'None',
'RecordWrapperType': 'RecordIO'
aiplatform_sdk.AutoMLTabularTrainingJob,
aiplatform_sdk.AutoMLTabularTrainingJob.run,
)
AutoMLForecastingTrainingJobRunOp = utils.convert_method_to_component(
aiplatform_sdk.AutoMLForecastingTrainingJob,
aiplatform_sdk.AutoMLForecastingTrainingJob.run,
)
AutoMLVideoTrainingJobRunOp = utils.convert_method_to_component(
aiplatform_sdk.AutoMLVideoTrainingJob,
aiplatform_sdk.AutoMLVideoTrainingJob.run,
)
ModelExportOp = components.load_component_from_file(
os.path.join(os.path.dirname(__file__),
'model/export_model/component.yaml'))
ModelDeployOp = components.load_component_from_file(
os.path.join(os.path.dirname(__file__),
'endpoint/deploy_model/component.yaml'))
ModelBatchPredictOp = components.load_component_from_file(
os.path.join(os.path.dirname(__file__),
'batch_predict_job/component.yaml'))
ModelUploadOp = components.load_component_from_file(
os.path.join(os.path.dirname(__file__),
'model/upload_model/component.yaml'))
EndpointCreateOp = components.load_component_from_file(
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pathlib
from kfp import components
from kfp.v2 import dsl
import kfp.v2.compiler as compiler
test_data_dir = pathlib.Path(__file__).parent / 'component_yaml'
trainer_op = components.load_component_from_file(
str(test_data_dir / 'trainer_component.yaml'))
serving_op = components.load_component_from_file(
str(test_data_dir / 'serving_component.yaml'))
@dsl.pipeline(name='two-step-pipeline-with-importer',
pipeline_root='dummy_root',
description='A linear two-step pipeline.')
def my_pipeline(input_gcs='gs://test-bucket/pipeline_root',
optimizer: str = 'sgd',
epochs: int = 200):
trainer = trainer_op(input_location=input_gcs,
train_optimizer=optimizer,
num_epochs=epochs)
serving = serving_op(model=trainer.outputs['model_output'],
model_cfg=trainer.outputs['model_config'])
import kfp
import json
import os
import copy
from kfp import components
from kfp import dsl
from kfp.aws import use_aws_secret
from kfp.components import load_component_from_file, load_component_from_url
cur_file_dir = os.path.dirname(__file__)
components_dir = os.path.join(cur_file_dir, "../pytorch")
bert_data_prep_op = components.load_component_from_file(
components_dir + "/data_prep/component.yaml"
)
bert_train_op = components.load_component_from_file(
components_dir + "/train/component.yaml"
)
@dsl.pipeline(name="Training pipeline", description="Sample training job test")
def training(input_directory = "/pvc/input",
output_directory = "/pvc/output", handlerFile = "image_classifier"):
vop = dsl.VolumeOp(
name="volume_creation",
resource_name="pvcm",
modes=dsl.VOLUME_MODE_RWO,
size="1Gi"
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from kfp import dsl
from kfp import compiler
from kfp import components
from kubernetes import client as k8s_client
import ai_pipeline_params as params
import json
notebook_ops = components.load_component_from_file('notebook.yaml')
setup_ops = components.load_component_from_file('setup.yaml')
post_model_ops = components.load_component_from_file('postprocessing.yaml')
@dsl.pipeline(
name='icp4d-demo',
description=
'A pipeline for training using Jupyter notebook and Serve models with KFServing'
)
def icpdPipeline(
notebook_url='https://raw.githubusercontent.com/animeshsingh/notebooks/master/sklearn.ipynb',
notebook_params='',
api_token='',
endpoint_url='minio-service:9000',
bucket_name='mlpipeline',
import argparse
parser = argparse.ArgumentParser(description='Process inputs.')
parser.add_argument('--image_name',
type=str,
default='kubeflow_synapse_component')
parser.add_argument('--image_repo_name', type=str, default='kubeflowdemo')
args = parser.parse_args()
component_root = os.path.join(os.path.dirname(os.path.abspath(__file__)), ".")
image_repo_name = args.image_repo_name # the container registery for the container operation and path in the ACR
image_name = args.image_name
file_path = os.path.join(component_root, "component.yaml")
# Loading the component.yaml file for deployment operation
run_job_operation = components.load_component_from_file(file_path)
# The run_job_image_name shall be the container image for the operation
# It shall be something like <your_acr_name>.azurecr.io/deploy/aml-deploy-model:latest
# If you are using a container registry other than Azure Container Registry, update the image name correspondingly
run_job_image_name = image_repo_name + '.azurecr.io/deploy/' + image_name + ':latest'
print(run_job_image_name)
def use_image(image_name):
def _use_image(task):
task.image = image_name
return task
return _use_image
