def sequential_pipeline(model_data_url = "https://test-epi.s3.fr-par.scw.cloud/kc_house_data.csv?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=SCWTTHE3QW9915A46GMW%2F20200206%2Ffr-par%2Fs3%2Faws4_request&X-Amz-Date=20200206T200330Z&X-Amz-Expires=521789&X-Amz-Signature=56952a33bf8a12b255ed573a2a4e05dd901eec9985a98ed332f89c61ad55a2cd&X-Amz-SignedHeaders=host"):
model_data_dest = "/data/kc_house_data.csv"
vop = dsl.VolumeOp(
name="vol",
resource_name="newpvc",
size="1Gi",
modes=dsl.VOLUME_MODE_RWM
)
op_download = dsl.ContainerOp(
name='download',
image='toune/epi-saas-project2-download',
pvolumes={"/data": vop.volume}
)
op_download.container.add_env_variable(V1EnvVar(name='DOWNLOAD_FILE_URL', value=model_data_url))
op_download.container.add_env_variable(V1EnvVar(name='DOWNLOAD_FILE_DEST', value=model_data_dest))
op_predict = dsl.ContainerOp(
name='predict',
image='toune/epi-saas-project2-app',
pvolumes={"/data": vop.volume}
)
op_predict.container.add_env_variable(V1EnvVar(name='TRAINED_MODEL_PATH', value="/data/model-artifacts"))
op_predict.container.add_env_variable(V1EnvVar(name='DATA_PATH', value=model_data_dest))
op_predict.after(op_download)
def notebook_pipeline():
"""A pipeline to run a Jupyter notebook with elyra-ai/kfp-notebook and Papermill."""
from kfp_notebook.pipeline import NotebookOp
notebook_op = NotebookOp(name="${name}",
notebook="${notebook}",
cos_endpoint="${cos_endpoint}",
cos_bucket="${cos_bucket}",
cos_directory="${cos_directory}",
cos_dependencies_archive="${cos_dependencies_archive}",
requirements_url="${requirements_url}",
image="${image}")
from kubernetes.client.models import V1EnvVar
notebook_op.container.add_env_variable(V1EnvVar(name='AWS_ACCESS_KEY_ID', value="${cos_username}"))
notebook_op.container.add_env_variable(V1EnvVar(name='AWS_SECRET_ACCESS_KEY', value="${cos_password}"))
from kfp import onprem
notebook_op.container.add_env_variable(V1EnvVar(name='DATA_DIR', value="${mount_path}"))
notebook_op.apply(onprem.mount_pvc(pvc_name='${dataset_pvc}',
volume_name='${dataset_pvc}',
volume_mount_path='${mount_path}'))
def test_deprecation_warnings(self):
"""Test deprecation warnings."""
op = ContainerOp(name='op1', image='image')
with self.assertWarns(PendingDeprecationWarning):
op.env_variables = [V1EnvVar(name="foo", value="bar")]
with self.assertWarns(PendingDeprecationWarning):
op.image = 'image2'
with self.assertWarns(PendingDeprecationWarning):
op.set_memory_request('10M')
with self.assertWarns(PendingDeprecationWarning):
op.set_memory_limit('10M')
with self.assertWarns(PendingDeprecationWarning):
op.set_cpu_request('100m')
with self.assertWarns(PendingDeprecationWarning):
op.set_cpu_limit('1')
with self.assertWarns(PendingDeprecationWarning):
op.set_gpu_limit('1')
with self.assertWarns(PendingDeprecationWarning):
op.add_env_variable(V1EnvVar(name="foo", value="bar"))
with self.assertWarns(PendingDeprecationWarning):
op.add_volume_mount(
V1VolumeMount(mount_path='/secret/gcp-credentials',
name='gcp-credentials'))
def ml_pipeline():
pvc1 = dsl.VolumeOp(
name = "PVC-data",
resource_name = "pvc-ojt-ml",
storage_class = "ojt-tibero-ml",
modes = dsl.VOLUME_MODE_RWM,
size = "1Gi"
)
pvc2 = dsl.VolumeOp(
name = "PVC-was",
resource_name = "pvc-ojt-result",
storage_class = "ojt-tibero-result",
modes = dsl.VOLUME_MODE_RWM,
size = "1Gi"
)
pod1 = dsl.ContainerOp(
name = 'Tibero-JDBC agent',
image = 'hanjoo8821/jdbc-tibero:colsout-' + ver,
container_kwargs={'env':[V1EnvVar('id', 'hanjoo'), V1EnvVar('pw', '1010'), V1EnvVar('sql', 'SELECT * FROM AB1_2 LEFT JOIN ACCOUNTING ON AB1_2.EMP_NUM = ACCOUNTING.EMP_NUM'), V1EnvVar('col1', 'EMP_NUM'), V1EnvVar('col2', 'BIRTH'), V1EnvVar('col3', 'SALARY')]},
pvolumes = {"/Output": pvc1.volume}
)
pod2a = dsl.ContainerOp(
name = 'Print-EMP_NUM',
image = 'alpine:3.6',
command = ['cat', '/Output/EMP_NUM.txt'],
pvolumes = {"/Output": pod1.pvolume}
)
pod2b = dsl.ContainerOp(
name = 'Trans-BIRTH',
image = 'hanjoo8821/jdbc-tibero:trans-birth-' + ver,
container_kwargs={'env':[V1EnvVar('col2', 'BIRTH')]},
pvolumes = {"/Output": pod1.pvolume}
)
pod2c = dsl.ContainerOp(
name = 'Trans-SALARY',
image = 'hanjoo8821/jdbc-tibero:trans-salary-' + ver,
container_kwargs={'env':[V1EnvVar('col3', 'SALARY')]},
pvolumes = {"/Output": pod1.pvolume}
)
pod3 = dsl.ContainerOp(
name = 'ML-Linear Regression',
image = 'hanjoo8821/jdbc-tibero:ml-' + ver,
container_kwargs={'env':[V1EnvVar('col2', 'BIRTH'), V1EnvVar('col3', 'SALARY')]},
pvolumes = {"/Output": pod2b.pvolume}
).after(pod2b, pod2c)
pod4 = dsl.ContainerOp(
name = 'Visualization',
image = 'hanjoo8821/jdbc-tibero:graph-' + ver,
container_kwargs={'env':[V1EnvVar('col2', 'BIRTH'), V1EnvVar('col3', 'SALARY')]},
pvolumes = {"/Output": pod3.pvolume, "/WAS": pvc2.volume}
)
def test_no_errors(repo, tag, full):
test_dag = DAG(
"test-dag",
default_args={},
description="testing",
start_date=datetime(2020, 1, 1),
schedule_interval=None,
)
k = basic_kubernetes_pod_operator(
task_id="task1",
dag=test_dag,
role="a_role",
repo_name=repo,
release=tag,
full_image_name=full,
)
assert isinstance(k, KubernetesPodOperator)
assert k.task_id == "task1"
assert k.name == "task1"
assert k.image == f"{ecr_base_path}my-repo:v0.0.0"
assert k.namespace == "airflow"
assert k.annotations == {"iam.amazonaws.com/role": "a_role"}
assert k.is_delete_operator_pod is True
assert k.cluster_context == "aws"
assert k.config_file == "/usr/local/airflow/dags/.kube/config"
assert k.in_cluster is False
assert k.get_logs is True
assert k.security_context == {
"allowPrivilegeEscalation": False,
"runAsNonRoot": True,
"privileged": False,
}
expected_env = [
V1EnvVar(
name="AWS_METADATA_SERVICE_TIMEOUT",
value="60",
value_from=None,
),
V1EnvVar(
name="AWS_METADATA_SERVICE_NUM_ATTEMPTS",
value="5",
value_from=None,
),
V1EnvVar(
name="AWS_DEFAULT_REGION",
value="eu-west-1",
value_from=None,
),
]
assert k.env_vars == expected_env
def query_op_vol(n, vol):
return dsl.ContainerOp(
name=n,
image="hanjoo8821/jdbc-tibero:null",
container_kwargs={
'env': [V1EnvVar('id', 'hanjoo'),
V1EnvVar('pw', '1010')]
},
command=['sh', '-c'],
arguments=[
'java -cp "./JdbcAgentEx-null.jar:./tibero6jdbc-1.0.jar" local.hanjoo.JdbcAgent "$id" "$pw" | tee /data/output'
],
pvolumes={"/data": vol})
def pipelineparams_pipeline(tag: str = 'latest', sleep_ms: int = 10):
echo = dsl.Sidecar(
name='echo',
image='hashicorp/http-echo:%s' % tag,
args=['-text="hello world"'],
)
op1 = dsl.ContainerOp(
name='download',
image='busybox:%s' % tag,
command=['sh', '-c'],
arguments=[
'sleep %s; wget localhost:5678 -O /tmp/results.txt' % sleep_ms
],
sidecars=[echo],
file_outputs={'downloaded': '/tmp/results.txt'})
op2 = dsl.ContainerOp(name='echo',
image='library/bash',
command=['sh', '-c'],
arguments=['echo $MSG %s' % op1.output])
op2.container.add_env_variable(
V1EnvVar(name='MSG', value='pipelineParams: '))
def train_op(docker_image_path: str, pvolume: PipelineVolume,
dataset_root_dir: str, dataset_name: str, num_gpu: int,
gpu_list: str):
# num_gpus = len(gpu_list.value.split(','))
# gpu_list = ""
# for i in range(num_gpu):
# gpu_list += (str(i) + ",")
env_var = V1EnvVar(name="CUDA_VISIBLE_DEVICES", value="0,1")
return dsl.ContainerOp(
name="train",
image=docker_image_path,
command=["python", f"{PROJ_PATH}/devolopment/train.py"],
arguments=[
"--dataset_root_dir", dataset_root_dir, "--dataset_name",
dataset_name
],
file_outputs={
'output': f'{PROJ_PATH}/devolopment/output.txt'
},
container_kwargs={
'image_pull_policy': 'IfNotPresent'
},
pvolumes={
"/workspace": pvolume
}).container.set_gpu_limit(2).add_env_variable(env_var)
def test_basic(self):
"""Test basic usage."""
param1 = PipelineParam('param1')
param2 = PipelineParam('param2')
op1 = (ContainerOp(
name='op1',
image='image',
arguments=['%s hello %s %s' % (param1, param2, param1)],
sidecars=[Sidecar(name='sidecar0', image='image0')],
container_kwargs={
'env': [V1EnvVar(name='env1', value='value1')]
},
file_outputs={
'out1': '/tmp/b'
}).add_sidecar(Sidecar(name='sidecar1',
image='image1')).add_sidecar(
Sidecar(name='sidecar2',
image='image2')))
self.assertCountEqual([x.name for x in op1.inputs],
['param1', 'param2'])
self.assertCountEqual(list(op1.outputs.keys()), ['out1'])
self.assertCountEqual([x.op_name for x in op1.outputs.values()],
[op1.name])
self.assertEqual(op1.output.name, 'out1')
self.assertCountEqual([sidecar.name for sidecar in op1.sidecars],
['sidecar0', 'sidecar1', 'sidecar2'])
self.assertCountEqual([sidecar.image for sidecar in op1.sidecars],
['image0', 'image1', 'image2'])
self.assertCountEqual([env.name for env in op1.container.env],
['env1'])
def echo_op(port_number):
return dsl.ContainerOp(
name='echo',
image='busybox',
command=['sh', '-c'],
arguments=['echo "Got scheduled"']
).container.add_env_variable(V1EnvVar(name='PORT', value=port_number))
def mnist_pipeline():
ENV_MANAGE_URL = V1EnvVar(name='MANAGE_URL', value='http://220.116.228.93:8088/send')
data_0 = dsl.ContainerOp(
name="load & preprocess data pipeline",
image="byeongjokim/mnist-pre-data:latest",
).set_display_name('collect & preprocess data')\
.apply(onprem.mount_pvc("data-pvc", volume_name="data", volume_mount_path="/data"))
data_1 = dsl.ContainerOp(
name="validate data pipeline",
image="byeongjokim/mnist-val-data:latest",
).set_display_name('validate data').after(data_0)\
.apply(onprem.mount_pvc("data-pvc", volume_name="data", volume_mount_path="/data"))
train_model = dsl.ContainerOp(
name="train embedding model",
image="byeongjokim/mnist-train-model:latest",
).set_display_name('train model').after(data_1)\
.apply(onprem.mount_pvc("data-pvc", volume_name="data", volume_mount_path="/data"))\
.apply(onprem.mount_pvc("train-model-pvc", volume_name="train-model", volume_mount_path="/model"))
embedding = dsl.ContainerOp(
name="embedding data using embedding model",
image="byeongjokim/mnist-embedding:latest",
).set_display_name('embedding').after(train_model)\
.apply(onprem.mount_pvc("data-pvc", volume_name="data", volume_mount_path="/data"))\
.apply(onprem.mount_pvc("train-model-pvc", volume_name="train-model", volume_mount_path="/model"))
train_faiss = dsl.ContainerOp(
name="train faiss",
image="byeongjokim/mnist-train-faiss:latest",
).set_display_name('train faiss').after(embedding)\
.apply(onprem.mount_pvc("data-pvc", volume_name="data", volume_mount_path="/data"))\
.apply(onprem.mount_pvc("train-model-pvc", volume_name="train-model", volume_mount_path="/model"))
analysis = dsl.ContainerOp(
name="analysis total",
image="byeongjokim/mnist-analysis:latest",
file_outputs={
"confusion_matrix": "/confusion_matrix.csv",
"mlpipeline-ui-metadata": "/mlpipeline-ui-metadata.json",
"accuracy": "/accuracy.json",
"mlpipeline_metrics": "/mlpipeline-metrics.json"
}
).add_env_variable(ENV_MANAGE_URL).set_display_name('analysis').after(train_faiss)\
.apply(onprem.mount_pvc("data-pvc", volume_name="data", volume_mount_path="/data"))\
.apply(onprem.mount_pvc("train-model-pvc", volume_name="train-model", volume_mount_path="/model"))
baseline = 0.8
with dsl.Condition(analysis.outputs["accuracy"] > baseline) as check_deploy:
deploy = dsl.ContainerOp(
name="deploy mar",
image="byeongjokim/mnist-deploy:latest",
).add_env_variable(ENV_MANAGE_URL).set_display_name('deploy').after(analysis)\
.apply(onprem.mount_pvc("train-model-pvc", volume_name="train-model", volume_mount_path="/model"))\
.apply(onprem.mount_pvc("deploy-model-pvc", volume_name="deploy-model", volume_mount_path="/deploy-model"))
def run_notebook_op(op_name):
notebook_op = NotebookOp(name=op_name,
notebook='Untitled.ipynb',
cos_endpoint=cos_endpoint,
cos_bucket=cos_bucket,
cos_directory=cos_directory,
cos_dependencies_archive=cos_dependencies_archive,
pipeline_outputs=outputs,
pipeline_inputs=inputs,
image=image)
notebook_op.container.add_env_variable(
V1EnvVar(name='AWS_ACCESS_KEY_ID', value=cos_username))
notebook_op.container.add_env_variable(
V1EnvVar(name='AWS_SECRET_ACCESS_KEY', value=cos_password))
notebook_op.container.set_image_pull_policy('Always')
return notebook_op
def test_env_vars():
test_dag = DAG(
"test-dag",
default_args={},
description="testing",
start_date=datetime(2020, 1, 1),
schedule_interval=None,
)
k = basic_kubernetes_pod_operator(
task_id="task1",
dag=test_dag,
role="a_role",
repo_name="my_repo",
release="v0.0.0",
env_vars={
"AWS_DEFAULT_REGION": "us-east-1",
"EXTRA": "test"
},
)
expected_env = [
V1EnvVar(
name="AWS_DEFAULT_REGION",
value="us-east-1",
value_from=None,
),
V1EnvVar(
name="EXTRA",
value="test",
value_from=None,
),
V1EnvVar(
name="AWS_METADATA_SERVICE_TIMEOUT",
value="60",
value_from=None,
),
V1EnvVar(
name="AWS_METADATA_SERVICE_NUM_ATTEMPTS",
value="5",
value_from=None,
),
]
assert k.env_vars == expected_env
def transformer(containerOp):
containerOp.arguments = ['/scripts/pipelineWrapper.py', 'Privacy', 'python'] + containerOp.arguments
# shouldn't hard code this experiment name
containerOp.container.set_image_pull_policy("Always")
containerOp.add_volume(
k8s_client.V1Volume(
name='azure',
persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(
claim_name='azure-managed-disk')
)
).add_volume_mount(k8s_client.V1VolumeMount(mount_path='/mnt/azure', name='azure'))
containerOp.container.add_env_variable(V1EnvVar(name='AZ_NAME', value=ws.name))\
.add_env_variable(V1EnvVar(name='AZ_SUBSCRIPTION_ID', value=ws.subscription_id))\
.add_env_variable(V1EnvVar(name='AZ_RESOURCE_GROUP', value=ws.resource_group))
containerOp.apply(use_azure_secret('azcreds'))
return containerOp
def notebook_pipeline():
"""A pipeline to run a Jupyter notebook with elyra-ai/kfp-notebook and Papermill."""
from kfp_notebook.pipeline import NotebookOp
notebook_op = NotebookOp(name="${name}",
notebook="${notebook}",
cos_endpoint="${cos_endpoint}",
cos_bucket="${cos_bucket}",
cos_directory="${cos_directory}",
cos_dependencies_archive="${cos_dependencies_archive}",
requirements_url="${requirements_url}",
image="${image}")
from kubernetes.client.models import V1EnvVar
notebook_op.container.add_env_variable(V1EnvVar(name='AWS_ACCESS_KEY_ID', value="${cos_username}"))
notebook_op.container.add_env_variable(V1EnvVar(name='AWS_SECRET_ACCESS_KEY', value="${cos_password}"))
notebook_op.container.set_image_pull_policy('Always')
def titanic_pipline(token,
project_id,
dataset,
version='v1',
claimname="titanic-test-pvc"):
pipelineConfig = kfp.dsl.PipelineConf()
pipelineConfig.set_image_pull_policy("Always")
input_volumes = json.dumps([f"{claimname}@dataset://{dataset}/{version}"])
storage_op = ContainerOp(
name="get_dataset",
image="ocdr/dkubepl:storage_v3",
command=[
"dkubepl",
"storage",
"--token",
token,
"--namespace",
"kubeflow",
"--input_volumes",
input_volumes,
],
)
predict_op = ContainerOp(
name="predict",
image="ocdr/titanic_submission",
command=["python", "predict.py"],
pvolumes={"/titanic-test/": kfp.dsl.PipelineVolume(pvc=claimname)},
file_outputs={"output": "/tmp/prediction.csv"},
)
predict_op.after(storage_op)
predictions = kfp.dsl.InputArgumentPath(predict_op.outputs["output"])
submit_op = ContainerOp(
name="submit",
image="ocdr/d3project_eval",
command=[
"python",
"submit.py",
kfp.dsl.RUN_ID_PLACEHOLDER,
"-t",
token,
predictions,
],
file_outputs={
"mlpipeline-ui-metadata": "/metadata.json",
"results": "/results",
},
)
env_var = V1EnvVar(name="DKUBE_PROJECT_ID", value=project_id)
submit_op.add_env_variable(env_var)
def _serialize_pod_spec(self,
settings: SerializationSettings) -> Dict[str, Any]:
containers = self.task_config.pod_spec.containers
primary_exists = False
for container in containers:
if container.name == self.task_config.primary_container_name:
primary_exists = True
break
if not primary_exists:
# insert a placeholder primary container if it is not defined in the pod spec.
containers.append(
V1Container(name=self.task_config.primary_container_name))
final_containers = []
for container in containers:
# In the case of the primary container, we overwrite specific container attributes with the default values
# used in an SDK runnable task.
if container.name == self.task_config.primary_container_name:
sdk_default_container = super().get_container(settings)
container.image = sdk_default_container.image
# clear existing commands
container.command = sdk_default_container.command
# also clear existing args
container.args = sdk_default_container.args
limits, requests = {}, {}
for resource in sdk_default_container.resources.limits:
limits[_core_task.Resources.ResourceName.Name(
resource.name).lower()] = resource.value
for resource in sdk_default_container.resources.requests:
requests[_core_task.Resources.ResourceName.Name(
resource.name).lower()] = resource.value
resource_requirements = V1ResourceRequirements(
limits=limits, requests=requests)
if len(limits) > 0 or len(requests) > 0:
# Important! Only copy over resource requirements if they are non-empty.
container.resources = resource_requirements
container.env = [
V1EnvVar(name=key, value=val)
for key, val in sdk_default_container.env.items()
]
final_containers.append(container)
self.task_config._pod_spec.containers = final_containers
return ApiClient().sanitize_for_serialization(
self.task_config.pod_spec)
def test_extract_pipelineparams_from_any(self):
"""Test extract_pipeleineparams."""
p1 = PipelineParam(name='param1', op_name='op1')
p2 = PipelineParam(name='param2')
p3 = PipelineParam(name='param3', value='value3')
stuff_chars = ' between '
payload = str(p1) + stuff_chars + str(p2) + stuff_chars + str(p3)
container = V1Container(name=p1,
image=p2,
env=[V1EnvVar(name="foo", value=payload)])
params = extract_pipelineparams_from_any(container)
self.assertListEqual(sorted([p1, p2, p3]), sorted(params))
def _build_container_op(self, overrides=dict()):
artifacts_volume = k8s_client.V1Volume(
name="artifacts", empty_dir=k8s_client.V1EmptyDirVolumeSource())
op = dsl.ContainerOp(name=f"{self.pipeline_name}-{self.op_name}",
image=self.container_image_url,
command=self.container_command,
arguments=self.container_args,
file_outputs=self.output_files,
output_artifact_paths=self.output_artifact_paths)
# Mount an empty direcotry for us to write output to
self._mount_empty_dir(op, "artifacts", self.kfp_artifact_path)
# Apply the GCP Auth secret
if not self.gcp_auth_secret is None:
op.apply(use_gcp_secret(self.gcp_auth_secret))
# Apply other secrets
for secret_name in self.secrets:
mount_path = self.secrets[secret_name]
op.apply(self._bind_secret(secret_name, mount_path))
logging.info(f"Build Container {self.pipeline_name}.{self.op_name}")
# All my parameters
for name, value in self.data.items():
logging.info(f"\tContainer.ENV: {name}={value}")
op.container.add_env_variable(V1EnvVar(name=name,
value=str(value)))
for name, value in overrides.items():
logging.info(f"\tContainer.ENV: {name}={value} (override)")
op.container.add_env_variable(V1EnvVar(name=name,
value=str(value)))
return op
def my_pipeline(param1, param2):
op1 = (
ContainerOp(
name='op1',
image='image',
arguments=['%s hello %s %s' % (param1, param2, param1)],
init_containers=[
UserContainer(
name='initcontainer0', image='initimage0')
],
sidecars=[Sidecar(name='sidecar0', image='image0')],
container_kwargs={
'env': [V1EnvVar(name='env1', value='value1')]
},
file_outputs={
'out1': '/tmp/b'
}).add_init_container(
UserContainer(
name='initcontainer1',
image='initimage1')).add_init_container(
UserContainer(
name='initcontainer2',
image='initimage2')).add_sidecar(
Sidecar(
name='sidecar1',
image='image1')).add_sidecar(
Sidecar(
name='sidecar2',
image='image2')))
self.assertCountEqual([x.name for x in op1.inputs],
['param1', 'param2'])
self.assertCountEqual(list(op1.outputs.keys()), ['out1'])
self.assertCountEqual([x.op_name for x in op1.outputs.values()],
[op1.name])
self.assertEqual(op1.output.name, 'out1')
self.assertCountEqual(
[init_container.name for init_container in op1.init_containers],
['initcontainer0', 'initcontainer1', 'initcontainer2'])
self.assertCountEqual([
init_container.image for init_container in op1.init_containers
], ['initimage0', 'initimage1', 'initimage2'])
self.assertCountEqual([sidecar.name for sidecar in op1.sidecars],
['sidecar0', 'sidecar1', 'sidecar2'])
self.assertCountEqual([sidecar.image for sidecar in op1.sidecars],
['image0', 'image1', 'image2'])
self.assertCountEqual([env.name for env in op1.container.env],
['env1'])
def pipeline_gcs():
GCSCredentialFileName = "user-gcp-sa.json"
GCSCredentialVolumeName = "user-gcp-sa"
GCSCredentialVolumeMountPath = "/var/secrets/"
GCSCredentialEnvKey = "GOOGLE_APPLICATION_CREDENTIALS"
GCSCredentialFilePath = os.path.join(GCSCredentialVolumeMountPath, GCSCredentialFileName)
secret_name = 'user-gcp-sa'
dsl.ContainerOp(
name='mnist-gcs',
image='kangwoo/kfp-mnist-storage:0.0.1',
arguments=['--model', 'gs://kfp-bucket/kfp/mnist/model']
).add_volume(V1Volume(name=GCSCredentialVolumeName, secret=V1SecretVolumeSource(secret_name=secret_name))) \
.add_volume_mount(V1VolumeMount(name=GCSCredentialVolumeName, mount_path=GCSCredentialVolumeMountPath)) \
.add_env_variable(V1EnvVar(name=GCSCredentialEnvKey, value=GCSCredentialFilePath))
def lightweight_tensorboard_pipeline(tb_log_dir):
s3_endpoint = 'minio-service.kubeflow.svc.cluster.local:9000'
minio_endpoint = "http://" + s3_endpoint
minio_username = "******"
minio_key = "minio123"
minio_region = "us-east-1"
train_op(tb_log_dir).add_env_variable(V1EnvVar(name='S3_ENDPOINT', value=s3_endpoint)) \
.add_env_variable(V1EnvVar(name='AWS_ENDPOINT_URL', value=minio_endpoint)) \
.add_env_variable(V1EnvVar(name='AWS_ACCESS_KEY_ID', value=minio_username)) \
.add_env_variable(V1EnvVar(name='AWS_SECRET_ACCESS_KEY', value=minio_key)) \
.add_env_variable(V1EnvVar(name='AWS_REGION', value=minio_region)) \
.add_env_variable(V1EnvVar(name='S3_USE_HTTPS', value='0')) \
.add_env_variable(V1EnvVar(name='S3_VERIFY_SSL', value='0'))
def mnist_pipeline(learning_rate, dropout_rate, checkpoint_dir, model_version,
saved_model_dir, tensorboard_log):
exit_task = echo_op("Done!")
with dsl.ExitHandler(exit_task):
vop = dsl.VolumeOp(name="mnist_model_volume",
resource_name="mnist_model",
storage_class="nfs-client",
modes=dsl.VOLUME_MODE_RWM,
size="10Gi")
mnist = dsl.ContainerOp(
name='Mnist',
image=
'kubeflow-registry.default.svc.cluster.local:30000/katib-job:B67AEB5C',
command=['python', '/app/mnist.py'],
arguments=[
"--learning_rate", learning_rate, "--dropout_rate",
dropout_rate, "--checkpoint_dir", checkpoint_dir,
"--model_version", model_version, "--saved_model_dir",
saved_model_dir, "--tensorboard_log", tensorboard_log
],
pvolumes={"/result": vop.volume},
output_artifact_paths={
'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'
},
container_kwargs={
'env': [
V1EnvVar('S3_ENDPOINT',
'minio-service.kubeflow.svc.cluster.local:9000'),
V1EnvVar(
'AWS_ENDPOINT_URL',
'http://minio-service.kubeflow.svc.cluster.local:9000'
),
V1EnvVar('AWS_ACCESS_KEY_ID', 'minio'),
V1EnvVar('AWS_SECRET_ACCESS_KEY', 'minio123'),
V1EnvVar('AWS_REGION', 'us-east-1'),
V1EnvVar('S3_USE_HTTPS', '0'),
V1EnvVar('S3_VERIFY_SSL', '0'),
]
})
result = dsl.ContainerOp(name='list_list',
image='library/bash:4.4.23',
command=['ls', '-R', '/result'],
pvolumes={"/result": mnist.pvolume})
mnist.after(vop)
result.after(mnist)
def pipelineparams_pipeline(tag: str = 'latest', sleep_ms: int = 10):
echo = dsl.Sidecar(
name='echo',
image='hashicorp/http-echo:%s' % tag,
args=['-text="hello world"'],
)
op1 = components.load_component_from_text("""
name: download
description: download
inputs:
- {name: sleep_ms, type: Integer}
outputs:
- {name: data, type: String}
implementation:
container:
image: busy:placeholder
command:
- sh
- -c
args:
- |
sleep $0; wget localhost:5678 -O $1
- {inputValue: sleep_ms}
- {outputPath: data}
""")(sleep_ms)
op1.container.image = "busy:%s" % tag
op2 = components.load_component_from_text("""
name: echo
description: echo
inputs:
- {name: message, type: String}
implementation:
container:
image: library/bash
command:
- sh
- -c
args:
- |
echo $MSG $0
- {inputValue: message}
""")(op1.output)
op2.container.add_env_variable(V1EnvVar(name='MSG', value='pipelineParams: '))
def with_env(self, variable_name, value) -> Optional["HmlContainerOp"]:
"""
Bind an environment variable with the name `variable_name` and `value` specified
Args:
variable_name (str): The name of the environment variable
value (str): The value to bind to the variable
Returns:
A reference to the current `HmlContainerOp` (self)
"""
# # Update our current environment, so everything works in local development
# os.environ[variable_name] = value
# logging.info(f"Binding: ${variable_name} = {value}")
self.op.container.add_env_variable(V1EnvVar(name=variable_name, value=str(value)))
return self
def tensorboard_pipeline(tb_log_dir):
s3_endpoint = 'minio-service.kubeflow.svc.cluster.local:9000'
minio_endpoint = "http://" + s3_endpoint
minio_username = "******"
minio_key = "minio123"
minio_region = "us-east-1"
dsl.ContainerOp(
name='tensorboard',
image='kangwoo/kfp-tensorboard:0.0.1',
arguments=['--tb_log_dir', tb_log_dir],
output_artifact_paths={'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}
).add_env_variable(V1EnvVar(name='S3_ENDPOINT', value=s3_endpoint)) \
.add_env_variable(V1EnvVar(name='AWS_ENDPOINT_URL', value=minio_endpoint)) \
.add_env_variable(V1EnvVar(name='AWS_ACCESS_KEY_ID', value=minio_username)) \
.add_env_variable(V1EnvVar(name='AWS_SECRET_ACCESS_KEY', value=minio_key)) \
.add_env_variable(V1EnvVar(name='AWS_REGION', value=minio_region)) \
.add_env_variable(V1EnvVar(name='S3_USE_HTTPS', value='0')) \
.add_env_variable(V1EnvVar(name='S3_VERIFY_SSL', value='0'))
def pipeline_s3():
secret_name = "kfp-aws-secret"
s3_endpoint = 'minio-service.kubeflow.svc.cluster.local:9000'
minio_endpoint = "http://" + s3_endpoint
minio_region = "us-east-1"
dsl.ContainerOp(
name='mnist-s3',
image='kangwoo/kfp-mnist-storage:0.0.1',
arguments=['--model', 's3://tensorflow/kfp/mnist/model']
).add_env_variable(V1EnvVar(name='S3_ENDPOINT', value=s3_endpoint)) \
.add_env_variable(V1EnvVar(name='AWS_ENDPOINT_URL', value=minio_endpoint)) \
.add_env_variable(V1EnvVar(name='AWS_ACCESS_KEY_ID',
value_from=V1EnvVarSource(
secret_key_ref=V1SecretKeySelector(name=secret_name, key='AWS_ACCESS_KEY_ID')))) \
.add_env_variable(V1EnvVar(name='AWS_SECRET_ACCESS_KEY',
value_from=V1EnvVarSource(secret_key_ref=V1SecretKeySelector(name=secret_name,
key='AWS_SECRET_ACCESS_KEY')))) \
.add_env_variable(V1EnvVar(name='AWS_REGION', value=minio_region)) \
.add_env_variable(V1EnvVar(name='S3_USE_HTTPS', value='0')) \
.add_env_variable(V1EnvVar(name='S3_VERIFY_SSL', value='0'))
def make_pod(
name,
cmd,
port,
image,
image_pull_policy,
image_pull_secrets=None,
node_selector=None,
run_as_uid=None,
run_as_gid=None,
fs_gid=None,
supplemental_gids=None,
run_privileged=False,
allow_privilege_escalation=True,
env=None,
working_dir=None,
volumes=None,
volume_mounts=None,
labels=None,
annotations=None,
cpu_limit=None,
cpu_guarantee=None,
mem_limit=None,
mem_guarantee=None,
extra_resource_limits=None,
extra_resource_guarantees=None,
lifecycle_hooks=None,
init_containers=None,
service_account=None,
extra_container_config=None,
extra_pod_config=None,
extra_containers=None,
scheduler_name=None,
tolerations=None,
node_affinity_preferred=None,
node_affinity_required=None,
pod_affinity_preferred=None,
pod_affinity_required=None,
pod_anti_affinity_preferred=None,
pod_anti_affinity_required=None,
priority_class_name=None,
logger=None,
):
"""
Make a k8s pod specification for running a user notebook.
Parameters
----------
name:
Name of pod. Must be unique within the namespace the object is
going to be created in. Must be a valid DNS label.
image:
Image specification - usually a image name and tag in the form
of image_name:tag. Same thing you would use with docker commandline
arguments
image_pull_policy:
Image pull policy - one of 'Always', 'IfNotPresent' or 'Never'. Decides
when kubernetes will check for a newer version of image and pull it when
running a pod.
image_pull_secrets:
Image pull secrets - a list of references to Kubernetes Secret resources
with credentials to pull images from image registries. This list can
either have strings in it or objects with the string value nested under
a name field.
port:
Port the notebook server is going to be listening on
cmd:
The command used to execute the singleuser server.
node_selector:
Dictionary Selector to match nodes where to launch the Pods
run_as_uid:
The UID used to run single-user pods. The default is to run as the user
specified in the Dockerfile, if this is set to None.
run_as_gid:
The GID used to run single-user pods. The default is to run as the primary
group of the user specified in the Dockerfile, if this is set to None.
Setting this parameter requires that *feature-gate* **RunAsGroup** be enabled,
otherwise the effective GID of the pod will be 0 (root). In addition, not
setting `run_as_gid` once feature-gate RunAsGroup is enabled will also
result in an effective GID of 0 (root).
fs_gid
The gid that will own any fresh volumes mounted into this pod, if using
volume types that support this (such as GCE). This should be a group that
the uid the process is running as should be a member of, so that it can
read / write to the volumes mounted.
supplemental_gids:
A list of GIDs that should be set as additional supplemental groups to
the user that the container runs as. You may have to set this if you are
deploying to an environment with RBAC/SCC enforced and pods run with a
'restricted' SCC which results in the image being run as an assigned
user ID. The supplemental group IDs would need to include the
corresponding group ID of the user ID the image normally would run as.
The image must setup all directories/files any application needs access
to, as group writable.
run_privileged:
Whether the container should be run in privileged mode.
allow_privilege_escalation:
Controls whether a process can gain more privileges than its parent process.
env:
Dictionary of environment variables.
volumes:
List of dictionaries containing the volumes of various types this pod
will be using. See k8s documentation about volumes on how to specify
these
volume_mounts:
List of dictionaries mapping paths in the container and the volume(
specified in volumes) that should be mounted on them. See the k8s
documentaiton for more details
working_dir:
String specifying the working directory for the notebook container
labels:
Labels to add to the spawned pod.
annotations:
Annotations to add to the spawned pod.
cpu_limit:
Float specifying the max number of CPU cores the user's pod is
allowed to use.
cpu_guarentee:
Float specifying the max number of CPU cores the user's pod is
guaranteed to have access to, by the scheduler.
mem_limit:
String specifying the max amount of RAM the user's pod is allowed
to use. String instead of float/int since common suffixes are allowed
mem_guarantee:
String specifying the max amount of RAM the user's pod is guaranteed
to have access to. String ins loat/int since common suffixes
are allowed
lifecycle_hooks:
Dictionary of lifecycle hooks
init_containers:
List of initialization containers belonging to the pod.
service_account:
Service account to mount on the pod. None disables mounting
extra_container_config:
Extra configuration (e.g. envFrom) for notebook container which is not covered by parameters above.
extra_pod_config:
Extra configuration (e.g. tolerations) for pod which is not covered by parameters above.
extra_containers:
Extra containers besides notebook container. Used for some housekeeping jobs (e.g. crontab).
scheduler_name:
The pod's scheduler explicitly named.
tolerations:
Tolerations can allow a pod to schedule or execute on a tainted node. To
learn more about pod tolerations, see
https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/.
Pass this field an array of "Toleration" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#nodeselectorterm-v1-core
node_affinity_preferred:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "PreferredSchedulingTerm" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#preferredschedulingterm-v1-core
node_affinity_required:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "NodeSelectorTerm" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#nodeselectorterm-v1-core
pod_affinity_preferred:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "WeightedPodAffinityTerm" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#weightedpodaffinityterm-v1-core
pod_affinity_required:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "PodAffinityTerm" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#podaffinityterm-v1-core
pod_anti_affinity_preferred:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "WeightedPodAffinityTerm" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#weightedpodaffinityterm-v1-core
pod_anti_affinity_required:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "PodAffinityTerm" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#podaffinityterm-v1-core
priority_class_name:
The name of the PriorityClass to be assigned the pod. This feature is Beta available in K8s 1.11.
"""
pod = V1Pod()
pod.kind = "Pod"
pod.api_version = "v1"
pod.metadata = V1ObjectMeta(name=name,
labels=(labels or {}).copy(),
annotations=(annotations or {}).copy())
pod.spec = V1PodSpec(containers=[])
pod.spec.restart_policy = 'OnFailure'
if image_pull_secrets is not None:
# image_pull_secrets as received by the make_pod function should always
# be a list, but it is allowed to have "a-string" elements or {"name":
# "a-string"} elements.
pod.spec.image_pull_secrets = [
V1LocalObjectReference(name=secret_ref) if type(secret_ref) == str
else get_k8s_model(V1LocalObjectReference, secret_ref)
for secret_ref in image_pull_secrets
]
if node_selector:
pod.spec.node_selector = node_selector
if lifecycle_hooks:
lifecycle_hooks = get_k8s_model(V1Lifecycle, lifecycle_hooks)
# There are security contexts both on the Pod level or the Container level.
# The security settings that you specify for a Pod apply to all Containers
# in the Pod, but settings on the container level can override them.
#
# We configure the pod to be spawned on the container level unless the
# option is only available on the pod level, such as for those relating to
# the volumes as compared to the running user of the container. Volumes
# belong to the pod and are only mounted by containers after all.
#
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
# ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.16/#securitycontext-v1-core (container)
# ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.16/#podsecuritycontext-v1-core (pod)
pod_security_context = V1PodSecurityContext()
if fs_gid is not None:
pod_security_context.fs_group = int(fs_gid)
if supplemental_gids is not None and supplemental_gids:
pod_security_context.supplemental_groups = [
int(gid) for gid in supplemental_gids
]
# Only clutter pod spec with actual content
if not all([e is None for e in pod_security_context.to_dict().values()]):
pod.spec.security_context = pod_security_context
container_security_context = V1SecurityContext()
if run_as_uid is not None:
container_security_context.run_as_user = int(run_as_uid)
if run_as_gid is not None:
container_security_context.run_as_group = int(run_as_gid)
if run_privileged:
container_security_context.privileged = True
if not allow_privilege_escalation:
container_security_context.allow_privilege_escalation = False
# Only clutter container spec with actual content
if all([e is None for e in container_security_context.to_dict().values()]):
container_security_context = None
# Transform a dict into valid Kubernetes EnvVar Python representations. This
# representation shall always have a "name" field as well as either a
# "value" field or "value_from" field. For examples see the
# test_make_pod_with_env function.
prepared_env = []
for k, v in (env or {}).items():
if type(v) == dict:
if not "name" in v:
v["name"] = k
prepared_env.append(get_k8s_model(V1EnvVar, v))
else:
prepared_env.append(V1EnvVar(name=k, value=v))
notebook_container = V1Container(
name='notebook',
image=image,
working_dir=working_dir,
ports=[V1ContainerPort(name='notebook-port', container_port=port)],
env=prepared_env,
args=cmd,
image_pull_policy=image_pull_policy,
lifecycle=lifecycle_hooks,
resources=V1ResourceRequirements(),
volume_mounts=[
get_k8s_model(V1VolumeMount, obj) for obj in (volume_mounts or [])
],
security_context=container_security_context,
)
if service_account is None:
# This makes sure that we don't accidentally give access to the whole
# kubernetes API to the users in the spawned pods.
pod.spec.automount_service_account_token = False
else:
pod.spec.service_account_name = service_account
notebook_container.resources.requests = {}
if cpu_guarantee:
notebook_container.resources.requests['cpu'] = cpu_guarantee
if mem_guarantee:
notebook_container.resources.requests['memory'] = mem_guarantee
if extra_resource_guarantees:
notebook_container.resources.requests.update(extra_resource_guarantees)
notebook_container.resources.limits = {}
if cpu_limit:
notebook_container.resources.limits['cpu'] = cpu_limit
if mem_limit:
notebook_container.resources.limits['memory'] = mem_limit
if extra_resource_limits:
notebook_container.resources.limits.update(extra_resource_limits)
if extra_container_config:
notebook_container = update_k8s_model(
target=notebook_container,
changes=extra_container_config,
logger=logger,
target_name="notebook_container",
changes_name="extra_container_config",
)
pod.spec.containers.append(notebook_container)
if extra_containers:
pod.spec.containers.extend(
[get_k8s_model(V1Container, obj) for obj in extra_containers])
if tolerations:
pod.spec.tolerations = [
get_k8s_model(V1Toleration, obj) for obj in tolerations
]
if init_containers:
pod.spec.init_containers = [
get_k8s_model(V1Container, obj) for obj in init_containers
]
if volumes:
pod.spec.volumes = [get_k8s_model(V1Volume, obj) for obj in volumes]
else:
# Keep behaving exactly like before by not cleaning up generated pod
# spec by setting the volumes field even though it is an empty list.
pod.spec.volumes = []
if scheduler_name:
pod.spec.scheduler_name = scheduler_name
node_affinity = None
if node_affinity_preferred or node_affinity_required:
node_selector = None
if node_affinity_required:
node_selector = V1NodeSelector(node_selector_terms=[
get_k8s_model(V1NodeSelectorTerm, obj)
for obj in node_affinity_required
], )
preferred_scheduling_terms = None
if node_affinity_preferred:
preferred_scheduling_terms = [
get_k8s_model(V1PreferredSchedulingTerm, obj)
for obj in node_affinity_preferred
]
node_affinity = V1NodeAffinity(
preferred_during_scheduling_ignored_during_execution=
preferred_scheduling_terms,
required_during_scheduling_ignored_during_execution=node_selector,
)
pod_affinity = None
if pod_affinity_preferred or pod_affinity_required:
weighted_pod_affinity_terms = None
if pod_affinity_preferred:
weighted_pod_affinity_terms = [
get_k8s_model(V1WeightedPodAffinityTerm, obj)
for obj in pod_affinity_preferred
]
pod_affinity_terms = None
if pod_affinity_required:
pod_affinity_terms = [
get_k8s_model(V1PodAffinityTerm, obj)
for obj in pod_affinity_required
]
pod_affinity = V1PodAffinity(
preferred_during_scheduling_ignored_during_execution=
weighted_pod_affinity_terms,
required_during_scheduling_ignored_during_execution=
pod_affinity_terms,
)
pod_anti_affinity = None
if pod_anti_affinity_preferred or pod_anti_affinity_required:
weighted_pod_affinity_terms = None
if pod_anti_affinity_preferred:
weighted_pod_affinity_terms = [
get_k8s_model(V1WeightedPodAffinityTerm, obj)
for obj in pod_anti_affinity_preferred
]
pod_affinity_terms = None
if pod_anti_affinity_required:
pod_affinity_terms = [
get_k8s_model(V1PodAffinityTerm, obj)
for obj in pod_anti_affinity_required
]
pod_anti_affinity = V1PodAffinity(
preferred_during_scheduling_ignored_during_execution=
weighted_pod_affinity_terms,
required_during_scheduling_ignored_during_execution=
pod_affinity_terms,
)
affinity = None
if (node_affinity or pod_affinity or pod_anti_affinity):
affinity = V1Affinity(
node_affinity=node_affinity,
pod_affinity=pod_affinity,
pod_anti_affinity=pod_anti_affinity,
)
if affinity:
pod.spec.affinity = affinity
if priority_class_name:
pod.spec.priority_class_name = priority_class_name
if extra_pod_config:
pod.spec = update_k8s_model(
target=pod.spec,
changes=extra_pod_config,
logger=logger,
target_name="pod.spec",
changes_name="extra_pod_config",
)
return pod
def __init__(self,
pipeline_name: str,
experiment_name: str,
notebook: str,
cos_endpoint: str,
cos_bucket: str,
cos_directory: str,
cos_dependencies_archive: str,
pipeline_version: Optional[str] = '',
pipeline_source: Optional[str] = None,
pipeline_outputs: Optional[List[str]] = None,
pipeline_inputs: Optional[List[str]] = None,
pipeline_envs: Optional[Dict[str, str]] = None,
requirements_url: Optional[str] = None,
bootstrap_script_url: Optional[str] = None,
emptydir_volume_size: Optional[str] = None,
cpu_request: Optional[str] = None,
mem_request: Optional[str] = None,
gpu_limit: Optional[str] = None,
workflow_engine: Optional[str] = 'argo',
**kwargs):
"""Create a new instance of ContainerOp.
Args:
pipeline_name: pipeline that this op belongs to
experiment_name: the experiment where pipeline_name is executed
notebook: name of the notebook that will be executed per this operation
cos_endpoint: object storage endpoint e.g weaikish1.fyre.ibm.com:30442
cos_bucket: bucket to retrieve archive from
cos_directory: name of the directory in the object storage bucket to pull
cos_dependencies_archive: archive file name to get from object storage bucket e.g archive1.tar.gz
pipeline_version: optional version identifier
pipeline_source: pipeline source
pipeline_outputs: comma delimited list of files produced by the notebook
pipeline_inputs: comma delimited list of files to be consumed/are required by the notebook
pipeline_envs: dictionary of environmental variables to set in the container prior to execution
requirements_url: URL to a python requirements.txt file to be installed prior to running the notebook
bootstrap_script_url: URL to a custom python bootstrap script to run
emptydir_volume_size: Size(GB) of the volume to create for the workspace when using CRIO container runtime
cpu_request: number of CPUs requested for the operation
mem_request: memory requested for the operation (in Gi)
gpu_limit: maximum number of GPUs allowed for the operation
workflow_engine: Kubeflow workflow engine, defaults to 'argo'
kwargs: additional key value pairs to pass e.g. name, image, sidecars & is_exit_handler.
See Kubeflow pipelines ContainerOp definition for more parameters or how to use
https://kubeflow-pipelines.readthedocs.io/en/latest/source/kfp.dsl.html#kfp.dsl.ContainerOp
"""
self.pipeline_name = pipeline_name
self.pipeline_version = pipeline_version
self.pipeline_source = pipeline_source
self.experiment_name = experiment_name
self.notebook = notebook
self.notebook_name = os.path.basename(notebook)
self.cos_endpoint = cos_endpoint
self.cos_bucket = cos_bucket
self.cos_directory = cos_directory
self.cos_dependencies_archive = cos_dependencies_archive
self.container_work_dir_root_path = "./"
self.container_work_dir_name = "jupyter-work-dir/"
self.container_work_dir = self.container_work_dir_root_path + self.container_work_dir_name
self.bootstrap_script_url = bootstrap_script_url
self.requirements_url = requirements_url
self.pipeline_outputs = pipeline_outputs
self.pipeline_inputs = pipeline_inputs
self.pipeline_envs = pipeline_envs
self.cpu_request = cpu_request
self.mem_request = mem_request
self.gpu_limit = gpu_limit
argument_list = []
""" CRI-o support for kfp pipelines
We need to attach an emptydir volume for each notebook that runs since CRI-o runtime does not allow
us to write to the base image layer file system, only to volumes.
"""
self.emptydir_volume_name = "workspace"
self.emptydir_volume_size = emptydir_volume_size
self.python_user_lib_path = ''
self.python_user_lib_path_target = ''
self.python_pip_config_url = ''
if self.emptydir_volume_size:
self.container_work_dir_root_path = "/opt/app-root/src/"
self.container_python_dir_name = "python3/"
self.container_work_dir = self.container_work_dir_root_path + self.container_work_dir_name
self.python_user_lib_path = self.container_work_dir + self.container_python_dir_name
self.python_user_lib_path_target = '--target=' + self.python_user_lib_path
self.python_pip_config_url = ELYRA_PIP_CONFIG_URL
if not self.bootstrap_script_url:
self.bootstrap_script_url = ELYRA_BOOTSTRAP_SCRIPT_URL
if not self.requirements_url:
self.requirements_url = ELYRA_REQUIREMENTS_URL
if 'name' not in kwargs:
raise TypeError("You need to provide a name for the operation.")
elif not kwargs.get('name'):
raise ValueError("You need to provide a name for the operation.")
if 'image' not in kwargs:
raise ValueError("You need to provide an image.")
if not notebook:
raise ValueError("You need to provide a notebook.")
if 'arguments' not in kwargs:
""" If no arguments are passed, we use our own.
If ['arguments'] are set, we assume container's ENTRYPOINT is set and dependencies are installed
NOTE: Images being pulled must have python3 available on PATH and cURL utility
"""
argument_list.append('mkdir -p {container_work_dir} && cd {container_work_dir} && '
'curl -H "Cache-Control: no-cache" -L {bootscript_url} --output bootstrapper.py && '
'curl -H "Cache-Control: no-cache" -L {reqs_url} --output requirements-elyra.txt && '
.format(container_work_dir=self.container_work_dir,
bootscript_url=self.bootstrap_script_url,
reqs_url=self.requirements_url)
)
if self.emptydir_volume_size:
argument_list.append('mkdir {container_python_dir} && cd {container_python_dir} && '
'curl -H "Cache-Control: no-cache" -L {python_pip_config_url} '
'--output pip.conf && cd .. &&'
.format(python_pip_config_url=self.python_pip_config_url,
container_python_dir=self.container_python_dir_name)
)
argument_list.append('python3 -m pip install {python_user_lib_path_target} packaging && '
'python3 -m pip freeze > requirements-current.txt && '
'python3 bootstrapper.py '
'--cos-endpoint {cos_endpoint} '
'--cos-bucket {cos_bucket} '
'--cos-directory "{cos_directory}" '
'--cos-dependencies-archive "{cos_dependencies_archive}" '
'--file "{notebook}" '
.format(cos_endpoint=self.cos_endpoint,
cos_bucket=self.cos_bucket,
cos_directory=self.cos_directory,
cos_dependencies_archive=self.cos_dependencies_archive,
notebook=self.notebook,
python_user_lib_path_target=self.python_user_lib_path_target)
)
if self.pipeline_inputs:
inputs_str = self._artifact_list_to_str(self.pipeline_inputs)
argument_list.append('--inputs "{}" '.format(inputs_str))
if self.pipeline_outputs:
outputs_str = self._artifact_list_to_str(self.pipeline_outputs)
argument_list.append('--outputs "{}" '.format(outputs_str))
if self.emptydir_volume_size:
argument_list.append('--user-volume-path "{}" '.format(self.python_user_lib_path))
kwargs['command'] = ['sh', '-c']
kwargs['arguments'] = "".join(argument_list)
super().__init__(**kwargs)
# We must deal with the envs after the superclass initialization since these amend the
# container attribute that isn't available until now.
if self.pipeline_envs:
for key, value in self.pipeline_envs.items(): # Convert dict entries to format kfp needs
self.container.add_env_variable(V1EnvVar(name=key, value=value))
# If crio volume size is found then assume kubeflow pipelines environment is using CRI-o as
# its container runtime
if self.emptydir_volume_size:
self.add_volume(V1Volume(empty_dir=V1EmptyDirVolumeSource(
medium="",
size_limit=self.emptydir_volume_size),
name=self.emptydir_volume_name))
self.container.add_volume_mount(V1VolumeMount(mount_path=self.container_work_dir_root_path,
name=self.emptydir_volume_name))
# Append to PYTHONPATH location of elyra dependencies in installed in Volume
self.container.add_env_variable(V1EnvVar(name='PYTHONPATH',
value=self.python_user_lib_path))
if self.cpu_request:
self.container.set_cpu_request(cpu=str(cpu_request))
if self.mem_request:
self.container.set_memory_request(memory=str(mem_request) + "G")
if self.gpu_limit:
gpu_vendor = self.pipeline_envs.get('GPU_VENDOR', 'nvidia')
self.container.set_gpu_limit(gpu=str(gpu_limit), vendor=gpu_vendor)
# Generate unique ELYRA_RUN_NAME value and expose it as an environment
# variable in the container
if workflow_engine and workflow_engine.lower() == 'argo':
run_name_placeholder = '{{workflow.annotations.pipelines.kubeflow.org/run_name}}'
self.container.add_env_variable(V1EnvVar(name='ELYRA_RUN_NAME',
value=run_name_placeholder))
else:
# For Tekton derive the value from the specified pod annotation
annotation = 'pipelines.kubeflow.org/run_name'
field_path = f"metadata.annotations['{annotation}']"
self.container.add_env_variable(V1EnvVar(name='ELYRA_RUN_NAME',
value_from=V1EnvVarSource(
field_ref=V1ObjectFieldSelector(field_path=field_path))))
# Attach metadata to the pod
# Node type (a static type for this op)
self.add_pod_label('elyra/node-type',
NotebookOp._normalize_label_value(
'notebook-script'))
# Pipeline name
self.add_pod_label('elyra/pipeline-name',
NotebookOp._normalize_label_value(self.pipeline_name))
# Pipeline version
self.add_pod_label('elyra/pipeline-version',
NotebookOp._normalize_label_value(self.pipeline_version))
# Experiment name
self.add_pod_label('elyra/experiment-name',
NotebookOp._normalize_label_value(self.experiment_name))
# Pipeline node name
self.add_pod_label('elyra/node-name',
NotebookOp._normalize_label_value(kwargs.get('name')))
# Pipeline node file
self.add_pod_annotation('elyra/node-file-name',
self.notebook)
# Identify the pipeline source, which can be a
# pipeline file (mypipeline.pipeline), a Python
# script or notebook that was submitted
if self.pipeline_source is not None:
self.add_pod_annotation('elyra/pipeline-source',
self.pipeline_source)
def taxi_cab_classification(
project,
output='s3://mlpipeline/tfx/output',
column_names='s3://mlpipeline/tfx/taxi-cab-classification/column-names.json',
key_columns='trip_start_timestamp',
train='s3://mlpipeline/tfx/taxi-cab-classification/train.csv',
evaluation='s3://mlpipeline/tfx/taxi-cab-classification/eval.csv',
mode='local',
preprocess_module='s3://mlpipeline/tfx/taxi-cab-classification/preprocessing.py',
learning_rate=0.1,
hidden_layer_size='1500',
steps=3000,
analyze_slice_column='trip_start_hour'
):
output_template = str(output) + '/{{workflow.uid}}/{{pod.name}}/data'
target_lambda = """lambda x: (x['target'] > x['fare'] * 0.2)"""
target_class_lambda = """lambda x: 1 if (x['target'] > x['fare'] * 0.2) else 0"""
tf_server_name = 'taxi-cab-classification-model-{{workflow.uid}}'
if platform == 'onprem':
if storage == 'minio':
data_preparation = dsl.ContainerOp(
name="data_preparation",
image="aiven86/minio_mc-git",
command=["sh", "/bin/run.sh"],
).set_image_pull_policy('IfNotPresent')
data_preparation.container.add_env_variable(V1EnvVar(name='GITPAHT', value=GITPAHT))
data_preparation.container.add_env_variable(V1EnvVar(name='GITDIR', value=GITDIR))
data_preparation.container.add_env_variable(V1EnvVar(name='MINIOPATH', value=MINIOPATH))
data_preparation.container.add_env_variable(V1EnvVar(name='DATAPATH', value=DATAPATH))
else:
vop = dsl.VolumeOp(
name="create_pvc",
storage_class="rook-ceph-fs",
resource_name="pipeline-pvc",
modes=dsl.VOLUME_MODE_RWM,
size="1Gi"
)
data_preparation = dsl.ContainerOp(
name="data_preparation",
image="aiven86/git",
command=["git", "clone", "https://github.com/kubeflow/pipelines.git", str(output) + "/pipelines"],
).apply(onprem.mount_pvc(vop.outputs["name"], 'local-storage', output))
data_preparation.after(vop)
validation = dataflow_tf_data_validation_op(
inference_data=train,
validation_data=evaluation,
column_names=column_names,
key_columns=key_columns,
gcp_project=project,
run_mode=mode,
validation_output=output_template,
)
if platform == 'onprem':
validation.after(data_preparation)
preprocess = dataflow_tf_transform_op(
training_data_file_pattern=train,
evaluation_data_file_pattern=evaluation,
schema=validation.outputs['schema'],
gcp_project=project,
run_mode=mode,
preprocessing_module=preprocess_module,
transformed_data_dir=output_template
)
training = tf_train_op(
transformed_data_dir=preprocess.output,
schema=validation.outputs['schema'],
learning_rate=learning_rate,
hidden_layer_size=hidden_layer_size,
steps=steps,
target='tips',
preprocessing_module=preprocess_module,
training_output_dir=output_template
)
analysis = dataflow_tf_model_analyze_op(
model=training.output,
evaluation_data=evaluation,
schema=validation.outputs['schema'],
gcp_project=project,
run_mode=mode,
slice_columns=analyze_slice_column,
analysis_results_dir=output_template
)
prediction = dataflow_tf_predict_op(
data_file_pattern=evaluation,
schema=validation.outputs['schema'],
target_column='tips',
model=training.output,
run_mode=mode,
gcp_project=project,
predictions_dir=output_template
)
cm = confusion_matrix_op(
predictions=prediction.output,
target_lambda=target_lambda,
output_dir=output_template
)
roc = roc_op(
predictions_dir=prediction.output,
target_lambda=target_class_lambda,
output_dir=output_template
)
if platform == 'GCP' or storage == 'minio':
deploy = kubeflow_deploy_op(
model_dir=str(training.output) + '/export/export',
server_name=tf_server_name
)
elif platform == 'onprem' and storage != 'minio':
deploy = kubeflow_deploy_op(
cluster_name=project,
model_dir=str(training.output) + '/export/export',
pvc_name=vop.outputs["name"],
server_name=tf_server_name
)
steps = [validation, preprocess, training, analysis, prediction, cm, roc, deploy]
for step in steps:
if platform == 'GCP':
step.apply(gcp.use_gcp_secret('user-gcp-sa'))
elif platform == 'onprem':
if storage == 'minio':
step.container.add_env_variable(V1EnvVar(name='AWS_ACCESS_KEY_ID', value=AWS_ACCESS_KEY_ID))
step.container.add_env_variable(V1EnvVar(name='AWS_SECRET_ACCESS_KEY', value=AWS_SECRET_ACCESS_KEY))
step.container.add_env_variable(V1EnvVar(name='AWS_REGION', value=AWS_REGION))
step.container.add_env_variable(V1EnvVar(name='S3_ENDPOINT', value=S3_ENDPOINT))
step.container.add_env_variable(V1EnvVar(name='S3_USE_HTTPS', value=S3_USE_HTTPS))
step.container.add_env_variable(V1EnvVar(name='S3_VERIFY_SSL', value=S3_VERIFY_SSL))
else:
step.apply(onprem.mount_pvc(vop.outputs["name"], 'local-storage', output))