def sequential_pipeline(model_data_url = "https://test-epi.s3.fr-par.scw.cloud/kc_house_data.csv?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=SCWTTHE3QW9915A46GMW%2F20200206%2Ffr-par%2Fs3%2Faws4_request&X-Amz-Date=20200206T200330Z&X-Amz-Expires=521789&X-Amz-Signature=56952a33bf8a12b255ed573a2a4e05dd901eec9985a98ed332f89c61ad55a2cd&X-Amz-SignedHeaders=host"): model_data_dest = "/data/kc_house_data.csv" vop = dsl.VolumeOp( name="vol", resource_name="newpvc", size="1Gi", modes=dsl.VOLUME_MODE_RWM ) op_download = dsl.ContainerOp( name='download', image='toune/epi-saas-project2-download', pvolumes={"/data": vop.volume} ) op_download.container.add_env_variable(V1EnvVar(name='DOWNLOAD_FILE_URL', value=model_data_url)) op_download.container.add_env_variable(V1EnvVar(name='DOWNLOAD_FILE_DEST', value=model_data_dest)) op_predict = dsl.ContainerOp( name='predict', image='toune/epi-saas-project2-app', pvolumes={"/data": vop.volume} ) op_predict.container.add_env_variable(V1EnvVar(name='TRAINED_MODEL_PATH', value="/data/model-artifacts")) op_predict.container.add_env_variable(V1EnvVar(name='DATA_PATH', value=model_data_dest)) op_predict.after(op_download)
def notebook_pipeline(): """A pipeline to run a Jupyter notebook with elyra-ai/kfp-notebook and Papermill.""" from kfp_notebook.pipeline import NotebookOp notebook_op = NotebookOp(name="${name}", notebook="${notebook}", cos_endpoint="${cos_endpoint}", cos_bucket="${cos_bucket}", cos_directory="${cos_directory}", cos_dependencies_archive="${cos_dependencies_archive}", requirements_url="${requirements_url}", image="${image}") from kubernetes.client.models import V1EnvVar notebook_op.container.add_env_variable(V1EnvVar(name='AWS_ACCESS_KEY_ID', value="${cos_username}")) notebook_op.container.add_env_variable(V1EnvVar(name='AWS_SECRET_ACCESS_KEY', value="${cos_password}")) from kfp import onprem notebook_op.container.add_env_variable(V1EnvVar(name='DATA_DIR', value="${mount_path}")) notebook_op.apply(onprem.mount_pvc(pvc_name='${dataset_pvc}', volume_name='${dataset_pvc}', volume_mount_path='${mount_path}'))
def test_deprecation_warnings(self): """Test deprecation warnings.""" op = ContainerOp(name='op1', image='image') with self.assertWarns(PendingDeprecationWarning): op.env_variables = [V1EnvVar(name="foo", value="bar")] with self.assertWarns(PendingDeprecationWarning): op.image = 'image2' with self.assertWarns(PendingDeprecationWarning): op.set_memory_request('10M') with self.assertWarns(PendingDeprecationWarning): op.set_memory_limit('10M') with self.assertWarns(PendingDeprecationWarning): op.set_cpu_request('100m') with self.assertWarns(PendingDeprecationWarning): op.set_cpu_limit('1') with self.assertWarns(PendingDeprecationWarning): op.set_gpu_limit('1') with self.assertWarns(PendingDeprecationWarning): op.add_env_variable(V1EnvVar(name="foo", value="bar")) with self.assertWarns(PendingDeprecationWarning): op.add_volume_mount( V1VolumeMount(mount_path='/secret/gcp-credentials', name='gcp-credentials'))
def ml_pipeline(): pvc1 = dsl.VolumeOp( name = "PVC-data", resource_name = "pvc-ojt-ml", storage_class = "ojt-tibero-ml", modes = dsl.VOLUME_MODE_RWM, size = "1Gi" ) pvc2 = dsl.VolumeOp( name = "PVC-was", resource_name = "pvc-ojt-result", storage_class = "ojt-tibero-result", modes = dsl.VOLUME_MODE_RWM, size = "1Gi" ) pod1 = dsl.ContainerOp( name = 'Tibero-JDBC agent', image = 'hanjoo8821/jdbc-tibero:colsout-' + ver, container_kwargs={'env':[V1EnvVar('id', 'hanjoo'), V1EnvVar('pw', '1010'), V1EnvVar('sql', 'SELECT * FROM AB1_2 LEFT JOIN ACCOUNTING ON AB1_2.EMP_NUM = ACCOUNTING.EMP_NUM'), V1EnvVar('col1', 'EMP_NUM'), V1EnvVar('col2', 'BIRTH'), V1EnvVar('col3', 'SALARY')]}, pvolumes = {"/Output": pvc1.volume} ) pod2a = dsl.ContainerOp( name = 'Print-EMP_NUM', image = 'alpine:3.6', command = ['cat', '/Output/EMP_NUM.txt'], pvolumes = {"/Output": pod1.pvolume} ) pod2b = dsl.ContainerOp( name = 'Trans-BIRTH', image = 'hanjoo8821/jdbc-tibero:trans-birth-' + ver, container_kwargs={'env':[V1EnvVar('col2', 'BIRTH')]}, pvolumes = {"/Output": pod1.pvolume} ) pod2c = dsl.ContainerOp( name = 'Trans-SALARY', image = 'hanjoo8821/jdbc-tibero:trans-salary-' + ver, container_kwargs={'env':[V1EnvVar('col3', 'SALARY')]}, pvolumes = {"/Output": pod1.pvolume} ) pod3 = dsl.ContainerOp( name = 'ML-Linear Regression', image = 'hanjoo8821/jdbc-tibero:ml-' + ver, container_kwargs={'env':[V1EnvVar('col2', 'BIRTH'), V1EnvVar('col3', 'SALARY')]}, pvolumes = {"/Output": pod2b.pvolume} ).after(pod2b, pod2c) pod4 = dsl.ContainerOp( name = 'Visualization', image = 'hanjoo8821/jdbc-tibero:graph-' + ver, container_kwargs={'env':[V1EnvVar('col2', 'BIRTH'), V1EnvVar('col3', 'SALARY')]}, pvolumes = {"/Output": pod3.pvolume, "/WAS": pvc2.volume} )
def test_no_errors(repo, tag, full): test_dag = DAG( "test-dag", default_args={}, description="testing", start_date=datetime(2020, 1, 1), schedule_interval=None, ) k = basic_kubernetes_pod_operator( task_id="task1", dag=test_dag, role="a_role", repo_name=repo, release=tag, full_image_name=full, ) assert isinstance(k, KubernetesPodOperator) assert k.task_id == "task1" assert k.name == "task1" assert k.image == f"{ecr_base_path}my-repo:v0.0.0" assert k.namespace == "airflow" assert k.annotations == {"iam.amazonaws.com/role": "a_role"} assert k.is_delete_operator_pod is True assert k.cluster_context == "aws" assert k.config_file == "/usr/local/airflow/dags/.kube/config" assert k.in_cluster is False assert k.get_logs is True assert k.security_context == { "allowPrivilegeEscalation": False, "runAsNonRoot": True, "privileged": False, } expected_env = [ V1EnvVar( name="AWS_METADATA_SERVICE_TIMEOUT", value="60", value_from=None, ), V1EnvVar( name="AWS_METADATA_SERVICE_NUM_ATTEMPTS", value="5", value_from=None, ), V1EnvVar( name="AWS_DEFAULT_REGION", value="eu-west-1", value_from=None, ), ] assert k.env_vars == expected_env
def query_op_vol(n, vol): return dsl.ContainerOp( name=n, image="hanjoo8821/jdbc-tibero:null", container_kwargs={ 'env': [V1EnvVar('id', 'hanjoo'), V1EnvVar('pw', '1010')] }, command=['sh', '-c'], arguments=[ 'java -cp "./JdbcAgentEx-null.jar:./tibero6jdbc-1.0.jar" local.hanjoo.JdbcAgent "$id" "$pw" | tee /data/output' ], pvolumes={"/data": vol})
def pipelineparams_pipeline(tag: str = 'latest', sleep_ms: int = 10): echo = dsl.Sidecar( name='echo', image='hashicorp/http-echo:%s' % tag, args=['-text="hello world"'], ) op1 = dsl.ContainerOp( name='download', image='busybox:%s' % tag, command=['sh', '-c'], arguments=[ 'sleep %s; wget localhost:5678 -O /tmp/results.txt' % sleep_ms ], sidecars=[echo], file_outputs={'downloaded': '/tmp/results.txt'}) op2 = dsl.ContainerOp(name='echo', image='library/bash', command=['sh', '-c'], arguments=['echo $MSG %s' % op1.output]) op2.container.add_env_variable( V1EnvVar(name='MSG', value='pipelineParams: '))
def train_op(docker_image_path: str, pvolume: PipelineVolume, dataset_root_dir: str, dataset_name: str, num_gpu: int, gpu_list: str): # num_gpus = len(gpu_list.value.split(',')) # gpu_list = "" # for i in range(num_gpu): # gpu_list += (str(i) + ",") env_var = V1EnvVar(name="CUDA_VISIBLE_DEVICES", value="0,1") return dsl.ContainerOp( name="train", image=docker_image_path, command=["python", f"{PROJ_PATH}/devolopment/train.py"], arguments=[ "--dataset_root_dir", dataset_root_dir, "--dataset_name", dataset_name ], file_outputs={ 'output': f'{PROJ_PATH}/devolopment/output.txt' }, container_kwargs={ 'image_pull_policy': 'IfNotPresent' }, pvolumes={ "/workspace": pvolume }).container.set_gpu_limit(2).add_env_variable(env_var)
def test_basic(self): """Test basic usage.""" param1 = PipelineParam('param1') param2 = PipelineParam('param2') op1 = (ContainerOp( name='op1', image='image', arguments=['%s hello %s %s' % (param1, param2, param1)], sidecars=[Sidecar(name='sidecar0', image='image0')], container_kwargs={ 'env': [V1EnvVar(name='env1', value='value1')] }, file_outputs={ 'out1': '/tmp/b' }).add_sidecar(Sidecar(name='sidecar1', image='image1')).add_sidecar( Sidecar(name='sidecar2', image='image2'))) self.assertCountEqual([x.name for x in op1.inputs], ['param1', 'param2']) self.assertCountEqual(list(op1.outputs.keys()), ['out1']) self.assertCountEqual([x.op_name for x in op1.outputs.values()], [op1.name]) self.assertEqual(op1.output.name, 'out1') self.assertCountEqual([sidecar.name for sidecar in op1.sidecars], ['sidecar0', 'sidecar1', 'sidecar2']) self.assertCountEqual([sidecar.image for sidecar in op1.sidecars], ['image0', 'image1', 'image2']) self.assertCountEqual([env.name for env in op1.container.env], ['env1'])
def echo_op(port_number): return dsl.ContainerOp( name='echo', image='busybox', command=['sh', '-c'], arguments=['echo "Got scheduled"'] ).container.add_env_variable(V1EnvVar(name='PORT', value=port_number))
def mnist_pipeline(): ENV_MANAGE_URL = V1EnvVar(name='MANAGE_URL', value='http://220.116.228.93:8088/send') data_0 = dsl.ContainerOp( name="load & preprocess data pipeline", image="byeongjokim/mnist-pre-data:latest", ).set_display_name('collect & preprocess data')\ .apply(onprem.mount_pvc("data-pvc", volume_name="data", volume_mount_path="/data")) data_1 = dsl.ContainerOp( name="validate data pipeline", image="byeongjokim/mnist-val-data:latest", ).set_display_name('validate data').after(data_0)\ .apply(onprem.mount_pvc("data-pvc", volume_name="data", volume_mount_path="/data")) train_model = dsl.ContainerOp( name="train embedding model", image="byeongjokim/mnist-train-model:latest", ).set_display_name('train model').after(data_1)\ .apply(onprem.mount_pvc("data-pvc", volume_name="data", volume_mount_path="/data"))\ .apply(onprem.mount_pvc("train-model-pvc", volume_name="train-model", volume_mount_path="/model")) embedding = dsl.ContainerOp( name="embedding data using embedding model", image="byeongjokim/mnist-embedding:latest", ).set_display_name('embedding').after(train_model)\ .apply(onprem.mount_pvc("data-pvc", volume_name="data", volume_mount_path="/data"))\ .apply(onprem.mount_pvc("train-model-pvc", volume_name="train-model", volume_mount_path="/model")) train_faiss = dsl.ContainerOp( name="train faiss", image="byeongjokim/mnist-train-faiss:latest", ).set_display_name('train faiss').after(embedding)\ .apply(onprem.mount_pvc("data-pvc", volume_name="data", volume_mount_path="/data"))\ .apply(onprem.mount_pvc("train-model-pvc", volume_name="train-model", volume_mount_path="/model")) analysis = dsl.ContainerOp( name="analysis total", image="byeongjokim/mnist-analysis:latest", file_outputs={ "confusion_matrix": "/confusion_matrix.csv", "mlpipeline-ui-metadata": "/mlpipeline-ui-metadata.json", "accuracy": "/accuracy.json", "mlpipeline_metrics": "/mlpipeline-metrics.json" } ).add_env_variable(ENV_MANAGE_URL).set_display_name('analysis').after(train_faiss)\ .apply(onprem.mount_pvc("data-pvc", volume_name="data", volume_mount_path="/data"))\ .apply(onprem.mount_pvc("train-model-pvc", volume_name="train-model", volume_mount_path="/model")) baseline = 0.8 with dsl.Condition(analysis.outputs["accuracy"] > baseline) as check_deploy: deploy = dsl.ContainerOp( name="deploy mar", image="byeongjokim/mnist-deploy:latest", ).add_env_variable(ENV_MANAGE_URL).set_display_name('deploy').after(analysis)\ .apply(onprem.mount_pvc("train-model-pvc", volume_name="train-model", volume_mount_path="/model"))\ .apply(onprem.mount_pvc("deploy-model-pvc", volume_name="deploy-model", volume_mount_path="/deploy-model"))
def run_notebook_op(op_name): notebook_op = NotebookOp(name=op_name, notebook='Untitled.ipynb', cos_endpoint=cos_endpoint, cos_bucket=cos_bucket, cos_directory=cos_directory, cos_dependencies_archive=cos_dependencies_archive, pipeline_outputs=outputs, pipeline_inputs=inputs, image=image) notebook_op.container.add_env_variable( V1EnvVar(name='AWS_ACCESS_KEY_ID', value=cos_username)) notebook_op.container.add_env_variable( V1EnvVar(name='AWS_SECRET_ACCESS_KEY', value=cos_password)) notebook_op.container.set_image_pull_policy('Always') return notebook_op
def test_env_vars(): test_dag = DAG( "test-dag", default_args={}, description="testing", start_date=datetime(2020, 1, 1), schedule_interval=None, ) k = basic_kubernetes_pod_operator( task_id="task1", dag=test_dag, role="a_role", repo_name="my_repo", release="v0.0.0", env_vars={ "AWS_DEFAULT_REGION": "us-east-1", "EXTRA": "test" }, ) expected_env = [ V1EnvVar( name="AWS_DEFAULT_REGION", value="us-east-1", value_from=None, ), V1EnvVar( name="EXTRA", value="test", value_from=None, ), V1EnvVar( name="AWS_METADATA_SERVICE_TIMEOUT", value="60", value_from=None, ), V1EnvVar( name="AWS_METADATA_SERVICE_NUM_ATTEMPTS", value="5", value_from=None, ), ] assert k.env_vars == expected_env
def transformer(containerOp): containerOp.arguments = ['/scripts/pipelineWrapper.py', 'Privacy', 'python'] + containerOp.arguments # shouldn't hard code this experiment name containerOp.container.set_image_pull_policy("Always") containerOp.add_volume( k8s_client.V1Volume( name='azure', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource( claim_name='azure-managed-disk') ) ).add_volume_mount(k8s_client.V1VolumeMount(mount_path='/mnt/azure', name='azure')) containerOp.container.add_env_variable(V1EnvVar(name='AZ_NAME', value=ws.name))\ .add_env_variable(V1EnvVar(name='AZ_SUBSCRIPTION_ID', value=ws.subscription_id))\ .add_env_variable(V1EnvVar(name='AZ_RESOURCE_GROUP', value=ws.resource_group)) containerOp.apply(use_azure_secret('azcreds')) return containerOp
def notebook_pipeline(): """A pipeline to run a Jupyter notebook with elyra-ai/kfp-notebook and Papermill.""" from kfp_notebook.pipeline import NotebookOp notebook_op = NotebookOp(name="${name}", notebook="${notebook}", cos_endpoint="${cos_endpoint}", cos_bucket="${cos_bucket}", cos_directory="${cos_directory}", cos_dependencies_archive="${cos_dependencies_archive}", requirements_url="${requirements_url}", image="${image}") from kubernetes.client.models import V1EnvVar notebook_op.container.add_env_variable(V1EnvVar(name='AWS_ACCESS_KEY_ID', value="${cos_username}")) notebook_op.container.add_env_variable(V1EnvVar(name='AWS_SECRET_ACCESS_KEY', value="${cos_password}")) notebook_op.container.set_image_pull_policy('Always')
def titanic_pipline(token, project_id, dataset, version='v1', claimname="titanic-test-pvc"): pipelineConfig = kfp.dsl.PipelineConf() pipelineConfig.set_image_pull_policy("Always") input_volumes = json.dumps([f"{claimname}@dataset://{dataset}/{version}"]) storage_op = ContainerOp( name="get_dataset", image="ocdr/dkubepl:storage_v3", command=[ "dkubepl", "storage", "--token", token, "--namespace", "kubeflow", "--input_volumes", input_volumes, ], ) predict_op = ContainerOp( name="predict", image="ocdr/titanic_submission", command=["python", "predict.py"], pvolumes={"/titanic-test/": kfp.dsl.PipelineVolume(pvc=claimname)}, file_outputs={"output": "/tmp/prediction.csv"}, ) predict_op.after(storage_op) predictions = kfp.dsl.InputArgumentPath(predict_op.outputs["output"]) submit_op = ContainerOp( name="submit", image="ocdr/d3project_eval", command=[ "python", "submit.py", kfp.dsl.RUN_ID_PLACEHOLDER, "-t", token, predictions, ], file_outputs={ "mlpipeline-ui-metadata": "/metadata.json", "results": "/results", }, ) env_var = V1EnvVar(name="DKUBE_PROJECT_ID", value=project_id) submit_op.add_env_variable(env_var)
def _serialize_pod_spec(self, settings: SerializationSettings) -> Dict[str, Any]: containers = self.task_config.pod_spec.containers primary_exists = False for container in containers: if container.name == self.task_config.primary_container_name: primary_exists = True break if not primary_exists: # insert a placeholder primary container if it is not defined in the pod spec. containers.append( V1Container(name=self.task_config.primary_container_name)) final_containers = [] for container in containers: # In the case of the primary container, we overwrite specific container attributes with the default values # used in an SDK runnable task. if container.name == self.task_config.primary_container_name: sdk_default_container = super().get_container(settings) container.image = sdk_default_container.image # clear existing commands container.command = sdk_default_container.command # also clear existing args container.args = sdk_default_container.args limits, requests = {}, {} for resource in sdk_default_container.resources.limits: limits[_core_task.Resources.ResourceName.Name( resource.name).lower()] = resource.value for resource in sdk_default_container.resources.requests: requests[_core_task.Resources.ResourceName.Name( resource.name).lower()] = resource.value resource_requirements = V1ResourceRequirements( limits=limits, requests=requests) if len(limits) > 0 or len(requests) > 0: # Important! Only copy over resource requirements if they are non-empty. container.resources = resource_requirements container.env = [ V1EnvVar(name=key, value=val) for key, val in sdk_default_container.env.items() ] final_containers.append(container) self.task_config._pod_spec.containers = final_containers return ApiClient().sanitize_for_serialization( self.task_config.pod_spec)
def test_extract_pipelineparams_from_any(self): """Test extract_pipeleineparams.""" p1 = PipelineParam(name='param1', op_name='op1') p2 = PipelineParam(name='param2') p3 = PipelineParam(name='param3', value='value3') stuff_chars = ' between ' payload = str(p1) + stuff_chars + str(p2) + stuff_chars + str(p3) container = V1Container(name=p1, image=p2, env=[V1EnvVar(name="foo", value=payload)]) params = extract_pipelineparams_from_any(container) self.assertListEqual(sorted([p1, p2, p3]), sorted(params))
def _build_container_op(self, overrides=dict()): artifacts_volume = k8s_client.V1Volume( name="artifacts", empty_dir=k8s_client.V1EmptyDirVolumeSource()) op = dsl.ContainerOp(name=f"{self.pipeline_name}-{self.op_name}", image=self.container_image_url, command=self.container_command, arguments=self.container_args, file_outputs=self.output_files, output_artifact_paths=self.output_artifact_paths) # Mount an empty direcotry for us to write output to self._mount_empty_dir(op, "artifacts", self.kfp_artifact_path) # Apply the GCP Auth secret if not self.gcp_auth_secret is None: op.apply(use_gcp_secret(self.gcp_auth_secret)) # Apply other secrets for secret_name in self.secrets: mount_path = self.secrets[secret_name] op.apply(self._bind_secret(secret_name, mount_path)) logging.info(f"Build Container {self.pipeline_name}.{self.op_name}") # All my parameters for name, value in self.data.items(): logging.info(f"\tContainer.ENV: {name}={value}") op.container.add_env_variable(V1EnvVar(name=name, value=str(value))) for name, value in overrides.items(): logging.info(f"\tContainer.ENV: {name}={value} (override)") op.container.add_env_variable(V1EnvVar(name=name, value=str(value))) return op
def my_pipeline(param1, param2): op1 = ( ContainerOp( name='op1', image='image', arguments=['%s hello %s %s' % (param1, param2, param1)], init_containers=[ UserContainer( name='initcontainer0', image='initimage0') ], sidecars=[Sidecar(name='sidecar0', image='image0')], container_kwargs={ 'env': [V1EnvVar(name='env1', value='value1')] }, file_outputs={ 'out1': '/tmp/b' }).add_init_container( UserContainer( name='initcontainer1', image='initimage1')).add_init_container( UserContainer( name='initcontainer2', image='initimage2')).add_sidecar( Sidecar( name='sidecar1', image='image1')).add_sidecar( Sidecar( name='sidecar2', image='image2'))) self.assertCountEqual([x.name for x in op1.inputs], ['param1', 'param2']) self.assertCountEqual(list(op1.outputs.keys()), ['out1']) self.assertCountEqual([x.op_name for x in op1.outputs.values()], [op1.name]) self.assertEqual(op1.output.name, 'out1') self.assertCountEqual( [init_container.name for init_container in op1.init_containers], ['initcontainer0', 'initcontainer1', 'initcontainer2']) self.assertCountEqual([ init_container.image for init_container in op1.init_containers ], ['initimage0', 'initimage1', 'initimage2']) self.assertCountEqual([sidecar.name for sidecar in op1.sidecars], ['sidecar0', 'sidecar1', 'sidecar2']) self.assertCountEqual([sidecar.image for sidecar in op1.sidecars], ['image0', 'image1', 'image2']) self.assertCountEqual([env.name for env in op1.container.env], ['env1'])
def pipeline_gcs(): GCSCredentialFileName = "user-gcp-sa.json" GCSCredentialVolumeName = "user-gcp-sa" GCSCredentialVolumeMountPath = "/var/secrets/" GCSCredentialEnvKey = "GOOGLE_APPLICATION_CREDENTIALS" GCSCredentialFilePath = os.path.join(GCSCredentialVolumeMountPath, GCSCredentialFileName) secret_name = 'user-gcp-sa' dsl.ContainerOp( name='mnist-gcs', image='kangwoo/kfp-mnist-storage:0.0.1', arguments=['--model', 'gs://kfp-bucket/kfp/mnist/model'] ).add_volume(V1Volume(name=GCSCredentialVolumeName, secret=V1SecretVolumeSource(secret_name=secret_name))) \ .add_volume_mount(V1VolumeMount(name=GCSCredentialVolumeName, mount_path=GCSCredentialVolumeMountPath)) \ .add_env_variable(V1EnvVar(name=GCSCredentialEnvKey, value=GCSCredentialFilePath))
def lightweight_tensorboard_pipeline(tb_log_dir): s3_endpoint = 'minio-service.kubeflow.svc.cluster.local:9000' minio_endpoint = "http://" + s3_endpoint minio_username = "******" minio_key = "minio123" minio_region = "us-east-1" train_op(tb_log_dir).add_env_variable(V1EnvVar(name='S3_ENDPOINT', value=s3_endpoint)) \ .add_env_variable(V1EnvVar(name='AWS_ENDPOINT_URL', value=minio_endpoint)) \ .add_env_variable(V1EnvVar(name='AWS_ACCESS_KEY_ID', value=minio_username)) \ .add_env_variable(V1EnvVar(name='AWS_SECRET_ACCESS_KEY', value=minio_key)) \ .add_env_variable(V1EnvVar(name='AWS_REGION', value=minio_region)) \ .add_env_variable(V1EnvVar(name='S3_USE_HTTPS', value='0')) \ .add_env_variable(V1EnvVar(name='S3_VERIFY_SSL', value='0'))
def mnist_pipeline(learning_rate, dropout_rate, checkpoint_dir, model_version, saved_model_dir, tensorboard_log): exit_task = echo_op("Done!") with dsl.ExitHandler(exit_task): vop = dsl.VolumeOp(name="mnist_model_volume", resource_name="mnist_model", storage_class="nfs-client", modes=dsl.VOLUME_MODE_RWM, size="10Gi") mnist = dsl.ContainerOp( name='Mnist', image= 'kubeflow-registry.default.svc.cluster.local:30000/katib-job:B67AEB5C', command=['python', '/app/mnist.py'], arguments=[ "--learning_rate", learning_rate, "--dropout_rate", dropout_rate, "--checkpoint_dir", checkpoint_dir, "--model_version", model_version, "--saved_model_dir", saved_model_dir, "--tensorboard_log", tensorboard_log ], pvolumes={"/result": vop.volume}, output_artifact_paths={ 'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json' }, container_kwargs={ 'env': [ V1EnvVar('S3_ENDPOINT', 'minio-service.kubeflow.svc.cluster.local:9000'), V1EnvVar( 'AWS_ENDPOINT_URL', 'http://minio-service.kubeflow.svc.cluster.local:9000' ), V1EnvVar('AWS_ACCESS_KEY_ID', 'minio'), V1EnvVar('AWS_SECRET_ACCESS_KEY', 'minio123'), V1EnvVar('AWS_REGION', 'us-east-1'), V1EnvVar('S3_USE_HTTPS', '0'), V1EnvVar('S3_VERIFY_SSL', '0'), ] }) result = dsl.ContainerOp(name='list_list', image='library/bash:4.4.23', command=['ls', '-R', '/result'], pvolumes={"/result": mnist.pvolume}) mnist.after(vop) result.after(mnist)
def pipelineparams_pipeline(tag: str = 'latest', sleep_ms: int = 10): echo = dsl.Sidecar( name='echo', image='hashicorp/http-echo:%s' % tag, args=['-text="hello world"'], ) op1 = components.load_component_from_text(""" name: download description: download inputs: - {name: sleep_ms, type: Integer} outputs: - {name: data, type: String} implementation: container: image: busy:placeholder command: - sh - -c args: - | sleep $0; wget localhost:5678 -O $1 - {inputValue: sleep_ms} - {outputPath: data} """)(sleep_ms) op1.container.image = "busy:%s" % tag op2 = components.load_component_from_text(""" name: echo description: echo inputs: - {name: message, type: String} implementation: container: image: library/bash command: - sh - -c args: - | echo $MSG $0 - {inputValue: message} """)(op1.output) op2.container.add_env_variable(V1EnvVar(name='MSG', value='pipelineParams: '))
def with_env(self, variable_name, value) -> Optional["HmlContainerOp"]: """ Bind an environment variable with the name `variable_name` and `value` specified Args: variable_name (str): The name of the environment variable value (str): The value to bind to the variable Returns: A reference to the current `HmlContainerOp` (self) """ # # Update our current environment, so everything works in local development # os.environ[variable_name] = value # logging.info(f"Binding: ${variable_name} = {value}") self.op.container.add_env_variable(V1EnvVar(name=variable_name, value=str(value))) return self
def tensorboard_pipeline(tb_log_dir): s3_endpoint = 'minio-service.kubeflow.svc.cluster.local:9000' minio_endpoint = "http://" + s3_endpoint minio_username = "******" minio_key = "minio123" minio_region = "us-east-1" dsl.ContainerOp( name='tensorboard', image='kangwoo/kfp-tensorboard:0.0.1', arguments=['--tb_log_dir', tb_log_dir], output_artifact_paths={'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'} ).add_env_variable(V1EnvVar(name='S3_ENDPOINT', value=s3_endpoint)) \ .add_env_variable(V1EnvVar(name='AWS_ENDPOINT_URL', value=minio_endpoint)) \ .add_env_variable(V1EnvVar(name='AWS_ACCESS_KEY_ID', value=minio_username)) \ .add_env_variable(V1EnvVar(name='AWS_SECRET_ACCESS_KEY', value=minio_key)) \ .add_env_variable(V1EnvVar(name='AWS_REGION', value=minio_region)) \ .add_env_variable(V1EnvVar(name='S3_USE_HTTPS', value='0')) \ .add_env_variable(V1EnvVar(name='S3_VERIFY_SSL', value='0'))
def pipeline_s3(): secret_name = "kfp-aws-secret" s3_endpoint = 'minio-service.kubeflow.svc.cluster.local:9000' minio_endpoint = "http://" + s3_endpoint minio_region = "us-east-1" dsl.ContainerOp( name='mnist-s3', image='kangwoo/kfp-mnist-storage:0.0.1', arguments=['--model', 's3://tensorflow/kfp/mnist/model'] ).add_env_variable(V1EnvVar(name='S3_ENDPOINT', value=s3_endpoint)) \ .add_env_variable(V1EnvVar(name='AWS_ENDPOINT_URL', value=minio_endpoint)) \ .add_env_variable(V1EnvVar(name='AWS_ACCESS_KEY_ID', value_from=V1EnvVarSource( secret_key_ref=V1SecretKeySelector(name=secret_name, key='AWS_ACCESS_KEY_ID')))) \ .add_env_variable(V1EnvVar(name='AWS_SECRET_ACCESS_KEY', value_from=V1EnvVarSource(secret_key_ref=V1SecretKeySelector(name=secret_name, key='AWS_SECRET_ACCESS_KEY')))) \ .add_env_variable(V1EnvVar(name='AWS_REGION', value=minio_region)) \ .add_env_variable(V1EnvVar(name='S3_USE_HTTPS', value='0')) \ .add_env_variable(V1EnvVar(name='S3_VERIFY_SSL', value='0'))
def make_pod( name, cmd, port, image, image_pull_policy, image_pull_secrets=None, node_selector=None, run_as_uid=None, run_as_gid=None, fs_gid=None, supplemental_gids=None, run_privileged=False, allow_privilege_escalation=True, env=None, working_dir=None, volumes=None, volume_mounts=None, labels=None, annotations=None, cpu_limit=None, cpu_guarantee=None, mem_limit=None, mem_guarantee=None, extra_resource_limits=None, extra_resource_guarantees=None, lifecycle_hooks=None, init_containers=None, service_account=None, extra_container_config=None, extra_pod_config=None, extra_containers=None, scheduler_name=None, tolerations=None, node_affinity_preferred=None, node_affinity_required=None, pod_affinity_preferred=None, pod_affinity_required=None, pod_anti_affinity_preferred=None, pod_anti_affinity_required=None, priority_class_name=None, logger=None, ): """ Make a k8s pod specification for running a user notebook. Parameters ---------- name: Name of pod. Must be unique within the namespace the object is going to be created in. Must be a valid DNS label. image: Image specification - usually a image name and tag in the form of image_name:tag. Same thing you would use with docker commandline arguments image_pull_policy: Image pull policy - one of 'Always', 'IfNotPresent' or 'Never'. Decides when kubernetes will check for a newer version of image and pull it when running a pod. image_pull_secrets: Image pull secrets - a list of references to Kubernetes Secret resources with credentials to pull images from image registries. This list can either have strings in it or objects with the string value nested under a name field. port: Port the notebook server is going to be listening on cmd: The command used to execute the singleuser server. node_selector: Dictionary Selector to match nodes where to launch the Pods run_as_uid: The UID used to run single-user pods. The default is to run as the user specified in the Dockerfile, if this is set to None. run_as_gid: The GID used to run single-user pods. The default is to run as the primary group of the user specified in the Dockerfile, if this is set to None. Setting this parameter requires that *feature-gate* **RunAsGroup** be enabled, otherwise the effective GID of the pod will be 0 (root). In addition, not setting `run_as_gid` once feature-gate RunAsGroup is enabled will also result in an effective GID of 0 (root). fs_gid The gid that will own any fresh volumes mounted into this pod, if using volume types that support this (such as GCE). This should be a group that the uid the process is running as should be a member of, so that it can read / write to the volumes mounted. supplemental_gids: A list of GIDs that should be set as additional supplemental groups to the user that the container runs as. You may have to set this if you are deploying to an environment with RBAC/SCC enforced and pods run with a 'restricted' SCC which results in the image being run as an assigned user ID. The supplemental group IDs would need to include the corresponding group ID of the user ID the image normally would run as. The image must setup all directories/files any application needs access to, as group writable. run_privileged: Whether the container should be run in privileged mode. allow_privilege_escalation: Controls whether a process can gain more privileges than its parent process. env: Dictionary of environment variables. volumes: List of dictionaries containing the volumes of various types this pod will be using. See k8s documentation about volumes on how to specify these volume_mounts: List of dictionaries mapping paths in the container and the volume( specified in volumes) that should be mounted on them. See the k8s documentaiton for more details working_dir: String specifying the working directory for the notebook container labels: Labels to add to the spawned pod. annotations: Annotations to add to the spawned pod. cpu_limit: Float specifying the max number of CPU cores the user's pod is allowed to use. cpu_guarentee: Float specifying the max number of CPU cores the user's pod is guaranteed to have access to, by the scheduler. mem_limit: String specifying the max amount of RAM the user's pod is allowed to use. String instead of float/int since common suffixes are allowed mem_guarantee: String specifying the max amount of RAM the user's pod is guaranteed to have access to. String ins loat/int since common suffixes are allowed lifecycle_hooks: Dictionary of lifecycle hooks init_containers: List of initialization containers belonging to the pod. service_account: Service account to mount on the pod. None disables mounting extra_container_config: Extra configuration (e.g. envFrom) for notebook container which is not covered by parameters above. extra_pod_config: Extra configuration (e.g. tolerations) for pod which is not covered by parameters above. extra_containers: Extra containers besides notebook container. Used for some housekeeping jobs (e.g. crontab). scheduler_name: The pod's scheduler explicitly named. tolerations: Tolerations can allow a pod to schedule or execute on a tainted node. To learn more about pod tolerations, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/. Pass this field an array of "Toleration" objects.* * https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#nodeselectorterm-v1-core node_affinity_preferred: Affinities describe where pods prefer or require to be scheduled, they may prefer or require a node to have a certain label or be in proximity / remoteness to another pod. To learn more visit https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ Pass this field an array of "PreferredSchedulingTerm" objects.* * https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#preferredschedulingterm-v1-core node_affinity_required: Affinities describe where pods prefer or require to be scheduled, they may prefer or require a node to have a certain label or be in proximity / remoteness to another pod. To learn more visit https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ Pass this field an array of "NodeSelectorTerm" objects.* * https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#nodeselectorterm-v1-core pod_affinity_preferred: Affinities describe where pods prefer or require to be scheduled, they may prefer or require a node to have a certain label or be in proximity / remoteness to another pod. To learn more visit https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ Pass this field an array of "WeightedPodAffinityTerm" objects.* * https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#weightedpodaffinityterm-v1-core pod_affinity_required: Affinities describe where pods prefer or require to be scheduled, they may prefer or require a node to have a certain label or be in proximity / remoteness to another pod. To learn more visit https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ Pass this field an array of "PodAffinityTerm" objects.* * https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#podaffinityterm-v1-core pod_anti_affinity_preferred: Affinities describe where pods prefer or require to be scheduled, they may prefer or require a node to have a certain label or be in proximity / remoteness to another pod. To learn more visit https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ Pass this field an array of "WeightedPodAffinityTerm" objects.* * https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#weightedpodaffinityterm-v1-core pod_anti_affinity_required: Affinities describe where pods prefer or require to be scheduled, they may prefer or require a node to have a certain label or be in proximity / remoteness to another pod. To learn more visit https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ Pass this field an array of "PodAffinityTerm" objects.* * https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#podaffinityterm-v1-core priority_class_name: The name of the PriorityClass to be assigned the pod. This feature is Beta available in K8s 1.11. """ pod = V1Pod() pod.kind = "Pod" pod.api_version = "v1" pod.metadata = V1ObjectMeta(name=name, labels=(labels or {}).copy(), annotations=(annotations or {}).copy()) pod.spec = V1PodSpec(containers=[]) pod.spec.restart_policy = 'OnFailure' if image_pull_secrets is not None: # image_pull_secrets as received by the make_pod function should always # be a list, but it is allowed to have "a-string" elements or {"name": # "a-string"} elements. pod.spec.image_pull_secrets = [ V1LocalObjectReference(name=secret_ref) if type(secret_ref) == str else get_k8s_model(V1LocalObjectReference, secret_ref) for secret_ref in image_pull_secrets ] if node_selector: pod.spec.node_selector = node_selector if lifecycle_hooks: lifecycle_hooks = get_k8s_model(V1Lifecycle, lifecycle_hooks) # There are security contexts both on the Pod level or the Container level. # The security settings that you specify for a Pod apply to all Containers # in the Pod, but settings on the container level can override them. # # We configure the pod to be spawned on the container level unless the # option is only available on the pod level, such as for those relating to # the volumes as compared to the running user of the container. Volumes # belong to the pod and are only mounted by containers after all. # # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ # ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.16/#securitycontext-v1-core (container) # ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.16/#podsecuritycontext-v1-core (pod) pod_security_context = V1PodSecurityContext() if fs_gid is not None: pod_security_context.fs_group = int(fs_gid) if supplemental_gids is not None and supplemental_gids: pod_security_context.supplemental_groups = [ int(gid) for gid in supplemental_gids ] # Only clutter pod spec with actual content if not all([e is None for e in pod_security_context.to_dict().values()]): pod.spec.security_context = pod_security_context container_security_context = V1SecurityContext() if run_as_uid is not None: container_security_context.run_as_user = int(run_as_uid) if run_as_gid is not None: container_security_context.run_as_group = int(run_as_gid) if run_privileged: container_security_context.privileged = True if not allow_privilege_escalation: container_security_context.allow_privilege_escalation = False # Only clutter container spec with actual content if all([e is None for e in container_security_context.to_dict().values()]): container_security_context = None # Transform a dict into valid Kubernetes EnvVar Python representations. This # representation shall always have a "name" field as well as either a # "value" field or "value_from" field. For examples see the # test_make_pod_with_env function. prepared_env = [] for k, v in (env or {}).items(): if type(v) == dict: if not "name" in v: v["name"] = k prepared_env.append(get_k8s_model(V1EnvVar, v)) else: prepared_env.append(V1EnvVar(name=k, value=v)) notebook_container = V1Container( name='notebook', image=image, working_dir=working_dir, ports=[V1ContainerPort(name='notebook-port', container_port=port)], env=prepared_env, args=cmd, image_pull_policy=image_pull_policy, lifecycle=lifecycle_hooks, resources=V1ResourceRequirements(), volume_mounts=[ get_k8s_model(V1VolumeMount, obj) for obj in (volume_mounts or []) ], security_context=container_security_context, ) if service_account is None: # This makes sure that we don't accidentally give access to the whole # kubernetes API to the users in the spawned pods. pod.spec.automount_service_account_token = False else: pod.spec.service_account_name = service_account notebook_container.resources.requests = {} if cpu_guarantee: notebook_container.resources.requests['cpu'] = cpu_guarantee if mem_guarantee: notebook_container.resources.requests['memory'] = mem_guarantee if extra_resource_guarantees: notebook_container.resources.requests.update(extra_resource_guarantees) notebook_container.resources.limits = {} if cpu_limit: notebook_container.resources.limits['cpu'] = cpu_limit if mem_limit: notebook_container.resources.limits['memory'] = mem_limit if extra_resource_limits: notebook_container.resources.limits.update(extra_resource_limits) if extra_container_config: notebook_container = update_k8s_model( target=notebook_container, changes=extra_container_config, logger=logger, target_name="notebook_container", changes_name="extra_container_config", ) pod.spec.containers.append(notebook_container) if extra_containers: pod.spec.containers.extend( [get_k8s_model(V1Container, obj) for obj in extra_containers]) if tolerations: pod.spec.tolerations = [ get_k8s_model(V1Toleration, obj) for obj in tolerations ] if init_containers: pod.spec.init_containers = [ get_k8s_model(V1Container, obj) for obj in init_containers ] if volumes: pod.spec.volumes = [get_k8s_model(V1Volume, obj) for obj in volumes] else: # Keep behaving exactly like before by not cleaning up generated pod # spec by setting the volumes field even though it is an empty list. pod.spec.volumes = [] if scheduler_name: pod.spec.scheduler_name = scheduler_name node_affinity = None if node_affinity_preferred or node_affinity_required: node_selector = None if node_affinity_required: node_selector = V1NodeSelector(node_selector_terms=[ get_k8s_model(V1NodeSelectorTerm, obj) for obj in node_affinity_required ], ) preferred_scheduling_terms = None if node_affinity_preferred: preferred_scheduling_terms = [ get_k8s_model(V1PreferredSchedulingTerm, obj) for obj in node_affinity_preferred ] node_affinity = V1NodeAffinity( preferred_during_scheduling_ignored_during_execution= preferred_scheduling_terms, required_during_scheduling_ignored_during_execution=node_selector, ) pod_affinity = None if pod_affinity_preferred or pod_affinity_required: weighted_pod_affinity_terms = None if pod_affinity_preferred: weighted_pod_affinity_terms = [ get_k8s_model(V1WeightedPodAffinityTerm, obj) for obj in pod_affinity_preferred ] pod_affinity_terms = None if pod_affinity_required: pod_affinity_terms = [ get_k8s_model(V1PodAffinityTerm, obj) for obj in pod_affinity_required ] pod_affinity = V1PodAffinity( preferred_during_scheduling_ignored_during_execution= weighted_pod_affinity_terms, required_during_scheduling_ignored_during_execution= pod_affinity_terms, ) pod_anti_affinity = None if pod_anti_affinity_preferred or pod_anti_affinity_required: weighted_pod_affinity_terms = None if pod_anti_affinity_preferred: weighted_pod_affinity_terms = [ get_k8s_model(V1WeightedPodAffinityTerm, obj) for obj in pod_anti_affinity_preferred ] pod_affinity_terms = None if pod_anti_affinity_required: pod_affinity_terms = [ get_k8s_model(V1PodAffinityTerm, obj) for obj in pod_anti_affinity_required ] pod_anti_affinity = V1PodAffinity( preferred_during_scheduling_ignored_during_execution= weighted_pod_affinity_terms, required_during_scheduling_ignored_during_execution= pod_affinity_terms, ) affinity = None if (node_affinity or pod_affinity or pod_anti_affinity): affinity = V1Affinity( node_affinity=node_affinity, pod_affinity=pod_affinity, pod_anti_affinity=pod_anti_affinity, ) if affinity: pod.spec.affinity = affinity if priority_class_name: pod.spec.priority_class_name = priority_class_name if extra_pod_config: pod.spec = update_k8s_model( target=pod.spec, changes=extra_pod_config, logger=logger, target_name="pod.spec", changes_name="extra_pod_config", ) return pod
def __init__(self, pipeline_name: str, experiment_name: str, notebook: str, cos_endpoint: str, cos_bucket: str, cos_directory: str, cos_dependencies_archive: str, pipeline_version: Optional[str] = '', pipeline_source: Optional[str] = None, pipeline_outputs: Optional[List[str]] = None, pipeline_inputs: Optional[List[str]] = None, pipeline_envs: Optional[Dict[str, str]] = None, requirements_url: Optional[str] = None, bootstrap_script_url: Optional[str] = None, emptydir_volume_size: Optional[str] = None, cpu_request: Optional[str] = None, mem_request: Optional[str] = None, gpu_limit: Optional[str] = None, workflow_engine: Optional[str] = 'argo', **kwargs): """Create a new instance of ContainerOp. Args: pipeline_name: pipeline that this op belongs to experiment_name: the experiment where pipeline_name is executed notebook: name of the notebook that will be executed per this operation cos_endpoint: object storage endpoint e.g weaikish1.fyre.ibm.com:30442 cos_bucket: bucket to retrieve archive from cos_directory: name of the directory in the object storage bucket to pull cos_dependencies_archive: archive file name to get from object storage bucket e.g archive1.tar.gz pipeline_version: optional version identifier pipeline_source: pipeline source pipeline_outputs: comma delimited list of files produced by the notebook pipeline_inputs: comma delimited list of files to be consumed/are required by the notebook pipeline_envs: dictionary of environmental variables to set in the container prior to execution requirements_url: URL to a python requirements.txt file to be installed prior to running the notebook bootstrap_script_url: URL to a custom python bootstrap script to run emptydir_volume_size: Size(GB) of the volume to create for the workspace when using CRIO container runtime cpu_request: number of CPUs requested for the operation mem_request: memory requested for the operation (in Gi) gpu_limit: maximum number of GPUs allowed for the operation workflow_engine: Kubeflow workflow engine, defaults to 'argo' kwargs: additional key value pairs to pass e.g. name, image, sidecars & is_exit_handler. See Kubeflow pipelines ContainerOp definition for more parameters or how to use https://kubeflow-pipelines.readthedocs.io/en/latest/source/kfp.dsl.html#kfp.dsl.ContainerOp """ self.pipeline_name = pipeline_name self.pipeline_version = pipeline_version self.pipeline_source = pipeline_source self.experiment_name = experiment_name self.notebook = notebook self.notebook_name = os.path.basename(notebook) self.cos_endpoint = cos_endpoint self.cos_bucket = cos_bucket self.cos_directory = cos_directory self.cos_dependencies_archive = cos_dependencies_archive self.container_work_dir_root_path = "./" self.container_work_dir_name = "jupyter-work-dir/" self.container_work_dir = self.container_work_dir_root_path + self.container_work_dir_name self.bootstrap_script_url = bootstrap_script_url self.requirements_url = requirements_url self.pipeline_outputs = pipeline_outputs self.pipeline_inputs = pipeline_inputs self.pipeline_envs = pipeline_envs self.cpu_request = cpu_request self.mem_request = mem_request self.gpu_limit = gpu_limit argument_list = [] """ CRI-o support for kfp pipelines We need to attach an emptydir volume for each notebook that runs since CRI-o runtime does not allow us to write to the base image layer file system, only to volumes. """ self.emptydir_volume_name = "workspace" self.emptydir_volume_size = emptydir_volume_size self.python_user_lib_path = '' self.python_user_lib_path_target = '' self.python_pip_config_url = '' if self.emptydir_volume_size: self.container_work_dir_root_path = "/opt/app-root/src/" self.container_python_dir_name = "python3/" self.container_work_dir = self.container_work_dir_root_path + self.container_work_dir_name self.python_user_lib_path = self.container_work_dir + self.container_python_dir_name self.python_user_lib_path_target = '--target=' + self.python_user_lib_path self.python_pip_config_url = ELYRA_PIP_CONFIG_URL if not self.bootstrap_script_url: self.bootstrap_script_url = ELYRA_BOOTSTRAP_SCRIPT_URL if not self.requirements_url: self.requirements_url = ELYRA_REQUIREMENTS_URL if 'name' not in kwargs: raise TypeError("You need to provide a name for the operation.") elif not kwargs.get('name'): raise ValueError("You need to provide a name for the operation.") if 'image' not in kwargs: raise ValueError("You need to provide an image.") if not notebook: raise ValueError("You need to provide a notebook.") if 'arguments' not in kwargs: """ If no arguments are passed, we use our own. If ['arguments'] are set, we assume container's ENTRYPOINT is set and dependencies are installed NOTE: Images being pulled must have python3 available on PATH and cURL utility """ argument_list.append('mkdir -p {container_work_dir} && cd {container_work_dir} && ' 'curl -H "Cache-Control: no-cache" -L {bootscript_url} --output bootstrapper.py && ' 'curl -H "Cache-Control: no-cache" -L {reqs_url} --output requirements-elyra.txt && ' .format(container_work_dir=self.container_work_dir, bootscript_url=self.bootstrap_script_url, reqs_url=self.requirements_url) ) if self.emptydir_volume_size: argument_list.append('mkdir {container_python_dir} && cd {container_python_dir} && ' 'curl -H "Cache-Control: no-cache" -L {python_pip_config_url} ' '--output pip.conf && cd .. &&' .format(python_pip_config_url=self.python_pip_config_url, container_python_dir=self.container_python_dir_name) ) argument_list.append('python3 -m pip install {python_user_lib_path_target} packaging && ' 'python3 -m pip freeze > requirements-current.txt && ' 'python3 bootstrapper.py ' '--cos-endpoint {cos_endpoint} ' '--cos-bucket {cos_bucket} ' '--cos-directory "{cos_directory}" ' '--cos-dependencies-archive "{cos_dependencies_archive}" ' '--file "{notebook}" ' .format(cos_endpoint=self.cos_endpoint, cos_bucket=self.cos_bucket, cos_directory=self.cos_directory, cos_dependencies_archive=self.cos_dependencies_archive, notebook=self.notebook, python_user_lib_path_target=self.python_user_lib_path_target) ) if self.pipeline_inputs: inputs_str = self._artifact_list_to_str(self.pipeline_inputs) argument_list.append('--inputs "{}" '.format(inputs_str)) if self.pipeline_outputs: outputs_str = self._artifact_list_to_str(self.pipeline_outputs) argument_list.append('--outputs "{}" '.format(outputs_str)) if self.emptydir_volume_size: argument_list.append('--user-volume-path "{}" '.format(self.python_user_lib_path)) kwargs['command'] = ['sh', '-c'] kwargs['arguments'] = "".join(argument_list) super().__init__(**kwargs) # We must deal with the envs after the superclass initialization since these amend the # container attribute that isn't available until now. if self.pipeline_envs: for key, value in self.pipeline_envs.items(): # Convert dict entries to format kfp needs self.container.add_env_variable(V1EnvVar(name=key, value=value)) # If crio volume size is found then assume kubeflow pipelines environment is using CRI-o as # its container runtime if self.emptydir_volume_size: self.add_volume(V1Volume(empty_dir=V1EmptyDirVolumeSource( medium="", size_limit=self.emptydir_volume_size), name=self.emptydir_volume_name)) self.container.add_volume_mount(V1VolumeMount(mount_path=self.container_work_dir_root_path, name=self.emptydir_volume_name)) # Append to PYTHONPATH location of elyra dependencies in installed in Volume self.container.add_env_variable(V1EnvVar(name='PYTHONPATH', value=self.python_user_lib_path)) if self.cpu_request: self.container.set_cpu_request(cpu=str(cpu_request)) if self.mem_request: self.container.set_memory_request(memory=str(mem_request) + "G") if self.gpu_limit: gpu_vendor = self.pipeline_envs.get('GPU_VENDOR', 'nvidia') self.container.set_gpu_limit(gpu=str(gpu_limit), vendor=gpu_vendor) # Generate unique ELYRA_RUN_NAME value and expose it as an environment # variable in the container if workflow_engine and workflow_engine.lower() == 'argo': run_name_placeholder = '{{workflow.annotations.pipelines.kubeflow.org/run_name}}' self.container.add_env_variable(V1EnvVar(name='ELYRA_RUN_NAME', value=run_name_placeholder)) else: # For Tekton derive the value from the specified pod annotation annotation = 'pipelines.kubeflow.org/run_name' field_path = f"metadata.annotations['{annotation}']" self.container.add_env_variable(V1EnvVar(name='ELYRA_RUN_NAME', value_from=V1EnvVarSource( field_ref=V1ObjectFieldSelector(field_path=field_path)))) # Attach metadata to the pod # Node type (a static type for this op) self.add_pod_label('elyra/node-type', NotebookOp._normalize_label_value( 'notebook-script')) # Pipeline name self.add_pod_label('elyra/pipeline-name', NotebookOp._normalize_label_value(self.pipeline_name)) # Pipeline version self.add_pod_label('elyra/pipeline-version', NotebookOp._normalize_label_value(self.pipeline_version)) # Experiment name self.add_pod_label('elyra/experiment-name', NotebookOp._normalize_label_value(self.experiment_name)) # Pipeline node name self.add_pod_label('elyra/node-name', NotebookOp._normalize_label_value(kwargs.get('name'))) # Pipeline node file self.add_pod_annotation('elyra/node-file-name', self.notebook) # Identify the pipeline source, which can be a # pipeline file (mypipeline.pipeline), a Python # script or notebook that was submitted if self.pipeline_source is not None: self.add_pod_annotation('elyra/pipeline-source', self.pipeline_source)
def taxi_cab_classification( project, output='s3://mlpipeline/tfx/output', column_names='s3://mlpipeline/tfx/taxi-cab-classification/column-names.json', key_columns='trip_start_timestamp', train='s3://mlpipeline/tfx/taxi-cab-classification/train.csv', evaluation='s3://mlpipeline/tfx/taxi-cab-classification/eval.csv', mode='local', preprocess_module='s3://mlpipeline/tfx/taxi-cab-classification/preprocessing.py', learning_rate=0.1, hidden_layer_size='1500', steps=3000, analyze_slice_column='trip_start_hour' ): output_template = str(output) + '/{{workflow.uid}}/{{pod.name}}/data' target_lambda = """lambda x: (x['target'] > x['fare'] * 0.2)""" target_class_lambda = """lambda x: 1 if (x['target'] > x['fare'] * 0.2) else 0""" tf_server_name = 'taxi-cab-classification-model-{{workflow.uid}}' if platform == 'onprem': if storage == 'minio': data_preparation = dsl.ContainerOp( name="data_preparation", image="aiven86/minio_mc-git", command=["sh", "/bin/run.sh"], ).set_image_pull_policy('IfNotPresent') data_preparation.container.add_env_variable(V1EnvVar(name='GITPAHT', value=GITPAHT)) data_preparation.container.add_env_variable(V1EnvVar(name='GITDIR', value=GITDIR)) data_preparation.container.add_env_variable(V1EnvVar(name='MINIOPATH', value=MINIOPATH)) data_preparation.container.add_env_variable(V1EnvVar(name='DATAPATH', value=DATAPATH)) else: vop = dsl.VolumeOp( name="create_pvc", storage_class="rook-ceph-fs", resource_name="pipeline-pvc", modes=dsl.VOLUME_MODE_RWM, size="1Gi" ) data_preparation = dsl.ContainerOp( name="data_preparation", image="aiven86/git", command=["git", "clone", "https://github.com/kubeflow/pipelines.git", str(output) + "/pipelines"], ).apply(onprem.mount_pvc(vop.outputs["name"], 'local-storage', output)) data_preparation.after(vop) validation = dataflow_tf_data_validation_op( inference_data=train, validation_data=evaluation, column_names=column_names, key_columns=key_columns, gcp_project=project, run_mode=mode, validation_output=output_template, ) if platform == 'onprem': validation.after(data_preparation) preprocess = dataflow_tf_transform_op( training_data_file_pattern=train, evaluation_data_file_pattern=evaluation, schema=validation.outputs['schema'], gcp_project=project, run_mode=mode, preprocessing_module=preprocess_module, transformed_data_dir=output_template ) training = tf_train_op( transformed_data_dir=preprocess.output, schema=validation.outputs['schema'], learning_rate=learning_rate, hidden_layer_size=hidden_layer_size, steps=steps, target='tips', preprocessing_module=preprocess_module, training_output_dir=output_template ) analysis = dataflow_tf_model_analyze_op( model=training.output, evaluation_data=evaluation, schema=validation.outputs['schema'], gcp_project=project, run_mode=mode, slice_columns=analyze_slice_column, analysis_results_dir=output_template ) prediction = dataflow_tf_predict_op( data_file_pattern=evaluation, schema=validation.outputs['schema'], target_column='tips', model=training.output, run_mode=mode, gcp_project=project, predictions_dir=output_template ) cm = confusion_matrix_op( predictions=prediction.output, target_lambda=target_lambda, output_dir=output_template ) roc = roc_op( predictions_dir=prediction.output, target_lambda=target_class_lambda, output_dir=output_template ) if platform == 'GCP' or storage == 'minio': deploy = kubeflow_deploy_op( model_dir=str(training.output) + '/export/export', server_name=tf_server_name ) elif platform == 'onprem' and storage != 'minio': deploy = kubeflow_deploy_op( cluster_name=project, model_dir=str(training.output) + '/export/export', pvc_name=vop.outputs["name"], server_name=tf_server_name ) steps = [validation, preprocess, training, analysis, prediction, cm, roc, deploy] for step in steps: if platform == 'GCP': step.apply(gcp.use_gcp_secret('user-gcp-sa')) elif platform == 'onprem': if storage == 'minio': step.container.add_env_variable(V1EnvVar(name='AWS_ACCESS_KEY_ID', value=AWS_ACCESS_KEY_ID)) step.container.add_env_variable(V1EnvVar(name='AWS_SECRET_ACCESS_KEY', value=AWS_SECRET_ACCESS_KEY)) step.container.add_env_variable(V1EnvVar(name='AWS_REGION', value=AWS_REGION)) step.container.add_env_variable(V1EnvVar(name='S3_ENDPOINT', value=S3_ENDPOINT)) step.container.add_env_variable(V1EnvVar(name='S3_USE_HTTPS', value=S3_USE_HTTPS)) step.container.add_env_variable(V1EnvVar(name='S3_VERIFY_SSL', value=S3_VERIFY_SSL)) else: step.apply(onprem.mount_pvc(vop.outputs["name"], 'local-storage', output))