def auto_generated_pipeline(date_of_processing='02_06_2021', dir_mask_specie='Ponca_DV', dir_specie='Ponca_DV_loc', dir_years='forest_jEquihua_mar', file_mask_specie='poncamask.tif', file_specie='poncadav2', specie='pan_onca', vol_shared_volume='hostpath-pvc'): _kale_pvolumes_dict = OrderedDict() _kale_volume_step_names = [] _kale_volume_name_parameters = [] _kale_annotations = {} _kale_volume = _kfp_dsl.PipelineVolume(pvc=vol_shared_volume) _kale_pvolumes_dict['/shared_volume'] = _kale_volume _kale_volume_step_names.sort() _kale_volume_name_parameters.sort() _kale_downloadfroms3_task = _kale_downloadfroms3_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after() _kale_downloadfroms3_task.container.working_dir = "//shared_volume/kube_sipecam_playground/hsi/notebooks" _kale_downloadfroms3_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'downloadfroms3': '/downloadfroms3.html'}) _kale_downloadfroms3_task.output_artifact_paths.update( _kale_output_artifacts) _kale_downloadfroms3_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_downloadfroms3_task.dependent_names + _kale_volume_step_names) _kale_downloadfroms3_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_downloadfroms3_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_readdatainput_task = _kale_readdatainput_op(dir_mask_specie, dir_specie, file_mask_specie, file_specie)\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_downloadfroms3_task) _kale_readdatainput_task.container.working_dir = "//shared_volume/kube_sipecam_playground/hsi/notebooks" _kale_readdatainput_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'readdatainput': '/readdatainput.html'}) _kale_readdatainput_task.output_artifact_paths.update( _kale_output_artifacts) _kale_readdatainput_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_readdatainput_task.dependent_names + _kale_volume_step_names) _kale_readdatainput_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_readdatainput_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_reproject_task = _kale_reproject_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_readdatainput_task) _kale_reproject_task.container.working_dir = "//shared_volume/kube_sipecam_playground/hsi/notebooks" _kale_reproject_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'reproject': '/reproject.html'}) _kale_reproject_task.output_artifact_paths.update(_kale_output_artifacts) _kale_reproject_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_reproject_task.dependent_names + _kale_volume_step_names) _kale_reproject_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_reproject_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_createtestdata_task = _kale_createtestdata_op(dir_years)\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_reproject_task) _kale_createtestdata_task.container.working_dir = "//shared_volume/kube_sipecam_playground/hsi/notebooks" _kale_createtestdata_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'createtestdata': '/createtestdata.html'}) _kale_createtestdata_task.output_artifact_paths.update( _kale_output_artifacts) _kale_createtestdata_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_createtestdata_task.dependent_names + _kale_volume_step_names) _kale_createtestdata_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_createtestdata_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_maskandextract_task = _kale_maskandextract_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_createtestdata_task) _kale_maskandextract_task.container.working_dir = "//shared_volume/kube_sipecam_playground/hsi/notebooks" _kale_maskandextract_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'maskandextract': '/maskandextract.html'}) _kale_maskandextract_task.output_artifact_paths.update( _kale_output_artifacts) _kale_maskandextract_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_maskandextract_task.dependent_names + _kale_volume_step_names) _kale_maskandextract_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_maskandextract_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_bestmodel_task = _kale_bestmodel_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_maskandextract_task) _kale_bestmodel_task.container.working_dir = "//shared_volume/kube_sipecam_playground/hsi/notebooks" _kale_bestmodel_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'bestmodel': '/bestmodel.html'}) _kale_bestmodel_task.output_artifact_paths.update(_kale_output_artifacts) _kale_bestmodel_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_bestmodel_task.dependent_names + _kale_volume_step_names) _kale_bestmodel_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_bestmodel_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_temporalprojection_task = _kale_temporalprojection_op(date_of_processing, specie)\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_bestmodel_task) _kale_temporalprojection_task.container.working_dir = "//shared_volume/kube_sipecam_playground/hsi/notebooks" _kale_temporalprojection_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update( {'temporalprojection': '/temporalprojection.html'}) _kale_temporalprojection_task.output_artifact_paths.update( _kale_output_artifacts) _kale_temporalprojection_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_temporalprojection_task.dependent_names + _kale_volume_step_names) _kale_temporalprojection_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_temporalprojection_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_uploadtos3_task = _kale_uploadtos3_op(date_of_processing)\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_temporalprojection_task) _kale_uploadtos3_task.container.working_dir = "//shared_volume/kube_sipecam_playground/hsi/notebooks" _kale_uploadtos3_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'uploadtos3': '/uploadtos3.html'}) _kale_uploadtos3_task.output_artifact_paths.update(_kale_output_artifacts) _kale_uploadtos3_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_uploadtos3_task.dependent_names + _kale_volume_step_names) _kale_uploadtos3_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_uploadtos3_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters))
def auto_generated_pipeline(DEPLOY_NAMESPACE='admin', EXPLAINER_MODEL_PATH='sklearn/income/explainer', INCOME_MODEL_PATH='sklearn/income/model', MINIO_ACCESS_KEY='minio', MINIO_HOST='minio-service.kubeflow:9000', MINIO_MODEL_BUCKET='seldon', MINIO_SECRET_KEY='minio123', OUTLIER_MODEL_PATH='sklearn/income/outlier'): pvolumes_dict = OrderedDict() volume_step_names = [] volume_name_parameters = [] marshal_vop = dsl.VolumeOp(name="kale-marshal-volume", resource_name="kale-marshal-pvc", modes=dsl.VOLUME_MODE_RWM, size="1Gi") volume_step_names.append(marshal_vop.name) volume_name_parameters.append(marshal_vop.outputs["name"].full_name) pvolumes_dict['/marshal'] = marshal_vop.volume volume_step_names.sort() volume_name_parameters.sort() setup_task = setup_op(MINIO_ACCESS_KEY, MINIO_HOST, MINIO_MODEL_BUCKET, MINIO_SECRET_KEY)\ .add_pvolumes(pvolumes_dict)\ .after() setup_task.container.working_dir = "/home/jovyan" setup_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'setup': '/setup.html'}) setup_task.output_artifact_paths.update(output_artifacts) setup_task.add_pod_label("pipelines.kubeflow.org/metadata_written", "true") dep_names = setup_task.dependent_names + volume_step_names setup_task.add_pod_annotation("kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: setup_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters)) build_model_task = build_model_op(INCOME_MODEL_PATH, MINIO_ACCESS_KEY, MINIO_HOST, MINIO_MODEL_BUCKET, MINIO_SECRET_KEY)\ .add_pvolumes(pvolumes_dict)\ .after(setup_task) build_model_task.container.working_dir = "/home/jovyan" build_model_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'build_model': '/build_model.html'}) build_model_task.output_artifact_paths.update(output_artifacts) build_model_task.add_pod_label("pipelines.kubeflow.org/metadata_written", "true") dep_names = build_model_task.dependent_names + volume_step_names build_model_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: build_model_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters)) build_outlier_task = build_outlier_op(MINIO_ACCESS_KEY, MINIO_HOST, MINIO_MODEL_BUCKET, MINIO_SECRET_KEY, OUTLIER_MODEL_PATH)\ .add_pvolumes(pvolumes_dict)\ .after(build_model_task) build_outlier_task.container.working_dir = "/home/jovyan" build_outlier_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'build_outlier': '/build_outlier.html'}) build_outlier_task.output_artifact_paths.update(output_artifacts) build_outlier_task.add_pod_label("pipelines.kubeflow.org/metadata_written", "true") dep_names = build_outlier_task.dependent_names + volume_step_names build_outlier_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: build_outlier_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters)) train_explainer_task = train_explainer_op(EXPLAINER_MODEL_PATH, MINIO_ACCESS_KEY, MINIO_HOST, MINIO_MODEL_BUCKET, MINIO_SECRET_KEY)\ .add_pvolumes(pvolumes_dict)\ .after(build_model_task) train_explainer_task.container.working_dir = "/home/jovyan" train_explainer_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'train_explainer': '/train_explainer.html'}) train_explainer_task.output_artifact_paths.update(output_artifacts) train_explainer_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") dep_names = train_explainer_task.dependent_names + volume_step_names train_explainer_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: train_explainer_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters)) deploy_seldon_task = deploy_seldon_op(DEPLOY_NAMESPACE, EXPLAINER_MODEL_PATH, INCOME_MODEL_PATH, MINIO_ACCESS_KEY, MINIO_HOST, MINIO_MODEL_BUCKET, MINIO_SECRET_KEY)\ .add_pvolumes(pvolumes_dict)\ .after(train_explainer_task) deploy_seldon_task.container.working_dir = "/home/jovyan" deploy_seldon_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'deploy_seldon': '/deploy_seldon.html'}) deploy_seldon_task.output_artifact_paths.update(output_artifacts) deploy_seldon_task.add_pod_label("pipelines.kubeflow.org/metadata_written", "true") dep_names = deploy_seldon_task.dependent_names + volume_step_names deploy_seldon_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: deploy_seldon_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters)) test_model_task = test_model_op(DEPLOY_NAMESPACE, MINIO_ACCESS_KEY, MINIO_HOST, MINIO_SECRET_KEY)\ .add_pvolumes(pvolumes_dict)\ .after(deploy_seldon_task) test_model_task.container.working_dir = "/home/jovyan" test_model_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'test_model': '/test_model.html'}) test_model_task.output_artifact_paths.update(output_artifacts) test_model_task.add_pod_label("pipelines.kubeflow.org/metadata_written", "true") dep_names = test_model_task.dependent_names + volume_step_names test_model_task.add_pod_annotation("kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: test_model_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters)) deploy_outlier_task = deploy_outlier_op(DEPLOY_NAMESPACE, MINIO_ACCESS_KEY, MINIO_HOST, MINIO_MODEL_BUCKET, MINIO_SECRET_KEY, OUTLIER_MODEL_PATH)\ .add_pvolumes(pvolumes_dict)\ .after(build_outlier_task, test_model_task) deploy_outlier_task.container.working_dir = "/home/jovyan" deploy_outlier_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'deploy_outlier': '/deploy_outlier.html'}) deploy_outlier_task.output_artifact_paths.update(output_artifacts) deploy_outlier_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") dep_names = deploy_outlier_task.dependent_names + volume_step_names deploy_outlier_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: deploy_outlier_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters)) deploy_event_display_task = deploy_event_display_op(DEPLOY_NAMESPACE, MINIO_ACCESS_KEY, MINIO_HOST, MINIO_SECRET_KEY)\ .add_pvolumes(pvolumes_dict)\ .after(deploy_outlier_task) deploy_event_display_task.container.working_dir = "/home/jovyan" deploy_event_display_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update( {'deploy_event_display': '/deploy_event_display.html'}) deploy_event_display_task.output_artifact_paths.update(output_artifacts) deploy_event_display_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") dep_names = deploy_event_display_task.dependent_names + volume_step_names deploy_event_display_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: deploy_event_display_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters)) test_outliers_task = test_outliers_op(DEPLOY_NAMESPACE, MINIO_ACCESS_KEY, MINIO_HOST, MINIO_SECRET_KEY)\ .add_pvolumes(pvolumes_dict)\ .after(deploy_event_display_task) test_outliers_task.container.working_dir = "/home/jovyan" test_outliers_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'test_outliers': '/test_outliers.html'}) test_outliers_task.output_artifact_paths.update(output_artifacts) test_outliers_task.add_pod_label("pipelines.kubeflow.org/metadata_written", "true") dep_names = test_outliers_task.dependent_names + volume_step_names test_outliers_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: test_outliers_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters)) explain_task = explain_op(MINIO_ACCESS_KEY, MINIO_HOST, MINIO_SECRET_KEY)\ .add_pvolumes(pvolumes_dict)\ .after(train_explainer_task) explain_task.container.working_dir = "/home/jovyan" explain_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'explain': '/explain.html'}) explain_task.output_artifact_paths.update(output_artifacts) explain_task.add_pod_label("pipelines.kubeflow.org/metadata_written", "true") dep_names = explain_task.dependent_names + volume_step_names explain_task.add_pod_annotation("kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: explain_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters))
def dispatch(job=None, namespace='default', debug=False, **kwargs): logger.info(f"starting {job['id']}") from kubernetes import client, config config.load_incluster_config() coreV1 = client.CoreV1Api() batchV1 = client.BatchV1Api() batchV1.create_namespaced_job( namespace=namespace, body=client.V1Job( api_version='batch/v1', kind='Job', metadata=client.V1ObjectMeta( name=job['id'], annotations={ f"container.apparmor.security.beta.kubernetes.io/appyter-{job['id']}": 'unconfined' }, ), spec=client.V1JobSpec( template=client.V1PodTemplateSpec(spec=client.V1PodSpec( restart_policy='Never', containers=[ client.V1Container( name=f"appyter-{job['id']}", image=job['image'], command=[ 'appyter', 'orchestration', 'job', json.dumps(job) ], security_context=client.V1SecurityContext( privileged=True, capabilities=client.V1Capabilities( add=['SYS_ADMIN'], ), ), volume_mounts=[ client.V1VolumeMount( name='fuse', mount_path='/dev/fuse', ) ], ), ], volumes=[ client.V1Volume( name='fuse', host_path=client.V1HostPathVolumeSource( path='/dev/fuse', )), ], ), ), backoff_limit=1, ), ), ) # for event in endless_watch(batchV1.list_namespaced_job, namespace, label_selector=f"job-name={job['id']}"): logger.debug(str(event)) event_type = event['type'] event_job = event['object'] if event_type == 'MODIFIED': if event_job.status.succeeded or event_job.status.failed: break # if not debug: batchV1.delete_namespaced_job(job['id'], namespace) # delete associated pod(s) for event_pod in coreV1.list_namespaced_pod( namespace, label_selector=f"job-name={job['id']}", watch=False).items: coreV1.delete_namespaced_pod(event_pod.metadata.name, namespace) # logger.info(f"{job['id']} completed")
def _execute_blight_job(self, ident_fault: str, on: bool, loc: orchestrator.DeviceLightLoc) -> str: operation_id = str(hash(loc)) message = "" # job definition job_metadata = client.V1ObjectMeta(name=operation_id, namespace=self.rook_env.namespace, labels={"ident": operation_id}) pod_metadata = client.V1ObjectMeta(labels={"ident": operation_id}) pod_container = client.V1Container( name="ceph-lsmcli-command", security_context=client.V1SecurityContext(privileged=True), image=self.get_ceph_image(), command=[ "lsmcli", ], args=[ 'local-disk-%s-led-%s' % (ident_fault, 'on' if on else 'off'), '--path', loc.path or loc.dev, ], volume_mounts=[ client.V1VolumeMount(name="devices", mount_path="/dev"), client.V1VolumeMount(name="run-udev", mount_path="/run/udev") ]) pod_spec = client.V1PodSpec( containers=[pod_container], active_deadline_seconds=30, # Max time to terminate pod restart_policy="Never", node_selector={"kubernetes.io/hostname": loc.host}, volumes=[ client.V1Volume( name="devices", host_path=client.V1HostPathVolumeSource(path="/dev")), client.V1Volume( name="run-udev", host_path=client.V1HostPathVolumeSource(path="/run/udev")) ]) pod_template = client.V1PodTemplateSpec(metadata=pod_metadata, spec=pod_spec) job_spec = client.V1JobSpec( active_deadline_seconds=60, # Max time to terminate job ttl_seconds_after_finished= 10, # Alfa. Lifetime after finishing (either Complete or Failed) backoff_limit=0, template=pod_template) job = client.V1Job(api_version="batch/v1", kind="Job", metadata=job_metadata, spec=job_spec) # delete previous job if it exists try: try: api_response = self.batchV1_api.delete_namespaced_job( operation_id, self.rook_env.namespace, propagation_policy="Background") except ApiException as e: if e.status != 404: # No problem if the job does not exist raise # wait until the job is not present deleted = False retries = 0 while not deleted and retries < 10: api_response = self.batchV1_api.list_namespaced_job( self.rook_env.namespace, label_selector="ident=%s" % operation_id, timeout_seconds=10) deleted = not api_response.items if retries > 5: sleep(0.1) retries += 1 if retries == 10 and not deleted: raise orchestrator.OrchestratorError( "Light <{}> in <{}:{}> cannot be executed. Cannot delete previous job <{}>" .format(on, loc.host, loc.path or loc.dev, operation_id)) # create the job api_response = self.batchV1_api.create_namespaced_job( self.rook_env.namespace, job) # get the result finished = False while not finished: api_response = self.batchV1_api.read_namespaced_job( operation_id, self.rook_env.namespace) finished = api_response.status.succeeded or api_response.status.failed if finished: message = api_response.status.conditions[-1].message # get the result of the lsmcli command api_response = self.coreV1_api.list_namespaced_pod( self.rook_env.namespace, label_selector="ident=%s" % operation_id, timeout_seconds=10) if api_response.items: pod_name = api_response.items[-1].metadata.name message = self.coreV1_api.read_namespaced_pod_log( pod_name, self.rook_env.namespace) except ApiException as e: log.exception('K8s API failed. {}'.format(e)) raise # Finally, delete the job. # The job uses <ttl_seconds_after_finished>. This makes that the TTL controller delete automatically the job. # This feature is in Alpha state, so extra explicit delete operations trying to delete the Job has been used strategically try: api_response = self.batchV1_api.delete_namespaced_job( operation_id, self.rook_env.namespace, propagation_policy="Background") except ApiException as e: if e.status != 404: # No problem if the job does not exist raise return message
def test_sanitize_k8s_container_attribute(self): # test cases for implicit type sanitization(conversion) op = dsl.ContainerOp(name='echo', image='image', command=['sh', '-c'], arguments=['echo test | tee /tmp/message.txt'], file_outputs={'merged': '/tmp/message.txt'}) op.container \ .add_volume_mount(k8s_client.V1VolumeMount( mount_path='/secret/gcp-credentials', name='gcp-credentials')) \ .add_env_variable(k8s_client.V1EnvVar( name=80, value=80)) \ .add_env_variable(k8s_client.V1EnvVar( name=80, value_from=k8s_client.V1EnvVarSource( config_map_key_ref=k8s_client.V1ConfigMapKeySelector(key=80, name=8080, optional='False'), field_ref=k8s_client.V1ObjectFieldSelector(api_version=80, field_path=8080), resource_field_ref=k8s_client.V1ResourceFieldSelector(container_name=80, divisor=8080, resource=8888), secret_key_ref=k8s_client.V1SecretKeySelector(key=80, name=8080, optional='False') ) )) \ .add_env_from(k8s_client.V1EnvFromSource( config_map_ref=k8s_client.V1ConfigMapEnvSource(name=80, optional='True'), prefix=999 )) \ .add_env_from(k8s_client.V1EnvFromSource( secret_ref=k8s_client.V1SecretEnvSource(name=80, optional='True'), prefix=888 )) \ .add_volume_mount(k8s_client.V1VolumeMount( mount_path=111, mount_propagation=222, name=333, read_only='False', sub_path=444, sub_path_expr=555 )) \ .add_volume_devices(k8s_client.V1VolumeDevice( device_path=111, name=222 )) \ .add_port(k8s_client.V1ContainerPort( container_port='8080', host_ip=111, host_port='8888', name=222, protocol=333 )) \ .set_security_context(k8s_client.V1SecurityContext( allow_privilege_escalation='True', capabilities=k8s_client.V1Capabilities(add=[11, 22], drop=[33, 44]), privileged='False', proc_mount=111, read_only_root_filesystem='False', run_as_group='222', run_as_non_root='True', run_as_user='******', se_linux_options=k8s_client.V1SELinuxOptions(level=11, role=22, type=33, user=44), windows_options=k8s_client.V1WindowsSecurityContextOptions( gmsa_credential_spec=11, gmsa_credential_spec_name=22) )) \ .set_stdin(stdin='False') \ .set_stdin_once(stdin_once='False') \ .set_termination_message_path(termination_message_path=111) \ .set_tty(tty='False') \ .set_readiness_probe(readiness_probe=k8s_client.V1Probe( _exec=k8s_client.V1ExecAction(command=[11, 22, 33]), failure_threshold='111', http_get=k8s_client.V1HTTPGetAction( host=11, http_headers=[k8s_client.V1HTTPHeader(name=22, value=33)], path=44, port='55', scheme=66), initial_delay_seconds='222', period_seconds='333', success_threshold='444', tcp_socket=k8s_client.V1TCPSocketAction(host=555, port='666'), timeout_seconds='777' )) \ .set_liveness_probe(liveness_probe=k8s_client.V1Probe( _exec=k8s_client.V1ExecAction(command=[11, 22, 33]), failure_threshold='111', http_get=k8s_client.V1HTTPGetAction( host=11, http_headers=[k8s_client.V1HTTPHeader(name=22, value=33)], path=44, port='55', scheme=66), initial_delay_seconds='222', period_seconds='333', success_threshold='444', tcp_socket=k8s_client.V1TCPSocketAction(host=555, port='666'), timeout_seconds='777' )) \ .set_lifecycle(lifecycle=k8s_client.V1Lifecycle( post_start=k8s_client.V1Handler( _exec=k8s_client.V1ExecAction(command=[11, 22, 33]), http_get=k8s_client.V1HTTPGetAction( host=11, http_headers=[k8s_client.V1HTTPHeader(name=22, value=33)], path=44, port='55', scheme=66), tcp_socket=k8s_client.V1TCPSocketAction(host=555, port='666') ), pre_stop=k8s_client.V1Handler( _exec=k8s_client.V1ExecAction(command=[11, 22, 33]), http_get=k8s_client.V1HTTPGetAction( host=11, http_headers=[k8s_client.V1HTTPHeader(name=22, value=33)], path=44, port='55', scheme=66), tcp_socket=k8s_client.V1TCPSocketAction(host=555, port='666') ) )) sanitize_k8s_object(op.container) for e in op.container.env: self.assertIsInstance(e.name, str) if e.value: self.assertIsInstance(e.value, str) if e.value_from: if e.value_from.config_map_key_ref: self.assertIsInstance(e.value_from.config_map_key_ref.key, str) if e.value_from.config_map_key_ref.name: self.assertIsInstance(e.value_from.config_map_key_ref.name, str) if e.value_from.config_map_key_ref.optional: self.assertIsInstance(e.value_from.config_map_key_ref.optional, bool) if e.value_from.field_ref: self.assertIsInstance(e.value_from.field_ref.field_path, str) if e.value_from.field_ref.api_version: self.assertIsInstance(e.value_from.field_ref.api_version, str) if e.value_from.resource_field_ref: self.assertIsInstance(e.value_from.resource_field_ref.resource, str) if e.value_from.resource_field_ref.container_name: self.assertIsInstance(e.value_from.resource_field_ref.container_name, str) if e.value_from.resource_field_ref.divisor: self.assertIsInstance(e.value_from.resource_field_ref.divisor, str) if e.value_from.secret_key_ref: self.assertIsInstance(e.value_from.secret_key_ref.key, str) if e.value_from.secret_key_ref.name: self.assertIsInstance(e.value_from.secret_key_ref.name, str) if e.value_from.secret_key_ref.optional: self.assertIsInstance(e.value_from.secret_key_ref.optional, bool) for e in op.container.env_from: if e.prefix: self.assertIsInstance(e.prefix, str) if e.config_map_ref: if e.config_map_ref.name: self.assertIsInstance(e.config_map_ref.name, str) if e.config_map_ref.optional: self.assertIsInstance(e.config_map_ref.optional, bool) if e.secret_ref: if e.secret_ref.name: self.assertIsInstance(e.secret_ref.name, str) if e.secret_ref.optional: self.assertIsInstance(e.secret_ref.optional, bool) for e in op.container.volume_mounts: if e.mount_path: self.assertIsInstance(e.mount_path, str) if e.mount_propagation: self.assertIsInstance(e.mount_propagation, str) if e.name: self.assertIsInstance(e.name, str) if e.read_only: self.assertIsInstance(e.read_only, bool) if e.sub_path: self.assertIsInstance(e.sub_path, str) if e.sub_path_expr: self.assertIsInstance(e.sub_path_expr, str) for e in op.container.volume_devices: if e.device_path: self.assertIsInstance(e.device_path, str) if e.name: self.assertIsInstance(e.name, str) for e in op.container.ports: if e.container_port: self.assertIsInstance(e.container_port, int) if e.host_ip: self.assertIsInstance(e.host_ip, str) if e.host_port: self.assertIsInstance(e.host_port, int) if e.name: self.assertIsInstance(e.name, str) if e.protocol: self.assertIsInstance(e.protocol, str) if op.container.security_context: e = op.container.security_context if e.allow_privilege_escalation: self.assertIsInstance(e.allow_privilege_escalation, bool) if e.capabilities: for a in e.capabilities.add: self.assertIsInstance(a, str) for d in e.capabilities.drop: self.assertIsInstance(d, str) if e.privileged: self.assertIsInstance(e.privileged, bool) if e.proc_mount: self.assertIsInstance(e.proc_mount, str) if e.read_only_root_filesystem: self.assertIsInstance(e.read_only_root_filesystem, bool) if e.run_as_group: self.assertIsInstance(e.run_as_group, int) if e.run_as_non_root: self.assertIsInstance(e.run_as_non_root, bool) if e.run_as_user: self.assertIsInstance(e.run_as_user, int) if e.se_linux_options: if e.se_linux_options.level: self.assertIsInstance(e.se_linux_options.level, str) if e.se_linux_options.role: self.assertIsInstance(e.se_linux_options.role, str) if e.se_linux_options.type: self.assertIsInstance(e.se_linux_options.type, str) if e.se_linux_options.user: self.assertIsInstance(e.se_linux_options.user, str) if e.windows_options: if e.windows_options.gmsa_credential_spec: self.assertIsInstance(e.windows_options.gmsa_credential_spec, str) if e.windows_options.gmsa_credential_spec_name: self.assertIsInstance(e.windows_options.gmsa_credential_spec_name, str) if op.container.stdin: self.assertIsInstance(op.container.stdin, bool) if op.container.stdin_once: self.assertIsInstance(op.container.stdin_once, bool) if op.container.termination_message_path: self.assertIsInstance(op.container.termination_message_path, str) if op.container.tty: self.assertIsInstance(op.container.tty, bool) for e in [op.container.readiness_probe, op.container.liveness_probe]: if e: if e._exec: for c in e._exec.command: self.assertIsInstance(c, str) if e.failure_threshold: self.assertIsInstance(e.failure_threshold, int) if e.http_get: if e.http_get.host: self.assertIsInstance(e.http_get.host, str) if e.http_get.http_headers: for h in e.http_get.http_headers: if h.name: self.assertIsInstance(h.name, str) if h.value: self.assertIsInstance(h.value, str) if e.http_get.path: self.assertIsInstance(e.http_get.path, str) if e.http_get.port: self.assertIsInstance(e.http_get.port, (str, int)) if e.http_get.scheme: self.assertIsInstance(e.http_get.scheme, str) if e.initial_delay_seconds: self.assertIsInstance(e.initial_delay_seconds, int) if e.period_seconds: self.assertIsInstance(e.period_seconds, int) if e.success_threshold: self.assertIsInstance(e.success_threshold, int) if e.tcp_socket: if e.tcp_socket.host: self.assertIsInstance(e.tcp_socket.host, str) if e.tcp_socket.port: self.assertIsInstance(e.tcp_socket.port, (str, int)) if e.timeout_seconds: self.assertIsInstance(e.timeout_seconds, int) if op.container.lifecycle: for e in [op.container.lifecycle.post_start, op.container.lifecycle.pre_stop]: if e: if e._exec: for c in e._exec.command: self.assertIsInstance(c, str) if e.http_get: if e.http_get.host: self.assertIsInstance(e.http_get.host, str) if e.http_get.http_headers: for h in e.http_get.http_headers: if h.name: self.assertIsInstance(h.name, str) if h.value: self.assertIsInstance(h.value, str) if e.http_get.path: self.assertIsInstance(e.http_get.path, str) if e.http_get.port: self.assertIsInstance(e.http_get.port, (str, int)) if e.http_get.scheme: self.assertIsInstance(e.http_get.scheme, str) if e.tcp_socket: if e.tcp_socket.host: self.assertIsInstance(e.tcp_socket.host, str) if e.tcp_socket.port: self.assertIsInstance(e.tcp_socket.port, (str, int)) # test cases for checking value after sanitization check_value_op = dsl.ContainerOp(name='echo', image='image', command=['sh', '-c'], arguments=['echo test | tee /tmp/message.txt'], file_outputs={'merged': '/tmp/message.txt'}) check_value_op.container \ .add_env_variable(k8s_client.V1EnvVar( name=80, value=8080)) \ .set_security_context(k8s_client.V1SecurityContext( allow_privilege_escalation='true', capabilities=k8s_client.V1Capabilities(add=[11, 22], drop=[33, 44]), privileged='false', proc_mount=111, read_only_root_filesystem='False', run_as_group='222', run_as_non_root='True', run_as_user='******', se_linux_options=k8s_client.V1SELinuxOptions(level=11, role=22, type=33, user=44), windows_options=k8s_client.V1WindowsSecurityContextOptions( gmsa_credential_spec=11, gmsa_credential_spec_name=22) )) sanitize_k8s_object(check_value_op.container) self.assertEqual(check_value_op.container.env[0].name, '80') self.assertEqual(check_value_op.container.env[0].value, '8080') self.assertEqual(check_value_op.container.security_context.allow_privilege_escalation, True) self.assertEqual(check_value_op.container.security_context.capabilities.add[0], '11') self.assertEqual(check_value_op.container.security_context.capabilities.add[1], '22') self.assertEqual(check_value_op.container.security_context.capabilities.drop[0], '33') self.assertEqual(check_value_op.container.security_context.capabilities.drop[1], '44') self.assertEqual(check_value_op.container.security_context.privileged, False) self.assertEqual(check_value_op.container.security_context.proc_mount, '111') self.assertEqual(check_value_op.container.security_context.read_only_root_filesystem, False) self.assertEqual(check_value_op.container.security_context.run_as_group, 222) self.assertEqual(check_value_op.container.security_context.run_as_non_root, True) self.assertEqual(check_value_op.container.security_context.run_as_user, 333) self.assertEqual(check_value_op.container.security_context.se_linux_options.level, '11') self.assertEqual(check_value_op.container.security_context.se_linux_options.role, '22') self.assertEqual(check_value_op.container.security_context.se_linux_options.type, '33') self.assertEqual(check_value_op.container.security_context.se_linux_options.user, '44') self.assertEqual(check_value_op.container.security_context.windows_options.gmsa_credential_spec, '11') self.assertEqual(check_value_op.container.security_context.windows_options.gmsa_credential_spec_name, '22') # test cases for exception with self.assertRaises(ValueError, msg='Invalid boolean string 2. Should be boolean.'): exception_op = dsl.ContainerOp(name='echo', image='image') exception_op.container \ .set_security_context(k8s_client.V1SecurityContext( allow_privilege_escalation=1 )) sanitize_k8s_object(exception_op.container) with self.assertRaises(ValueError, msg='Invalid boolean string Test. Should be "true" or "false".'): exception_op = dsl.ContainerOp(name='echo', image='image') exception_op.container \ .set_security_context(k8s_client.V1SecurityContext( allow_privilege_escalation='Test' )) sanitize_k8s_object(exception_op.container) with self.assertRaises(ValueError, msg='Invalid test. Should be integer.'): exception_op = dsl.ContainerOp(name='echo', image='image') exception_op.container \ .set_security_context(k8s_client.V1SecurityContext( run_as_group='test', )) sanitize_k8s_object(exception_op.container)
def createStatefulSet(cls, cluster_object: V1MongoClusterConfiguration) -> client.V1beta1StatefulSet: """ Creates a the stateful set configuration for the given cluster. :param cluster_object: The cluster object from the YAML file. :return: The stateful set object. """ # Parse cluster data object. name = cluster_object.metadata.name namespace = cluster_object.metadata.namespace replicas = cluster_object.spec.mongodb.replicas storage_mount_path = cluster_object.spec.mongodb.host_path or cls.DEFAULT_STORAGE_MOUNT_PATH host_path = cluster_object.spec.mongodb.host_path cpu_limit = cluster_object.spec.mongodb.cpu_limit or cls.DEFAULT_CPU_LIMIT memory_limit = cluster_object.spec.mongodb.memory_limit or cls.DEFAULT_MEMORY_LIMIT run_as_user = cluster_object.spec.mongodb.run_as_user or cls.DEFAULT_RUN_AS_USER service_account = cluster_object.spec.mongodb.service_account or cls.DEFAULT_SERVICE_ACCOUNT wired_tiger_cache_size = cluster_object.spec.mongodb.wired_tiger_cache_size or cls.DEFAULT_CACHE_SIZE secret_name = cls.ADMIN_SECRET_NAME_FORMAT.format(name) # create container mongo_container = client.V1Container( name=name, env=[client.V1EnvVar( name="POD_IP", value_from=client.V1EnvVarSource( field_ref = client.V1ObjectFieldSelector( api_version = "v1", field_path = "status.podIP" ) ) ), client.V1EnvVar( name="MONGODB_PASSWORD", value_from=client.V1EnvVarSource( secret_key_ref=client.V1SecretKeySelector( key="database-password", name=secret_name ) ) ), client.V1EnvVar( name="MONGODB_USER", value_from=client.V1EnvVarSource( secret_key_ref=client.V1SecretKeySelector( key="database-user", name=secret_name ) ) ), client.V1EnvVar( name="MONGODB_DATABASE", value_from=client.V1EnvVarSource( secret_key_ref=client.V1SecretKeySelector( key="database-name", name=secret_name ) ) ), client.V1EnvVar( name="MONGODB_ADMIN_PASSWORD", value_from=client.V1EnvVarSource( secret_key_ref=client.V1SecretKeySelector( key="database-admin-password", name=secret_name ) ) ), client.V1EnvVar( name="WIREDTIGER_CACHE_SIZE", value=wired_tiger_cache_size ), client.V1EnvVar( name="MONGODB_REPLICA_NAME", value=name ), client.V1EnvVar( name="MONGODB_SERVICE_NAME", value="svc-" + name + "-internal" ), client.V1EnvVar( name="MONGODB_KEYFILE_VALUE", value="supersecretkeyfile123" )], liveness_probe=client.V1Probe(failure_threshold=3, initial_delay_seconds=30, period_seconds=30, success_threshold=1, tcp_socket=client.V1TCPSocketAction(port=cls.MONGO_PORT), timeout_seconds=1 ), command=cls.MONGO_COMMAND.split(), image=cls.MONGO_IMAGE, image_pull_policy="Always", ports=[client.V1ContainerPort( name="mongodb", container_port=cls.MONGO_PORT, protocol="TCP" )], readiness_probe=client.V1Probe(_exec=client.V1ExecAction(command=["/bin/sh", "-i", "-c", "mongo 127.0.0.1:27017/$MONGODB_DATABASE -u $MONGODB_USER -p $MONGODB_PASSWORD --eval=\"quit()\""]), failure_threshold=3, initial_delay_seconds=10, period_seconds=10, success_threshold=1, timeout_seconds=1 ), security_context=client.V1SecurityContext( run_as_user=int(run_as_user), se_linux_options=client.V1SELinuxOptions( level="s0", type="spc_t" ) ), termination_message_path="/dev/termination-log", volume_mounts=[client.V1VolumeMount( name="mongo-data", read_only=False, mount_path=storage_mount_path )], resources=client.V1ResourceRequirements( limits={"cpu": cpu_limit, "memory": memory_limit}, requests={"cpu": cpu_limit, "memory": memory_limit} ) ) #create affinity rules affinity = client.V1Affinity( pod_anti_affinity=client.V1PodAntiAffinity( required_during_scheduling_ignored_during_execution=[ client.V1PodAffinityTerm(label_selector=client.V1LabelSelector( match_expressions=[client.V1LabelSelectorRequirement( key="app", operator="In", values=[name] )] ), topology_key="kubernetes.io/hostname") ] ) ) volumes = [client.V1Volume( name="mongo-data", host_path=client.V1HostPathVolumeSource(path=host_path) )] # Create stateful set. return client.V1beta1StatefulSet( metadata = client.V1ObjectMeta(annotations={"service.alpha.kubernetes.io/tolerate-unready-endpoints": "true"}, name=name, namespace=namespace, labels=cls.createDefaultLabels(name)), spec = client.V1beta1StatefulSetSpec( replicas = replicas, service_name = "svc-" + name + "-internal", template = client.V1PodTemplateSpec( metadata = client.V1ObjectMeta(labels=cls.createDefaultLabels(name)), spec = client.V1PodSpec(affinity = affinity, containers=[mongo_container], node_selector={"compute":"mongodb"}, service_account=service_account, #restart_policy="Never", volumes=volumes ) ), ), )
def add_run_with_root_permissions(self): """Run interactive session with root.""" security_context = client.V1SecurityContext(run_as_user=0) self.kubernetes_objects["deployment"].spec.template.spec.containers[ 0].security_context = security_context
def deployment_object(self, instance_uuid, cnf_yaml, service_uuid, vim_uuid): """ CNF modeling method. This build a deployment object in kubernetes instance_uuid: k8s deployment name cnf_yaml: CNF Descriptor in yaml format """ t0 = time.time() LOG.debug("CNFD: {}".format(cnf_yaml)) container_list = [] pod_volume_list = [] deployment_k8s = None privileged = False node_selector = {} host_network = False if "cloudnative_deployment_units" in cnf_yaml: cdu = cnf_yaml.get('cloudnative_deployment_units') for cdu_obj in cdu: env_vars = env_from = cpu = memory = huge_pages = gpu = sr_iov = resources = volume_mounts = None port_list = [] environment = [] capabilities_list = [] cdu_id = cdu_obj.get('id') image = cdu_obj.get('image') cdu_conex = cdu_obj.get('connection_points') container_name = cdu_id config_map_id = cdu_id if cdu_obj.get('parameters'): env_vars = cdu_obj['parameters'].get('env') volume_mounts = cdu_obj['parameters'].get('volume_mounts') capabilities_list = cdu_obj['parameters'].get('capabilities') if cdu_obj['parameters'].get('privileged'): privileged = cdu_obj['parameters'].get('privileged') if cdu_obj['parameters'].get('host_network'): privileged = cdu_obj['parameters']['host_network'] if not isinstance(capabilities_list, list): capabilities_list = [] if cdu_obj['parameters'].get('node_selector'): node_selector = cdu_obj['parameters']['node_selector'] if cdu_obj.get('resource_requirements'): gpu = cdu_obj['resource_requirements'].get('gpu') cpu = cdu_obj['resource_requirements'].get('cpu') memory = cdu_obj['resource_requirements'].get('memory') sr_iov = cdu_obj['resource_requirements'].get('sr-iov') huge_pages = cdu_obj['resource_requirements'].get('huge-pages') if cdu_conex: for po in cdu_conex: port = po.get('port') port_name = po.get('id') protocol = "TCP" if po.get("protocol"): protocol = po["protocol"] port_list.append(client.V1ContainerPort(container_port = port, name = port_name, protocol=protocol)) limits = {} requests = {} if gpu: LOG.debug("Features requested: {}".format(gpu)) # gpu_type can be amd or nvidia for gpu_type, amount in gpu.items(): limits["{}.com/gpu".format(gpu_type)] = amount if cpu: # TODO pass if memory: # TODO pass if sr_iov: # TODO pass if huge_pages: # TODO pass resources = client.V1ResourceRequirements(limits=limits, requests=requests) # Environment variables from descriptor if env_vars: LOG.debug("Configmap: {}".format(config_map_id)) KubernetesWrapperEngine.create_configmap(self, config_map_id, instance_uuid, env_vars, service_uuid, vim_uuid, namespace = "default") else: env_vars = {"sonata": "rules"} LOG.debug("Configmap: {}".format(config_map_id)) KubernetesWrapperEngine.create_configmap(self, config_map_id, instance_uuid, env_vars, service_uuid, vim_uuid, namespace = "default") env_from = client.V1EnvFromSource(config_map_ref = client.V1ConfigMapEnvSource(name = config_map_id, optional = False)) # Default static environment variables environment.append(client.V1EnvVar(name="instance_uuid", value=instance_uuid)) environment.append(client.V1EnvVar(name="service_uuid", value=service_uuid)) environment.append(client.V1EnvVar(name="container_name", value=container_name)) environment.append(client.V1EnvVar(name="vendor", value=KubernetesWrapperEngine.normalize(self, cnf_yaml.get('vendor')))) environment.append(client.V1EnvVar(name="name", value=KubernetesWrapperEngine.normalize(self, cnf_yaml.get('name')))) environment.append(client.V1EnvVar(name="version", value=KubernetesWrapperEngine.normalize(self, cnf_yaml.get('version')))) image_pull_policy = KubernetesWrapperEngine.check_connection(self) # Volume mounts container_volume_mount_list = [] if volume_mounts: LOG.debug("volume mounts: {}".format(volume_mounts)) # Create the specification of volumes for volume_mounts_item in volume_mounts: if volume_mounts_item.get('id') and volume_mounts_item.get('location'): if volume_mounts_item.get('persistent'): volumes = client.V1Volume(name=volume_mounts_item['id'], host_path=client.V1HostPathVolumeSource(path='/mnt/data', type='DirectoryOrCreate' )) else: volumes = client.V1Volume(name=volume_mounts_item['id'], empty_dir=client.V1EmptyDirVolumeSource(medium='' )) if volumes not in pod_volume_list: pod_volume_list.append(volumes) container_volume_mount = client.V1VolumeMount(name=volume_mounts_item['id'], mount_path=volume_mounts_item['location'] ) container_volume_mount_list.append(container_volume_mount) LOG.debug("Security capabilities: {}, privileged: {} applied to {}".format(capabilities_list, privileged, container_name)) sec_context = client.V1SecurityContext(privileged=privileged, capabilities=client.V1Capabilities(add=capabilities_list)) # Configureate Pod template container container = client.V1Container( env = environment, name = container_name, resources = resources, image = image, image_pull_policy = image_pull_policy, ports = port_list, env_from = [env_from], volume_mounts = container_volume_mount_list, security_context=sec_context) container_list.append(container) else: return deployment_k8s # Create and configurate a spec section deployment_label = ("{}-{}-{}-{}".format(cnf_yaml.get("vendor"), cnf_yaml.get("name"), cnf_yaml.get("version"), instance_uuid.split("-")[0])).replace(".", "-") template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta(labels={'deployment': deployment_label, 'instance_uuid': cnf_yaml['instance_uuid'], 'service_uuid': service_uuid, 'sp': "sonata", 'descriptor_uuid': cnf_yaml['uuid']} ), spec=client.V1PodSpec(containers=container_list, volumes=pod_volume_list, node_selector=node_selector, host_network=host_network)) selector=client.V1LabelSelector(match_labels={'deployment': deployment_label, 'instance_uuid': cnf_yaml['instance_uuid'], 'service_uuid': service_uuid, 'sp': "sonata", 'descriptor_uuid': cnf_yaml['uuid']} ) # Create the specification of deployment spec = client.V1DeploymentSpec( replicas=1, template=template, selector=selector) # Instantiate the deployment object deployment_k8s = client.V1Deployment( api_version="apps/v1", kind="Deployment", metadata=client.V1ObjectMeta(name=deployment_label, labels={'deployment': deployment_label, 'instance_uuid': cnf_yaml['instance_uuid'], 'service_uuid': service_uuid, 'sp': "sonata", 'descriptor_uuid': cnf_yaml['uuid']} ), spec=spec) LOG.info("Deployment object: {}".format(deployment_k8s)) LOG.info("CreatingDeploymentObject-time: {} ms".format(int((time.time() - t0)* 1000))) return deployment_k8s
def _create_pod(self, image, pod_name, job_name, port=80, cmd_string=None, volumes=[]): """ Create a kubernetes pod for the job. Args: - image (string) : Docker image to launch - pod_name (string) : Name of the pod - job_name (string) : App label KWargs: - port (integer) : Container port Returns: - None """ security_context = None if self.user_id and self.group_id: security_context = client.V1SecurityContext( run_as_group=self.group_id, run_as_user=self.user_id, run_as_non_root=self.run_as_non_root) # Create the enviornment variables and command to initiate IPP environment_vars = client.V1EnvVar(name="TEST", value="SOME DATA") launch_args = ["-c", "{0}".format(cmd_string)] volume_mounts = [] # Create mount paths for the volumes for volume in volumes: volume_mounts.append( client.V1VolumeMount(mount_path=volume[1], name=volume[0])) resources = client.V1ResourceRequirements(limits={ 'cpu': str(self.max_cpu), 'memory': self.max_mem }, requests={ 'cpu': str(self.init_cpu), 'memory': self.init_mem }) # Configure Pod template container container = client.V1Container( name=pod_name, image=image, resources=resources, ports=[client.V1ContainerPort(container_port=port)], volume_mounts=volume_mounts, command=['/bin/bash'], args=launch_args, env=[environment_vars], security_context=security_context) # Create a secret to enable pulling images from secure repositories secret = None if self.secret: secret = client.V1LocalObjectReference(name=self.secret) # Create list of volumes from (pvc, mount) tuples volume_defs = [] for volume in volumes: volume_defs.append( client.V1Volume( name=volume[0], persistent_volume_claim=client. V1PersistentVolumeClaimVolumeSource(claim_name=volume[0]))) metadata = client.V1ObjectMeta(name=pod_name, labels={"app": job_name}) spec = client.V1PodSpec(containers=[container], image_pull_secrets=[secret], volumes=volume_defs) pod = client.V1Pod(spec=spec, metadata=metadata) api_response = self.kube_client.create_namespaced_pod( namespace=self.namespace, body=pod) logger.debug("Pod created. status='{0}'".format( str(api_response.status)))
def deploy_k8s_nfs(self) -> bool: """ Deploy the NFS server in the Kubernetes orchestrator. """ from kubernetes import client as k8sclient name = "nfs-server-{}".format(uuid.uuid4()) container = k8sclient.V1Container( name=name, image="k8s.gcr.io/volume-nfs:0.8", ports=[ k8sclient.V1ContainerPort(name="nfs", container_port=2049, protocol="TCP"), k8sclient.V1ContainerPort(name="rpcbind", container_port=111), k8sclient.V1ContainerPort(name="mountd", container_port=20048), ], volume_mounts=[ k8sclient.V1VolumeMount(name='nfs-host-path', mount_path='/exports') ], security_context=k8sclient.V1SecurityContext(privileged=True)) template = k8sclient.V1PodTemplateSpec( metadata=k8sclient.V1ObjectMeta(labels={'app': name}), spec=k8sclient.V1PodSpec( containers=[container], volumes=[ k8sclient.V1Volume( name="nfs-host-path", host_path=k8sclient.V1HostPathVolumeSource( path='/tmp/nfsexports-{}'.format(uuid.uuid4()))) ])) deployment_spec = k8sclient.V1DeploymentSpec( replicas=1, template=template, selector=k8sclient.V1LabelSelector(match_labels={'app': name})) deployment = k8sclient.V1Deployment(api_version='apps/v1', kind='Deployment', metadata=k8sclient.V1ObjectMeta( name=name, labels={'app': name}), spec=deployment_spec) k8s_apps_v1_api_client = k8sclient.AppsV1Api() try: k8s_apps_v1_api_client.create_namespaced_deployment( self.params.namespace, deployment) self.params.name = name except k8sclient.rest.ApiException as e: screen.print("Got exception: %s\n while creating nfs-server", e) return False k8s_core_v1_api_client = k8sclient.CoreV1Api() svc_name = "nfs-service-{}".format(uuid.uuid4()) service = k8sclient.V1Service( api_version='v1', kind='Service', metadata=k8sclient.V1ObjectMeta(name=svc_name), spec=k8sclient.V1ServiceSpec(selector={'app': self.params.name}, ports=[ k8sclient.V1ServicePort( protocol='TCP', port=2049, target_port=2049) ])) try: svc_response = k8s_core_v1_api_client.create_namespaced_service( self.params.namespace, service) self.params.svc_name = svc_name self.params.server = svc_response.spec.cluster_ip except k8sclient.rest.ApiException as e: screen.print( "Got exception: %s\n while creating a service for nfs-server", e) return False return True
def auto_generated_pipeline(booltest='True', d1='5', d2='6', strtest='test'): pvolumes_dict = OrderedDict() volume_step_names = [] volume_name_parameters = [] marshal_vop = dsl.VolumeOp(name="kale-marshal-volume", resource_name="kale-marshal-pvc", modes=dsl.VOLUME_MODE_RWM, size="1Gi") volume_step_names.append(marshal_vop.name) volume_name_parameters.append(marshal_vop.outputs["name"].full_name) pvolumes_dict['/marshal'] = marshal_vop.volume volume_step_names.sort() volume_name_parameters.sort() create_matrix_task = create_matrix_op(d1, d2)\ .add_pvolumes(pvolumes_dict)\ .after() step_limits = {'nvidia.com/gpu': '2'} for k, v in step_limits.items(): create_matrix_task.container.add_resource_limit(k, v) create_matrix_task.container.working_dir = "/kale" create_matrix_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update({'mlpipeline-metrics': '/mlpipeline-metrics.json'}) output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'create_matrix': '/create_matrix.html'}) create_matrix_task.output_artifact_paths.update(output_artifacts) create_matrix_task.add_pod_label("pipelines.kubeflow.org/metadata_written", "true") dep_names = create_matrix_task.dependent_names + volume_step_names create_matrix_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: create_matrix_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters)) sum_matrix_task = sum_matrix_op()\ .add_pvolumes(pvolumes_dict)\ .after(create_matrix_task) sum_matrix_task.container.working_dir = "/kale" sum_matrix_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update({'mlpipeline-metrics': '/mlpipeline-metrics.json'}) output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'sum_matrix': '/sum_matrix.html'}) sum_matrix_task.output_artifact_paths.update(output_artifacts) sum_matrix_task.add_pod_label("pipelines.kubeflow.org/metadata_written", "true") dep_names = sum_matrix_task.dependent_names + volume_step_names sum_matrix_task.add_pod_annotation("kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: sum_matrix_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters))
def _build_definition(machine, config_map): volume_mounts = [] if config_map: # Define volume mounts for hostlab if a ConfigMap is defined. volume_mounts.append( client.V1VolumeMount(name="hostlab", mount_path="/tmp/kathara")) if Setting.get_instance().host_shared: volume_mounts.append( client.V1VolumeMount(name="shared", mount_path="/shared")) # Machine must be executed in privileged mode to run sysctls. security_context = client.V1SecurityContext(privileged=True) ports_info = machine.get_ports() container_ports = None if ports_info: container_ports = [] for (host_port, protocol), guest_port in ports_info.items(): container_ports.append( client.V1ContainerPort(name=str(uuid.uuid4()).replace( '-', '')[0:15], container_port=guest_port, host_port=host_port, protocol=protocol.upper())) resources = None memory = machine.get_mem() cpus = machine.get_cpu(multiplier=1000) if memory or cpus: limits = dict() if memory: limits["memory"] = memory.upper() if cpus: limits["cpu"] = "%dm" % cpus resources = client.V1ResourceRequirements(limits=limits) # postStart lifecycle hook is launched asynchronously by k8s master when the main container is Ready # On Ready state, the pod has volumes and network interfaces up, so this hook is used # to execute custom commands coming from .startup file and "exec" option # Build the final startup commands string sysctl_commands = "; ".join([ "sysctl -w -q %s=%d" % item for item in machine.meta["sysctls"].items() ]) startup_commands_string = "; ".join(STARTUP_COMMANDS) \ .format(machine_name=machine.name, sysctl_commands=sysctl_commands, machine_commands="; ".join(machine.startup_commands) ) post_start = client.V1Handler(_exec=client.V1ExecAction(command=[ Setting.get_instance().device_shell, "-c", startup_commands_string ])) lifecycle = client.V1Lifecycle(post_start=post_start) env = [ client.V1EnvVar( "_MEGALOS_SHELL", machine.meta["shell"] if "shell" in machine.meta else Setting.get_instance().device_shell) ] container_definition = client.V1Container( name=machine.meta['real_name'], image=machine.get_image(), lifecycle=lifecycle, stdin=True, tty=True, image_pull_policy=Setting.get_instance().image_pull_policy, ports=container_ports, resources=resources, volume_mounts=volume_mounts, security_context=security_context, env=env) pod_annotations = {} network_interfaces = [] for (idx, machine_link) in machine.interfaces.items(): network_interfaces.append({ "name": machine_link.api_object["metadata"]["name"], "namespace": machine.lab.folder_hash, "interface": "net%d" % idx }) pod_annotations["k8s.v1.cni.cncf.io/networks"] = json.dumps( network_interfaces) # Create labels (so Deployment can match them) pod_labels = {"name": machine.name, "app": "kathara"} pod_metadata = client.V1ObjectMeta(deletion_grace_period_seconds=0, annotations=pod_annotations, labels=pod_labels) # Add fake DNS just to override k8s one dns_config = client.V1PodDNSConfig(nameservers=["127.0.0.1"]) volumes = [] if config_map: # Hostlab is the lab base64 encoded .tar.gz of the machine files, deployed as a ConfigMap in the cluster # The base64 file is mounted into /tmp and it's extracted by the postStart hook volumes.append( client.V1Volume(name="hostlab", config_map=client.V1ConfigMapVolumeSource( name=config_map.metadata.name))) # Container /shared mounts in /home/shared folder if Setting.get_instance().host_shared: volumes.append( client.V1Volume(name="shared", host_path=client.V1HostPathVolumeSource( path='/home/shared', type='DirectoryOrCreate'))) pod_spec = client.V1PodSpec(containers=[container_definition], hostname=machine.meta['real_name'], dns_policy="None", dns_config=dns_config, volumes=volumes) pod_template = client.V1PodTemplateSpec(metadata=pod_metadata, spec=pod_spec) selector_rules = client.V1LabelSelector(match_labels=pod_labels) deployment_spec = client.V1DeploymentSpec(replicas=1, template=pod_template, selector=selector_rules) deployment_metadata = client.V1ObjectMeta( name=machine.meta['real_name'], labels=pod_labels) return client.V1Deployment(api_version="apps/v1", kind="Deployment", metadata=deployment_metadata, spec=deployment_spec)
def _create_deployment_object(self): """Creates the deployment object for the grader service using environment variables Returns: V1Deployment: a valid kubernetes deployment object """ # Configureate Pod template container # Volumes to mount as subPaths of PV sub_path_grader_home = str(self.course_dir.parent).strip("/") sub_path_exchange = str(self.exchange_dir.relative_to(EXCHANGE_MNT_ROOT)) # define the container to launch container = client.V1Container( name="grader-notebook", image=GRADER_IMAGE_NAME, image_pull_policy=GRADER_IMAGE_PULL_POLICY, command=["start-notebook.sh", f"--group=formgrade-{self.course_id}"], ports=[client.V1ContainerPort(container_port=8888)], working_dir=f"/home/{self.grader_name}", resources=client.V1ResourceRequirements( requests={ "cpu": GRADER_REQUESTS_CPU, "memory": GRADER_REQUESTS_MEM, }, limits={ "cpu": GRADER_LIMITS_CPU, "memory": GRADER_LIMITS_MEM, }, ), security_context=client.V1SecurityContext(allow_privilege_escalation=False), env=[ client.V1EnvVar(name="JUPYTERHUB_SERVICE_NAME", value=self.course_id), client.V1EnvVar( name="JUPYTERHUB_SERVICE_URL", value=f"http://{self.course_id}.{NAMESPACE}.svc.cluster.local:8888", ), client.V1EnvVar(name="JUPYTERHUB_API_TOKEN", value=self.grader_token), # we're using the K8s Service name 'hub' (defined in the jhub helm chart) # to connect from our grader-notebooks client.V1EnvVar(name="JUPYTERHUB_API_URL", value=JUPYTERHUB_API_URL), client.V1EnvVar(name="JUPYTERHUB_BASE_URL", value=JUPYTERHUB_BASE_URL), client.V1EnvVar( name="JUPYTERHUB_SERVICE_PREFIX", value=f"/services/{self.course_id}/", ), client.V1EnvVar( name="JUPYTERHUB_CLIENT_ID", value=f"service-{self.course_id}" ), client.V1EnvVar(name="JUPYTERHUB_USER", value=self.grader_name), client.V1EnvVar(name="NAMESPACE", value=str(NAMESPACE)), client.V1EnvVar(name="NB_UID", value=str(NB_UID)), client.V1EnvVar(name="NB_GID", value=str(NB_GID)), client.V1EnvVar(name="NB_USER", value=self.grader_name), ], volume_mounts=[ client.V1VolumeMount( mount_path=f"/home/{self.grader_name}", name=GRADER_PVC, sub_path=sub_path_grader_home, ), client.V1VolumeMount( mount_path="/srv/nbgrader/exchange", name=GRADER_EXCHANGE_SHARED_PVC, sub_path=sub_path_exchange, ), ], ) # Create and configure a spec section template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta( labels={"component": self.grader_name, "app": "illumidesk"} ), spec=client.V1PodSpec( containers=[container], security_context=client.V1PodSecurityContext(run_as_user=0), volumes=[ client.V1Volume( name=GRADER_PVC, persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource( claim_name=GRADER_PVC ), ), client.V1Volume( name=GRADER_EXCHANGE_SHARED_PVC, persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource( claim_name=GRADER_EXCHANGE_SHARED_PVC ), ), ], ), ) # Create the specification of deployment spec = client.V1DeploymentSpec( replicas=1, template=template, selector={"matchLabels": {"component": self.grader_name}}, ) # Instantiate the deployment object deployment = client.V1Deployment( api_version="apps/v1", kind="Deployment", metadata=client.V1ObjectMeta(name=self.grader_name), spec=spec, ) return deployment
def template(context): labels = {"app": context["name"]} # Create volume mount lists and populate them if they are declared in the command pod_spec_volume_mounts = [] pod_spec_volumes = [] if "volumeName" in context: # Create volume mounts if "subPath" in context: pod_spec_volume_mounts = [ client.V1VolumeMount(name=context["volumeName"], mount_path=context["mountPath"], sub_path=context["subPath"]) ] else: pod_spec_volume_mounts = [ client.V1VolumeMount(name=context["volumeName"], mount_path=context["mountPath"]) ] # Create volumes pod_spec_volumes = [ client.V1Volume(name=context["volumeName"], nfs=client.V1NFSVolumeSource( path=context["nfsPath"], server=context["nfsServer"])) ] pod_init_containers = [] if "vpn_init" in context: for key, val in dict( zip(context["vpn_init"].split(), context["vpn_cmds"].split(';'))).items(): pod_init_containers.append( client.V1Container(name=context["name"] + "-" + key, image=context["image"], command=val.split(), volume_mounts=pod_spec_volume_mounts)) # Create Environment variable list and populate if it is declared in the command env_list = [] if "env" in context: envs = dict(zip(context["env"].split(), context["vals"].split())) for key, val in envs.items(): env_var = client.V1EnvVar(name=key, value=val) env_list.append(env_var) args = [] if "attack_args" in context: args.append(context["attack_args"].split()) # Define the template specification template_spec = client.V1PodSpec(containers=[ client.V1Container( name=context["name"], image=context["image"], env=env_list, args=args, security_context=client.V1SecurityContext(privileged=True), volume_mounts=pod_spec_volume_mounts) ], init_containers=pod_init_containers, volumes=pod_spec_volumes) # Create dictionary for network attachment definition # This is required in a dictionary format for the template.metadata.annotations field net_dict = {} net_dict[context["netkey"]] = ', '.join(context["netval"].split()) # Return deployment specification and tie together all the above components return client.V1Deployment( api_version="extensions/v1beta1", kind="Deployment", metadata=client.V1ObjectMeta(name=context["name"]), spec=client.V1DeploymentSpec( replicas=int(context["replicas"]), selector=client.V1LabelSelector(match_labels=labels), template=client.V1PodTemplateSpec(metadata=client.V1ObjectMeta( labels=labels, annotations=net_dict), spec=template_spec), ), )
def auto_generated_pipeline(): pvolumes_dict = OrderedDict() marshal_vop = dsl.VolumeOp(name="kale_marshal_volume", resource_name="kale-marshal-pvc", modes=dsl.VOLUME_MODE_RWM, size="1Gi") pvolumes_dict['/marshal'] = marshal_vop.volume loaddata_task = loaddata_op()\ .add_pvolumes(pvolumes_dict)\ .after() loaddata_task.container.working_dir = "/kale" loaddata_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'loaddata': '/loaddata.html'}) loaddata_task.output_artifact_paths.update(output_artifacts) datapreprocessing_task = datapreprocessing_op()\ .add_pvolumes(pvolumes_dict)\ .after(loaddata_task) datapreprocessing_task.container.working_dir = "/kale" datapreprocessing_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'datapreprocessing': '/datapreprocessing.html'}) datapreprocessing_task.output_artifact_paths.update(output_artifacts) featureengineering_task = featureengineering_op()\ .add_pvolumes(pvolumes_dict)\ .after(datapreprocessing_task) featureengineering_task.container.working_dir = "/kale" featureengineering_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'featureengineering': '/featureengineering.html'}) featureengineering_task.output_artifact_paths.update(output_artifacts) decisiontree_task = decisiontree_op()\ .add_pvolumes(pvolumes_dict)\ .after(featureengineering_task) decisiontree_task.container.working_dir = "/kale" decisiontree_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'decisiontree': '/decisiontree.html'}) decisiontree_task.output_artifact_paths.update(output_artifacts) svm_task = svm_op()\ .add_pvolumes(pvolumes_dict)\ .after(featureengineering_task) svm_task.container.working_dir = "/kale" svm_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'svm': '/svm.html'}) svm_task.output_artifact_paths.update(output_artifacts) naivebayes_task = naivebayes_op()\ .add_pvolumes(pvolumes_dict)\ .after(featureengineering_task) naivebayes_task.container.working_dir = "/kale" naivebayes_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'naivebayes': '/naivebayes.html'}) naivebayes_task.output_artifact_paths.update(output_artifacts) logisticregression_task = logisticregression_op()\ .add_pvolumes(pvolumes_dict)\ .after(featureengineering_task) logisticregression_task.container.working_dir = "/kale" logisticregression_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'logisticregression': '/logisticregression.html'}) logisticregression_task.output_artifact_paths.update(output_artifacts) randomforest_task = randomforest_op()\ .add_pvolumes(pvolumes_dict)\ .after(featureengineering_task) randomforest_task.container.working_dir = "/kale" randomforest_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'randomforest': '/randomforest.html'}) randomforest_task.output_artifact_paths.update(output_artifacts) results_task = results_op()\ .add_pvolumes(pvolumes_dict)\ .after(randomforest_task, logisticregression_task, naivebayes_task, svm_task, decisiontree_task) results_task.container.working_dir = "/kale" results_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'results': '/results.html'}) results_task.output_artifact_paths.update(output_artifacts)
def auto_generated_pipeline(): _kale_pvolumes_dict = OrderedDict() _kale_volume_step_names = [] _kale_volume_name_parameters = [] _kale_marshal_vop = _kfp_dsl.VolumeOp( name="kale-marshal-volume", resource_name="kale-marshal-pvc", modes=['ReadWriteMany'], size="1Gi" ) _kale_volume_step_names.append(_kale_marshal_vop.name) _kale_volume_name_parameters.append( _kale_marshal_vop.outputs["name"].full_name) _kale_pvolumes_dict['/marshal'] = _kale_marshal_vop.volume _kale_volume_step_names.sort() _kale_volume_name_parameters.sort() _kale_step1_task = _kale_step1_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after() _kale_step_labels = {'common-label': 'true'} for _kale_k, _kale_v in _kale_step_labels.items(): _kale_step1_task.add_pod_label(_kale_k, _kale_v) _kale_step_limits = {'amd/gpu': '1'} for _kale_k, _kale_v in _kale_step_limits.items(): _kale_step1_task.container.add_resource_limit(_kale_k, _kale_v) _kale_step1_task.container.working_dir = "/test" _kale_step1_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_step1_task.output_artifact_paths.update(_kale_output_artifacts) _kale_step1_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_step1_task.dependent_names + _kale_volume_step_names) _kale_step1_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_step1_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_step2_task = _kale_step2_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_step1_task) _kale_step_labels = {'common-label': 'true'} for _kale_k, _kale_v in _kale_step_labels.items(): _kale_step2_task.add_pod_label(_kale_k, _kale_v) _kale_step2_task.set_retry_strategy( num_retries=5, retry_policy="Always", backoff_duration="20", backoff_factor=2, backoff_max_duration=None) _kale_step2_task.container.working_dir = "/test" _kale_step2_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_step2_task.output_artifact_paths.update(_kale_output_artifacts) _kale_step2_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_step2_task.dependent_names + _kale_volume_step_names) _kale_step2_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_step2_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_step3_task = _kale_step3_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_step2_task, _kale_step1_task) _kale_step_annotations = {'step3-annotation': 'test'} for _kale_k, _kale_v in _kale_step_annotations.items(): _kale_step3_task.add_pod_annotation(_kale_k, _kale_v) _kale_step_labels = {'common-label': 'true'} for _kale_k, _kale_v in _kale_step_labels.items(): _kale_step3_task.add_pod_label(_kale_k, _kale_v) _kale_step3_task.container.working_dir = "/test" _kale_step3_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_step3_task.output_artifact_paths.update(_kale_output_artifacts) _kale_step3_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_step3_task.dependent_names + _kale_volume_step_names) _kale_step3_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_step3_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters))
def auto_generated_pipeline(density_p='1000000000', vol_shared_volume='hostpath-pvc'): _kale_pvolumes_dict = OrderedDict() _kale_volume_step_names = [] _kale_volume_name_parameters = [] _kale_annotations = {} _kale_volume = _kfp_dsl.PipelineVolume(pvc=vol_shared_volume) _kale_pvolumes_dict['/shared_volume'] = _kale_volume _kale_volume_step_names.sort() _kale_volume_name_parameters.sort() _kale_integral1_task = _kale_integral1_op(density_p)\ .add_pvolumes(_kale_pvolumes_dict)\ .after() #see: https://kubeflow-pipelines.readthedocs.io/en/latest/source/kfp.dsl.html#kfp.dsl.Sidecar.add_resource_limit #can also be: _kale_step_limits = {'nvidia.com/gpu': '1'} _kale_step_limits = {'cpu': '1', 'memory': '40Gi'} for _kale_k, _kale_v in _kale_step_limits.items(): _kale_integral1_task.container.add_resource_limit(_kale_k, _kale_v) _kale_integral1_task.container.working_dir = "//shared_volume" _kale_integral1_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'integral1': '/integral1.html'}) _kale_integral1_task.output_artifact_paths.update(_kale_output_artifacts) _kale_integral1_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_integral1_task.dependent_names + _kale_volume_step_names) _kale_integral1_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_integral1_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_integral2_task = _kale_integral2_op(density_p)\ .add_pvolumes(_kale_pvolumes_dict)\ .after() _kale_step_limits = {'cpu': '1', 'memory': '40Gi'} for _kale_k, _kale_v in _kale_step_limits.items(): _kale_integral2_task.container.add_resource_limit(_kale_k, _kale_v) _kale_integral2_task.container.working_dir = "//shared_volume" _kale_integral2_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'integral2': '/integral2.html'}) _kale_integral2_task.output_artifact_paths.update(_kale_output_artifacts) _kale_integral2_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_integral2_task.dependent_names + _kale_volume_step_names) _kale_integral2_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_integral2_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_integral3_task = _kale_integral3_op(density_p)\ .add_pvolumes(_kale_pvolumes_dict)\ .after() _kale_step_limits = {'cpu': '1', 'memory': '40Gi'} for _kale_k, _kale_v in _kale_step_limits.items(): _kale_integral3_task.container.add_resource_limit(_kale_k, _kale_v) _kale_integral3_task.container.working_dir = "//shared_volume" _kale_integral3_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'integral3': '/integral3.html'}) _kale_integral3_task.output_artifact_paths.update(_kale_output_artifacts) _kale_integral3_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_integral3_task.dependent_names + _kale_volume_step_names) _kale_integral3_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_integral3_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_integral4_task = _kale_integral4_op(density_p)\ .add_pvolumes(_kale_pvolumes_dict)\ .after() _kale_step_limits = {'cpu': '1', 'memory': '40Gi'} for _kale_k, _kale_v in _kale_step_limits.items(): _kale_integral4_task.container.add_resource_limit(_kale_k, _kale_v) _kale_integral4_task.container.working_dir = "//shared_volume" _kale_integral4_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'integral4': '/integral4.html'}) _kale_integral4_task.output_artifact_paths.update(_kale_output_artifacts) _kale_integral4_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_integral4_task.dependent_names + _kale_volume_step_names) _kale_integral4_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_integral4_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_integral5_task = _kale_integral5_op(density_p)\ .add_pvolumes(_kale_pvolumes_dict)\ .after() _kale_step_limits = {'cpu': '1', 'memory': '40Gi'} for _kale_k, _kale_v in _kale_step_limits.items(): _kale_integral5_task.container.add_resource_limit(_kale_k, _kale_v) _kale_integral5_task.container.working_dir = "//shared_volume" _kale_integral5_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'integral5': '/integral5.html'}) _kale_integral5_task.output_artifact_paths.update(_kale_output_artifacts) _kale_integral5_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_integral5_task.dependent_names + _kale_volume_step_names) _kale_integral5_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_integral5_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_integral6_task = _kale_integral6_op(density_p)\ .add_pvolumes(_kale_pvolumes_dict)\ .after() _kale_step_limits = {'cpu': '1', 'memory': '40Gi'} for _kale_k, _kale_v in _kale_step_limits.items(): _kale_integral6_task.container.add_resource_limit(_kale_k, _kale_v) _kale_integral6_task.container.working_dir = "//shared_volume" _kale_integral6_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'integral6': '/integral6.html'}) _kale_integral6_task.output_artifact_paths.update(_kale_output_artifacts) _kale_integral6_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_integral6_task.dependent_names + _kale_volume_step_names) _kale_integral6_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_integral6_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters))
def _create_deployment_object(self): """Creates the deployment object for the grader service using environment variables Returns: V1Deployment: a valid kubernetes deployment object """ # Configureate Pod template container # Volumes to mount as subPaths of PV sub_path_grader_home = str(self.course_dir.parent).strip('/') sub_path_exchange = str( self.exchange_dir.relative_to(EXCHANGE_MNT_ROOT)) # define the container to launch container = client.V1Container( name='grader-notebook', image=GRADER_IMAGE_NAME, command=[ 'start-notebook.sh', f'--group=formgrade-{self.course_id}' ], ports=[client.V1ContainerPort(container_port=8888)], working_dir=f'/home/{self.grader_name}', resources=client.V1ResourceRequirements(requests={ "cpu": "100m", "memory": "200Mi" }, limits={ "cpu": "500m", "memory": "1G" }), security_context=client.V1SecurityContext( allow_privilege_escalation=False), env=[ client.V1EnvVar(name='JUPYTERHUB_SERVICE_NAME', value=self.course_id), client.V1EnvVar(name='JUPYTERHUB_API_TOKEN', value=self.grader_token), # we're using the K8s Service name 'hub' (defined in the jhub helm chart) # to connect from our grader-notebooks client.V1EnvVar(name='JUPYTERHUB_API_URL', value='http://hub:8081/hub/api'), client.V1EnvVar(name='JUPYTERHUB_BASE_URL', value='/'), client.V1EnvVar(name='JUPYTERHUB_SERVICE_PREFIX', value=f'/services/{self.course_id}/'), client.V1EnvVar(name='JUPYTERHUB_CLIENT_ID', value=f'service-{self.course_id}'), client.V1EnvVar(name='JUPYTERHUB_USER', value=self.grader_name), client.V1EnvVar(name='NB_UID', value=str(NB_UID)), client.V1EnvVar(name='NB_GID', value=str(NB_GID)), client.V1EnvVar(name='NB_USER', value=self.grader_name), ], volume_mounts=[ client.V1VolumeMount(mount_path=f'/home/{self.grader_name}', name=GRADER_PVC, sub_path=sub_path_grader_home), client.V1VolumeMount(mount_path='/srv/nbgrader/exchange', name=GRADER_EXCHANGE_SHARED_PVC, sub_path=sub_path_exchange), ], ) # Create and configurate a spec section template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta(labels={ 'component': self.grader_name, 'app': 'illumidesk' }), spec=client.V1PodSpec( containers=[container], security_context=client.V1PodSecurityContext(run_as_user=0), volumes=[ client.V1Volume( name=GRADER_PVC, persistent_volume_claim=client. V1PersistentVolumeClaimVolumeSource( claim_name=GRADER_PVC), ), client.V1Volume( name=GRADER_EXCHANGE_SHARED_PVC, persistent_volume_claim=client. V1PersistentVolumeClaimVolumeSource( claim_name=GRADER_EXCHANGE_SHARED_PVC), ), ], ), ) # Create the specification of deployment spec = client.V1DeploymentSpec( replicas=1, template=template, selector={'matchLabels': { 'component': self.grader_name }}) # Instantiate the deployment object deployment = client.V1Deployment( api_version="apps/v1", kind="Deployment", metadata=client.V1ObjectMeta(name=self.grader_name), spec=spec) return deployment
def auto_generated_pipeline(a='1', b='Some string', c='5'): _kale_pvolumes_dict = OrderedDict() _kale_volume_step_names = [] _kale_volume_name_parameters = [] _kale_marshal_vop = _kfp_dsl.VolumeOp( name="kale-marshal-volume", resource_name="kale-marshal-pvc", modes=['ReadWriteMany'], size="1Gi" ) _kale_volume_step_names.append(_kale_marshal_vop.name) _kale_volume_name_parameters.append( _kale_marshal_vop.outputs["name"].full_name) _kale_pvolumes_dict['/marshal'] = _kale_marshal_vop.volume _kale_volume_step_names.sort() _kale_volume_name_parameters.sort() _kale_step1_task = _kale_step1_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after() _kale_step1_task.container.working_dir = "/test" _kale_step1_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_step1_task.output_artifact_paths.update(_kale_output_artifacts) _kale_step1_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_step1_task.dependent_names + _kale_volume_step_names) _kale_step1_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_step1_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_step3_task = _kale_step3_op(b)\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_step1_task) _kale_step3_task.container.working_dir = "/test" _kale_step3_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_step3_task.output_artifact_paths.update(_kale_output_artifacts) _kale_step3_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_step3_task.dependent_names + _kale_volume_step_names) _kale_step3_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_step3_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_step2_task = _kale_step2_op(a, c)\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_step1_task) _kale_step2_task.container.working_dir = "/test" _kale_step2_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_step2_task.output_artifact_paths.update(_kale_output_artifacts) _kale_step2_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_step2_task.dependent_names + _kale_volume_step_names) _kale_step2_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_step2_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_final_auto_snapshot_task = _kale_final_auto_snapshot_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_step3_task, _kale_step2_task) _kale_final_auto_snapshot_task.container.working_dir = "/test" _kale_final_auto_snapshot_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_final_auto_snapshot_task.output_artifact_paths.update( _kale_output_artifacts) _kale_final_auto_snapshot_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_final_auto_snapshot_task.dependent_names + _kale_volume_step_names) _kale_final_auto_snapshot_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_final_auto_snapshot_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters))
'-y', '%s' % get_base64_encoded_str(namelist_input), '-k', '/wrf/config/gcs.json', '-v', 'curwsl_nfs_1:/wrf/output', '-v', 'curwsl_archive_1:/wrf/archive', ] logging.info('Creating container spec for image ' + image) container = client.V1Container(name='test-container', image_pull_policy='IfNotPresent') container.image = image container.command = command container.args = command_args container.volume_mounts = [client.V1VolumeMount(mount_path='/wrf/geog', name='test-vol'), client.V1VolumeMount(mount_path='/wrf/config', name='test-sec-vol')] container.resources = client.V1ResourceRequirements(requests={'cpu': 4}) container.security_context = client.V1SecurityContext(privileged=True) logging.info('Initializing pod') pod = client.V1Pod() logging.info('Creating pod metadata ') pod_metadata = client.V1ObjectMeta(name='test-pod') pod.metadata = pod_metadata logging.info('Creating volume list') vols = [client.V1Volume(name='test-vol', gce_persistent_disk=client.V1GCEPersistentDiskVolumeSource(pd_name='wrf-geog-disk1', read_only=True)), client.V1Volume(name='test-sec-vol', secret=client.V1SecretVolumeSource(secret_name='google-app-creds'))]
def _create_deployment_object( self, job_name, job_image, deployment_name, port=80, replicas=1, cmd_string=None, engine_json_file='~/.ipython/profile_default/security/ipcontroller-engine.json', engine_dir='.'): """ Create a kubernetes deployment for the job. Args: - job_name (string) : Name of the job and deployment - job_image (string) : Docker image to launch KWargs: - port (integer) : Container port - replicas : Number of replica containers to maintain Returns: - True: The deployment object to launch """ # sorry, quick hack that doesn't pass this stuff through to test it works. # TODO it also doesn't only add what is set :( security_context = None if 'security' in self.config['execution']: security_context = client.V1SecurityContext( run_as_group=self.group_id, run_as_user=self.user_id, run_as_non_root=self.run_as_non_root) # self.user_id = None # self.group_id = None # self.run_as_non_root = None # Create the enviornment variables and command to initiate IPP environment_vars = client.V1EnvVar(name="TEST", value="SOME DATA") launch_args = ["-c", "{0}; /app/deploy.sh;".format(cmd_string)] print(launch_args) # Configureate Pod template container container = None if security_context: container = client.V1Container( name=job_name, image=job_image, ports=[client.V1ContainerPort(container_port=port)], command=['/bin/bash'], args=launch_args, env=[environment_vars], security_context=security_context) else: container = client.V1Container( name=job_name, image=job_image, ports=[client.V1ContainerPort(container_port=port)], command=['/bin/bash'], args=launch_args, env=[environment_vars]) # Create a secret to enable pulling images from secure repositories secret = None if self.secret: secret = client.V1LocalObjectReference(name=self.secret) # Create and configurate a spec section template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta(labels={"app": job_name}), spec=client.V1PodSpec(containers=[container], image_pull_secrets=[secret])) # Create the specification of deployment spec = client.ExtensionsV1beta1DeploymentSpec(replicas=replicas, template=template) # Instantiate the deployment object deployment = client.ExtensionsV1beta1Deployment( api_version="extensions/v1beta1", kind="Deployment", metadata=client.V1ObjectMeta(name=deployment_name), spec=spec) return deployment
def _create_job_spec( self, name, command=None, image=None, env_vars=None, overwrite_input_parameters=None, overwrite_operational_options=None, ): """Instantiate a Kubernetes job. :param name: Name of the job. :param image: Docker image to use to run the job on. :param command: List of commands to run on the given job. :param env_vars: List of environment variables (dictionaries) to inject into the workflow engine container. :param interactive_session_type: One of the available interactive session types. :param overwrite_input_params: Dictionary with parameters to be overwritten or added to the current workflow run. :param type: Dict :param overwrite_operational_options: Dictionary with operational options to be overwritten or added to the current workflow run. :param type: Dict """ image = image or self._workflow_engine_image() command = command or self._workflow_engine_command( overwrite_input_parameters=overwrite_input_parameters, overwrite_operational_options=overwrite_operational_options, ) workflow_engine_env_vars = env_vars or self._workflow_engine_env_vars() job_controller_env_vars = [] owner_id = str(self.workflow.owner_id) command = format_cmd(command) workspace_mount, workspace_volume = get_workspace_volume( self.workflow.workspace_path) db_mount, shared_volume = get_shared_volume("db") workflow_metadata = client.V1ObjectMeta( name=name, labels={ "reana_workflow_mode": "batch", "reana-run-batch-workflow-uuid": str(self.workflow.id_), }, namespace=REANA_RUNTIME_KUBERNETES_NAMESPACE, ) job = client.V1Job() job.api_version = "batch/v1" job.kind = "Job" job.metadata = workflow_metadata spec = client.V1JobSpec(template=client.V1PodTemplateSpec()) spec.template.metadata = workflow_metadata workflow_engine_container = client.V1Container( name=current_app.config["WORKFLOW_ENGINE_NAME"], image=image, image_pull_policy="IfNotPresent", env=[], volume_mounts=[], command=["/bin/bash", "-c"], args=command, ) workflow_engine_env_vars.extend([ { "name": "REANA_JOB_CONTROLLER_SERVICE_PORT_HTTP", "value": str(current_app.config["JOB_CONTROLLER_CONTAINER_PORT"]), }, { "name": "REANA_JOB_CONTROLLER_SERVICE_HOST", "value": "localhost" }, { "name": "REANA_COMPONENT_PREFIX", "value": REANA_COMPONENT_PREFIX }, { "name": "REANA_COMPONENT_NAMING_SCHEME", "value": REANA_COMPONENT_NAMING_SCHEME, }, { "name": "REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE", "value": REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE, }, { "name": "REANA_RUNTIME_KUBERNETES_NAMESPACE", "value": REANA_RUNTIME_KUBERNETES_NAMESPACE, }, { "name": "REANA_JOB_CONTROLLER_CONNECTION_CHECK_SLEEP", "value": str(REANA_JOB_CONTROLLER_CONNECTION_CHECK_SLEEP), }, ]) workflow_engine_container.env.extend(workflow_engine_env_vars) workflow_engine_container.security_context = client.V1SecurityContext( run_as_group=WORKFLOW_RUNTIME_USER_GID, run_as_user=WORKFLOW_RUNTIME_USER_UID, ) workflow_engine_container.volume_mounts = [workspace_mount] secrets_store = REANAUserSecretsStore(owner_id) job_controller_env_secrets = secrets_store.get_env_secrets_as_k8s_spec( ) user = secrets_store.get_secret_value( "CERN_USER") or WORKFLOW_RUNTIME_USER_NAME job_controller_container = client.V1Container( name=current_app.config["JOB_CONTROLLER_NAME"], image=current_app.config["JOB_CONTROLLER_IMAGE"], image_pull_policy="IfNotPresent", env=[], volume_mounts=[], command=["/bin/bash", "-c"], args=self._create_job_controller_startup_cmd(user), ports=[], ) job_controller_env_vars.extend([ { "name": "REANA_USER_ID", "value": owner_id }, { "name": "CERN_USER", "value": user }, { "name": "USER", "value": user }, # Required by HTCondor { "name": "K8S_CERN_EOS_AVAILABLE", "value": K8S_CERN_EOS_AVAILABLE }, { "name": "IMAGE_PULL_SECRETS", "value": ",".join(IMAGE_PULL_SECRETS) }, { "name": "REANA_SQLALCHEMY_DATABASE_URI", "value": SQLALCHEMY_DATABASE_URI, }, { "name": "REANA_STORAGE_BACKEND", "value": REANA_STORAGE_BACKEND }, { "name": "REANA_COMPONENT_PREFIX", "value": REANA_COMPONENT_PREFIX }, { "name": "REANA_COMPONENT_NAMING_SCHEME", "value": REANA_COMPONENT_NAMING_SCHEME, }, { "name": "REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE", "value": REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE, }, { "name": "REANA_RUNTIME_KUBERNETES_NAMESPACE", "value": REANA_RUNTIME_KUBERNETES_NAMESPACE, }, { "name": "REANA_JOB_HOSTPATH_MOUNTS", "value": json.dumps(REANA_JOB_HOSTPATH_MOUNTS), }, { "name": "REANA_RUNTIME_KUBERNETES_KEEP_ALIVE_JOBS_WITH_STATUSES", "value": ",".join( REANA_RUNTIME_KUBERNETES_KEEP_ALIVE_JOBS_WITH_STATUSES), }, { "name": "REANA_KUBERNETES_JOBS_MEMORY_LIMIT", "value": REANA_KUBERNETES_JOBS_MEMORY_LIMIT, }, { "name": "REANA_KUBERNETES_JOBS_MAX_USER_MEMORY_LIMIT", "value": REANA_KUBERNETES_JOBS_MAX_USER_MEMORY_LIMIT, }, { "name": "REANA_KUBERNETES_JOBS_TIMEOUT_LIMIT", "value": REANA_KUBERNETES_JOBS_TIMEOUT_LIMIT, }, { "name": "REANA_KUBERNETES_JOBS_MAX_USER_TIMEOUT_LIMIT", "value": REANA_KUBERNETES_JOBS_MAX_USER_TIMEOUT_LIMIT, }, { "name": "WORKSPACE_PATHS", "value": json.dumps(WORKSPACE_PATHS) }, ]) job_controller_container.env.extend(job_controller_env_vars) job_controller_container.env.extend(job_controller_env_secrets) if REANA_RUNTIME_JOBS_KUBERNETES_NODE_LABEL: job_controller_container.env.append( { "name": "REANA_RUNTIME_JOBS_KUBERNETES_NODE_LABEL", "value": os.getenv("REANA_RUNTIME_JOBS_KUBERNETES_NODE_LABEL"), }, ) secrets_volume_mount = secrets_store.get_secrets_volume_mount_as_k8s_spec( ) job_controller_container.volume_mounts = [workspace_mount, db_mount] job_controller_container.volume_mounts.append(secrets_volume_mount) job_controller_container.ports = [{ "containerPort": current_app.config["JOB_CONTROLLER_CONTAINER_PORT"] }] containers = [workflow_engine_container, job_controller_container] spec.template.spec = client.V1PodSpec( containers=containers, node_selector=REANA_RUNTIME_BATCH_KUBERNETES_NODE_LABEL, ) spec.template.spec.service_account_name = ( REANA_RUNTIME_KUBERNETES_SERVICEACCOUNT_NAME) volumes = [ workspace_volume, shared_volume, secrets_store.get_file_secrets_volume_as_k8s_specs(), ] # filter out volumes with the same name spec.template.spec.volumes = list({v["name"]: v for v in volumes}.values()) if os.getenv("FLASK_ENV") == "development": code_volume_name = "reana-code" code_mount_path = "/code" k8s_code_volume = client.V1Volume(name=code_volume_name) k8s_code_volume.host_path = client.V1HostPathVolumeSource( code_mount_path) spec.template.spec.volumes.append(k8s_code_volume) for container in spec.template.spec.containers: container.env.extend(current_app.config["DEBUG_ENV_VARS"]) sub_path = f"reana-{container.name}" if container.name == "workflow-engine": sub_path += f"-{self.workflow.type_}" container.volume_mounts.append({ "name": code_volume_name, "mountPath": code_mount_path, "subPath": sub_path, }) job.spec = spec job.spec.template.spec.restart_policy = "Never" job.spec.backoff_limit = 0 return job
def auto_generated_pipeline(booltest='True', d1='5', d2='6', strtest='test'): _kale_pvolumes_dict = OrderedDict() _kale_volume_step_names = [] _kale_volume_name_parameters = [] _kale_marshal_vop = _kfp_dsl.VolumeOp(name="kale-marshal-volume", resource_name="kale-marshal-pvc", modes=['ReadWriteMany'], size="1Gi") _kale_volume_step_names.append(_kale_marshal_vop.name) _kale_volume_name_parameters.append( _kale_marshal_vop.outputs["name"].full_name) _kale_pvolumes_dict['/marshal'] = _kale_marshal_vop.volume _kale_volume_step_names.sort() _kale_volume_name_parameters.sort() _kale_create_matrix_task = _kale_create_matrix_op(d1, d2)\ .add_pvolumes(_kale_pvolumes_dict)\ .after() _kale_step_limits = {'nvidia.com/gpu': '2'} for _kale_k, _kale_v in _kale_step_limits.items(): _kale_create_matrix_task.container.add_resource_limit(_kale_k, _kale_v) _kale_create_matrix_task.container.working_dir = "/kale" _kale_create_matrix_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-metrics': '/tmp/mlpipeline-metrics.json'}) _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'create_matrix': '/create_matrix.html'}) _kale_create_matrix_task.output_artifact_paths.update( _kale_output_artifacts) _kale_create_matrix_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_create_matrix_task.dependent_names + _kale_volume_step_names) _kale_create_matrix_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_create_matrix_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_sum_matrix_task = _kale_sum_matrix_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_create_matrix_task) _kale_sum_matrix_task.container.working_dir = "/kale" _kale_sum_matrix_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-metrics': '/tmp/mlpipeline-metrics.json'}) _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'sum_matrix': '/sum_matrix.html'}) _kale_sum_matrix_task.output_artifact_paths.update(_kale_output_artifacts) _kale_sum_matrix_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_sum_matrix_task.dependent_names + _kale_volume_step_names) _kale_sum_matrix_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_sum_matrix_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters))
def auto_generated_pipeline(vol_shared_volume_kale_volumes='titanic'): #pvolumes_dict = OrderedDict() # #annotations = {} # #volume = dsl.PipelineVolume(pvc=vol_shared_volume_kale_volumes) # #pvolumes_dict['/shared_volume/kale_volumes/'] = volume # #marshal_vop = dsl.VolumeOp( # name="kale_marshal_volume", # resource_name="kale-marshal-pvc", # modes=dsl.VOLUME_MODE_RWM, # size="1Gi" #) #pvolumes_dict['/marshal'] = marshal_vop.volume #loaddata_task = loaddata_op(vol_shared_volume_kale_volumes)\ # .add_pvolumes(pvolumes_dict)\ # .after() #loaddata_task.container.working_dir = "/shared_volume/notebooks" #loaddata_task.container.set_security_context( # k8s_client.V1SecurityContext(run_as_user=0)) # #datapreprocessing_task = datapreprocessing_op(vol_shared_volume_kale_volumes)\ # .add_pvolumes(pvolumes_dict)\ # .after(loaddata_task) #datapreprocessing_task.container.working_dir = "/shared_volume/notebooks" #datapreprocessing_task.container.set_security_context( # k8s_client.V1SecurityContext(run_as_user=0)) # #featureengineering_task = featureengineering_op(vol_shared_volume_kale_volumes)\ # .add_pvolumes(pvolumes_dict)\ # .after(datapreprocessing_task) #featureengineering_task.container.working_dir = "/shared_volume/notebooks" #featureengineering_task.container.set_security_context( # k8s_client.V1SecurityContext(run_as_user=0)) # #decisiontree_task = decisiontree_op(vol_shared_volume_kale_volumes)\ # .add_pvolumes(pvolumes_dict)\ # .after(featureengineering_task) #decisiontree_task.container.working_dir = "/shared_volume/notebooks" #decisiontree_task.container.set_security_context( # k8s_client.V1SecurityContext(run_as_user=0)) # #svm_task = svm_op(vol_shared_volume_kale_volumes)\ # .add_pvolumes(pvolumes_dict)\ # .after(featureengineering_task) #svm_task.container.working_dir = "/shared_volume/notebooks" #svm_task.container.set_security_context( # k8s_client.V1SecurityContext(run_as_user=0)) # #naivebayes_task = naivebayes_op(vol_shared_volume_kale_volumes)\ # .add_pvolumes(pvolumes_dict)\ # .after(featureengineering_task) #naivebayes_task.container.working_dir = "/shared_volume/notebooks" #naivebayes_task.container.set_security_context( # k8s_client.V1SecurityContext(run_as_user=0)) # #logisticregression_task = logisticregression_op(vol_shared_volume_kale_volumes)\ # .add_pvolumes(pvolumes_dict)\ # .after(featureengineering_task) #logisticregression_task.container.working_dir = "/shared_volume/notebooks" #logisticregression_task.container.set_security_context( # k8s_client.V1SecurityContext(run_as_user=0)) # #randomforest_task = randomforest_op(vol_shared_volume_kale_volumes)\ # .add_pvolumes(pvolumes_dict)\ # .after(featureengineering_task) #randomforest_task.container.working_dir = "/shared_volume/notebooks" #randomforest_task.container.set_security_context( # k8s_client.V1SecurityContext(run_as_user=0)) # #results_task = results_op(vol_shared_volume_kale_volumes)\ # .add_pvolumes(pvolumes_dict)\ # .after(randomforest_task, logisticregression_task, naivebayes_task, svm_task, decisiontree_task) #results_task.container.working_dir = "/shared_volume/notebooks" #results_task.container.set_security_context( # k8s_client.V1SecurityContext(run_as_user=0)) loaddata_task = loaddata_op(vol_shared_volume_kale_volumes)\ .add_volume(k8s_client.V1Volume(name='efs-pvc', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(claim_name='efs')))\ .add_volume_mount(k8s_client.V1VolumeMount(mount_path='/shared_volume', name='efs-pvc'))\ .after() loaddata_task.add_pod_annotation('iam.amazonaws.com/role', 'arn:aws:iam::<micuenta>:role/<mirol>') loaddata_task.container.working_dir = "/shared_volume/notebooks" loaddata_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) datapreprocessing_task = datapreprocessing_op(vol_shared_volume_kale_volumes)\ .add_volume(k8s_client.V1Volume(name='efs-pvc', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(claim_name='efs')))\ .add_volume_mount(k8s_client.V1VolumeMount(mount_path='/shared_volume', name='efs-pvc'))\ .after(loaddata_task) datapreprocessing_task.add_pod_annotation( 'iam.amazonaws.com/role', 'arn:aws:iam::<micuenta>:role/<mirol>') datapreprocessing_task.container.working_dir = "/shared_volume/notebooks" datapreprocessing_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) featureengineering_task = featureengineering_op(vol_shared_volume_kale_volumes)\ .add_volume(k8s_client.V1Volume(name='efs-pvc', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(claim_name='efs')))\ .add_volume_mount(k8s_client.V1VolumeMount(mount_path='/shared_volume', name='efs-pvc'))\ .after(datapreprocessing_task) featureengineering_task.add_pod_annotation( 'iam.amazonaws.com/role', 'arn:aws:iam::<micuenta>:role/<mirol>') featureengineering_task.container.working_dir = "/shared_volume/notebooks" featureengineering_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) decisiontree_task = decisiontree_op(vol_shared_volume_kale_volumes)\ .add_volume(k8s_client.V1Volume(name='efs-pvc', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(claim_name='efs')))\ .add_volume_mount(k8s_client.V1VolumeMount(mount_path='/shared_volume', name='efs-pvc'))\ .after(featureengineering_task) decisiontree_task.add_pod_annotation( 'iam.amazonaws.com/role', 'arn:aws:iam::<micuenta>:role/<mirol>') decisiontree_task.container.working_dir = "/shared_volume/notebooks" decisiontree_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) svm_task = svm_op(vol_shared_volume_kale_volumes)\ .add_volume(k8s_client.V1Volume(name='efs-pvc', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(claim_name='efs')))\ .add_volume_mount(k8s_client.V1VolumeMount(mount_path='/shared_volume', name='efs-pvc'))\ .after(featureengineering_task) svm_task.add_pod_annotation('iam.amazonaws.com/role', 'arn:aws:iam::<micuenta>:role/<mirol>') svm_task.container.working_dir = "/shared_volume/notebooks" svm_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) # naivebayes_task = naivebayes_op(vol_shared_volume_kale_volumes)\ .add_volume(k8s_client.V1Volume(name='efs-pvc', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(claim_name='efs')))\ .add_volume_mount(k8s_client.V1VolumeMount(mount_path='/shared_volume', name='efs-pvc'))\ .after(featureengineering_task) naivebayes_task.add_pod_annotation('iam.amazonaws.com/role', 'arn:aws:iam::<micuenta>:role/<mirol>') naivebayes_task.container.working_dir = "/shared_volume/notebooks" naivebayes_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) # logisticregression_task = logisticregression_op(vol_shared_volume_kale_volumes)\ .add_volume(k8s_client.V1Volume(name='efs-pvc', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(claim_name='efs')))\ .add_volume_mount(k8s_client.V1VolumeMount(mount_path='/shared_volume', name='efs-pvc'))\ .after(featureengineering_task) logisticregression_task.add_pod_annotation( 'iam.amazonaws.com/role', 'arn:aws:iam::<micuenta>:role/<mirol>') logisticregression_task.container.working_dir = "/shared_volume/notebooks" logisticregression_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) # randomforest_task = randomforest_op(vol_shared_volume_kale_volumes)\ .add_volume(k8s_client.V1Volume(name='efs-pvc', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(claim_name='efs')))\ .add_volume_mount(k8s_client.V1VolumeMount(mount_path='/shared_volume', name='efs-pvc'))\ .after(featureengineering_task) randomforest_task.add_pod_annotation( 'iam.amazonaws.com/role', 'arn:aws:iam::<micuenta>:role/<mirol>') randomforest_task.container.working_dir = "/shared_volume/notebooks" randomforest_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) # results_task = results_op(vol_shared_volume_kale_volumes)\ .add_volume(k8s_client.V1Volume(name='efs-pvc', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(claim_name='efs')))\ .add_volume_mount(k8s_client.V1VolumeMount(mount_path='/shared_volume', name='efs-pvc'))\ .after(randomforest_task, logisticregression_task, naivebayes_task, svm_task, decisiontree_task) results_task.add_pod_annotation('iam.amazonaws.com/role', 'arn:aws:iam::<micuenta>:role/<mirol>') results_task.container.working_dir = "/shared_volume/notebooks" results_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0))
def auto_generated_pipeline(): pvolumes_dict = OrderedDict() volume_step_names = [] volume_name_parameters = [] marshal_vop = dsl.VolumeOp( name="kale-marshal-volume", resource_name="kale-marshal-pvc", modes=dsl.VOLUME_MODE_RWM, size="1Gi" ) volume_step_names.append(marshal_vop.name) volume_name_parameters.append(marshal_vop.outputs["name"].full_name) pvolumes_dict['/marshal'] = marshal_vop.volume volume_step_names.sort() volume_name_parameters.sort() loaddata_task = loaddata_op()\ .add_pvolumes(pvolumes_dict)\ .after() loaddata_task.container.working_dir = "/kale" loaddata_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'loaddata': '/loaddata.html'}) loaddata_task.output_artifact_paths.update(output_artifacts) loaddata_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") dep_names = loaddata_task.dependent_names + volume_step_names loaddata_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: loaddata_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters)) datapreprocessing_task = datapreprocessing_op()\ .add_pvolumes(pvolumes_dict)\ .after(loaddata_task) datapreprocessing_task.container.working_dir = "/kale" datapreprocessing_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'datapreprocessing': '/datapreprocessing.html'}) datapreprocessing_task.output_artifact_paths.update(output_artifacts) datapreprocessing_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") dep_names = datapreprocessing_task.dependent_names + volume_step_names datapreprocessing_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: datapreprocessing_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters)) featureengineering_task = featureengineering_op()\ .add_pvolumes(pvolumes_dict)\ .after(datapreprocessing_task) featureengineering_task.container.working_dir = "/kale" featureengineering_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'featureengineering': '/featureengineering.html'}) featureengineering_task.output_artifact_paths.update(output_artifacts) featureengineering_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") dep_names = featureengineering_task.dependent_names + volume_step_names featureengineering_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: featureengineering_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters)) decisiontree_task = decisiontree_op()\ .add_pvolumes(pvolumes_dict)\ .after(featureengineering_task) decisiontree_task.container.working_dir = "/kale" decisiontree_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'decisiontree': '/decisiontree.html'}) decisiontree_task.output_artifact_paths.update(output_artifacts) decisiontree_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") dep_names = decisiontree_task.dependent_names + volume_step_names decisiontree_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: decisiontree_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters)) svm_task = svm_op()\ .add_pvolumes(pvolumes_dict)\ .after(featureengineering_task) svm_task.container.working_dir = "/kale" svm_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'svm': '/svm.html'}) svm_task.output_artifact_paths.update(output_artifacts) svm_task.add_pod_label("pipelines.kubeflow.org/metadata_written", "true") dep_names = svm_task.dependent_names + volume_step_names svm_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: svm_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters)) naivebayes_task = naivebayes_op()\ .add_pvolumes(pvolumes_dict)\ .after(featureengineering_task) naivebayes_task.container.working_dir = "/kale" naivebayes_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'naivebayes': '/naivebayes.html'}) naivebayes_task.output_artifact_paths.update(output_artifacts) naivebayes_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") dep_names = naivebayes_task.dependent_names + volume_step_names naivebayes_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: naivebayes_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters)) logisticregression_task = logisticregression_op()\ .add_pvolumes(pvolumes_dict)\ .after(featureengineering_task) logisticregression_task.container.working_dir = "/kale" logisticregression_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'logisticregression': '/logisticregression.html'}) logisticregression_task.output_artifact_paths.update(output_artifacts) logisticregression_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") dep_names = logisticregression_task.dependent_names + volume_step_names logisticregression_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: logisticregression_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters)) randomforest_task = randomforest_op()\ .add_pvolumes(pvolumes_dict)\ .after(featureengineering_task) randomforest_task.container.working_dir = "/kale" randomforest_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'randomforest': '/randomforest.html'}) randomforest_task.output_artifact_paths.update(output_artifacts) randomforest_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") dep_names = randomforest_task.dependent_names + volume_step_names randomforest_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: randomforest_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters)) results_task = results_op()\ .add_pvolumes(pvolumes_dict)\ .after(randomforest_task, logisticregression_task, naivebayes_task, svm_task, decisiontree_task) results_task.container.working_dir = "/kale" results_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) output_artifacts = {} output_artifacts.update( {'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'}) output_artifacts.update({'results': '/results.html'}) results_task.output_artifact_paths.update(output_artifacts) results_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") dep_names = results_task.dependent_names + volume_step_names results_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(dep_names)) if volume_name_parameters: results_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(volume_name_parameters))
def template(context): """ handle yml env """ name = context.get("name") version = context.get("version") labels = { "app": name, "version": "v1" } if not version else { "app": name.strip("-v2"), "version": version } image_tag = context["name"].split('-')[1] image = context["image_namespace"] + image_tag + ":" + context[ "image_branch"] args = [arg for arg in context["args"]] if context.get("args") else None limits, requests = context["resources"]["limits"], context["resources"][ "requests"] replicas = context.get("replicas", 1) workingDir = context["workingDir"] if name == "backend-logproxy": annotations = {"sidecar.istio.io/inject": "false"} else: annotations = {"traffic.sidecar.istio.io/excludeOutboundPorts": "6379"} """ handle cmdb env """ filename = "env_" + name.split("-")[1] + ".yml" env = handle_env("/tmp/{}".format(filename)) """ k8s yaml 组件模块 """ #从svn分支configmap目录中获取相关的目录结构 parentDir, subdir = handle_configmap("configmap") volumemounts = [ client.V1VolumeMount(mount_path="/{}".format(parentDir), name="mainfiles") ] volumes = [ client.V1Volume( name="mainfiles", config_map=client.V1ConfigMapVolumeSource(name="mainfiles")) ] for dir in subdir: volumemounts.append( client.V1VolumeMount(mount_path="/{}/{}".format(parentDir, dir), name=dir)) volumes.append( client.V1Volume( name=dir, config_map=client.V1ConfigMapVolumeSource(name=dir))) if name.startswith("frontend-dispatch"): containers = [ client.V1Container( name=name, image=image, env=env, args=args, volume_mounts=volumemounts, image_pull_policy="Always", lifecycle=client.V1Lifecycle(pre_stop=client.V1Handler( _exec=client.V1ExecAction( command=["nginx", "-s", "quit"]))), readiness_probe=client.V1Probe(_exec=client.V1ExecAction( command=['cat', '/tmp/container_ready']), initial_delay_seconds=10, period_seconds=5), resources=client.V1ResourceRequirements(limits=limits, requests=requests), security_context=client.V1SecurityContext(privileged=True), working_dir=workingDir, ) ] else: containers = [ client.V1Container( name=name, image=image, env=env, args=args, volume_mounts=volumemounts, image_pull_policy="Always", readiness_probe=client.V1Probe(_exec=client.V1ExecAction( command=['cat', '/tmp/container_ready']), initial_delay_seconds=10, period_seconds=5), resources=client.V1ResourceRequirements(limits=limits, requests=requests), security_context=client.V1SecurityContext(privileged=True), working_dir=workingDir, ) ] template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta(labels=labels, annotations=annotations), spec=client.V1PodSpec( containers=containers, dns_policy="ClusterFirst", image_pull_secrets=[ client.V1LocalObjectReference(name="image-pull-secret") ], restart_policy="Always", volumes=volumes)) spec = client.V1DeploymentSpec( replicas=replicas, selector=client.V1LabelSelector(match_labels=labels), template=template, strategy=client.ExtensionsV1beta1DeploymentStrategy( rolling_update=client.ExtensionsV1beta1RollingUpdateDeployment( max_surge=1, max_unavailable='25%'), type="RollingUpdate", ), ) return client.V1Deployment(api_version="apps/v1", kind="Deployment", metadata=client.V1ObjectMeta(name=name, labels=labels), spec=spec)
def get_pod_manifest(self): """Include volume with the git repository.""" repository = yield self.git_repository() options = self.user_options # make sure the pod name is less than 64 characters - if longer, keep # the last 16 untouched since it is the server hash if len(self.pod_name) > 63: self.pod_name = self.pod_name[:47] + self.pod_name[-16:] # Process the requested server options server_options = options.get("server_options", {}) self.default_url = server_options.get("defaultUrl") self.cpu_guarantee = float(server_options.get("cpu_request", 0.1)) # Make the user pods be in Guaranteed QoS class if the user # had specified a memory request. Otherwise use a sensible default. self.mem_guarantee = server_options.get("mem_request", "500M") self.mem_limit = server_options.get("mem_request", "1G") gpu = server_options.get("gpu_request", {}) if gpu: self.extra_resource_limits = {"nvidia.com/gpu": str(gpu)} # Configure the git repository volume git_volume_name = self.pod_name[:54] + "-git-repo" # 1. Define a new empty volume. self.volumes = [ volume for volume in self.volumes if volume["name"] != git_volume_name ] volume = {"name": git_volume_name, "emptyDir": {}} self.volumes.append(volume) # 2. Define a volume mount for both init and notebook containers. mount_path = f'/work/{options["project"]}' volume_mount = {"mountPath": mount_path, "name": git_volume_name} # 3. Configure the init container init_container_name = "git-clone" self.init_containers = [ container for container in self.init_containers if not container.name.startswith(init_container_name) ] lfs_auto_fetch = server_options.get("lfs_auto_fetch") init_container = client.V1Container( name=init_container_name, env=[ client.V1EnvVar(name="MOUNT_PATH", value=mount_path), client.V1EnvVar(name="REPOSITORY", value=repository), client.V1EnvVar( name="LFS_AUTO_FETCH", value="1" if lfs_auto_fetch else "0", ), client.V1EnvVar(name="COMMIT_SHA", value=str(options.get("commit_sha"))), client.V1EnvVar(name="BRANCH", value=options.get("branch", "master")), client.V1EnvVar(name="JUPYTERHUB_USER", value=self.user.name), ], image=options.get("git_clone_image"), volume_mounts=[volume_mount], working_dir=mount_path, security_context=client.V1SecurityContext(run_as_user=0), ) self.init_containers.append(init_container) # 4. Configure notebook container git repo volume mount self.volume_mounts = [ volume_mount for volume_mount in self.volume_mounts if volume_mount["mountPath"] != mount_path ] self.volume_mounts.append(volume_mount) # 5. Configure autosaving script execution hook self.lifecycle_hooks = { "preStop": { "exec": { "command": [ "/bin/sh", "-c", "/usr/local/bin/pre-stop.sh", "||", "true", ] } } } # Finalize the pod configuration # Set the repository path to the working directory self.working_dir = mount_path self.notebook_dir = mount_path # add git project-specific annotations self.extra_annotations = { RENKU_ANNOTATION_PREFIX + "namespace": options.get("namespace"), RENKU_ANNOTATION_PREFIX + "projectName": options.get("project"), RENKU_ANNOTATION_PREFIX + "projectId": "{}".format(options.get("project_id")), RENKU_ANNOTATION_PREFIX + "branch": options.get("branch"), RENKU_ANNOTATION_PREFIX + "commit-sha": options.get("commit_sha"), } # add username to labels safe_username = escapism.escape(self.user.name, escape_char="-").lower() self.extra_labels = { RENKU_ANNOTATION_PREFIX + "username": safe_username } self.delete_grace_period = 30 pod = yield super().get_pod_manifest() # Because repository comes from a coroutine, we can't put it simply in `get_env()` pod.spec.containers[0].env.append( client.V1EnvVar("CI_REPOSITORY_URL", repository)) # Add image pull secrets if options.get("image_pull_secrets"): secrets = [ client.V1LocalObjectReference(name=name) for name in options.get("image_pull_secrets") ] pod.spec.image_pull_secrets = secrets return pod
def auto_generated_pipeline(): _kale_pvolumes_dict = OrderedDict() _kale_volume_step_names = [] _kale_volume_name_parameters = [] _kale_marshal_vop = _kfp_dsl.VolumeOp(name="kale-marshal-volume", resource_name="kale-marshal-pvc", modes=_kfp_dsl.VOLUME_MODE_RWM, size="1Gi") _kale_volume_step_names.append(_kale_marshal_vop.name) _kale_volume_name_parameters.append( _kale_marshal_vop.outputs["name"].full_name) _kale_pvolumes_dict['/marshal'] = _kale_marshal_vop.volume _kale_volume_step_names.sort() _kale_volume_name_parameters.sort() _kale_loaddata_task = _kale_loaddata_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after() _kale_loaddata_task.container.working_dir = "/Users/animeshsingh/go/src/github.com/kubeflow/kale/~/go/src/github.com/kubeflow/kale/examples/titanic-ml-dataset" _kale_loaddata_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'loaddata': '/tmp/loaddata.html'}) _kale_loaddata_task.output_artifact_paths.update(_kale_output_artifacts) _kale_loaddata_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_loaddata_task.dependent_names + _kale_volume_step_names) _kale_loaddata_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_loaddata_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_datapreprocessing_task = _kale_datapreprocessing_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_loaddata_task) _kale_datapreprocessing_task.container.working_dir = "/Users/animeshsingh/go/src/github.com/kubeflow/kale/~/go/src/github.com/kubeflow/kale/examples/titanic-ml-dataset" _kale_datapreprocessing_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update( {'datapreprocessing': '/tmp/datapreprocessing.html'}) _kale_datapreprocessing_task.output_artifact_paths.update( _kale_output_artifacts) _kale_datapreprocessing_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_datapreprocessing_task.dependent_names + _kale_volume_step_names) _kale_datapreprocessing_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_datapreprocessing_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_featureengineering_task = _kale_featureengineering_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_datapreprocessing_task) _kale_featureengineering_task.container.working_dir = "/Users/animeshsingh/go/src/github.com/kubeflow/kale/~/go/src/github.com/kubeflow/kale/examples/titanic-ml-dataset" _kale_featureengineering_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update( {'featureengineering': '/tmp/featureengineering.html'}) _kale_featureengineering_task.output_artifact_paths.update( _kale_output_artifacts) _kale_featureengineering_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_featureengineering_task.dependent_names + _kale_volume_step_names) _kale_featureengineering_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_featureengineering_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_decisiontree_task = _kale_decisiontree_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_featureengineering_task) _kale_decisiontree_task.container.working_dir = "/Users/animeshsingh/go/src/github.com/kubeflow/kale/~/go/src/github.com/kubeflow/kale/examples/titanic-ml-dataset" _kale_decisiontree_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'decisiontree': '/tmp/decisiontree.html'}) _kale_decisiontree_task.output_artifact_paths.update( _kale_output_artifacts) _kale_decisiontree_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_decisiontree_task.dependent_names + _kale_volume_step_names) _kale_decisiontree_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_decisiontree_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_svm_task = _kale_svm_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_featureengineering_task) _kale_svm_task.container.working_dir = "/Users/animeshsingh/go/src/github.com/kubeflow/kale/~/go/src/github.com/kubeflow/kale/examples/titanic-ml-dataset" _kale_svm_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'svm': '/tmp/svm.html'}) _kale_svm_task.output_artifact_paths.update(_kale_output_artifacts) _kale_svm_task.add_pod_label("pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_svm_task.dependent_names + _kale_volume_step_names) _kale_svm_task.add_pod_annotation("kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_svm_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_naivebayes_task = _kale_naivebayes_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_featureengineering_task) _kale_naivebayes_task.container.working_dir = "/Users/animeshsingh/go/src/github.com/kubeflow/kale/~/go/src/github.com/kubeflow/kale/examples/titanic-ml-dataset" _kale_naivebayes_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'naivebayes': '/tmp/naivebayes.html'}) _kale_naivebayes_task.output_artifact_paths.update(_kale_output_artifacts) _kale_naivebayes_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_naivebayes_task.dependent_names + _kale_volume_step_names) _kale_naivebayes_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_naivebayes_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_logisticregression_task = _kale_logisticregression_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_featureengineering_task) _kale_logisticregression_task.container.working_dir = "/Users/animeshsingh/go/src/github.com/kubeflow/kale/~/go/src/github.com/kubeflow/kale/examples/titanic-ml-dataset" _kale_logisticregression_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update( {'logisticregression': '/tmp/logisticregression.html'}) _kale_logisticregression_task.output_artifact_paths.update( _kale_output_artifacts) _kale_logisticregression_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_logisticregression_task.dependent_names + _kale_volume_step_names) _kale_logisticregression_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_logisticregression_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_randomforest_task = _kale_randomforest_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_featureengineering_task) _kale_randomforest_task.container.working_dir = "/Users/animeshsingh/go/src/github.com/kubeflow/kale/~/go/src/github.com/kubeflow/kale/examples/titanic-ml-dataset" _kale_randomforest_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'randomforest': '/tmp/randomforest.html'}) _kale_randomforest_task.output_artifact_paths.update( _kale_output_artifacts) _kale_randomforest_task.add_pod_label( "pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_randomforest_task.dependent_names + _kale_volume_step_names) _kale_randomforest_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_randomforest_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters)) _kale_results_task = _kale_results_op()\ .add_pvolumes(_kale_pvolumes_dict)\ .after(_kale_randomforest_task, _kale_logisticregression_task, _kale_naivebayes_task, _kale_svm_task, _kale_decisiontree_task) _kale_results_task.container.working_dir = "/Users/animeshsingh/go/src/github.com/kubeflow/kale/~/go/src/github.com/kubeflow/kale/examples/titanic-ml-dataset" _kale_results_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) _kale_output_artifacts = {} _kale_output_artifacts.update( {'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) _kale_output_artifacts.update({'results': '/tmp/results.html'}) _kale_results_task.output_artifact_paths.update(_kale_output_artifacts) _kale_results_task.add_pod_label("pipelines.kubeflow.org/metadata_written", "true") _kale_dep_names = (_kale_results_task.dependent_names + _kale_volume_step_names) _kale_results_task.add_pod_annotation( "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) if _kale_volume_name_parameters: _kale_results_task.add_pod_annotation( "kubeflow-kale.org/volume-name-parameters", json.dumps(_kale_volume_name_parameters))
cloud_metadata = get_config('singleuser.cloudMetadata', {}) if not cloud_metadata.get('enabled', False): # Use iptables to block access to cloud metadata by default network_tools_image_name = get_config('singleuser.networkTools.image.name') network_tools_image_tag = get_config('singleuser.networkTools.image.tag') ip_block_container = client.V1Container( name="block-cloud-metadata", image=f"{network_tools_image_name}:{network_tools_image_tag}", command=[ 'iptables', '-A', 'OUTPUT', '-d', cloud_metadata.get('ip', '169.254.169.254'), '-j', 'DROP' ], security_context=client.V1SecurityContext( privileged=True, run_as_user=0, capabilities=client.V1Capabilities(add=['NET_ADMIN']))) c.KubeSpawner.init_containers.append(ip_block_container) if get_config('debug.enabled', False): c.JupyterHub.log_level = 'DEBUG' c.Spawner.debug = True extra_config = get_config('hub.extraConfig', {}) if isinstance(extra_config, str): from textwrap import indent, dedent msg = dedent(""" hub.extraConfig should be a dict of strings, but found a single string instead.
def auto_generated_pipeline(vol_shared_volume='efs'): pvolumes_dict = OrderedDict() annotations = {} volume = dsl.PipelineVolume(pvc=vol_shared_volume) pvolumes_dict['/shared_volume/'] = volume loaddata_task = loaddata_op(vol_shared_volume)\ .add_pvolumes(pvolumes_dict)\ .after() loaddata_task.container.working_dir = "/shared_volume/notebooks/titanic" loaddata_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) datapreprocessing_task = datapreprocessing_op(vol_shared_volume)\ .add_pvolumes(pvolumes_dict)\ .after(loaddata_task) datapreprocessing_task.container.working_dir = "/shared_volume/notebooks/titanic" datapreprocessing_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) featureengineering_task = featureengineering_op(vol_shared_volume)\ .add_pvolumes(pvolumes_dict)\ .after(datapreprocessing_task) featureengineering_task.container.working_dir = "/shared_volume/notebooks/titanic" featureengineering_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) decisiontree_task = decisiontree_op(vol_shared_volume)\ .add_pvolumes(pvolumes_dict)\ .after(featureengineering_task) decisiontree_task.container.working_dir = "/shared_volume/notebooks/titanic" decisiontree_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) svm_task = svm_op(vol_shared_volume)\ .add_pvolumes(pvolumes_dict)\ .after(featureengineering_task) svm_task.container.working_dir = "/shared_volume/notebooks/titanic" svm_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) naivebayes_task = naivebayes_op(vol_shared_volume)\ .add_pvolumes(pvolumes_dict)\ .after(featureengineering_task) naivebayes_task.container.working_dir = "/shared_volume/notebooks/titanic" naivebayes_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) logisticregression_task = logisticregression_op(vol_shared_volume)\ .add_pvolumes(pvolumes_dict)\ .after(featureengineering_task) logisticregression_task.container.working_dir = "/shared_volume/notebooks/titanic" logisticregression_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) randomforest_task = randomforest_op(vol_shared_volume)\ .add_pvolumes(pvolumes_dict)\ .after(featureengineering_task) randomforest_task.container.working_dir = "/shared_volume/notebooks/titanic" randomforest_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0)) results_task = results_op(vol_shared_volume)\ .add_pvolumes(pvolumes_dict)\ .after(randomforest_task, logisticregression_task, naivebayes_task, svm_task, decisiontree_task) results_task.container.working_dir = "/shared_volume/notebooks/titanic" results_task.container.set_security_context( k8s_client.V1SecurityContext(run_as_user=0))