def get_container(train_op, train_env, train_num_gpus, drive='coco-headset-vol-1'): (train_op.container.set_memory_request('56Gi').set_memory_limit( '56Gi').set_cpu_request('7.5').set_cpu_limit('7.5').set_gpu_limit( str(train_num_gpus)).add_volume_mount( V1VolumeMount( name='tensorboard', mount_path='/shared/tensorboard')).add_volume_mount( V1VolumeMount(name='data', mount_path='/data/')).add_volume_mount( V1VolumeMount( name='shm', mount_path='/dev/shm'))) (add_env(add_ssh_volume(train_op), train_env).add_toleration( V1Toleration(key='nvidia.com/gpu', operator='Exists', effect='NoSchedule')).add_node_selector_constraint( 'beta.kubernetes.io/instance-type', f'p3.{2 * train_num_gpus}xlarge'). add_volume( V1Volume(name='tensorboard', persistent_volume_claim=V1PersistentVolumeClaimVolumeSource( 'tensorboard-research-kf')) ).add_volume( V1Volume(name='data', persistent_volume_claim=V1PersistentVolumeClaimVolumeSource( drive))) # .add_volume(V1Volume(name='shm', host_path=V1HostPathVolumeSource(path='/dev/shm'))) .add_volume( V1Volume(name='shm', empty_dir=V1EmptyDirVolumeSource(medium='Memory'))))
def train_eval_epic(owner, project, experiment, model, git_rev, pretrained_s3, mode, train_additional_args='', eval_additional_args=''): train_env = {} train_num_gpus = 1 train_op = components.load_component_from_file('components/train.yaml')( owner=owner, project=project, experiment=experiment, model=model, git_rev=git_rev, pretrained_s3=pretrained_s3, mode=mode, additional_args=train_additional_args) (train_op.container.set_memory_request('56Gi').set_memory_limit( '56Gi').set_cpu_request('7.5').set_cpu_limit('7.5').set_gpu_limit( str(train_num_gpus)).add_volume_mount( V1VolumeMount( name='tensorboard', mount_path='/shared/tensorboard')).add_volume_mount( V1VolumeMount(name='data', mount_path='/data/')).add_volume_mount( V1VolumeMount( name='shm', mount_path='/dev/shm'))) (add_env(add_ssh_volume(train_op), train_env).add_toleration( V1Toleration(key='nvidia.com/gpu', operator='Exists', effect='NoSchedule')).add_node_selector_constraint( 'beta.kubernetes.io/instance-type', f'p3.{2*train_num_gpus}xlarge'). add_volume( V1Volume(name='tensorboard', persistent_volume_claim=V1PersistentVolumeClaimVolumeSource( 'tensorboard-research-kf')) ).add_volume( V1Volume(name='data', persistent_volume_claim=V1PersistentVolumeClaimVolumeSource( 'dataset-epic-kitchen'))) # .add_volume(V1Volume(name='shm', host_path=V1HostPathVolumeSource(path='/dev/shm'))) .add_volume( V1Volume(name='shm', empty_dir=V1EmptyDirVolumeSource(medium='Memory'))))
def my_pipeline(param1, param2): vol = VolumeOp(name="myvol_creation", resource_name=param1, size=param2, annotations={"test": "annotation"}) self.assertCountEqual([x.name for x in vol.inputs], ["param1", "param2"]) self.assertEqual(vol.k8s_resource.metadata.name, "{{workflow.name}}-%s" % str(param1)) expected_attribute_outputs = { "manifest": "{}", "name": "{.metadata.name}", "size": "{.status.capacity.storage}" } self.assertEqual(vol.attribute_outputs, expected_attribute_outputs) expected_outputs = { "manifest": PipelineParam(name="manifest", op_name=vol.name), "name": PipelineParam(name="name", op_name=vol.name), "size": PipelineParam(name="size", op_name=vol.name) } self.assertEqual(vol.outputs, expected_outputs) self.assertEqual(vol.output, PipelineParam(name="name", op_name=vol.name)) self.assertEqual(vol.dependent_names, []) expected_volume = PipelineVolume( name="myvol-creation", persistent_volume_claim=V1PersistentVolumeClaimVolumeSource( claim_name=PipelineParam(name="name", op_name=vol.name)))
def __init__(self, pvc: str = None, volume: V1Volume = None, **kwargs): if volume and kwargs: raise ValueError("You can't pass a volume along with other " "kwargs.") name_provided = True init_volume = {} if volume: init_volume = { attr: getattr(volume, attr) for attr in self.attribute_map.keys() } else: if "name" in kwargs: if len(kwargs["name"]) > 63: raise ValueError("PipelineVolume name must be no more than" " 63 characters") init_volume = {"name": kwargs.pop("name")} else: name_provided = False init_volume = {"name": "pvolume-placeholder"} if pvc and kwargs: raise ValueError("You can only pass 'name' along with 'pvc'.") elif pvc and not kwargs: pvc_volume_source = V1PersistentVolumeClaimVolumeSource( claim_name=str(pvc)) init_volume["persistent_volume_claim"] = pvc_volume_source super().__init__(**init_volume, **kwargs) if not name_provided: volume_dict = prune_none_dict_values(self.to_dict()) hash_value = hashlib.sha256( bytes(json.dumps(volume_dict, sort_keys=True), "utf-8")).hexdigest() name = "pvolume-{}".format(hash_value) self.name = name[0:63] if len(name) > 63 else name self.dependent_names = []
def pipeline_mount_pvc(): pvc_name = "kfp-pvc" volume_name = 'pipeline' volume_mount_path = '/mnt/pipeline' dsl.ContainerOp( name='mnist_pvc', image='kangwoo/kfp-mnist-storage:0.0.1', arguments=['--model', '/mnt/pipeline/kfp/mnist/model'] ).add_volume(V1Volume(name=volume_name, persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(claim_name=pvc_name))) \ .add_volume_mount(V1VolumeMount(mount_path=volume_mount_path, name=volume_name))
def __init__(self, pvc: str = None, volume: V1Volume = None, **kwargs): """Create a new instance of PipelineVolume. Args: pvc: The name of an existing PVC volume: Create a deep copy out of a V1Volume or PipelineVolume with no deps Raises: ValueError: if volume is not None and kwargs is not None if pvc is not None and kwargs.pop("name") is not None """ if volume and kwargs: raise ValueError("You can't pass a volume along with other " "kwargs.") name_provided = True init_volume = {} if volume: init_volume = {attr: getattr(volume, attr) for attr in self.attribute_map.keys()} else: if "name" in kwargs: init_volume = {"name": kwargs.pop("name")} else: name_provided = False init_volume = {"name": "pvolume-placeholder"} if pvc and kwargs: raise ValueError("You can only pass 'name' along with 'pvc'.") elif pvc and not kwargs: pvc_volume_source = V1PersistentVolumeClaimVolumeSource( claim_name=str(pvc) ) init_volume["persistent_volume_claim"] = pvc_volume_source super().__init__(**init_volume, **kwargs) if not name_provided: self.name = "pvolume-%s" % hashlib.sha256( bytes(json.dumps(self.to_dict(), sort_keys=True), "utf-8") ).hexdigest() self.dependent_names = []
def __init__(self, pvc: str = None, volume: V1Volume = None, **kwargs): """Create a new instance of PipelineVolume. Args: pvc: The name of an existing PVC volume: Create a deep copy out of a V1Volume or PipelineVolume with no deps Raises: ValueError: if pvc is not None and name is None if volume is not None and kwargs is not None if pvc is not None and kwargs.pop("name") is not None """ if pvc and "name" not in kwargs: raise ValueError("Please provide name.") elif volume and kwargs: raise ValueError("You can't pass a volume along with other " "kwargs.") init_volume = {} if volume: init_volume = {attr: getattr(volume, attr) for attr in self.attribute_map.keys()} else: init_volume = {"name": kwargs.pop("name") if "name" in kwargs else None} if pvc and kwargs: raise ValueError("You can only pass 'name' along with 'pvc'.") elif pvc and not kwargs: pvc_volume_source = V1PersistentVolumeClaimVolumeSource( claim_name=pvc ) init_volume["persistent_volume_claim"] = pvc_volume_source super().__init__(**init_volume, **kwargs) self.dependent_names = []
def _splice_volumes(self, vollist): namespace = self.get_user_namespace() already_vols = [] if self.volumes: already_vols = [x["name"] for x in self.volumes] self.log.debug("Already_vols: %r" % already_vols) for vol in vollist: mountpoint = vol["mountpoint"] if not mountpoint: self.log.error( "Mountpoint not specified for volume '{}'!".format(vol)) continue volname = self._get_volume_name_for_mountpoint(mountpoint) shortname = mountpoint[1:].replace("/", "-") if shortname in already_vols: self.log.info( "Volume '{}' already exists for pod.".format(volname)) continue k8s_vol = vol["k8s_vol"] hostpath = vol["hostpath"] if k8s_vol and not hostpath: # If hostpath is set, k8s_vol should NOT be, but... # Create shadow PV and namespaced PVC for volume kvol = self._get_nfs_volume(k8s_vol) ns_vol = self._replicate_nfs_pv_with_suffix(kvol, namespace) self._create_pvc_for_pv(ns_vol) mode = "ReadOnlyMany" vmro = True if vol["mode"] == "rw": mode = "ReadWriteMany" vmro = False vvol = { "name": shortname, } if hostpath: vsrc = V1HostPathVolumeSource(path=hostpath, type="Directory") vvol["host_path"] = vsrc elif k8s_vol: pvcvs = V1PersistentVolumeClaimVolumeSource( claim_name=ns_vol.metadata.name, read_only=vmro) vvol["persistent_volume_claim"] = pvcvs else: vvol["nfs"] = { "server": vol["host"], "path": vol["export"], "accessModes": [mode] } self.volumes.append(vvol) options = vol.get("options") if options: optlist = options.split(',') # This does not work. # To get NFS with mount_options, you need to specify the # 'kubernetes-volume' parameter and create your PV with the # appropriate volumes in the first place. vvol["nfs"]["mount_options"] = optlist vmount = {"name": shortname, "mountPath": mountpoint} if vmro: vmount["readOnly"] = True self.volume_mounts.append(vmount)
def __init__( self, pipeline_name: str, experiment_name: str, notebook: str, cos_endpoint: str, cos_bucket: str, cos_directory: str, cos_dependencies_archive: str, pipeline_version: Optional[str] = "", pipeline_source: Optional[str] = None, pipeline_outputs: Optional[List[str]] = None, pipeline_inputs: Optional[List[str]] = None, pipeline_envs: Optional[Dict[str, str]] = None, requirements_url: Optional[str] = None, bootstrap_script_url: Optional[str] = None, emptydir_volume_size: Optional[str] = None, cpu_request: Optional[str] = None, mem_request: Optional[str] = None, gpu_limit: Optional[str] = None, workflow_engine: Optional[str] = "argo", volume_mounts: Optional[List[VolumeMount]] = None, kubernetes_secrets: Optional[List[KubernetesSecret]] = None, **kwargs, ): """Create a new instance of ContainerOp. Args: pipeline_name: pipeline that this op belongs to experiment_name: the experiment where pipeline_name is executed notebook: name of the notebook that will be executed per this operation cos_endpoint: object storage endpoint e.g weaikish1.fyre.ibm.com:30442 cos_bucket: bucket to retrieve archive from cos_directory: name of the directory in the object storage bucket to pull cos_dependencies_archive: archive file name to get from object storage bucket e.g archive1.tar.gz pipeline_version: optional version identifier pipeline_source: pipeline source pipeline_outputs: comma delimited list of files produced by the notebook pipeline_inputs: comma delimited list of files to be consumed/are required by the notebook pipeline_envs: dictionary of environmental variables to set in the container prior to execution requirements_url: URL to a python requirements.txt file to be installed prior to running the notebook bootstrap_script_url: URL to a custom python bootstrap script to run emptydir_volume_size: Size(GB) of the volume to create for the workspace when using CRIO container runtime cpu_request: number of CPUs requested for the operation mem_request: memory requested for the operation (in Gi) gpu_limit: maximum number of GPUs allowed for the operation workflow_engine: Kubeflow workflow engine, defaults to 'argo' volume_mounts: data volumes to be mounted kubernetes_secrets: secrets to be made available as environment variables kwargs: additional key value pairs to pass e.g. name, image, sidecars & is_exit_handler. See Kubeflow pipelines ContainerOp definition for more parameters or how to use https://kubeflow-pipelines.readthedocs.io/en/latest/source/kfp.dsl.html#kfp.dsl.ContainerOp """ self.pipeline_name = pipeline_name self.pipeline_version = pipeline_version self.pipeline_source = pipeline_source self.experiment_name = experiment_name self.notebook = notebook self.notebook_name = os.path.basename(notebook) self.cos_endpoint = cos_endpoint self.cos_bucket = cos_bucket self.cos_directory = cos_directory self.cos_dependencies_archive = cos_dependencies_archive self.container_work_dir_root_path = "./" self.container_work_dir_name = "jupyter-work-dir/" self.container_work_dir = self.container_work_dir_root_path + self.container_work_dir_name self.bootstrap_script_url = bootstrap_script_url self.requirements_url = requirements_url self.pipeline_outputs = pipeline_outputs self.pipeline_inputs = pipeline_inputs self.pipeline_envs = pipeline_envs self.cpu_request = cpu_request self.mem_request = mem_request self.gpu_limit = gpu_limit self.volume_mounts = volume_mounts # optional data volumes to be mounted to the pod self.kubernetes_secrets = kubernetes_secrets # optional secrets to be made available as env vars argument_list = [] """ CRI-o support for kfp pipelines We need to attach an emptydir volume for each notebook that runs since CRI-o runtime does not allow us to write to the base image layer file system, only to volumes. """ self.emptydir_volume_name = "workspace" self.emptydir_volume_size = emptydir_volume_size self.python_user_lib_path = "" self.python_user_lib_path_target = "" self.python_pip_config_url = "" if self.emptydir_volume_size: self.container_work_dir_root_path = "/opt/app-root/src/" self.container_python_dir_name = "python3/" self.container_work_dir = self.container_work_dir_root_path + self.container_work_dir_name self.python_user_lib_path = self.container_work_dir + self.container_python_dir_name self.python_user_lib_path_target = "--target=" + self.python_user_lib_path self.python_pip_config_url = ELYRA_PIP_CONFIG_URL if not self.bootstrap_script_url: self.bootstrap_script_url = ELYRA_BOOTSTRAP_SCRIPT_URL if not self.requirements_url: self.requirements_url = ELYRA_REQUIREMENTS_URL if "name" not in kwargs: raise TypeError("You need to provide a name for the operation.") elif not kwargs.get("name"): raise ValueError("You need to provide a name for the operation.") if "image" not in kwargs: raise ValueError("You need to provide an image.") if not notebook: raise ValueError("You need to provide a notebook.") if "arguments" not in kwargs: """If no arguments are passed, we use our own. If ['arguments'] are set, we assume container's ENTRYPOINT is set and dependencies are installed NOTE: Images being pulled must have python3 available on PATH and cURL utility """ common_curl_options = '--fail -H "Cache-Control: no-cache"' argument_list.append( f"mkdir -p {self.container_work_dir} && cd {self.container_work_dir} && " f"echo 'Downloading {self.bootstrap_script_url}' && " f"curl {common_curl_options} -L {self.bootstrap_script_url} --output bootstrapper.py && " f"echo 'Downloading {self.requirements_url}' && " f"curl {common_curl_options} -L {self.requirements_url} --output requirements-elyra.txt && " f"echo 'Downloading {ELYRA_REQUIREMENTS_URL_PY37}' && " f"curl {common_curl_options} -L {ELYRA_REQUIREMENTS_URL_PY37} --output requirements-elyra-py37.txt && " ) if self.emptydir_volume_size: argument_list.append( f"mkdir {self.container_python_dir_name} && cd {self.container_python_dir_name} && " f"echo 'Downloading {self.python_pip_config_url}' && " f"curl {common_curl_options} -L {self.python_pip_config_url} --output pip.conf && cd .. &&" ) argument_list.append( f"python3 -m pip install {self.python_user_lib_path_target} packaging && " "python3 -m pip freeze > requirements-current.txt && " "python3 bootstrapper.py " f'--pipeline-name "{self.pipeline_name}" ' f"--cos-endpoint {self.cos_endpoint} " f"--cos-bucket {self.cos_bucket} " f'--cos-directory "{self.cos_directory}" ' f'--cos-dependencies-archive "{self.cos_dependencies_archive}" ' f'--file "{self.notebook}" ') if self.pipeline_inputs: inputs_str = self._artifact_list_to_str(self.pipeline_inputs) argument_list.append(f'--inputs "{inputs_str}" ') if self.pipeline_outputs: outputs_str = self._artifact_list_to_str(self.pipeline_outputs) argument_list.append(f'--outputs "{outputs_str}" ') if self.emptydir_volume_size: argument_list.append( f'--user-volume-path "{self.python_user_lib_path}" ') kwargs["command"] = ["sh", "-c"] kwargs["arguments"] = "".join(argument_list) super().__init__(**kwargs) # add user-specified volume mounts: the referenced PVCs must exist # or this generic operation will fail if self.volume_mounts: unique_pvcs = [] for volume_mount in self.volume_mounts: if volume_mount.pvc_name not in unique_pvcs: self.add_volume( V1Volume( name=volume_mount.pvc_name, persistent_volume_claim= V1PersistentVolumeClaimVolumeSource( claim_name=volume_mount.pvc_name), )) unique_pvcs.append(volume_mount.pvc_name) self.container.add_volume_mount( V1VolumeMount(mount_path=volume_mount.path, name=volume_mount.pvc_name)) # We must deal with the envs after the superclass initialization since these amend the # container attribute that isn't available until now. if self.pipeline_envs: for key, value in self.pipeline_envs.items( ): # Convert dict entries to format kfp needs self.container.add_env_variable(V1EnvVar(name=key, value=value)) if self.kubernetes_secrets: for secret in self.kubernetes_secrets: # Convert tuple entries to format kfp needs self.container.add_env_variable( V1EnvVar( name=secret.env_var, value_from=V1EnvVarSource( secret_key_ref=V1SecretKeySelector( name=secret.name, key=secret.key)), )) # If crio volume size is found then assume kubeflow pipelines environment is using CRI-o as # its container runtime if self.emptydir_volume_size: self.add_volume( V1Volume( empty_dir=V1EmptyDirVolumeSource( medium="", size_limit=self.emptydir_volume_size), name=self.emptydir_volume_name, )) self.container.add_volume_mount( V1VolumeMount(mount_path=self.container_work_dir_root_path, name=self.emptydir_volume_name)) # Append to PYTHONPATH location of elyra dependencies in installed in Volume self.container.add_env_variable( V1EnvVar(name="PYTHONPATH", value=self.python_user_lib_path)) if self.cpu_request: self.container.set_cpu_request(cpu=str(cpu_request)) if self.mem_request: self.container.set_memory_request(memory=str(mem_request) + "G") if self.gpu_limit: gpu_vendor = self.pipeline_envs.get("GPU_VENDOR", "nvidia") self.container.set_gpu_limit(gpu=str(gpu_limit), vendor=gpu_vendor) # Generate unique ELYRA_RUN_NAME value and expose it as an environment # variable in the container if not workflow_engine: raise ValueError( "workflow_engine is missing and needs to be specified.") if workflow_engine.lower() == "argo": # attach RUN_ID_PLACEHOLDER as run name # '{{workflow.annotations.pipelines.kubeflow.org/run_name}}' variable # cannot be resolved by Argo in KF 1.4 run_name_placeholder = RUN_ID_PLACEHOLDER self.container.add_env_variable( V1EnvVar(name="ELYRA_RUN_NAME", value=run_name_placeholder)) elif workflow_engine.lower() == "tekton": try: from kfp_tekton import TektonClient # noqa: F401 except ImportError: raise ValueError( "kfp-tekton not installed. Please install using elyra[kfp-tekton] to use Tekton engine." ) # For Tekton derive the value from the specified pod annotation annotation = "pipelines.kubeflow.org/run_name" field_path = f"metadata.annotations['{annotation}']" self.container.add_env_variable( V1EnvVar( name="ELYRA_RUN_NAME", value_from=V1EnvVarSource(field_ref=V1ObjectFieldSelector( field_path=field_path)), )) else: raise ValueError( f"{workflow_engine} is not a supported workflow engine.") # Attach metadata to the pod # Node type (a static type for this op) self.add_pod_label( "elyra/node-type", ExecuteFileOp._normalize_label_value("notebook-script")) # Pipeline name self.add_pod_label( "elyra/pipeline-name", ExecuteFileOp._normalize_label_value(self.pipeline_name)) # Pipeline version self.add_pod_label( "elyra/pipeline-version", ExecuteFileOp._normalize_label_value(self.pipeline_version)) # Experiment name self.add_pod_label( "elyra/experiment-name", ExecuteFileOp._normalize_label_value(self.experiment_name)) # Pipeline node name self.add_pod_label( "elyra/node-name", ExecuteFileOp._normalize_label_value(kwargs.get("name"))) # Pipeline node file self.add_pod_annotation("elyra/node-file-name", self.notebook) # Identify the pipeline source, which can be a # pipeline file (mypipeline.pipeline), a Python # script or notebook that was submitted if self.pipeline_source is not None: self.add_pod_annotation("elyra/pipeline-source", self.pipeline_source)
producer_task = producer_op() processor_task = processor_op(producer_task.outputs['output_1'], producer_task.outputs['output_2']) consumer_task = consumer_op(processor_task.outputs['output_1'], processor_task.outputs['output_2']) markdown_task = create_component_from_func(func=metadata_and_metrics)() # This line is only needed for compiling using dsl-compile to work kfp.dsl.get_pipeline_conf( ).data_passing_method = volume_based_data_passing_method from kubernetes.client.models import V1Volume, V1PersistentVolumeClaimVolumeSource from kfp.dsl import data_passing_methods volume_based_data_passing_method = data_passing_methods.KubernetesVolume( volume=V1Volume( name='data', persistent_volume_claim=V1PersistentVolumeClaimVolumeSource( claim_name='data-volume', ), ), path_prefix='artifact_data/', ) if __name__ == '__main__': pipeline_conf = kfp.dsl.PipelineConf() pipeline_conf.data_passing_method = volume_based_data_passing_method kfp.compiler.Compiler().compile(artifact_passing_pipeline, __file__ + '.yaml', pipeline_conf=pipeline_conf)