def execute(self): """Execute a job in Kubernetes.""" backend_job_id = build_unique_component_name("run-job") self.job = { "kind": "Job", "apiVersion": "batch/v1", "metadata": { "name": backend_job_id, "namespace": REANA_RUNTIME_KUBERNETES_NAMESPACE, }, "spec": { "automountServiceAccountToken": False, "backoffLimit": KubernetesJobManager.MAX_NUM_JOB_RESTARTS, "autoSelector": True, "template": { "metadata": { "name": backend_job_id, "labels": {"reana-run-job-workflow-uuid": self.workflow_uuid}, }, "spec": { "containers": [ { "image": self.docker_img, "command": ["bash", "-c"], "args": [self.cmd], "name": "job", "env": [], "volumeMounts": [], } ], "initContainers": [], "volumes": [], "restartPolicy": "Never", "enableServiceLinks": False, }, }, }, } user_id = os.getenv("REANA_USER_ID") secrets_store = REANAUserSecretsStore(user_id) secret_env_vars = secrets_store.get_env_secrets_as_k8s_spec() job_spec = self.job["spec"]["template"]["spec"] job_spec["containers"][0]["env"].extend(secret_env_vars) job_spec["volumes"].append(secrets_store.get_file_secrets_volume_as_k8s_specs()) secrets_volume_mount = secrets_store.get_secrets_volume_mount_as_k8s_spec() job_spec["containers"][0]["volumeMounts"].append(secrets_volume_mount) if self.env_vars: for var, value in self.env_vars.items(): job_spec["containers"][0]["env"].append({"name": var, "value": value}) self.add_memory_limit(job_spec) self.add_hostpath_volumes() self.add_workspace_volume() self.add_shared_volume() self.add_eos_volume() self.add_image_pull_secrets() self.add_kubernetes_job_timeout() if self.cvmfs_mounts != "false": cvmfs_map = {} for cvmfs_mount_path in ast.literal_eval(self.cvmfs_mounts): if cvmfs_mount_path in CVMFS_REPOSITORIES: cvmfs_map[CVMFS_REPOSITORIES[cvmfs_mount_path]] = cvmfs_mount_path for repository, mount_path in cvmfs_map.items(): volume = get_k8s_cvmfs_volume(repository) ( job_spec["containers"][0]["volumeMounts"].append( { "name": volume["name"], "mountPath": "/cvmfs/{}".format(mount_path), "readOnly": volume["readOnly"], } ) ) job_spec["volumes"].append(volume) self.job["spec"]["template"]["spec"][ "securityContext" ] = client.V1PodSecurityContext( run_as_group=WORKFLOW_RUNTIME_USER_GID, run_as_user=self.kubernetes_uid ) if self.kerberos: self._add_krb5_init_container(secrets_store) if self.voms_proxy: self._add_voms_proxy_init_container(secrets_volume_mount, secret_env_vars) if REANA_RUNTIME_JOBS_KUBERNETES_NODE_LABEL: self.job["spec"]["template"]["spec"][ "nodeSelector" ] = REANA_RUNTIME_JOBS_KUBERNETES_NODE_LABEL backend_job_id = self._submit() return backend_job_id
def _create_job_spec( self, name, command=None, image=None, env_vars=None, overwrite_input_parameters=None, overwrite_operational_options=None, ): """Instantiate a Kubernetes job. :param name: Name of the job. :param image: Docker image to use to run the job on. :param command: List of commands to run on the given job. :param env_vars: List of environment variables (dictionaries) to inject into the workflow engine container. :param interactive_session_type: One of the available interactive session types. :param overwrite_input_params: Dictionary with parameters to be overwritten or added to the current workflow run. :param type: Dict :param overwrite_operational_options: Dictionary with operational options to be overwritten or added to the current workflow run. :param type: Dict """ image = image or self._workflow_engine_image() command = command or self._workflow_engine_command( overwrite_input_parameters=overwrite_input_parameters, overwrite_operational_options=overwrite_operational_options, ) workflow_engine_env_vars = env_vars or self._workflow_engine_env_vars() job_controller_env_vars = [] owner_id = str(self.workflow.owner_id) command = format_cmd(command) workspace_mount, workspace_volume = get_workspace_volume( self.workflow.workspace_path) db_mount, shared_volume = get_shared_volume("db") workflow_metadata = client.V1ObjectMeta( name=name, labels={ "reana_workflow_mode": "batch", "reana-run-batch-workflow-uuid": str(self.workflow.id_), }, namespace=REANA_RUNTIME_KUBERNETES_NAMESPACE, ) secrets_store = REANAUserSecretsStore(owner_id) kerberos = None if self.requires_kerberos(): kerberos = get_kerberos_k8s_config( secrets_store, kubernetes_uid=WORKFLOW_RUNTIME_USER_UID, ) job = client.V1Job() job.api_version = "batch/v1" job.kind = "Job" job.metadata = workflow_metadata spec = client.V1JobSpec(template=client.V1PodTemplateSpec()) spec.template.metadata = workflow_metadata workflow_engine_container = client.V1Container( name=current_app.config["WORKFLOW_ENGINE_NAME"], image=image, image_pull_policy="IfNotPresent", env=[], volume_mounts=[], command=["/bin/bash", "-c"], args=command, ) workflow_engine_env_vars.extend([ { "name": "REANA_JOB_CONTROLLER_SERVICE_PORT_HTTP", "value": str(current_app.config["JOB_CONTROLLER_CONTAINER_PORT"]), }, { "name": "REANA_JOB_CONTROLLER_SERVICE_HOST", "value": "localhost" }, { "name": "REANA_COMPONENT_PREFIX", "value": REANA_COMPONENT_PREFIX }, { "name": "REANA_COMPONENT_NAMING_SCHEME", "value": REANA_COMPONENT_NAMING_SCHEME, }, { "name": "REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE", "value": REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE, }, { "name": "REANA_RUNTIME_KUBERNETES_NAMESPACE", "value": REANA_RUNTIME_KUBERNETES_NAMESPACE, }, { "name": "REANA_JOB_CONTROLLER_CONNECTION_CHECK_SLEEP", "value": str(REANA_JOB_CONTROLLER_CONNECTION_CHECK_SLEEP), }, ]) workflow_engine_container.env.extend(workflow_engine_env_vars) workflow_engine_container.security_context = client.V1SecurityContext( run_as_group=WORKFLOW_RUNTIME_USER_GID, run_as_user=WORKFLOW_RUNTIME_USER_UID, ) workflow_engine_container.volume_mounts = [workspace_mount] if kerberos: workflow_engine_container.volume_mounts += kerberos.volume_mounts workflow_engine_container.env += kerberos.env job_controller_env_secrets = secrets_store.get_env_secrets_as_k8s_spec( ) user = secrets_store.get_secret_value( "CERN_USER") or WORKFLOW_RUNTIME_USER_NAME job_controller_container = client.V1Container( name=current_app.config["JOB_CONTROLLER_NAME"], image=current_app.config["JOB_CONTROLLER_IMAGE"], image_pull_policy="IfNotPresent", env=[], volume_mounts=[], command=["/bin/bash", "-c"], args=self._create_job_controller_startup_cmd(user), ports=[], ) job_controller_env_vars.extend([ { "name": "REANA_USER_ID", "value": owner_id }, { "name": "CERN_USER", "value": user }, { "name": "USER", "value": user }, # Required by HTCondor { "name": "K8S_CERN_EOS_AVAILABLE", "value": K8S_CERN_EOS_AVAILABLE }, { "name": "IMAGE_PULL_SECRETS", "value": ",".join(IMAGE_PULL_SECRETS) }, { "name": "REANA_SQLALCHEMY_DATABASE_URI", "value": SQLALCHEMY_DATABASE_URI, }, { "name": "REANA_STORAGE_BACKEND", "value": REANA_STORAGE_BACKEND }, { "name": "REANA_COMPONENT_PREFIX", "value": REANA_COMPONENT_PREFIX }, { "name": "REANA_COMPONENT_NAMING_SCHEME", "value": REANA_COMPONENT_NAMING_SCHEME, }, { "name": "REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE", "value": REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE, }, { "name": "REANA_RUNTIME_KUBERNETES_NAMESPACE", "value": REANA_RUNTIME_KUBERNETES_NAMESPACE, }, { "name": "REANA_JOB_HOSTPATH_MOUNTS", "value": json.dumps(REANA_JOB_HOSTPATH_MOUNTS), }, { "name": "REANA_RUNTIME_KUBERNETES_KEEP_ALIVE_JOBS_WITH_STATUSES", "value": ",".join( REANA_RUNTIME_KUBERNETES_KEEP_ALIVE_JOBS_WITH_STATUSES), }, { "name": "REANA_KUBERNETES_JOBS_MEMORY_LIMIT", "value": REANA_KUBERNETES_JOBS_MEMORY_LIMIT, }, { "name": "REANA_KUBERNETES_JOBS_MAX_USER_MEMORY_LIMIT", "value": REANA_KUBERNETES_JOBS_MAX_USER_MEMORY_LIMIT, }, { "name": "REANA_KUBERNETES_JOBS_TIMEOUT_LIMIT", "value": REANA_KUBERNETES_JOBS_TIMEOUT_LIMIT, }, { "name": "REANA_KUBERNETES_JOBS_MAX_USER_TIMEOUT_LIMIT", "value": REANA_KUBERNETES_JOBS_MAX_USER_TIMEOUT_LIMIT, }, { "name": "WORKSPACE_PATHS", "value": json.dumps(WORKSPACE_PATHS) }, ]) job_controller_container.env.extend(job_controller_env_vars) job_controller_container.env.extend(job_controller_env_secrets) if REANA_RUNTIME_JOBS_KUBERNETES_NODE_LABEL: job_controller_container.env.append( { "name": "REANA_RUNTIME_JOBS_KUBERNETES_NODE_LABEL", "value": os.getenv("REANA_RUNTIME_JOBS_KUBERNETES_NODE_LABEL"), }, ) secrets_volume_mount = secrets_store.get_secrets_volume_mount_as_k8s_spec( ) job_controller_container.volume_mounts = [workspace_mount, db_mount] job_controller_container.volume_mounts.append(secrets_volume_mount) job_controller_container.ports = [{ "containerPort": current_app.config["JOB_CONTROLLER_CONTAINER_PORT"] }] containers = [workflow_engine_container, job_controller_container] spec.template.spec = client.V1PodSpec( containers=containers, node_selector=REANA_RUNTIME_BATCH_KUBERNETES_NODE_LABEL, init_containers=[], ) spec.template.spec.service_account_name = ( REANA_RUNTIME_KUBERNETES_SERVICEACCOUNT_NAME) volumes = [ workspace_volume, shared_volume, secrets_store.get_file_secrets_volume_as_k8s_specs(), ] if kerberos: volumes += kerberos.volumes spec.template.spec.init_containers.append(kerberos.init_container) # filter out volumes with the same name spec.template.spec.volumes = list({v["name"]: v for v in volumes}.values()) if os.getenv("FLASK_ENV") == "development": code_volume_name = "reana-code" code_mount_path = "/code" k8s_code_volume = client.V1Volume(name=code_volume_name) k8s_code_volume.host_path = client.V1HostPathVolumeSource( code_mount_path) spec.template.spec.volumes.append(k8s_code_volume) for container in spec.template.spec.containers: container.env.extend(current_app.config["DEBUG_ENV_VARS"]) sub_path = f"reana-{container.name}" if container.name == "workflow-engine": sub_path += f"-{self.workflow.type_}" container.volume_mounts.append({ "name": code_volume_name, "mountPath": code_mount_path, "subPath": sub_path, }) job.spec = spec job.spec.template.spec.restart_policy = "Never" job.spec.backoff_limit = 0 return job
def execute(self): """Execute a job in Kubernetes.""" backend_job_id = str(uuid.uuid4()) self.job = { 'kind': 'Job', 'apiVersion': 'batch/v1', 'metadata': { 'name': backend_job_id, 'namespace': K8S_DEFAULT_NAMESPACE }, 'spec': { 'backoffLimit': KubernetesJobManager.MAX_NUM_JOB_RESTARTS, 'autoSelector': True, 'template': { 'metadata': { 'name': backend_job_id }, 'spec': { 'containers': [ { 'image': self.docker_img, 'command': self.cmd, 'name': 'job', 'env': [], 'volumeMounts': [], } ], 'initContainers': [], 'volumes': [], 'restartPolicy': 'Never' } } } } user_id = os.getenv('REANA_USER_ID') secrets_store = REANAUserSecretsStore(user_id) secret_env_vars = secrets_store.get_env_secrets_as_k8s_spec() self.job['spec']['template']['spec']['containers'][0]['env'].extend( secret_env_vars ) self.job['spec']['template']['spec']['volumes'].append( secrets_store.get_file_secrets_volume_as_k8s_specs() ) secrets_volume_mount = \ secrets_store.get_secrets_volume_mount_as_k8s_spec() self.job['spec']['template']['spec']['containers'][0]['volumeMounts'] \ .append(secrets_volume_mount) if self.env_vars: for var, value in self.env_vars.items(): self.job['spec']['template']['spec'][ 'containers'][0]['env'].append({'name': var, 'value': value}) self.add_hostpath_volumes() self.add_shared_volume() self.add_eos_volume() self.add_image_pull_secrets() if self.cvmfs_mounts != 'false': cvmfs_map = {} for cvmfs_mount_path in ast.literal_eval(self.cvmfs_mounts): if cvmfs_mount_path in CVMFS_REPOSITORIES: cvmfs_map[ CVMFS_REPOSITORIES[cvmfs_mount_path]] = \ cvmfs_mount_path for repository, mount_path in cvmfs_map.items(): volume = get_k8s_cvmfs_volume(repository) (self.job['spec']['template']['spec']['containers'][0] ['volumeMounts'].append( {'name': volume['name'], 'mountPath': '/cvmfs/{}'.format(mount_path), 'readOnly': volume['readOnly']} )) self.job['spec']['template']['spec']['volumes'].append(volume) self.job['spec']['template']['spec']['securityContext'] = \ client.V1PodSecurityContext( run_as_group=WORKFLOW_RUNTIME_USER_GID, run_as_user=self.kubernetes_uid) if self.kerberos: self._add_krb5_init_container(secrets_volume_mount) backend_job_id = self._submit() return backend_job_id