def create_workflow_workspace(path, user_id=None, git_url=None, git_branch=None, git_ref=None): """Create workflow workspace. :param path: Relative path to workspace directory. :return: Absolute workspace path. """ os.umask(REANA_WORKFLOW_UMASK) reana_fs = fs.open_fs(app.config["SHARED_VOLUME_PATH"]) reana_fs.makedirs(path, recreate=True) if git_url and git_ref: secret_store = REANAUserSecretsStore(user_id) gitlab_access_token = secret_store.get_secret_value( "gitlab_access_token") url = "https://*****:*****@{1}/{2}.git".format(gitlab_access_token, REANA_GITLAB_HOST, git_url) repo = Repo.clone_from( url=url, to_path=os.path.abspath(reana_fs.root_path + "/" + path), branch=git_branch, depth=1, ) repo.head.reset(commit=git_ref)
def gitlab_projects(): # noqa r"""Endpoint to retrieve GitLab projects. --- get: summary: Get user project from GitLab operationId: gitlab_projects description: >- Retrieve projects from GitLab. produces: - application/json responses: 200: description: >- This resource return all projects owned by the user on GitLab in JSON format. 403: description: >- Request failed. User token not valid. examples: application/json: { "message": "Token is not valid." } 500: description: >- Request failed. Internal controller error. """ try: if current_user.is_authenticated: user = _get_user_from_invenio_user(current_user.email) else: user = get_user_from_token(request.args.get("access_token")) secrets_store = REANAUserSecretsStore(str(user.id_)) gitlab_token = secrets_store.get_secret_value("gitlab_access_token") gitlab_user = secrets_store.get_secret_value("gitlab_user") gitlab_url = REANA_GITLAB_URL + "/api/v4/users/{0}/projects?access_token={1}" response = requests.get(gitlab_url.format(gitlab_user, gitlab_token)) projects = dict() if response.status_code == 200: for gitlab_project in response.json(): hook_id = _get_gitlab_hook_id(response, gitlab_project["id"], gitlab_token) projects[gitlab_project["id"]] = { "name": gitlab_project["name"], "path": gitlab_project["path_with_namespace"], "url": gitlab_project["web_url"], "hook_id": hook_id, } return jsonify(projects), 200 else: return ( jsonify({"message": "Project list could not be retrieved"}), response.status_code, ) except ValueError: return jsonify({"message": "Token is not valid."}), 403 except Exception as e: logging.error(traceback.format_exc()) return jsonify({"message": str(e)}), 500
def test_get_secrets(corev1_api_client_with_user_secrets, user_secrets, no_db_user): """Test listing user secrests.""" with patch('reana_commons.k8s.secrets.' 'current_k8s_corev1_api_client', corev1_api_client_with_user_secrets(user_secrets)): secrets_store = REANAUserSecretsStore(no_db_user.id_) secrets_list = secrets_store.get_secrets() for secret in secrets_list: assert user_secrets[secret['name']]['type'] == secret['type']
def test_overwrite_secret(corev1_api_client_with_user_secrets, user_secrets, test_user): """Test overwriting secrets.""" with patch('reana_commons.k8s.secrets.' 'current_k8s_corev1_api_client', corev1_api_client_with_user_secrets): secrets_store = REANAUserSecretsStore(test_user) secrets_list = secrets_store.add_secrets(user_secrets, overwrite=True) corev1_api_client_with_user_secrets. \ replace_namespaced_secret.assert_called()
def test_delete_secrets(corev1_api_client_with_user_secrets, user_secrets, no_db_user): """Test deletion of user secrets.""" with patch('reana_commons.k8s.secrets.' 'current_k8s_corev1_api_client', corev1_api_client_with_user_secrets(user_secrets)): secrets_store = REANAUserSecretsStore(no_db_user.id_) secret_names_list = user_secrets.keys() deleted_secrets = set(secrets_store.delete_secrets(secret_names_list)) assert bool(deleted_secrets.intersection(secret_names_list)) \ and not bool(deleted_secrets.difference(secret_names_list))
def test_create_secret(user_secrets, no_db_user): """Test creation of user secrets.""" corev1_api_client = Mock() corev1_api_client.read_namespaced_secret = Mock( side_effect=ApiException(reason="Secret does not exist.", status=404)) with patch("reana_commons.k8s.secrets." "current_k8s_corev1_api_client", corev1_api_client): secrets_store = REANAUserSecretsStore(no_db_user.id_) secrets_store.add_secrets(user_secrets) corev1_api_client.create_namespaced_secret.assert_called_once() corev1_api_client.replace_namespaced_secret.assert_called_once()
def gitlab_webhook(): # noqa r"""Endpoint to setup a GitLab webhook. --- post: summary: Set a webhook on a user project from GitLab operationId: gitlab_webhook description: >- Setup a webhook for a GitLab project on GitLab. produces: - application/json responses: 201: description: >- The webhook was created. 403: description: >- Request failed. User token not valid. examples: application/json: { "message": "Token is not valid." } 500: description: >- Request failed. Internal controller error. """ try: if current_user.is_authenticated: user = _get_user_from_invenio_user(current_user.email) else: user = get_user_from_token(request.args.get('access_token')) secrets_store = REANAUserSecretsStore(str(user.id_)) gitlab_token = secrets_store.get_secret_value('gitlab_access_token') parameters = request.json gitlab_url = REANA_GITLAB_URL + "/api/v4/projects/" + \ "{0}/hooks?access_token={1}" webhook_payload = { "url": "https://{}/api/workflows".format(REANA_URL), "push_events": True, "push_events_branch_filter": "master", "merge_requests_events": True, "enable_ssl_verification": False, "token": user.access_token, } webhook = requests.post(gitlab_url.format( parameters['project_id'], gitlab_token), data=webhook_payload) return webhook.content, 201 except ValueError: return jsonify({"message": "Token is not valid."}), 403 except Exception as e: logging.error(traceback.format_exc()) return jsonify({"message": str(e)}), 500
def test_delete_unknown_secret(corev1_api_client_with_user_secrets, test_user): """Test delete a non existing secret.""" with patch('reana_commons.k8s.secrets.' 'current_k8s_corev1_api_client', corev1_api_client_with_user_secrets): secrets_store = REANAUserSecretsStore(test_user) secret_name = 'unknown-secret' with pytest.raises(REANASecretDoesNotExist): secrets_store.delete_secrets([secret_name]) corev1_api_client_with_user_secrets. \ replace_namespaced_secret.assert_not_called()
def test_create_existing_secrets_fail(corev1_api_client_with_user_secrets, user_secrets, test_user): """Test create secrets which already exist without overwrite.""" with patch('reana_commons.k8s.secrets.' 'current_k8s_corev1_api_client', corev1_api_client_with_user_secrets): secrets_store = REANAUserSecretsStore(test_user) with pytest.raises(REANASecretAlreadyExists): secrets_list = secrets_store.add_secrets(user_secrets) corev1_api_client_with_user_secrets. \ replace_namespaced_secret.assert_not_called()
def test_overwrite_secret(corev1_api_client_with_user_secrets, user_secrets, no_db_user): """Test overwriting secrets.""" with patch( "reana_commons.k8s.secrets." "current_k8s_corev1_api_client", corev1_api_client_with_user_secrets(user_secrets), ) as api_client: secrets_store = REANAUserSecretsStore(no_db_user.id_) secrets_store.add_secrets(user_secrets, overwrite=True) api_client.replace_namespaced_secret.assert_called()
def test_delete_unknown_secret(corev1_api_client_with_user_secrets, user_secrets, no_db_user): """Test delete a non existing secret.""" with patch( "reana_commons.k8s.secrets." "current_k8s_corev1_api_client", corev1_api_client_with_user_secrets(user_secrets), ) as api_client: secrets_store = REANAUserSecretsStore(no_db_user.id_) secret_name = "unknown-secret" with pytest.raises(REANASecretDoesNotExist): secrets_store.delete_secrets([secret_name]) api_client.replace_namespaced_secret.assert_not_called()
def _update_commit_status(workflow, status): if status == WorkflowStatus.finished: state = "success" elif status == WorkflowStatus.failed: state = "failed" elif status == WorkflowStatus.stopped or status == WorkflowStatus.deleted: state = "canceled" else: state = "running" secret_store = REANAUserSecretsStore(workflow.owner_id) gitlab_access_token = secret_store.get_secret_value('gitlab_access_token') target_url = REANA_URL + "/api/workflows/{0}/logs".format(workflow.id_) commit_status_url = REANA_GITLAB_URL + "/api/v4/projects/{0}/" + \ "statuses/{1}?access_token={2}&state={3}&target_url={4}" requests.post( commit_status_url.format(workflow.name, workflow.git_ref, gitlab_access_token, state, target_url))
def _update_commit_status(workflow, status): if status == RunStatus.finished: state = "success" elif status == RunStatus.failed: state = "failed" elif status == RunStatus.stopped or status == RunStatus.deleted: state = "canceled" else: state = "running" secret_store = REANAUserSecretsStore(workflow.owner_id) gitlab_access_token = secret_store.get_secret_value("gitlab_access_token") target_url = f"https://{REANA_HOSTNAME}/api/workflows/{workflow.id_}/logs" workflow_name = urlparse.quote_plus(workflow.git_repo) commit_status_url = ( f"{REANA_GITLAB_URL}/api/v4/projects/{workflow_name}/statuses/" f"{workflow.git_ref}?access_token={gitlab_access_token}&state={state}&" f"target_url={target_url}") requests.post(commit_status_url)
def _get_reana_yaml_from_gitlab(webhook_data, user_id): gitlab_api = REANA_GITLAB_URL + "/api/v4/projects/{0}" + \ "/repository/files/{1}/raw?ref={2}&access_token={3}" reana_yaml = 'reana.yaml' if webhook_data['object_kind'] == 'push': branch = webhook_data['project']['default_branch'] commit_sha = webhook_data['checkout_sha'] elif webhook_data['object_kind'] == 'merge_request': branch = webhook_data['object_attributes']['source_branch'] commit_sha = webhook_data['object_attributes']['last_commit']['id'] secrets_store = REANAUserSecretsStore(str(user_id)) gitlab_token = secrets_store.get_secret_value('gitlab_access_token') project_id = webhook_data['project']['id'] yaml_file = requests.get( gitlab_api.format(project_id, reana_yaml, branch, gitlab_token)) return yaml.load(yaml_file.content), \ webhook_data['project']['path_with_namespace'], branch, \ commit_sha
def _get_reana_yaml_from_gitlab(webhook_data, user_id): gitlab_api = (REANA_GITLAB_URL + "/api/v4/projects/{0}" + "/repository/files/{1}/raw?ref={2}&access_token={3}") reana_yaml = "reana.yaml" if webhook_data["object_kind"] == "push": branch = webhook_data["project"]["default_branch"] commit_sha = webhook_data["checkout_sha"] elif webhook_data["object_kind"] == "merge_request": branch = webhook_data["object_attributes"]["source_branch"] commit_sha = webhook_data["object_attributes"]["last_commit"]["id"] secrets_store = REANAUserSecretsStore(str(user_id)) gitlab_token = secrets_store.get_secret_value("gitlab_access_token") project_id = webhook_data["project"]["id"] yaml_file = requests.get( gitlab_api.format(project_id, reana_yaml, branch, gitlab_token)) return ( yaml.load(yaml_file.content, Loader=yaml.FullLoader), webhook_data["project"]["path_with_namespace"], webhook_data["project"]["name"], branch, commit_sha, )
def gitlab_webhook(): # noqa r"""Endpoint to setup a GitLab webhook. --- post: summary: Set a webhook on a user project from GitLab operationId: create_gitlab_webhook description: >- Setup a webhook for a GitLab project on GitLab. produces: - application/json parameters: - name: project_id in: path description: The GitLab project id. required: true type: integer responses: 201: description: >- The webhook was created. 403: description: >- Request failed. User token not valid. examples: application/json: { "message": "Token is not valid." } 500: description: >- Request failed. Internal controller error. delete: summary: Delete an existing webhook from GitLab operationId: delete_gitlab_webhook description: >- Remove an existing REANA webhook from a project on GitLab produces: - application/json parameters: - name: project_id in: path description: The GitLab project id. required: true type: integer - name: hook_id in: path description: The GitLab webhook id of the project. required: true type: integer responses: 204: description: >- The webhook was properly deleted. 404: description: >- No webhook found with provided id. 403: description: >- Request failed. User token not valid. examples: application/json: { "message": "Token is not valid." } 500: description: >- Request failed. Internal controller error. """ try: if current_user.is_authenticated: user = _get_user_from_invenio_user(current_user.email) else: user = get_user_from_token(request.args.get("access_token")) secrets_store = REANAUserSecretsStore(str(user.id_)) gitlab_token = secrets_store.get_secret_value("gitlab_access_token") parameters = request.json if request.method == "POST": gitlab_url = (REANA_GITLAB_URL + "/api/v4/projects/" + "{0}/hooks?access_token={1}") webhook_payload = { "url": "https://{}/api/workflows".format(REANA_URL), "push_events": True, "push_events_branch_filter": "master", "merge_requests_events": True, "enable_ssl_verification": False, "token": user.access_token, } webhook = requests.post( gitlab_url.format(parameters["project_id"], gitlab_token), data=webhook_payload, ) return jsonify({"id": webhook.json()["id"]}), 201 elif request.method == "DELETE": gitlab_url = (REANA_GITLAB_URL + "/api/v4/projects/" + "{0}/hooks/{1}?access_token={2}") resp = requests.delete( gitlab_url.format(parameters["project_id"], parameters["hook_id"], gitlab_token)) return resp.content, resp.status_code except ValueError: return jsonify({"message": "Token is not valid."}), 403 except Exception as e: logging.error(traceback.format_exc()) return jsonify({"message": str(e)}), 500
def gitlab_oauth(): # noqa r"""Endpoint to authorize REANA on GitLab. --- get: summary: Get access token from GitLab operationId: gitlab_oauth description: >- Authorize REANA on GitLab. produces: - application/json responses: 200: description: >- Ping succeeded. schema: type: object properties: message: type: string status: type: string examples: application/json: message: OK status: 200 201: description: >- Authorization succeeded. GitLab secret created. schema: type: object properties: message: type: string status: type: string examples: application/json: message: GitLab secret created status: 201 403: description: >- Request failed. User token not valid. examples: application/json: { "message": "Token is not valid." } 500: description: >- Request failed. Internal controller error. """ try: if current_user.is_authenticated: user = _get_user_from_invenio_user(current_user.email) else: user = get_user_from_token(request.args.get('access_token')) if 'code' in request.args: gitlab_code = request.args.get('code') parameters = "client_id={0}&" + \ "client_secret={1}&code={2}&" + \ "grant_type=authorization_code&redirect_uri={3}" parameters = parameters.format(REANA_GITLAB_OAUTH_APP_ID, REANA_GITLAB_OAUTH_APP_SECRET, gitlab_code, REANA_GITLAB_OAUTH_REDIRECT_URL) gitlab_response = requests.post(REANA_GITLAB_URL + '/oauth/token', data=parameters).content secrets_store = REANAUserSecretsStore(str(user.id_)) secrets_store.add_secrets(_format_gitlab_secrets(gitlab_response), overwrite=True) return jsonify({"message": "GitLab secret created"}), 201 else: return jsonify({"message": "OK"}), 200 except ValueError: return jsonify({"message": "Token is not valid."}), 403 except Exception as e: logging.error(traceback.format_exc()) return jsonify({"message": str(e)}), 500
def _create_job_spec( self, name, command=None, image=None, env_vars=None, overwrite_input_parameters=None, overwrite_operational_options=None, ): """Instantiate a Kubernetes job. :param name: Name of the job. :param image: Docker image to use to run the job on. :param command: List of commands to run on the given job. :param env_vars: List of environment variables (dictionaries) to inject into the workflow engine container. :param interactive_session_type: One of the available interactive session types. :param overwrite_input_params: Dictionary with parameters to be overwritten or added to the current workflow run. :param type: Dict :param overwrite_operational_options: Dictionary with operational options to be overwritten or added to the current workflow run. :param type: Dict """ image = image or self._workflow_engine_image() command = command or self._workflow_engine_command( overwrite_input_parameters=overwrite_input_parameters, overwrite_operational_options=overwrite_operational_options, ) workflow_engine_env_vars = env_vars or self._workflow_engine_env_vars() job_controller_env_vars = [] owner_id = str(self.workflow.owner_id) command = format_cmd(command) workspace_mount, workspace_volume = get_workspace_volume( self.workflow.workspace_path) db_mount, shared_volume = get_shared_volume("db") workflow_metadata = client.V1ObjectMeta( name=name, labels={ "reana_workflow_mode": "batch", "reana-run-batch-workflow-uuid": str(self.workflow.id_), }, namespace=REANA_RUNTIME_KUBERNETES_NAMESPACE, ) secrets_store = REANAUserSecretsStore(owner_id) kerberos = None if self.requires_kerberos(): kerberos = get_kerberos_k8s_config( secrets_store, kubernetes_uid=WORKFLOW_RUNTIME_USER_UID, ) job = client.V1Job() job.api_version = "batch/v1" job.kind = "Job" job.metadata = workflow_metadata spec = client.V1JobSpec(template=client.V1PodTemplateSpec()) spec.template.metadata = workflow_metadata workflow_engine_container = client.V1Container( name=current_app.config["WORKFLOW_ENGINE_NAME"], image=image, image_pull_policy="IfNotPresent", env=[], volume_mounts=[], command=["/bin/bash", "-c"], args=command, ) workflow_engine_env_vars.extend([ { "name": "REANA_JOB_CONTROLLER_SERVICE_PORT_HTTP", "value": str(current_app.config["JOB_CONTROLLER_CONTAINER_PORT"]), }, { "name": "REANA_JOB_CONTROLLER_SERVICE_HOST", "value": "localhost" }, { "name": "REANA_COMPONENT_PREFIX", "value": REANA_COMPONENT_PREFIX }, { "name": "REANA_COMPONENT_NAMING_SCHEME", "value": REANA_COMPONENT_NAMING_SCHEME, }, { "name": "REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE", "value": REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE, }, { "name": "REANA_RUNTIME_KUBERNETES_NAMESPACE", "value": REANA_RUNTIME_KUBERNETES_NAMESPACE, }, { "name": "REANA_JOB_CONTROLLER_CONNECTION_CHECK_SLEEP", "value": str(REANA_JOB_CONTROLLER_CONNECTION_CHECK_SLEEP), }, ]) workflow_engine_container.env.extend(workflow_engine_env_vars) workflow_engine_container.security_context = client.V1SecurityContext( run_as_group=WORKFLOW_RUNTIME_USER_GID, run_as_user=WORKFLOW_RUNTIME_USER_UID, ) workflow_engine_container.volume_mounts = [workspace_mount] if kerberos: workflow_engine_container.volume_mounts += kerberos.volume_mounts workflow_engine_container.env += kerberos.env job_controller_env_secrets = secrets_store.get_env_secrets_as_k8s_spec( ) user = secrets_store.get_secret_value( "CERN_USER") or WORKFLOW_RUNTIME_USER_NAME job_controller_container = client.V1Container( name=current_app.config["JOB_CONTROLLER_NAME"], image=current_app.config["JOB_CONTROLLER_IMAGE"], image_pull_policy="IfNotPresent", env=[], volume_mounts=[], command=["/bin/bash", "-c"], args=self._create_job_controller_startup_cmd(user), ports=[], ) job_controller_env_vars.extend([ { "name": "REANA_USER_ID", "value": owner_id }, { "name": "CERN_USER", "value": user }, { "name": "USER", "value": user }, # Required by HTCondor { "name": "K8S_CERN_EOS_AVAILABLE", "value": K8S_CERN_EOS_AVAILABLE }, { "name": "IMAGE_PULL_SECRETS", "value": ",".join(IMAGE_PULL_SECRETS) }, { "name": "REANA_SQLALCHEMY_DATABASE_URI", "value": SQLALCHEMY_DATABASE_URI, }, { "name": "REANA_STORAGE_BACKEND", "value": REANA_STORAGE_BACKEND }, { "name": "REANA_COMPONENT_PREFIX", "value": REANA_COMPONENT_PREFIX }, { "name": "REANA_COMPONENT_NAMING_SCHEME", "value": REANA_COMPONENT_NAMING_SCHEME, }, { "name": "REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE", "value": REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE, }, { "name": "REANA_RUNTIME_KUBERNETES_NAMESPACE", "value": REANA_RUNTIME_KUBERNETES_NAMESPACE, }, { "name": "REANA_JOB_HOSTPATH_MOUNTS", "value": json.dumps(REANA_JOB_HOSTPATH_MOUNTS), }, { "name": "REANA_RUNTIME_KUBERNETES_KEEP_ALIVE_JOBS_WITH_STATUSES", "value": ",".join( REANA_RUNTIME_KUBERNETES_KEEP_ALIVE_JOBS_WITH_STATUSES), }, { "name": "REANA_KUBERNETES_JOBS_MEMORY_LIMIT", "value": REANA_KUBERNETES_JOBS_MEMORY_LIMIT, }, { "name": "REANA_KUBERNETES_JOBS_MAX_USER_MEMORY_LIMIT", "value": REANA_KUBERNETES_JOBS_MAX_USER_MEMORY_LIMIT, }, { "name": "REANA_KUBERNETES_JOBS_TIMEOUT_LIMIT", "value": REANA_KUBERNETES_JOBS_TIMEOUT_LIMIT, }, { "name": "REANA_KUBERNETES_JOBS_MAX_USER_TIMEOUT_LIMIT", "value": REANA_KUBERNETES_JOBS_MAX_USER_TIMEOUT_LIMIT, }, { "name": "WORKSPACE_PATHS", "value": json.dumps(WORKSPACE_PATHS) }, ]) job_controller_container.env.extend(job_controller_env_vars) job_controller_container.env.extend(job_controller_env_secrets) if REANA_RUNTIME_JOBS_KUBERNETES_NODE_LABEL: job_controller_container.env.append( { "name": "REANA_RUNTIME_JOBS_KUBERNETES_NODE_LABEL", "value": os.getenv("REANA_RUNTIME_JOBS_KUBERNETES_NODE_LABEL"), }, ) secrets_volume_mount = secrets_store.get_secrets_volume_mount_as_k8s_spec( ) job_controller_container.volume_mounts = [workspace_mount, db_mount] job_controller_container.volume_mounts.append(secrets_volume_mount) job_controller_container.ports = [{ "containerPort": current_app.config["JOB_CONTROLLER_CONTAINER_PORT"] }] containers = [workflow_engine_container, job_controller_container] spec.template.spec = client.V1PodSpec( containers=containers, node_selector=REANA_RUNTIME_BATCH_KUBERNETES_NODE_LABEL, init_containers=[], ) spec.template.spec.service_account_name = ( REANA_RUNTIME_KUBERNETES_SERVICEACCOUNT_NAME) volumes = [ workspace_volume, shared_volume, secrets_store.get_file_secrets_volume_as_k8s_specs(), ] if kerberos: volumes += kerberos.volumes spec.template.spec.init_containers.append(kerberos.init_container) # filter out volumes with the same name spec.template.spec.volumes = list({v["name"]: v for v in volumes}.values()) if os.getenv("FLASK_ENV") == "development": code_volume_name = "reana-code" code_mount_path = "/code" k8s_code_volume = client.V1Volume(name=code_volume_name) k8s_code_volume.host_path = client.V1HostPathVolumeSource( code_mount_path) spec.template.spec.volumes.append(k8s_code_volume) for container in spec.template.spec.containers: container.env.extend(current_app.config["DEBUG_ENV_VARS"]) sub_path = f"reana-{container.name}" if container.name == "workflow-engine": sub_path += f"-{self.workflow.type_}" container.volume_mounts.append({ "name": code_volume_name, "mountPath": code_mount_path, "subPath": sub_path, }) job.spec = spec job.spec.template.spec.restart_policy = "Never" job.spec.backoff_limit = 0 return job
def execute(self): """Execute a job in Kubernetes.""" backend_job_id = build_unique_component_name("run-job") self.job = { "kind": "Job", "apiVersion": "batch/v1", "metadata": { "name": backend_job_id, "namespace": REANA_RUNTIME_KUBERNETES_NAMESPACE, }, "spec": { "automountServiceAccountToken": False, "backoffLimit": KubernetesJobManager.MAX_NUM_JOB_RESTARTS, "autoSelector": True, "template": { "metadata": { "name": backend_job_id, "labels": {"reana-run-job-workflow-uuid": self.workflow_uuid}, }, "spec": { "containers": [ { "image": self.docker_img, "command": ["bash", "-c"], "args": [self.cmd], "name": "job", "env": [], "volumeMounts": [], } ], "initContainers": [], "volumes": [], "restartPolicy": "Never", "enableServiceLinks": False, }, }, }, } user_id = os.getenv("REANA_USER_ID") secrets_store = REANAUserSecretsStore(user_id) secret_env_vars = secrets_store.get_env_secrets_as_k8s_spec() job_spec = self.job["spec"]["template"]["spec"] job_spec["containers"][0]["env"].extend(secret_env_vars) job_spec["volumes"].append(secrets_store.get_file_secrets_volume_as_k8s_specs()) secrets_volume_mount = secrets_store.get_secrets_volume_mount_as_k8s_spec() job_spec["containers"][0]["volumeMounts"].append(secrets_volume_mount) if self.env_vars: for var, value in self.env_vars.items(): job_spec["containers"][0]["env"].append({"name": var, "value": value}) self.add_memory_limit(job_spec) self.add_hostpath_volumes() self.add_workspace_volume() self.add_shared_volume() self.add_eos_volume() self.add_image_pull_secrets() self.add_kubernetes_job_timeout() if self.cvmfs_mounts != "false": cvmfs_map = {} for cvmfs_mount_path in ast.literal_eval(self.cvmfs_mounts): if cvmfs_mount_path in CVMFS_REPOSITORIES: cvmfs_map[CVMFS_REPOSITORIES[cvmfs_mount_path]] = cvmfs_mount_path for repository, mount_path in cvmfs_map.items(): volume = get_k8s_cvmfs_volume(repository) ( job_spec["containers"][0]["volumeMounts"].append( { "name": volume["name"], "mountPath": "/cvmfs/{}".format(mount_path), "readOnly": volume["readOnly"], } ) ) job_spec["volumes"].append(volume) self.job["spec"]["template"]["spec"][ "securityContext" ] = client.V1PodSecurityContext( run_as_group=WORKFLOW_RUNTIME_USER_GID, run_as_user=self.kubernetes_uid ) if self.kerberos: self._add_krb5_init_container(secrets_store) if self.voms_proxy: self._add_voms_proxy_init_container(secrets_volume_mount, secret_env_vars) if REANA_RUNTIME_JOBS_KUBERNETES_NODE_LABEL: self.job["spec"]["template"]["spec"][ "nodeSelector" ] = REANA_RUNTIME_JOBS_KUBERNETES_NODE_LABEL backend_job_id = self._submit() return backend_job_id
def execute(self): """Execute a job in Kubernetes.""" backend_job_id = str(uuid.uuid4()) self.job = { 'kind': 'Job', 'apiVersion': 'batch/v1', 'metadata': { 'name': backend_job_id, 'namespace': K8S_DEFAULT_NAMESPACE }, 'spec': { 'backoffLimit': KubernetesJobManager.MAX_NUM_JOB_RESTARTS, 'autoSelector': True, 'template': { 'metadata': { 'name': backend_job_id }, 'spec': { 'containers': [ { 'image': self.docker_img, 'command': self.cmd, 'name': 'job', 'env': [], 'volumeMounts': [], } ], 'initContainers': [], 'volumes': [], 'restartPolicy': 'Never' } } } } user_id = os.getenv('REANA_USER_ID') secrets_store = REANAUserSecretsStore(user_id) secret_env_vars = secrets_store.get_env_secrets_as_k8s_spec() self.job['spec']['template']['spec']['containers'][0]['env'].extend( secret_env_vars ) self.job['spec']['template']['spec']['volumes'].append( secrets_store.get_file_secrets_volume_as_k8s_specs() ) secrets_volume_mount = \ secrets_store.get_secrets_volume_mount_as_k8s_spec() self.job['spec']['template']['spec']['containers'][0]['volumeMounts'] \ .append(secrets_volume_mount) if self.env_vars: for var, value in self.env_vars.items(): self.job['spec']['template']['spec'][ 'containers'][0]['env'].append({'name': var, 'value': value}) self.add_hostpath_volumes() self.add_shared_volume() self.add_eos_volume() self.add_image_pull_secrets() if self.cvmfs_mounts != 'false': cvmfs_map = {} for cvmfs_mount_path in ast.literal_eval(self.cvmfs_mounts): if cvmfs_mount_path in CVMFS_REPOSITORIES: cvmfs_map[ CVMFS_REPOSITORIES[cvmfs_mount_path]] = \ cvmfs_mount_path for repository, mount_path in cvmfs_map.items(): volume = get_k8s_cvmfs_volume(repository) (self.job['spec']['template']['spec']['containers'][0] ['volumeMounts'].append( {'name': volume['name'], 'mountPath': '/cvmfs/{}'.format(mount_path), 'readOnly': volume['readOnly']} )) self.job['spec']['template']['spec']['volumes'].append(volume) self.job['spec']['template']['spec']['securityContext'] = \ client.V1PodSecurityContext( run_as_group=WORKFLOW_RUNTIME_USER_GID, run_as_user=self.kubernetes_uid) if self.kerberos: self._add_krb5_init_container(secrets_volume_mount) backend_job_id = self._submit() return backend_job_id
def _create_job_spec(self, name, command=None, image=None, env_vars=None, overwrite_input_parameters=None, overwrite_operational_options=None): """Instantiate a Kubernetes job. :param name: Name of the job. :param image: Docker image to use to run the job on. :param command: List of commands to run on the given job. :param env_vars: List of environment variables (dictionaries) to inject into the workflow engine container. :param interactive_session_type: One of the available interactive session types. :param overwrite_input_params: Dictionary with parameters to be overwritten or added to the current workflow run. :param type: Dict :param overwrite_operational_options: Dictionary with operational options to be overwritten or added to the current workflow run. :param type: Dict """ image = image or self._workflow_engine_image() command = command or self._workflow_engine_command( overwrite_input_parameters=overwrite_input_parameters, overwrite_operational_options=overwrite_operational_options) workflow_engine_env_vars = env_vars or self._workflow_engine_env_vars() job_controller_env_vars = [] owner_id = str(self.workflow.owner_id) command = format_cmd(command) workspace_mount, workspace_volume = \ get_shared_volume(self.workflow.workspace_path) db_mount, _ = get_shared_volume('db') workflow_metadata = client.V1ObjectMeta( name=name, labels={'reana_workflow_mode': 'batch'}) job = client.V1Job() job.api_version = 'batch/v1' job.kind = 'Job' job.metadata = workflow_metadata spec = client.V1JobSpec(template=client.V1PodTemplateSpec()) spec.template.metadata = workflow_metadata workflow_engine_container = client.V1Container( name=current_app.config['WORKFLOW_ENGINE_NAME'], image=image, image_pull_policy='IfNotPresent', env=[], volume_mounts=[], command=['/bin/bash', '-c'], args=command) job_controller_address = [{ 'name': 'REANA_JOB_CONTROLLER_SERVICE_PORT_HTTP', 'value': str(current_app.config['JOB_CONTROLLER_CONTAINER_PORT']) }, { 'name': 'REANA_JOB_CONTROLLER_SERVICE_HOST', 'value': 'localhost' }] workflow_engine_env_vars.extend(job_controller_address) workflow_engine_container.env.extend(workflow_engine_env_vars) workflow_engine_container.security_context = \ client.V1SecurityContext( run_as_group=WORKFLOW_RUNTIME_USER_GID, run_as_user=WORKFLOW_RUNTIME_USER_UID ) workflow_engine_container.volume_mounts = [workspace_mount] secrets_store = REANAUserSecretsStore(owner_id) job_controller_env_secrets = secrets_store.\ get_env_secrets_as_k8s_spec() user = \ secrets_store.get_secret_value('CERN_USER') or \ WORKFLOW_RUNTIME_USER_NAME job_controller_container = client.V1Container( name=current_app.config['JOB_CONTROLLER_NAME'], image=current_app.config['JOB_CONTROLLER_IMAGE'], image_pull_policy='IfNotPresent', env=[], volume_mounts=[], command=['/bin/bash', '-c'], args=self._create_job_controller_startup_cmd(user), ports=[]) job_controller_env_vars.extend([ { 'name': 'REANA_USER_ID', 'value': owner_id }, { 'name': 'CERN_USER', 'value': user }, { 'name': 'USER', # Required by HTCondor 'value': user }, { 'name': 'K8S_CERN_EOS_AVAILABLE', 'value': K8S_CERN_EOS_AVAILABLE }, { 'name': 'IMAGE_PULL_SECRETS', 'value': ','.join(IMAGE_PULL_SECRETS) } ]) job_controller_container.env.extend(job_controller_env_vars) job_controller_container.env.extend(job_controller_env_secrets) job_controller_container.env.extend([{ 'name': 'REANA_SQLALCHEMY_DATABASE_URI', 'value': SQLALCHEMY_DATABASE_URI }, { 'name': 'REANA_STORAGE_BACKEND', 'value': REANA_STORAGE_BACKEND }]) secrets_volume_mount = \ secrets_store.get_secrets_volume_mount_as_k8s_spec() job_controller_container.volume_mounts = [workspace_mount, db_mount] job_controller_container.volume_mounts.append(secrets_volume_mount) job_controller_container.ports = [{ "containerPort": current_app.config['JOB_CONTROLLER_CONTAINER_PORT'] }] containers = [workflow_engine_container, job_controller_container] spec.template.spec = client.V1PodSpec(containers=containers) spec.template.spec.service_account_name = \ K8S_REANA_SERVICE_ACCOUNT_NAME spec.template.spec.volumes = [ workspace_volume, secrets_store.get_file_secrets_volume_as_k8s_specs(), ] if os.getenv('FLASK_ENV') == 'development': code_volume_name = 'reana-code' code_mount_path = '/code' k8s_code_volume = client.V1Volume(name=code_volume_name) k8s_code_volume.host_path = client.V1HostPathVolumeSource( code_mount_path) spec.template.spec.volumes.append(k8s_code_volume) for container in spec.template.spec.containers: container.env.extend(current_app.config['DEBUG_ENV_VARS']) sub_path = f'reana-{container.name}' if container.name == 'workflow-engine': sub_path += f'-{self.workflow.type_}' container.volume_mounts.append({ 'name': code_volume_name, 'mountPath': code_mount_path, 'subPath': sub_path }) job.spec = spec job.spec.template.spec.restart_policy = 'Never' job.spec.ttl_seconds_after_finished = TTL_SECONDS_AFTER_FINISHED job.spec.backoff_limit = 0 return job
def add_secrets(): # noqa r"""Endpoint to create user secrets. --- post: summary: Add user secrets to REANA. description: >- This resource adds secrets for the authenticated user. operationId: add_secrets produces: - application/json parameters: - name: access_token in: query description: Secrets owner access token. required: false type: string - name: overwrite in: query description: Whether existing secret keys should be overwritten. required: false type: boolean - name: secrets in: body description: >- Optional. List of secrets to be added. required: true schema: type: object additionalProperties: type: object description: Secret definition. properties: name: type: string description: Secret name value: type: string description: Secret value type: type: string enum: - env - file description: >- How will be the secret assigned to the jobs, either exported as an environment variable or mounted as a file. responses: 201: description: >- Secrets successfully added. schema: type: object properties: message: type: string examples: application/json: { "message": "Secret(s) successfully added." } 403: description: >- Request failed. Token is not valid. examples: application/json: { "message": "Token is not valid" } 409: description: >- Request failed. Secrets could not be added due to a conflict. examples: application/json: { "message": "The submitted secrets api_key, password, username already exist." } 500: description: >- Request failed. Internal server error. examples: application/json: { "message": "Internal server error." } """ try: if current_user.is_authenticated: user = _get_user_from_invenio_user(current_user.email) else: user = get_user_from_token(request.args.get('access_token')) secrets_store = REANAUserSecretsStore(str(user.id_)) overwrite = json.loads(request.args.get('overwrite')) secrets_store.add_secrets(request.json, overwrite=overwrite) return jsonify({"message": "Secret(s) successfully added."}), 201 except REANASecretAlreadyExists as e: return jsonify({"message": str(e)}), 409 except ValueError: return jsonify({"message": "Token is not valid."}), 403 except Exception as e: logging.error(traceback.format_exc()) return jsonify({"message": str(e)}), 500
def delete_secrets(): # noqa r"""Endpoint to delete user secrets. --- delete: summary: Deletes the specified secret(s). description: >- This resource deletes the requested secrets. operationId: delete_secrets produces: - application/json parameters: - name: access_token in: query description: API key of the admin. required: false type: string - name: secrets in: body description: >- Optional. List of secrets to be deleted. required: true schema: type: array description: List of secret names to be deleted. items: type: string description: Secret name to be deleted. responses: 200: description: >- Secrets successfully deleted. schema: type: array description: List of secret names that have been deleted. items: type: string description: Name of the secret that have been deleted. examples: application/json: [ ".keytab", "username", ] 403: description: >- Request failed. Token is not valid. examples: application/json: { "message": "Token is not valid" } 404: description: >- Request failed. Secrets do not exist. schema: type: array description: List of secret names that could not be deleted. items: type: string description: Name of the secret which does not exist. examples: application/json: [ "certificate.pem", "PASSWORD", ] 500: description: >- Request failed. Internal server error. examples: application/json: { "message": "Internal server error." } """ try: if current_user.is_authenticated: user = _get_user_from_invenio_user(current_user.email) else: user = get_user_from_token(request.args.get('access_token')) secrets_store = REANAUserSecretsStore(str(user.id_)) deleted_secrets_list = secrets_store.delete_secrets(request.json) return jsonify(deleted_secrets_list), 200 except REANASecretDoesNotExist as e: return jsonify(e.missing_secrets_list), 404 except ValueError: return jsonify({"message": "Token is not valid."}), 403 except Exception as e: logging.error(traceback.format_exc()) return jsonify({"message": str(e)}), 500
def get_secrets(): # noqa r"""Endpoint to retrieve user secrets. --- get: summary: Get user secrets. Requires an user access token. description: >- Get user secrets. operationId: get_secrets produces: - application/json parameters: - name: access_token in: query description: Secrets owner access token. required: false type: string responses: 200: description: >- List of user secrets. schema: type: array items: properties: name: type: string description: Secret name type: type: string enum: - env - file description: >- How will be the secret assigned to the jobs, either exported as an environment variable or mounted as a file. examples: application/json: [ { "name": ".keytab", "value": "SGVsbG8gUkVBTkEh", }, { "name": "username", "value": "reanauser", }, ] 403: description: >- Request failed. Token is not valid. examples: application/json: { "message": "Token is not valid" } 500: description: >- Request failed. Internal server error. examples: application/json: { "message": "Error while querying." } """ try: if current_user.is_authenticated: user = _get_user_from_invenio_user(current_user.email) else: user = get_user_from_token(request.args.get('access_token')) secrets_store = REANAUserSecretsStore(str(user.id_)) user_secrets = secrets_store.get_secrets() return jsonify(user_secrets), 200 except ValueError: return jsonify({"message": "Token is not valid."}), 403 except Exception as e: logging.error(traceback.format_exc()) return jsonify({"message": str(e)}), 500
def _create_job_spec(self, name, command=None, image=None, env_vars=None): """Instantiate a Kubernetes job. :param name: Name of the job. :param image: Docker image to use to run the job on. :param command: List of commands to run on the given job. :param env_vars: List of environment variables (dictionaries) to inject into the workflow engine container. """ image = image or self._workflow_engine_image() command = command or self._workflow_engine_command() workflow_engine_env_vars = env_vars or self._workflow_engine_env_vars() job_controller_env_vars = [] owner_id = str(self.workflow.owner_id) command = format_cmd(command) workspace_mount, _ = get_shared_volume( self.workflow.get_workspace(), SHARED_VOLUME_PATH ) db_mount, _ = get_shared_volume( 'db', SHARED_VOLUME_PATH ) workflow_metadata = client.V1ObjectMeta(name=name) job = client.V1Job() job.api_version = 'batch/v1' job.kind = 'Job' job.metadata = workflow_metadata spec = client.V1JobSpec( template=client.V1PodTemplateSpec()) spec.template.metadata = workflow_metadata workflow_enginge_container = client.V1Container( name=current_app.config['WORKFLOW_ENGINE_NAME'], image=image, image_pull_policy='IfNotPresent', env=[], volume_mounts=[], command=['/bin/bash', '-c'], args=command) job_controller_address = [ { 'name': 'JOB_CONTROLLER_SERVICE_PORT_HTTP', 'value': str(current_app.config['JOB_CONTROLLER_CONTAINER_PORT']) }, { 'name': 'JOB_CONTROLLER_SERVICE_HOST', 'value': 'localhost'} ] workflow_engine_env_vars.extend(job_controller_address) workflow_enginge_container.env.extend(workflow_engine_env_vars) workflow_enginge_container.security_context = \ client.V1SecurityContext( run_as_group=WORKFLOW_RUNTIME_USER_GID, run_as_user=WORKFLOW_RUNTIME_USER_UID ) workflow_enginge_container.volume_mounts = [workspace_mount] secrets_store = REANAUserSecretsStore(owner_id) job_controller_env_secrets = secrets_store.\ get_env_secrets_as_k8s_spec() user = \ secrets_store.get_secret_value('HTCONDORCERN_USERNAME') or \ WORKFLOW_RUNTIME_USER_NAME job_controller_container = client.V1Container( name=current_app.config['JOB_CONTROLLER_NAME'], image=current_app.config['JOB_CONTROLLER_IMAGE'], image_pull_policy='IfNotPresent', env=[], volume_mounts=[], command=['/bin/bash', '-c'], args=self._create_job_controller_startup_cmd(user), ports=[]) if os.getenv('FLASK_ENV') == 'development': job_controller_env_vars.extend( current_app.config['DEBUG_ENV_VARS']) job_controller_env_vars.extend([ { 'name': 'REANA_USER_ID', 'value': owner_id }, { 'name': 'CERN_USER', 'value': user }, { 'name': 'USER', # Required by HTCondor 'value': user } ]) job_controller_container.env.extend(job_controller_env_vars) job_controller_container.env.extend(job_controller_env_secrets) job_controller_container.env.extend([ { 'name': 'REANA_SQLALCHEMY_DATABASE_URI', 'value': SQLALCHEMY_DATABASE_URI }, { 'name': 'REANA_STORAGE_BACKEND', 'value': REANA_STORAGE_BACKEND } ]) secrets_volume_mount = \ secrets_store.get_secrets_volume_mount_as_k8s_spec() job_controller_container.volume_mounts = [workspace_mount, db_mount] job_controller_container.volume_mounts.append(secrets_volume_mount) job_controller_container.ports = [{ "containerPort": current_app.config['JOB_CONTROLLER_CONTAINER_PORT'] }] containers = [workflow_enginge_container, job_controller_container] spec.template.spec = client.V1PodSpec( containers=containers) spec.template.spec.volumes = [ KubernetesWorkflowRunManager.k8s_shared_volume [REANA_STORAGE_BACKEND], secrets_store.get_file_secrets_volume_as_k8s_specs(), ] job.spec = spec job.spec.template.spec.restart_policy = 'Never' job.spec.ttl_seconds_after_finished = TTL_SECONDS_AFTER_FINISHED job.spec.backoff_limit = 0 return job
def gitlab_oauth(): # noqa r"""Endpoint to authorize REANA on GitLab. --- get: summary: Get access token from GitLab operationId: gitlab_oauth description: >- Authorize REANA on GitLab. produces: - application/json responses: 200: description: >- Ping succeeded. schema: type: object properties: message: type: string status: type: string examples: application/json: message: OK status: 200 201: description: >- Authorization succeeded. GitLab secret created. schema: type: object properties: message: type: string status: type: string examples: application/json: message: GitLab secret created status: 201 403: description: >- Request failed. User token not valid. examples: application/json: { "message": "Token is not valid." } 500: description: >- Request failed. Internal controller error. """ try: if current_user.is_authenticated: user = _get_user_from_invenio_user(current_user.email) else: user = get_user_from_token(request.args.get("access_token")) if "code" in request.args: # Verifies state parameter and obtain next url state_token = request.args.get("state") assert state_token # Checks authenticity and integrity of state and decodes the value. state = serializer.loads(state_token) # Verifies that state is for this session and that next parameter # has not been modified. assert state["sid"] == _create_identifier() # Stores next URL next_url = state["next"] gitlab_code = request.args.get("code") params = { "client_id": REANA_GITLAB_OAUTH_APP_ID, "client_secret": REANA_GITLAB_OAUTH_APP_SECRET, "redirect_uri": url_for(".gitlab_oauth", _external=True), "code": gitlab_code, "grant_type": "authorization_code", } gitlab_response = requests.post(REANA_GITLAB_URL + "/oauth/token", data=params).content secrets_store = REANAUserSecretsStore(str(user.id_)) secrets_store.add_secrets(_format_gitlab_secrets(gitlab_response), overwrite=True) return redirect(next_url), 201 else: return jsonify({"message": "OK"}), 200 except ValueError: return jsonify({"message": "Token is not valid."}), 403 except (AssertionError, BadData): return jsonify({"message": "State param is invalid."}), 403 except Exception as e: logging.error(traceback.format_exc()) return jsonify({"message": str(e)}), 500