示例#1
0
    def _update_disk_quota(workflow):
        if ResourceType.disk.name not in WORKFLOW_TERMINATION_QUOTA_UPDATE_POLICY:
            return

        try:
            update_users_disk_quota(user=workflow.owner)
            store_workflow_disk_quota(workflow)
        except Exception as e:
            logging.error(f"Failed to update disk quota: \n{e}\nContinuing...")
示例#2
0
def disk_usage_update():
    """Update users disk quota usage based on user workspace."""
    try:
        update_users_disk_quota()
        click.secho("Users disk quota usage updated successfully.", fg="green")
    except Exception as e:
        click.secho(
            "[ERROR]: An error occurred when updating users disk quota usage: {}"
            .format(repr(e)),
            fg="red",
        )
        sys.exit(1)
示例#3
0
 def _resource_usage_update(resource: ResourceType) -> None:
     """Update users resource quota usage."""
     try:
         if resource == ResourceType.disk:
             update_users_disk_quota()
             for workflow in Workflow.query.all():
                 store_workflow_disk_quota(workflow)
         elif resource == ResourceType.cpu:
             update_users_cpu_quota()
         click.secho(
             f"Users {resource.name} quota usage updated successfully.",
             fg="green")
     except Exception as e:
         click.secho(
             f"[ERROR]: An error occurred when updating users {resource.name} quota usage: {repr(e)}",
             fg="red",
         )
         sys.exit(1)
示例#4
0
 def _update_disk_quota(workflow):
     update_users_disk_quota(user=workflow.owner)
     store_workflow_disk_quota(workflow)
示例#5
0
def upload_file(workflow_id_or_name):
    r"""Upload file to workspace.

    ---
    post:
      summary: Adds a file to the workspace.
      description: >-
        This resource is expecting a workflow UUID and a file to place in the
        workspace.
      operationId: upload_file
      consumes:
        - application/octet-stream
      produces:
        - application/json
      parameters:
        - name: user
          in: query
          description: Required. UUID of workflow owner.
          required: true
          type: string
        - name: workflow_id_or_name
          in: path
          description: Required. Workflow UUID or name.
          required: true
          type: string
        - name: file
          in: body
          description: Required. File to add to the workspace.
          required: true
          schema:
            type: string
        - name: file_name
          in: query
          description: Required. File name.
          required: true
          type: string
      responses:
        200:
          description: >-
            Request succeeded. The file has been added to the workspace.
          schema:
            type: object
            properties:
              message:
                type: string
          examples:
            application/json:
              {
                "message": "`file_name` has been successfully uploaded.",
              }
        400:
          description: >-
            Request failed. The incoming data specification seems malformed
        404:
          description: >-
            Request failed. Workflow does not exist.
          schema:
            type: object
            properties:
              message:
                type: string
          examples:
            application/json:
              {
                "message": "Workflow cdcf48b1-c2f3-4693-8230-b066e088c6ac does
                            not exist",
              }
        500:
          description: >-
            Request failed. Internal controller error.
    """
    try:
        if not ("application/octet-stream" in request.headers.get("Content-Type")):
            return (
                jsonify(
                    {
                        "message": f"Wrong Content-Type "
                        f'{request.headers.get("Content-Type")} '
                        f"use application/octet-stream"
                    }
                ),
                400,
            )
        user_uuid = request.args["user"]
        full_file_name = request.args["file_name"]
        if not full_file_name:
            raise ValueError("The file transferred needs to have name.")

        workflow = _get_workflow_with_uuid_or_name(workflow_id_or_name, user_uuid)

        filename = full_file_name.split("/")[-1]

        # Remove starting '/' in path
        if full_file_name[0] == "/":
            full_file_name = full_file_name[1:]
        elif ".." in full_file_name.split("/"):
            raise REANAUploadPathError('Path cannot contain "..".')
        absolute_workspace_path = workflow.workspace_path
        if len(full_file_name.split("/")) > 1:
            dirs = full_file_name.split("/")[:-1]
            absolute_workspace_path = os.path.join(
                workflow.workspace_path, "/".join(dirs)
            )
            if not os.path.exists(absolute_workspace_path):
                os.makedirs(absolute_workspace_path)
        absolute_file_path = os.path.join(absolute_workspace_path, filename)

        FileStorage(request.stream).save(absolute_file_path, buffer_size=32768)
        # update user and workflow resource disk quota
        store_workflow_disk_quota(workflow, bytes_to_sum=request.content_length)
        update_users_disk_quota(workflow.owner, bytes_to_sum=request.content_length)
        return (
            jsonify(
                {"message": "{} has been successfully uploaded.".format(full_file_name)}
            ),
            200,
        )

    except ValueError:
        return (
            jsonify(
                {
                    "message": "REANA_WORKON is set to {0}, but "
                    "that workflow does not exist. "
                    "Please set your REANA_WORKON environment"
                    "variable appropriately.".format(workflow_id_or_name)
                }
            ),
            404,
        )
    except KeyError as e:
        return jsonify({"message": str(e)}), 400
    except Exception as e:
        return jsonify({"message": str(e)}), 500
示例#6
0
def delete_file(workflow_id_or_name, file_name):  # noqa
    r"""Delete a file from the workspace.

    ---
    delete:
      summary: Delete the specified file.
      description: >-
        This resource is expecting a workflow UUID and a filename existing
        inside the workspace to be deleted.
      operationId: delete_file
      produces:
        - application/json
      parameters:
        - name: user
          in: query
          description: Required. UUID of workflow owner.
          required: true
          type: string
        - name: workflow_id_or_name
          in: path
          description: Required. Workflow UUID or name
          required: true
          type: string
        - name: file_name
          in: path
          description: Required. Name (or path) of the file to be deleted.
          required: true
          type: string
      responses:
        200:
          description: >-
            Requests succeeded. The file has been downloaded.
          schema:
            type: file
        404:
          description: >-
            Request failed. `file_name` does not exist.
          examples:
            application/json:
              {
                "message": "input.csv does not exist"
              }
        500:
          description: >-
            Request failed. Internal controller error.
          examples:
            application/json:
              {
                "message": "Internal workflow controller error."
              }
    """
    try:
        user_uuid = request.args["user"]
        user = User.query.filter(User.id_ == user_uuid).first()
        if not user:
            return jsonify({"message": "User {} does not exist".format(user)}), 404

        workflow = _get_workflow_with_uuid_or_name(workflow_id_or_name, user_uuid)
        deleted = remove_files_recursive_wildcard(workflow.workspace_path, file_name)
        # update user and workflow resource disk quota
        freed_up_bytes = sum(
            size.get("size", 0) for size in deleted["deleted"].values()
        )
        store_workflow_disk_quota(workflow, bytes_to_sum=-freed_up_bytes)
        update_users_disk_quota(user, bytes_to_sum=-freed_up_bytes)
        return jsonify(deleted), 200

    except ValueError:
        return (
            jsonify(
                {
                    "message": "REANA_WORKON is set to {0}, but "
                    "that workflow does not exist. "
                    "Please set your REANA_WORKON environment "
                    "variable appropriately.".format(workflow_id_or_name)
                }
            ),
            404,
        )
    except KeyError:
        return jsonify({"message": "Malformed request."}), 400
    except NotFound:
        return jsonify({"message": "{0} does not exist.".format(file_name)}), 404
    except OSError:
        return jsonify({"message": "Error while deleting {}.".format(file_name)}), 500
    except Exception as e:
        return jsonify({"message": str(e)}), 500
def test_workspace_deletion(
    mock_update_user_quota,
    mock_update_workflow_quota,
    app,
    session,
    default_user,
    sample_yadage_workflow_in_db,
    workspace,
):
    """Test workspace deletion."""
    workflow = sample_yadage_workflow_in_db
    create_workflow_workspace(sample_yadage_workflow_in_db.workspace_path)

    # Add file to the worskpace
    file_size = 123
    file_path = os.path.join(sample_yadage_workflow_in_db.workspace_path, "temp.txt")
    with open(file_path, "w") as f:
        f.write("A" * file_size)

    # Get disk usage
    disk_usage = get_disk_usage_or_zero(sample_yadage_workflow_in_db.workspace_path)
    assert disk_usage

    # Update disk quotas
    store_workflow_disk_quota(sample_yadage_workflow_in_db)
    update_users_disk_quota(sample_yadage_workflow_in_db.owner)

    # create a job for the workflow
    workflow_job = Job(id_=uuid.uuid4(), workflow_uuid=workflow.id_)
    job_cache_entry = JobCache(job_id=workflow_job.id_)
    session.add(workflow_job)
    session.commit()
    session.add(job_cache_entry)
    session.commit()

    # create cached workspace
    cache_dir_path = os.path.join(
        sample_yadage_workflow_in_db.workspace_path,
        "..",
        "archive",
        str(workflow_job.id_),
    )

    os.makedirs(cache_dir_path)

    # check that the workflow workspace exists
    assert os.path.exists(sample_yadage_workflow_in_db.workspace_path)
    assert os.path.exists(cache_dir_path)
    delete_workflow(workflow, workspace=workspace)
    if workspace:
        assert not os.path.exists(sample_yadage_workflow_in_db.workspace_path)
        mock_update_user_quota.assert_called_once_with(
            sample_yadage_workflow_in_db.owner,
            bytes_to_sum=-disk_usage,
            override_policy_checks=True,
        )
        mock_update_workflow_quota.assert_called_once_with(
            sample_yadage_workflow_in_db,
            bytes_to_sum=-disk_usage,
            override_policy_checks=True,
        )
    else:
        assert not mock_update_user_quota.called
        assert not mock_update_workflow_quota.called

    # check that all cache entries for jobs
    # of the deleted workflow are removed
    cache_entries_after_delete = JobCache.query.filter_by(job_id=workflow_job.id_).all()
    assert not cache_entries_after_delete
    assert not os.path.exists(cache_dir_path)
示例#8
0
def delete_workflow(workflow, all_runs=False, workspace=False):
    """Delete workflow."""
    if workflow.status in [
            RunStatus.created,
            RunStatus.finished,
            RunStatus.stopped,
            RunStatus.deleted,
            RunStatus.failed,
            RunStatus.queued,
            RunStatus.pending,
    ]:
        try:
            to_be_deleted = [workflow]
            if all_runs:
                to_be_deleted += (Session.query(Workflow).filter(
                    Workflow.name == workflow.name,
                    Workflow.status != RunStatus.running,
                ).all())
            for workflow in to_be_deleted:
                if workspace:
                    remove_workflow_workspace(workflow.workspace_path)

                    disk_resource = get_default_quota_resource(
                        ResourceType.disk.name)
                    workflow_disk_resource = WorkflowResource.query.filter(
                        WorkflowResource.workflow_id == workflow.id_,
                        WorkflowResource.resource_id == disk_resource.id_,
                    ).one_or_none()
                    disk_usage = None
                    if workflow_disk_resource:
                        disk_usage = workflow_disk_resource.quota_used

                    if disk_usage:
                        # We override the quota update policy checks so that the quotas
                        # are updated immediately and the user can reuse the freed
                        # resources without waiting.
                        store_workflow_disk_quota(
                            workflow,
                            bytes_to_sum=-disk_usage,
                            override_policy_checks=True,
                        )
                        update_users_disk_quota(
                            workflow.owner,
                            bytes_to_sum=-disk_usage,
                            override_policy_checks=True,
                        )
                _mark_workflow_as_deleted_in_db(workflow)
                remove_workflow_jobs_from_cache(workflow)

            return (
                jsonify({
                    "message": "Workflow successfully deleted",
                    "workflow_id": workflow.id_,
                    "workflow_name": get_workflow_name(workflow),
                    "status": workflow.status.name,
                    "user": str(workflow.owner_id),
                }),
                200,
            )
        except Exception as e:
            logging.error(traceback.format_exc())
            return jsonify({"message": str(e)}), 500
    elif workflow.status == RunStatus.running:
        raise REANAWorkflowDeletionError(
            "Workflow {0}.{1} cannot be deleted as it"
            " is currently running.".format(workflow.name,
                                            workflow.run_number))