def _update_disk_quota(workflow):
if ResourceType.disk.name not in WORKFLOW_TERMINATION_QUOTA_UPDATE_POLICY:
return
try:
update_users_disk_quota(user=workflow.owner)
store_workflow_disk_quota(workflow)
except Exception as e:
logging.error(f"Failed to update disk quota: \n{e}\nContinuing...")
def get_workspace_disk_usage(self, summarize=False, search=None):
"""Retrieve disk usage information of a workspace."""
from functools import partial
if not summarize:
# size break down per directory so we can't query DB (`r-client du`)
return get_disk_usage(
self.workspace_path,
summarize,
search,
to_human_readable_units=partial(
ResourceUnit.human_readable_unit, ResourceUnit.bytes_),
)
disk_usage = self.get_quota_usage().get("disk", {}).get("usage", {})
if not disk_usage:
# recalculate disk workflow resource
workflow_resource = store_workflow_disk_quota(self)
disk_usage = dict(
raw=workflow_resource.quota_used,
to_human_readable_units=ResourceUnit.human_readable_unit(
ResourceUnit.bytes_, workflow_resource.quota_used),
)
return [{"name": "", "size": disk_usage}]
def _resource_usage_update(resource: ResourceType) -> None:
"""Update users resource quota usage."""
try:
if resource == ResourceType.disk:
update_users_disk_quota()
for workflow in Workflow.query.all():
store_workflow_disk_quota(workflow)
elif resource == ResourceType.cpu:
update_users_cpu_quota()
click.secho(
f"Users {resource.name} quota usage updated successfully.",
fg="green")
except Exception as e:
click.secho(
f"[ERROR]: An error occurred when updating users {resource.name} quota usage: {repr(e)}",
fg="red",
)
sys.exit(1)
def _update_disk_quota(workflow):
update_users_disk_quota(user=workflow.owner)
store_workflow_disk_quota(workflow)
def upload_file(workflow_id_or_name):
r"""Upload file to workspace.
---
post:
summary: Adds a file to the workspace.
description: >-
This resource is expecting a workflow UUID and a file to place in the
workspace.
operationId: upload_file
consumes:
- application/octet-stream
produces:
- application/json
parameters:
- name: user
in: query
description: Required. UUID of workflow owner.
required: true
type: string
- name: workflow_id_or_name
in: path
description: Required. Workflow UUID or name.
required: true
type: string
- name: file
in: body
description: Required. File to add to the workspace.
required: true
schema:
type: string
- name: file_name
in: query
description: Required. File name.
required: true
type: string
responses:
200:
description: >-
Request succeeded. The file has been added to the workspace.
schema:
type: object
properties:
message:
type: string
examples:
application/json:
{
"message": "`file_name` has been successfully uploaded.",
}
400:
description: >-
Request failed. The incoming data specification seems malformed
404:
description: >-
Request failed. Workflow does not exist.
schema:
type: object
properties:
message:
type: string
examples:
application/json:
{
"message": "Workflow cdcf48b1-c2f3-4693-8230-b066e088c6ac does
not exist",
}
500:
description: >-
Request failed. Internal controller error.
"""
try:
if not ("application/octet-stream" in request.headers.get("Content-Type")):
return (
jsonify(
{
"message": f"Wrong Content-Type "
f'{request.headers.get("Content-Type")} '
f"use application/octet-stream"
}
),
400,
)
user_uuid = request.args["user"]
full_file_name = request.args["file_name"]
if not full_file_name:
raise ValueError("The file transferred needs to have name.")
workflow = _get_workflow_with_uuid_or_name(workflow_id_or_name, user_uuid)
filename = full_file_name.split("/")[-1]
# Remove starting '/' in path
if full_file_name[0] == "/":
full_file_name = full_file_name[1:]
elif ".." in full_file_name.split("/"):
raise REANAUploadPathError('Path cannot contain "..".')
absolute_workspace_path = workflow.workspace_path
if len(full_file_name.split("/")) > 1:
dirs = full_file_name.split("/")[:-1]
absolute_workspace_path = os.path.join(
workflow.workspace_path, "/".join(dirs)
)
if not os.path.exists(absolute_workspace_path):
os.makedirs(absolute_workspace_path)
absolute_file_path = os.path.join(absolute_workspace_path, filename)
FileStorage(request.stream).save(absolute_file_path, buffer_size=32768)
# update user and workflow resource disk quota
store_workflow_disk_quota(workflow, bytes_to_sum=request.content_length)
update_users_disk_quota(workflow.owner, bytes_to_sum=request.content_length)
return (
jsonify(
{"message": "{} has been successfully uploaded.".format(full_file_name)}
),
200,
)
except ValueError:
return (
jsonify(
{
"message": "REANA_WORKON is set to {0}, but "
"that workflow does not exist. "
"Please set your REANA_WORKON environment"
"variable appropriately.".format(workflow_id_or_name)
}
),
404,
)
except KeyError as e:
return jsonify({"message": str(e)}), 400
except Exception as e:
return jsonify({"message": str(e)}), 500
def delete_file(workflow_id_or_name, file_name): # noqa
r"""Delete a file from the workspace.
---
delete:
summary: Delete the specified file.
description: >-
This resource is expecting a workflow UUID and a filename existing
inside the workspace to be deleted.
operationId: delete_file
produces:
- application/json
parameters:
- name: user
in: query
description: Required. UUID of workflow owner.
required: true
type: string
- name: workflow_id_or_name
in: path
description: Required. Workflow UUID or name
required: true
type: string
- name: file_name
in: path
description: Required. Name (or path) of the file to be deleted.
required: true
type: string
responses:
200:
description: >-
Requests succeeded. The file has been downloaded.
schema:
type: file
404:
description: >-
Request failed. `file_name` does not exist.
examples:
application/json:
{
"message": "input.csv does not exist"
}
500:
description: >-
Request failed. Internal controller error.
examples:
application/json:
{
"message": "Internal workflow controller error."
}
"""
try:
user_uuid = request.args["user"]
user = User.query.filter(User.id_ == user_uuid).first()
if not user:
return jsonify({"message": "User {} does not exist".format(user)}), 404
workflow = _get_workflow_with_uuid_or_name(workflow_id_or_name, user_uuid)
deleted = remove_files_recursive_wildcard(workflow.workspace_path, file_name)
# update user and workflow resource disk quota
freed_up_bytes = sum(
size.get("size", 0) for size in deleted["deleted"].values()
)
store_workflow_disk_quota(workflow, bytes_to_sum=-freed_up_bytes)
update_users_disk_quota(user, bytes_to_sum=-freed_up_bytes)
return jsonify(deleted), 200
except ValueError:
return (
jsonify(
{
"message": "REANA_WORKON is set to {0}, but "
"that workflow does not exist. "
"Please set your REANA_WORKON environment "
"variable appropriately.".format(workflow_id_or_name)
}
),
404,
)
except KeyError:
return jsonify({"message": "Malformed request."}), 400
except NotFound:
return jsonify({"message": "{0} does not exist.".format(file_name)}), 404
except OSError:
return jsonify({"message": "Error while deleting {}.".format(file_name)}), 500
except Exception as e:
return jsonify({"message": str(e)}), 500
def test_workspace_deletion(
mock_update_user_quota,
mock_update_workflow_quota,
app,
session,
default_user,
sample_yadage_workflow_in_db,
workspace,
):
"""Test workspace deletion."""
workflow = sample_yadage_workflow_in_db
create_workflow_workspace(sample_yadage_workflow_in_db.workspace_path)
# Add file to the worskpace
file_size = 123
file_path = os.path.join(sample_yadage_workflow_in_db.workspace_path, "temp.txt")
with open(file_path, "w") as f:
f.write("A" * file_size)
# Get disk usage
disk_usage = get_disk_usage_or_zero(sample_yadage_workflow_in_db.workspace_path)
assert disk_usage
# Update disk quotas
store_workflow_disk_quota(sample_yadage_workflow_in_db)
update_users_disk_quota(sample_yadage_workflow_in_db.owner)
# create a job for the workflow
workflow_job = Job(id_=uuid.uuid4(), workflow_uuid=workflow.id_)
job_cache_entry = JobCache(job_id=workflow_job.id_)
session.add(workflow_job)
session.commit()
session.add(job_cache_entry)
session.commit()
# create cached workspace
cache_dir_path = os.path.join(
sample_yadage_workflow_in_db.workspace_path,
"..",
"archive",
str(workflow_job.id_),
)
os.makedirs(cache_dir_path)
# check that the workflow workspace exists
assert os.path.exists(sample_yadage_workflow_in_db.workspace_path)
assert os.path.exists(cache_dir_path)
delete_workflow(workflow, workspace=workspace)
if workspace:
assert not os.path.exists(sample_yadage_workflow_in_db.workspace_path)
mock_update_user_quota.assert_called_once_with(
sample_yadage_workflow_in_db.owner,
bytes_to_sum=-disk_usage,
override_policy_checks=True,
)
mock_update_workflow_quota.assert_called_once_with(
sample_yadage_workflow_in_db,
bytes_to_sum=-disk_usage,
override_policy_checks=True,
)
else:
assert not mock_update_user_quota.called
assert not mock_update_workflow_quota.called
# check that all cache entries for jobs
# of the deleted workflow are removed
cache_entries_after_delete = JobCache.query.filter_by(job_id=workflow_job.id_).all()
assert not cache_entries_after_delete
assert not os.path.exists(cache_dir_path)
def delete_workflow(workflow, all_runs=False, workspace=False):
"""Delete workflow."""
if workflow.status in [
RunStatus.created,
RunStatus.finished,
RunStatus.stopped,
RunStatus.deleted,
RunStatus.failed,
RunStatus.queued,
RunStatus.pending,
]:
try:
to_be_deleted = [workflow]
if all_runs:
to_be_deleted += (Session.query(Workflow).filter(
Workflow.name == workflow.name,
Workflow.status != RunStatus.running,
).all())
for workflow in to_be_deleted:
if workspace:
remove_workflow_workspace(workflow.workspace_path)
disk_resource = get_default_quota_resource(
ResourceType.disk.name)
workflow_disk_resource = WorkflowResource.query.filter(
WorkflowResource.workflow_id == workflow.id_,
WorkflowResource.resource_id == disk_resource.id_,
).one_or_none()
disk_usage = None
if workflow_disk_resource:
disk_usage = workflow_disk_resource.quota_used
if disk_usage:
# We override the quota update policy checks so that the quotas
# are updated immediately and the user can reuse the freed
# resources without waiting.
store_workflow_disk_quota(
workflow,
bytes_to_sum=-disk_usage,
override_policy_checks=True,
)
update_users_disk_quota(
workflow.owner,
bytes_to_sum=-disk_usage,
override_policy_checks=True,
)
_mark_workflow_as_deleted_in_db(workflow)
remove_workflow_jobs_from_cache(workflow)
return (
jsonify({
"message": "Workflow successfully deleted",
"workflow_id": workflow.id_,
"workflow_name": get_workflow_name(workflow),
"status": workflow.status.name,
"user": str(workflow.owner_id),
}),
200,
)
except Exception as e:
logging.error(traceback.format_exc())
return jsonify({"message": str(e)}), 500
elif workflow.status == RunStatus.running:
raise REANAWorkflowDeletionError(
"Workflow {0}.{1} cannot be deleted as it"
" is currently running.".format(workflow.name,
workflow.run_number))
