def test_deletion_of_workspace_of_an_already_deleted_workflow(
app,
session,
default_user,
sample_yadage_workflow_in_db,
tmp_shared_volume_path):
"""Test workspace deletion of an already deleted workflow."""
create_workflow_workspace(sample_yadage_workflow_in_db.workspace_path)
absolute_workflow_workspace = os.path.join(
tmp_shared_volume_path,
sample_yadage_workflow_in_db.workspace_path)
# check that the workflow workspace exists
assert os.path.exists(absolute_workflow_workspace)
delete_workflow(sample_yadage_workflow_in_db,
hard_delete=False,
workspace=False)
assert os.path.exists(absolute_workflow_workspace)
delete_workflow(sample_yadage_workflow_in_db,
hard_delete=False,
workspace=True)
assert not os.path.exists(absolute_workflow_workspace)
delete_workflow(sample_yadage_workflow_in_db,
hard_delete=True,
workspace=True)
def test_workspace_permissions(
app, session, default_user, sample_yadage_workflow_in_db, tmp_shared_volume_path
):
"""Test workspace dir permissions."""
create_workflow_workspace(sample_yadage_workflow_in_db.workspace_path)
expected_worspace_permissions = "drwxrwxr-x"
workspace_permissions = stat.filemode(
os.stat(sample_yadage_workflow_in_db.workspace_path).st_mode
)
assert os.path.exists(sample_yadage_workflow_in_db.workspace_path)
assert workspace_permissions == expected_worspace_permissions
delete_workflow(sample_yadage_workflow_in_db, workspace=True)
def test_deletion_of_workspace_of_an_already_deleted_workflow(
app, session, default_user, sample_yadage_workflow_in_db
):
"""Test workspace deletion of an already deleted workflow."""
create_workflow_workspace(sample_yadage_workflow_in_db.workspace_path)
# check that the workflow workspace exists
assert os.path.exists(sample_yadage_workflow_in_db.workspace_path)
delete_workflow(sample_yadage_workflow_in_db, workspace=False)
assert os.path.exists(sample_yadage_workflow_in_db.workspace_path)
delete_workflow(sample_yadage_workflow_in_db, workspace=True)
assert not os.path.exists(sample_yadage_workflow_in_db.workspace_path)
def test_workspace_permissions(app, session, default_user,
sample_yadage_workflow_in_db,
tmp_shared_volume_path):
"""Test workspace dir permissions."""
create_workflow_workspace(sample_yadage_workflow_in_db.workspace_path)
expeted_worspace_permissions = 'drwxrwxr-x'
absolute_workflow_workspace = os.path.join(
tmp_shared_volume_path,
sample_yadage_workflow_in_db.workspace_path)
workspace_permissions = \
stat.filemode(os.stat(absolute_workflow_workspace).st_mode)
assert os.path.exists(absolute_workflow_workspace)
assert workspace_permissions == expeted_worspace_permissions
delete_workflow(sample_yadage_workflow_in_db,
hard_delete=True,
workspace=True)
def test_workspace_deletion(app,
session,
default_user,
sample_yadage_workflow_in_db,
tmp_shared_volume_path,
workspace,
hard_delete):
"""Test workspace deletion."""
workflow = sample_yadage_workflow_in_db
create_workflow_workspace(sample_yadage_workflow_in_db.workspace_path)
absolute_workflow_workspace = os.path.join(
tmp_shared_volume_path,
workflow.workspace_path)
# create a job for the workflow
workflow_job = Job(id_=uuid.uuid4(), workflow_uuid=workflow.id_)
job_cache_entry = JobCache(job_id=workflow_job.id_)
session.add(workflow_job)
session.add(job_cache_entry)
session.commit()
# create cached workspace
cache_dir_path = os.path.abspath(os.path.join(
absolute_workflow_workspace, os.pardir,
'archive', str(workflow_job.id_)))
os.makedirs(cache_dir_path)
# check that the workflow workspace exists
assert os.path.exists(absolute_workflow_workspace)
assert os.path.exists(cache_dir_path)
delete_workflow(workflow,
hard_delete=hard_delete,
workspace=workspace)
if hard_delete or workspace:
assert not os.path.exists(absolute_workflow_workspace)
# check that all cache entries for jobs
# of the deleted workflow are removed
cache_entries_after_delete = JobCache.query.filter_by(
job_id=workflow_job.id_).all()
assert not cache_entries_after_delete
assert not os.path.exists(cache_dir_path)
def create_workflow(): # noqa
r"""Create workflow and its workspace.
---
post:
summary: Create workflow and its workspace.
description: >-
This resource expects all necessary data to represent a workflow so
it is stored in database and its workspace is created.
operationId: create_workflow
produces:
- application/json
parameters:
- name: user
in: query
description: Required. UUID of workflow owner.
required: true
type: string
- name: workflow
in: body
description: >-
JSON object including workflow parameters and workflow
specification in JSON format (`yadageschemas.load()` output)
with necessary data to instantiate a yadage workflow.
required: true
schema:
type: object
properties:
operational_options:
type: object
description: Operational options.
reana_specification:
type: object
description: >-
Workflow specification in JSON format.
workflow_name:
type: string
description: Workflow name. If empty name will be generated.
git_data:
type: object
description: >-
GitLab data.
required: [reana_specification,
workflow_name,
operational_options]
responses:
201:
description: >-
Request succeeded. The workflow has been created along
with its workspace
schema:
type: object
properties:
message:
type: string
workflow_id:
type: string
workflow_name:
type: string
examples:
application/json:
{
"message": "Workflow workspace has been created.",
"workflow_id": "cdcf48b1-c2f3-4693-8230-b066e088c6ac",
"workflow_name": "mytest-1"
}
400:
description: >-
Request failed. The incoming data specification seems malformed
404:
description: >-
Request failed. User does not exist.
examples:
application/json:
{
"message": "User 00000000-0000-0000-0000-000000000000 does not
exist"
}
"""
try:
user_uuid = request.args["user"]
user = User.query.filter(User.id_ == user_uuid).first()
if not user:
return (
jsonify({
"message":
"User with id:{} does not exist".format(user_uuid)
}),
404,
)
workflow_uuid = str(uuid4())
# Use name prefix user specified or use default name prefix
# Actual name is prefix + autoincremented run_number.
workflow_name = request.json.get("workflow_name", "")
if workflow_name == "":
workflow_name = DEFAULT_NAME_FOR_WORKFLOWS
else:
try:
workflow_name.encode("ascii")
except UnicodeEncodeError:
# `workflow_name` contains something else than just ASCII.
raise REANAWorkflowNameError(
"Workflow name {} is not valid.".format(workflow_name))
git_ref = ""
git_repo = ""
if "git_data" in request.json:
git_data = request.json["git_data"]
git_ref = git_data["git_commit_sha"]
git_repo = git_data["git_url"]
# add spec and params to DB as JSON
workflow = Workflow(
id_=workflow_uuid,
name=workflow_name,
owner_id=request.args["user"],
reana_specification=request.json["reana_specification"],
operational_options=request.json.get("operational_options", {}),
type_=request.json["reana_specification"]["workflow"]["type"],
logs="",
git_ref=git_ref,
git_repo=git_repo,
)
Session.add(workflow)
Session.object_session(workflow).commit()
if git_ref:
create_workflow_workspace(
workflow.workspace_path,
user_id=user.id_,
git_url=git_data["git_url"],
git_branch=git_data["git_branch"],
git_ref=git_ref,
)
else:
create_workflow_workspace(workflow.workspace_path)
return (
jsonify({
"message": "Workflow workspace created",
"workflow_id": workflow.id_,
"workflow_name": get_workflow_name(workflow),
}),
201,
)
except (REANAWorkflowNameError, KeyError) as e:
return jsonify({"message": str(e)}), 400
except Exception as e:
return jsonify({"message": str(e)}), 500
def test_workspace_deletion(
mock_update_user_quota,
mock_update_workflow_quota,
app,
session,
default_user,
sample_yadage_workflow_in_db,
workspace,
):
"""Test workspace deletion."""
workflow = sample_yadage_workflow_in_db
create_workflow_workspace(sample_yadage_workflow_in_db.workspace_path)
# Add file to the worskpace
file_size = 123
file_path = os.path.join(sample_yadage_workflow_in_db.workspace_path, "temp.txt")
with open(file_path, "w") as f:
f.write("A" * file_size)
# Get disk usage
disk_usage = get_disk_usage_or_zero(sample_yadage_workflow_in_db.workspace_path)
assert disk_usage
# Update disk quotas
store_workflow_disk_quota(sample_yadage_workflow_in_db)
update_users_disk_quota(sample_yadage_workflow_in_db.owner)
# create a job for the workflow
workflow_job = Job(id_=uuid.uuid4(), workflow_uuid=workflow.id_)
job_cache_entry = JobCache(job_id=workflow_job.id_)
session.add(workflow_job)
session.commit()
session.add(job_cache_entry)
session.commit()
# create cached workspace
cache_dir_path = os.path.join(
sample_yadage_workflow_in_db.workspace_path,
"..",
"archive",
str(workflow_job.id_),
)
os.makedirs(cache_dir_path)
# check that the workflow workspace exists
assert os.path.exists(sample_yadage_workflow_in_db.workspace_path)
assert os.path.exists(cache_dir_path)
delete_workflow(workflow, workspace=workspace)
if workspace:
assert not os.path.exists(sample_yadage_workflow_in_db.workspace_path)
mock_update_user_quota.assert_called_once_with(
sample_yadage_workflow_in_db.owner,
bytes_to_sum=-disk_usage,
override_policy_checks=True,
)
mock_update_workflow_quota.assert_called_once_with(
sample_yadage_workflow_in_db,
bytes_to_sum=-disk_usage,
override_policy_checks=True,
)
else:
assert not mock_update_user_quota.called
assert not mock_update_workflow_quota.called
# check that all cache entries for jobs
# of the deleted workflow are removed
cache_entries_after_delete = JobCache.query.filter_by(job_id=workflow_job.id_).all()
assert not cache_entries_after_delete
assert not os.path.exists(cache_dir_path)
