def _run_workflow(time_elapsed_seconds=0.5, finish=True):
"""Mock a workflow run."""
id_ = uuid4()
workflow = Workflow(
id_=str(id_),
name="test_{}".format(id_),
owner_id=new_user.id_,
reana_specification=[],
type_="serial",
logs="",
status=RunStatus.created,
)
# start workflow
workflow.status = RunStatus.running
session.add(workflow)
session.commit()
termination_value = datetime.now() + timedelta(
seconds=time_elapsed_seconds)
class MockDatetime(datetime):
@classmethod
def now(cls):
return termination_value
if finish:
with mock.patch("reana_db.models.datetime", MockDatetime):
Workflow.update_workflow_status(session, workflow.id_,
RunStatus.finished)
return workflow
def test_delete_all_workflow_runs(app, session, default_user,
yadage_workflow_with_name):
"""Test deletion of all runs of a given workflow."""
# add 5 workflows in the database with the same name
for i in range(5):
workflow = Workflow(
id_=uuid.uuid4(),
name=yadage_workflow_with_name["name"],
owner_id=default_user.id_,
reana_specification=yadage_workflow_with_name[
"reana_specification"],
operational_options={},
type_=yadage_workflow_with_name["reana_specification"]["workflow"]
["type"],
logs="",
)
session.add(workflow)
if i == 4:
workflow.status = RunStatus.running
not_deleted_one = workflow.id_
session.commit()
first_workflow = (session.query(Workflow).filter_by(
name=yadage_workflow_with_name["name"]).first())
delete_workflow(first_workflow, all_runs=True)
for workflow in session.query(Workflow).filter_by(
name=first_workflow.name).all():
if not_deleted_one == workflow.id_:
assert workflow.status == RunStatus.running
else:
assert workflow.status == RunStatus.deleted
def publish_workflow_submission(workflow, user_id, parameters):
"""Publish workflow submission."""
from reana_server.status import NodesStatus
Workflow.update_workflow_status(Session, workflow.id_, RunStatus.queued)
scheduling_policy = REANA_WORKFLOW_SCHEDULING_POLICY
if scheduling_policy not in REANA_WORKFLOW_SCHEDULING_POLICIES:
raise ValueError(
'Workflow scheduling policy "{0}" is not valid.'.format(
scheduling_policy))
# No need to estimate the complexity for "fifo" strategy
if scheduling_policy == "fifo":
workflow_priority = 0
workflow_min_job_memory = 0
else:
total_cluster_memory = NodesStatus().get_total_memory()
complexity = _calculate_complexity(workflow)
workflow_priority = workflow.get_priority(total_cluster_memory)
workflow_min_job_memory = get_workflow_min_job_memory(complexity)
current_workflow_submission_publisher.publish_workflow_submission(
user_id=str(user_id),
workflow_id_or_name=workflow.get_full_workflow_name(),
parameters=parameters,
priority=workflow_priority,
min_job_memory=workflow_min_job_memory,
)
def _update_workflow_status(workflow_uuid, status, logs):
"""Update workflow status in DB."""
Workflow.update_workflow_status(Session, workflow_uuid, status, logs, None)
alive_statuses = \
[WorkflowStatus.created, WorkflowStatus.running, WorkflowStatus.queued]
if status not in alive_statuses:
_delete_workflow_engine_pod(workflow_uuid)
def _update_workflow_status(workflow_uuid, status, logs):
"""Update workflow status in DB."""
Workflow.update_workflow_status(Session, workflow_uuid, status, logs, None)
workflow = Session.query(Workflow).filter_by(id_=workflow_uuid)\
.one_or_none()
if workflow.git_ref:
_update_commit_status(workflow, status)
def initialize_krb5_token(workflow_uuid):
"""Create kerberos ticket from mounted keytab_file."""
cern_user = os.environ.get("CERN_USER")
keytab_file = os.environ.get("CERN_KEYTAB")
cmd = "kinit -kt /etc/reana/secrets/{} {}@CERN.CH".format(
keytab_file, cern_user)
if cern_user:
try:
subprocess.check_output(cmd, shell=True)
except subprocess.CalledProcessError as err:
msg = "Executing: {} \n Authentication failed: {}".format(cmd, err)
Workflow.update_workflow_status(
db_session=Session,
workflow_uuid=workflow_uuid,
status=None,
new_logs=msg,
)
logging.error(msg, exc_info=True)
sys.exit(1)
else:
msg = "CERN_USER is not set."
logging.error(msg, exc_info=True)
Workflow.update_workflow_status(db_session=Session,
workflow_uuid=workflow_uuid,
status=None,
new_logs=msg)
logging.error(msg, exc_info=True)
def _update_workflow_status(workflow, status, logs):
"""Update workflow status in DB."""
if workflow.status != status:
Workflow.update_workflow_status(Session, workflow.id_, status, logs,
None)
if workflow.git_ref:
_update_commit_status(workflow, status)
if status not in ALIVE_STATUSES:
workflow.run_finished_at = datetime.now()
workflow.logs = workflow.logs or ""
try:
workflow_engine_logs = _get_workflow_engine_pod_logs(workflow)
workflow.logs += workflow_engine_logs + "\n"
except REANAWorkflowControllerError as exception:
logging.error(
f"Could not fetch workflow engine pod logs for workflow {workflow.id_}."
f" Error: {exception}")
workflow.logs += "Workflow engine logs could not be retrieved.\n"
if RunStatus.should_cleanup_job(status):
try:
_delete_workflow_job(workflow)
except REANAWorkflowControllerError as exception:
logging.error(
f"Could not clean up workflow job for workflow {workflow.id_}."
f" Error: {exception}")
def _update_workflow_status(workflow_uuid, status, logs):
"""Update workflow status in DB."""
Workflow.update_workflow_status(Session, workflow_uuid, status, logs, None)
workflow = Session.query(Workflow).filter_by(id_=workflow_uuid)\
.one_or_none()
if workflow.git_ref:
_update_commit_status(workflow, status)
alive_statuses = \
[WorkflowStatus.created, WorkflowStatus.running, WorkflowStatus.queued]
if status not in alive_statuses:
_delete_workflow_engine_pod(workflow_uuid)
def test_workflow_run_number_assignment(db, session, new_user):
"""Test workflow run number assignment."""
workflow_name = "workflow"
first_workflow = Workflow(
id_=str(uuid4()),
name=workflow_name,
owner_id=new_user.id_,
reana_specification=[],
type_="serial",
logs="",
)
session.add(first_workflow)
session.commit()
assert first_workflow.run_number == 1
second_workflow = Workflow(
id_=str(uuid4()),
name=workflow_name,
owner_id=new_user.id_,
reana_specification=[],
type_="serial",
logs="",
)
session.add(second_workflow)
session.commit()
assert second_workflow.run_number == 2
first_workflow_restart = Workflow(
id_=str(uuid4()),
name=workflow_name,
owner_id=new_user.id_,
reana_specification=[],
type_="serial",
logs="",
restart=True,
run_number=first_workflow.run_number,
)
session.add(first_workflow_restart)
session.commit()
assert first_workflow_restart.run_number == 1.1
first_workflow_second_restart = Workflow(
id_=str(uuid4()),
name=workflow_name,
owner_id=new_user.id_,
reana_specification=[],
type_="serial",
logs="",
restart=True,
run_number=first_workflow_restart.run_number,
)
session.add(first_workflow_second_restart)
session.commit()
assert first_workflow_second_restart.run_number == 1.2
def test_workflow_run_number_assignment(db, session):
"""Test workflow run number assignment."""
workflow_name = 'workflow'
owner_id = str(uuid4())
first_workflow = Workflow(
id_=str(uuid4()),
name=workflow_name,
owner_id=owner_id,
reana_specification=[],
type_='serial',
logs='',
)
session.add(first_workflow)
session.commit()
assert first_workflow.run_number == 1
second_workflow = Workflow(
id_=str(uuid4()),
name=workflow_name,
owner_id=owner_id,
reana_specification=[],
type_='serial',
logs='',
)
session.add(second_workflow)
session.commit()
assert second_workflow.run_number == 2
first_workflow_restart = Workflow(
id_=str(uuid4()),
name=workflow_name,
owner_id=owner_id,
reana_specification=[],
type_='serial',
logs='',
restart=True,
run_number=first_workflow.run_number,
)
session.add(first_workflow_restart)
session.commit()
assert first_workflow_restart.run_number == 1.1
first_workflow_second_restart = Workflow(
id_=str(uuid4()),
name=workflow_name,
owner_id=owner_id,
reana_specification=[],
type_='serial',
logs='',
restart=True,
run_number=first_workflow_restart.run_number,
)
session.add(first_workflow_second_restart)
session.commit()
assert first_workflow_second_restart.run_number == 1.2
def _update_workflow_status(workflow, status, logs):
"""Update workflow status in DB."""
if workflow.status != status:
Workflow.update_workflow_status(Session, workflow.id_, status, logs, None)
if workflow.git_ref:
_update_commit_status(workflow, status)
alive_statuses = [
RunStatus.created,
RunStatus.running,
RunStatus.queued,
]
if status not in alive_statuses:
_delete_workflow_engine_pod(workflow)
def _update_workflow_status(workflow, status, logs):
"""Update workflow status in DB."""
if workflow.status != status:
Workflow.update_workflow_status(Session, workflow.id_, status, logs,
None)
if workflow.git_ref:
_update_commit_status(workflow, status)
alive_statuses = [
WorkflowStatus.created,
WorkflowStatus.running,
WorkflowStatus.queued,
]
if status not in alive_statuses:
workflow.run_finished_at = datetime.now()
_delete_workflow_engine_pod(workflow)
def sample_serial_workflow_in_db(
app, default_user, session, serial_workflow, sample_workflow_workspace
):
"""Create a sample workflow in the database.
Scope: function
Adds a sample serial workflow in the DB.
"""
from reana_db.models import Workflow
workflow_id = uuid4()
relative_workspace_path = build_workspace_path(default_user.id_, workflow_id)
next(sample_workflow_workspace(relative_workspace_path))
workflow = Workflow(
id_=workflow_id,
name="sample_serial_workflow_1",
owner_id=default_user.id_,
reana_specification=serial_workflow["reana_specification"],
operational_options={},
type_=serial_workflow["reana_specification"]["workflow"]["type"],
logs="",
workspace_path=relative_workspace_path,
)
session.add(workflow)
session.commit()
yield workflow
for resource in workflow.resources:
session.delete(resource)
session.delete(workflow)
session.commit()
def test_get_workflows(app, session, default_user, cwl_workflow_with_name):
"""Test listing all workflows."""
with app.test_client() as client:
workflow_uuid = uuid.uuid4()
workflow_name = 'my_test_workflow'
workflow = Workflow(
id_=workflow_uuid,
name=workflow_name,
status=WorkflowStatus.finished,
owner_id=default_user.id_,
reana_specification=cwl_workflow_with_name['reana_specification'],
type_=cwl_workflow_with_name[
'reana_specification']['type'],
logs='')
session.add(workflow)
session.commit()
res = client.get(url_for('api.get_workflows'),
query_string={"user": default_user.id_})
assert res.status_code == 200
response_data = json.loads(res.get_data(as_text=True))
expected_data = [
{
"id": str(workflow.id_),
"name": workflow.name + '.1', # Add run_number
"status": workflow.status.name,
"user": str(workflow.owner_id),
"created": response_data[0]["created"],
"size": "-"
}
]
assert response_data == expected_data
def test_workflow_can_transition_to(db, session, from_status, to_status,
can_transition, new_user):
"""Test workflow run number assignment."""
workflow_name = "test-workflow"
workflow = Workflow(
id_=str(uuid4()),
name=workflow_name,
owner_id=new_user.id_,
reana_specification=[],
type_="serial",
logs="",
status=from_status,
)
session.add(workflow)
session.commit()
assert workflow.can_transition_to(to_status) is can_transition
def on_message(self, workflow_submission, message):
"""On new workflow_submission event handler."""
if reana_ready():
message.ack()
workflow_submission = json.loads(workflow_submission)
logging.info(
'Starting queued workflow: {}'.format(workflow_submission))
workflow_submission['status'] = 'start'
response, http_response = current_rwc_api_client.api.\
set_workflow_status(**workflow_submission).result()
http_response_json = http_response.json()
if http_response.status_code == 200:
workflow_uuid = http_response_json['workflow_id']
status = http_response_json['status']
Workflow.update_workflow_status(Session, workflow_uuid, status)
else:
message.requeue()
def _update_workflow_status(workflow, status, logs):
"""Update workflow status in DB."""
if workflow.status != status:
Workflow.update_workflow_status(Session, workflow.id_, status, logs,
None)
if workflow.git_ref:
_update_commit_status(workflow, status)
if status not in ALIVE_STATUSES:
try:
workflow.run_finished_at = datetime.now()
if RunStatus.should_cleanup_job(status):
_delete_workflow_engine_pod(workflow)
except REANAWorkflowControllerError:
logging.error(
f"Could not clean up workflow engine for workflow {workflow.id_}"
)
workflow.logs += "Workflow engine logs could not be retrieved.\n"
def test_get_workflow_status_with_name(app, session, default_user,
cwl_workflow_with_name):
"""Test get workflow status."""
with app.test_client() as client:
# create workflow
workflow_uuid = uuid.uuid4()
workflow_name = 'my_test_workflow'
workflow = Workflow(
id_=workflow_uuid,
name=workflow_name,
status=WorkflowStatus.finished,
owner_id=default_user.id_,
reana_specification=cwl_workflow_with_name['reana_specification'],
type_=cwl_workflow_with_name[
'reana_specification']['type'],
logs='')
session.add(workflow)
session.commit()
workflow = Workflow.query.filter(
Workflow.name == workflow_name).first()
res = client.get(url_for('api.get_workflow_status',
workflow_id_or_name=workflow_name + '.1'),
query_string={"user": default_user.id_},
content_type='application/json',
data=json.dumps(cwl_workflow_with_name))
json_response = json.loads(res.data.decode())
assert json_response.get('status') == workflow.status.name
workflow.status = WorkflowStatus.finished
session.commit()
res = client.get(url_for('api.get_workflow_status',
workflow_id_or_name=workflow_name + '.1'),
query_string={"user": default_user.id_},
content_type='application/json',
data=json.dumps(cwl_workflow_with_name))
json_response = json.loads(res.data.decode())
assert json_response.get('status') == workflow.status.name
def test_delete_all_workflow_runs(app,
session,
default_user,
yadage_workflow_with_name,
hard_delete):
"""Test deletion of all runs of a given workflow."""
# add 5 workflows in the database with the same name
for i in range(5):
workflow = Workflow(id_=uuid.uuid4(),
name=yadage_workflow_with_name['name'],
owner_id=default_user.id_,
reana_specification=yadage_workflow_with_name[
'reana_specification'],
operational_options={},
type_=yadage_workflow_with_name[
'reana_specification']['workflow']['type'],
logs='')
session.add(workflow)
if i == 4:
workflow.status = WorkflowStatus.running
not_deleted_one = workflow.id_
session.commit()
first_workflow = session.query(Workflow).\
filter_by(name=yadage_workflow_with_name['name']).first()
delete_workflow(first_workflow,
all_runs=True,
hard_delete=hard_delete)
if not hard_delete:
for workflow in session.query(Workflow).\
filter_by(name=first_workflow.name).all():
if not_deleted_one == workflow.id_:
assert workflow.status == WorkflowStatus.running
else:
assert workflow.status == WorkflowStatus.deleted
else:
# the one running should not be deleted
assert len(session.query(Workflow).
filter_by(name=first_workflow.name).all()) == 1
def sample_serial_workflow_in_db(app, default_user, session, serial_workflow):
"""Create a sample workflow in the database.
Scope: function
Adds a sample serial workflow in the DB.
"""
workflow = Workflow(
id_=uuid4(),
name='sample_serial_workflow_1',
owner_id=default_user.id_,
reana_specification=serial_workflow['reana_specification'],
operational_options={},
type_=serial_workflow['reana_specification']['workflow']['type'],
logs='')
session.add(workflow)
session.commit()
yield workflow
session.delete(workflow)
session.commit()
def test_delete_all_workflow_runs(app,
session,
default_user,
yadage_workflow_with_name,
hard_delete):
"""Test deletion of all runs of a given workflow."""
# add 5 workflows in the database with the same name
for i in range(5):
workflow = Workflow(id_=uuid.uuid4(),
name=yadage_workflow_with_name['name'],
owner_id=default_user.id_,
reana_specification=yadage_workflow_with_name[
'reana_specification'],
operational_options={},
type_=yadage_workflow_with_name[
'reana_specification']['workflow']['type'],
logs='')
session.add(workflow)
session.commit()
first_workflow = session.query(Workflow).\
filter_by(name=yadage_workflow_with_name['name']).first()
with app.test_client() as client:
res = client.put(
url_for('api.set_workflow_status',
workflow_id_or_name=first_workflow.id_),
query_string={
'user': default_user.id_,
'status': 'deleted'
},
content_type='application/json',
data=json.dumps({'hard_delete': hard_delete,
'all_runs': True}))
if not hard_delete:
for workflow in session.query(Workflow).\
filter_by(name=first_workflow.name).all():
assert workflow.status == WorkflowStatus.deleted
else:
assert session.query(Workflow).\
filter_by(name=first_workflow.name).all() == []
def clone_workflow(workflow, reana_spec, restart_type):
"""Create a copy of workflow in DB for restarting."""
try:
cloned_workflow = Workflow(
id_=str(uuid4()),
name=workflow.name,
owner_id=workflow.owner_id,
reana_specification=reana_spec or workflow.reana_specification,
type_=restart_type or workflow.type_,
logs="",
workspace_path=workflow.workspace_path,
restart=True,
run_number=workflow.run_number,
)
Session.add(cloned_workflow)
Session.object_session(cloned_workflow).commit()
return cloned_workflow
except SQLAlchemyError as e:
message = "Database connection failed, please retry."
logging.error(
f"Error while creating {cloned_workflow.id_}: {message}\n{e}",
exc_info=True)
def sample_yadage_workflow_in_db(app, default_user, session,
yadage_workflow_with_name):
"""Create a sample workflow in the database.
Scope: function
Adds a sample yadage workflow in the DB.
"""
workflow = Workflow(
id_=uuid4(),
name="sample_serial_workflow_1",
owner_id=default_user.id_,
reana_specification=yadage_workflow_with_name["reana_specification"],
operational_options={},
type_=yadage_workflow_with_name["reana_specification"]["workflow"]
["type"],
logs="",
)
session.add(workflow)
session.commit()
yield workflow
session.delete(workflow)
session.commit()
def create_workflow(): # noqa
r"""Create workflow and its workspace.
---
post:
summary: Create workflow and its workspace.
description: >-
This resource expects all necessary data to represent a workflow so
it is stored in database and its workspace is created.
operationId: create_workflow
produces:
- application/json
parameters:
- name: user
in: query
description: Required. UUID of workflow owner.
required: true
type: string
- name: workflow
in: body
description: >-
JSON object including workflow parameters and workflow
specification in JSON format (`yadageschemas.load()` output)
with necessary data to instantiate a yadage workflow.
required: true
schema:
type: object
properties:
operational_options:
type: object
description: Operational options.
reana_specification:
type: object
description: >-
Workflow specification in JSON format.
workflow_name:
type: string
description: Workflow name. If empty name will be generated.
git_data:
type: object
description: >-
GitLab data.
required: [reana_specification,
workflow_name,
operational_options]
responses:
201:
description: >-
Request succeeded. The workflow has been created along
with its workspace
schema:
type: object
properties:
message:
type: string
workflow_id:
type: string
workflow_name:
type: string
examples:
application/json:
{
"message": "Workflow workspace has been created.",
"workflow_id": "cdcf48b1-c2f3-4693-8230-b066e088c6ac",
"workflow_name": "mytest-1"
}
400:
description: >-
Request failed. The incoming data specification seems malformed
404:
description: >-
Request failed. User does not exist.
examples:
application/json:
{
"message": "User 00000000-0000-0000-0000-000000000000 does not
exist"
}
"""
try:
user_uuid = request.args["user"]
user = User.query.filter(User.id_ == user_uuid).first()
if not user:
return (
jsonify({
"message":
"User with id:{} does not exist".format(user_uuid)
}),
404,
)
workflow_uuid = str(uuid4())
# Use name prefix user specified or use default name prefix
# Actual name is prefix + autoincremented run_number.
workflow_name = request.json.get("workflow_name", "")
if workflow_name == "":
workflow_name = DEFAULT_NAME_FOR_WORKFLOWS
else:
try:
workflow_name.encode("ascii")
except UnicodeEncodeError:
# `workflow_name` contains something else than just ASCII.
raise REANAWorkflowNameError(
"Workflow name {} is not valid.".format(workflow_name))
git_ref = ""
git_repo = ""
if "git_data" in request.json:
git_data = request.json["git_data"]
git_ref = git_data["git_commit_sha"]
git_repo = git_data["git_url"]
# add spec and params to DB as JSON
workflow = Workflow(
id_=workflow_uuid,
name=workflow_name,
owner_id=request.args["user"],
reana_specification=request.json["reana_specification"],
operational_options=request.json.get("operational_options", {}),
type_=request.json["reana_specification"]["workflow"]["type"],
logs="",
git_ref=git_ref,
git_repo=git_repo,
)
Session.add(workflow)
Session.object_session(workflow).commit()
if git_ref:
create_workflow_workspace(
workflow.workspace_path,
user_id=user.id_,
git_url=git_data["git_url"],
git_branch=git_data["git_branch"],
git_ref=git_ref,
)
else:
create_workflow_workspace(workflow.workspace_path)
return (
jsonify({
"message": "Workflow workspace created",
"workflow_id": workflow.id_,
"workflow_name": get_workflow_name(workflow),
}),
201,
)
except (REANAWorkflowNameError, KeyError) as e:
return jsonify({"message": str(e)}), 400
except Exception as e:
return jsonify({"message": str(e)}), 500
def _update_workflow_status(workflow_uuid, status, logs):
"""Update workflow status in DB."""
Workflow.update_workflow_status(Session, workflow_uuid, status, logs, None)
