def _run_workflow(time_elapsed_seconds=0.5, finish=True): """Mock a workflow run.""" id_ = uuid4() workflow = Workflow( id_=str(id_), name="test_{}".format(id_), owner_id=new_user.id_, reana_specification=[], type_="serial", logs="", status=RunStatus.created, ) # start workflow workflow.status = RunStatus.running session.add(workflow) session.commit() termination_value = datetime.now() + timedelta( seconds=time_elapsed_seconds) class MockDatetime(datetime): @classmethod def now(cls): return termination_value if finish: with mock.patch("reana_db.models.datetime", MockDatetime): Workflow.update_workflow_status(session, workflow.id_, RunStatus.finished) return workflow
def test_delete_all_workflow_runs(app, session, default_user, yadage_workflow_with_name): """Test deletion of all runs of a given workflow.""" # add 5 workflows in the database with the same name for i in range(5): workflow = Workflow( id_=uuid.uuid4(), name=yadage_workflow_with_name["name"], owner_id=default_user.id_, reana_specification=yadage_workflow_with_name[ "reana_specification"], operational_options={}, type_=yadage_workflow_with_name["reana_specification"]["workflow"] ["type"], logs="", ) session.add(workflow) if i == 4: workflow.status = RunStatus.running not_deleted_one = workflow.id_ session.commit() first_workflow = (session.query(Workflow).filter_by( name=yadage_workflow_with_name["name"]).first()) delete_workflow(first_workflow, all_runs=True) for workflow in session.query(Workflow).filter_by( name=first_workflow.name).all(): if not_deleted_one == workflow.id_: assert workflow.status == RunStatus.running else: assert workflow.status == RunStatus.deleted
def publish_workflow_submission(workflow, user_id, parameters): """Publish workflow submission.""" from reana_server.status import NodesStatus Workflow.update_workflow_status(Session, workflow.id_, RunStatus.queued) scheduling_policy = REANA_WORKFLOW_SCHEDULING_POLICY if scheduling_policy not in REANA_WORKFLOW_SCHEDULING_POLICIES: raise ValueError( 'Workflow scheduling policy "{0}" is not valid.'.format( scheduling_policy)) # No need to estimate the complexity for "fifo" strategy if scheduling_policy == "fifo": workflow_priority = 0 workflow_min_job_memory = 0 else: total_cluster_memory = NodesStatus().get_total_memory() complexity = _calculate_complexity(workflow) workflow_priority = workflow.get_priority(total_cluster_memory) workflow_min_job_memory = get_workflow_min_job_memory(complexity) current_workflow_submission_publisher.publish_workflow_submission( user_id=str(user_id), workflow_id_or_name=workflow.get_full_workflow_name(), parameters=parameters, priority=workflow_priority, min_job_memory=workflow_min_job_memory, )
def _update_workflow_status(workflow_uuid, status, logs): """Update workflow status in DB.""" Workflow.update_workflow_status(Session, workflow_uuid, status, logs, None) alive_statuses = \ [WorkflowStatus.created, WorkflowStatus.running, WorkflowStatus.queued] if status not in alive_statuses: _delete_workflow_engine_pod(workflow_uuid)
def _update_workflow_status(workflow_uuid, status, logs): """Update workflow status in DB.""" Workflow.update_workflow_status(Session, workflow_uuid, status, logs, None) workflow = Session.query(Workflow).filter_by(id_=workflow_uuid)\ .one_or_none() if workflow.git_ref: _update_commit_status(workflow, status)
def initialize_krb5_token(workflow_uuid): """Create kerberos ticket from mounted keytab_file.""" cern_user = os.environ.get("CERN_USER") keytab_file = os.environ.get("CERN_KEYTAB") cmd = "kinit -kt /etc/reana/secrets/{} {}@CERN.CH".format( keytab_file, cern_user) if cern_user: try: subprocess.check_output(cmd, shell=True) except subprocess.CalledProcessError as err: msg = "Executing: {} \n Authentication failed: {}".format(cmd, err) Workflow.update_workflow_status( db_session=Session, workflow_uuid=workflow_uuid, status=None, new_logs=msg, ) logging.error(msg, exc_info=True) sys.exit(1) else: msg = "CERN_USER is not set." logging.error(msg, exc_info=True) Workflow.update_workflow_status(db_session=Session, workflow_uuid=workflow_uuid, status=None, new_logs=msg) logging.error(msg, exc_info=True)
def _update_workflow_status(workflow, status, logs): """Update workflow status in DB.""" if workflow.status != status: Workflow.update_workflow_status(Session, workflow.id_, status, logs, None) if workflow.git_ref: _update_commit_status(workflow, status) if status not in ALIVE_STATUSES: workflow.run_finished_at = datetime.now() workflow.logs = workflow.logs or "" try: workflow_engine_logs = _get_workflow_engine_pod_logs(workflow) workflow.logs += workflow_engine_logs + "\n" except REANAWorkflowControllerError as exception: logging.error( f"Could not fetch workflow engine pod logs for workflow {workflow.id_}." f" Error: {exception}") workflow.logs += "Workflow engine logs could not be retrieved.\n" if RunStatus.should_cleanup_job(status): try: _delete_workflow_job(workflow) except REANAWorkflowControllerError as exception: logging.error( f"Could not clean up workflow job for workflow {workflow.id_}." f" Error: {exception}")
def _update_workflow_status(workflow_uuid, status, logs): """Update workflow status in DB.""" Workflow.update_workflow_status(Session, workflow_uuid, status, logs, None) workflow = Session.query(Workflow).filter_by(id_=workflow_uuid)\ .one_or_none() if workflow.git_ref: _update_commit_status(workflow, status) alive_statuses = \ [WorkflowStatus.created, WorkflowStatus.running, WorkflowStatus.queued] if status not in alive_statuses: _delete_workflow_engine_pod(workflow_uuid)
def test_workflow_run_number_assignment(db, session, new_user): """Test workflow run number assignment.""" workflow_name = "workflow" first_workflow = Workflow( id_=str(uuid4()), name=workflow_name, owner_id=new_user.id_, reana_specification=[], type_="serial", logs="", ) session.add(first_workflow) session.commit() assert first_workflow.run_number == 1 second_workflow = Workflow( id_=str(uuid4()), name=workflow_name, owner_id=new_user.id_, reana_specification=[], type_="serial", logs="", ) session.add(second_workflow) session.commit() assert second_workflow.run_number == 2 first_workflow_restart = Workflow( id_=str(uuid4()), name=workflow_name, owner_id=new_user.id_, reana_specification=[], type_="serial", logs="", restart=True, run_number=first_workflow.run_number, ) session.add(first_workflow_restart) session.commit() assert first_workflow_restart.run_number == 1.1 first_workflow_second_restart = Workflow( id_=str(uuid4()), name=workflow_name, owner_id=new_user.id_, reana_specification=[], type_="serial", logs="", restart=True, run_number=first_workflow_restart.run_number, ) session.add(first_workflow_second_restart) session.commit() assert first_workflow_second_restart.run_number == 1.2
def test_workflow_run_number_assignment(db, session): """Test workflow run number assignment.""" workflow_name = 'workflow' owner_id = str(uuid4()) first_workflow = Workflow( id_=str(uuid4()), name=workflow_name, owner_id=owner_id, reana_specification=[], type_='serial', logs='', ) session.add(first_workflow) session.commit() assert first_workflow.run_number == 1 second_workflow = Workflow( id_=str(uuid4()), name=workflow_name, owner_id=owner_id, reana_specification=[], type_='serial', logs='', ) session.add(second_workflow) session.commit() assert second_workflow.run_number == 2 first_workflow_restart = Workflow( id_=str(uuid4()), name=workflow_name, owner_id=owner_id, reana_specification=[], type_='serial', logs='', restart=True, run_number=first_workflow.run_number, ) session.add(first_workflow_restart) session.commit() assert first_workflow_restart.run_number == 1.1 first_workflow_second_restart = Workflow( id_=str(uuid4()), name=workflow_name, owner_id=owner_id, reana_specification=[], type_='serial', logs='', restart=True, run_number=first_workflow_restart.run_number, ) session.add(first_workflow_second_restart) session.commit() assert first_workflow_second_restart.run_number == 1.2
def _update_workflow_status(workflow, status, logs): """Update workflow status in DB.""" if workflow.status != status: Workflow.update_workflow_status(Session, workflow.id_, status, logs, None) if workflow.git_ref: _update_commit_status(workflow, status) alive_statuses = [ RunStatus.created, RunStatus.running, RunStatus.queued, ] if status not in alive_statuses: _delete_workflow_engine_pod(workflow)
def _update_workflow_status(workflow, status, logs): """Update workflow status in DB.""" if workflow.status != status: Workflow.update_workflow_status(Session, workflow.id_, status, logs, None) if workflow.git_ref: _update_commit_status(workflow, status) alive_statuses = [ WorkflowStatus.created, WorkflowStatus.running, WorkflowStatus.queued, ] if status not in alive_statuses: workflow.run_finished_at = datetime.now() _delete_workflow_engine_pod(workflow)
def sample_serial_workflow_in_db( app, default_user, session, serial_workflow, sample_workflow_workspace ): """Create a sample workflow in the database. Scope: function Adds a sample serial workflow in the DB. """ from reana_db.models import Workflow workflow_id = uuid4() relative_workspace_path = build_workspace_path(default_user.id_, workflow_id) next(sample_workflow_workspace(relative_workspace_path)) workflow = Workflow( id_=workflow_id, name="sample_serial_workflow_1", owner_id=default_user.id_, reana_specification=serial_workflow["reana_specification"], operational_options={}, type_=serial_workflow["reana_specification"]["workflow"]["type"], logs="", workspace_path=relative_workspace_path, ) session.add(workflow) session.commit() yield workflow for resource in workflow.resources: session.delete(resource) session.delete(workflow) session.commit()
def test_get_workflows(app, session, default_user, cwl_workflow_with_name): """Test listing all workflows.""" with app.test_client() as client: workflow_uuid = uuid.uuid4() workflow_name = 'my_test_workflow' workflow = Workflow( id_=workflow_uuid, name=workflow_name, status=WorkflowStatus.finished, owner_id=default_user.id_, reana_specification=cwl_workflow_with_name['reana_specification'], type_=cwl_workflow_with_name[ 'reana_specification']['type'], logs='') session.add(workflow) session.commit() res = client.get(url_for('api.get_workflows'), query_string={"user": default_user.id_}) assert res.status_code == 200 response_data = json.loads(res.get_data(as_text=True)) expected_data = [ { "id": str(workflow.id_), "name": workflow.name + '.1', # Add run_number "status": workflow.status.name, "user": str(workflow.owner_id), "created": response_data[0]["created"], "size": "-" } ] assert response_data == expected_data
def test_workflow_can_transition_to(db, session, from_status, to_status, can_transition, new_user): """Test workflow run number assignment.""" workflow_name = "test-workflow" workflow = Workflow( id_=str(uuid4()), name=workflow_name, owner_id=new_user.id_, reana_specification=[], type_="serial", logs="", status=from_status, ) session.add(workflow) session.commit() assert workflow.can_transition_to(to_status) is can_transition
def on_message(self, workflow_submission, message): """On new workflow_submission event handler.""" if reana_ready(): message.ack() workflow_submission = json.loads(workflow_submission) logging.info( 'Starting queued workflow: {}'.format(workflow_submission)) workflow_submission['status'] = 'start' response, http_response = current_rwc_api_client.api.\ set_workflow_status(**workflow_submission).result() http_response_json = http_response.json() if http_response.status_code == 200: workflow_uuid = http_response_json['workflow_id'] status = http_response_json['status'] Workflow.update_workflow_status(Session, workflow_uuid, status) else: message.requeue()
def _update_workflow_status(workflow, status, logs): """Update workflow status in DB.""" if workflow.status != status: Workflow.update_workflow_status(Session, workflow.id_, status, logs, None) if workflow.git_ref: _update_commit_status(workflow, status) if status not in ALIVE_STATUSES: try: workflow.run_finished_at = datetime.now() if RunStatus.should_cleanup_job(status): _delete_workflow_engine_pod(workflow) except REANAWorkflowControllerError: logging.error( f"Could not clean up workflow engine for workflow {workflow.id_}" ) workflow.logs += "Workflow engine logs could not be retrieved.\n"
def test_get_workflow_status_with_name(app, session, default_user, cwl_workflow_with_name): """Test get workflow status.""" with app.test_client() as client: # create workflow workflow_uuid = uuid.uuid4() workflow_name = 'my_test_workflow' workflow = Workflow( id_=workflow_uuid, name=workflow_name, status=WorkflowStatus.finished, owner_id=default_user.id_, reana_specification=cwl_workflow_with_name['reana_specification'], type_=cwl_workflow_with_name[ 'reana_specification']['type'], logs='') session.add(workflow) session.commit() workflow = Workflow.query.filter( Workflow.name == workflow_name).first() res = client.get(url_for('api.get_workflow_status', workflow_id_or_name=workflow_name + '.1'), query_string={"user": default_user.id_}, content_type='application/json', data=json.dumps(cwl_workflow_with_name)) json_response = json.loads(res.data.decode()) assert json_response.get('status') == workflow.status.name workflow.status = WorkflowStatus.finished session.commit() res = client.get(url_for('api.get_workflow_status', workflow_id_or_name=workflow_name + '.1'), query_string={"user": default_user.id_}, content_type='application/json', data=json.dumps(cwl_workflow_with_name)) json_response = json.loads(res.data.decode()) assert json_response.get('status') == workflow.status.name
def test_delete_all_workflow_runs(app, session, default_user, yadage_workflow_with_name, hard_delete): """Test deletion of all runs of a given workflow.""" # add 5 workflows in the database with the same name for i in range(5): workflow = Workflow(id_=uuid.uuid4(), name=yadage_workflow_with_name['name'], owner_id=default_user.id_, reana_specification=yadage_workflow_with_name[ 'reana_specification'], operational_options={}, type_=yadage_workflow_with_name[ 'reana_specification']['workflow']['type'], logs='') session.add(workflow) if i == 4: workflow.status = WorkflowStatus.running not_deleted_one = workflow.id_ session.commit() first_workflow = session.query(Workflow).\ filter_by(name=yadage_workflow_with_name['name']).first() delete_workflow(first_workflow, all_runs=True, hard_delete=hard_delete) if not hard_delete: for workflow in session.query(Workflow).\ filter_by(name=first_workflow.name).all(): if not_deleted_one == workflow.id_: assert workflow.status == WorkflowStatus.running else: assert workflow.status == WorkflowStatus.deleted else: # the one running should not be deleted assert len(session.query(Workflow). filter_by(name=first_workflow.name).all()) == 1
def sample_serial_workflow_in_db(app, default_user, session, serial_workflow): """Create a sample workflow in the database. Scope: function Adds a sample serial workflow in the DB. """ workflow = Workflow( id_=uuid4(), name='sample_serial_workflow_1', owner_id=default_user.id_, reana_specification=serial_workflow['reana_specification'], operational_options={}, type_=serial_workflow['reana_specification']['workflow']['type'], logs='') session.add(workflow) session.commit() yield workflow session.delete(workflow) session.commit()
def test_delete_all_workflow_runs(app, session, default_user, yadage_workflow_with_name, hard_delete): """Test deletion of all runs of a given workflow.""" # add 5 workflows in the database with the same name for i in range(5): workflow = Workflow(id_=uuid.uuid4(), name=yadage_workflow_with_name['name'], owner_id=default_user.id_, reana_specification=yadage_workflow_with_name[ 'reana_specification'], operational_options={}, type_=yadage_workflow_with_name[ 'reana_specification']['workflow']['type'], logs='') session.add(workflow) session.commit() first_workflow = session.query(Workflow).\ filter_by(name=yadage_workflow_with_name['name']).first() with app.test_client() as client: res = client.put( url_for('api.set_workflow_status', workflow_id_or_name=first_workflow.id_), query_string={ 'user': default_user.id_, 'status': 'deleted' }, content_type='application/json', data=json.dumps({'hard_delete': hard_delete, 'all_runs': True})) if not hard_delete: for workflow in session.query(Workflow).\ filter_by(name=first_workflow.name).all(): assert workflow.status == WorkflowStatus.deleted else: assert session.query(Workflow).\ filter_by(name=first_workflow.name).all() == []
def clone_workflow(workflow, reana_spec, restart_type): """Create a copy of workflow in DB for restarting.""" try: cloned_workflow = Workflow( id_=str(uuid4()), name=workflow.name, owner_id=workflow.owner_id, reana_specification=reana_spec or workflow.reana_specification, type_=restart_type or workflow.type_, logs="", workspace_path=workflow.workspace_path, restart=True, run_number=workflow.run_number, ) Session.add(cloned_workflow) Session.object_session(cloned_workflow).commit() return cloned_workflow except SQLAlchemyError as e: message = "Database connection failed, please retry." logging.error( f"Error while creating {cloned_workflow.id_}: {message}\n{e}", exc_info=True)
def sample_yadage_workflow_in_db(app, default_user, session, yadage_workflow_with_name): """Create a sample workflow in the database. Scope: function Adds a sample yadage workflow in the DB. """ workflow = Workflow( id_=uuid4(), name="sample_serial_workflow_1", owner_id=default_user.id_, reana_specification=yadage_workflow_with_name["reana_specification"], operational_options={}, type_=yadage_workflow_with_name["reana_specification"]["workflow"] ["type"], logs="", ) session.add(workflow) session.commit() yield workflow session.delete(workflow) session.commit()
def create_workflow(): # noqa r"""Create workflow and its workspace. --- post: summary: Create workflow and its workspace. description: >- This resource expects all necessary data to represent a workflow so it is stored in database and its workspace is created. operationId: create_workflow produces: - application/json parameters: - name: user in: query description: Required. UUID of workflow owner. required: true type: string - name: workflow in: body description: >- JSON object including workflow parameters and workflow specification in JSON format (`yadageschemas.load()` output) with necessary data to instantiate a yadage workflow. required: true schema: type: object properties: operational_options: type: object description: Operational options. reana_specification: type: object description: >- Workflow specification in JSON format. workflow_name: type: string description: Workflow name. If empty name will be generated. git_data: type: object description: >- GitLab data. required: [reana_specification, workflow_name, operational_options] responses: 201: description: >- Request succeeded. The workflow has been created along with its workspace schema: type: object properties: message: type: string workflow_id: type: string workflow_name: type: string examples: application/json: { "message": "Workflow workspace has been created.", "workflow_id": "cdcf48b1-c2f3-4693-8230-b066e088c6ac", "workflow_name": "mytest-1" } 400: description: >- Request failed. The incoming data specification seems malformed 404: description: >- Request failed. User does not exist. examples: application/json: { "message": "User 00000000-0000-0000-0000-000000000000 does not exist" } """ try: user_uuid = request.args["user"] user = User.query.filter(User.id_ == user_uuid).first() if not user: return ( jsonify({ "message": "User with id:{} does not exist".format(user_uuid) }), 404, ) workflow_uuid = str(uuid4()) # Use name prefix user specified or use default name prefix # Actual name is prefix + autoincremented run_number. workflow_name = request.json.get("workflow_name", "") if workflow_name == "": workflow_name = DEFAULT_NAME_FOR_WORKFLOWS else: try: workflow_name.encode("ascii") except UnicodeEncodeError: # `workflow_name` contains something else than just ASCII. raise REANAWorkflowNameError( "Workflow name {} is not valid.".format(workflow_name)) git_ref = "" git_repo = "" if "git_data" in request.json: git_data = request.json["git_data"] git_ref = git_data["git_commit_sha"] git_repo = git_data["git_url"] # add spec and params to DB as JSON workflow = Workflow( id_=workflow_uuid, name=workflow_name, owner_id=request.args["user"], reana_specification=request.json["reana_specification"], operational_options=request.json.get("operational_options", {}), type_=request.json["reana_specification"]["workflow"]["type"], logs="", git_ref=git_ref, git_repo=git_repo, ) Session.add(workflow) Session.object_session(workflow).commit() if git_ref: create_workflow_workspace( workflow.workspace_path, user_id=user.id_, git_url=git_data["git_url"], git_branch=git_data["git_branch"], git_ref=git_ref, ) else: create_workflow_workspace(workflow.workspace_path) return ( jsonify({ "message": "Workflow workspace created", "workflow_id": workflow.id_, "workflow_name": get_workflow_name(workflow), }), 201, ) except (REANAWorkflowNameError, KeyError) as e: return jsonify({"message": str(e)}), 400 except Exception as e: return jsonify({"message": str(e)}), 500
def _update_workflow_status(workflow_uuid, status, logs): """Update workflow status in DB.""" Workflow.update_workflow_status(Session, workflow_uuid, status, logs, None)