def test_atomic_creation_of_interactive_session(sample_serial_workflow_in_db): """Test atomic creation of interactive sessions. All interactive session should be created as well as writing the state to DB, either all should be done or nothing. """ mocked_k8s_client = Mock() mocked_k8s_client.create_namespaced_deployment = Mock( side_effect=ApiException(reason="Error while creating deployment") ) # Raise 404 when deleting Deployment, because it doesn't exist mocked_k8s_client.delete_namespaced_deployment = Mock( side_effect=ApiException(reason="Not Found") ) with patch.multiple( "reana_workflow_controller.k8s", current_k8s_appsv1_api_client=mocked_k8s_client, current_k8s_networking_api_client=DEFAULT, current_k8s_corev1_api_client=DEFAULT, ) as mocks: try: kwrm = KubernetesWorkflowRunManager(sample_serial_workflow_in_db) if len(InteractiveSessionType): kwrm.start_interactive_session(InteractiveSessionType(0).name) except REANAInteractiveSessionError: mocks[ "current_k8s_corev1_api_client" ].delete_namespaced_service.assert_called_once() mocks[ "current_k8s_networking_api_client" ].delete_namespaced_ingress.assert_called_once() mocked_k8s_client.delete_namespaced_deployment.assert_called_once() assert not sample_serial_workflow_in_db.sessions.all()
def test_atomic_creation_of_interactive_session(sample_serial_workflow_in_db): """Test the correct creation of all objects related to an interactive sesison as well as writing the state to DB, either all should be done or nothing..""" mocked_k8s_client = Mock() mocked_k8s_client.create_namespaced_deployment =\ Mock(side_effect=ApiException( reason='Error while creating deployment')) # Raise 404 when deleting Deployment, because it doesn't exist mocked_k8s_client.delete_namespaced_deployment =\ Mock(side_effect=ApiException( reason='Not Found')) with patch.multiple('reana_workflow_controller.k8s', current_k8s_extensions_v1beta1=mocked_k8s_client, current_k8s_corev1_api_client=DEFAULT) as mocks: try: kwrm = KubernetesWorkflowRunManager(sample_serial_workflow_in_db) if len(INTERACTIVE_SESSION_TYPES): kwrm.start_interactive_session(INTERACTIVE_SESSION_TYPES[0]) except REANAInteractiveSessionError: mocks['current_k8s_corev1_api_client']\ .delete_namespaced_service.assert_called_once() mocked_k8s_client.delete_namespaced_ingress.assert_called_once() mocked_k8s_client.delete_namespaced_deployment.assert_called_once() assert sample_serial_workflow_in_db.interactive_session is None
def test_create_job_spec_kerberos( sample_serial_workflow_in_db, kerberos_user_secrets, corev1_api_client_with_user_secrets, ): """Test creation of k8s job specification when Kerberos is required.""" workflow = sample_serial_workflow_in_db workflow.reana_specification["workflow"].setdefault("resources", {})[ "kerberos" ] = True with patch( "reana_commons.k8s.secrets.current_k8s_corev1_api_client", corev1_api_client_with_user_secrets(kerberos_user_secrets), ): kwrm = KubernetesWorkflowRunManager(workflow) job = kwrm._create_job_spec("run-batch-test") init_containers = job.spec.template.spec.init_containers assert len(init_containers) == 1 assert init_containers[0]["name"] == KRB5_CONTAINER_NAME volumes = [volume["name"] for volume in job.spec.template.spec.volumes] assert len(set(volumes)) == len(volumes) # volumes have unique names assert any(volume.startswith("reana-secretsstore") for volume in volumes) assert "krb5-cache" in volumes assert "krb5-conf" in volumes
def start_workflow(workflow, parameters): """Start a workflow.""" def _start_workflow_db(workflow, parameters): workflow.run_started_at = datetime.now() workflow.status = WorkflowStatus.running if parameters: workflow.input_parameters = parameters.get("input_parameters") workflow.operational_options = parameters.get("operational_options") current_db_sessions.add(workflow) current_db_sessions.commit() current_db_sessions = Session.object_session(workflow) kwrm = KubernetesWorkflowRunManager(workflow) failure_message = ( "Workflow {id_} could not be started because it {verb} " "already {status}." ).format( id_=workflow.id_, verb=get_workflow_status_change_verb(workflow.status.name), status=str(workflow.status.name), ) if "restart" in parameters.keys(): if parameters["restart"]: if workflow.status not in [ WorkflowStatus.failed, WorkflowStatus.finished, WorkflowStatus.queued, ]: raise REANAWorkflowControllerError(failure_message) elif workflow.status not in [WorkflowStatus.created, WorkflowStatus.queued]: if workflow.status == WorkflowStatus.deleted: raise REANAWorkflowStatusError(failure_message) raise REANAWorkflowControllerError(failure_message) try: kwrm.start_batch_workflow_run( overwrite_input_params=parameters.get("input_parameters"), overwrite_operational_options=parameters.get("operational_options"), ) _start_workflow_db(workflow, parameters) except SQLAlchemyError as e: message = "Database connection failed, please retry." logging.error( f"Error while creating {workflow.id_}: {message}\n{e}", exc_info=True ) # Rollback Kubernetes job creation kwrm.stop_batch_workflow_run() logging.error( f"Stopping Kubernetes jobs associated with workflow " f"{workflow.id_} ..." ) raise REANAExternalCallError(message) except ApiException as e: message = "Kubernetes connection failed, please retry." logging.error( f"Error while creating {workflow.id_}: {message}\n{e}", exc_info=True ) raise REANAExternalCallError(message)
def stop_workflow(workflow): """Stop a given workflow.""" if workflow.status == RunStatus.running: kwrm = KubernetesWorkflowRunManager(workflow) kwrm.stop_batch_workflow_run() workflow.status = RunStatus.stopped Session.add(workflow) Session.commit() else: message = ("Workflow {id_} is not running.").format(id_=workflow.id_) raise REANAWorkflowControllerError(message)
def test_start_interactive_workflow_k8s_failure(sample_serial_workflow_in_db): """Test failure of an interactive workflow run deployment because of .""" mocked_k8s_client = Mock() mocked_k8s_client.create_namespaced_deployment =\ Mock(side_effect=ApiException(reason='some reason')) with patch.multiple('reana_workflow_controller.k8s', current_k8s_extensions_v1beta1=mocked_k8s_client, current_k8s_corev1_api_client=DEFAULT): with pytest.raises(REANAInteractiveSessionError, match=r'.*Kubernetes has failed.*'): kwrm = KubernetesWorkflowRunManager(sample_serial_workflow_in_db) if len(INTERACTIVE_SESSION_TYPES): kwrm.start_interactive_session(INTERACTIVE_SESSION_TYPES[0])
def stop_workflow(workflow): """Stop a given workflow.""" if workflow.status == WorkflowStatus.running: kwrm = KubernetesWorkflowRunManager(workflow) workflow.run_stopped_at = datetime.now() kwrm.stop_batch_workflow_run() workflow.status = WorkflowStatus.stopped current_db_sessions = Session.object_session(workflow) current_db_sessions.add(workflow) current_db_sessions.commit() else: message = ("Workflow {id_} is not running.").format(id_=workflow.id_) raise REANAWorkflowControllerError(message)
def test_start_interactive_session(sample_serial_workflow_in_db): """Test interactive workflow run deployment.""" with patch.multiple("reana_workflow_controller.k8s", current_k8s_corev1_api_client=DEFAULT, current_k8s_extensions_v1beta1=DEFAULT) as mocks: kwrm = KubernetesWorkflowRunManager(sample_serial_workflow_in_db) if len(INTERACTIVE_SESSION_TYPES): kwrm.start_interactive_session(INTERACTIVE_SESSION_TYPES[0]) mocks['current_k8s_extensions_v1beta1'].\ create_namespaced_deployment.assert_called_once() mocks['current_k8s_corev1_api_client'].\ create_namespaced_service.assert_called_once() mocks['current_k8s_extensions_v1beta1'].\ create_namespaced_ingress.assert_called_once()
def test_stop_workflow_backend_only_kubernetes( sample_serial_workflow_in_db, add_kubernetes_jobs_to_workflow): """Test deletion of workflows with only Kubernetes based jobs.""" workflow = sample_serial_workflow_in_db workflow.status = WorkflowStatus.running workflow_jobs = add_kubernetes_jobs_to_workflow(workflow) backend_job_ids = [job.backend_job_id for job in workflow_jobs] with patch("reana_workflow_controller.workflow_run_manager." "current_k8s_batchv1_api_client") as api_client: kwrm = KubernetesWorkflowRunManager(workflow) kwrm.stop_batch_workflow_run() for delete_call in api_client.delete_namespaced_job.call_args_list: if delete_call.args[0] in backend_job_ids: del backend_job_ids[backend_job_ids.index(delete_call.args[0])] assert not backend_job_ids
def test_start_interactive_workflow_k8s_failure(sample_serial_workflow_in_db): """Test failure of an interactive workflow run deployment because of .""" mocked_k8s_client = Mock() mocked_k8s_client.create_namespaced_deployment = Mock( side_effect=ApiException(reason="some reason")) with patch.multiple( "reana_workflow_controller.k8s", current_k8s_appsv1_api_client=mocked_k8s_client, current_k8s_corev1_api_client=DEFAULT, current_k8s_networking_v1beta1=DEFAULT, ): with pytest.raises(REANAInteractiveSessionError, match=r".*Kubernetes has failed.*"): kwrm = KubernetesWorkflowRunManager(sample_serial_workflow_in_db) if len(InteractiveSessionType): kwrm.start_interactive_session(InteractiveSessionType(0).name)
def test_interactive_session_closure(sample_serial_workflow_in_db): """Test closure of an interactive sessions.""" mocked_k8s_client = Mock() workflow = sample_serial_workflow_in_db with patch.multiple('reana_workflow_controller.k8s', current_k8s_appsv1_api_client=mocked_k8s_client, current_k8s_networking_v1beta1=DEFAULT, current_k8s_corev1_api_client=DEFAULT) as mocks: kwrm = KubernetesWorkflowRunManager(workflow) if len(INTERACTIVE_SESSION_TYPES): kwrm.start_interactive_session(INTERACTIVE_SESSION_TYPES[0]) assert workflow.interactive_session_name assert workflow.interactive_session assert workflow.interactive_session_type kwrm.stop_interactive_session() assert workflow.interactive_session_name is None assert workflow.interactive_session is None assert workflow.interactive_session_type is None
def test_start_interactive_session(sample_serial_workflow_in_db): """Test interactive workflow run deployment.""" with patch.multiple( "reana_workflow_controller.k8s", current_k8s_corev1_api_client=DEFAULT, current_k8s_networking_api_client=DEFAULT, current_k8s_appsv1_api_client=DEFAULT, ) as mocks: kwrm = KubernetesWorkflowRunManager(sample_serial_workflow_in_db) if len(InteractiveSessionType): kwrm.start_interactive_session(InteractiveSessionType(0).name) mocks[ "current_k8s_appsv1_api_client" ].create_namespaced_deployment.assert_called_once() mocks[ "current_k8s_corev1_api_client" ].create_namespaced_service.assert_called_once() mocks[ "current_k8s_networking_api_client" ].create_namespaced_ingress.assert_called_once()
def test_interactive_session_closure(sample_serial_workflow_in_db, session): """Test closure of an interactive sessions.""" mocked_k8s_client = Mock() workflow = sample_serial_workflow_in_db with patch.multiple( "reana_workflow_controller.k8s", current_k8s_appsv1_api_client=mocked_k8s_client, current_k8s_networking_api_client=DEFAULT, current_k8s_corev1_api_client=DEFAULT, ): kwrm = KubernetesWorkflowRunManager(workflow) if len(InteractiveSessionType): kwrm.start_interactive_session(InteractiveSessionType(0).name) int_session = InteractiveSession.query.filter_by( owner_id=workflow.owner_id, type_=InteractiveSessionType(0).name, ).first() assert int_session.status == RunStatus.created kwrm.stop_interactive_session(int_session.id_) assert not workflow.sessions.first()
def open_interactive_session(workflow_id_or_name, interactive_session_type): # noqa r"""Start an interactive session inside the workflow workspace. --- post: summary: Start an interactive session inside the workflow workspace. description: >- This resource is expecting a workflow to start an interactive session within its workspace. operationId: open_interactive_session consumes: - application/json produces: - application/json parameters: - name: user in: query description: Required. UUID of workflow owner. required: true type: string - name: workflow_id_or_name in: path description: Required. Workflow UUID or name. required: true type: string - name: interactive_session_type in: path description: >- Optional. Type of interactive session to use, by default Jupyter Notebook. required: false type: string - name: interactive_session_configuration in: body description: >- Interactive session configuration. required: false schema: type: object properties: image: type: string description: >- Replaces the default Docker image of an interactive session. responses: 200: description: >- Request succeeded. The interactive session has been opened. schema: type: object properties: path: type: string examples: application/json: { "path": "/dd4e93cf-e6d0-4714-a601-301ed97eec60", } 400: description: >- Request failed. The incoming data specification seems malformed. examples: application/json: { "message": "Malformed request." } 404: description: >- Request failed. Either User or Workflow does not exist. examples: application/json: { "message": "Interactive session type terminl not found, try with one of: [jupyter]" } application/json: { "message": "Workflow 256b25f4-4cfb-4684-b7a8-73872ef455a1 does not exist" } 500: description: >- Request failed. Internal controller error. """ try: if interactive_session_type not in INTERACTIVE_SESSION_TYPES: return jsonify({ "message": "Interactive session type {0} not found, try " "with one of: {1}".format(interactive_session_type, INTERACTIVE_SESSION_TYPES) }), 404 interactive_session_configuration = request.json or {} user_uuid = request.args["user"] workflow = None workflow = _get_workflow_with_uuid_or_name(workflow_id_or_name, user_uuid) kwrm = KubernetesWorkflowRunManager(workflow) access_path = kwrm.start_interactive_session( interactive_session_type, image=interactive_session_configuration.get("image", None)) return jsonify({"path": "{}".format(access_path)}), 200 except (KeyError, ValueError) as e: status_code = 400 if workflow else 404 return jsonify({"message": str(e)}), status_code except Exception as e: return jsonify({"message": str(e)}), 500
def close_interactive_session(workflow_id_or_name): # noqa r"""Close an interactive workflow session. --- post: summary: Close an interactive workflow session. description: >- This resource is expecting a workflow to close an interactive session within its workspace. operationId: close_interactive_session consumes: - application/json produces: - application/json parameters: - name: user in: query description: Required. UUID of workflow owner. required: true type: string - name: workflow_id_or_name in: path description: Required. Workflow UUID or name. required: true type: string responses: 200: description: >- Request succeeded. The interactive session has been closed. schema: type: object properties: message: type: string examples: application/json: { "message": "The interactive session has been closed", } 400: description: >- Request failed. The incoming data specification seems malformed. examples: application/json: { "message": "Malformed request." } 404: description: >- Request failed. Either User or Workflow does not exist. examples: application/json: { "message": "Workflow 256b25f4-4cfb-4684-b7a8-73872ef455a1 does not exist" } 500: description: >- Request failed. Internal controller error. """ try: user_uuid = request.args["user"] workflow = None workflow = _get_workflow_with_uuid_or_name(workflow_id_or_name, user_uuid) if workflow.interactive_session_name is None: return jsonify({ "message": "Workflow - {} has no open interactive session.".format( workflow_id_or_name) }), 404 kwrm = KubernetesWorkflowRunManager(workflow) kwrm.stop_interactive_session() return jsonify({"message": "The interactive session has been closed"}), 200 except (KeyError, ValueError) as e: status_code = 400 if workflow else 404 return jsonify({"message": str(e)}), status_code except Exception as e: return jsonify({"message": str(e)}), 500