def test_atomic_creation_of_interactive_session(sample_serial_workflow_in_db):
"""Test atomic creation of interactive sessions.
All interactive session should be created as well as writing the state
to DB, either all should be done or nothing.
"""
mocked_k8s_client = Mock()
mocked_k8s_client.create_namespaced_deployment = Mock(
side_effect=ApiException(reason="Error while creating deployment")
)
# Raise 404 when deleting Deployment, because it doesn't exist
mocked_k8s_client.delete_namespaced_deployment = Mock(
side_effect=ApiException(reason="Not Found")
)
with patch.multiple(
"reana_workflow_controller.k8s",
current_k8s_appsv1_api_client=mocked_k8s_client,
current_k8s_networking_api_client=DEFAULT,
current_k8s_corev1_api_client=DEFAULT,
) as mocks:
try:
kwrm = KubernetesWorkflowRunManager(sample_serial_workflow_in_db)
if len(InteractiveSessionType):
kwrm.start_interactive_session(InteractiveSessionType(0).name)
except REANAInteractiveSessionError:
mocks[
"current_k8s_corev1_api_client"
].delete_namespaced_service.assert_called_once()
mocks[
"current_k8s_networking_api_client"
].delete_namespaced_ingress.assert_called_once()
mocked_k8s_client.delete_namespaced_deployment.assert_called_once()
assert not sample_serial_workflow_in_db.sessions.all()
def test_atomic_creation_of_interactive_session(sample_serial_workflow_in_db):
"""Test the correct creation of all objects related to an interactive
sesison as well as writing the state to DB, either all should be done
or nothing.."""
mocked_k8s_client = Mock()
mocked_k8s_client.create_namespaced_deployment =\
Mock(side_effect=ApiException(
reason='Error while creating deployment'))
# Raise 404 when deleting Deployment, because it doesn't exist
mocked_k8s_client.delete_namespaced_deployment =\
Mock(side_effect=ApiException(
reason='Not Found'))
with patch.multiple('reana_workflow_controller.k8s',
current_k8s_extensions_v1beta1=mocked_k8s_client,
current_k8s_corev1_api_client=DEFAULT) as mocks:
try:
kwrm = KubernetesWorkflowRunManager(sample_serial_workflow_in_db)
if len(INTERACTIVE_SESSION_TYPES):
kwrm.start_interactive_session(INTERACTIVE_SESSION_TYPES[0])
except REANAInteractiveSessionError:
mocks['current_k8s_corev1_api_client']\
.delete_namespaced_service.assert_called_once()
mocked_k8s_client.delete_namespaced_ingress.assert_called_once()
mocked_k8s_client.delete_namespaced_deployment.assert_called_once()
assert sample_serial_workflow_in_db.interactive_session is None
def test_create_job_spec_kerberos(
sample_serial_workflow_in_db,
kerberos_user_secrets,
corev1_api_client_with_user_secrets,
):
"""Test creation of k8s job specification when Kerberos is required."""
workflow = sample_serial_workflow_in_db
workflow.reana_specification["workflow"].setdefault("resources", {})[
"kerberos"
] = True
with patch(
"reana_commons.k8s.secrets.current_k8s_corev1_api_client",
corev1_api_client_with_user_secrets(kerberos_user_secrets),
):
kwrm = KubernetesWorkflowRunManager(workflow)
job = kwrm._create_job_spec("run-batch-test")
init_containers = job.spec.template.spec.init_containers
assert len(init_containers) == 1
assert init_containers[0]["name"] == KRB5_CONTAINER_NAME
volumes = [volume["name"] for volume in job.spec.template.spec.volumes]
assert len(set(volumes)) == len(volumes) # volumes have unique names
assert any(volume.startswith("reana-secretsstore") for volume in volumes)
assert "krb5-cache" in volumes
assert "krb5-conf" in volumes
def stop_workflow(workflow):
"""Stop a given workflow."""
if workflow.status == RunStatus.running:
kwrm = KubernetesWorkflowRunManager(workflow)
kwrm.stop_batch_workflow_run()
workflow.status = RunStatus.stopped
Session.add(workflow)
Session.commit()
else:
message = ("Workflow {id_} is not running.").format(id_=workflow.id_)
raise REANAWorkflowControllerError(message)
def test_start_interactive_workflow_k8s_failure(sample_serial_workflow_in_db):
"""Test failure of an interactive workflow run deployment because of ."""
mocked_k8s_client = Mock()
mocked_k8s_client.create_namespaced_deployment =\
Mock(side_effect=ApiException(reason='some reason'))
with patch.multiple('reana_workflow_controller.k8s',
current_k8s_extensions_v1beta1=mocked_k8s_client,
current_k8s_corev1_api_client=DEFAULT):
with pytest.raises(REANAInteractiveSessionError,
match=r'.*Kubernetes has failed.*'):
kwrm = KubernetesWorkflowRunManager(sample_serial_workflow_in_db)
if len(INTERACTIVE_SESSION_TYPES):
kwrm.start_interactive_session(INTERACTIVE_SESSION_TYPES[0])
def stop_workflow(workflow):
"""Stop a given workflow."""
if workflow.status == WorkflowStatus.running:
kwrm = KubernetesWorkflowRunManager(workflow)
workflow.run_stopped_at = datetime.now()
kwrm.stop_batch_workflow_run()
workflow.status = WorkflowStatus.stopped
current_db_sessions = Session.object_session(workflow)
current_db_sessions.add(workflow)
current_db_sessions.commit()
else:
message = ("Workflow {id_} is not running.").format(id_=workflow.id_)
raise REANAWorkflowControllerError(message)
def test_start_interactive_session(sample_serial_workflow_in_db):
"""Test interactive workflow run deployment."""
with patch.multiple("reana_workflow_controller.k8s",
current_k8s_corev1_api_client=DEFAULT,
current_k8s_extensions_v1beta1=DEFAULT) as mocks:
kwrm = KubernetesWorkflowRunManager(sample_serial_workflow_in_db)
if len(INTERACTIVE_SESSION_TYPES):
kwrm.start_interactive_session(INTERACTIVE_SESSION_TYPES[0])
mocks['current_k8s_extensions_v1beta1'].\
create_namespaced_deployment.assert_called_once()
mocks['current_k8s_corev1_api_client'].\
create_namespaced_service.assert_called_once()
mocks['current_k8s_extensions_v1beta1'].\
create_namespaced_ingress.assert_called_once()
def test_stop_workflow_backend_only_kubernetes(
sample_serial_workflow_in_db, add_kubernetes_jobs_to_workflow):
"""Test deletion of workflows with only Kubernetes based jobs."""
workflow = sample_serial_workflow_in_db
workflow.status = WorkflowStatus.running
workflow_jobs = add_kubernetes_jobs_to_workflow(workflow)
backend_job_ids = [job.backend_job_id for job in workflow_jobs]
with patch("reana_workflow_controller.workflow_run_manager."
"current_k8s_batchv1_api_client") as api_client:
kwrm = KubernetesWorkflowRunManager(workflow)
kwrm.stop_batch_workflow_run()
for delete_call in api_client.delete_namespaced_job.call_args_list:
if delete_call.args[0] in backend_job_ids:
del backend_job_ids[backend_job_ids.index(delete_call.args[0])]
assert not backend_job_ids
def test_start_interactive_workflow_k8s_failure(sample_serial_workflow_in_db):
"""Test failure of an interactive workflow run deployment because of ."""
mocked_k8s_client = Mock()
mocked_k8s_client.create_namespaced_deployment = Mock(
side_effect=ApiException(reason="some reason"))
with patch.multiple(
"reana_workflow_controller.k8s",
current_k8s_appsv1_api_client=mocked_k8s_client,
current_k8s_corev1_api_client=DEFAULT,
current_k8s_networking_v1beta1=DEFAULT,
):
with pytest.raises(REANAInteractiveSessionError,
match=r".*Kubernetes has failed.*"):
kwrm = KubernetesWorkflowRunManager(sample_serial_workflow_in_db)
if len(InteractiveSessionType):
kwrm.start_interactive_session(InteractiveSessionType(0).name)
def test_start_interactive_session(sample_serial_workflow_in_db):
"""Test interactive workflow run deployment."""
with patch.multiple(
"reana_workflow_controller.k8s",
current_k8s_corev1_api_client=DEFAULT,
current_k8s_networking_api_client=DEFAULT,
current_k8s_appsv1_api_client=DEFAULT,
) as mocks:
kwrm = KubernetesWorkflowRunManager(sample_serial_workflow_in_db)
if len(InteractiveSessionType):
kwrm.start_interactive_session(InteractiveSessionType(0).name)
mocks[
"current_k8s_appsv1_api_client"
].create_namespaced_deployment.assert_called_once()
mocks[
"current_k8s_corev1_api_client"
].create_namespaced_service.assert_called_once()
mocks[
"current_k8s_networking_api_client"
].create_namespaced_ingress.assert_called_once()
def start_workflow(workflow, parameters):
"""Start a workflow."""
def _start_workflow_db(workflow, parameters):
workflow.run_started_at = datetime.now()
workflow.status = WorkflowStatus.running
if parameters:
workflow.input_parameters = parameters.get("input_parameters")
workflow.operational_options = parameters.get("operational_options")
current_db_sessions.add(workflow)
current_db_sessions.commit()
current_db_sessions = Session.object_session(workflow)
kwrm = KubernetesWorkflowRunManager(workflow)
failure_message = (
"Workflow {id_} could not be started because it {verb} " "already {status}."
).format(
id_=workflow.id_,
verb=get_workflow_status_change_verb(workflow.status.name),
status=str(workflow.status.name),
)
if "restart" in parameters.keys():
if parameters["restart"]:
if workflow.status not in [
WorkflowStatus.failed,
WorkflowStatus.finished,
WorkflowStatus.queued,
]:
raise REANAWorkflowControllerError(failure_message)
elif workflow.status not in [WorkflowStatus.created, WorkflowStatus.queued]:
if workflow.status == WorkflowStatus.deleted:
raise REANAWorkflowStatusError(failure_message)
raise REANAWorkflowControllerError(failure_message)
try:
kwrm.start_batch_workflow_run(
overwrite_input_params=parameters.get("input_parameters"),
overwrite_operational_options=parameters.get("operational_options"),
)
_start_workflow_db(workflow, parameters)
except SQLAlchemyError as e:
message = "Database connection failed, please retry."
logging.error(
f"Error while creating {workflow.id_}: {message}\n{e}", exc_info=True
)
# Rollback Kubernetes job creation
kwrm.stop_batch_workflow_run()
logging.error(
f"Stopping Kubernetes jobs associated with workflow " f"{workflow.id_} ..."
)
raise REANAExternalCallError(message)
except ApiException as e:
message = "Kubernetes connection failed, please retry."
logging.error(
f"Error while creating {workflow.id_}: {message}\n{e}", exc_info=True
)
raise REANAExternalCallError(message)
def test_interactive_session_closure(sample_serial_workflow_in_db):
"""Test closure of an interactive sessions."""
mocked_k8s_client = Mock()
workflow = sample_serial_workflow_in_db
with patch.multiple('reana_workflow_controller.k8s',
current_k8s_appsv1_api_client=mocked_k8s_client,
current_k8s_networking_v1beta1=DEFAULT,
current_k8s_corev1_api_client=DEFAULT) as mocks:
kwrm = KubernetesWorkflowRunManager(workflow)
if len(INTERACTIVE_SESSION_TYPES):
kwrm.start_interactive_session(INTERACTIVE_SESSION_TYPES[0])
assert workflow.interactive_session_name
assert workflow.interactive_session
assert workflow.interactive_session_type
kwrm.stop_interactive_session()
assert workflow.interactive_session_name is None
assert workflow.interactive_session is None
assert workflow.interactive_session_type is None
def test_interactive_session_closure(sample_serial_workflow_in_db, session):
"""Test closure of an interactive sessions."""
mocked_k8s_client = Mock()
workflow = sample_serial_workflow_in_db
with patch.multiple(
"reana_workflow_controller.k8s",
current_k8s_appsv1_api_client=mocked_k8s_client,
current_k8s_networking_api_client=DEFAULT,
current_k8s_corev1_api_client=DEFAULT,
):
kwrm = KubernetesWorkflowRunManager(workflow)
if len(InteractiveSessionType):
kwrm.start_interactive_session(InteractiveSessionType(0).name)
int_session = InteractiveSession.query.filter_by(
owner_id=workflow.owner_id,
type_=InteractiveSessionType(0).name,
).first()
assert int_session.status == RunStatus.created
kwrm.stop_interactive_session(int_session.id_)
assert not workflow.sessions.first()
def open_interactive_session(workflow_id_or_name,
interactive_session_type): # noqa
r"""Start an interactive session inside the workflow workspace.
---
post:
summary: Start an interactive session inside the workflow workspace.
description: >-
This resource is expecting a workflow to start an interactive session
within its workspace.
operationId: open_interactive_session
consumes:
- application/json
produces:
- application/json
parameters:
- name: user
in: query
description: Required. UUID of workflow owner.
required: true
type: string
- name: workflow_id_or_name
in: path
description: Required. Workflow UUID or name.
required: true
type: string
- name: interactive_session_type
in: path
description: >-
Optional. Type of interactive session to use, by default Jupyter
Notebook.
required: false
type: string
- name: interactive_session_configuration
in: body
description: >-
Interactive session configuration.
required: false
schema:
type: object
properties:
image:
type: string
description: >-
Replaces the default Docker image of an interactive session.
responses:
200:
description: >-
Request succeeded. The interactive session has been opened.
schema:
type: object
properties:
path:
type: string
examples:
application/json:
{
"path": "/dd4e93cf-e6d0-4714-a601-301ed97eec60",
}
400:
description: >-
Request failed. The incoming data specification seems malformed.
examples:
application/json:
{
"message": "Malformed request."
}
404:
description: >-
Request failed. Either User or Workflow does not exist.
examples:
application/json:
{
"message": "Interactive session type terminl not found, try
with one of: [jupyter]"
}
application/json:
{
"message": "Workflow 256b25f4-4cfb-4684-b7a8-73872ef455a1
does not exist"
}
500:
description: >-
Request failed. Internal controller error.
"""
try:
if interactive_session_type not in INTERACTIVE_SESSION_TYPES:
return jsonify({
"message":
"Interactive session type {0} not found, try "
"with one of: {1}".format(interactive_session_type,
INTERACTIVE_SESSION_TYPES)
}), 404
interactive_session_configuration = request.json or {}
user_uuid = request.args["user"]
workflow = None
workflow = _get_workflow_with_uuid_or_name(workflow_id_or_name,
user_uuid)
kwrm = KubernetesWorkflowRunManager(workflow)
access_path = kwrm.start_interactive_session(
interactive_session_type,
image=interactive_session_configuration.get("image", None))
return jsonify({"path": "{}".format(access_path)}), 200
except (KeyError, ValueError) as e:
status_code = 400 if workflow else 404
return jsonify({"message": str(e)}), status_code
except Exception as e:
return jsonify({"message": str(e)}), 500
def close_interactive_session(workflow_id_or_name): # noqa
r"""Close an interactive workflow session.
---
post:
summary: Close an interactive workflow session.
description: >-
This resource is expecting a workflow to close an interactive session
within its workspace.
operationId: close_interactive_session
consumes:
- application/json
produces:
- application/json
parameters:
- name: user
in: query
description: Required. UUID of workflow owner.
required: true
type: string
- name: workflow_id_or_name
in: path
description: Required. Workflow UUID or name.
required: true
type: string
responses:
200:
description: >-
Request succeeded. The interactive session has been closed.
schema:
type: object
properties:
message:
type: string
examples:
application/json:
{
"message": "The interactive session has been closed",
}
400:
description: >-
Request failed. The incoming data specification seems malformed.
examples:
application/json:
{
"message": "Malformed request."
}
404:
description: >-
Request failed. Either User or Workflow does not exist.
examples:
application/json:
{
"message": "Workflow 256b25f4-4cfb-4684-b7a8-73872ef455a1
does not exist"
}
500:
description: >-
Request failed. Internal controller error.
"""
try:
user_uuid = request.args["user"]
workflow = None
workflow = _get_workflow_with_uuid_or_name(workflow_id_or_name,
user_uuid)
if workflow.interactive_session_name is None:
return jsonify({
"message":
"Workflow - {} has no open interactive session.".format(
workflow_id_or_name)
}), 404
kwrm = KubernetesWorkflowRunManager(workflow)
kwrm.stop_interactive_session()
return jsonify({"message":
"The interactive session has been closed"}), 200
except (KeyError, ValueError) as e:
status_code = 400 if workflow else 404
return jsonify({"message": str(e)}), status_code
except Exception as e:
return jsonify({"message": str(e)}), 500