async def test_websocket_disconnected_after_logout(
client: TestClient,
logged_user: Dict[str, Any],
socketio_client_factory: Callable,
client_session_id_factory: Callable[[], str],
expected,
mocker: MockerFixture,
):
app = client.server.app
socket_registry = get_registry(app)
# connect first socket
cur_client_session_id1 = client_session_id_factory()
sio = await socketio_client_factory(cur_client_session_id1)
socket_logout_mock_callable = mocker.Mock()
sio.on("logout", handler=socket_logout_mock_callable)
# connect second socket
cur_client_session_id2 = client_session_id_factory()
sio2 = await socketio_client_factory(cur_client_session_id2)
socket_logout_mock_callable2 = mocker.Mock()
sio2.on("logout", handler=socket_logout_mock_callable2)
# connect third socket
cur_client_session_id3 = client_session_id_factory()
sio3 = await socketio_client_factory(cur_client_session_id3)
socket_logout_mock_callable3 = mocker.Mock()
sio3.on("logout", handler=socket_logout_mock_callable3)
# logout client with socket 2
logout_url = client.app.router["auth_logout"].url_for()
r = await client.post(f"{logout_url}",
json={"client_session_id": cur_client_session_id2})
assert r.url_obj.path == logout_url.path
await assert_status(r, expected)
# the socket2 should be gone
await asyncio.sleep(1)
assert not sio2.sid
socket_logout_mock_callable2.assert_not_called()
# the others should receive a logout message through their respective sockets
await asyncio.sleep(3)
socket_logout_mock_callable.assert_called_once()
socket_logout_mock_callable2.assert_not_called(
) # note 2 should be not called ever
socket_logout_mock_callable3.assert_called_once()
await asyncio.sleep(3)
# first socket should be closed now
assert not sio.sid
# second socket also closed
assert not sio3.sid
async def test_disconnected_backend_send_computation_task(
dask_spec_local_cluster: SpecCluster,
dask_client: DaskClient,
user_id: UserID,
project_id: ProjectID,
node_id: NodeID,
cluster_id: ClusterID,
cluster_id_resource: str,
image: Image,
expected_annotations: Dict[str, Any],
mocker: MockerFixture,
):
# INIT
expected_annotations["resources"].update(
{cluster_id_resource: CLUSTER_RESOURCE_MOCK_USAGE})
fake_task = {node_id: image}
mocked_done_callback_fct = mocker.Mock()
# DISCONNECT THE CLUSTER
await dask_spec_local_cluster.close()
#
with pytest.raises(ComputationalBackendNotConnectedError):
await dask_client.send_computation_tasks(
user_id=user_id,
project_id=project_id,
cluster_id=cluster_id,
tasks=fake_task,
callback=mocked_done_callback_fct,
remote_fct=None,
)
assert (len(dask_client._taskid_to_future_map) == 0
), "dask client should not store any future here"
async def test_too_many_resource_send_computation_task(
dask_client: DaskClient,
user_id: UserID,
project_id: ProjectID,
node_id: NodeID,
cluster_id: ClusterID,
cluster_id_resource: str,
image: Image,
expected_annotations: Dict[str, Any],
mocker: MockerFixture,
):
# INIT
expected_annotations["resources"].update(
{cluster_id_resource: CLUSTER_RESOURCE_MOCK_USAGE})
fake_task = {node_id: image}
mocked_done_callback_fct = mocker.Mock()
# let's have a big number of CPUs
with pytest.raises(InsuficientComputationalResourcesError):
await dask_client.send_computation_tasks(
user_id=user_id,
project_id=project_id,
cluster_id=cluster_id,
tasks=fake_task,
callback=mocked_done_callback_fct,
remote_fct=None,
)
assert (len(dask_client._taskid_to_future_map) == 0
), "dask client should not store any future here"
def test_unknown_completion_mode_returns_default(
self, mocker: MockerFixture) -> None:
mock_config = mocker.Mock()
mock_config.completion.mode = "unknown"
config = GalaxyToolsConfiguration(mock_config)
assert config.completion_mode == CompletionMode.AUTO
def test_init_sets_properties(self, mocker: MockerFixture) -> None:
mock_config = mocker.Mock()
mock_config.completion.mode = "invoke"
mock_config.completion.autoCloseTags = "false"
config = GalaxyToolsConfiguration(mock_config)
assert config.completion_mode
assert not config.auto_close_tags
async def test_send_computation_task(
dask_client: DaskClient,
user_id: UserID,
project_id: ProjectID,
node_id: NodeID,
cluster_id: ClusterID,
cluster_id_resource: str,
image: Image,
expected_annotations: Dict[str, Any],
mocker: MockerFixture,
):
# INIT
expected_annotations["resources"].update(
{cluster_id_resource: CLUSTER_RESOURCE_MOCK_USAGE})
fake_task = {node_id: image}
mocked_done_callback_fct = mocker.Mock()
# NOTE: We pass another fct so it can run in our localy created dask cluster
def fake_sidecar_fct(job_id: str, u_id: str, prj_id: str,
n_id: str) -> int:
from dask.distributed import get_worker
worker = get_worker()
task: TaskState = worker.tasks.get(worker.get_current_task())
assert task is not None
assert task.annotations == expected_annotations
assert u_id == user_id
assert prj_id == project_id
assert n_id == node_id
return 123
# TEST COMPUTATION RUNS THROUGH
await dask_client.send_computation_tasks(
user_id=user_id,
project_id=project_id,
cluster_id=cluster_id,
tasks=fake_task,
callback=mocked_done_callback_fct,
remote_fct=fake_sidecar_fct,
)
assert (len(dask_client._taskid_to_future_map) == 1
), "dask client did not store the future of the task sent"
job_id, future = list(dask_client._taskid_to_future_map.items())[0]
# this waits for the computation to run
task_result = await future.result(timeout=2)
assert task_result == 123
assert future.key == job_id
await _wait_for_call(mocked_done_callback_fct)
mocked_done_callback_fct.assert_called_once()
mocked_done_callback_fct.reset_mock()
assert (len(dask_client._taskid_to_future_map) == 0
), "the list of futures was not cleaned correctly"
async def test_interactive_services_remain_after_websocket_reconnection_from_2_tabs(
client,
logged_user,
empty_user_project,
mocked_director_v2_api,
create_dynamic_service_mock,
socketio_client_factory: Callable,
client_session_id_factory: Callable[[], str],
storage_subsystem_mock, # when guest user logs out garbage is collected
expected_save_state: bool,
mocker: MockerFixture,
):
# login - logged_user fixture
# create empty study - empty_user_project fixture
# create dynamic service - create_dynamic_service_mock fixture
service = await create_dynamic_service_mock(logged_user["id"],
empty_user_project["uuid"])
# create first websocket
client_session_id1 = client_session_id_factory()
sio = await socketio_client_factory(client_session_id1)
# open project in client 1
await open_project(client, empty_user_project["uuid"], client_session_id1)
# create second websocket
client_session_id2 = client_session_id_factory()
sio2 = await socketio_client_factory(client_session_id2)
assert sio.sid != sio2.sid
socket_project_state_update_mock_callable = mocker.Mock()
sio2.on(
SOCKET_IO_PROJECT_UPDATED_EVENT,
handler=socket_project_state_update_mock_callable,
)
# disconnect first websocket
# NOTE: since the service deletion delay is set to 1 second for the test, we should not sleep as long here, or the user will be deleted
# We have no mock-up for the heatbeat...
await sio.disconnect()
assert not sio.sid
await asyncio.sleep(0.5) # let the thread call the method
socket_project_state_update_mock_callable.assert_called_with(
json.dumps({
"project_uuid": empty_user_project["uuid"],
"data": {
"locked": {
"value": False,
"owner": {
"user_id": logged_user["id"],
"first_name": logged_user["name"],
"last_name": "",
},
"status": "OPENED",
},
"state": {
"value": "NOT_STARTED"
},
},
}))
# open project in second client
await open_project(client, empty_user_project["uuid"], client_session_id2)
# ensure sufficient time is wasted here
await asyncio.sleep(SERVICE_DELETION_DELAY + 1)
await garbage_collector_core.collect_garbage(client.app)
# assert dynamic service is still around
mocked_director_v2_api["director_v2_api.stop_service"].assert_not_called()
# disconnect second websocket
await sio2.disconnect()
assert not sio2.sid
# assert dynamic service is still around for now
mocked_director_v2_api["director_v2_api.stop_service"].assert_not_called()
# reconnect websocket
sio2 = await socketio_client_factory(client_session_id2)
# it should still be there even after waiting for auto deletion from garbage collector
await asyncio.sleep(SERVICE_DELETION_DELAY + 1)
await garbage_collector_core.collect_garbage(client.app)
mocked_director_v2_api["director_v2_api.stop_service"].assert_not_called()
# now really disconnect
await sio2.disconnect()
assert not sio2.sid
# run the garbage collector
# event after waiting some time
await asyncio.sleep(SERVICE_DELETION_DELAY + 1)
await garbage_collector_core.collect_garbage(client.app)
await asyncio.sleep(1)
# assert dynamic service is gone
calls = [
call(
app=client.server.app,
save_state=expected_save_state,
service_uuid=service["service_uuid"],
)
]
mocked_director_v2_api["director_v2_core.stop_service"].assert_has_calls(
calls)
def create_run_mock(run_api_mock, mocker: MockerFixture):
run_api_mock.create_run = mocker.Mock()
return run_api_mock.create_run