async def test_monitor_crash(running_backend, event_bus, alice,
during_bootstrap):
async def _bad_monitor(*, task_status=trio.TASK_STATUS_IGNORED):
if during_bootstrap:
raise RuntimeError("D'oh !")
task_status.started()
await trio.sleep(0)
raise RuntimeError("D'oh !")
conn = BackendAuthenticatedConn(alice.organization_addr, alice.device_id,
alice.signing_key, event_bus)
with event_bus.listen() as spy:
conn.register_monitor(_bad_monitor)
async with conn.run():
await spy.wait_with_timeout(
"backend.connection.changed",
{
"status": BackendConnStatus.CRASHED,
"status_exc": spy.ANY
},
)
assert conn.status == BackendConnStatus.CRASHED
# Test command not possible
with pytest.raises(BackendNotAvailable) as exc:
await conn.cmds.ping()
assert str(
exc.value) == "Backend connection manager has crashed: D'oh !"
async def test_concurrency_sends(running_backend, alice, event_bus):
CONCURRENCY = 10
work_done_counter = 0
work_all_done = trio.Event()
async def sender(cmds, x):
nonlocal work_done_counter
rep = await cmds.ping(x)
assert rep == {"status": "ok", "pong": str(x)}
work_done_counter += 1
if work_done_counter == CONCURRENCY:
work_all_done.set()
conn = BackendAuthenticatedConn(
alice.organization_addr,
alice.device_id,
alice.signing_key,
event_bus,
max_pool=CONCURRENCY // 2,
)
async with conn.run():
async with trio.open_service_nursery() as nursery:
for x in range(CONCURRENCY):
nursery.start_soon(sender, conn.cmds, str(x))
async with real_clock_timeout():
await work_all_done.wait()
async def test_monitor_crash(caplog, running_backend, event_bus, alice, during_bootstrap):
async def _bad_monitor(*, task_status=trio.TASK_STATUS_IGNORED):
if during_bootstrap:
raise RuntimeError("D'oh !")
task_status.started()
await trio.sleep(0)
raise RuntimeError("D'oh !")
conn = BackendAuthenticatedConn(
alice.organization_addr, alice.device_id, alice.signing_key, event_bus
)
with event_bus.listen() as spy:
conn.register_monitor(_bad_monitor)
async with conn.run():
await spy.wait_with_timeout(
CoreEvent.BACKEND_CONNECTION_CHANGED,
{"status": BackendConnStatus.CRASHED, "status_exc": spy.ANY},
)
assert conn.status == BackendConnStatus.CRASHED
caplog.assert_occured_once(
"[error ] Unhandled exception [parsec.core.backend_connection.authenticated]"
)
# Test command not possible
with pytest.raises(BackendNotAvailable) as exc:
await conn.cmds.ping()
assert str(exc.value) == "Backend connection manager has crashed: D'oh !"
async def test_init_with_backend_online(running_backend, event_bus, alice):
monitor_initialized = False
monitor_teardown = False
async def _monitor(*, task_status=trio.TASK_STATUS_IGNORED):
nonlocal monitor_initialized, monitor_teardown
monitor_initialized = True
try:
task_status.started()
await trio.sleep_forever()
finally:
monitor_teardown = True
conn = BackendAuthenticatedConn(alice.organization_addr, alice.device_id,
alice.signing_key, event_bus)
assert conn.status == BackendConnStatus.LOST
conn.register_monitor(_monitor)
with event_bus.listen() as spy:
async with conn.run():
await spy.wait_multiple_with_timeout([
(
CoreEvent.BACKEND_CONNECTION_CHANGED,
{
"status": BackendConnStatus.INITIALIZING,
"status_exc": None
},
),
(
CoreEvent.BACKEND_CONNECTION_CHANGED,
{
"status": BackendConnStatus.READY,
"status_exc": None
},
),
])
assert conn.status == BackendConnStatus.READY
assert monitor_initialized
assert not monitor_teardown
# Test command
rep = await conn.cmds.ping("foo")
assert rep == {"status": "ok", "pong": "foo"}
# Test events
running_backend.backend.event_bus.send(
BackendEvent.PINGED,
organization_id=alice.organization_id,
author="bob@test",
ping="foo",
)
await spy.wait_with_timeout(CoreEvent.BACKEND_PINGED,
{"ping": "foo"})
assert monitor_teardown
async def test_switch_offline(mock_clock, running_backend, event_bus, alice):
mock_clock.rate = 1.0
conn = BackendAuthenticatedConn(alice.organization_addr, alice.device_id,
alice.signing_key, event_bus)
with event_bus.listen() as spy:
async with conn.run():
# Wait for the connection to be initialized
await spy.wait_with_timeout(
CoreEvent.BACKEND_CONNECTION_CHANGED,
{
"status": BackendConnStatus.READY,
"status_exc": None
},
)
# Switch backend offline and wait for according event
spy.clear()
with running_backend.offline():
await spy.wait_with_timeout(
CoreEvent.BACKEND_CONNECTION_CHANGED,
{
"status": BackendConnStatus.LOST,
"status_exc": spy.ANY
},
)
assert conn.status == BackendConnStatus.LOST
# Here backend switch back online, wait for the corresponding event
spy.clear()
# Backend event manager waits before retrying to connect
mock_clock.jump(5.0)
await spy.wait_with_timeout(
CoreEvent.BACKEND_CONNECTION_CHANGED,
{
"status": BackendConnStatus.READY,
"status_exc": None
},
)
assert conn.status == BackendConnStatus.READY
# Make sure event system still works as expected
spy.clear()
running_backend.backend.event_bus.send(
BackendEvent.PINGED,
organization_id=alice.organization_id,
author="bob@test",
ping="foo",
)
await spy.wait_with_timeout(CoreEvent.BACKEND_PINGED,
{"ping": "foo"})
async def alice_backend_conn(alice, event_bus_factory, running_backend_ready):
await running_backend_ready()
event_bus = event_bus_factory()
conn = BackendAuthenticatedConn(
alice.organization_addr, alice.device_id, alice.signing_key, event_bus
)
with event_bus.listen() as spy:
async with conn.run():
await spy.wait_with_timeout(
CoreEvent.BACKEND_CONNECTION_CHANGED,
{"status": BackendConnStatus.READY, "status_exc": None},
)
yield conn
async def alice_backend_conn(alice, event_bus_factory, running_backend_ready):
await running_backend_ready.wait()
event_bus = event_bus_factory()
conn = BackendAuthenticatedConn(alice.organization_addr, alice.device_id,
alice.signing_key, event_bus)
with event_bus.listen() as spy:
async with conn.run():
await spy.wait_with_timeout(
"backend.connection.changed",
{
"status": BackendConnStatus.READY,
"status_exc": None
},
)
yield conn
async def test_connection_refused(running_backend, event_bus, mallory):
conn = BackendAuthenticatedConn(
mallory.organization_addr, mallory.device_id, mallory.signing_key, event_bus
)
with event_bus.listen() as spy:
async with conn.run():
# Wait for the connection to be initialized
await spy.wait_with_timeout(
CoreEvent.BACKEND_CONNECTION_CHANGED,
{"status": BackendConnStatus.REFUSED, "status_exc": spy.ANY},
)
# Trying to use the connection should endup with an exception
with pytest.raises(BackendConnectionRefused):
await conn.cmds.ping()
async def test_init_with_backend_offline(event_bus, alice):
conn = BackendAuthenticatedConn(alice.organization_addr, alice.device_id,
alice.signing_key, event_bus)
assert conn.status == BackendConnStatus.LOST
with event_bus.listen() as spy:
async with conn.run():
await spy.wait_with_timeout(
CoreEvent.BACKEND_CONNECTION_CHANGED,
{
"status": BackendConnStatus.LOST,
"status_exc": spy.ANY
},
)
assert conn.status == BackendConnStatus.LOST
# Test command not possible
with pytest.raises(BackendNotAvailable):
await conn.cmds.ping("foo")
async def logged_core_factory(config: CoreConfig,
device: LocalDevice,
event_bus: Optional[EventBus] = None):
event_bus = event_bus or EventBus()
prevent_sync_pattern = get_prevent_sync_pattern(
config.prevent_sync_pattern_path)
backend_conn = BackendAuthenticatedConn(
addr=device.organization_addr,
device_id=device.device_id,
signing_key=device.signing_key,
event_bus=event_bus,
max_cooldown=config.backend_max_cooldown,
max_pool=config.backend_max_connections,
keepalive=config.backend_connection_keepalive,
)
remote_devices_manager = RemoteDevicesManager(backend_conn.cmds,
device.root_verify_key)
async with UserFS.run(
data_base_dir=config.data_base_dir,
device=device,
backend_cmds=backend_conn.cmds,
remote_devices_manager=remote_devices_manager,
event_bus=event_bus,
prevent_sync_pattern=prevent_sync_pattern,
preferred_language=config.gui_language,
workspace_storage_cache_size=config.workspace_storage_cache_size,
) as user_fs:
backend_conn.register_monitor(
partial(monitor_messages, user_fs, event_bus))
backend_conn.register_monitor(partial(monitor_sync, user_fs,
event_bus))
async with backend_conn.run():
async with mountpoint_manager_factory(
user_fs,
event_bus,
config.mountpoint_base_dir,
mount_all=config.mountpoint_enabled,
mount_on_workspace_created=config.mountpoint_enabled,
mount_on_workspace_shared=config.mountpoint_enabled,
unmount_on_workspace_revoked=config.mountpoint_enabled,
exclude_from_mount_all=config.disabled_workspaces,
) as mountpoint_manager:
yield LoggedCore(
config=config,
device=device,
event_bus=event_bus,
mountpoint_manager=mountpoint_manager,
user_fs=user_fs,
remote_devices_manager=remote_devices_manager,
backend_conn=backend_conn,
)
async def test_init_with_backend_offline(event_bus, alice):
conn = BackendAuthenticatedConn(
alice.organization_addr, alice.device_id, alice.signing_key, event_bus
)
assert conn.status == BackendConnStatus.LOST
default_organization_config = OrganizationConfig(
user_profile_outsider_allowed=False, active_users_limit=None
)
assert conn.get_organization_config() == default_organization_config
with event_bus.listen() as spy:
async with conn.run():
await spy.wait_with_timeout(
CoreEvent.BACKEND_CONNECTION_CHANGED,
{"status": BackendConnStatus.LOST, "status_exc": spy.ANY},
)
assert conn.status == BackendConnStatus.LOST
assert conn.get_organization_config() == default_organization_config
# Test command not possible
with pytest.raises(BackendNotAvailable):
await conn.cmds.ping("foo")
async def logged_core_factory(
config: CoreConfig, device: LocalDevice, event_bus: Optional[EventBus] = None
):
event_bus = event_bus or EventBus()
backend_conn = BackendAuthenticatedConn(
addr=device.organization_addr,
device_id=device.device_id,
signing_key=device.signing_key,
event_bus=event_bus,
max_cooldown=config.backend_max_cooldown,
max_pool=config.backend_max_connections,
keepalive=config.backend_connection_keepalive,
)
path = config.data_base_dir / device.slug
remote_devices_manager = RemoteDevicesManager(backend_conn.cmds, device.root_verify_key)
async with UserFS.run(
device, path, backend_conn.cmds, remote_devices_manager, event_bus
) as user_fs:
backend_conn.register_monitor(partial(monitor_messages, user_fs, event_bus))
backend_conn.register_monitor(partial(monitor_sync, user_fs, event_bus))
async with backend_conn.run():
async with mountpoint_manager_factory(
user_fs, event_bus, config.mountpoint_base_dir
) as mountpoint_manager:
yield LoggedCore(
config=config,
device=device,
event_bus=event_bus,
remote_devices_manager=remote_devices_manager,
mountpoint_manager=mountpoint_manager,
backend_conn=backend_conn,
user_fs=user_fs,
)
async def test_init_with_backend_online(
running_backend, event_bus, alice, apiv22_organization_cmd_supported
):
monitor_initialized = False
monitor_teardown = False
# Mock organization config command
async def _mocked_organization_config(client_ctx, msg):
if apiv22_organization_cmd_supported:
return {
"user_profile_outsider_allowed": True,
"active_users_limit": None,
"status": "ok",
}
else:
# Backend with API support <2.2, the client should be able to fallback
return {"status": "unknown_command"}
running_backend.backend.apis[ClientType.AUTHENTICATED][
"organization_config"
] = _mocked_organization_config
async def _monitor(*, task_status=trio.TASK_STATUS_IGNORED):
nonlocal monitor_initialized, monitor_teardown
monitor_initialized = True
try:
task_status.started()
await trio.sleep_forever()
finally:
monitor_teardown = True
conn = BackendAuthenticatedConn(
alice.organization_addr, alice.device_id, alice.signing_key, event_bus
)
assert conn.status == BackendConnStatus.LOST
default_organization_config = OrganizationConfig(
user_profile_outsider_allowed=False, active_users_limit=None
)
assert conn.get_organization_config() == default_organization_config
conn.register_monitor(_monitor)
with event_bus.listen() as spy:
async with conn.run():
await spy.wait_multiple_with_timeout(
[
(
CoreEvent.BACKEND_CONNECTION_CHANGED,
{"status": BackendConnStatus.INITIALIZING, "status_exc": None},
),
(
CoreEvent.BACKEND_CONNECTION_CHANGED,
{"status": BackendConnStatus.READY, "status_exc": None},
),
]
)
assert conn.status == BackendConnStatus.READY
assert monitor_initialized
assert not monitor_teardown
# Test organization config retrieval
if apiv22_organization_cmd_supported:
assert conn.get_organization_config() == OrganizationConfig(
user_profile_outsider_allowed=True, active_users_limit=None
)
else:
# Default value
assert conn.get_organization_config() == default_organization_config
# Test command
rep = await conn.cmds.ping("foo")
assert rep == {"status": "ok", "pong": "foo"}
# Test events
running_backend.backend.event_bus.send(
BackendEvent.PINGED,
organization_id=alice.organization_id,
author="bob@test",
ping="foo",
)
await spy.wait_with_timeout(CoreEvent.BACKEND_PINGED, {"ping": "foo"})
assert monitor_teardown