async def test_concurrency_sends(running_backend, alice, event_bus):
CONCURRENCY = 10
work_done_counter = 0
work_all_done = trio.Event()
async def sender(cmds, x):
nonlocal work_done_counter
rep = await cmds.ping(x)
assert rep == {"status": "ok", "pong": str(x)}
work_done_counter += 1
if work_done_counter == CONCURRENCY:
work_all_done.set()
conn = BackendAuthenticatedConn(
alice.organization_addr,
alice.device_id,
alice.signing_key,
event_bus,
max_pool=CONCURRENCY // 2,
)
async with conn.run():
async with trio.open_service_nursery() as nursery:
for x in range(CONCURRENCY):
nursery.start_soon(sender, conn.cmds, str(x))
with trio.fail_after(1):
await work_all_done.wait()
async def alice_backend_conn(alice, event_bus_factory, running_backend_ready):
await running_backend_ready.wait()
event_bus = event_bus_factory()
conn = BackendAuthenticatedConn(
alice.organization_addr, alice.device_id, alice.signing_key, event_bus
)
with event_bus.listen() as spy:
async with conn.run():
await spy.wait_with_timeout(
ClientEvent.BACKEND_CONNECTION_CHANGED,
{"status": BackendConnStatus.READY, "status_exc": None},
)
yield conn
async def test_init_with_backend_online(running_backend, event_bus, alice):
monitor_initialized = False
monitor_teardown = False
async def _monitor(*, task_status=trio.TASK_STATUS_IGNORED):
nonlocal monitor_initialized, monitor_teardown
monitor_initialized = True
try:
task_status.started()
await trio.sleep_forever()
finally:
monitor_teardown = True
conn = BackendAuthenticatedConn(
alice.organization_addr, alice.device_id, alice.signing_key, event_bus
)
assert conn.status == BackendConnStatus.LOST
conn.register_monitor(_monitor)
with event_bus.listen() as spy:
async with conn.run():
await spy.wait_multiple_with_timeout(
[
(
ClientEvent.BACKEND_CONNECTION_CHANGED,
{"status": BackendConnStatus.INITIALIZING, "status_exc": None},
),
(
ClientEvent.BACKEND_CONNECTION_CHANGED,
{"status": BackendConnStatus.READY, "status_exc": None},
),
]
)
assert conn.status == BackendConnStatus.READY
assert monitor_initialized
assert not monitor_teardown
# Test command
rep = await conn.cmds.ping("foo")
assert rep == {"status": "ok", "pong": "foo"}
# Test events
running_backend.backend.event_bus.send(
BackendEvent.PINGED,
organization_id=alice.organization_id,
author="bob@test",
ping="foo",
)
await spy.wait_with_timeout(ClientEvent.BACKEND_PINGED, {"ping": "foo"})
assert monitor_teardown
async def test_init_with_backend_offline(event_bus, alice):
conn = BackendAuthenticatedConn(
alice.organization_addr, alice.device_id, alice.signing_key, event_bus
)
assert conn.status == BackendConnStatus.LOST
with event_bus.listen() as spy:
async with conn.run():
await spy.wait_with_timeout(
ClientEvent.BACKEND_CONNECTION_CHANGED,
{"status": BackendConnStatus.LOST, "status_exc": spy.ANY},
)
assert conn.status == BackendConnStatus.LOST
# Test command not possible
with pytest.raises(BackendNotAvailable):
await conn.cmds.ping("foo")
async def test_connection_refused(mock_clock, running_backend, event_bus, mallory):
mock_clock.rate = 1.0
conn = BackendAuthenticatedConn(
mallory.organization_addr, mallory.device_id, mallory.signing_key, event_bus
)
with event_bus.listen() as spy:
async with conn.run():
# Wait for the connection to be initialized
await spy.wait_with_timeout(
ClientEvent.BACKEND_CONNECTION_CHANGED,
{"status": BackendConnStatus.REFUSED, "status_exc": spy.ANY},
)
# Trying to use the connection should endup with an exception
with pytest.raises(BackendConnectionRefused):
await conn.cmds.ping()
async def test_switch_offline(mock_clock, running_backend, event_bus, alice):
mock_clock.rate = 1.0
conn = BackendAuthenticatedConn(
alice.organization_addr, alice.device_id, alice.signing_key, event_bus
)
with event_bus.listen() as spy:
async with conn.run():
# Wait for the connection to be initialized
await spy.wait_with_timeout(
ClientEvent.BACKEND_CONNECTION_CHANGED,
{"status": BackendConnStatus.READY, "status_exc": None},
)
# Switch backend offline and wait for according event
spy.clear()
with running_backend.offline():
await spy.wait_with_timeout(
ClientEvent.BACKEND_CONNECTION_CHANGED,
{"status": BackendConnStatus.LOST, "status_exc": spy.ANY},
)
assert conn.status == BackendConnStatus.LOST
# Here backend switch back online, wait for the corresponding event
spy.clear()
# Backend event manager waits before retrying to connect
mock_clock.jump(5.0)
await spy.wait_with_timeout(
ClientEvent.BACKEND_CONNECTION_CHANGED,
{"status": BackendConnStatus.READY, "status_exc": None},
)
assert conn.status == BackendConnStatus.READY
# Make sure event system still works as expected
spy.clear()
running_backend.backend.event_bus.send(
BackendEvent.PINGED,
organization_id=alice.organization_id,
author="bob@test",
ping="foo",
)
await spy.wait_with_timeout(ClientEvent.BACKEND_PINGED, {"ping": "foo"})
async def logged_client_factory(config: ClientConfig,
device: LocalDevice,
event_bus: Optional[EventBus] = None):
event_bus = event_bus or EventBus()
# Get the pattern filter
if config.pattern_filter is None:
pattern_filter = get_pattern_filter(config.pattern_filter_path)
else:
try:
pattern_filter = re.compile(config.pattern_filter)
except re.error:
pattern_filter = get_pattern_filter(config.pattern_filter_path)
backend_conn = BackendAuthenticatedConn(
addr=device.organization_addr,
device_id=device.device_id,
signing_key=device.signing_key,
event_bus=event_bus,
max_cooldown=config.backend_max_cooldown,
max_pool=config.backend_max_connections,
keepalive=config.backend_connection_keepalive,
)
path = config.data_base_dir / device.slug
remote_devices_manager = RemoteDevicesManager(backend_conn.cmds,
device.root_verify_key)
async with UserFS.run(device, path, backend_conn.cmds,
remote_devices_manager, event_bus,
pattern_filter) as user_fs:
backend_conn.register_monitor(
partial(monitor_messages, user_fs, event_bus))
backend_conn.register_monitor(partial(monitor_sync, user_fs,
event_bus))
async with backend_conn.run():
async with mountpoint_manager_factory(
user_fs,
event_bus,
config.mountpoint_base_dir,
mount_all=config.mountpoint_enabled,
mount_on_workspace_created=config.mountpoint_enabled,
mount_on_workspace_shared=config.mountpoint_enabled,
unmount_on_workspace_revoked=config.mountpoint_enabled,
exclude_from_mount_all=config.disabled_workspaces,
) as mountpoint_manager:
yield LoggedClient(
config=config,
device=device,
event_bus=event_bus,
mountpoint_manager=mountpoint_manager,
user_fs=user_fs,
remote_devices_manager=remote_devices_manager,
backend_conn=backend_conn,
)
async def test_monitor_crash(running_backend, event_bus, alice, during_bootstrap):
async def _bad_monitor(*, task_status=trio.TASK_STATUS_IGNORED):
if during_bootstrap:
raise RuntimeError("D'oh !")
task_status.started()
await trio.sleep(0)
raise RuntimeError("D'oh !")
conn = BackendAuthenticatedConn(
alice.organization_addr, alice.device_id, alice.signing_key, event_bus
)
with event_bus.listen() as spy:
conn.register_monitor(_bad_monitor)
async with conn.run():
await spy.wait_with_timeout(
ClientEvent.BACKEND_CONNECTION_CHANGED,
{"status": BackendConnStatus.CRASHED, "status_exc": spy.ANY},
)
assert conn.status == BackendConnStatus.CRASHED
# Test command not possible
with pytest.raises(BackendNotAvailable) as exc:
await conn.cmds.ping()
assert str(exc.value) == "Backend connection manager has crashed: D'oh !"