async def test_sync_under_concurrency(running_backend, alice_user_fs,
alice2_user_fs, alice, alice2):
with freeze_time("2000-01-02"):
waid = await alice_user_fs.workspace_create("wa")
with freeze_time("2000-01-03"):
wa2id = await alice2_user_fs.workspace_create("wa2")
with freeze_time("2000-01-04"):
await alice_user_fs.sync()
with freeze_time("2000-01-05"):
await alice2_user_fs.sync()
# Fetch back alice2's changes
with freeze_time("2000-01-06"):
await alice_user_fs.sync()
um = alice_user_fs.get_user_manifest()
um2 = alice2_user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=alice2.device_id,
timestamp=datetime(2000, 1, 5),
id=alice2.user_manifest_id,
version=3,
created=datetime(2000, 1, 1),
updated=datetime(2000, 1, 3),
last_processed_message=0,
workspaces=(
WorkspaceEntry(
name="wa",
id=waid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
),
WorkspaceEntry(
name="wa2",
id=wa2id,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 3),
role_cached_on=datetime(2000, 1, 3),
role=WorkspaceRole.OWNER,
),
),
)
expected_um = LocalUserManifest.from_remote(expected_base_um)
assert um == expected_um
assert um2 == expected_um
async def test_rename_workspace(initial_user_manifest_state, alice_user_fs,
alice):
with freeze_time("2000-01-02"):
wid = await alice_user_fs.workspace_create("w1")
with freeze_time("2000-01-03"):
await alice_user_fs.workspace_rename(wid, "w2")
um = alice_user_fs.get_user_manifest()
expected_base_um = initial_user_manifest_state.get_user_manifest_v1_for_backend(
alice)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=True,
updated=datetime(2000, 1, 3),
last_processed_message=expected_base_um.last_processed_message,
workspaces=(WorkspaceEntry(
name="w2",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
assert um == expected_um
async def test_create_workspace(initial_user_manifest_state, alice_user_fs,
alice):
with freeze_time("2000-01-02"):
wid = await alice_user_fs.workspace_create("w1")
um = alice_user_fs.get_user_manifest()
expected_base_um = initial_user_manifest_state.get_user_manifest_v1_for_backend(
alice)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=True,
updated=datetime(2000, 1, 2),
last_processed_message=expected_base_um.last_processed_message,
workspaces=(WorkspaceEntry(
name="w1",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
assert um == expected_um
w_manifest = await alice_user_fs.get_workspace(
wid).local_storage.get_manifest(wid)
expected_w_manifest = LocalWorkspaceManifest.new_placeholder(
alice.device_id, id=w_manifest.id, now=datetime(2000, 1, 2))
assert w_manifest == expected_w_manifest
async def test_modify_user_manifest_placeholder(running_backend,
backend_data_binder,
local_device_factory,
user_fs_factory):
device = local_device_factory()
await backend_data_binder.bind_device(device,
initial_user_manifest_in_v0=True)
async with user_fs_factory(device, initialize_in_v0=True) as user_fs:
um_v0 = user_fs.get_user_manifest()
with freeze_time("2000-01-02"):
wid = await user_fs.workspace_create("w1")
um = user_fs.get_user_manifest()
expected_um = um_v0.evolve(
updated=datetime(2000, 1, 2),
workspaces=(WorkspaceEntry(
name="w1",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
assert um == expected_um
# Make sure we can fetch back data from the database on user_fs restart
async with user_fs_factory(device, initialize_in_v0=True) as user_fs2:
um2 = user_fs2.get_user_manifest()
assert um2 == expected_um
async def test_new_sharing_trigger_event(alice_client, bob_client,
running_backend):
# First, create a folder and sync it on backend
with freeze_time("2000-01-01"):
wid = await alice_client.user_fs.workspace_create("foo")
workspace = alice_client.user_fs.get_workspace(wid)
with freeze_time("2000-01-02"):
await workspace.sync()
# Now we can share this workspace with Bob
with bob_client.event_bus.listen() as spy:
with freeze_time("2000-01-03"):
await alice_client.user_fs.workspace_share(
wid, recipient="bob", role=WorkspaceRole.MANAGER)
# Bob should get a notification
await spy.wait_with_timeout(
ClientEvent.SHARING_UPDATED,
{
"new_entry":
WorkspaceEntry(
name="foo",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=ANY,
role=WorkspaceRole.MANAGER,
),
"previous_entry":
None,
},
timeout=3,
)
async def _transactions_factory(device,
backend_cmds,
local_storage,
cls=SyncTransactions):
def _get_workspace_entry():
return workspace_entry
workspace_entry = WorkspaceEntry.new("test")
workspace_manifest = LocalWorkspaceManifest.new_placeholder(
device.device_id, id=workspace_entry.id, now=datetime(2000, 1, 1))
async with local_storage.lock_entry_id(workspace_entry.id):
await local_storage.set_manifest(workspace_entry.id,
workspace_manifest)
remote_devices_manager = remote_devices_manager_factory(device)
remote_loader = RemoteLoader(
device,
workspace_entry.id,
_get_workspace_entry,
backend_cmds,
remote_devices_manager,
local_storage,
)
return cls(
workspace_entry.id,
_get_workspace_entry,
device,
local_storage,
remote_loader,
event_bus,
)
async def test_sync_placeholder(running_backend, backend_data_binder,
local_device_factory, user_fs_factory,
with_workspace):
device = local_device_factory()
await backend_data_binder.bind_device(device,
initial_user_manifest_in_v0=True)
async with user_fs_factory(device, initialize_in_v0=True) as user_fs:
um_v0 = user_fs.get_user_manifest()
expected_um = LocalUserManifest.new_placeholder(
device.device_id, id=device.user_manifest_id, now=um_v0.created)
assert um_v0 == expected_um
if with_workspace:
with freeze_time("2000-01-02"):
wid = await user_fs.workspace_create("w1")
um = user_fs.get_user_manifest()
expected_um = um_v0.evolve(
updated=datetime(2000, 1, 2),
workspaces=(WorkspaceEntry(
name="w1",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
assert um == expected_um
with freeze_time("2000-01-02"):
await user_fs.sync()
um = user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=device.device_id,
timestamp=datetime(2000, 1, 2),
id=device.user_manifest_id,
version=1,
created=expected_um.created,
updated=expected_um.updated,
last_processed_message=0,
workspaces=expected_um.workspaces,
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=expected_um.updated,
last_processed_message=0,
workspaces=expected_base_um.workspaces,
)
assert um == expected_um
async def test_revoke_sharing_trigger_event(alice_client, bob_client,
running_backend):
with freeze_time("2000-01-02"):
wid = await create_shared_workspace("w", alice_client, bob_client)
with bob_client.event_bus.listen() as spy:
with freeze_time("2000-01-03"):
await alice_client.user_fs.workspace_share(wid,
recipient="bob",
role=None)
# Each workspace participant should get the message
await spy.wait_with_timeout(
ClientEvent.SHARING_UPDATED,
{
"new_entry":
WorkspaceEntry(
name="w",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=ANY,
role=None,
),
"previous_entry":
WorkspaceEntry(
name="w",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=ANY,
role=WorkspaceRole.MANAGER,
),
},
timeout=3,
)
def __init__(
self,
device: LocalDevice,
path: Path,
backend_cmds: BackendAuthenticatedCmds,
remote_devices_manager: RemoteDevicesManager,
event_bus: EventBus,
pattern_filter: Pattern,
):
self.device = device
self.path = path
self.backend_cmds = backend_cmds
self.remote_devices_manager = remote_devices_manager
self.event_bus = event_bus
self.pattern_filter = pattern_filter
self.storage: UserStorage # Setup by UserStorage.run factory
# Message processing is done in-order, hence it is pointless to do
# it concurrently
self._workspace_storage_nursery: trio.Nursery # Setup by UserStorage.run factory
self._process_messages_lock = trio.Lock()
self._update_user_manifest_lock = trio.Lock()
self._workspace_storages: Dict[EntryID, WorkspaceFS] = {}
now = pendulum_now()
wentry = WorkspaceEntry(
name="<user manifest>",
id=device.user_manifest_id,
key=device.user_manifest_key,
encryption_revision=1,
encrypted_on=now,
role_cached_on=now,
role=WorkspaceRole.OWNER,
)
self.remote_loader = RemoteLoader(
self.device,
self.device.user_manifest_id,
lambda: wentry,
self.backend_cmds,
self.remote_devices_manager,
# Hack, but fine as long as we only call `load_realm_current_roles`
None,
)
async def test_sync_remote_changes(running_backend, alice_user_fs,
alice2_user_fs, alice, alice2):
# Alice 2 update the user manifest
with freeze_time("2000-01-02"):
wid = await alice2_user_fs.workspace_create("wa")
with freeze_time("2000-01-03"):
await alice2_user_fs.sync()
# Alice retrieve the changes
um = alice_user_fs.get_user_manifest()
await alice_user_fs.sync()
um = alice_user_fs.get_user_manifest()
um2 = alice2_user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=alice2.device_id,
timestamp=datetime(2000, 1, 3),
id=alice2.user_manifest_id,
version=2,
created=datetime(2000, 1, 1),
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=(WorkspaceEntry(
name="wa",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
)
assert um == expected_um
assert um2 == expected_um
async def workspace_create(self, name: AnyEntryName) -> EntryID:
"""
Raises: Nothing !
"""
name = EntryName(name)
workspace_entry = WorkspaceEntry.new(name)
workspace_manifest = LocalWorkspaceManifest.new_placeholder(
self.device.device_id, id=workspace_entry.id)
async with self._update_user_manifest_lock:
user_manifest = self.get_user_manifest()
user_manifest = user_manifest.evolve_workspaces_and_mark_updated(
workspace_entry)
await self._create_workspace(workspace_entry.id,
workspace_manifest)
await self.set_user_manifest(user_manifest)
self.event_bus.send(ClientEvent.FS_ENTRY_UPDATED,
id=self.user_manifest_id)
self.event_bus.send(ClientEvent.FS_WORKSPACE_CREATED,
new_entry=workspace_entry)
return workspace_entry.id
async def test_concurrent_sync_placeholder(running_backend,
backend_data_binder,
local_device_factory,
user_fs_factory, dev2_has_changes):
device1 = local_device_factory("a@1")
await backend_data_binder.bind_device(device1,
initial_user_manifest_in_v0=True)
device2 = local_device_factory("a@2")
await backend_data_binder.bind_device(device2,
initial_user_manifest_in_v0=True)
async with user_fs_factory(
device1, initialize_in_v0=True) as user_fs1, user_fs_factory(
device2, initialize_in_v0=True) as user_fs2:
# fs2's created value is different and will be overwritten when
# merging synced manifest from fs1
um_created_v0_fs1 = user_fs1.get_user_manifest().created
with freeze_time("2000-01-01"):
w1id = await user_fs1.workspace_create("w1")
if dev2_has_changes:
with freeze_time("2000-01-02"):
w2id = await user_fs2.workspace_create("w2")
with freeze_time("2000-01-03"):
await user_fs1.sync()
with freeze_time("2000-01-04"):
await user_fs2.sync()
if dev2_has_changes:
with freeze_time("2000-01-05"):
await user_fs1.sync()
um1 = user_fs1.get_user_manifest()
um2 = user_fs2.get_user_manifest()
if dev2_has_changes:
expected_base_um = UserManifest(
author=device2.device_id,
id=device2.user_manifest_id,
timestamp=datetime(2000, 1, 4),
version=2,
created=um_created_v0_fs1,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=(
WorkspaceEntry(
name="w1",
id=w1id,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=datetime(2000, 1, 1),
role=WorkspaceRole.OWNER,
),
WorkspaceEntry(
name="w2",
id=w2id,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
),
),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
)
else:
expected_base_um = UserManifest(
author=device1.device_id,
timestamp=datetime(2000, 1, 3),
id=device1.user_manifest_id,
version=1,
created=um_created_v0_fs1,
updated=datetime(2000, 1, 1),
last_processed_message=0,
workspaces=(WorkspaceEntry(
name="w1",
id=w1id,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=datetime(2000, 1, 1),
role=WorkspaceRole.OWNER,
), ),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 1),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
)
assert um1 == expected_um
assert um2 == expected_um
async def _process_message_sharing_granted(
self, msg: Union[SharingGrantedMessageContent,
SharingReencryptedMessageContent]):
"""
Raises:
FSError
FSBackendOfflineError
FSSharingNotAllowedError
"""
# We cannot blindly trust the message sender ! Hence we first
# interrogate the backend to make sure he is a workspace manager/owner.
# Note this means we refuse to process messages from a former-manager,
# even if the message was sent at a time the user was manager (in such
# case the user can still ask for another manager to re-do the sharing
# so it's no big deal).
try:
roles = await self.remote_loader.load_realm_current_roles(msg.id)
except FSWorkspaceNoAccess:
# Seems we lost the access roles anyway, nothing to do then
return
if roles.get(msg.author.user_id,
None) not in (WorkspaceRole.OWNER, WorkspaceRole.MANAGER):
raise FSSharingNotAllowedError(
f"User {msg.author.user_id} cannot share workspace `{msg.id}`"
" with us (requires owner or manager right)")
# Determine the access roles we have been given to
self_role = roles.get(self.device.user_id)
# Finally insert the new workspace entry into our user manifest
workspace_entry = WorkspaceEntry(
# Name are not required to be unique across workspaces, so no check to do here
name=msg.name,
id=msg.id,
key=msg.key,
encryption_revision=msg.encryption_revision,
encrypted_on=msg.encrypted_on,
role=self_role,
role_cached_on=pendulum_now(),
)
async with self._update_user_manifest_lock:
user_manifest = self.get_user_manifest()
# Check if we already know this workspace
already_existing_entry = user_manifest.get_workspace_entry(msg.id)
if already_existing_entry:
# Merge with existing as target to keep possible workpace rename
workspace_entry = merge_workspace_entry(
None, workspace_entry, already_existing_entry)
user_manifest = user_manifest.evolve_workspaces_and_mark_updated(
workspace_entry)
await self.set_user_manifest(user_manifest)
self.event_bus.send(ClientEvent.USERFS_UPDATED)
if not already_existing_entry:
# TODO: remove this event ?
self.event_bus.send(ClientEvent.FS_ENTRY_SYNCED,
id=workspace_entry.id)
self.event_bus.send(
ClientEvent.SHARING_UPDATED,
new_entry=workspace_entry,
previous_entry=already_existing_entry,
)