async def test_sync_under_concurrency(running_backend, alice_user_fs,
alice2_user_fs, alice, alice2):
with freeze_time("2000-01-02"):
waid = await alice_user_fs.workspace_create(EntryName("wa"))
with freeze_time("2000-01-03"):
wa2id = await alice2_user_fs.workspace_create(EntryName("wa2"))
with freeze_time("2000-01-04"):
await alice_user_fs.sync()
with freeze_time("2000-01-05"):
await alice2_user_fs.sync()
# Fetch back alice2's changes
with freeze_time("2000-01-06"):
await alice_user_fs.sync()
um = alice_user_fs.get_user_manifest()
um2 = alice2_user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=alice2.device_id,
timestamp=datetime(2000, 1, 5),
id=alice2.user_manifest_id,
version=3,
created=datetime(2000, 1, 1),
updated=datetime(2000, 1, 3),
last_processed_message=0,
workspaces=(
WorkspaceEntry(
name=EntryName("wa"),
id=waid,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
),
WorkspaceEntry(
name=EntryName("wa2"),
id=wa2id,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 3),
role_cached_on=datetime(2000, 1, 3),
role=WorkspaceRole.OWNER,
),
),
)
expected_um = LocalUserManifest.from_remote(expected_base_um)
um = _update_user_manifest_key(um)
um2 = _update_user_manifest_key(um2)
assert um == expected_um
assert um2 == expected_um
async def workspace_create(self, name: EntryName) -> EntryID:
assert isinstance(name, EntryName)
async with self._update_user_manifest_lock:
timestamp = self.device.timestamp()
workspace_entry = WorkspaceEntry.new(name, timestamp=timestamp)
user_manifest = self.get_user_manifest()
user_manifest = user_manifest.evolve_workspaces_and_mark_updated(
timestamp, workspace_entry)
# Given *we* are the creator of the workspace, our placeholder is
# the only non-speculative one.
#
# Note the save order is important given there is no atomicity
# between saving the non-speculative workspace manifest placeholder
# and the save of the user manifest containing the workspace entry.
# Indeed, if we would save the user manifest first and a crash
# occured before saving the placeholder, we would endup in the same
# situation as if the workspace has been created by someone else
# (i.e. a workspace entry but no local data about this workspace)
# so we would fallback to a local speculative workspace manifest.
# However a speculative manifest means the workspace have been
# created by somebody else, and hence we shouldn't try to create
# it corresponding realm in the backend !
await workspace_storage_non_speculative_init(
data_base_dir=self.data_base_dir,
device=self.device,
workspace_id=workspace_entry.id,
)
await self.set_user_manifest(user_manifest)
self.event_bus.send(CoreEvent.FS_ENTRY_UPDATED,
id=self.user_manifest_id)
self.event_bus.send(CoreEvent.FS_WORKSPACE_CREATED,
new_entry=workspace_entry)
return workspace_entry.id
async def test_sync(running_backend, alice2_user_fs, alice2):
with freeze_time("2000-01-02"):
wid = await alice2_user_fs.workspace_create("w1")
with freeze_time("2000-01-03"):
await alice2_user_fs.sync()
um = alice2_user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=alice2.device_id,
timestamp=datetime(2000, 1, 3),
id=alice2.user_manifest_id,
version=2,
created=datetime(2000, 1, 1),
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=(WorkspaceEntry(
name="w1",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
expected_um = LocalUserManifest.from_remote(expected_base_um)
assert um == expected_um
async def _transactions_factory(device, backend_cmds, local_storage, cls=SyncTransactions):
def _get_workspace_entry():
return workspace_entry
workspace_entry = WorkspaceEntry.new("test")
workspace_manifest = LocalWorkspaceManifest.new_placeholder(
id=workspace_entry.id, now=Pendulum(2000, 1, 1)
)
async with local_storage.lock_entry_id(workspace_entry.id):
await local_storage.set_manifest(workspace_entry.id, workspace_manifest)
remote_devices_manager = remote_devices_manager_factory(device)
remote_loader = RemoteLoader(
device,
workspace_entry.id,
_get_workspace_entry,
backend_cmds,
remote_devices_manager,
local_storage,
)
return cls(
workspace_entry.id,
_get_workspace_entry,
device,
local_storage,
remote_loader,
event_bus,
)
async def test_modify_user_manifest_placeholder(running_backend,
backend_data_binder,
local_device_factory,
user_fs_factory):
device = local_device_factory()
await backend_data_binder.bind_device(device,
initial_user_manifest_in_v0=True)
async with user_fs_factory(device, initialize_in_v0=True) as user_fs:
um_v0 = user_fs.get_user_manifest()
with freeze_time("2000-01-02"):
wid = await user_fs.workspace_create("w1")
um = user_fs.get_user_manifest()
expected_um = um_v0.evolve(
updated=datetime(2000, 1, 2),
workspaces=(WorkspaceEntry(
name="w1",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
assert um == expected_um
# Make sure we can fetch back data from the database on user_fs restart
async with user_fs_factory(device, initialize_in_v0=True) as user_fs2:
um2 = user_fs2.get_user_manifest()
assert um2 == expected_um
async def test_create_workspace(initial_user_manifest_state, alice_user_fs,
alice):
with freeze_time("2000-01-02"):
wid = await alice_user_fs.workspace_create("w1")
um = alice_user_fs.get_user_manifest()
expected_base_um = initial_user_manifest_state.get_user_manifest_v1_for_backend(
alice)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=True,
updated=datetime(2000, 1, 2),
last_processed_message=expected_base_um.last_processed_message,
workspaces=(WorkspaceEntry(
name="w1",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
assert um == expected_um
w_manifest = await alice_user_fs.get_workspace(
wid).local_storage.get_manifest(wid)
expected_w_manifest = LocalWorkspaceManifest.new_placeholder(
alice.device_id, id=w_manifest.id, now=datetime(2000, 1, 2))
assert w_manifest == expected_w_manifest
async def test_rename_workspace(initial_user_manifest_state, alice_user_fs,
alice):
with freeze_time("2000-01-02"):
wid = await alice_user_fs.workspace_create("w1")
with freeze_time("2000-01-03"):
await alice_user_fs.workspace_rename(wid, "w2")
um = alice_user_fs.get_user_manifest()
expected_base_um = initial_user_manifest_state.get_user_manifest_v1_for_backend(
alice)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=True,
updated=datetime(2000, 1, 3),
last_processed_message=expected_base_um.last_processed_message,
workspaces=(WorkspaceEntry(
name="w2",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
assert um == expected_um
async def test_new_sharing_trigger_event(alice_core, bob_core,
running_backend):
# First, create a folder and sync it on backend
with freeze_time("2000-01-01"):
wid = await alice_core.user_fs.workspace_create("foo")
workspace = alice_core.user_fs.get_workspace(wid)
with freeze_time("2000-01-02"):
await workspace.sync()
# Now we can share this workspace with Bob
with bob_core.event_bus.listen() as spy:
with freeze_time("2000-01-03"):
await alice_core.user_fs.workspace_share(
wid, recipient="bob", role=WorkspaceRole.MANAGER)
# Bob should get a notification
await spy.wait_with_timeout(
"sharing.updated",
{
"new_entry":
WorkspaceEntry(
name="foo (shared by alice)",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=Pendulum(2000, 1, 1),
role_cached_on=ANY,
role=WorkspaceRole.MANAGER,
),
"previous_entry":
None,
},
)
async def test_revoke_sharing_trigger_event(alice_core, bob_core, running_backend):
KEY = SecretKey.generate()
def _update_event(event):
if event.event == CoreEvent.SHARING_UPDATED:
event.kwargs["new_entry"] = event.kwargs["new_entry"].evolve(
key=KEY, role_cached_on=datetime(2000, 1, 2)
)
event.kwargs["previous_entry"] = event.kwargs["previous_entry"].evolve(
key=KEY, role_cached_on=datetime(2000, 1, 2)
)
return event
with freeze_time("2000-01-02"):
wid = await create_shared_workspace(EntryName("w"), alice_core, bob_core)
with bob_core.event_bus.listen() as spy:
with freeze_time("2000-01-03"):
await alice_core.user_fs.workspace_share(wid, recipient=UserID("bob"), role=None)
# Each workspace participant should get the message
await spy.wait_with_timeout(
CoreEvent.SHARING_UPDATED,
{
"new_entry": WorkspaceEntry(
name=EntryName("w"),
id=wid,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=None,
),
"previous_entry": WorkspaceEntry(
name=EntryName("w"),
id=wid,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.MANAGER,
),
},
update_event_func=_update_event,
)
async def test_sync_placeholder(
running_backend, backend_data_binder, local_device_factory, user_fs_factory, with_workspace
):
device = local_device_factory()
await backend_data_binder.bind_device(device, initial_user_manifest_in_v0=True)
async with user_fs_factory(device, initialize_in_v0=True) as user_fs:
um_v0 = user_fs.get_user_manifest()
expected_um = LocalUserManifest.new_placeholder(
id=device.user_manifest_id, now=um_v0.created
)
assert um_v0 == expected_um
if with_workspace:
with freeze_time("2000-01-02"):
wid = await user_fs.workspace_create("w1")
um = user_fs.get_user_manifest()
expected_um = um_v0.evolve(
updated=Pendulum(2000, 1, 2),
workspaces=(
WorkspaceEntry(
name="w1",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=Pendulum(2000, 1, 2),
role_cached_on=Pendulum(2000, 1, 2),
role=WorkspaceRole.OWNER,
),
),
)
assert um == expected_um
with freeze_time("2000-01-02"):
await user_fs.sync()
um = user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=device.device_id,
timestamp=Pendulum(2000, 1, 2),
id=device.user_manifest_id,
version=1,
created=expected_um.created,
updated=expected_um.updated,
last_processed_message=0,
workspaces=expected_um.workspaces,
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=expected_um.updated,
last_processed_message=0,
workspaces=expected_base_um.workspaces,
)
assert um == expected_um
def __init__(
self,
data_base_dir: Path,
device: LocalDevice,
backend_cmds: BackendAuthenticatedCmds,
remote_devices_manager: RemoteDevicesManager,
event_bus: EventBus,
prevent_sync_pattern: Pattern[str],
preferred_language: str,
workspace_storage_cache_size: int,
):
self.data_base_dir = data_base_dir
self.device = device
self.backend_cmds = backend_cmds
self.remote_devices_manager = remote_devices_manager
self.event_bus = event_bus
self.prevent_sync_pattern = prevent_sync_pattern
self.preferred_language = preferred_language
self.workspace_storage_cache_size = workspace_storage_cache_size
self.storage: UserStorage # Setup by UserStorage.run factory
# Message processing is done in-order, hence it is pointless to do
# it concurrently
self._workspace_storage_nursery: trio.Nursery # Setup by UserStorage.run factory
self._process_messages_lock = trio.Lock()
self._update_user_manifest_lock = trio.Lock()
self._workspaces: Dict[EntryID, WorkspaceFS] = {}
timestamp = self.device.timestamp()
wentry = WorkspaceEntry(
name=EntryName("<user manifest>"),
id=device.user_manifest_id,
key=device.user_manifest_key,
encryption_revision=1,
encrypted_on=timestamp,
role_cached_on=timestamp,
role=WorkspaceRole.OWNER,
)
async def _get_previous_entry() -> WorkspaceEntry:
raise NotImplementedError
self.remote_loader = UserRemoteLoader(
self.device,
self.device.user_manifest_id,
lambda: wentry,
_get_previous_entry,
self.backend_cmds,
self.remote_devices_manager,
)
async def test_revoke_sharing_trigger_event(alice_core, bob_core,
running_backend):
with freeze_time("2000-01-02"):
wid = await create_shared_workspace("w", alice_core, bob_core)
with bob_core.event_bus.listen() as spy:
with freeze_time("2000-01-03"):
await alice_core.user_fs.workspace_share(wid,
recipient="bob",
role=None)
# Each workspace participant should get the message
await spy.wait_with_timeout(
"sharing.updated",
{
"new_entry":
WorkspaceEntry(
name="w (shared by alice)",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=Pendulum(2000, 1, 2),
role_cached_on=ANY,
role=None,
),
"previous_entry":
WorkspaceEntry(
name="w (shared by alice)",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=Pendulum(2000, 1, 2),
role_cached_on=ANY,
role=WorkspaceRole.MANAGER,
),
},
)
async def workspace_create(self, name: AnyEntryName) -> EntryID:
"""
Raises: Nothing !
"""
name = EntryName(name)
workspace_entry = WorkspaceEntry.new(name)
workspace_manifest = LocalWorkspaceManifest.new_placeholder(id=workspace_entry.id)
async with self._update_user_manifest_lock:
user_manifest = self.get_user_manifest()
user_manifest = user_manifest.evolve_workspaces_and_mark_updated(workspace_entry)
await self._create_workspace(workspace_entry.id, workspace_manifest)
await self.set_user_manifest(user_manifest)
self.event_bus.send("fs.entry.updated", id=self.user_manifest_id)
self.event_bus.send("fs.workspace.created", new_entry=workspace_entry)
return workspace_entry.id
async def test_new_workspace(running_backend, alice, alice_user_fs,
alice2_user_fs):
with freeze_time("2000-01-02"):
wid = await alice_user_fs.workspace_create(EntryName("w"))
workspace = alice_user_fs.get_workspace(wid)
with alice_user_fs.event_bus.listen() as spy:
with freeze_time("2000-01-03"):
await workspace.sync()
spy.assert_events_occured([(CoreEvent.FS_ENTRY_SYNCED, {
"workspace_id": wid,
"id": wid
}, datetime(2000, 1, 3))])
workspace2 = alice_user_fs.get_workspace(wid)
await alice_user_fs.sync()
await workspace2.sync()
workspace_entry = workspace.get_workspace_entry()
path_info = await workspace.path_info("/")
assert path_info == {
"type": "folder",
"id": wid,
"is_placeholder": False,
"need_sync": False,
"base_version": 1,
"children": [],
"created": datetime(2000, 1, 2),
"updated": datetime(2000, 1, 2),
"confinement_point": None,
}
KEY = SecretKey.generate()
workspace_entry = workspace_entry.evolve(key=KEY)
assert workspace_entry == WorkspaceEntry(
name=EntryName("w"),
id=wid,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
)
workspace_entry2 = workspace.get_workspace_entry()
workspace_entry2 = workspace_entry2.evolve(key=KEY)
path_info2 = await workspace.path_info("/")
assert workspace_entry == workspace_entry2
assert path_info == path_info2
async def test_sync_remote_changes(running_backend, alice_user_fs,
alice2_user_fs, alice, alice2):
# Alice 2 update the user manifest
with freeze_time("2000-01-02"):
wid = await alice2_user_fs.workspace_create(EntryName("wa"))
with freeze_time("2000-01-03"):
await alice2_user_fs.sync()
# Alice retrieve the changes
um = alice_user_fs.get_user_manifest()
await alice_user_fs.sync()
um = alice_user_fs.get_user_manifest()
um2 = alice2_user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=alice2.device_id,
timestamp=datetime(2000, 1, 3),
id=alice2.user_manifest_id,
version=2,
created=datetime(2000, 1, 1),
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=(WorkspaceEntry(
name=EntryName("wa"),
id=wid,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
speculative=False,
)
um = _update_user_manifest_key(um)
um2 = _update_user_manifest_key(um2)
assert um == expected_um
assert um2 == expected_um
async def workspace_create(self, name: AnyEntryName) -> EntryID:
"""
Raises: Nothing !
"""
name = EntryName(name)
workspace_entry = WorkspaceEntry.new(name)
workspace_manifest = LocalWorkspaceManifest.new_placeholder(
self.device.device_id, id=workspace_entry.id
)
async with self._update_user_manifest_lock:
user_manifest = self.get_user_manifest()
user_manifest = user_manifest.evolve_workspaces_and_mark_updated(workspace_entry)
await self._create_workspace(workspace_entry.id, workspace_manifest)
await self.set_user_manifest(user_manifest)
self.event_bus.send(CoreEvent.FS_ENTRY_UPDATED, id=self.user_manifest_id)
self.event_bus.send(CoreEvent.FS_WORKSPACE_CREATED, new_entry=workspace_entry)
return workspace_entry.id
async def _transactions_factory(device,
local_storage,
cls=SyncTransactions):
def _get_workspace_entry():
return workspace_entry
async def _get_previous_workspace_entry():
# The tests shouldn't need this yet
assert False
workspace_entry = WorkspaceEntry.new(EntryName("test"),
device.timestamp())
workspace_manifest = LocalWorkspaceManifest.new_placeholder(
device.device_id,
id=workspace_entry.id,
timestamp=datetime(2000, 1, 1))
async with local_storage.lock_entry_id(workspace_entry.id):
await local_storage.set_manifest(workspace_entry.id,
workspace_manifest)
async with backend_authenticated_cmds_factory(
device.organization_addr, device.device_id,
device.signing_key) as cmds:
remote_devices_manager = remote_devices_manager_factory(device)
remote_loader = RemoteLoader(
device,
workspace_entry.id,
_get_workspace_entry,
_get_previous_workspace_entry,
cmds,
remote_devices_manager,
local_storage,
)
yield cls(
workspace_entry.id,
_get_workspace_entry,
device,
local_storage,
remote_loader,
event_bus,
core_config,
)
def __init__(
self,
device: LocalDevice,
path: Path,
backend_cmds: BackendAuthenticatedCmds,
remote_devices_manager: RemoteDevicesManager,
event_bus: EventBus,
):
self.device = device
self.path = path
self.backend_cmds = backend_cmds
self.remote_devices_manager = remote_devices_manager
self.event_bus = event_bus
self.storage = None
# Message processing is done in-order, hence it is pointless to do
# it concurrently
self._workspace_storage_nursery = None
self._process_messages_lock = trio.Lock()
self._update_user_manifest_lock = trio.Lock()
self._workspace_storages = {}
now = pendulum_now()
wentry = WorkspaceEntry(
name="<user manifest>",
id=device.user_manifest_id,
key=device.user_manifest_key,
encryption_revision=1,
encrypted_on=now,
role_cached_on=now,
role=WorkspaceRole.OWNER,
)
self.remote_loader = RemoteLoader(
self.device,
self.device.user_manifest_id,
lambda: wentry,
self.backend_cmds,
self.remote_devices_manager,
# Hack, but fine as long as we only call `load_realm_current_roles`
None,
)
def __init__(
self,
device: LocalDevice,
path: Path,
backend_cmds: BackendAuthenticatedCmds,
remote_devices_manager: RemoteDevicesManager,
event_bus: EventBus,
prevent_sync_pattern: Pattern[str],
):
self.device = device
self.path = path
self.backend_cmds = backend_cmds
self.remote_devices_manager = remote_devices_manager
self.event_bus = event_bus
self.prevent_sync_pattern = prevent_sync_pattern
self.storage: UserStorage # Setup by UserStorage.run factory
# Message processing is done in-order, hence it is pointless to do
# it concurrently
self._workspace_storage_nursery: trio.Nursery # Setup by UserStorage.run factory
self._process_messages_lock = trio.Lock()
self._update_user_manifest_lock = trio.Lock()
self._workspace_storages: Dict[EntryID, WorkspaceFS] = {}
now = pendulum_now()
wentry = WorkspaceEntry(
name="<user manifest>",
id=device.user_manifest_id,
key=device.user_manifest_key,
encryption_revision=1,
encrypted_on=now,
role_cached_on=now,
role=WorkspaceRole.OWNER,
)
self.remote_loader = UserRemoteLoader(
self.device,
self.device.user_manifest_id,
lambda: wentry,
self.backend_cmds,
self.remote_devices_manager,
)
async def test_new_workspace(running_backend, alice, alice_user_fs, alice2_user_fs):
with freeze_time("2000-01-02"):
wid = await alice_user_fs.workspace_create("w")
workspace = alice_user_fs.get_workspace(wid)
with alice_user_fs.event_bus.listen() as spy:
with freeze_time("2000-01-03"):
await workspace.sync()
spy.assert_events_occured(
[("fs.entry.synced", {"workspace_id": wid, "id": wid}, Pendulum(2000, 1, 3))]
)
workspace2 = alice_user_fs.get_workspace(wid)
await alice_user_fs.sync()
await workspace2.sync()
workspace_entry = workspace.get_workspace_entry()
path_info = await workspace.path_info("/")
assert path_info == {
"type": "folder",
"id": wid,
"is_placeholder": False,
"need_sync": False,
"base_version": 1,
"children": [],
"created": Pendulum(2000, 1, 2),
"updated": Pendulum(2000, 1, 2),
}
assert workspace_entry == WorkspaceEntry(
name="w",
id=wid,
key=spy.ANY,
encryption_revision=1,
encrypted_on=Pendulum(2000, 1, 2),
role_cached_on=Pendulum(2000, 1, 2),
role=WorkspaceRole.OWNER,
)
workspace_entry2 = workspace.get_workspace_entry()
path_info2 = await workspace.path_info("/")
assert workspace_entry == workspace_entry2
assert path_info == path_info2
async def test_modify_user_manifest_placeholder(
running_backend,
backend_data_binder,
local_device_factory,
user_fs_factory,
data_base_dir,
initialize_local_user_manifest,
):
device = local_device_factory()
await backend_data_binder.bind_device(device,
initial_user_manifest="not_synced")
await initialize_local_user_manifest(
data_base_dir, device, initial_user_manifest="non_speculative_v0")
async with user_fs_factory(device) as user_fs:
um_v0 = user_fs.get_user_manifest()
with freeze_time("2000-01-02"):
wid = await user_fs.workspace_create(EntryName("w1"))
um = user_fs.get_user_manifest()
um = _update_user_manifest_key(um)
expected_um = um_v0.evolve(
updated=datetime(2000, 1, 2),
workspaces=(WorkspaceEntry(
name=EntryName("w1"),
id=wid,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
assert um == expected_um
# Make sure we can fetch back data from the database on user_fs restart
async with user_fs_factory(device) as user_fs2:
um2 = user_fs2.get_user_manifest()
um2 = _update_user_manifest_key(um2)
assert um2 == expected_um
async def test_new_sharing_trigger_event(alice_core, bob_core, running_backend):
KEY = SecretKey.generate()
# First, create a folder and sync it on backend
with freeze_time("2000-01-01"):
wid = await alice_core.user_fs.workspace_create(EntryName("foo"))
workspace = alice_core.user_fs.get_workspace(wid)
with freeze_time("2000-01-02"):
await workspace.sync()
# Now we can share this workspace with Bob
with bob_core.event_bus.listen() as spy:
with freeze_time("2000-01-03"):
await alice_core.user_fs.workspace_share(
wid, recipient=UserID("bob"), role=WorkspaceRole.MANAGER
)
def _update_event(event):
if event.event == CoreEvent.SHARING_UPDATED:
event.kwargs["new_entry"] = event.kwargs["new_entry"].evolve(
key=KEY, role_cached_on=datetime(2000, 1, 1)
)
return event
# Bob should get a notification
await spy.wait_with_timeout(
CoreEvent.SHARING_UPDATED,
{
"new_entry": WorkspaceEntry(
name=EntryName("foo"),
id=wid,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=datetime(2000, 1, 1),
role=WorkspaceRole.MANAGER,
),
"previous_entry": None,
},
update_event_func=_update_event,
)
async def _transactions_factory(device,
backend_cmds,
local_storage,
cls=SyncTransactions):
def _get_workspace_entry():
return workspace_entry
async def _get_previous_workspace_entry():
# The tests shouldn't need this yet
assert False
workspace_entry = WorkspaceEntry.new("test")
workspace_manifest = LocalWorkspaceManifest.new_placeholder(
device.device_id, id=workspace_entry.id, now=datetime(2000, 1, 1))
async with local_storage.lock_entry_id(workspace_entry.id):
await local_storage.set_manifest(workspace_entry.id,
workspace_manifest)
remote_devices_manager = remote_devices_manager_factory(device)
remote_loader = RemoteLoader(
device,
workspace_entry.id,
_get_workspace_entry,
_get_previous_workspace_entry,
backend_cmds,
remote_devices_manager,
local_storage,
)
return cls(
workspace_entry.id,
_get_workspace_entry,
device,
local_storage,
remote_loader,
event_bus,
)
async def _process_message_sharing_granted(
self, msg: Union[SharingRevokedMessageContent, SharingReencryptedMessageContent]
):
"""
Raises:
FSError
FSBackendOfflineError
FSSharingNotAllowedError
"""
# We cannot blindly trust the message sender ! Hence we first
# interrogate the backend to make sure he is a workspace manager/owner.
# Note this means we refuse to process messages from a former-manager,
# even if the message was sent at a time the user was manager (in such
# case the user can still ask for another manager to re-do the sharing
# so it's no big deal).
try:
roles = await self.remote_loader.load_realm_current_roles(msg.id)
except FSWorkspaceNoAccess:
# Seems we lost the access roles anyway, nothing to do then
return
if roles.get(msg.author.user_id, None) not in (WorkspaceRole.OWNER, WorkspaceRole.MANAGER):
raise FSSharingNotAllowedError(
f"User {msg.author.user_id} cannot share workspace `{msg.id}`"
" with us (requires owner or manager right)"
)
# Determine the access roles we have been given to
self_role = roles.get(self.device.user_id)
# Finally insert the new workspace entry into our user manifest
workspace_entry = WorkspaceEntry(
# Name are not required to be unique across workspaces, so no check to do here
name=f"{msg.name} (shared by {msg.author.user_id})",
id=msg.id,
key=msg.key,
encryption_revision=msg.encryption_revision,
encrypted_on=msg.encrypted_on,
role=self_role,
role_cached_on=pendulum_now(),
)
async with self._update_user_manifest_lock:
user_manifest = self.get_user_manifest()
# Check if we already know this workspace
already_existing_entry = user_manifest.get_workspace_entry(msg.id)
if already_existing_entry:
# Merge with existing as target to keep possible workpace rename
workspace_entry = merge_workspace_entry(
None, workspace_entry, already_existing_entry
)
user_manifest = user_manifest.evolve_workspaces_and_mark_updated(workspace_entry)
await self.set_user_manifest(user_manifest)
self.event_bus.send("userfs.updated")
if not already_existing_entry:
# TODO: remove this event ?
self.event_bus.send("fs.entry.synced", id=workspace_entry.id)
self.event_bus.send(
"sharing.updated", new_entry=workspace_entry, previous_entry=already_existing_entry
)
async def test_concurrent_sync_placeholder(
running_backend,
backend_data_binder,
local_device_factory,
user_fs_factory,
data_base_dir,
initialize_local_user_manifest,
dev2_has_changes,
):
device1 = local_device_factory("a@1")
await backend_data_binder.bind_device(device1,
initial_user_manifest="not_synced")
await initialize_local_user_manifest(
data_base_dir, device1, initial_user_manifest="non_speculative_v0")
device2 = local_device_factory("a@2")
await backend_data_binder.bind_device(device2)
await initialize_local_user_manifest(
data_base_dir, device2, initial_user_manifest="speculative_v0")
async with user_fs_factory(device1) as user_fs1, user_fs_factory(
device2) as user_fs2:
# fs2's created value is different and will be overwritten when
# merging synced manifest from fs1
um_created_v0_fs1 = user_fs1.get_user_manifest().created
with freeze_time("2000-01-01"):
# Sync user manifests now to avoid extra milliseconds from restamping
await user_fs1.sync()
await user_fs2.sync()
w1id = await user_fs1.workspace_create(EntryName("w1"))
if dev2_has_changes:
with freeze_time("2000-01-02"):
w2id = await user_fs2.workspace_create(EntryName("w2"))
with freeze_time("2000-01-03"):
await user_fs1.sync()
with freeze_time("2000-01-04"):
await user_fs2.sync()
if dev2_has_changes:
with freeze_time("2000-01-05"):
await user_fs1.sync()
um1 = user_fs1.get_user_manifest()
um2 = user_fs2.get_user_manifest()
if dev2_has_changes:
expected_base_um = UserManifest(
author=device2.device_id,
id=device2.user_manifest_id,
timestamp=datetime(2000, 1, 4),
version=3,
created=um_created_v0_fs1,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=(
WorkspaceEntry(
name=EntryName("w1"),
id=w1id,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=datetime(2000, 1, 1),
role=WorkspaceRole.OWNER,
),
WorkspaceEntry(
name=EntryName("w2"),
id=w2id,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
),
),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
speculative=False,
)
else:
expected_base_um = UserManifest(
author=device1.device_id,
timestamp=datetime(2000, 1, 3),
id=device1.user_manifest_id,
version=2,
created=um_created_v0_fs1,
updated=datetime(2000, 1, 1),
last_processed_message=0,
workspaces=(WorkspaceEntry(
name=EntryName("w1"),
id=w1id,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=datetime(2000, 1, 1),
role=WorkspaceRole.OWNER,
), ),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 1),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
speculative=False,
)
um1 = _update_user_manifest_key(um1)
um2 = _update_user_manifest_key(um2)
assert um1 == expected_um
assert um2 == expected_um
async def test_new_reencryption_trigger_event(alice_core, bob_core,
running_backend):
with freeze_time("2000-01-02"):
wid = await create_shared_workspace("w", alice_core, bob_core)
with alice_core.event_bus.listen() as aspy, bob_core.event_bus.listen(
) as bspy:
with freeze_time("2000-01-03"):
await alice_core.user_fs.workspace_start_reencryption(wid)
# Each workspace participant should get the message
await aspy.wait_with_timeout(
CoreEvent.SHARING_UPDATED,
{
"new_entry":
WorkspaceEntry(
name="w",
id=wid,
key=ANY,
encryption_revision=2,
encrypted_on=datetime(2000, 1, 3),
role_cached_on=ANY,
role=WorkspaceRole.OWNER,
),
"previous_entry":
WorkspaceEntry(
name="w",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=ANY,
role=WorkspaceRole.OWNER,
),
},
)
await bspy.wait_with_timeout(
CoreEvent.SHARING_UPDATED,
{
"new_entry":
WorkspaceEntry(
name="w",
id=wid,
key=ANY,
encryption_revision=2,
encrypted_on=datetime(2000, 1, 3),
role_cached_on=ANY,
role=WorkspaceRole.MANAGER,
),
"previous_entry":
WorkspaceEntry(
name="w",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=ANY,
role=WorkspaceRole.MANAGER,
),
},
)
async def test_concurrent_sync_placeholder(running_backend,
backend_data_binder,
local_device_factory,
user_fs_factory, dev2_has_changes):
device1 = local_device_factory("a@1")
await backend_data_binder.bind_device(device1,
initial_user_manifest_in_v0=True)
device2 = local_device_factory("a@2")
await backend_data_binder.bind_device(device2,
initial_user_manifest_in_v0=True)
async with user_fs_factory(
device1, initialize_in_v0=True) as user_fs1, user_fs_factory(
device2, initialize_in_v0=True) as user_fs2:
# fs2's created value is different and will be overwritten when
# merging synced manifest from fs1
um_created_v0_fs1 = user_fs1.get_user_manifest().created
with freeze_time("2000-01-01"):
w1id = await user_fs1.workspace_create("w1")
if dev2_has_changes:
with freeze_time("2000-01-02"):
w2id = await user_fs2.workspace_create("w2")
with freeze_time("2000-01-03"):
await user_fs1.sync()
with freeze_time("2000-01-04"):
await user_fs2.sync()
if dev2_has_changes:
with freeze_time("2000-01-05"):
await user_fs1.sync()
um1 = user_fs1.get_user_manifest()
um2 = user_fs2.get_user_manifest()
if dev2_has_changes:
expected_base_um = UserManifest(
author=device2.device_id,
id=device2.user_manifest_id,
timestamp=datetime(2000, 1, 4),
version=2,
created=um_created_v0_fs1,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=(
WorkspaceEntry(
name="w1",
id=w1id,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=datetime(2000, 1, 1),
role=WorkspaceRole.OWNER,
),
WorkspaceEntry(
name="w2",
id=w2id,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
),
),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
)
else:
expected_base_um = UserManifest(
author=device1.device_id,
timestamp=datetime(2000, 1, 3),
id=device1.user_manifest_id,
version=1,
created=um_created_v0_fs1,
updated=datetime(2000, 1, 1),
last_processed_message=0,
workspaces=(WorkspaceEntry(
name="w1",
id=w1id,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=datetime(2000, 1, 1),
role=WorkspaceRole.OWNER,
), ),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 1),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
)
assert um1 == expected_um
assert um2 == expected_um
async def test_sync_placeholder(
running_backend,
backend_data_binder,
local_device_factory,
user_fs_factory,
data_base_dir,
initialize_local_user_manifest,
with_workspace,
initial_user_manifest,
):
device = local_device_factory()
await backend_data_binder.bind_device(device,
initial_user_manifest="not_synced")
await initialize_local_user_manifest(
data_base_dir, device, initial_user_manifest=initial_user_manifest)
async with user_fs_factory(device) as user_fs:
um_v0 = user_fs.get_user_manifest()
expected_um = LocalUserManifest.new_placeholder(
device.device_id,
id=device.user_manifest_id,
timestamp=um_v0.created,
speculative=(initial_user_manifest == "speculative_v0"),
)
assert um_v0 == expected_um
if with_workspace:
with freeze_time("2000-01-02"):
wid = await user_fs.workspace_create(EntryName("w1"))
um = user_fs.get_user_manifest()
expected_um = um_v0.evolve(
updated=datetime(2000, 1, 2),
workspaces=(WorkspaceEntry(
name=EntryName("w1"),
id=wid,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
um = _update_user_manifest_key(um)
assert um == expected_um
with freeze_time("2000-01-02"):
await user_fs.sync()
um = user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=device.device_id,
# Add extra time due to the user realm being already created at 2000-01-02
timestamp=datetime(2000, 1,
2).add(microseconds=MANIFEST_STAMP_AHEAD_US),
id=device.user_manifest_id,
version=1,
created=expected_um.created,
updated=expected_um.updated,
last_processed_message=0,
workspaces=expected_um.workspaces,
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=expected_um.updated,
last_processed_message=0,
workspaces=expected_base_um.workspaces,
speculative=False,
)
um = _update_user_manifest_key(um)
assert um.base == expected_base_um
assert um == expected_um
