def test_merge_local_user_manifest_no_changes_in_diverged_placeholder(
gen_date, alice, alice2, with_ignored_changes
):
w1 = WorkspaceEntry.new(name="w1")
d1, d2, d3, d4 = [gen_date() for _ in range(4)]
base = UserManifest(
author=alice.device_id,
timestamp=d2,
id=alice.user_manifest_id,
version=1,
created=d1,
updated=d2,
last_processed_message=0,
workspaces=(w1,),
)
diverged = LocalUserManifest.from_remote(base)
if with_ignored_changes:
diverged = diverged.evolve(updated=d4, need_sync=True)
target = UserManifest(
author=alice2.device_id,
timestamp=d2,
id=alice2.user_manifest_id,
version=2,
created=d1,
updated=d3,
last_processed_message=0,
workspaces=(w1,),
)
expected_merged = LocalUserManifest.from_remote(target)
merged = merge_local_user_manifests(diverged, target)
assert merged == expected_merged
def test_merge_local_user_manifest_changes_placeholder(gen_date, alice, speculative_placeholder):
d1, d2, d3, d4 = [gen_date() for _ in range(4)]
w1 = WorkspaceEntry.new(name=EntryName("w1"), timestamp=d2)
w2 = WorkspaceEntry.new(name=EntryName("w2"), timestamp=d2)
w3 = WorkspaceEntry.new(name=EntryName("w3"), timestamp=d2)
diverged = LocalUserManifest.new_placeholder(
alice.device_id,
id=alice.user_manifest_id,
timestamp=d4,
speculative=speculative_placeholder,
).evolve(last_processed_message=30, workspaces=(w1, w3))
target = UserManifest(
author=alice.device_id,
timestamp=d2,
id=alice.user_manifest_id,
version=3,
created=d1,
updated=d3,
last_processed_message=20,
workspaces=(w1, w2),
)
expected_merged = LocalUserManifest(
base=target,
updated=d4,
last_processed_message=30,
workspaces=(w1, w2, w3),
need_sync=True,
speculative=False,
)
merged = merge_local_user_manifests(diverged, target)
assert merged == expected_merged
async def test_sync_placeholder(
running_backend, backend_data_binder, local_device_factory, user_fs_factory, with_workspace
):
device = local_device_factory()
await backend_data_binder.bind_device(device, initial_user_manifest_in_v0=True)
async with user_fs_factory(device, initialize_in_v0=True) as user_fs:
um_v0 = user_fs.get_user_manifest()
expected_um = LocalUserManifest.new_placeholder(
id=device.user_manifest_id, now=um_v0.created
)
assert um_v0 == expected_um
if with_workspace:
with freeze_time("2000-01-02"):
wid = await user_fs.workspace_create("w1")
um = user_fs.get_user_manifest()
expected_um = um_v0.evolve(
updated=Pendulum(2000, 1, 2),
workspaces=(
WorkspaceEntry(
name="w1",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=Pendulum(2000, 1, 2),
role_cached_on=Pendulum(2000, 1, 2),
role=WorkspaceRole.OWNER,
),
),
)
assert um == expected_um
with freeze_time("2000-01-02"):
await user_fs.sync()
um = user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=device.device_id,
timestamp=Pendulum(2000, 1, 2),
id=device.user_manifest_id,
version=1,
created=expected_um.created,
updated=expected_um.updated,
last_processed_message=0,
workspaces=expected_um.workspaces,
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=expected_um.updated,
last_processed_message=0,
workspaces=expected_base_um.workspaces,
)
assert um == expected_um
def test_created_field_modified_by_remote(gen_date, alice, with_local_changes):
d1, d2, d3, d4 = [gen_date() for _ in range(4)]
w1 = WorkspaceEntry.new(name=EntryName("w1"), timestamp=d2)
base = UserManifest(
author=alice.device_id,
timestamp=d2,
id=alice.user_manifest_id,
version=1,
created=d1,
updated=d2,
last_processed_message=0,
workspaces=(w1,),
)
local = LocalUserManifest.from_remote(base)
if with_local_changes:
w2 = WorkspaceEntry.new(name=EntryName("w1"), timestamp=d3)
local = local.evolve(
need_sync=True, updated=d3, last_processed_message=1, workspaces=(w1, w2)
)
target = base.evolve(created=d4, version=2)
expected_merged = local.evolve(base=target)
merged = merge_local_user_manifests(local, target)
# Remote always control the value of the create field
assert merged == expected_merged
async def test_rename_workspace(initial_user_manifest_state, alice_user_fs,
alice):
with freeze_time("2000-01-02"):
wid = await alice_user_fs.workspace_create("w1")
with freeze_time("2000-01-03"):
await alice_user_fs.workspace_rename(wid, "w2")
um = alice_user_fs.get_user_manifest()
expected_base_um = initial_user_manifest_state.get_user_manifest_v1_for_backend(
alice)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=True,
updated=datetime(2000, 1, 3),
last_processed_message=expected_base_um.last_processed_message,
workspaces=(WorkspaceEntry(
name="w2",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
assert um == expected_um
async def _initialize_local_user_manifest(
data_base_dir, device, initial_user_manifest: str
) -> None:
assert initial_user_manifest in ("non_speculative_v0", "speculative_v0", "v1")
# Create a storage just for this operation (the underlying database
# will be reused by the core's storage thanks to `persistent_mockup`)
with freeze_time("2000-01-01", device=device) as timestamp:
async with UserStorage.run(data_base_dir, device) as storage:
assert storage.get_user_manifest().base_version == 0
if initial_user_manifest == "v1":
user_manifest = initial_user_manifest_state.get_user_manifest_v1_for_device(
storage.device
)
await storage.set_user_manifest(user_manifest)
# Chcekpoint 1 *is* the upload of user manifest v1
await storage.update_realm_checkpoint(1, {})
elif initial_user_manifest == "non_speculative_v0":
user_manifest = LocalUserManifest.new_placeholder(
author=storage.device.device_id,
id=storage.device.user_manifest_id,
timestamp=timestamp,
speculative=False,
)
await storage.set_user_manifest(user_manifest)
else:
# Nothing to do given speculative placeholder is the default
assert initial_user_manifest == "speculative_v0"
async def test_sync(running_backend, alice2_user_fs, alice2):
with freeze_time("2000-01-02"):
wid = await alice2_user_fs.workspace_create("w1")
with freeze_time("2000-01-03"):
await alice2_user_fs.sync()
um = alice2_user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=alice2.device_id,
timestamp=datetime(2000, 1, 3),
id=alice2.user_manifest_id,
version=2,
created=datetime(2000, 1, 1),
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=(WorkspaceEntry(
name="w1",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
expected_um = LocalUserManifest.from_remote(expected_base_um)
assert um == expected_um
async def test_create_workspace(initial_user_manifest_state, alice_user_fs,
alice):
with freeze_time("2000-01-02"):
wid = await alice_user_fs.workspace_create("w1")
um = alice_user_fs.get_user_manifest()
expected_base_um = initial_user_manifest_state.get_user_manifest_v1_for_backend(
alice)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=True,
updated=datetime(2000, 1, 2),
last_processed_message=expected_base_um.last_processed_message,
workspaces=(WorkspaceEntry(
name="w1",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
assert um == expected_um
w_manifest = await alice_user_fs.get_workspace(
wid).local_storage.get_manifest(wid)
expected_w_manifest = LocalWorkspaceManifest.new_placeholder(
alice.device_id, id=w_manifest.id, now=datetime(2000, 1, 2))
assert w_manifest == expected_w_manifest
def test_merge_local_user_manifest_no_changes_in_diverged_placeholder(
gen_date, alice, alice2, with_ignored_changes
):
d1, d2, d3, d4, d5, d6, d7 = [gen_date() for _ in range(7)]
w1 = WorkspaceEntry.new(name=EntryName("w1"), timestamp=d2)
base = UserManifest(
author=alice.device_id,
timestamp=d4,
id=alice.user_manifest_id,
version=1,
created=d1,
updated=d3,
last_processed_message=0,
workspaces=(w1,),
)
diverged = LocalUserManifest.from_remote(base)
if with_ignored_changes:
w1_bis = WorkspaceEntry(
name=w1.name,
id=w1.id,
key=w1.key,
# Same encryption revision than remote (so encryption date should be ignored)
encryption_revision=1,
encrypted_on=d5,
# Cache older than remote
role_cached_on=d1,
role=RealmRole.MANAGER,
)
diverged = diverged.evolve(updated=d4, need_sync=True, workspaces=(w1_bis,))
target = UserManifest(
author=alice2.device_id,
timestamp=d7,
id=alice2.user_manifest_id,
version=2,
created=d1,
updated=d6,
last_processed_message=0,
workspaces=(w1,),
)
expected_merged = LocalUserManifest.from_remote(target)
merged = merge_local_user_manifests(diverged, target)
assert merged == expected_merged
def create_manifest(device, type=LocalWorkspaceManifest):
if type is LocalUserManifest:
manifest = LocalUserManifest.new_placeholder(parent=EntryID())
elif type is LocalWorkspaceManifest:
manifest = type.new_placeholder()
else:
manifest = type.new_placeholder(parent=EntryID())
return manifest
def test_merge_speculative_with_it_unsuspected_former_self(alice, local_changes):
d1 = datetime(2000, 1, 1)
d2 = datetime(2000, 1, 2)
d3 = datetime(2000, 1, 3)
# 1) User manifest is originally created by our device
local = LocalUserManifest.new_placeholder(
author=alice.device_id, id=alice.user_manifest_id, timestamp=d1, speculative=False
)
w1 = WorkspaceEntry.new(EntryName("foo"), timestamp=d1)
local = local.evolve(workspaces=(w1,), last_processed_message=1)
# 2) We sync the user manifest
v1 = local.to_remote(author=alice.device_id, timestamp=d2)
# 3) Now let's pretend we lost local storage, hence creating a new speculative manifest
new_local = LocalUserManifest.new_placeholder(
author=alice.device_id, id=alice.user_manifest_id, timestamp=d3, speculative=True
)
if local_changes:
w2 = WorkspaceEntry.new(EntryName("bar"), timestamp=d3)
new_local = new_local.evolve(workspaces=(w2,), last_processed_message=2)
# 4) When syncing the manifest, we shouldn't remove any data from the remote
merged = merge_local_user_manifests(new_local, v1)
if local_changes:
assert merged == LocalUserManifest(
base=v1,
updated=d3,
last_processed_message=2,
workspaces=(w2, w1),
need_sync=True,
speculative=False,
)
else:
assert merged == LocalUserManifest(
base=v1,
updated=v1.updated,
last_processed_message=1,
workspaces=(w1,),
need_sync=False,
speculative=False,
)
async def test_sync_under_concurrency(running_backend, alice_user_fs,
alice2_user_fs, alice, alice2):
with freeze_time("2000-01-02"):
waid = await alice_user_fs.workspace_create(EntryName("wa"))
with freeze_time("2000-01-03"):
wa2id = await alice2_user_fs.workspace_create(EntryName("wa2"))
with freeze_time("2000-01-04"):
await alice_user_fs.sync()
with freeze_time("2000-01-05"):
await alice2_user_fs.sync()
# Fetch back alice2's changes
with freeze_time("2000-01-06"):
await alice_user_fs.sync()
um = alice_user_fs.get_user_manifest()
um2 = alice2_user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=alice2.device_id,
timestamp=datetime(2000, 1, 5),
id=alice2.user_manifest_id,
version=3,
created=datetime(2000, 1, 1),
updated=datetime(2000, 1, 3),
last_processed_message=0,
workspaces=(
WorkspaceEntry(
name=EntryName("wa"),
id=waid,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
),
WorkspaceEntry(
name=EntryName("wa2"),
id=wa2id,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 3),
role_cached_on=datetime(2000, 1, 3),
role=WorkspaceRole.OWNER,
),
),
)
expected_um = LocalUserManifest.from_remote(expected_base_um)
um = _update_user_manifest_key(um)
um2 = _update_user_manifest_key(um2)
assert um == expected_um
assert um2 == expected_um
def create_manifest(device, type=LocalWorkspaceManifest, use_legacy_none_author=False):
author = device.device_id
if type is LocalUserManifest:
manifest = LocalUserManifest.new_placeholder(author)
elif type is LocalWorkspaceManifest:
manifest = type.new_placeholder(author)
else:
manifest = type.new_placeholder(author, parent=EntryID())
if use_legacy_none_author:
base = manifest.base.evolve(author=None)
manifest = manifest.evolve(base=base)
return manifest
async def _load_user_manifest(self) -> None:
try:
await self.manifest_storage.get_manifest(self.user_manifest_id)
except FSLocalMissError:
# In the unlikely event the user manifest is not present in
# local (e.g. device just created or during tests), we fall
# back on an empty manifest which is a good aproximation of
# the very first version of the manifest (field `created` is
# invalid, but it will be corrected by the merge during sync).
manifest = LocalUserManifest.new_placeholder(
self.device.device_id, id=self.device.user_manifest_id)
await self.manifest_storage.set_manifest(self.user_manifest_id,
manifest)
async def test_sync_remote_changes(running_backend, alice_user_fs,
alice2_user_fs, alice, alice2):
# Alice 2 update the user manifest
with freeze_time("2000-01-02"):
wid = await alice2_user_fs.workspace_create(EntryName("wa"))
with freeze_time("2000-01-03"):
await alice2_user_fs.sync()
# Alice retrieve the changes
um = alice_user_fs.get_user_manifest()
await alice_user_fs.sync()
um = alice_user_fs.get_user_manifest()
um2 = alice2_user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=alice2.device_id,
timestamp=datetime(2000, 1, 3),
id=alice2.user_manifest_id,
version=2,
created=datetime(2000, 1, 1),
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=(WorkspaceEntry(
name=EntryName("wa"),
id=wid,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
speculative=False,
)
um = _update_user_manifest_key(um)
um2 = _update_user_manifest_key(um2)
assert um == expected_um
assert um2 == expected_um
async def user_storage_non_speculative_init(data_base_dir: Path,
device: LocalDevice) -> None:
data_path = get_user_data_storage_db_path(data_base_dir, device)
# Local data storage service
async with LocalDatabase.run(data_path) as localdb:
# Manifest storage service
async with ManifestStorage.run(
device, localdb, device.user_manifest_id) as manifest_storage:
timestamp = device.timestamp()
manifest = LocalUserManifest.new_placeholder(
author=device.device_id,
id=device.user_manifest_id,
timestamp=timestamp,
speculative=False,
)
await manifest_storage.set_manifest(device.user_manifest_id,
manifest)
async def _load_user_manifest(self) -> None:
try:
await self.manifest_storage.get_manifest(self.user_manifest_id)
except FSLocalMissError:
# It is possible to lack the user manifest in local if our
# device hasn't tried to access it yet (and we are not the
# initial device of our user, in which case the user local db is
# initialized with a non-speculative local manifest placeholder).
# In such case it is easy to fall back on an empty manifest
# which is a good enough aproximation of the very first version
# of the manifest (field `created` is invalid, but it will be
# correction by the merge during sync).
timestamp = self.device.timestamp()
manifest = LocalUserManifest.new_placeholder(
self.device.device_id,
id=self.device.user_manifest_id,
timestamp=timestamp,
speculative=True,
)
await self.manifest_storage.set_manifest(self.user_manifest_id,
manifest)
def merge_local_user_manifests(diverged: LocalUserManifest,
target: UserManifest) -> LocalUserManifest:
assert isinstance(diverged, LocalUserManifest)
assert isinstance(target, UserManifest)
assert diverged.id == target.id
base_version = diverged.base_version
base_workspaces = diverged.base.workspaces if diverged.base is not None else None
assert target.version > base_version
# `created` should never change, so in theory we should have
# `diverged.created == target.created`, but there is no strict guarantee
# (e.g. remote manifest may have been uploaded by a buggy client) so
# we have no choice but to accept whatever value remote provides.
workspaces, need_sync = merge_workspace_entries(base_workspaces,
diverged.workspaces,
target.workspaces)
last_processed_message = max(diverged.last_processed_message,
target.last_processed_message)
need_sync = need_sync or last_processed_message != target.last_processed_message
if not need_sync:
updated = target.updated
else:
if target.updated > diverged.updated:
updated = target.updated
else:
updated = diverged.updated
return LocalUserManifest(
base=target,
need_sync=need_sync,
updated=updated,
last_processed_message=last_processed_message,
workspaces=workspaces,
speculative=False,
)
def _generate_or_retrieve_user_manifest_v1(self, device):
try:
return self._v1[(device.organization_id, device.user_id)]
except KeyError:
timestamp = device.timestamp()
remote_user_manifest = UserManifest(
author=device.device_id,
timestamp=timestamp,
id=device.user_manifest_id,
version=1,
created=timestamp,
updated=timestamp,
last_processed_message=0,
workspaces=(),
)
local_user_manifest = LocalUserManifest.from_remote(remote_user_manifest)
self._v1[(device.organization_id, device.user_id)] = (
remote_user_manifest,
local_user_manifest,
)
return self._v1[(device.organization_id, device.user_id)]
def _generate_or_retrieve_user_manifest_v1(self, device):
try:
return self._v1[device.user_id]
except KeyError:
now = pendulum.now()
remote_user_manifest = UserManifest(
author=device.device_id,
timestamp=now,
id=device.user_manifest_id,
version=1,
created=now,
updated=now,
last_processed_message=0,
workspaces=(),
)
local_user_manifest = LocalUserManifest.from_remote(
remote_user_manifest)
self._v1[device.user_id] = (remote_user_manifest,
local_user_manifest)
return self._v1[device.user_id]
def _get_manifest_read_only(self, access: Access) -> LocalManifest:
try:
return self._manifests_cache[access.id]
except KeyError:
pass
try:
raw = self._local_db.get(access)
except LocalDBMissingEntry as exc:
# Last chance: if we are looking for the user manifest, we can
# fake to know it version 0, which is useful during boostrap step
if access == self.root_access:
manifest = LocalUserManifest(self.local_author)
else:
raise FSManifestLocalMiss(access) from exc
else:
manifest = local_manifest_serializer.loads(raw)
self._manifests_cache[access.id] = manifest
# TODO: shouldn't be processed in multiple places like this...
if is_workspace_manifest(manifest):
path, *_ = self.get_entry_path(access.id)
self.event_bus.send("fs.workspace.loaded",
path=str(path),
id=access.id)
return manifest
def merge_local_user_manifests(diverged: LocalUserManifest,
target: UserManifest) -> LocalUserManifest:
assert isinstance(diverged, LocalUserManifest)
assert isinstance(target, UserManifest)
assert diverged.id == target.id
base_version = diverged.base_version
base_workspaces = diverged.base.workspaces if diverged.base is not None else None
assert target.version > base_version
# Not true when merging user manifest v1 given v0 is lazily generated
assert base_version == 0 or diverged.created == target.created
workspaces, need_sync = merge_workspace_entries(base_workspaces,
diverged.workspaces,
target.workspaces)
last_processed_message = max(diverged.last_processed_message,
target.last_processed_message)
need_sync = need_sync or last_processed_message != target.last_processed_message
if not need_sync:
updated = target.updated
else:
if target.updated > diverged.updated:
updated = target.updated
else:
updated = diverged.updated
return LocalUserManifest(
base=target,
need_sync=need_sync,
updated=updated,
last_processed_message=last_processed_message,
workspaces=workspaces,
)
async def test_concurrent_sync_placeholder(running_backend,
backend_data_binder,
local_device_factory,
user_fs_factory, dev2_has_changes):
device1 = local_device_factory("a@1")
await backend_data_binder.bind_device(device1,
initial_user_manifest_in_v0=True)
device2 = local_device_factory("a@2")
await backend_data_binder.bind_device(device2,
initial_user_manifest_in_v0=True)
async with user_fs_factory(
device1, initialize_in_v0=True) as user_fs1, user_fs_factory(
device2, initialize_in_v0=True) as user_fs2:
# fs2's created value is different and will be overwritten when
# merging synced manifest from fs1
um_created_v0_fs1 = user_fs1.get_user_manifest().created
with freeze_time("2000-01-01"):
w1id = await user_fs1.workspace_create("w1")
if dev2_has_changes:
with freeze_time("2000-01-02"):
w2id = await user_fs2.workspace_create("w2")
with freeze_time("2000-01-03"):
await user_fs1.sync()
with freeze_time("2000-01-04"):
await user_fs2.sync()
if dev2_has_changes:
with freeze_time("2000-01-05"):
await user_fs1.sync()
um1 = user_fs1.get_user_manifest()
um2 = user_fs2.get_user_manifest()
if dev2_has_changes:
expected_base_um = UserManifest(
author=device2.device_id,
id=device2.user_manifest_id,
timestamp=datetime(2000, 1, 4),
version=2,
created=um_created_v0_fs1,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=(
WorkspaceEntry(
name="w1",
id=w1id,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=datetime(2000, 1, 1),
role=WorkspaceRole.OWNER,
),
WorkspaceEntry(
name="w2",
id=w2id,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
),
),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
)
else:
expected_base_um = UserManifest(
author=device1.device_id,
timestamp=datetime(2000, 1, 3),
id=device1.user_manifest_id,
version=1,
created=um_created_v0_fs1,
updated=datetime(2000, 1, 1),
last_processed_message=0,
workspaces=(WorkspaceEntry(
name="w1",
id=w1id,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=datetime(2000, 1, 1),
role=WorkspaceRole.OWNER,
), ),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 1),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
)
assert um1 == expected_um
assert um2 == expected_um
def user_manifest(alice):
return LocalUserManifest.new_placeholder(alice.device_id,
id=alice.user_manifest_id)
async def test_share_workspace_then_conflict_on_rights(
running_backend, alice_user_fs, alice2_user_fs, bob_user_fs, alice, alice2, bob, first_to_sync
):
# Bob shares a workspace with Alice...
with freeze_time("2000-01-01"):
wid = await bob_user_fs.workspace_create("w")
with freeze_time("2000-01-02"):
await bob_user_fs.workspace_share(wid, alice.user_id, WorkspaceRole.MANAGER)
# ...but only Alice's first device get the information
with freeze_time("2000-01-03"):
await alice_user_fs.process_last_messages()
# Now Bob change the sharing rights...
with freeze_time("2000-01-04"):
await bob_user_fs.workspace_share(wid, alice.user_id, WorkspaceRole.CONTRIBUTOR)
# ...this time it's Alice's second device which get the info
with freeze_time("2000-01-05"):
# Note we will process the 2 sharing messages bob sent us, this
# will attribute role_cached_on to the first message timestamp even
# if we cache the second message role...
await alice2_user_fs.process_last_messages()
if first_to_sync == "alice":
first = alice_user_fs
second = alice2_user_fs
synced_timestamp = datetime(2000, 1, 7)
synced_version = 3
else:
first = alice2_user_fs
second = alice_user_fs
synced_timestamp = datetime(2000, 1, 6)
synced_version = 2
# Finally Alice devices try to reconciliate
with freeze_time("2000-01-06"):
await first.sync()
with freeze_time("2000-01-07"):
await second.sync()
# Resync first device to get changes from the 2nd
with freeze_time("2000-01-08"):
await first.sync()
am = alice_user_fs.get_user_manifest()
a2m = alice2_user_fs.get_user_manifest()
expected_remote = UserManifest(
author=alice2.device_id,
timestamp=synced_timestamp,
id=alice2.user_manifest_id,
version=synced_version,
created=datetime(2000, 1, 1),
updated=datetime(2000, 1, 5),
last_processed_message=2,
workspaces=(
WorkspaceEntry(
name="w",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=datetime(2000, 1, 5),
role=WorkspaceRole.CONTRIBUTOR,
),
),
)
expected = LocalUserManifest(
base=expected_remote,
need_sync=False,
updated=expected_remote.updated,
last_processed_message=expected_remote.last_processed_message,
workspaces=expected_remote.workspaces,
)
assert am == expected
assert a2m == expected
a_w = alice_user_fs.get_workspace(wid)
a2_w = alice2_user_fs.get_workspace(wid)
a_w_stat = await a_w.path_info("/")
a2_w_stat = await a2_w.path_info("/")
a_w_entry = a_w.get_workspace_entry()
a2_w_entry = a2_w.get_workspace_entry()
assert a_w_stat == {
"type": "folder",
"is_placeholder": False,
"id": wid,
"created": ANY,
"updated": ANY,
"base_version": 1,
"need_sync": False,
"children": [],
"confinement_point": None,
}
assert a_w_stat == a2_w_stat
assert a_w_entry == WorkspaceEntry(
name=f"w",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=datetime(2000, 1, 5),
role=WorkspaceRole.CONTRIBUTOR,
)
assert a2_w_entry == a_w_entry
async def test_concurrent_devices_agree_on_user_manifest(
running_backend,
backend_data_binder,
data_base_dir,
user_fs_factory,
coolorg,
alice,
alice2,
with_speculative,
):
KEY = SecretKey.generate()
async def _switch_running_backend_offline(task_status):
should_switch_online = trio.Event()
backend_online = trio.Event()
async def _switch_backend_online():
should_switch_online.set()
await backend_online.wait()
with running_backend.offline():
task_status.started(_switch_backend_online)
await should_switch_online.wait()
backend_online.set()
# I call this "diagonal programming"...
async with trio.open_nursery() as nursery:
switch_back_online = await nursery.start(
_switch_running_backend_offline)
with freeze_time("2000-01-01"):
if with_speculative != "both":
await user_storage_non_speculative_init(
data_base_dir=data_base_dir, device=alice)
async with user_fs_factory(
alice, data_base_dir=data_base_dir) as user_fs1:
wksp1_id = await user_fs1.workspace_create(EntryName("wksp1"))
with freeze_time("2000-01-02"):
if with_speculative not in ("both", "alice2"):
await user_storage_non_speculative_init(
data_base_dir=data_base_dir, device=alice2)
async with user_fs_factory(
alice2, data_base_dir=data_base_dir) as user_fs2:
wksp2_id = await user_fs2.workspace_create(
EntryName("wksp2"))
with freeze_time("2000-01-03"):
# Only now the backend appear offline, this is to ensure each
# userfs has created a user manifest in isolation
await backend_data_binder.bind_organization(
coolorg,
alice,
initial_user_manifest="not_synced")
await backend_data_binder.bind_device(
alice2, certifier=alice)
await switch_back_online()
# Sync user_fs2 first to ensure created_on field is
# kept even if further syncs have an earlier value
with freeze_time("2000-01-04"):
await user_fs2.sync()
with freeze_time("2000-01-05"):
await user_fs1.sync()
with freeze_time("2000-01-06"):
await user_fs2.sync()
# Now, both user fs should have the same view on data
expected_workspaces_entries = (
WorkspaceEntry(
name=EntryName("wksp1"),
id=wksp1_id,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=datetime(2000, 1, 1),
role=WorkspaceRole.OWNER,
),
WorkspaceEntry(
name=EntryName("wksp2"),
id=wksp2_id,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
),
)
expected_user_manifest = LocalUserManifest(
base=UserManifest(
id=alice.user_manifest_id,
version=2,
timestamp=datetime(2000, 1, 5),
author=alice.device_id,
created=datetime(2000, 1, 2),
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=expected_workspaces_entries,
),
need_sync=False,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=expected_workspaces_entries,
speculative=False,
)
user_fs1_manifest = user_fs1.get_user_manifest()
user_fs2_manifest = user_fs2.get_user_manifest()
# We use to use ANY for the "key" argument in expected_user_manifest,
# so that we could compare the two instances safely. Sadly, ANY doesn't
# play nicely with the Rust bindings, so we instead update the instances
# to change the key.
user_fs1_manifest = user_fs1_manifest.evolve(
workspaces=tuple(
w.evolve(key=KEY)
for w in user_fs1_manifest.workspaces),
base=user_fs1_manifest.base.evolve(
workspaces=tuple(
w.evolve(key=KEY) for w in
user_fs1_manifest.base.workspaces)),
)
user_fs2_manifest = user_fs2_manifest.evolve(
workspaces=tuple(
w.evolve(key=KEY)
for w in user_fs2_manifest.workspaces),
base=user_fs2_manifest.base.evolve(
workspaces=tuple(
w.evolve(key=KEY) for w in
user_fs2_manifest.base.workspaces)),
)
assert user_fs1_manifest == expected_user_manifest
assert user_fs2_manifest == expected_user_manifest
async def test_sync_placeholder(
running_backend,
backend_data_binder,
local_device_factory,
user_fs_factory,
data_base_dir,
initialize_local_user_manifest,
with_workspace,
initial_user_manifest,
):
device = local_device_factory()
await backend_data_binder.bind_device(device,
initial_user_manifest="not_synced")
await initialize_local_user_manifest(
data_base_dir, device, initial_user_manifest=initial_user_manifest)
async with user_fs_factory(device) as user_fs:
um_v0 = user_fs.get_user_manifest()
expected_um = LocalUserManifest.new_placeholder(
device.device_id,
id=device.user_manifest_id,
timestamp=um_v0.created,
speculative=(initial_user_manifest == "speculative_v0"),
)
assert um_v0 == expected_um
if with_workspace:
with freeze_time("2000-01-02"):
wid = await user_fs.workspace_create(EntryName("w1"))
um = user_fs.get_user_manifest()
expected_um = um_v0.evolve(
updated=datetime(2000, 1, 2),
workspaces=(WorkspaceEntry(
name=EntryName("w1"),
id=wid,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
um = _update_user_manifest_key(um)
assert um == expected_um
with freeze_time("2000-01-02"):
await user_fs.sync()
um = user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=device.device_id,
# Add extra time due to the user realm being already created at 2000-01-02
timestamp=datetime(2000, 1,
2).add(microseconds=MANIFEST_STAMP_AHEAD_US),
id=device.user_manifest_id,
version=1,
created=expected_um.created,
updated=expected_um.updated,
last_processed_message=0,
workspaces=expected_um.workspaces,
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=expected_um.updated,
last_processed_message=0,
workspaces=expected_base_um.workspaces,
speculative=False,
)
um = _update_user_manifest_key(um)
assert um.base == expected_base_um
assert um == expected_um
async def test_concurrent_sync_placeholder(
running_backend,
backend_data_binder,
local_device_factory,
user_fs_factory,
data_base_dir,
initialize_local_user_manifest,
dev2_has_changes,
):
device1 = local_device_factory("a@1")
await backend_data_binder.bind_device(device1,
initial_user_manifest="not_synced")
await initialize_local_user_manifest(
data_base_dir, device1, initial_user_manifest="non_speculative_v0")
device2 = local_device_factory("a@2")
await backend_data_binder.bind_device(device2)
await initialize_local_user_manifest(
data_base_dir, device2, initial_user_manifest="speculative_v0")
async with user_fs_factory(device1) as user_fs1, user_fs_factory(
device2) as user_fs2:
# fs2's created value is different and will be overwritten when
# merging synced manifest from fs1
um_created_v0_fs1 = user_fs1.get_user_manifest().created
with freeze_time("2000-01-01"):
# Sync user manifests now to avoid extra milliseconds from restamping
await user_fs1.sync()
await user_fs2.sync()
w1id = await user_fs1.workspace_create(EntryName("w1"))
if dev2_has_changes:
with freeze_time("2000-01-02"):
w2id = await user_fs2.workspace_create(EntryName("w2"))
with freeze_time("2000-01-03"):
await user_fs1.sync()
with freeze_time("2000-01-04"):
await user_fs2.sync()
if dev2_has_changes:
with freeze_time("2000-01-05"):
await user_fs1.sync()
um1 = user_fs1.get_user_manifest()
um2 = user_fs2.get_user_manifest()
if dev2_has_changes:
expected_base_um = UserManifest(
author=device2.device_id,
id=device2.user_manifest_id,
timestamp=datetime(2000, 1, 4),
version=3,
created=um_created_v0_fs1,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=(
WorkspaceEntry(
name=EntryName("w1"),
id=w1id,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=datetime(2000, 1, 1),
role=WorkspaceRole.OWNER,
),
WorkspaceEntry(
name=EntryName("w2"),
id=w2id,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
),
),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
speculative=False,
)
else:
expected_base_um = UserManifest(
author=device1.device_id,
timestamp=datetime(2000, 1, 3),
id=device1.user_manifest_id,
version=2,
created=um_created_v0_fs1,
updated=datetime(2000, 1, 1),
last_processed_message=0,
workspaces=(WorkspaceEntry(
name=EntryName("w1"),
id=w1id,
key=KEY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=datetime(2000, 1, 1),
role=WorkspaceRole.OWNER,
), ),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 1),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
speculative=False,
)
um1 = _update_user_manifest_key(um1)
um2 = _update_user_manifest_key(um2)
assert um1 == expected_um
assert um2 == expected_um
def user_manifest(alice):
timestamp = alice.timestamp()
return LocalUserManifest.new_placeholder(
alice.device_id, id=alice.user_manifest_id, timestamp=timestamp
)