def test_created_field_modified_by_remote(gen_date, alice, with_local_changes):
d1, d2, d3, d4 = [gen_date() for _ in range(4)]
w1 = WorkspaceEntry.new(name=EntryName("w1"), timestamp=d2)
base = UserManifest(
author=alice.device_id,
timestamp=d2,
id=alice.user_manifest_id,
version=1,
created=d1,
updated=d2,
last_processed_message=0,
workspaces=(w1,),
)
local = LocalUserManifest.from_remote(base)
if with_local_changes:
w2 = WorkspaceEntry.new(name=EntryName("w1"), timestamp=d3)
local = local.evolve(
need_sync=True, updated=d3, last_processed_message=1, workspaces=(w1, w2)
)
target = base.evolve(created=d4, version=2)
expected_merged = local.evolve(base=target)
merged = merge_local_user_manifests(local, target)
# Remote always control the value of the create field
assert merged == expected_merged
def test_merge_local_user_manifest_changes_placeholder(gen_date, alice, speculative_placeholder):
d1, d2, d3, d4 = [gen_date() for _ in range(4)]
w1 = WorkspaceEntry.new(name=EntryName("w1"), timestamp=d2)
w2 = WorkspaceEntry.new(name=EntryName("w2"), timestamp=d2)
w3 = WorkspaceEntry.new(name=EntryName("w3"), timestamp=d2)
diverged = LocalUserManifest.new_placeholder(
alice.device_id,
id=alice.user_manifest_id,
timestamp=d4,
speculative=speculative_placeholder,
).evolve(last_processed_message=30, workspaces=(w1, w3))
target = UserManifest(
author=alice.device_id,
timestamp=d2,
id=alice.user_manifest_id,
version=3,
created=d1,
updated=d3,
last_processed_message=20,
workspaces=(w1, w2),
)
expected_merged = LocalUserManifest(
base=target,
updated=d4,
last_processed_message=30,
workspaces=(w1, w2, w3),
need_sync=True,
speculative=False,
)
merged = merge_local_user_manifests(diverged, target)
assert merged == expected_merged
async def _inbound_sync(self) -> None:
# Retrieve remote
target_um = await self._fetch_remote_user_manifest()
diverged_um = self.get_user_manifest()
if target_um.version == diverged_um.base_version:
# Nothing new
return
# New things in remote, merge is needed
async with self._update_user_manifest_lock:
diverged_um = self.get_user_manifest()
if target_um.version <= diverged_um.base_version:
# Sync already achieved by a concurrent operation
return
merged_um = merge_local_user_manifests(diverged_um, target_um)
await self.set_user_manifest(merged_um)
# In case we weren't online when the sharing message arrived,
# we will learn about the change in the sharing only now.
# Hence send the corresponding events !
self._detect_and_send_shared_events(diverged_um, merged_um)
# TODO: deprecated event ?
self.event_bus.send(CoreEvent.FS_ENTRY_REMOTE_CHANGED,
path="/",
id=self.user_manifest_id)
return
def test_merge_local_user_manifest_no_changes_in_diverged_placeholder(
gen_date, alice, alice2, with_ignored_changes
):
w1 = WorkspaceEntry.new(name="w1")
d1, d2, d3, d4 = [gen_date() for _ in range(4)]
base = UserManifest(
author=alice.device_id,
timestamp=d2,
id=alice.user_manifest_id,
version=1,
created=d1,
updated=d2,
last_processed_message=0,
workspaces=(w1,),
)
diverged = LocalUserManifest.from_remote(base)
if with_ignored_changes:
diverged = diverged.evolve(updated=d4, need_sync=True)
target = UserManifest(
author=alice2.device_id,
timestamp=d2,
id=alice2.user_manifest_id,
version=2,
created=d1,
updated=d3,
last_processed_message=0,
workspaces=(w1,),
)
expected_merged = LocalUserManifest.from_remote(target)
merged = merge_local_user_manifests(diverged, target)
assert merged == expected_merged
def test_merge_local_user_manifest_no_changes_in_diverged_placeholder(
gen_date, alice, alice2, with_ignored_changes
):
d1, d2, d3, d4, d5, d6, d7 = [gen_date() for _ in range(7)]
w1 = WorkspaceEntry.new(name=EntryName("w1"), timestamp=d2)
base = UserManifest(
author=alice.device_id,
timestamp=d4,
id=alice.user_manifest_id,
version=1,
created=d1,
updated=d3,
last_processed_message=0,
workspaces=(w1,),
)
diverged = LocalUserManifest.from_remote(base)
if with_ignored_changes:
w1_bis = WorkspaceEntry(
name=w1.name,
id=w1.id,
key=w1.key,
# Same encryption revision than remote (so encryption date should be ignored)
encryption_revision=1,
encrypted_on=d5,
# Cache older than remote
role_cached_on=d1,
role=RealmRole.MANAGER,
)
diverged = diverged.evolve(updated=d4, need_sync=True, workspaces=(w1_bis,))
target = UserManifest(
author=alice2.device_id,
timestamp=d7,
id=alice2.user_manifest_id,
version=2,
created=d1,
updated=d6,
last_processed_message=0,
workspaces=(w1,),
)
expected_merged = LocalUserManifest.from_remote(target)
merged = merge_local_user_manifests(diverged, target)
assert merged == expected_merged
def test_merge_speculative_with_it_unsuspected_former_self(alice, local_changes):
d1 = datetime(2000, 1, 1)
d2 = datetime(2000, 1, 2)
d3 = datetime(2000, 1, 3)
# 1) User manifest is originally created by our device
local = LocalUserManifest.new_placeholder(
author=alice.device_id, id=alice.user_manifest_id, timestamp=d1, speculative=False
)
w1 = WorkspaceEntry.new(EntryName("foo"), timestamp=d1)
local = local.evolve(workspaces=(w1,), last_processed_message=1)
# 2) We sync the user manifest
v1 = local.to_remote(author=alice.device_id, timestamp=d2)
# 3) Now let's pretend we lost local storage, hence creating a new speculative manifest
new_local = LocalUserManifest.new_placeholder(
author=alice.device_id, id=alice.user_manifest_id, timestamp=d3, speculative=True
)
if local_changes:
w2 = WorkspaceEntry.new(EntryName("bar"), timestamp=d3)
new_local = new_local.evolve(workspaces=(w2,), last_processed_message=2)
# 4) When syncing the manifest, we shouldn't remove any data from the remote
merged = merge_local_user_manifests(new_local, v1)
if local_changes:
assert merged == LocalUserManifest(
base=v1,
updated=d3,
last_processed_message=2,
workspaces=(w2, w1),
need_sync=True,
speculative=False,
)
else:
assert merged == LocalUserManifest(
base=v1,
updated=v1.updated,
last_processed_message=1,
workspaces=(w1,),
need_sync=False,
speculative=False,
)
async def _outbound_sync_inner(self) -> bool:
base_um = self.get_user_manifest()
if not base_um.need_sync:
return True
# Make sure the corresponding realm has been created in the backend
if base_um.is_placeholder:
certif = RealmRoleCertificateContent.build_realm_root_certif(
author=self.device.device_id,
timestamp=pendulum_now(),
realm_id=self.device.user_manifest_id,
).dump_and_sign(self.device.signing_key)
try:
rep = await self.backend_cmds.realm_create(certif)
except BackendNotAvailable as exc:
raise FSBackendOfflineError(str(exc)) from exc
except BackendConnectionError as exc:
raise FSError(f"Cannot create user manifest's realm in backend: {exc}") from exc
if rep["status"] == "already_exists":
# It's possible a previous attempt to create this realm
# succeeded but we didn't receive the confirmation, hence
# we play idempotent here.
pass
elif rep["status"] != "ok":
raise FSError(f"Cannot create user manifest's realm in backend: {rep}")
# Sync placeholders
for w in base_um.workspaces:
await self._workspace_minimal_sync(w)
# Build vlob
now = pendulum_now()
to_sync_um = base_um.to_remote(author=self.device.device_id, timestamp=now)
ciphered = to_sync_um.dump_sign_and_encrypt(
author_signkey=self.device.signing_key, key=self.device.user_manifest_key
)
# Sync the vlob with backend
try:
# Note encryption_revision is always 1 given we never reencrypt
# the user manifest's realm
if to_sync_um.version == 1:
rep = await self.backend_cmds.vlob_create(
self.user_manifest_id, 1, self.user_manifest_id, now, ciphered
)
else:
rep = await self.backend_cmds.vlob_update(
1, self.user_manifest_id, to_sync_um.version, now, ciphered
)
except BackendNotAvailable as exc:
raise FSBackendOfflineError(str(exc)) from exc
except BackendConnectionError as exc:
raise FSError(f"Cannot sync user manifest: {exc}") from exc
if rep["status"] in ("already_exists", "bad_version"):
# Concurrency error (handled by the caller)
return False
elif rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
f"Cannot modify workspace data while it is in maintenance: {rep}"
)
elif rep["status"] != "ok":
raise FSError(f"Cannot sync user manifest: {rep}")
# Merge back the manifest in local
async with self._update_user_manifest_lock:
diverged_um = self.get_user_manifest()
# Final merge could have been achieved by a concurrent operation
if to_sync_um.version > diverged_um.base_version:
merged_um = merge_local_user_manifests(diverged_um, to_sync_um)
await self.set_user_manifest(merged_um)
self.event_bus.send("fs.entry.synced", id=self.user_manifest_id)
return True