def test_merge_local_user_manifest_no_changes_in_diverged_placeholder(
gen_date, alice, alice2, with_ignored_changes):
w1 = WorkspaceEntry.new(name="w1")
d1, d2, d3, d4 = [gen_date() for _ in range(4)]
base = UserManifest(
author=alice.device_id,
timestamp=d2,
id=alice.user_manifest_id,
version=1,
created=d1,
updated=d2,
last_processed_message=0,
workspaces=(w1, ),
)
diverged = LocalUserManifest.from_remote(base)
if with_ignored_changes:
diverged = diverged.evolve(updated=d4, need_sync=True)
target = UserManifest(
author=alice2.device_id,
timestamp=d2,
id=alice2.user_manifest_id,
version=2,
created=d1,
updated=d3,
last_processed_message=0,
workspaces=(w1, ),
)
expected_merged = LocalUserManifest.from_remote(target)
merged = merge_local_user_manifests(diverged, target)
assert merged == expected_merged
async def test_sync(running_backend, alice2_user_fs, alice2):
with freeze_time("2000-01-02"):
wid = await alice2_user_fs.workspace_create("w1")
with freeze_time("2000-01-03"):
await alice2_user_fs.sync()
um = alice2_user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=alice2.device_id,
timestamp=datetime(2000, 1, 3),
id=alice2.user_manifest_id,
version=2,
created=datetime(2000, 1, 1),
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=(WorkspaceEntry(
name="w1",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
expected_um = LocalUserManifest.from_remote(expected_base_um)
assert um == expected_um
def test_merge_local_user_manifest_changes_placeholder(gen_date, alice):
d1, d2, d3, d4 = [gen_date() for _ in range(4)]
w1 = WorkspaceEntry.new(name="w1")
w2 = WorkspaceEntry.new(name="w2")
w3 = WorkspaceEntry.new(name="w3")
diverged = LocalUserManifest.new_placeholder(alice.device_id,
id=alice.user_manifest_id,
now=d4).evolve(
last_processed_message=30,
workspaces=(w1, w3))
target = UserManifest(
author=alice.device_id,
timestamp=d2,
id=alice.user_manifest_id,
version=3,
created=d1,
updated=d3,
last_processed_message=20,
workspaces=(w1, w2),
)
expected_merged = LocalUserManifest(base=target,
updated=d4,
last_processed_message=30,
workspaces=(w1, w2, w3),
need_sync=True)
merged = merge_local_user_manifests(diverged, target)
assert merged == expected_merged
async def test_sync_placeholder(running_backend, backend_data_binder,
local_device_factory, user_fs_factory,
with_workspace):
device = local_device_factory()
await backend_data_binder.bind_device(device,
initial_user_manifest_in_v0=True)
async with user_fs_factory(device, initialize_in_v0=True) as user_fs:
um_v0 = user_fs.get_user_manifest()
expected_um = LocalUserManifest.new_placeholder(
device.device_id, id=device.user_manifest_id, now=um_v0.created)
assert um_v0 == expected_um
if with_workspace:
with freeze_time("2000-01-02"):
wid = await user_fs.workspace_create("w1")
um = user_fs.get_user_manifest()
expected_um = um_v0.evolve(
updated=datetime(2000, 1, 2),
workspaces=(WorkspaceEntry(
name="w1",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
assert um == expected_um
with freeze_time("2000-01-02"):
await user_fs.sync()
um = user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=device.device_id,
timestamp=datetime(2000, 1, 2),
id=device.user_manifest_id,
version=1,
created=expected_um.created,
updated=expected_um.updated,
last_processed_message=0,
workspaces=expected_um.workspaces,
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=expected_um.updated,
last_processed_message=0,
workspaces=expected_base_um.workspaces,
)
assert um == expected_um
async def _fetch_remote_user_manifest(self,
version: Optional[int] = None
) -> UserManifest:
"""
Raises:
FSError
FSWorkspaceInMaintenance
FSBackendOfflineError
"""
try:
# Note encryption_revision is always 1 given we never reencrypt
# the user manifest's realm
rep = await self.backend_cmds.vlob_read(1, self.user_manifest_id,
version)
except BackendNotAvailable as exc:
raise FSBackendOfflineError(str(exc)) from exc
if rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
"Cannot access workspace data while it is in maintenance")
elif rep["status"] != "ok":
raise FSError(f"Cannot fetch user manifest from backend: {rep}")
expected_author = rep["author"]
expected_timestamp = rep["timestamp"]
expected_version = rep["version"]
blob = rep["blob"]
author = await self.remote_loader.get_device(expected_author)
try:
manifest = UserManifest.decrypt_verify_and_load(
blob,
key=self.device.user_manifest_key,
author_verify_key=author.verify_key,
expected_id=self.device.user_manifest_id,
expected_author=expected_author,
expected_timestamp=expected_timestamp,
expected_version=version
if version is not None else expected_version,
)
except DataError as exc:
raise FSError(f"Invalid user manifest: {exc}") from exc
return manifest
async def test_sync_remote_changes(running_backend, alice_user_fs,
alice2_user_fs, alice, alice2):
# Alice 2 update the user manifest
with freeze_time("2000-01-02"):
wid = await alice2_user_fs.workspace_create("wa")
with freeze_time("2000-01-03"):
await alice2_user_fs.sync()
# Alice retrieve the changes
um = alice_user_fs.get_user_manifest()
await alice_user_fs.sync()
um = alice_user_fs.get_user_manifest()
um2 = alice2_user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=alice2.device_id,
timestamp=datetime(2000, 1, 3),
id=alice2.user_manifest_id,
version=2,
created=datetime(2000, 1, 1),
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=(WorkspaceEntry(
name="wa",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
)
assert um == expected_um
assert um2 == expected_um
def _generate_or_retrieve_user_manifest_v1(self, device):
try:
return self._v1[(device.organization_id, device.user_id)]
except KeyError:
now = pendulum.now()
remote_user_manifest = UserManifest(
author=device.device_id,
timestamp=now,
id=device.user_manifest_id,
version=1,
created=now,
updated=now,
last_processed_message=0,
workspaces=(),
)
local_user_manifest = LocalUserManifest.from_remote(
remote_user_manifest)
self._v1[(device.organization_id, device.user_id)] = (
remote_user_manifest,
local_user_manifest,
)
return self._v1[(device.organization_id, device.user_id)]
async def test_concurrent_sync_placeholder(running_backend,
backend_data_binder,
local_device_factory,
user_fs_factory, dev2_has_changes):
device1 = local_device_factory("a@1")
await backend_data_binder.bind_device(device1,
initial_user_manifest_in_v0=True)
device2 = local_device_factory("a@2")
await backend_data_binder.bind_device(device2,
initial_user_manifest_in_v0=True)
async with user_fs_factory(
device1, initialize_in_v0=True) as user_fs1, user_fs_factory(
device2, initialize_in_v0=True) as user_fs2:
# fs2's created value is different and will be overwritten when
# merging synced manifest from fs1
um_created_v0_fs1 = user_fs1.get_user_manifest().created
with freeze_time("2000-01-01"):
w1id = await user_fs1.workspace_create("w1")
if dev2_has_changes:
with freeze_time("2000-01-02"):
w2id = await user_fs2.workspace_create("w2")
with freeze_time("2000-01-03"):
await user_fs1.sync()
with freeze_time("2000-01-04"):
await user_fs2.sync()
if dev2_has_changes:
with freeze_time("2000-01-05"):
await user_fs1.sync()
um1 = user_fs1.get_user_manifest()
um2 = user_fs2.get_user_manifest()
if dev2_has_changes:
expected_base_um = UserManifest(
author=device2.device_id,
id=device2.user_manifest_id,
timestamp=datetime(2000, 1, 4),
version=2,
created=um_created_v0_fs1,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=(
WorkspaceEntry(
name="w1",
id=w1id,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=datetime(2000, 1, 1),
role=WorkspaceRole.OWNER,
),
WorkspaceEntry(
name="w2",
id=w2id,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
),
),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
)
else:
expected_base_um = UserManifest(
author=device1.device_id,
timestamp=datetime(2000, 1, 3),
id=device1.user_manifest_id,
version=1,
created=um_created_v0_fs1,
updated=datetime(2000, 1, 1),
last_processed_message=0,
workspaces=(WorkspaceEntry(
name="w1",
id=w1id,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=datetime(2000, 1, 1),
role=WorkspaceRole.OWNER,
), ),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 1),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
)
assert um1 == expected_um
assert um2 == expected_um