def test_merge_local_user_manifest_changes_placeholder(gen_date, alice):
d1, d2, d3, d4 = [gen_date() for _ in range(4)]
w1 = WorkspaceEntry.new(name="w1")
w2 = WorkspaceEntry.new(name="w2")
w3 = WorkspaceEntry.new(name="w3")
diverged = LocalUserManifest.new_placeholder(alice.device_id,
id=alice.user_manifest_id,
now=d4).evolve(
last_processed_message=30,
workspaces=(w1, w3))
target = UserManifest(
author=alice.device_id,
timestamp=d2,
id=alice.user_manifest_id,
version=3,
created=d1,
updated=d3,
last_processed_message=20,
workspaces=(w1, w2),
)
expected_merged = LocalUserManifest(base=target,
updated=d4,
last_processed_message=30,
workspaces=(w1, w2, w3),
need_sync=True)
merged = merge_local_user_manifests(diverged, target)
assert merged == expected_merged
def test_merge_local_user_manifest_no_changes_in_diverged_placeholder(
gen_date, alice, alice2, with_ignored_changes):
w1 = WorkspaceEntry.new(name="w1")
d1, d2, d3, d4 = [gen_date() for _ in range(4)]
base = UserManifest(
author=alice.device_id,
timestamp=d2,
id=alice.user_manifest_id,
version=1,
created=d1,
updated=d2,
last_processed_message=0,
workspaces=(w1, ),
)
diverged = LocalUserManifest.from_remote(base)
if with_ignored_changes:
diverged = diverged.evolve(updated=d4, need_sync=True)
target = UserManifest(
author=alice2.device_id,
timestamp=d2,
id=alice2.user_manifest_id,
version=2,
created=d1,
updated=d3,
last_processed_message=0,
workspaces=(w1, ),
)
expected_merged = LocalUserManifest.from_remote(target)
merged = merge_local_user_manifests(diverged, target)
assert merged == expected_merged
async def test_sync_placeholder(running_backend, backend_data_binder,
local_device_factory, user_fs_factory,
with_workspace):
device = local_device_factory()
await backend_data_binder.bind_device(device,
initial_user_manifest_in_v0=True)
async with user_fs_factory(device, initialize_in_v0=True) as user_fs:
um_v0 = user_fs.get_user_manifest()
expected_um = LocalUserManifest.new_placeholder(
device.device_id, id=device.user_manifest_id, now=um_v0.created)
assert um_v0 == expected_um
if with_workspace:
with freeze_time("2000-01-02"):
wid = await user_fs.workspace_create("w1")
um = user_fs.get_user_manifest()
expected_um = um_v0.evolve(
updated=datetime(2000, 1, 2),
workspaces=(WorkspaceEntry(
name="w1",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
assert um == expected_um
with freeze_time("2000-01-02"):
await user_fs.sync()
um = user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=device.device_id,
timestamp=datetime(2000, 1, 2),
id=device.user_manifest_id,
version=1,
created=expected_um.created,
updated=expected_um.updated,
last_processed_message=0,
workspaces=expected_um.workspaces,
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=expected_um.updated,
last_processed_message=0,
workspaces=expected_base_um.workspaces,
)
assert um == expected_um
async def test_rename_workspace(initial_user_manifest_state, alice_user_fs,
alice):
with freeze_time("2000-01-02"):
wid = await alice_user_fs.workspace_create("w1")
with freeze_time("2000-01-03"):
await alice_user_fs.workspace_rename(wid, "w2")
um = alice_user_fs.get_user_manifest()
expected_base_um = initial_user_manifest_state.get_user_manifest_v1_for_backend(
alice)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=True,
updated=datetime(2000, 1, 3),
last_processed_message=expected_base_um.last_processed_message,
workspaces=(WorkspaceEntry(
name="w2",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
assert um == expected_um
async def test_create_workspace(initial_user_manifest_state, alice_user_fs,
alice):
with freeze_time("2000-01-02"):
wid = await alice_user_fs.workspace_create("w1")
um = alice_user_fs.get_user_manifest()
expected_base_um = initial_user_manifest_state.get_user_manifest_v1_for_backend(
alice)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=True,
updated=datetime(2000, 1, 2),
last_processed_message=expected_base_um.last_processed_message,
workspaces=(WorkspaceEntry(
name="w1",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
assert um == expected_um
w_manifest = await alice_user_fs.get_workspace(
wid).local_storage.get_manifest(wid)
expected_w_manifest = LocalWorkspaceManifest.new_placeholder(
alice.device_id, id=w_manifest.id, now=datetime(2000, 1, 2))
assert w_manifest == expected_w_manifest
async def test_sync(running_backend, alice2_user_fs, alice2):
with freeze_time("2000-01-02"):
wid = await alice2_user_fs.workspace_create("w1")
with freeze_time("2000-01-03"):
await alice2_user_fs.sync()
um = alice2_user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=alice2.device_id,
timestamp=datetime(2000, 1, 3),
id=alice2.user_manifest_id,
version=2,
created=datetime(2000, 1, 1),
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=(WorkspaceEntry(
name="w1",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
expected_um = LocalUserManifest.from_remote(expected_base_um)
assert um == expected_um
async def _load_user_manifest(self) -> None:
try:
await self.manifest_storage.get_manifest(self.user_manifest_id)
except FSLocalMissError:
# In the unlikely event the user manifest is not present in
# local (e.g. device just created or during tests), we fall
# back on an empty manifest which is a good aproximation of
# the very first version of the manifest (field `created` is
# invalid, but it will be corrected by the merge during sync).
manifest = LocalUserManifest.new_placeholder(
self.device.device_id, id=self.device.user_manifest_id)
await self.manifest_storage.set_manifest(self.user_manifest_id,
manifest)
def create_manifest(device,
type=LocalWorkspaceManifest,
use_legacy_none_author=False):
author = device.device_id
if type is LocalUserManifest:
manifest = LocalUserManifest.new_placeholder(author)
elif type is LocalWorkspaceManifest:
manifest = type.new_placeholder(author)
else:
manifest = type.new_placeholder(author, parent=EntryID())
if use_legacy_none_author:
base = manifest.base.evolve(author=None)
manifest = manifest.evolve(base=base)
return manifest
async def test_sync_remote_changes(running_backend, alice_user_fs,
alice2_user_fs, alice, alice2):
# Alice 2 update the user manifest
with freeze_time("2000-01-02"):
wid = await alice2_user_fs.workspace_create("wa")
with freeze_time("2000-01-03"):
await alice2_user_fs.sync()
# Alice retrieve the changes
um = alice_user_fs.get_user_manifest()
await alice_user_fs.sync()
um = alice_user_fs.get_user_manifest()
um2 = alice2_user_fs.get_user_manifest()
expected_base_um = UserManifest(
author=alice2.device_id,
timestamp=datetime(2000, 1, 3),
id=alice2.user_manifest_id,
version=2,
created=datetime(2000, 1, 1),
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=(WorkspaceEntry(
name="wa",
id=wid,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
), ),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
)
assert um == expected_um
assert um2 == expected_um
def merge_local_user_manifests(diverged: LocalUserManifest,
target: UserManifest) -> LocalUserManifest:
assert isinstance(diverged, LocalUserManifest)
assert isinstance(target, UserManifest)
assert diverged.id == target.id
base_version = diverged.base_version
base_workspaces = diverged.base.workspaces if diverged.base is not None else None
assert target.version > base_version
# Not true when merging user manifest v1 given v0 is lazily generated
assert base_version == 0 or diverged.created == target.created
workspaces, need_sync = merge_workspace_entries(base_workspaces,
diverged.workspaces,
target.workspaces)
last_processed_message = max(diverged.last_processed_message,
target.last_processed_message)
need_sync = need_sync or last_processed_message != target.last_processed_message
if not need_sync:
updated = target.updated
else:
if target.updated > diverged.updated:
updated = target.updated
else:
updated = diverged.updated
return LocalUserManifest(
base=target,
need_sync=need_sync,
updated=updated,
last_processed_message=last_processed_message,
workspaces=workspaces,
)
def _generate_or_retrieve_user_manifest_v1(self, device):
try:
return self._v1[(device.organization_id, device.user_id)]
except KeyError:
now = pendulum.now()
remote_user_manifest = UserManifest(
author=device.device_id,
timestamp=now,
id=device.user_manifest_id,
version=1,
created=now,
updated=now,
last_processed_message=0,
workspaces=(),
)
local_user_manifest = LocalUserManifest.from_remote(
remote_user_manifest)
self._v1[(device.organization_id, device.user_id)] = (
remote_user_manifest,
local_user_manifest,
)
return self._v1[(device.organization_id, device.user_id)]
def user_manifest(alice):
return LocalUserManifest.new_placeholder(alice.device_id,
id=alice.user_manifest_id)
async def test_concurrent_sync_placeholder(running_backend,
backend_data_binder,
local_device_factory,
user_fs_factory, dev2_has_changes):
device1 = local_device_factory("a@1")
await backend_data_binder.bind_device(device1,
initial_user_manifest_in_v0=True)
device2 = local_device_factory("a@2")
await backend_data_binder.bind_device(device2,
initial_user_manifest_in_v0=True)
async with user_fs_factory(
device1, initialize_in_v0=True) as user_fs1, user_fs_factory(
device2, initialize_in_v0=True) as user_fs2:
# fs2's created value is different and will be overwritten when
# merging synced manifest from fs1
um_created_v0_fs1 = user_fs1.get_user_manifest().created
with freeze_time("2000-01-01"):
w1id = await user_fs1.workspace_create("w1")
if dev2_has_changes:
with freeze_time("2000-01-02"):
w2id = await user_fs2.workspace_create("w2")
with freeze_time("2000-01-03"):
await user_fs1.sync()
with freeze_time("2000-01-04"):
await user_fs2.sync()
if dev2_has_changes:
with freeze_time("2000-01-05"):
await user_fs1.sync()
um1 = user_fs1.get_user_manifest()
um2 = user_fs2.get_user_manifest()
if dev2_has_changes:
expected_base_um = UserManifest(
author=device2.device_id,
id=device2.user_manifest_id,
timestamp=datetime(2000, 1, 4),
version=2,
created=um_created_v0_fs1,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=(
WorkspaceEntry(
name="w1",
id=w1id,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=datetime(2000, 1, 1),
role=WorkspaceRole.OWNER,
),
WorkspaceEntry(
name="w2",
id=w2id,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 2),
role_cached_on=datetime(2000, 1, 2),
role=WorkspaceRole.OWNER,
),
),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 2),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
)
else:
expected_base_um = UserManifest(
author=device1.device_id,
timestamp=datetime(2000, 1, 3),
id=device1.user_manifest_id,
version=1,
created=um_created_v0_fs1,
updated=datetime(2000, 1, 1),
last_processed_message=0,
workspaces=(WorkspaceEntry(
name="w1",
id=w1id,
key=ANY,
encryption_revision=1,
encrypted_on=datetime(2000, 1, 1),
role_cached_on=datetime(2000, 1, 1),
role=WorkspaceRole.OWNER,
), ),
)
expected_um = LocalUserManifest(
base=expected_base_um,
need_sync=False,
updated=datetime(2000, 1, 1),
last_processed_message=0,
workspaces=expected_base_um.workspaces,
)
assert um1 == expected_um
assert um2 == expected_um