def test_merge_manifests_with_a_placeholder(alice, bob):
timestamp = alice.timestamp()
my_device = alice.device_id
other_device = bob.device_id
parent = EntryID.new()
m1 = LocalFolderManifest.new_placeholder(my_device,
parent=parent,
timestamp=timestamp)
m2 = merge_manifests(my_device, timestamp, empty_pattern, m1)
assert m2 == m1
v1 = m1.to_remote(author=my_device, timestamp=timestamp)
m2a = merge_manifests(my_device, timestamp, empty_pattern, m1, v1)
assert m2a == LocalFolderManifest.from_remote(v1, empty_pattern)
m2b = m1.evolve_children_and_mark_updated({EntryName("a"): EntryID.new()},
empty_pattern,
timestamp=timestamp)
m3b = merge_manifests(my_device, timestamp, empty_pattern, m2b, v1)
assert m3b == m2b.evolve(base=v1)
v2 = v1.evolve(version=2,
author=other_device,
children={EntryName("b"): EntryID.new()})
m2c = m1.evolve_children_and_mark_updated({EntryName("a"): EntryID.new()},
empty_pattern,
timestamp=timestamp)
m3c = merge_manifests(my_device, timestamp, empty_pattern, m2c, v2)
children = {**v2.children, **m2c.children}
assert m3c == m2c.evolve(base=v2, children=children, updated=m3c.updated)
async def test_realm_checkpoint(alice_workspace_storage):
aws = alice_workspace_storage
manifest = create_manifest(aws.device, LocalFileManifest)
assert await aws.get_realm_checkpoint() == 0
assert await aws.get_need_sync_entries() == (set(), set())
await aws.update_realm_checkpoint(11, {manifest.id: 22, EntryID(): 33})
assert await aws.get_realm_checkpoint() == 11
assert await aws.get_need_sync_entries() == (set(), set())
await aws.set_manifest(manifest.id, manifest, check_lock_status=False)
assert await aws.get_realm_checkpoint() == 11
assert await aws.get_need_sync_entries() == (set([manifest.id]), set())
await aws.set_manifest(manifest.id, manifest.evolve(need_sync=False), check_lock_status=False)
assert await aws.get_realm_checkpoint() == 11
assert await aws.get_need_sync_entries() == (set(), set())
await aws.update_realm_checkpoint(44, {manifest.id: 55, EntryID(): 66})
assert await aws.get_realm_checkpoint() == 44
assert await aws.get_need_sync_entries() == (set(), set([manifest.id]))
def create_manifest(device, type=LocalWorkspaceManifest):
if type is LocalUserManifest:
manifest = LocalUserManifest.new_placeholder(parent=EntryID())
elif type is LocalWorkspaceManifest:
manifest = type.new_placeholder()
else:
manifest = type.new_placeholder(parent=EntryID())
return manifest
def test_build_addrs():
backend_addr = BackendAddr.from_url(BackendAddrTestbed.url)
assert backend_addr.hostname == "parsec.cloud.com"
assert backend_addr.port == 443
assert backend_addr.use_ssl is True
organization_id = OrganizationID("MyOrg")
root_verify_key = SigningKey.generate().verify_key
organization_addr = BackendOrganizationAddr.build(
backend_addr=backend_addr, organization_id=organization_id, root_verify_key=root_verify_key
)
assert organization_addr.organization_id == organization_id
assert organization_addr.root_verify_key == root_verify_key
organization_bootstrap_addr = BackendOrganizationBootstrapAddr.build(
backend_addr=backend_addr,
organization_id=organization_id,
token="a0000000000000000000000000000001",
)
assert organization_bootstrap_addr.token == "a0000000000000000000000000000001"
assert organization_bootstrap_addr.organization_id == organization_id
organization_bootstrap_addr2 = BackendOrganizationBootstrapAddr.build(
backend_addr=backend_addr, organization_id=organization_id, token=None
)
assert organization_bootstrap_addr2.organization_id == organization_id
assert organization_bootstrap_addr2.token == ""
organization_file_link_addr = BackendOrganizationFileLinkAddr.build(
organization_addr=organization_addr,
workspace_id=EntryID.from_hex("2d4ded12-7406-4608-833b-7f57f01156e2"),
encrypted_path=b"<encrypted_payload>",
)
assert organization_file_link_addr.workspace_id == EntryID.from_hex(
"2d4ded12-7406-4608-833b-7f57f01156e2"
)
assert organization_file_link_addr.encrypted_path == b"<encrypted_payload>"
invitation_addr = BackendInvitationAddr.build(
backend_addr=backend_addr,
organization_id=organization_id,
invitation_type=InvitationType.USER,
token=InvitationToken.from_hex("a0000000000000000000000000000001"),
)
assert invitation_addr.organization_id == organization_id
assert invitation_addr.token == InvitationToken.from_hex("a0000000000000000000000000000001")
assert invitation_addr.invitation_type == InvitationType.USER
async def test_timestamped_storage(alice_workspace_storage):
timestamp = now()
aws = alice_workspace_storage
taws = aws.to_timestamped(timestamp)
assert taws.timestamp == timestamp
assert taws.device == aws.device
assert taws.path == aws.path
assert taws.workspace_id == aws.workspace_id
assert taws.manifest_storage is None
assert taws.block_storage == aws.block_storage
assert taws.chunk_storage == aws.chunk_storage
with pytest.raises(FSError):
await taws.set_chunk("chunk id", "data")
with pytest.raises(FSError):
await taws.clear_chunk("chunk id")
with pytest.raises(FSError):
await taws.clear_manifest("manifest id")
with pytest.raises(FSLocalMissError):
await taws.get_manifest(EntryID())
manifest = create_manifest(aws.device)
with pytest.raises(FSError):
await taws.set_manifest(manifest.id, manifest)
manifest = manifest.evolve(need_sync=False)
async with taws.lock_entry_id(manifest.id):
await taws.set_manifest(manifest.id, manifest)
assert await taws.get_manifest(manifest.id) == manifest
# No-op
await taws.ensure_manifest_persistent(manifest.id)
def __init__(self) -> None:
super().__init__()
self.oracle = open(tmpdir / "oracle.txt", "w+b")
self.manifest = LocalFileManifest.new_placeholder(DeviceID.new(),
parent=EntryID(),
blocksize=8)
self.storage = Storage()
async def init(self):
nonlocal tentative
tentative += 1
await self.reset_all()
await self.start_backend()
self.device = alice
await self.start_transactions()
self.file_transactions = self.transactions_controller.file_transactions
self.local_storage = self.file_transactions.local_storage
self.fresh_manifest = LocalFileManifest.new_placeholder(
alice.device_id,
parent=EntryID.new(),
timestamp=alice.timestamp())
self.entry_id = self.fresh_manifest.id
async with self.local_storage.lock_entry_id(self.entry_id):
await self.local_storage.set_manifest(self.entry_id,
self.fresh_manifest)
self.fd = self.local_storage.create_file_descriptor(
self.fresh_manifest)
self.file_oracle_path = tmpdir / f"oracle-test-{tentative}.txt"
self.file_oracle_fd = os.open(self.file_oracle_path,
os.O_RDWR | os.O_CREAT)
async def test_link_file_unknown_org(
aqtbot, core_config, gui_factory, autoclose_dialog, running_backend, alice
):
password = "******"
save_device_with_password_in_config(core_config.config_dir, alice, password)
# Cheating a bit but it does not matter, we just want a link that appears valid with
# an unknown organization
org_addr = BackendOrganizationAddr.build(
running_backend.addr, OrganizationID("UnknownOrg"), alice.organization_addr.root_verify_key
)
file_link = BackendOrganizationFileLinkAddr.build(
organization_addr=org_addr, workspace_id=EntryID.new(), encrypted_path=b"<whatever>"
)
gui = await gui_factory(core_config=core_config, start_arg=file_link.to_url())
lw = gui.test_get_login_widget()
assert len(autoclose_dialog.dialogs) == 1
assert autoclose_dialog.dialogs[0][0] == "Error"
assert autoclose_dialog.dialogs[0][1] == translate(
"TEXT_FILE_LINK_NOT_IN_ORG_organization"
).format(organization="UnknownOrg")
def _devices_listed():
assert lw.widget.layout().count() > 0
await aqtbot.wait_until(_devices_listed)
accounts_w = lw.widget.layout().itemAt(0).widget()
assert accounts_w
assert isinstance(accounts_w, LoginPasswordInputWidget)
async def test_link_file_unknown_org(core_config, gui_factory,
autoclose_dialog, running_backend, alice):
password = "******"
save_device_with_password(core_config.config_dir, alice, password)
# Cheating a bit but it does not matter, we just want a link that appears valid with
# an unknown organization
org_addr = BackendOrganizationAddr.build(
running_backend.addr, "UnknownOrg",
alice.organization_addr.root_verify_key)
file_link = BackendOrganizationFileLinkAddr.build(
org_addr, EntryID(), FsPath("/doesntmattereither"))
gui = await gui_factory(core_config=core_config,
start_arg=file_link.to_url())
lw = gui.test_get_login_widget()
assert len(autoclose_dialog.dialogs) == 1
assert autoclose_dialog.dialogs[0][0] == "Error"
assert autoclose_dialog.dialogs[0][1] == translate(
"TEXT_FILE_LINK_NOT_IN_ORG_organization").format(
organization="UnknownOrg")
accounts_w = lw.widget.layout().itemAt(0).widget()
assert accounts_w
assert isinstance(accounts_w, LoginPasswordInputWidget)
async def test_path_info_remote_loader_exceptions(monkeypatch, alice_workspace,
alice):
manifest, _ = await alice_workspace.transactions._get_manifest_from_path(
FsPath("/foo/bar"))
async with alice_workspace.local_storage.lock_entry_id(manifest.id):
await alice_workspace.local_storage.clear_manifest(manifest.id)
vanilla_file_manifest_deserialize = BaseRemoteManifest._deserialize
def mocked_file_manifest_deserialize(*args, **kwargs):
return vanilla_file_manifest_deserialize(
*args, **kwargs).evolve(**manifest_modifiers)
monkeypatch.setattr(BaseRemoteManifest, "_deserialize",
mocked_file_manifest_deserialize)
manifest_modifiers = {"id": EntryID.new()}
with pytest.raises(FSError) as exc:
await alice_workspace.path_info(FsPath("/foo/bar"))
assert f"Invalid entry ID: expected `{manifest.id}`, got `{manifest_modifiers['id']}`" in str(
exc.value)
manifest_modifiers = {"version": 4}
with pytest.raises(FSError) as exc:
await alice_workspace.path_info(FsPath("/foo/bar"))
assert "Invalid version: expected `1`, got `4`" in str(exc.value)
manifest_modifiers = {"author": DeviceID("mallory@pc1")}
with pytest.raises(FSError) as exc:
await alice_workspace.path_info(FsPath("/foo/bar"))
assert "Invalid author: expected `alice@dev1`, got `mallory@pc1`" in str(
exc.value)
async def test_synchronization_step_transaction(alice_sync_transactions, type):
sync_transactions = alice_sync_transactions
synchronization_step = sync_transactions.synchronization_step
entry_id = sync_transactions.get_workspace_entry().id
# Sync a placeholder
manifest = await synchronization_step(entry_id)
# Acknowledge a successful synchronization
assert await synchronization_step(entry_id, manifest) is None
# Local change
if type == "file":
a_id, fd = await sync_transactions.file_create(FsPath("/a"))
await sync_transactions.fd_write(fd, b"abc", 0)
await sync_transactions.fd_close(fd)
else:
a_id = await sync_transactions.folder_create(FsPath("/a"))
# Sync parent with a placeholder child
manifest = await synchronization_step(entry_id)
children = []
async for child in sync_transactions.get_placeholder_children(manifest):
children.append(child)
a_entry_id, = children
assert a_entry_id == a_id
# Sync child
if type == "file":
await synchronization_step(a_entry_id)
a_manifest = await synchronization_step(a_entry_id)
assert await synchronization_step(a_entry_id, a_manifest) is None
# Acknowledge the manifest
assert sorted(manifest.children) == ["a"]
assert await synchronization_step(entry_id, manifest) is None
# Local change
b_id = await sync_transactions.folder_create(FsPath("/b"))
# Remote change
children = {**manifest.children, "c": EntryID()}
manifest = manifest.evolve(version=5, children=children, author="b@b")
# Sync parent with a placeholder child
manifest = await synchronization_step(entry_id, manifest)
children = []
async for child in sync_transactions.get_placeholder_children(manifest):
children.append(child)
b_entry_id, = children
assert b_entry_id == b_id
# Sync child
b_manifest = await synchronization_step(b_entry_id)
assert await synchronization_step(b_entry_id, b_manifest) is None
# Acknowledge the manifest
assert sorted(manifest.children) == ["a", "b", "c"]
assert await synchronization_step(entry_id, manifest) is None
async def test_unknown_workspace(alice_user_fs):
bad_wid = EntryID()
with pytest.raises(FSWorkspaceNotFoundError):
await alice_user_fs.workspace_start_reencryption(bad_wid)
with pytest.raises(FSWorkspaceNotFoundError):
await alice_user_fs.workspace_continue_reencryption(bad_wid)
async def _transactions_controlled_cb(started_cb):
async with WorkspaceStorage.run(
tmp_path / f"file_operations-{tentative}", alice,
EntryID.new()) as local_storage:
async with file_transactions_factory(
self.device,
local_storage=local_storage) as file_transactions:
await started_cb(file_transactions=file_transactions)
async def _transactions_controlled_cb(started_cb):
async with WorkspaceStorage.run(alice, Path("/dummy"),
EntryID()) as local_storage:
file_transactions = await file_transactions_factory(
self.device,
alice_backend_cmds,
local_storage=local_storage)
await started_cb(file_transactions=file_transactions)
async def foo_txt(alice, alice_file_transactions):
local_storage = alice_file_transactions.local_storage
now = Pendulum(2000, 1, 2)
placeholder = LocalFileManifest.new_placeholder(parent=EntryID(), now=now)
remote_v1 = placeholder.to_remote(author=alice.device_id, timestamp=now)
manifest = LocalFileManifest.from_remote(remote_v1)
async with local_storage.lock_entry_id(manifest.id):
await local_storage.set_manifest(manifest.id, manifest)
return File(local_storage, manifest)
def __init__(self) -> None:
super().__init__()
self.oracle = open(tmpdir / "oracle.txt", "w+b")
self.manifest = LocalFileManifest.new_placeholder(
alice.device_id,
parent=EntryID.new(),
blocksize=8,
timestamp=alice.timestamp())
self.storage = Storage()
def _handle_event(event_bus: EventBus, rep: dict) -> None:
if rep["status"] != "ok":
logger.warning("Bad response to `events_listen` command", rep=rep)
return
if rep["event"] == APIEvent.MESSAGE_RECEIVED:
event_bus.send(CoreEvent.BACKEND_MESSAGE_RECEIVED, index=rep["index"])
elif rep["event"] == APIEvent.PINGED:
event_bus.send(CoreEvent.BACKEND_PINGED, ping=rep["ping"])
elif rep["event"] == APIEvent.REALM_ROLES_UPDATED:
event_bus.send(
CoreEvent.BACKEND_REALM_ROLES_UPDATED,
realm_id=EntryID(rep["realm_id"].uuid),
role=rep["role"],
)
elif rep["event"] == APIEvent.REALM_VLOBS_UPDATED:
event_bus.send(
CoreEvent.BACKEND_REALM_VLOBS_UPDATED,
realm_id=EntryID(rep["realm_id"].uuid),
checkpoint=rep["checkpoint"],
src_id=EntryID(rep["src_id"].uuid),
src_version=rep["src_version"],
)
elif rep["event"] == APIEvent.REALM_MAINTENANCE_STARTED:
event_bus.send(
CoreEvent.BACKEND_REALM_MAINTENANCE_STARTED,
realm_id=EntryID(rep["realm_id"].uuid),
encryption_revision=rep["encryption_revision"],
)
elif rep["event"] == APIEvent.REALM_MAINTENANCE_FINISHED:
event_bus.send(
CoreEvent.BACKEND_REALM_MAINTENANCE_FINISHED,
realm_id=EntryID(rep["realm_id"].uuid),
encryption_revision=rep["encryption_revision"],
)
elif rep["event"] == APIEvent.PKI_ENROLLMENTS_UPDATED:
event_bus.send(CoreEvent.PKI_ENROLLMENTS_UPDATED)
async def _load_changes(self) -> bool:
if self._changes_loaded:
return True
# Initialize due_time so that if we cannot retrieve the changes, we
# will wait until an external event (most likely a `sharing.updated`)
# make it worth to retry
self.due_time = math.inf
# 1) Fetch new checkpoint and changes
realm_checkpoint = await self._get_local_storage().get_realm_checkpoint()
try:
rep = await self._get_backend_cmds().vlob_poll_changes(
RealmID(self.id.uuid), realm_checkpoint
)
except BackendNotAvailable:
raise
# Another backend error
except BackendConnectionError as exc:
logger.warning("Unexpected backend response during sync bootstrap", exc_info=exc)
return False
if rep["status"] == "not_found":
# Workspace not yet synchronized with backend
new_checkpoint = 0
changes = {}
elif rep["status"] in ("in_maintenance", "not_allowed"):
return False
elif rep["status"] != "ok":
return False
else:
new_checkpoint = rep["current_checkpoint"]
changes = rep["changes"]
# 2) Store new checkpoint and changes
await self._get_local_storage().update_realm_checkpoint(
new_checkpoint, {EntryID.from_hex(name.hex): val for name, val in changes.items()}
)
# 3) Compute local and remote changes that need to be synced
need_sync_local, need_sync_remote = await self._get_local_storage().get_need_sync_entries()
now = current_time()
# Ignore local changes in read only mode
if not self.read_only:
self._local_changes = {entry_id: LocalChange(now) for entry_id in need_sync_local}
self._remote_changes = need_sync_remote
# 4) Finally refresh due time according to the changes
self._compute_due_time()
self._changes_loaded = True
return True
def create_manifest(device, type=LocalWorkspaceManifest, use_legacy_none_author=False):
author = device.device_id
if type is LocalUserManifest:
manifest = LocalUserManifest.new_placeholder(author)
elif type is LocalWorkspaceManifest:
manifest = type.new_placeholder(author)
else:
manifest = type.new_placeholder(author, parent=EntryID())
if use_legacy_none_author:
base = manifest.base.evolve(author=None)
manifest = manifest.evolve(base=base)
return manifest
async def make_workspace_dir_inconsistent(workspace: WorkspaceFS, dir: FsPath):
await workspace.mkdir(dir)
await workspace.touch(dir / "foo.txt")
rep_info = await workspace.transactions.entry_info(dir)
rep_manifest = await workspace.local_storage.get_manifest(rep_info["id"])
children = rep_manifest.children
children["newfail.txt"] = EntryID("b9295787-d9aa-6cbd-be27-1ff83ac72fa6")
rep_manifest.evolve(children=children)
async with workspace.local_storage.lock_manifest(rep_info["id"]):
await workspace.local_storage.set_manifest(rep_info["id"],
rep_manifest)
await workspace.sync()
async def test_realm_checkpoint(alice_workspace_storage):
aws = alice_workspace_storage
manifest = create_manifest(aws.device, LocalFileManifest)
assert await aws.get_realm_checkpoint() == 0
# Workspace storage starts with a speculative workspace manifest placeholder
assert await aws.get_need_sync_entries() == ({aws.workspace_id}, set())
workspace_manifest = create_manifest(aws.device, LocalWorkspaceManifest)
base = workspace_manifest.to_remote(aws.device.device_id,
timestamp=aws.device.timestamp())
workspace_manifest = workspace_manifest.evolve(base=base, need_sync=False)
await aws.set_manifest(aws.workspace_id,
workspace_manifest,
check_lock_status=False)
assert await aws.get_realm_checkpoint() == 0
assert await aws.get_need_sync_entries() == (set(), set())
await aws.update_realm_checkpoint(11, {manifest.id: 22, EntryID.new(): 33})
assert await aws.get_realm_checkpoint() == 11
assert await aws.get_need_sync_entries() == (set(), set())
await aws.set_manifest(manifest.id, manifest, check_lock_status=False)
assert await aws.get_realm_checkpoint() == 11
assert await aws.get_need_sync_entries() == (set([manifest.id]), set())
await aws.set_manifest(manifest.id,
manifest.evolve(need_sync=False),
check_lock_status=False)
assert await aws.get_realm_checkpoint() == 11
assert await aws.get_need_sync_entries() == (set(), set())
await aws.update_realm_checkpoint(44, {manifest.id: 55, EntryID.new(): 66})
assert await aws.get_realm_checkpoint() == 44
assert await aws.get_need_sync_entries() == (set(), set([manifest.id]))
def test_merge_manifests_with_a_placeholder():
my_device = DeviceID("b@b")
other_device = DeviceID("a@a")
parent = EntryID()
m1 = LocalFolderManifest.new_placeholder(my_device, parent=parent)
m2 = merge_manifests(my_device, empty_pattern, m1)
assert m2 == m1
v1 = m1.to_remote(author=my_device)
m2a = merge_manifests(my_device, empty_pattern, m1, v1)
assert m2a == LocalFolderManifest.from_remote(v1, empty_pattern)
m2b = m1.evolve_children_and_mark_updated({"a": EntryID()}, empty_pattern)
m3b = merge_manifests(my_device, empty_pattern, m2b, v1)
assert m3b == m2b.evolve(base=v1)
v2 = v1.evolve(version=2, author=other_device, children={"b": EntryID()})
m2c = m1.evolve_children_and_mark_updated({"a": EntryID()}, empty_pattern)
m3c = merge_manifests(my_device, empty_pattern, m2c, v2)
children = {**v2.children, **m2c.children}
assert m3c == m2c.evolve(base=v2, children=children, updated=m3c.updated)
def generate_new_device(device_id: DeviceID,
organization_addr: BackendOrganizationAddr,
is_admin: bool = False) -> LocalDevice:
return LocalDevice(
organization_addr=organization_addr,
device_id=device_id,
signing_key=SigningKey.generate(),
private_key=PrivateKey.generate(),
is_admin=is_admin,
user_manifest_id=EntryID(uuid4().hex),
user_manifest_key=SecretKey.generate(),
local_symkey=SecretKey.generate(),
)
def test_merge_folder_children():
m1 = EntryID()
m2 = EntryID()
a1 = {"a": m1}
a2 = {"a": m2}
b1 = {"b.txt": m1}
b2 = {"b.txt": m2}
c1 = {"c.tar.gz": m1}
c2 = {"c.tar.gz": m2}
# Empty folder
assert merge_folder_children({}, {}, {}, "a@a") == {}
# Adding children
assert merge_folder_children({}, a1, {}, "a@a") == a1
assert merge_folder_children({}, {}, a1, "a@a") == a1
assert merge_folder_children({}, a1, a1, "a@a") == a1
# Removing children
assert merge_folder_children(a1, {}, a1, "a@a") == {}
assert merge_folder_children(a1, a1, {}, "a@a") == {}
assert merge_folder_children(a1, {}, {}, "a@a") == {}
# Renaming children
assert merge_folder_children(a1, a1, b1, "a@a") == b1
assert merge_folder_children(a1, b1, a1, "a@a") == b1
assert merge_folder_children(a1, b1, b1, "a@a") == b1
# Conflicting renaming
result = merge_folder_children(a1, b1, c1, "a@a")
assert result == {"c (renamed by a@a).tar.gz": m1}
# Conflicting names
result = merge_folder_children({}, a1, a2, "a@a")
assert result == {"a": m2, "a (conflicting with a@a)": m1}
result = merge_folder_children({}, b1, b2, "a@a")
assert result == {"b.txt": m2, "b (conflicting with a@a).txt": m1}
result = merge_folder_children({}, c1, c2, "a@a")
assert result == {"c.tar.gz": m2, "c (conflicting with a@a).tar.gz": m1}
def test_merge_folder_manifests():
my_device = DeviceID("b@b")
other_device = DeviceID("a@a")
parent = EntryID()
v1 = LocalFolderManifest.new_placeholder(
my_device, parent=parent).to_remote(author=other_device)
# Initial base manifest
m1 = LocalFolderManifest.from_remote(v1, empty_pattern)
assert merge_manifests(my_device, empty_pattern, m1) == m1
# Local change
m2 = m1.evolve_children_and_mark_updated({"a": EntryID()}, empty_pattern)
assert merge_manifests(my_device, empty_pattern, m2) == m2
# Successful upload
v2 = m2.to_remote(author=my_device)
m3 = merge_manifests(my_device, empty_pattern, m2, v2)
assert m3 == LocalFolderManifest.from_remote(v2, empty_pattern)
# Two local changes
m4 = m3.evolve_children_and_mark_updated({"b": EntryID()}, empty_pattern)
assert merge_manifests(my_device, empty_pattern, m4) == m4
m5 = m4.evolve_children_and_mark_updated({"c": EntryID()}, empty_pattern)
assert merge_manifests(my_device, empty_pattern, m4) == m4
# M4 has been successfully uploaded
v3 = m4.to_remote(author=my_device)
m6 = merge_manifests(my_device, empty_pattern, m5, v3)
assert m6 == m5.evolve(base=v3)
# The remote has changed
v4 = v3.evolve(version=4,
children={
"d": EntryID(),
**v3.children
},
author=other_device)
m7 = merge_manifests(my_device, empty_pattern, m6, v4)
assert m7.base_version == 4
assert sorted(m7.children) == ["a", "b", "c", "d"]
assert m7.need_sync
# Successful upload
v5 = m7.to_remote(author=my_device)
m8 = merge_manifests(my_device, empty_pattern, m7, v5)
assert m8 == LocalFolderManifest.from_remote(v5, empty_pattern)
# The remote has changed
v6 = v5.evolve(version=6,
children={
"e": EntryID(),
**v5.children
},
author=other_device)
m9 = merge_manifests(my_device, empty_pattern, m8, v6)
assert m9 == LocalFolderManifest.from_remote(v6, empty_pattern)
def test_merge_file_manifests(alice, bob):
timestamp = alice.timestamp()
my_device = alice.device_id
other_device = bob.device_id
parent = EntryID.new()
v1 = LocalFileManifest.new_placeholder(my_device,
parent=parent,
timestamp=timestamp).to_remote(
author=other_device,
timestamp=timestamp)
def evolve(m, n):
chunk = Chunk.new(0, n).evolve_as_block(b"a" * n)
blocks = ((chunk, ), )
return m1.evolve_and_mark_updated(size=n,
blocks=blocks,
timestamp=timestamp)
# Initial base manifest
m1 = LocalFileManifest.from_remote(v1)
assert merge_manifests(my_device, timestamp, empty_pattern, m1) == m1
# Local change
m2 = evolve(m1, 1)
assert merge_manifests(my_device, timestamp, empty_pattern, m2) == m2
# Successful upload
v2 = m2.to_remote(author=my_device, timestamp=timestamp)
m3 = merge_manifests(my_device, timestamp, empty_pattern, m2, v2)
assert m3 == LocalFileManifest.from_remote(v2)
# Two local changes
m4 = evolve(m3, 2)
assert merge_manifests(my_device, timestamp, empty_pattern, m4) == m4
m5 = evolve(m4, 3)
assert merge_manifests(my_device, timestamp, empty_pattern, m4) == m4
# M4 has been successfully uploaded
v3 = m4.to_remote(author=my_device, timestamp=timestamp)
m6 = merge_manifests(my_device, timestamp, empty_pattern, m5, v3)
assert m6 == m5.evolve(base=v3)
# The remote has changed
v4 = v3.evolve(version=4, size=0, author=other_device)
with pytest.raises(FSFileConflictError):
merge_manifests(my_device, timestamp, empty_pattern, m6, v4)
async def test_realm_checkpoint(alice, alice_user_storage,
initial_user_manifest_state):
aws = alice_user_storage
user_manifest_id = alice.user_manifest_id
# Brand new user storage contains user manifest placeholder
assert await aws.get_realm_checkpoint() == 0
assert await aws.get_need_sync_entries() == ({user_manifest_id}, set())
# Modified entries not in storage should be ignored
await aws.update_realm_checkpoint(1, {EntryID.new(): 2})
assert await aws.get_realm_checkpoint() == 1
assert await aws.get_need_sync_entries() == ({user_manifest_id}, set())
# Another device updated the user manifest remotly
await aws.update_realm_checkpoint(2, {user_manifest_id: 1})
assert await aws.get_realm_checkpoint() == 2
assert await aws.get_need_sync_entries() == ({user_manifest_id},
{user_manifest_id})
# Load the up to date version of the user manifest
manifest_v1 = initial_user_manifest_state.get_user_manifest_v1_for_device(
alice)
await aws.set_user_manifest(manifest_v1)
assert await aws.get_realm_checkpoint() == 2
assert await aws.get_need_sync_entries() == (set(), set())
# Provide new remote changes
await aws.update_realm_checkpoint(3, {user_manifest_id: 2})
assert await aws.get_realm_checkpoint() == 3
assert await aws.get_need_sync_entries() == (set(), {user_manifest_id})
# Provide new local changes too
manifest_v1_modified = manifest_v1.evolve(need_sync=True)
await aws.set_user_manifest(manifest_v1_modified)
assert await aws.get_realm_checkpoint() == 3
assert await aws.get_need_sync_entries() == ({user_manifest_id},
{user_manifest_id})
# Checkpoint's remote version should be ignored if manifest base version is greater
manifest_v2 = manifest_v1_modified.evolve(base=manifest_v1.base.evolve(
version=4))
await aws.set_user_manifest(manifest_v2)
assert await aws.get_realm_checkpoint() == 3
assert await aws.get_need_sync_entries() == ({user_manifest_id}, set())
async def get_need_sync_entries(self) -> Tuple[Set[EntryID], Set[EntryID]]:
"""
Raises: Nothing !
"""
async with self._open_cursor() as cursor:
cursor.execute(
"SELECT vlob_id, need_sync, base_version, remote_version "
"FROM vlobs WHERE need_sync = 1 OR base_version != remote_version"
)
local_changes = set()
remote_changes = set()
for manifest_id, need_sync, bv, rv in cursor.fetchall():
manifest_id = EntryID(manifest_id)
if need_sync:
local_changes.add(manifest_id)
if bv != rv:
remote_changes.add(manifest_id)
return local_changes, remote_changes
async def make_workspace_dir_inconsistent(workspace: WorkspaceFS, dir: FsPath):
"""
Create directory and make it inconsistent by adding an entry refering
to an unknown EntryID.
"""
await workspace.mkdir(dir)
await workspace.touch(dir / "foo.txt")
rep_info = await workspace.transactions.entry_info(dir)
rep_manifest = await workspace.local_storage.get_manifest(rep_info["id"])
children = rep_manifest.children
children[EntryName("newfail.txt")] = EntryID.from_hex(
"b9295787-d9aa-6cbd-be27-1ff83ac72fa6")
rep_manifest = rep_manifest.evolve(children=children)
async with workspace.local_storage.lock_manifest(rep_info["id"]):
await workspace.local_storage.set_manifest(rep_info["id"],
rep_manifest)
await workspace.sync()
def selected_files(self):
files = []
# As it turns out, Qt can return several overlapping ranges
# Fix the overlap by using a sorted set
rows = {
row
for r in self.selectedRanges()
for row in range(r.topRow(),
r.bottomRow() + 1)
}
for row in sorted(rows):
item = self.item(row, Column.NAME)
files.append(
SelectedFile(
row,
item.data(TYPE_DATA_INDEX),
item.data(NAME_DATA_INDEX),
EntryID.from_hex(item.data(ENTRY_ID_DATA_INDEX)),
))
return files
