def test_merge_manifests_with_a_placeholder(alice, bob): timestamp = alice.timestamp() my_device = alice.device_id other_device = bob.device_id parent = EntryID.new() m1 = LocalFolderManifest.new_placeholder(my_device, parent=parent, timestamp=timestamp) m2 = merge_manifests(my_device, timestamp, empty_pattern, m1) assert m2 == m1 v1 = m1.to_remote(author=my_device, timestamp=timestamp) m2a = merge_manifests(my_device, timestamp, empty_pattern, m1, v1) assert m2a == LocalFolderManifest.from_remote(v1, empty_pattern) m2b = m1.evolve_children_and_mark_updated({EntryName("a"): EntryID.new()}, empty_pattern, timestamp=timestamp) m3b = merge_manifests(my_device, timestamp, empty_pattern, m2b, v1) assert m3b == m2b.evolve(base=v1) v2 = v1.evolve(version=2, author=other_device, children={EntryName("b"): EntryID.new()}) m2c = m1.evolve_children_and_mark_updated({EntryName("a"): EntryID.new()}, empty_pattern, timestamp=timestamp) m3c = merge_manifests(my_device, timestamp, empty_pattern, m2c, v2) children = {**v2.children, **m2c.children} assert m3c == m2c.evolve(base=v2, children=children, updated=m3c.updated)
async def test_realm_checkpoint(alice_workspace_storage): aws = alice_workspace_storage manifest = create_manifest(aws.device, LocalFileManifest) assert await aws.get_realm_checkpoint() == 0 assert await aws.get_need_sync_entries() == (set(), set()) await aws.update_realm_checkpoint(11, {manifest.id: 22, EntryID(): 33}) assert await aws.get_realm_checkpoint() == 11 assert await aws.get_need_sync_entries() == (set(), set()) await aws.set_manifest(manifest.id, manifest, check_lock_status=False) assert await aws.get_realm_checkpoint() == 11 assert await aws.get_need_sync_entries() == (set([manifest.id]), set()) await aws.set_manifest(manifest.id, manifest.evolve(need_sync=False), check_lock_status=False) assert await aws.get_realm_checkpoint() == 11 assert await aws.get_need_sync_entries() == (set(), set()) await aws.update_realm_checkpoint(44, {manifest.id: 55, EntryID(): 66}) assert await aws.get_realm_checkpoint() == 44 assert await aws.get_need_sync_entries() == (set(), set([manifest.id]))
def create_manifest(device, type=LocalWorkspaceManifest): if type is LocalUserManifest: manifest = LocalUserManifest.new_placeholder(parent=EntryID()) elif type is LocalWorkspaceManifest: manifest = type.new_placeholder() else: manifest = type.new_placeholder(parent=EntryID()) return manifest
def test_build_addrs(): backend_addr = BackendAddr.from_url(BackendAddrTestbed.url) assert backend_addr.hostname == "parsec.cloud.com" assert backend_addr.port == 443 assert backend_addr.use_ssl is True organization_id = OrganizationID("MyOrg") root_verify_key = SigningKey.generate().verify_key organization_addr = BackendOrganizationAddr.build( backend_addr=backend_addr, organization_id=organization_id, root_verify_key=root_verify_key ) assert organization_addr.organization_id == organization_id assert organization_addr.root_verify_key == root_verify_key organization_bootstrap_addr = BackendOrganizationBootstrapAddr.build( backend_addr=backend_addr, organization_id=organization_id, token="a0000000000000000000000000000001", ) assert organization_bootstrap_addr.token == "a0000000000000000000000000000001" assert organization_bootstrap_addr.organization_id == organization_id organization_bootstrap_addr2 = BackendOrganizationBootstrapAddr.build( backend_addr=backend_addr, organization_id=organization_id, token=None ) assert organization_bootstrap_addr2.organization_id == organization_id assert organization_bootstrap_addr2.token == "" organization_file_link_addr = BackendOrganizationFileLinkAddr.build( organization_addr=organization_addr, workspace_id=EntryID.from_hex("2d4ded12-7406-4608-833b-7f57f01156e2"), encrypted_path=b"<encrypted_payload>", ) assert organization_file_link_addr.workspace_id == EntryID.from_hex( "2d4ded12-7406-4608-833b-7f57f01156e2" ) assert organization_file_link_addr.encrypted_path == b"<encrypted_payload>" invitation_addr = BackendInvitationAddr.build( backend_addr=backend_addr, organization_id=organization_id, invitation_type=InvitationType.USER, token=InvitationToken.from_hex("a0000000000000000000000000000001"), ) assert invitation_addr.organization_id == organization_id assert invitation_addr.token == InvitationToken.from_hex("a0000000000000000000000000000001") assert invitation_addr.invitation_type == InvitationType.USER
async def test_timestamped_storage(alice_workspace_storage): timestamp = now() aws = alice_workspace_storage taws = aws.to_timestamped(timestamp) assert taws.timestamp == timestamp assert taws.device == aws.device assert taws.path == aws.path assert taws.workspace_id == aws.workspace_id assert taws.manifest_storage is None assert taws.block_storage == aws.block_storage assert taws.chunk_storage == aws.chunk_storage with pytest.raises(FSError): await taws.set_chunk("chunk id", "data") with pytest.raises(FSError): await taws.clear_chunk("chunk id") with pytest.raises(FSError): await taws.clear_manifest("manifest id") with pytest.raises(FSLocalMissError): await taws.get_manifest(EntryID()) manifest = create_manifest(aws.device) with pytest.raises(FSError): await taws.set_manifest(manifest.id, manifest) manifest = manifest.evolve(need_sync=False) async with taws.lock_entry_id(manifest.id): await taws.set_manifest(manifest.id, manifest) assert await taws.get_manifest(manifest.id) == manifest # No-op await taws.ensure_manifest_persistent(manifest.id)
def __init__(self) -> None: super().__init__() self.oracle = open(tmpdir / "oracle.txt", "w+b") self.manifest = LocalFileManifest.new_placeholder(DeviceID.new(), parent=EntryID(), blocksize=8) self.storage = Storage()
async def init(self): nonlocal tentative tentative += 1 await self.reset_all() await self.start_backend() self.device = alice await self.start_transactions() self.file_transactions = self.transactions_controller.file_transactions self.local_storage = self.file_transactions.local_storage self.fresh_manifest = LocalFileManifest.new_placeholder( alice.device_id, parent=EntryID.new(), timestamp=alice.timestamp()) self.entry_id = self.fresh_manifest.id async with self.local_storage.lock_entry_id(self.entry_id): await self.local_storage.set_manifest(self.entry_id, self.fresh_manifest) self.fd = self.local_storage.create_file_descriptor( self.fresh_manifest) self.file_oracle_path = tmpdir / f"oracle-test-{tentative}.txt" self.file_oracle_fd = os.open(self.file_oracle_path, os.O_RDWR | os.O_CREAT)
async def test_link_file_unknown_org( aqtbot, core_config, gui_factory, autoclose_dialog, running_backend, alice ): password = "******" save_device_with_password_in_config(core_config.config_dir, alice, password) # Cheating a bit but it does not matter, we just want a link that appears valid with # an unknown organization org_addr = BackendOrganizationAddr.build( running_backend.addr, OrganizationID("UnknownOrg"), alice.organization_addr.root_verify_key ) file_link = BackendOrganizationFileLinkAddr.build( organization_addr=org_addr, workspace_id=EntryID.new(), encrypted_path=b"<whatever>" ) gui = await gui_factory(core_config=core_config, start_arg=file_link.to_url()) lw = gui.test_get_login_widget() assert len(autoclose_dialog.dialogs) == 1 assert autoclose_dialog.dialogs[0][0] == "Error" assert autoclose_dialog.dialogs[0][1] == translate( "TEXT_FILE_LINK_NOT_IN_ORG_organization" ).format(organization="UnknownOrg") def _devices_listed(): assert lw.widget.layout().count() > 0 await aqtbot.wait_until(_devices_listed) accounts_w = lw.widget.layout().itemAt(0).widget() assert accounts_w assert isinstance(accounts_w, LoginPasswordInputWidget)
async def test_link_file_unknown_org(core_config, gui_factory, autoclose_dialog, running_backend, alice): password = "******" save_device_with_password(core_config.config_dir, alice, password) # Cheating a bit but it does not matter, we just want a link that appears valid with # an unknown organization org_addr = BackendOrganizationAddr.build( running_backend.addr, "UnknownOrg", alice.organization_addr.root_verify_key) file_link = BackendOrganizationFileLinkAddr.build( org_addr, EntryID(), FsPath("/doesntmattereither")) gui = await gui_factory(core_config=core_config, start_arg=file_link.to_url()) lw = gui.test_get_login_widget() assert len(autoclose_dialog.dialogs) == 1 assert autoclose_dialog.dialogs[0][0] == "Error" assert autoclose_dialog.dialogs[0][1] == translate( "TEXT_FILE_LINK_NOT_IN_ORG_organization").format( organization="UnknownOrg") accounts_w = lw.widget.layout().itemAt(0).widget() assert accounts_w assert isinstance(accounts_w, LoginPasswordInputWidget)
async def test_path_info_remote_loader_exceptions(monkeypatch, alice_workspace, alice): manifest, _ = await alice_workspace.transactions._get_manifest_from_path( FsPath("/foo/bar")) async with alice_workspace.local_storage.lock_entry_id(manifest.id): await alice_workspace.local_storage.clear_manifest(manifest.id) vanilla_file_manifest_deserialize = BaseRemoteManifest._deserialize def mocked_file_manifest_deserialize(*args, **kwargs): return vanilla_file_manifest_deserialize( *args, **kwargs).evolve(**manifest_modifiers) monkeypatch.setattr(BaseRemoteManifest, "_deserialize", mocked_file_manifest_deserialize) manifest_modifiers = {"id": EntryID.new()} with pytest.raises(FSError) as exc: await alice_workspace.path_info(FsPath("/foo/bar")) assert f"Invalid entry ID: expected `{manifest.id}`, got `{manifest_modifiers['id']}`" in str( exc.value) manifest_modifiers = {"version": 4} with pytest.raises(FSError) as exc: await alice_workspace.path_info(FsPath("/foo/bar")) assert "Invalid version: expected `1`, got `4`" in str(exc.value) manifest_modifiers = {"author": DeviceID("mallory@pc1")} with pytest.raises(FSError) as exc: await alice_workspace.path_info(FsPath("/foo/bar")) assert "Invalid author: expected `alice@dev1`, got `mallory@pc1`" in str( exc.value)
async def test_synchronization_step_transaction(alice_sync_transactions, type): sync_transactions = alice_sync_transactions synchronization_step = sync_transactions.synchronization_step entry_id = sync_transactions.get_workspace_entry().id # Sync a placeholder manifest = await synchronization_step(entry_id) # Acknowledge a successful synchronization assert await synchronization_step(entry_id, manifest) is None # Local change if type == "file": a_id, fd = await sync_transactions.file_create(FsPath("/a")) await sync_transactions.fd_write(fd, b"abc", 0) await sync_transactions.fd_close(fd) else: a_id = await sync_transactions.folder_create(FsPath("/a")) # Sync parent with a placeholder child manifest = await synchronization_step(entry_id) children = [] async for child in sync_transactions.get_placeholder_children(manifest): children.append(child) a_entry_id, = children assert a_entry_id == a_id # Sync child if type == "file": await synchronization_step(a_entry_id) a_manifest = await synchronization_step(a_entry_id) assert await synchronization_step(a_entry_id, a_manifest) is None # Acknowledge the manifest assert sorted(manifest.children) == ["a"] assert await synchronization_step(entry_id, manifest) is None # Local change b_id = await sync_transactions.folder_create(FsPath("/b")) # Remote change children = {**manifest.children, "c": EntryID()} manifest = manifest.evolve(version=5, children=children, author="b@b") # Sync parent with a placeholder child manifest = await synchronization_step(entry_id, manifest) children = [] async for child in sync_transactions.get_placeholder_children(manifest): children.append(child) b_entry_id, = children assert b_entry_id == b_id # Sync child b_manifest = await synchronization_step(b_entry_id) assert await synchronization_step(b_entry_id, b_manifest) is None # Acknowledge the manifest assert sorted(manifest.children) == ["a", "b", "c"] assert await synchronization_step(entry_id, manifest) is None
async def test_unknown_workspace(alice_user_fs): bad_wid = EntryID() with pytest.raises(FSWorkspaceNotFoundError): await alice_user_fs.workspace_start_reencryption(bad_wid) with pytest.raises(FSWorkspaceNotFoundError): await alice_user_fs.workspace_continue_reencryption(bad_wid)
async def _transactions_controlled_cb(started_cb): async with WorkspaceStorage.run( tmp_path / f"file_operations-{tentative}", alice, EntryID.new()) as local_storage: async with file_transactions_factory( self.device, local_storage=local_storage) as file_transactions: await started_cb(file_transactions=file_transactions)
async def _transactions_controlled_cb(started_cb): async with WorkspaceStorage.run(alice, Path("/dummy"), EntryID()) as local_storage: file_transactions = await file_transactions_factory( self.device, alice_backend_cmds, local_storage=local_storage) await started_cb(file_transactions=file_transactions)
async def foo_txt(alice, alice_file_transactions): local_storage = alice_file_transactions.local_storage now = Pendulum(2000, 1, 2) placeholder = LocalFileManifest.new_placeholder(parent=EntryID(), now=now) remote_v1 = placeholder.to_remote(author=alice.device_id, timestamp=now) manifest = LocalFileManifest.from_remote(remote_v1) async with local_storage.lock_entry_id(manifest.id): await local_storage.set_manifest(manifest.id, manifest) return File(local_storage, manifest)
def __init__(self) -> None: super().__init__() self.oracle = open(tmpdir / "oracle.txt", "w+b") self.manifest = LocalFileManifest.new_placeholder( alice.device_id, parent=EntryID.new(), blocksize=8, timestamp=alice.timestamp()) self.storage = Storage()
def _handle_event(event_bus: EventBus, rep: dict) -> None: if rep["status"] != "ok": logger.warning("Bad response to `events_listen` command", rep=rep) return if rep["event"] == APIEvent.MESSAGE_RECEIVED: event_bus.send(CoreEvent.BACKEND_MESSAGE_RECEIVED, index=rep["index"]) elif rep["event"] == APIEvent.PINGED: event_bus.send(CoreEvent.BACKEND_PINGED, ping=rep["ping"]) elif rep["event"] == APIEvent.REALM_ROLES_UPDATED: event_bus.send( CoreEvent.BACKEND_REALM_ROLES_UPDATED, realm_id=EntryID(rep["realm_id"].uuid), role=rep["role"], ) elif rep["event"] == APIEvent.REALM_VLOBS_UPDATED: event_bus.send( CoreEvent.BACKEND_REALM_VLOBS_UPDATED, realm_id=EntryID(rep["realm_id"].uuid), checkpoint=rep["checkpoint"], src_id=EntryID(rep["src_id"].uuid), src_version=rep["src_version"], ) elif rep["event"] == APIEvent.REALM_MAINTENANCE_STARTED: event_bus.send( CoreEvent.BACKEND_REALM_MAINTENANCE_STARTED, realm_id=EntryID(rep["realm_id"].uuid), encryption_revision=rep["encryption_revision"], ) elif rep["event"] == APIEvent.REALM_MAINTENANCE_FINISHED: event_bus.send( CoreEvent.BACKEND_REALM_MAINTENANCE_FINISHED, realm_id=EntryID(rep["realm_id"].uuid), encryption_revision=rep["encryption_revision"], ) elif rep["event"] == APIEvent.PKI_ENROLLMENTS_UPDATED: event_bus.send(CoreEvent.PKI_ENROLLMENTS_UPDATED)
async def _load_changes(self) -> bool: if self._changes_loaded: return True # Initialize due_time so that if we cannot retrieve the changes, we # will wait until an external event (most likely a `sharing.updated`) # make it worth to retry self.due_time = math.inf # 1) Fetch new checkpoint and changes realm_checkpoint = await self._get_local_storage().get_realm_checkpoint() try: rep = await self._get_backend_cmds().vlob_poll_changes( RealmID(self.id.uuid), realm_checkpoint ) except BackendNotAvailable: raise # Another backend error except BackendConnectionError as exc: logger.warning("Unexpected backend response during sync bootstrap", exc_info=exc) return False if rep["status"] == "not_found": # Workspace not yet synchronized with backend new_checkpoint = 0 changes = {} elif rep["status"] in ("in_maintenance", "not_allowed"): return False elif rep["status"] != "ok": return False else: new_checkpoint = rep["current_checkpoint"] changes = rep["changes"] # 2) Store new checkpoint and changes await self._get_local_storage().update_realm_checkpoint( new_checkpoint, {EntryID.from_hex(name.hex): val for name, val in changes.items()} ) # 3) Compute local and remote changes that need to be synced need_sync_local, need_sync_remote = await self._get_local_storage().get_need_sync_entries() now = current_time() # Ignore local changes in read only mode if not self.read_only: self._local_changes = {entry_id: LocalChange(now) for entry_id in need_sync_local} self._remote_changes = need_sync_remote # 4) Finally refresh due time according to the changes self._compute_due_time() self._changes_loaded = True return True
def create_manifest(device, type=LocalWorkspaceManifest, use_legacy_none_author=False): author = device.device_id if type is LocalUserManifest: manifest = LocalUserManifest.new_placeholder(author) elif type is LocalWorkspaceManifest: manifest = type.new_placeholder(author) else: manifest = type.new_placeholder(author, parent=EntryID()) if use_legacy_none_author: base = manifest.base.evolve(author=None) manifest = manifest.evolve(base=base) return manifest
async def make_workspace_dir_inconsistent(workspace: WorkspaceFS, dir: FsPath): await workspace.mkdir(dir) await workspace.touch(dir / "foo.txt") rep_info = await workspace.transactions.entry_info(dir) rep_manifest = await workspace.local_storage.get_manifest(rep_info["id"]) children = rep_manifest.children children["newfail.txt"] = EntryID("b9295787-d9aa-6cbd-be27-1ff83ac72fa6") rep_manifest.evolve(children=children) async with workspace.local_storage.lock_manifest(rep_info["id"]): await workspace.local_storage.set_manifest(rep_info["id"], rep_manifest) await workspace.sync()
async def test_realm_checkpoint(alice_workspace_storage): aws = alice_workspace_storage manifest = create_manifest(aws.device, LocalFileManifest) assert await aws.get_realm_checkpoint() == 0 # Workspace storage starts with a speculative workspace manifest placeholder assert await aws.get_need_sync_entries() == ({aws.workspace_id}, set()) workspace_manifest = create_manifest(aws.device, LocalWorkspaceManifest) base = workspace_manifest.to_remote(aws.device.device_id, timestamp=aws.device.timestamp()) workspace_manifest = workspace_manifest.evolve(base=base, need_sync=False) await aws.set_manifest(aws.workspace_id, workspace_manifest, check_lock_status=False) assert await aws.get_realm_checkpoint() == 0 assert await aws.get_need_sync_entries() == (set(), set()) await aws.update_realm_checkpoint(11, {manifest.id: 22, EntryID.new(): 33}) assert await aws.get_realm_checkpoint() == 11 assert await aws.get_need_sync_entries() == (set(), set()) await aws.set_manifest(manifest.id, manifest, check_lock_status=False) assert await aws.get_realm_checkpoint() == 11 assert await aws.get_need_sync_entries() == (set([manifest.id]), set()) await aws.set_manifest(manifest.id, manifest.evolve(need_sync=False), check_lock_status=False) assert await aws.get_realm_checkpoint() == 11 assert await aws.get_need_sync_entries() == (set(), set()) await aws.update_realm_checkpoint(44, {manifest.id: 55, EntryID.new(): 66}) assert await aws.get_realm_checkpoint() == 44 assert await aws.get_need_sync_entries() == (set(), set([manifest.id]))
def test_merge_manifests_with_a_placeholder(): my_device = DeviceID("b@b") other_device = DeviceID("a@a") parent = EntryID() m1 = LocalFolderManifest.new_placeholder(my_device, parent=parent) m2 = merge_manifests(my_device, empty_pattern, m1) assert m2 == m1 v1 = m1.to_remote(author=my_device) m2a = merge_manifests(my_device, empty_pattern, m1, v1) assert m2a == LocalFolderManifest.from_remote(v1, empty_pattern) m2b = m1.evolve_children_and_mark_updated({"a": EntryID()}, empty_pattern) m3b = merge_manifests(my_device, empty_pattern, m2b, v1) assert m3b == m2b.evolve(base=v1) v2 = v1.evolve(version=2, author=other_device, children={"b": EntryID()}) m2c = m1.evolve_children_and_mark_updated({"a": EntryID()}, empty_pattern) m3c = merge_manifests(my_device, empty_pattern, m2c, v2) children = {**v2.children, **m2c.children} assert m3c == m2c.evolve(base=v2, children=children, updated=m3c.updated)
def generate_new_device(device_id: DeviceID, organization_addr: BackendOrganizationAddr, is_admin: bool = False) -> LocalDevice: return LocalDevice( organization_addr=organization_addr, device_id=device_id, signing_key=SigningKey.generate(), private_key=PrivateKey.generate(), is_admin=is_admin, user_manifest_id=EntryID(uuid4().hex), user_manifest_key=SecretKey.generate(), local_symkey=SecretKey.generate(), )
def test_merge_folder_children(): m1 = EntryID() m2 = EntryID() a1 = {"a": m1} a2 = {"a": m2} b1 = {"b.txt": m1} b2 = {"b.txt": m2} c1 = {"c.tar.gz": m1} c2 = {"c.tar.gz": m2} # Empty folder assert merge_folder_children({}, {}, {}, "a@a") == {} # Adding children assert merge_folder_children({}, a1, {}, "a@a") == a1 assert merge_folder_children({}, {}, a1, "a@a") == a1 assert merge_folder_children({}, a1, a1, "a@a") == a1 # Removing children assert merge_folder_children(a1, {}, a1, "a@a") == {} assert merge_folder_children(a1, a1, {}, "a@a") == {} assert merge_folder_children(a1, {}, {}, "a@a") == {} # Renaming children assert merge_folder_children(a1, a1, b1, "a@a") == b1 assert merge_folder_children(a1, b1, a1, "a@a") == b1 assert merge_folder_children(a1, b1, b1, "a@a") == b1 # Conflicting renaming result = merge_folder_children(a1, b1, c1, "a@a") assert result == {"c (renamed by a@a).tar.gz": m1} # Conflicting names result = merge_folder_children({}, a1, a2, "a@a") assert result == {"a": m2, "a (conflicting with a@a)": m1} result = merge_folder_children({}, b1, b2, "a@a") assert result == {"b.txt": m2, "b (conflicting with a@a).txt": m1} result = merge_folder_children({}, c1, c2, "a@a") assert result == {"c.tar.gz": m2, "c (conflicting with a@a).tar.gz": m1}
def test_merge_folder_manifests(): my_device = DeviceID("b@b") other_device = DeviceID("a@a") parent = EntryID() v1 = LocalFolderManifest.new_placeholder( my_device, parent=parent).to_remote(author=other_device) # Initial base manifest m1 = LocalFolderManifest.from_remote(v1, empty_pattern) assert merge_manifests(my_device, empty_pattern, m1) == m1 # Local change m2 = m1.evolve_children_and_mark_updated({"a": EntryID()}, empty_pattern) assert merge_manifests(my_device, empty_pattern, m2) == m2 # Successful upload v2 = m2.to_remote(author=my_device) m3 = merge_manifests(my_device, empty_pattern, m2, v2) assert m3 == LocalFolderManifest.from_remote(v2, empty_pattern) # Two local changes m4 = m3.evolve_children_and_mark_updated({"b": EntryID()}, empty_pattern) assert merge_manifests(my_device, empty_pattern, m4) == m4 m5 = m4.evolve_children_and_mark_updated({"c": EntryID()}, empty_pattern) assert merge_manifests(my_device, empty_pattern, m4) == m4 # M4 has been successfully uploaded v3 = m4.to_remote(author=my_device) m6 = merge_manifests(my_device, empty_pattern, m5, v3) assert m6 == m5.evolve(base=v3) # The remote has changed v4 = v3.evolve(version=4, children={ "d": EntryID(), **v3.children }, author=other_device) m7 = merge_manifests(my_device, empty_pattern, m6, v4) assert m7.base_version == 4 assert sorted(m7.children) == ["a", "b", "c", "d"] assert m7.need_sync # Successful upload v5 = m7.to_remote(author=my_device) m8 = merge_manifests(my_device, empty_pattern, m7, v5) assert m8 == LocalFolderManifest.from_remote(v5, empty_pattern) # The remote has changed v6 = v5.evolve(version=6, children={ "e": EntryID(), **v5.children }, author=other_device) m9 = merge_manifests(my_device, empty_pattern, m8, v6) assert m9 == LocalFolderManifest.from_remote(v6, empty_pattern)
def test_merge_file_manifests(alice, bob): timestamp = alice.timestamp() my_device = alice.device_id other_device = bob.device_id parent = EntryID.new() v1 = LocalFileManifest.new_placeholder(my_device, parent=parent, timestamp=timestamp).to_remote( author=other_device, timestamp=timestamp) def evolve(m, n): chunk = Chunk.new(0, n).evolve_as_block(b"a" * n) blocks = ((chunk, ), ) return m1.evolve_and_mark_updated(size=n, blocks=blocks, timestamp=timestamp) # Initial base manifest m1 = LocalFileManifest.from_remote(v1) assert merge_manifests(my_device, timestamp, empty_pattern, m1) == m1 # Local change m2 = evolve(m1, 1) assert merge_manifests(my_device, timestamp, empty_pattern, m2) == m2 # Successful upload v2 = m2.to_remote(author=my_device, timestamp=timestamp) m3 = merge_manifests(my_device, timestamp, empty_pattern, m2, v2) assert m3 == LocalFileManifest.from_remote(v2) # Two local changes m4 = evolve(m3, 2) assert merge_manifests(my_device, timestamp, empty_pattern, m4) == m4 m5 = evolve(m4, 3) assert merge_manifests(my_device, timestamp, empty_pattern, m4) == m4 # M4 has been successfully uploaded v3 = m4.to_remote(author=my_device, timestamp=timestamp) m6 = merge_manifests(my_device, timestamp, empty_pattern, m5, v3) assert m6 == m5.evolve(base=v3) # The remote has changed v4 = v3.evolve(version=4, size=0, author=other_device) with pytest.raises(FSFileConflictError): merge_manifests(my_device, timestamp, empty_pattern, m6, v4)
async def test_realm_checkpoint(alice, alice_user_storage, initial_user_manifest_state): aws = alice_user_storage user_manifest_id = alice.user_manifest_id # Brand new user storage contains user manifest placeholder assert await aws.get_realm_checkpoint() == 0 assert await aws.get_need_sync_entries() == ({user_manifest_id}, set()) # Modified entries not in storage should be ignored await aws.update_realm_checkpoint(1, {EntryID.new(): 2}) assert await aws.get_realm_checkpoint() == 1 assert await aws.get_need_sync_entries() == ({user_manifest_id}, set()) # Another device updated the user manifest remotly await aws.update_realm_checkpoint(2, {user_manifest_id: 1}) assert await aws.get_realm_checkpoint() == 2 assert await aws.get_need_sync_entries() == ({user_manifest_id}, {user_manifest_id}) # Load the up to date version of the user manifest manifest_v1 = initial_user_manifest_state.get_user_manifest_v1_for_device( alice) await aws.set_user_manifest(manifest_v1) assert await aws.get_realm_checkpoint() == 2 assert await aws.get_need_sync_entries() == (set(), set()) # Provide new remote changes await aws.update_realm_checkpoint(3, {user_manifest_id: 2}) assert await aws.get_realm_checkpoint() == 3 assert await aws.get_need_sync_entries() == (set(), {user_manifest_id}) # Provide new local changes too manifest_v1_modified = manifest_v1.evolve(need_sync=True) await aws.set_user_manifest(manifest_v1_modified) assert await aws.get_realm_checkpoint() == 3 assert await aws.get_need_sync_entries() == ({user_manifest_id}, {user_manifest_id}) # Checkpoint's remote version should be ignored if manifest base version is greater manifest_v2 = manifest_v1_modified.evolve(base=manifest_v1.base.evolve( version=4)) await aws.set_user_manifest(manifest_v2) assert await aws.get_realm_checkpoint() == 3 assert await aws.get_need_sync_entries() == ({user_manifest_id}, set())
async def get_need_sync_entries(self) -> Tuple[Set[EntryID], Set[EntryID]]: """ Raises: Nothing ! """ async with self._open_cursor() as cursor: cursor.execute( "SELECT vlob_id, need_sync, base_version, remote_version " "FROM vlobs WHERE need_sync = 1 OR base_version != remote_version" ) local_changes = set() remote_changes = set() for manifest_id, need_sync, bv, rv in cursor.fetchall(): manifest_id = EntryID(manifest_id) if need_sync: local_changes.add(manifest_id) if bv != rv: remote_changes.add(manifest_id) return local_changes, remote_changes
async def make_workspace_dir_inconsistent(workspace: WorkspaceFS, dir: FsPath): """ Create directory and make it inconsistent by adding an entry refering to an unknown EntryID. """ await workspace.mkdir(dir) await workspace.touch(dir / "foo.txt") rep_info = await workspace.transactions.entry_info(dir) rep_manifest = await workspace.local_storage.get_manifest(rep_info["id"]) children = rep_manifest.children children[EntryName("newfail.txt")] = EntryID.from_hex( "b9295787-d9aa-6cbd-be27-1ff83ac72fa6") rep_manifest = rep_manifest.evolve(children=children) async with workspace.local_storage.lock_manifest(rep_info["id"]): await workspace.local_storage.set_manifest(rep_info["id"], rep_manifest) await workspace.sync()
def selected_files(self): files = [] # As it turns out, Qt can return several overlapping ranges # Fix the overlap by using a sorted set rows = { row for r in self.selectedRanges() for row in range(r.topRow(), r.bottomRow() + 1) } for row in sorted(rows): item = self.item(row, Column.NAME) files.append( SelectedFile( row, item.data(TYPE_DATA_INDEX), item.data(NAME_DATA_INDEX), EntryID.from_hex(item.data(ENTRY_ID_DATA_INDEX)), )) return files