async def test_realm_checkpoint(alice_workspace_storage): aws = alice_workspace_storage manifest = create_manifest(aws.device, LocalFileManifest) assert await aws.get_realm_checkpoint() == 0 assert await aws.get_need_sync_entries() == (set(), set()) await aws.update_realm_checkpoint(11, {manifest.id: 22, EntryID.new(): 33}) assert await aws.get_realm_checkpoint() == 11 assert await aws.get_need_sync_entries() == (set(), set()) await aws.set_manifest(manifest.id, manifest, check_lock_status=False) assert await aws.get_realm_checkpoint() == 11 assert await aws.get_need_sync_entries() == (set([manifest.id]), set()) await aws.set_manifest(manifest.id, manifest.evolve(need_sync=False), check_lock_status=False) assert await aws.get_realm_checkpoint() == 11 assert await aws.get_need_sync_entries() == (set(), set()) await aws.update_realm_checkpoint(44, {manifest.id: 55, EntryID.new(): 66}) assert await aws.get_realm_checkpoint() == 44 assert await aws.get_need_sync_entries() == (set(), set([manifest.id]))
def test_merge_manifests_with_a_placeholder(alice, bob): timestamp = alice.timestamp() my_device = alice.device_id other_device = bob.device_id parent = EntryID.new() m1 = LocalFolderManifest.new_placeholder(my_device, parent=parent, timestamp=timestamp) m2 = merge_manifests(my_device, timestamp, empty_pattern, m1) assert m2 == m1 v1 = m1.to_remote(author=my_device, timestamp=timestamp) m2a = merge_manifests(my_device, timestamp, empty_pattern, m1, v1) assert m2a == LocalFolderManifest.from_remote(v1, empty_pattern) m2b = m1.evolve_children_and_mark_updated({EntryName("a"): EntryID.new()}, empty_pattern, timestamp=timestamp) m3b = merge_manifests(my_device, timestamp, empty_pattern, m2b, v1) assert m3b == m2b.evolve(base=v1) v2 = v1.evolve(version=2, author=other_device, children={EntryName("b"): EntryID.new()}) m2c = m1.evolve_children_and_mark_updated({EntryName("a"): EntryID.new()}, empty_pattern, timestamp=timestamp) m3c = merge_manifests(my_device, timestamp, empty_pattern, m2c, v2) children = {**v2.children, **m2c.children} assert m3c == m2c.evolve(base=v2, children=children, updated=m3c.updated)
async def init(self): nonlocal tentative tentative += 1 await self.reset_all() await self.start_backend() self.device = alice await self.start_transactions() self.file_transactions = self.transactions_controller.file_transactions self.local_storage = self.file_transactions.local_storage self.fresh_manifest = LocalFileManifest.new_placeholder( alice.device_id, parent=EntryID.new(), timestamp=alice.timestamp()) self.entry_id = self.fresh_manifest.id async with self.local_storage.lock_entry_id(self.entry_id): await self.local_storage.set_manifest(self.entry_id, self.fresh_manifest) self.fd = self.local_storage.create_file_descriptor( self.fresh_manifest) self.file_oracle_path = tmpdir / f"oracle-test-{tentative}.txt" self.file_oracle_fd = os.open(self.file_oracle_path, os.O_RDWR | os.O_CREAT)
async def test_link_file_unknown_org( aqtbot, core_config, gui_factory, autoclose_dialog, running_backend, alice ): password = "******" save_device_with_password_in_config(core_config.config_dir, alice, password) # Cheating a bit but it does not matter, we just want a link that appears valid with # an unknown organization org_addr = BackendOrganizationAddr.build( running_backend.addr, OrganizationID("UnknownOrg"), alice.organization_addr.root_verify_key ) file_link = BackendOrganizationFileLinkAddr.build( organization_addr=org_addr, workspace_id=EntryID.new(), encrypted_path=b"<whatever>" ) gui = await gui_factory(core_config=core_config, start_arg=file_link.to_url()) lw = gui.test_get_login_widget() assert len(autoclose_dialog.dialogs) == 1 assert autoclose_dialog.dialogs[0][0] == "Error" assert autoclose_dialog.dialogs[0][1] == translate( "TEXT_FILE_LINK_NOT_IN_ORG_organization" ).format(organization="UnknownOrg") def _devices_listed(): assert lw.widget.layout().count() > 0 await aqtbot.wait_until(_devices_listed) accounts_w = lw.widget.layout().itemAt(0).widget() assert accounts_w assert isinstance(accounts_w, LoginPasswordInputWidget)
async def test_path_info_remote_loader_exceptions(monkeypatch, alice_workspace, alice): manifest, _ = await alice_workspace.transactions._get_manifest_from_path( FsPath("/foo/bar")) async with alice_workspace.local_storage.lock_entry_id(manifest.id): await alice_workspace.local_storage.clear_manifest(manifest.id) vanilla_file_manifest_deserialize = BaseRemoteManifest._deserialize def mocked_file_manifest_deserialize(*args, **kwargs): return vanilla_file_manifest_deserialize( *args, **kwargs).evolve(**manifest_modifiers) monkeypatch.setattr(BaseRemoteManifest, "_deserialize", mocked_file_manifest_deserialize) manifest_modifiers = {"id": EntryID.new()} with pytest.raises(FSError) as exc: await alice_workspace.path_info(FsPath("/foo/bar")) assert f"Invalid entry ID: expected `{manifest.id}`, got `{manifest_modifiers['id']}`" in str( exc.value) manifest_modifiers = {"version": 4} with pytest.raises(FSError) as exc: await alice_workspace.path_info(FsPath("/foo/bar")) assert "Invalid version: expected `1`, got `4`" in str(exc.value) manifest_modifiers = {"author": DeviceID("mallory@pc1")} with pytest.raises(FSError) as exc: await alice_workspace.path_info(FsPath("/foo/bar")) assert "Invalid author: expected `alice@dev1`, got `mallory@pc1`" in str( exc.value)
async def test_synchronization_step_transaction(alice_sync_transactions, type): sync_transactions = alice_sync_transactions synchronization_step = sync_transactions.synchronization_step entry_id = sync_transactions.get_workspace_entry().id # Sync a placeholder manifest = await synchronization_step(entry_id) # Acknowledge a successful synchronization assert await synchronization_step(entry_id, manifest) is None # Local change if type == "file": a_id, fd = await sync_transactions.file_create(FsPath("/a")) await sync_transactions.fd_write(fd, b"abc", 0) await sync_transactions.fd_close(fd) else: a_id = await sync_transactions.folder_create(FsPath("/a")) # Sync parent with a placeholder child manifest = await synchronization_step(entry_id) children = [] async for child in sync_transactions.get_placeholder_children(manifest): children.append(child) a_entry_id, = children assert a_entry_id == a_id # Sync child if type == "file": await synchronization_step(a_entry_id) a_manifest = await synchronization_step(a_entry_id) assert await synchronization_step(a_entry_id, a_manifest) is None # Acknowledge the manifest assert sorted(manifest.children) == ["a"] assert await synchronization_step(entry_id, manifest) is None # Local change b_id = await sync_transactions.folder_create(FsPath("/b")) # Remote change children = {**manifest.children, "c": EntryID.new()} manifest = manifest.evolve(version=5, children=children, author="b@b") # Sync parent with a placeholder child manifest = await synchronization_step(entry_id, manifest) children = [] async for child in sync_transactions.get_placeholder_children(manifest): children.append(child) b_entry_id, = children assert b_entry_id == b_id # Sync child b_manifest = await synchronization_step(b_entry_id) assert await synchronization_step(b_entry_id, b_manifest) is None # Acknowledge the manifest assert sorted(manifest.children) == ["a", "b", "c"] assert await synchronization_step(entry_id, manifest) is None
async def _transactions_controlled_cb(started_cb): async with WorkspaceStorage.run( tmp_path / f"file_operations-{tentative}", alice, EntryID.new()) as local_storage: async with file_transactions_factory( self.device, local_storage=local_storage) as file_transactions: await started_cb(file_transactions=file_transactions)
async def _transactions_controlled_cb(started_cb): async with WorkspaceStorage.run( alice, Path("/dummy"), EntryID.new()) as local_storage: file_transactions = await file_transactions_factory( self.device, alice_backend_cmds, local_storage=local_storage) await started_cb(file_transactions=file_transactions)
async def test_unknown_workspace(alice_user_fs): bad_wid = EntryID.new() with pytest.raises(FSWorkspaceNotFoundError): await alice_user_fs.workspace_start_reencryption(bad_wid) with pytest.raises(FSWorkspaceNotFoundError): await alice_user_fs.workspace_continue_reencryption(bad_wid)
def __init__(self) -> None: super().__init__() self.oracle = open(tmpdir / "oracle.txt", "w+b") self.manifest = LocalFileManifest.new_placeholder( alice.device_id, parent=EntryID.new(), blocksize=8, timestamp=alice.timestamp()) self.storage = Storage()
async def foo_txt(alice, alice_file_transactions): local_storage = alice_file_transactions.local_storage now = datetime(2000, 1, 2) placeholder = LocalFileManifest.new_placeholder(alice.device_id, parent=EntryID.new(), timestamp=now) remote_v1 = placeholder.to_remote(author=alice.device_id, timestamp=now) manifest = LocalFileManifest.from_remote(remote_v1) async with local_storage.lock_entry_id(manifest.id): await local_storage.set_manifest(manifest.id, manifest) return File(local_storage, manifest)
async def test_realm_checkpoint(alice_workspace_storage): aws = alice_workspace_storage manifest = create_manifest(aws.device, LocalFileManifest) assert await aws.get_realm_checkpoint() == 0 # Workspace storage starts with a speculative workspace manifest placeholder assert await aws.get_need_sync_entries() == ({aws.workspace_id}, set()) workspace_manifest = create_manifest(aws.device, LocalWorkspaceManifest) base = workspace_manifest.to_remote(aws.device.device_id, timestamp=aws.device.timestamp()) workspace_manifest = workspace_manifest.evolve(base=base, need_sync=False) await aws.set_manifest(aws.workspace_id, workspace_manifest, check_lock_status=False) assert await aws.get_realm_checkpoint() == 0 assert await aws.get_need_sync_entries() == (set(), set()) await aws.update_realm_checkpoint(11, {manifest.id: 22, EntryID.new(): 33}) assert await aws.get_realm_checkpoint() == 11 assert await aws.get_need_sync_entries() == (set(), set()) await aws.set_manifest(manifest.id, manifest, check_lock_status=False) assert await aws.get_realm_checkpoint() == 11 assert await aws.get_need_sync_entries() == (set([manifest.id]), set()) await aws.set_manifest(manifest.id, manifest.evolve(need_sync=False), check_lock_status=False) assert await aws.get_realm_checkpoint() == 11 assert await aws.get_need_sync_entries() == (set(), set()) await aws.update_realm_checkpoint(44, {manifest.id: 55, EntryID.new(): 66}) assert await aws.get_realm_checkpoint() == 44 assert await aws.get_need_sync_entries() == (set(), set([manifest.id]))
def test_merge_folder_children(): m1 = EntryID.new() m2 = EntryID.new() a1 = {"a": m1} a2 = {"a": m2} b1 = {"b.txt": m1} b2 = {"b.txt": m2} c1 = {"c.tar.gz": m1} c2 = {"c.tar.gz": m2} # Empty folder assert merge_folder_children({}, {}, {}, "a@a") == {} # Adding children assert merge_folder_children({}, a1, {}, "a@a") == a1 assert merge_folder_children({}, {}, a1, "a@a") == a1 assert merge_folder_children({}, a1, a1, "a@a") == a1 # Removing children assert merge_folder_children(a1, {}, a1, "a@a") == {} assert merge_folder_children(a1, a1, {}, "a@a") == {} assert merge_folder_children(a1, {}, {}, "a@a") == {} # Renaming children assert merge_folder_children(a1, a1, b1, "a@a") == b1 assert merge_folder_children(a1, b1, a1, "a@a") == b1 assert merge_folder_children(a1, b1, b1, "a@a") == b1 # Conflicting renaming result = merge_folder_children(a1, b1, c1, "a@a") assert result == {"c (renamed by a@a).tar.gz": m1} # Conflicting names result = merge_folder_children({}, a1, a2, "a@a") assert result == {"a": m2, "a (conflicting with a@a)": m1} result = merge_folder_children({}, b1, b2, "a@a") assert result == {"b.txt": m2, "b (conflicting with a@a).txt": m1} result = merge_folder_children({}, c1, c2, "a@a") assert result == {"c.tar.gz": m2, "c (conflicting with a@a).tar.gz": m1}
def test_merge_manifests_with_a_placeholder(): my_device = DeviceID("b@b") other_device = DeviceID("a@a") parent = EntryID.new() m1 = LocalFolderManifest.new_placeholder(my_device, parent=parent) m2 = merge_manifests(my_device, empty_pattern, m1) assert m2 == m1 v1 = m1.to_remote(author=my_device) m2a = merge_manifests(my_device, empty_pattern, m1, v1) assert m2a == LocalFolderManifest.from_remote(v1, empty_pattern) m2b = m1.evolve_children_and_mark_updated({"a": EntryID.new()}, empty_pattern) m3b = merge_manifests(my_device, empty_pattern, m2b, v1) assert m3b == m2b.evolve(base=v1) v2 = v1.evolve(version=2, author=other_device, children={"b": EntryID.new()}) m2c = m1.evolve_children_and_mark_updated({"a": EntryID.new()}, empty_pattern) m3c = merge_manifests(my_device, empty_pattern, m2c, v2) children = {**v2.children, **m2c.children} assert m3c == m2c.evolve(base=v2, children=children, updated=m3c.updated)
def create_manifest(device, type=LocalWorkspaceManifest, use_legacy_none_author=False): author = device.device_id if type is LocalUserManifest: manifest = LocalUserManifest.new_placeholder(author) elif type is LocalWorkspaceManifest: manifest = type.new_placeholder(author) else: manifest = type.new_placeholder(author, parent=EntryID.new()) if use_legacy_none_author: base = manifest.base.evolve(author=None) manifest = manifest.evolve(base=base) return manifest
async def test_timestamped_storage(alice_workspace_storage): timestamp = now() aws = alice_workspace_storage taws = aws.to_timestamped(timestamp) assert taws.timestamp == timestamp assert taws.device == aws.device assert taws.path == aws.path assert taws.workspace_id == aws.workspace_id assert taws.manifest_storage is None assert taws.block_storage == aws.block_storage assert taws.chunk_storage == aws.chunk_storage with pytest.raises(FSError): await taws.set_chunk("chunk id", "data") with pytest.raises(FSError): await taws.clear_chunk("chunk id") with pytest.raises(FSError): await taws.clear_manifest("manifest id") with pytest.raises(FSError): await taws.run_vacuum() with pytest.raises(FSError): await taws.get_need_sync_entries() with pytest.raises(FSError): await taws.get_realm_checkpoint() with pytest.raises(FSError): await taws.clear_memory_cache("flush") with pytest.raises(FSLocalMissError): await taws.get_manifest(EntryID.new()) manifest = create_manifest(aws.device) with pytest.raises(FSError): await taws.set_manifest(manifest.id, manifest) manifest = manifest.evolve(need_sync=False) async with taws.lock_entry_id(manifest.id): await taws.set_manifest(manifest.id, manifest) assert await taws.get_manifest(manifest.id) == manifest # No-op await taws.ensure_manifest_persistent(manifest.id)
def test_merge_file_manifests(alice, bob): timestamp = alice.timestamp() my_device = alice.device_id other_device = bob.device_id parent = EntryID.new() v1 = LocalFileManifest.new_placeholder(my_device, parent=parent, timestamp=timestamp).to_remote( author=other_device, timestamp=timestamp) def evolve(m, n): chunk = Chunk.new(0, n).evolve_as_block(b"a" * n) blocks = ((chunk, ), ) return m1.evolve_and_mark_updated(size=n, blocks=blocks, timestamp=timestamp) # Initial base manifest m1 = LocalFileManifest.from_remote(v1) assert merge_manifests(my_device, timestamp, empty_pattern, m1) == m1 # Local change m2 = evolve(m1, 1) assert merge_manifests(my_device, timestamp, empty_pattern, m2) == m2 # Successful upload v2 = m2.to_remote(author=my_device, timestamp=timestamp) m3 = merge_manifests(my_device, timestamp, empty_pattern, m2, v2) assert m3 == LocalFileManifest.from_remote(v2) # Two local changes m4 = evolve(m3, 2) assert merge_manifests(my_device, timestamp, empty_pattern, m4) == m4 m5 = evolve(m4, 3) assert merge_manifests(my_device, timestamp, empty_pattern, m4) == m4 # M4 has been successfully uploaded v3 = m4.to_remote(author=my_device, timestamp=timestamp) m6 = merge_manifests(my_device, timestamp, empty_pattern, m5, v3) assert m6 == m5.evolve(base=v3) # The remote has changed v4 = v3.evolve(version=4, size=0, author=other_device) with pytest.raises(FSFileConflictError): merge_manifests(my_device, timestamp, empty_pattern, m6, v4)
async def test_realm_checkpoint(alice, alice_user_storage, initial_user_manifest_state): aws = alice_user_storage user_manifest_id = alice.user_manifest_id # Brand new user storage contains user manifest placeholder assert await aws.get_realm_checkpoint() == 0 assert await aws.get_need_sync_entries() == ({user_manifest_id}, set()) # Modified entries not in storage should be ignored await aws.update_realm_checkpoint(1, {EntryID.new(): 2}) assert await aws.get_realm_checkpoint() == 1 assert await aws.get_need_sync_entries() == ({user_manifest_id}, set()) # Another device updated the user manifest remotly await aws.update_realm_checkpoint(2, {user_manifest_id: 1}) assert await aws.get_realm_checkpoint() == 2 assert await aws.get_need_sync_entries() == ({user_manifest_id}, {user_manifest_id}) # Load the up to date version of the user manifest manifest_v1 = initial_user_manifest_state.get_user_manifest_v1_for_device( alice) await aws.set_user_manifest(manifest_v1) assert await aws.get_realm_checkpoint() == 2 assert await aws.get_need_sync_entries() == (set(), set()) # Provide new remote changes await aws.update_realm_checkpoint(3, {user_manifest_id: 2}) assert await aws.get_realm_checkpoint() == 3 assert await aws.get_need_sync_entries() == (set(), {user_manifest_id}) # Provide new local changes too manifest_v1_modified = manifest_v1.evolve(need_sync=True) await aws.set_user_manifest(manifest_v1_modified) assert await aws.get_realm_checkpoint() == 3 assert await aws.get_need_sync_entries() == ({user_manifest_id}, {user_manifest_id}) # Checkpoint's remote version should be ignored if manifest base version is greater manifest_v2 = manifest_v1_modified.evolve(base=manifest_v1.base.evolve( version=4)) await aws.set_user_manifest(manifest_v2) assert await aws.get_realm_checkpoint() == 3 assert await aws.get_need_sync_entries() == ({user_manifest_id}, set())
def test_merge_folder_manifests(): my_device = DeviceID("b@b") other_device = DeviceID("a@a") parent = EntryID.new() v1 = LocalFolderManifest.new_placeholder(my_device, parent=parent).to_remote( author=other_device ) # Initial base manifest m1 = LocalFolderManifest.from_remote(v1, empty_pattern) assert merge_manifests(my_device, empty_pattern, m1) == m1 # Local change m2 = m1.evolve_children_and_mark_updated({"a": EntryID.new()}, empty_pattern) assert merge_manifests(my_device, empty_pattern, m2) == m2 # Successful upload v2 = m2.to_remote(author=my_device) m3 = merge_manifests(my_device, empty_pattern, m2, v2) assert m3 == LocalFolderManifest.from_remote(v2, empty_pattern) # Two local changes m4 = m3.evolve_children_and_mark_updated({"b": EntryID.new()}, empty_pattern) assert merge_manifests(my_device, empty_pattern, m4) == m4 m5 = m4.evolve_children_and_mark_updated({"c": EntryID.new()}, empty_pattern) assert merge_manifests(my_device, empty_pattern, m4) == m4 # M4 has been successfully uploaded v3 = m4.to_remote(author=my_device) m6 = merge_manifests(my_device, empty_pattern, m5, v3) assert m6 == m5.evolve(base=v3) # The remote has changed v4 = v3.evolve(version=4, children={"d": EntryID.new(), **v3.children}, author=other_device) m7 = merge_manifests(my_device, empty_pattern, m6, v4) assert m7.base_version == 4 assert sorted(m7.children) == ["a", "b", "c", "d"] assert m7.need_sync # Successful upload v5 = m7.to_remote(author=my_device) m8 = merge_manifests(my_device, empty_pattern, m7, v5) assert m8 == LocalFolderManifest.from_remote(v5, empty_pattern) # The remote has changed v6 = v5.evolve(version=6, children={"e": EntryID.new(), **v5.children}, author=other_device) m9 = merge_manifests(my_device, empty_pattern, m8, v6) assert m9 == LocalFolderManifest.from_remote(v6, empty_pattern)
async def _do_folder_stat(workspace_fs, path, default_selection): stats = {} dir_stat = await workspace_fs.path_info(path) # Retrieve children info, this is not an atomic operation so our view # on the folder might be slightly non-causal (typically if a file is # created during while we do the loop it won't appear in the final result, # but later update on existing files will appear). # We consider this fine enough given a change in the folder should lead # to an event from the corefs which in turn will re-trigger this stat code for child in dir_stat["children"]: try: child_stat = await workspace_fs.path_info(path / child) except FSFileNotFoundError: # The child entry as been concurrently removed, just ignore it continue except FSRemoteManifestNotFound: # Cannot get informations about this child entry, this can occur if # if the manifest is inconsistent (broken data or signature). child_stat = {"type": "inconsistency", "id": EntryID.new()} stats[child] = child_stat return path, dir_stat["id"], stats, default_selection
def generate_new_device( organization_addr: BackendOrganizationAddr, device_id: Optional[DeviceID] = None, profile: UserProfile = UserProfile.STANDARD, human_handle: Optional[HumanHandle] = None, device_label: Optional[DeviceLabel] = None, signing_key: Optional[SigningKey] = None, private_key: Optional[PrivateKey] = None, ) -> LocalDevice: return LocalDevice( organization_addr=organization_addr, device_id=device_id or DeviceID.new(), device_label=device_label, human_handle=human_handle, signing_key=signing_key or SigningKey.generate(), private_key=private_key or PrivateKey.generate(), profile=profile, user_manifest_id=EntryID.new(), user_manifest_key=SecretKey.generate(), local_symkey=SecretKey.generate(), )
async def test_link_file_unknown_workspace( aqtbot, core_config, gui_factory, autoclose_dialog, running_backend, alice ): password = "******" save_device_with_password_in_config(core_config.config_dir, alice, password) file_link = BackendOrganizationFileLinkAddr.build( organization_addr=alice.organization_addr, workspace_id=EntryID.new(), encrypted_path=b"<whatever>", ) gui = await gui_factory(core_config=core_config, start_arg=file_link.to_url()) lw = gui.test_get_login_widget() def _password_prompt(): assert len(autoclose_dialog.dialogs) == 0 lpi_w = lw.widget.layout().itemAt(0).widget() assert isinstance(lpi_w, LoginPasswordInputWidget) await aqtbot.wait_until(_password_prompt) lpi_w = lw.widget.layout().itemAt(0).widget() await aqtbot.key_clicks(lpi_w.line_edit_password, password) await aqtbot.wait_until(lambda: lpi_w.line_edit_password.text() == password) tabw = gui.test_get_tab() async with aqtbot.wait_signals([lw.login_with_password_clicked, tabw.logged_in]): aqtbot.mouse_click(lpi_w.button_login, QtCore.Qt.LeftButton) def _error_shown(): assert len(autoclose_dialog.dialogs) == 1 print(autoclose_dialog.dialogs) assert autoclose_dialog.dialogs[0] == ( "Error", "You do not have access to the workspace containing the file. It may not have been shared with you.", ) await aqtbot.wait_until(_error_shown)
async def alice_transaction_local_storage(alice, persistent_mockup): async with WorkspaceStorage.run(alice, Path("/dummy"), EntryID.new()) as storage: yield storage
def test_complete_scenario(): storage = Storage() with freeze_time("2000-01-01"): base = manifest = LocalFileManifest.new_placeholder( DeviceID.new(), parent=EntryID.new(), blocksize=16) assert manifest == base.evolve(size=0) with freeze_time("2000-01-02") as t2: manifest = storage.write(manifest, b"Hello ", 0) assert storage.read(manifest, 6, 0) == b"Hello " (chunk0, ), = manifest.blocks assert manifest == base.evolve(size=6, blocks=((chunk0, ), ), updated=t2) assert chunk0 == Chunk(id=chunk0.id, start=0, stop=6, raw_offset=0, raw_size=6, access=None) assert storage[chunk0.id] == b"Hello " with freeze_time("2000-01-03") as t3: manifest = storage.write(manifest, b"world !", 6) assert storage.read(manifest, 13, 0) == b"Hello world !" (_, chunk1), = manifest.blocks assert manifest == base.evolve(size=13, blocks=((chunk0, chunk1), ), updated=t3) assert chunk1 == Chunk(id=chunk1.id, start=6, stop=13, raw_offset=6, raw_size=7, access=None) assert storage[chunk1.id] == b"world !" with freeze_time("2000-01-04") as t4: manifest = storage.write(manifest, b"\n More kontent", 13) assert storage.read(manifest, 27, 0) == b"Hello world !\n More kontent" (_, _, chunk2), (chunk3, ) = manifest.blocks assert storage[chunk2.id] == b"\n M" assert storage[chunk3.id] == b"ore kontent" assert manifest == base.evolve(size=27, blocks=((chunk0, chunk1, chunk2), (chunk3, )), updated=t4) with freeze_time("2000-01-05") as t5: manifest = storage.write(manifest, b"c", 20) assert storage.read(manifest, 27, 0) == b"Hello world !\n More content" chunk4, chunk5, chunk6 = manifest.blocks[1] assert chunk3.id == chunk4.id == chunk6.id assert storage[chunk5.id] == b"c" assert manifest == base.evolve(size=27, blocks=((chunk0, chunk1, chunk2), (chunk4, chunk5, chunk6)), updated=t5) with freeze_time("2000-01-06") as t6: manifest = storage.resize(manifest, 40) expected = b"Hello world !\n More content" + b"\x00" * 13 assert storage.read(manifest, 40, 0) == expected (_, _, _, chunk7), (chunk8, ) = manifest.blocks[1:] assert storage[chunk7.id] == b"\x00" * 5 assert storage[chunk8.id] == b"\x00" * 8 assert manifest == base.evolve( size=40, blocks=((chunk0, chunk1, chunk2), (chunk4, chunk5, chunk6, chunk7), (chunk8, )), updated=t6, ) with freeze_time("2000-01-07") as t7: manifest = storage.resize(manifest, 25) expected = b"Hello world !\n More conte" assert storage.read(manifest, 25, 0) == expected (_, _, chunk9), = manifest.blocks[1:] assert chunk9.id == chunk6.id assert manifest == base.evolve(size=25, blocks=((chunk0, chunk1, chunk2), (chunk4, chunk5, chunk9)), updated=t7) with freeze_time("2000-01-08"): assert not manifest.is_reshaped() manifest = storage.reshape(manifest) expected = b"Hello world !\n More conte" assert storage.read(manifest, 25, 0) == expected assert manifest.is_reshaped() (chunk10, ), (chunk11, ) = manifest.blocks assert storage[chunk10.id] == b"Hello world !\n M" assert storage[chunk11.id] == b"ore conte" assert manifest == base.evolve(size=25, blocks=((chunk10, ), (chunk11, )), updated=t7)
def test_merge_folder_manifests(alice, bob): timestamp = alice.timestamp() my_device = alice.device_id other_device = bob.device_id parent = EntryID.new() v1 = LocalFolderManifest.new_placeholder(my_device, parent=parent, timestamp=timestamp).to_remote( author=other_device, timestamp=timestamp) # Initial base manifest m1 = LocalFolderManifest.from_remote(v1, empty_pattern) assert merge_manifests(my_device, timestamp, empty_pattern, m1) == m1 # Local change m2 = m1.evolve_children_and_mark_updated({EntryName("a"): EntryID.new()}, empty_pattern, timestamp=timestamp) assert merge_manifests(my_device, timestamp, empty_pattern, m2) == m2 # Successful upload v2 = m2.to_remote(author=my_device, timestamp=timestamp) m3 = merge_manifests(my_device, timestamp, empty_pattern, m2, v2) assert m3 == LocalFolderManifest.from_remote(v2, empty_pattern) # Two local changes m4 = m3.evolve_children_and_mark_updated({EntryName("b"): EntryID.new()}, empty_pattern, timestamp=timestamp) assert merge_manifests(my_device, timestamp, empty_pattern, m4) == m4 m5 = m4.evolve_children_and_mark_updated({EntryName("c"): EntryID.new()}, empty_pattern, timestamp=timestamp) assert merge_manifests(my_device, timestamp, empty_pattern, m4) == m4 # M4 has been successfully uploaded v3 = m4.to_remote(author=my_device, timestamp=timestamp) m6 = merge_manifests(my_device, timestamp, empty_pattern, m5, v3) assert m6 == m5.evolve(base=v3) # The remote has changed v4 = v3.evolve(version=4, children={ EntryName("d"): EntryID.new(), **v3.children }, author=other_device) m7 = merge_manifests(my_device, timestamp, empty_pattern, m6, v4) assert m7.base_version == 4 assert sorted(m7.children) == [ EntryName("a"), EntryName("b"), EntryName("c"), EntryName("d") ] assert m7.need_sync # Successful upload v5 = m7.to_remote(author=my_device, timestamp=timestamp) m8 = merge_manifests(my_device, timestamp, empty_pattern, m7, v5) assert m8 == LocalFolderManifest.from_remote(v5, empty_pattern) # The remote has changed v6 = v5.evolve(version=6, children={ EntryName("e"): EntryID.new(), **v5.children }, author=other_device) m9 = merge_manifests(my_device, timestamp, empty_pattern, m8, v6) assert m9 == LocalFolderManifest.from_remote(v6, empty_pattern)
def test_merge_folder_manifests_with_concurrent_remote_change( local_change, remote_change, alice, bob): timestamp = alice.timestamp() my_device = alice.device_id other_device = bob.device_id parent = EntryID.new() foo_txt = EntryID.new() remote_manifest_v1 = (LocalFolderManifest.new_placeholder( my_device, parent=parent, timestamp=timestamp).evolve(children={ EntryName("foo.txt"): foo_txt }).to_remote(author=my_device, timestamp=timestamp)) prevent_sync_pattern = re.compile(r".*\.tmp\Z") # Load the manifest in local local_manifest = LocalFolderManifest.from_remote( remote_manifest_v1, prevent_sync_pattern=prevent_sync_pattern) # In local, `foo.txt` is renamed if local_change == "rename": foo_txt_new_name = EntryName("foo2.txt") else: assert local_change == "prevent_sync_rename" foo_txt_new_name = EntryName("foo.txt.tmp") local_manifest = local_manifest.evolve_children_and_mark_updated( data={ EntryName("foo.txt"): None, foo_txt_new_name: foo_txt }, prevent_sync_pattern=prevent_sync_pattern, timestamp=timestamp, ) # In remote, a change also occurs if remote_change == "same_entry_moved": remote_manifest_v2_children = { EntryName("bar.txt"): remote_manifest_v1.children[EntryName("foo.txt")] } else: assert remote_change == "new_entry_added" remote_manifest_v2_children = { **remote_manifest_v1.children, EntryName("bar.txt"): EntryID.new(), } remote_manifest_v2 = remote_manifest_v1.evolve( author=other_device, version=remote_manifest_v1.version + 1, children=remote_manifest_v2_children, ) # Now merging should detect the duplication merged_manifest = merge_manifests( local_author=my_device, timestamp=timestamp, prevent_sync_pattern=prevent_sync_pattern, local_manifest=local_manifest, remote_manifest=remote_manifest_v2, force_apply_pattern=False, ) if remote_change == "same_entry_moved": assert list(merged_manifest.children) == [EntryName("bar.txt")] else: assert remote_change == "new_entry_added" if local_change == "rename": assert sorted(merged_manifest.children) == [ EntryName("bar.txt"), EntryName("foo2.txt") ] else: assert local_change == "prevent_sync_rename" assert sorted(merged_manifest.children) == [ EntryName("bar.txt"), EntryName("foo.txt.tmp"), ]
async def test_share_unknown(running_backend, alice_user_fs, bob): wid = EntryID.new() with pytest.raises(FSWorkspaceNotFoundError): await alice_user_fs.workspace_share(wid, bob.user_id, WorkspaceRole.MANAGER)
def workspace_id(): return EntryID.new()
def test_merge_folder_children(preferred_language, suffix): m1 = EntryID.new() m2 = EntryID.new() m3 = EntryID.new() a1 = {EntryName("a"): m1} a2 = {EntryName("a"): m2} b1 = {EntryName("b.txt"): m1} b2 = {EntryName("b.txt"): m2} c1 = {EntryName("c.tar.gz"): m1} c2 = {EntryName("c.tar.gz"): m2} # Empty folder assert merge_folder_children({}, {}, {}, preferred_language) == {} # Adding children assert merge_folder_children({}, a1, {}, preferred_language) == a1 assert merge_folder_children({}, {}, a1, preferred_language) == a1 assert merge_folder_children({}, a1, a1, preferred_language) == a1 # Removing children assert merge_folder_children(a1, {}, a1, preferred_language) == {} assert merge_folder_children(a1, a1, {}, preferred_language) == {} assert merge_folder_children(a1, {}, {}, preferred_language) == {} # Renaming children assert merge_folder_children(a1, a1, b1, preferred_language) == b1 assert merge_folder_children(a1, b1, a1, preferred_language) == b1 assert merge_folder_children(a1, b1, b1, preferred_language) == b1 # Conflicting renaming result = merge_folder_children(a1, b1, c1, preferred_language) assert result == {EntryName("c.tar.gz"): m1} # Conflicting names result = merge_folder_children({}, a1, a2, preferred_language) assert result == { EntryName("a"): m2, EntryName(f"a (Parsec - {suffix})"): m1 } result = merge_folder_children({}, b1, b2, preferred_language) assert result == { EntryName("b.txt"): m2, EntryName(f"b (Parsec - {suffix}).txt"): m1 } result = merge_folder_children({}, c1, c2, preferred_language) assert result == { EntryName("c.tar.gz"): m2, EntryName(f"c (Parsec - {suffix}).tar.gz"): m1 } # Conflicting name with special pattern filename base = {EntryName(f"a (Parsec - {suffix})"): m3} a3 = {**base, **a1} b3 = {**base, **a2} result = merge_folder_children(base, a3, b3, preferred_language) assert result == { EntryName("a"): m2, EntryName(f"a (Parsec - {suffix})"): m3, EntryName(f"a (Parsec - {suffix} (2))"): m1, } m4 = EntryID.new() base = {**base, EntryName(f"a (Parsec - {suffix} (2))"): m4} a3 = {**base, **a1} b3 = {**base, **a2} result = merge_folder_children(base, a3, b3, preferred_language) assert result == { EntryName("a"): m2, EntryName(f"a (Parsec - {suffix})"): m3, EntryName(f"a (Parsec - {suffix} (2))"): m4, EntryName(f"a (Parsec - {suffix} (3))"): m1, }
def test_merge_speculative_with_it_unsuspected_former_self( local_changes, core_config): d1 = datetime(2000, 1, 1) d2 = datetime(2000, 1, 2) d3 = datetime(2000, 1, 3) d4 = datetime(2000, 1, 4) d5 = datetime(2000, 1, 5) my_device = DeviceID("a@a") # 1) Workspace manifest is originally created by our device local = LocalWorkspaceManifest.new_placeholder(author=my_device, timestamp=d1) foo_id = EntryID.new() local = local.evolve(updated=d2, children=FrozenDict({EntryName("foo"): foo_id})) # 2) We sync the workspace manifest v1 = local.to_remote(author=my_device, timestamp=d3) # 3) Now let's pretend we lost local storage, hence creating a new speculative manifest new_local = LocalWorkspaceManifest.new_placeholder(author=my_device, id=local.id, timestamp=d3, speculative=True) if local_changes: bar_id = EntryID.new() new_local = new_local.evolve(updated=d4, children=FrozenDict( {EntryName("bar"): bar_id})) # 4) When syncing the manifest, we shouldn't remove any data from the remote merged = merge_manifests( local_author=my_device, timestamp=d5, prevent_sync_pattern=empty_pattern, local_manifest=new_local, remote_manifest=v1, ) if local_changes: assert merged == LocalWorkspaceManifest( base=v1, need_sync=True, updated=d5, children=FrozenDict({ **v1.children, **new_local.children }), local_confinement_points=frozenset(), remote_confinement_points=frozenset(), speculative=False, ) else: assert merged == LocalWorkspaceManifest( base=v1, need_sync=False, updated=v1.updated, children=v1.children, local_confinement_points=frozenset(), remote_confinement_points=frozenset(), speculative=False, )