async def test_path_info_remote_loader_exceptions(monkeypatch, alice_workspace, alice): manifest, _ = await alice_workspace.transactions._get_manifest_from_path( FsPath("/foo/bar")) async with alice_workspace.local_storage.lock_entry_id(manifest.id): await alice_workspace.local_storage.clear_manifest(manifest.id) vanilla_file_manifest_deserialize = BaseRemoteManifest._deserialize def mocked_file_manifest_deserialize(*args, **kwargs): return vanilla_file_manifest_deserialize( *args, **kwargs).evolve(**manifest_modifiers) monkeypatch.setattr(BaseRemoteManifest, "_deserialize", mocked_file_manifest_deserialize) manifest_modifiers = {"id": EntryID()} with pytest.raises(FSError) as exc: await alice_workspace.path_info(FsPath("/foo/bar")) assert f"Invalid entry ID: expected `{manifest.id}`, got `{manifest_modifiers['id']}`" in str( exc.value) manifest_modifiers = {"version": 4} with pytest.raises(FSError) as exc: await alice_workspace.path_info(FsPath("/foo/bar")) assert "Invalid version: expected `1`, got `4`" in str(exc.value) manifest_modifiers = {"author": DeviceID("mallory@pc1")} with pytest.raises(FSError) as exc: await alice_workspace.path_info(FsPath("/foo/bar")) assert "Invalid author: expected `alice@dev1`, got `mallory@pc1`" in str( exc.value)
async def test_files_history(alice_workspace): sync_by_id = partial(alice_workspace.sync_by_id, remote_changed=True) entry = alice_workspace.get_workspace_entry() wid = entry.id # Empty workspace await sync_by_id(wid) # Creating files and writing first byte await alice_workspace.touch("/f") await alice_workspace.touch("/f2") # Test version before first sync f_versions = await VersionLister(alice_workspace).list(FsPath("/f")) # Assert if task list have been completed assert f_versions[1] is True # Should have no version yet because we didn't sync assert not f_versions[0] # Syncronysing the files await sync_by_id(wid) # Checking again, should have the first version now f_versions = await VersionLister(alice_workspace).list(FsPath("/f")) # Assert if task list have been completed assert f_versions[1] is True # Assert there is a version in the list assert f_versions[0] # Checking the first version of the list assert getattr(f_versions[0][0], "version") == 1 f2_versions = await VersionLister(alice_workspace).list(FsPath("/f2")) # Assert if task list have been completed assert f2_versions[1] is True # Assert there is a version in the list assert f2_versions[0] # Checking the first version of the list assert getattr(f2_versions[0][0], "version") == 1 # Updating the file a couple of time and sync again to test the version list for i in range(20): f = await alice_workspace.open_file("/f", "ab") await f.write(str(i).encode()) await f.close() await sync_by_id(wid) f_versions = await VersionLister(alice_workspace).list(FsPath("/f")) # _sanitize_list is removing the first 1 version because it is the empty manifest set version_nb = 2 previous_late = None for version in f_versions[0]: assert version.version == version_nb version_nb += 1 if previous_late: assert previous_late == version.early previous_late = version.late
async def test_root_entry_info(alice_workspace_t2, alice_workspace_t4): stat2 = await alice_workspace_t2.transactions.entry_info(FsPath("/")) assert stat2 == { "type": "folder", "id": alice_workspace_t4.transactions.workspace_id, "base_version": 1, "is_placeholder": False, "need_sync": False, "created": datetime(1999, 12, 31), "updated": datetime(1999, 12, 31), "children": ["foo"], "confined": False, } stat4 = await alice_workspace_t4.transactions.entry_info(FsPath("/")) assert stat4 == { "type": "folder", "id": alice_workspace_t4.transactions.workspace_id, "base_version": 2, "is_placeholder": False, "need_sync": False, "created": datetime(1999, 12, 31), "updated": datetime(2000, 1, 4), "children": ["files", "foo"], "confined": False, }
async def test_version_non_existing_directory(alice_workspace, alice): version_lister = alice_workspace.get_version_lister() versions, version_list_is_complete = await version_lister.list( FsPath("/moved")) assert version_list_is_complete is True assert len(versions) == 2 assert versions[0][1:] == ( 5, _day(9), _day(10), alice.device_id, _day(8), True, None, FsPath("/files"), None, ) assert versions[1][1:] == ( 6, _day(10), _day(11), alice.device_id, _day(10), True, None, None, FsPath("/files"), )
async def copytree( self, source_path: AnyPath, target_path: AnyPath, source_workspace: Optional["WorkspaceFS"] = None, ) -> None: source_path = FsPath(source_path) target_path = FsPath(target_path) source_workspace = source_workspace or self source_files = await source_workspace.listdir(source_path) await self.mkdir(target_path) for source_file in source_files: target_file = target_path / source_file.name if await source_workspace.is_dir(source_file): await self.copytree( source_path=source_file, target_path=target_file, source_workspace=source_workspace, ) elif await source_workspace.is_file(source_file): await self.copyfile( source_path=source_file, target_path=target_file, source_workspace=source_workspace, )
async def test_access_not_loaded_entry(alice, bob, alice_entry_transactions): entry_transactions = alice_entry_transactions entry_id = entry_transactions.get_workspace_entry().id manifest = await entry_transactions.local_storage.get_manifest(entry_id) async with entry_transactions.local_storage.lock_entry_id(entry_id): await entry_transactions.local_storage.clear_manifest(entry_id) with pytest.raises(FSRemoteManifestNotFound): await entry_transactions.entry_info(FsPath("/")) async with entry_transactions.local_storage.lock_entry_id(entry_id): await entry_transactions.local_storage.set_manifest(entry_id, manifest) entry_info = await entry_transactions.entry_info(FsPath("/")) assert entry_info == { "type": "folder", "id": entry_id, "created": datetime(2000, 1, 1), "updated": datetime(2000, 1, 1), "base_version": 0, "is_placeholder": True, "need_sync": True, "children": [], "confined": False, }
async def test_file_create(alice_entry_transactions, alice_file_transactions, alice): entry_transactions = alice_entry_transactions file_transactions = alice_file_transactions with freeze_time("2000-01-02"): access_id, fd = await entry_transactions.file_create(FsPath("/foo.txt")) await file_transactions.fd_close(fd) assert fd == 1 root_stat = await entry_transactions.entry_info(FsPath("/")) assert root_stat == { "type": "folder", "id": entry_transactions.workspace_id, "base_version": 0, "is_placeholder": True, "need_sync": True, "created": datetime(2000, 1, 1), "updated": datetime(2000, 1, 2), "children": ["foo.txt"], "confined": False, } foo_stat = await entry_transactions.entry_info(FsPath("/foo.txt")) assert foo_stat == { "type": "file", "id": access_id, "base_version": 0, "is_placeholder": True, "need_sync": True, "created": datetime(2000, 1, 2), "updated": datetime(2000, 1, 2), "size": 0, "confined": False, }
async def test_mountpoint_iterdir_with_many_files(n, base_path, base_mountpoint, alice_user_fs, event_bus): wid = await alice_user_fs.workspace_create("w") workspace = alice_user_fs.get_workspace(wid) await workspace.mkdir(base_path, parents=True, exist_ok=True) names = [f"some_file_{i:03d}.txt" for i in range(n)] path_list = [FsPath(f"{base_path}/{name}") for name in names] for path in path_list: await workspace.touch(path) # Now we can start fuse async with mountpoint_manager_factory( alice_user_fs, event_bus, base_mountpoint) as mountpoint_manager: await mountpoint_manager.mount_workspace(wid) test_path = mountpoint_manager.get_path_in_mountpoint( wid, FsPath(base_path)) # Work around trio issue #1308 (https://github.com/python-trio/trio/issues/1308) items = await trio.to_thread.run_sync( lambda: [path.name for path in Path(test_path).iterdir()]) assert items == names for path in path_list: item_path = mountpoint_manager.get_path_in_mountpoint(wid, path) assert await trio.Path(item_path).exists()
async def test_get_path_in_mountpoint(base_mountpoint, alice_user_fs, event_bus): # Populate a bit the fs first... wid = await alice_user_fs.workspace_create("mounted_wksp") wid2 = await alice_user_fs.workspace_create("not_mounted_wksp") workspace1 = alice_user_fs.get_workspace(wid) workspace2 = alice_user_fs.get_workspace(wid2) await workspace1.touch("/bar.txt") await workspace2.touch("/foo.txt") # Now we can start fuse async with mountpoint_manager_factory( alice_user_fs, event_bus, base_mountpoint) as mountpoint_manager: await mountpoint_manager.mount_workspace(wid) bar_path = mountpoint_manager.get_path_in_mountpoint( wid, FsPath("/bar.txt")) assert isinstance(bar_path, PurePath) # Windows uses drives, not base_mountpoint if os.name != "nt": expected = base_mountpoint / "mounted_wksp" / "bar.txt" assert str(bar_path) == str(expected.absolute()) assert await trio.Path(bar_path).exists() with pytest.raises(MountpointNotMounted): mountpoint_manager.get_path_in_mountpoint(wid2, FsPath("/foo.txt"))
async def test_synchronization_step_transaction(alice_sync_transactions, type): sync_transactions = alice_sync_transactions synchronization_step = sync_transactions.synchronization_step entry_id = sync_transactions.get_workspace_entry().id # Sync a placeholder manifest = await synchronization_step(entry_id) # Acknowledge a successful synchronization assert await synchronization_step(entry_id, manifest) is None # Local change if type == "file": a_id, fd = await sync_transactions.file_create(FsPath("/a")) await sync_transactions.fd_write(fd, b"abc", 0) await sync_transactions.fd_close(fd) else: a_id = await sync_transactions.folder_create(FsPath("/a")) # Sync parent with a placeholder child manifest = await synchronization_step(entry_id) children = [] async for child in sync_transactions.get_placeholder_children(manifest): children.append(child) (a_entry_id, ) = children assert a_entry_id == a_id # Sync child if type == "file": await synchronization_step(a_entry_id) a_manifest = await synchronization_step(a_entry_id) assert await synchronization_step(a_entry_id, a_manifest) is None # Acknowledge the manifest assert sorted(manifest.children) == ["a"] assert await synchronization_step(entry_id, manifest) is None # Local change b_id = await sync_transactions.folder_create(FsPath("/b")) # Remote change children = {**manifest.children, "c": EntryID()} manifest = manifest.evolve(version=5, children=children, author="b@b") # Sync parent with a placeholder child manifest = await synchronization_step(entry_id, manifest) children = [] async for child in sync_transactions.get_placeholder_children(manifest): children.append(child) (b_entry_id, ) = children assert b_entry_id == b_id # Sync child b_manifest = await synchronization_step(b_entry_id) assert await synchronization_step(b_entry_id, b_manifest) is None # Acknowledge the manifest assert sorted(manifest.children) == ["a", "b", "c"] assert await synchronization_step(entry_id, manifest) is None
async def is_file(self, path: AnyPath) -> bool: """ Raises: FSError """ path = FsPath(path) info = await self.transactions.entry_info(FsPath(path)) return info["type"] == "file"
async def test_listdir(alice_workspace): lst = await alice_workspace.listdir("/") assert lst == [FsPath("/foo")] lst = await alice_workspace.listdir("/foo") assert lst == [FsPath("/foo/bar"), FsPath("/foo/baz")] with pytest.raises(NotADirectoryError): await alice_workspace.listdir("/foo/bar") with pytest.raises(FileNotFoundError): await alice_workspace.listdir("/baz")
async def test_iterdir(alice_workspace): lst = [child async for child in alice_workspace.iterdir("/")] assert lst == [FsPath("/foo")] lst = [child async for child in alice_workspace.iterdir("/foo")] assert lst == [FsPath("/foo/bar"), FsPath("/foo/baz")] with pytest.raises(NotADirectoryError): async for child in alice_workspace.iterdir("/foo/bar"): assert False, child with pytest.raises(FileNotFoundError): async for child in alice_workspace.iterdir("/baz"): assert False, child
async def move( self, source: AnyPath, destination: AnyPath, source_workspace: Optional["WorkspaceFS"] = None, ) -> None: """ Raises: FSError """ source = FsPath(source) destination = FsPath(destination) real_destination = destination # Source workspace will be either the same workspace # or another one when copy paste between two different workspace source_workspace = source_workspace or self # Testing if pasting files from the same workspace if source_workspace is self: if source.parts == destination.parts[:len(source.parts)]: raise FSInvalidArgumentError( f"Cannot move a directory {source} into itself {destination}" ) try: if await self.is_dir(destination): real_destination = destination / source.name if await self.exists(real_destination): raise FileExistsError # At this point, real_destination is the target either representing : # - the destination path if it didn't already exist, # - a new entry with the same name as source, but inside the destination directory except FileNotFoundError: pass # Rename if possible if source.parent == real_destination.parent: return await self.rename(source, real_destination) # Copy directory if await source_workspace.is_dir(source): await self.copytree(source_path=source, target_path=real_destination, source_workspace=source_workspace) await source_workspace.rmtree(source) return # Copy file await self.copyfile(source_path=source, target_path=real_destination, source_workspace=source_workspace) await source_workspace.unlink(source)
async def rename(self, source: AnyPath, destination: AnyPath, overwrite: bool = True) -> None: """ Raises: FSError """ source = FsPath(source) destination = FsPath(destination) await self.transactions.entry_rename(source, destination, overwrite=overwrite)
async def test_get_minimal_remote_manifest(alice, alice_sync_transactions): sync_transactions = alice_sync_transactions # Prepare w_id = sync_transactions.workspace_id a_id, fd = await sync_transactions.file_create(FsPath("/a")) await sync_transactions.fd_write(fd, b"abc", 0) await sync_transactions.fd_close(fd) b_id = await sync_transactions.folder_create(FsPath("/b")) c_id = await sync_transactions.folder_create(FsPath("/b/c")) # Workspace manifest minimal = await sync_transactions.get_minimal_remote_manifest(w_id) local = await sync_transactions.local_storage.get_manifest(w_id) expected = local.to_remote(author=alice.device_id, timestamp=minimal.timestamp).evolve( children={}, updated=local.created) assert minimal == expected await sync_transactions.synchronization_step(w_id, minimal) assert await sync_transactions.get_minimal_remote_manifest(w_id) is None # File manifest minimal = await sync_transactions.get_minimal_remote_manifest(a_id) local = await sync_transactions.local_storage.get_manifest(a_id) expected = local.evolve(blocks=(), updated=local.created, size=0).to_remote(author=alice.device_id, timestamp=minimal.timestamp) assert minimal == expected await sync_transactions.file_reshape(a_id) await sync_transactions.synchronization_step(a_id, minimal) assert await sync_transactions.get_minimal_remote_manifest(a_id) is None # Folder manifest minimal = await sync_transactions.get_minimal_remote_manifest(b_id) local = await sync_transactions.local_storage.get_manifest(b_id) expected = local.to_remote(author=alice.device_id, timestamp=minimal.timestamp).evolve( children={}, updated=local.created) assert minimal == expected await sync_transactions.synchronization_step(b_id, minimal) assert await sync_transactions.get_minimal_remote_manifest(b_id) is None # Empty folder manifest minimal = await sync_transactions.get_minimal_remote_manifest(c_id) local = await sync_transactions.local_storage.get_manifest(c_id) expected = local.to_remote(author=alice.device_id, timestamp=minimal.timestamp) assert minimal == expected await sync_transactions.synchronization_step(c_id, minimal) assert await sync_transactions.get_minimal_remote_manifest(c_id) is None
async def test_access_not_loaded_entry(alice_workspace_t4): entry_id = alice_workspace_t4.transactions.get_workspace_entry().id alice_workspace_t4.transactions.local_storage._cache.clear() with pytest.raises(FSLocalMissError): await alice_workspace_t4.transactions.local_storage.get_manifest( entry_id) await alice_workspace_t4.transactions.entry_info(FsPath("/"))
async def test_flush_file(alice_workspace_t4): _, fd4 = await alice_workspace_t4.transactions.file_open( FsPath("/files/content"), "r") assert isinstance(fd4, int) transactions_t4 = alice_workspace_t4.transactions await transactions_t4.fd_flush(fd4)
async def truncate(self, path: AnyPath, length: int) -> None: """ Raises: FSError """ path = FsPath(path) await self.transactions.file_resize(path, length)
async def unlink(self, path: AnyPath) -> None: """ Raises: FSError """ path = FsPath(path) await self.transactions.file_delete(path)
async def rmdir(self, path: AnyPath) -> None: """ Raises: FSError """ path = FsPath(path) await self.transactions.folder_delete(path)
async def test_versions_not_enough_download_permited(alice_workspace, alice): version_lister = alice_workspace.get_version_lister() version_lister = alice_workspace.get_version_lister() versions, version_list_is_complete = await version_lister.list( FsPath("/files/renamed"), skip_minimal_sync=False, max_manifest_queries=1) assert version_list_is_complete is False versions, version_list_is_complete = await version_lister.list( FsPath("/files/renamed"), skip_minimal_sync=False) assert version_list_is_complete is True versions, version_list_is_complete = await version_lister.list( FsPath("/files/renamed"), skip_minimal_sync=False, max_manifest_queries=1) assert version_list_is_complete is True
async def path_id(self, path: AnyPath) -> EntryID: """ Raises: FSError """ info = await self.transactions.entry_info(FsPath(path)) return info["id"]
async def _do_import(workspace_fs, files, total_size, progress_signal): current_size = 0 for src, dst in files: try: if dst.parent != FsPath("/"): await workspace_fs.mkdir(dst.parent, parents=True, exist_ok=True) progress_signal.emit(src.name, current_size) async with await trio.open_file(src, "rb") as f: async with await workspace_fs.open_file(dst, "wb") as dest_file: read_size = 0 while True: chunk = await f.read(DEFAULT_BLOCK_SIZE) if not chunk: break await dest_file.write(chunk) read_size += len(chunk) progress_signal.emit(src.name, current_size + read_size) current_size += read_size + 1 progress_signal.emit(src.name, current_size) except trio.Cancelled as exc: raise JobResultError("cancelled", last_file=dst) from exc
def test_stringify(path, sanitized_path): # Don't test '//' because according to POSIX path resolution: # http://pubs.opengroup.org/onlinepubs/009695399/basedefs/xbd_chap04.html#tag_04_11 # "A pathname that begins with two successive slashes may be # interpreted in an implementation-defined manner, although more # than two leading slashes shall be treated as a single slash". obj = FsPath(path) assert str(obj) == sanitized_path
def __call__(self, name, path, *args, **kwargs): # The path argument might be None or "-" in some special cases # related to `release` and `releasedir` (when the file descriptor # is available but the corresponding path is not). In those cases, # we can simply ignore the path. path = FsPath(path) if path not in (None, "-") else None with translate_error(self.event_bus, name, path): return super().__call__(name, path, *args, **kwargs)
async def test_cancel_mount_workspace(base_mountpoint, alice_user_fs, event_bus, timeout): wid = await alice_user_fs.workspace_create("w") async with mountpoint_manager_factory( alice_user_fs, event_bus, base_mountpoint) as mountpoint_manager: with trio.move_on_after(timeout) as cancel_scope: await mountpoint_manager.mount_workspace(wid) if cancel_scope.cancelled_caught: with pytest.raises(MountpointNotMounted): mountpoint_manager.get_path_in_mountpoint(wid, FsPath("/")) else: path = trio.Path( mountpoint_manager.get_path_in_mountpoint(wid, FsPath("/"))) await path.exists() assert not await (path / "foo").exists()
async def test_operations_on_file(alice_workspace_t4, alice_workspace_t5): _, fd4 = await alice_workspace_t4.transactions.file_open( FsPath("/files/content"), "r") assert isinstance(fd4, int) transactions_t4 = alice_workspace_t4.transactions _, fd5 = await alice_workspace_t5.transactions.file_open( FsPath("/files/content"), "r") assert isinstance(fd5, int) transactions_t5 = alice_workspace_t5.transactions data = await transactions_t4.fd_read(fd4, 1, 0) assert data == b"a" data = await transactions_t4.fd_read(fd4, 3, 1) assert data == b"bcd" data = await transactions_t4.fd_read(fd4, 100, 4) assert data == b"e" data = await transactions_t4.fd_read(fd4, 4, 0) assert data == b"abcd" data = await transactions_t4.fd_read(fd4, -1, 0) assert data == b"abcde" with pytest.raises( FSError): # if removed from local_storage, no write right error?.. await alice_workspace_t4.transactions.fd_write(fd4, b"hello ", 0) data = await transactions_t5.fd_read(fd5, 100, 0) assert data == b"fghij" data = await transactions_t5.fd_read(fd5, 1, 1) assert data == b"g" data = await transactions_t4.fd_read(fd4, 1, 2) assert data == b"c" await transactions_t5.fd_close(fd5) with pytest.raises(FSInvalidFileDescriptor): data = await transactions_t5.fd_read(fd5, 1, 0) data = await transactions_t4.fd_read(fd4, 1, 3) assert data == b"d" _, fd5 = await alice_workspace_t5.transactions.file_open( FsPath("/files/content"), "r") data = await transactions_t5.fd_read(fd5, 3, 0) assert data == b"fgh"
async def test_remote_error_event( tmpdir, monkeypatch, running_backend, alice_user_fs, bob_user_fs, monitor ): wid = await create_shared_workspace("w1", bob_user_fs, alice_user_fs) base_mountpoint = Path(tmpdir / "alice_mountpoint") async with mountpoint_manager_factory( alice_user_fs, alice_user_fs.event_bus, base_mountpoint, debug=False ) as mountpoint_manager: await mountpoint_manager.mount_workspace(wid) # Create shared data bob_w = bob_user_fs.get_workspace(wid) await bob_w.touch("/foo.txt") await bob_w.write_bytes("/foo.txt", b"hello") await bob_w.sync() alice_w = alice_user_fs.get_workspace(wid) await alice_w.sync() # Force manifest cache await alice_w.path_id("/") await alice_w.path_id("/foo.txt") trio_w = trio.Path(mountpoint_manager.get_path_in_mountpoint(wid, FsPath("/"))) # Switch the mountpoint in maintenance... await bob_user_fs.workspace_start_reencryption(wid) def _testbed(): # ...accessing workspace data in the backend should endup in remote error with alice_user_fs.event_bus.listen() as spy: fd = os.open(str(trio_w / "foo.txt"), os.O_RDONLY) with pytest.raises(OSError): os.read(fd, 10) spy.assert_event_occured(ClientEvent.MOUNTPOINT_REMOTE_ERROR) # But should still be able to do local stuff though without remote errors with alice_user_fs.event_bus.listen() as spy: os.open(str(trio_w / "bar.txt"), os.O_RDWR | os.O_CREAT) assert os.listdir(str(trio_w)) == ["bar.txt", "foo.txt"] assert ClientEvent.MOUNTPOINT_REMOTE_ERROR not in [e.event for e in spy.events] # Finally test unhandled error def _crash(*args, **kwargs): raise RuntimeError("Fake Error") monkeypatch.setattr( "guardata.client.fs.workspacefs.entry_transactions.EntryTransactions.folder_create", _crash, ) with alice_user_fs.event_bus.listen() as spy: with pytest.raises(OSError): os.mkdir(str(trio_w / "dummy")) spy.assert_event_occured(ClientEvent.MOUNTPOINT_UNHANDLED_ERROR) await trio.to_thread.run_sync(_testbed)
async def test_versions_non_existing_file_remove_minimal_synced( alice_workspace, alice, skip_minimal_sync): version_lister = alice_workspace.get_version_lister() versions, version_list_is_complete = await version_lister.list( FsPath("/moved/renamed"), skip_minimal_sync=skip_minimal_sync) assert version_list_is_complete is True assert len(versions) == 1 assert versions[0][1:] == ( 2, _day(9), _day(11), alice.device_id, _day(8), False, 6, FsPath("/files/renamed"), FsPath("/files/renamed"), )