async def _lock_parent_manifest_from_path( self, path: FsPath ) -> AsyncIterator[Tuple[LocalFolderishManifests, Optional[BaseLocalManifest]]]: # This is the most complicated locking scenario. # It requires locking the parent of the given entry and the entry itself # if it exists. # This is done in a two step process: # - 1. Lock the parent (it must exist). While the parent is locked, no # children can be added, renamed or removed. # - 2. Lock the children if exists. It it doesn't, there is nothing to lock # since the parent lock guarentees that it is not going to be added while # using the context. # This double locking is only required for a single use case: the overwriting # of empty directory during a move. We have to make sure that no one adds # something to the directory while it is being overwritten. # If read/write locks were to be implemented, the parent would be write locked # and the child read locked. This means that despite locking two entries, only # a single entry is modified at a time. # Source is root if path.is_root(): raise FSPermissionError(filename=str(path)) # Loop over attempts while True: # Lock parent first async with self._lock_manifest_from_path(path.parent) as parent: # Parent is not a directory if not isinstance( parent, (LocalFolderManifest, LocalWorkspaceManifest)): raise FSNotADirectoryError(filename=path.parent) # Child doesn't exist if path.name not in parent.children: yield parent, None return # Child exists entry_id = parent.children[path.name] try: async with self.local_storage.lock_manifest( entry_id) as manifest: yield parent, manifest return # Child is not available except FSLocalMissError as exc: assert exc.id == entry_id # Release the lock and download the child manifest await self._load_manifest(entry_id)
def open_workspace_file(self, workspace_fs, file_name): file_name = FsPath("/", file_name) if file_name else FsPath("/") try: # The Qt thread should never hit the core directly. # Synchronous calls can run directly in the job system # as they won't block the Qt loop for long path = self.jobs_ctx.run_sync( self.core.mountpoint_manager.get_path_in_mountpoint, workspace_fs.workspace_id, file_name, workspace_fs.timestamp if isinstance(workspace_fs, WorkspaceFSTimestamped) else None, ) except MountpointNotMounted: # The mountpoint has been umounted in our back, nothing left to do pass desktop.open_file(str(path))
def _on_folder_stat_error(self, job): self.table_files.clear() if isinstance(job.exc, FSFileNotFoundError): show_error(self, _("TEXT_FILE_FOLDER_NOT_FOUND")) self.table_files.add_parent_workspace() return if self.current_directory == FsPath("/"): self.table_files.add_parent_workspace() else: self.table_files.add_parent_folder()
async def test_versions_non_existing_file_remove_minimal_synced( alice_workspace, alice, skip_minimal_sync): version_lister = alice_workspace.get_version_lister() versions, version_list_is_complete = await version_lister.list( FsPath("/moved/renamed"), skip_minimal_sync=skip_minimal_sync) assert version_list_is_complete is True assert len(versions) == 1 assert versions[0][1:] == ( 2, _day(9), _day(11), alice.device_id, _day(8), False, 6, FsPath("/files/renamed"), FsPath("/files/renamed"), )
async def _do_rename(workspace_fs, paths): new_names = {} for (old_path, new_path, uuid) in paths: try: await workspace_fs.rename(old_path, new_path) new_names[uuid] = FsPath(new_path).name except FileExistsError as exc: raise JobResultError("already-exists", multi=len(paths) > 1) from exc except OSError as exc: raise JobResultError("not-empty", multi=len(paths) > 1) from exc
async def touch(self, path: AnyPath, exist_ok: bool = True) -> None: """ Raises: FSError """ path = FsPath(path) try: await self.transactions.file_create(path, open=False) except FileExistsError: if not exist_ok: raise
async def exists(self, path: AnyPath) -> bool: """ Raises: FSError """ path = FsPath(path) try: await self.transactions.entry_info(path) except (FileNotFoundError, NotADirectoryError): return False return True
async def iterdir(self, path: AnyPath) -> AsyncIterator[FsPath]: """ Raises: FSError """ path = FsPath(path) info = await self.transactions.entry_info(path) if "children" not in info: raise FSNotADirectoryError(filename=str(path)) for child in cast(Dict[str, EntryID], info["children"]): yield path / child
def decrypt_file_link_path( self, addr: BackendOrganizationFileLinkAddr) -> FsPath: """ Raises: ValueError """ workspace_entry = self.get_workspace_entry() try: raw_path = workspace_entry.key.decrypt(addr.encrypted_path) except CryptoError: raise ValueError("Cannot decrypt path") # FsPath raises ValueError, decode() raises UnicodeDecodeError which is a subclass of ValueError return FsPath(raw_path.decode("utf-8"))
async def test_unlink(alice_workspace): await alice_workspace.unlink("/foo/bar") lst = await alice_workspace.listdir("/foo") assert lst == [FsPath("/foo/baz")] with pytest.raises(FileNotFoundError): await alice_workspace.unlink("/foo/bar") with pytest.raises(IsADirectoryError): await alice_workspace.unlink("/foo") # TODO: should this be a `IsADirectoryError`? with pytest.raises(PermissionError): await alice_workspace.unlink("/")
async def test_root_entry_info(alice_entry_transactions): stat = await alice_entry_transactions.entry_info(FsPath("/")) assert stat == { "type": "folder", "id": alice_entry_transactions.workspace_id, "base_version": 0, "is_placeholder": True, "need_sync": True, "created": Pendulum(2000, 1, 1), "updated": Pendulum(2000, 1, 1), "children": [], }
async def _clear_directory( workspace_directory_path: FsPath, local_path: AnyPath, workspace_fs: WorkspaceFS, folder_manifest: FolderManifest, ): local_children_keys = [p.name for p in await local_path.iterdir()] for name, entry_id in folder_manifest.children.items(): if name not in local_children_keys: absolute_path = FsPath(workspace_directory_path / name) print("delete %s" % absolute_path) await _clear_path(workspace_fs, absolute_path)
async def rmtree(self, path: AnyPath) -> None: """ Raises: FSError """ path = FsPath(path) async for child in self.iterdir(path): if await self.is_dir(child): await self.rmtree(child) else: await self.unlink(child) await self.rmdir(path)
async def test_root_manifest_parent(alice_workspace): workspace_manifest = mock.Mock() _get_or_create_directory_mock = AsyncMock(spec=mock.Mock) with mock.patch("parsec.core.cli.rsync._get_or_create_directory", _get_or_create_directory_mock): root_manifest, parent = await rsync._root_manifest_parent( None, alice_workspace, workspace_manifest) _get_or_create_directory_mock.assert_not_called() assert root_manifest == workspace_manifest assert parent == FsPath("/") workspace_manifest.children = {"test": "id1"} workspace_test_save_manifest = mock.Mock() workspace_test_save_manifest.children = {} workspace_test_manifest = mock.Mock() _get_or_create_directory_mock.side_effect = [ workspace_test_manifest, workspace_test_save_manifest, ] with mock.patch("parsec.core.cli.rsync._get_or_create_directory", _get_or_create_directory_mock): root_manifest, parent = await rsync._root_manifest_parent( FsPath("/path_in_workspace"), alice_workspace, workspace_manifest) assert root_manifest == workspace_test_manifest assert parent == FsPath("/path_in_workspace") _get_or_create_directory_mock.side_effect = [ workspace_test_manifest, workspace_test_save_manifest, ] with mock.patch("parsec.core.cli.rsync._get_or_create_directory", _get_or_create_directory_mock): root_manifest, parent = await rsync._root_manifest_parent( FsPath("/path_in_workspace/save"), alice_workspace, workspace_manifest) assert root_manifest == workspace_test_save_manifest assert parent == FsPath("/path_in_workspace/save")
async def test_get_or_create_directory(alice_workspace): load_manifest_mock = AsyncMock(spec=mock.Mock(), side_effect=lambda x: "load_manifest_mock") alice_workspace.remote_loader.load_manifest = load_manifest_mock _create_path_mock = AsyncMock(spec=mock.Mock(), side_effect=lambda *x: "_create_path_mock") with mock.patch("parsec.core.cli.rsync._create_path", _create_path_mock): entry_id = EntryID() res = await rsync._get_or_create_directory( entry_id, alice_workspace, FsPath("/test_directory"), FsPath("/path_in_workspace")) load_manifest_mock.assert_called_once_with(entry_id) _create_path_mock.assert_not_called() assert res == "load_manifest_mock" load_manifest_mock.reset_mock() with mock.patch("parsec.core.cli.rsync._create_path", _create_path_mock): res = await rsync._get_or_create_directory( None, alice_workspace, FsPath("/test_directory"), FsPath("/path_in_workspace")) load_manifest_mock.assert_not_called() _create_path_mock.assert_called_once_with(alice_workspace, True, FsPath("/test_directory"), FsPath("/path_in_workspace")) assert res == "_create_path_mock"
async def test_operations_on_file(alice_workspace_t4, alice_workspace_t5): _, fd4 = await alice_workspace_t4.transactions.file_open(FsPath("/files/content"), "r") assert isinstance(fd4, int) transactions_t4 = alice_workspace_t4.transactions _, fd5 = await alice_workspace_t5.transactions.file_open(FsPath("/files/content"), "r") assert isinstance(fd5, int) transactions_t5 = alice_workspace_t5.transactions data = await transactions_t4.fd_read(fd4, 1, 0) assert data == b"a" data = await transactions_t4.fd_read(fd4, 3, 1) assert data == b"bcd" data = await transactions_t4.fd_read(fd4, 100, 4) assert data == b"e" data = await transactions_t4.fd_read(fd4, 4, 0) assert data == b"abcd" data = await transactions_t4.fd_read(fd4, -1, 0) assert data == b"abcde" with pytest.raises(FSError): # if removed from local_storage, no write right error?.. await alice_workspace_t4.transactions.fd_write(fd4, b"hello ", 0) data = await transactions_t5.fd_read(fd5, 100, 0) assert data == b"fghij" data = await transactions_t5.fd_read(fd5, 1, 1) assert data == b"g" data = await transactions_t4.fd_read(fd4, 1, 2) assert data == b"c" await transactions_t5.fd_close(fd5) with pytest.raises(FSInvalidFileDescriptor): data = await transactions_t5.fd_read(fd5, 1, 0) data = await transactions_t4.fd_read(fd4, 1, 3) assert data == b"d" _, fd5 = await alice_workspace_t5.transactions.file_open(FsPath("/files/content"), "r") data = await transactions_t5.fd_read(fd5, 3, 0) assert data == b"fgh"
async def get_path_at_timestamp(self, entry_id: EntryID, timestamp: DateTime) -> FsPath: """ Find a path for an entry_id at a specific timestamp. If the path is broken, will raise an EntryNotFound exception. All the other exceptions are thrown by the ManifestCache. Raises: FSError FSBackendOfflineError FSWorkspaceInMaintenance FSBadEncryptionRevision FSWorkspaceNoAccess EntryNotFound """ # Get first manifest try: current_id = entry_id current_manifest, _ = await self.load(current_id, timestamp=timestamp) except FSRemoteManifestNotFound: raise EntryNotFound(entry_id) # Loop over parts parts = [] while not isinstance(current_manifest, WorkspaceManifest): # Get the manifest try: current_manifest = cast(Union[FolderManifest, FileManifest], current_manifest) parent_manifest, _ = await self.load(current_manifest.parent, timestamp=timestamp) parent_manifest = cast( Union[FolderManifest, WorkspaceManifest], parent_manifest) except FSRemoteManifestNotFound: raise EntryNotFound(entry_id) # Find the child name for name, child_id in parent_manifest.children.items(): if child_id == current_id: parts.append(name) break else: raise EntryNotFound(entry_id) # Continue until root is found current_id = current_manifest.parent current_manifest = parent_manifest # Return the path return FsPath("/" + "/".join(reversed(parts)))
def set_workspace_fs(self, wk_fs, current_directory=FsPath("/"), default_selection=None): self.current_directory = current_directory self.workspace_fs = wk_fs ws_entry = self.jobs_ctx.run_sync( self.workspace_fs.get_workspace_entry) self.current_user_role = ws_entry.role # self.label_role.setText(self.ROLES_TEXTS[self.current_user_role]) self.table_files.current_user_role = self.current_user_role self.clipboard = None self.reset(default_selection)
def generate_file_link(self, path: AnyPath) -> BackendOrganizationFileLinkAddr: """ Raises: Nothing """ workspace_entry = self.get_workspace_entry() encrypted_path = workspace_entry.key.encrypt( str(FsPath(path)).encode("utf-8")) return BackendOrganizationFileLinkAddr.build( organization_addr=self.device.organization_addr, workspace_id=workspace_entry.id, encrypted_path=encrypted_path, )
async def test_folder_create_delete(alice_entry_transactions, alice_sync_transactions): entry_transactions = alice_entry_transactions sync_transactions = alice_sync_transactions # Create and delete a foo directory foo_id = await entry_transactions.folder_create(FsPath("/foo")) assert await entry_transactions.folder_delete(FsPath("/foo")) == foo_id # The directory is not synced manifest = await entry_transactions.local_storage.get_manifest(foo_id) assert manifest.need_sync # Create and sync a bar directory bar_id = await entry_transactions.folder_create(FsPath("/bar")) remote = await sync_transactions.synchronization_step(bar_id) assert await sync_transactions.synchronization_step(bar_id, remote) is None # Remove the bar directory, the manifest is synced assert await entry_transactions.folder_delete(FsPath("/bar")) == bar_id manifest = await entry_transactions.local_storage.get_manifest(bar_id) assert not manifest.need_sync
async def test_create_path(alice_workspace): mkdir_mock = AsyncMock(spec=mock.Mock) alice_workspace.mkdir = mkdir_mock sync_mock = AsyncMock(spec=mock.Mock) alice_workspace.sync = sync_mock path_info_mock = AsyncMock(spec=mock.Mock, side_effect=lambda x: {"id": "mock_id"}) alice_workspace.path_info = path_info_mock get_manifest_mock = AsyncMock(spec=mock.Mock, side_effect=lambda x: "mock_manifest") alice_workspace.local_storage.get_manifest = get_manifest_mock import_file_mock = AsyncMock(spec=mock.Mock) with mock.patch("parsec.core.cli.rsync._import_file", import_file_mock): is_dir = True res = await rsync._create_path(alice_workspace, is_dir, FsPath("/test"), FsPath("/path_in_workspace/test")) mkdir_mock.assert_called_once_with(FsPath("/path_in_workspace/test")) sync_mock.assert_called_once_with() path_info_mock.assert_called_once_with( FsPath("/path_in_workspace/test")) get_manifest_mock.assert_called_once_with("mock_id") import_file_mock.assert_not_called() assert res == "mock_manifest" mkdir_mock.reset_mock() sync_mock.reset_mock() path_info_mock.reset_mock() get_manifest_mock.reset_mock() import_file_mock.reset_mock() with mock.patch("parsec.core.cli.rsync._import_file", import_file_mock): is_dir = False res = await rsync._create_path(alice_workspace, is_dir, FsPath("/test"), FsPath("/path_in_workspace/test")) mkdir_mock.assert_not_called() path_info_mock.assert_not_called() get_manifest_mock.assert_not_called() import_file_mock.assert_called_once_with( alice_workspace, FsPath("/test"), FsPath("/path_in_workspace/test")) sync_mock.assert_called_once_with() assert res is None
async def _resolve_placeholders_in_path(self, path: FsPath, access: Access, manifest: LocalManifest) -> bool: """ Returns: If an additional sync is needed """ # Notes we sync recursively from children to parents, this is more # efficient given otherwise we would have to do: # 1) sync the parent # 2) sync the child # 3) re-sync the parent with the child is_placeholder = is_placeholder_manifest(manifest) if not is_placeholder: # Cannot have a non-placeholder with a placeholder parent, hence # we don't have to go any further. return manifest.need_sync else: if is_file_manifest(manifest): need_more_sync = await self._minimal_sync_file( path, access, manifest) else: need_more_sync = await self._minimal_sync_folder( path, access, manifest) # Once the entry is synced, we must sync it parent as well to have # the entry visible for other clients if not path.is_root(): try: parent_access, parent_manifest = self.local_folder_fs.get_entry( path.parent) except FSManifestLocalMiss: # Nothing to do if entry is no present locally return False if is_placeholder_manifest(parent_manifest): await self._resolve_placeholders_in_path( path.parent, parent_access, parent_manifest) else: await self._sync_folder(path.parent, parent_access, parent_manifest, recursive=False) if not need_more_sync: self.event_bus.send("fs.entry.synced", path=str(path), id=access.id) return need_more_sync
def test_parse_destination(): mock_workspace1 = mock.Mock() mock_workspace1.name = "workspace1" mock_workspace2 = mock.Mock() mock_workspace2.name = "workspace2" workspaces_mock = mock.Mock() workspaces_mock.workspaces = [mock_workspace1, mock_workspace2] get_user_manifest_mock = mock.Mock(return_value=workspaces_mock) alice_core = mock.Mock() alice_core.user_fs.get_user_manifest = get_user_manifest_mock workspace, path = rsync._parse_destination(alice_core, "workspace1") assert workspace == mock_workspace1 assert path is None workspace, path = rsync._parse_destination(alice_core, "workspace1:/test/save") assert workspace == mock_workspace1 assert path == FsPath("/test/save") workspace, path = rsync._parse_destination(alice_core, "workspace1") assert workspace == mock_workspace1 assert path is None workspace, path = rsync._parse_destination(alice_core, "workspace2:/test/save2") assert workspace == mock_workspace2 assert path == FsPath("/test/save2") with pytest.raises(SystemExit): workspace, path = rsync._parse_destination(alice_core, "unknown_workspace") with pytest.raises(SystemExit): workspace, path = rsync._parse_destination( alice_core, "unknown_workspace:/test/save3")
def get_beacon(self, path: FsPath) -> UUID: # The beacon is used to notify other clients that we modified an entry. # We try to use the id of workspace containing the modification as # beacon. This is not possible when directly modifying the user # manifest in which case we use the user manifest id as beacon. try: _, workspace_name, *_ = path.parts except ValueError: return self.root_access.id access, manifest = self._retrieve_entry_read_only( FsPath(f"/{workspace_name}")) assert is_workspace_manifest(manifest) return access.id
async def read_bytes(self, path: AnyPath, size: int = -1, offset: int = 0) -> bytes: """ Raises: FSError """ path = FsPath(path) _, fd = await self.transactions.file_open(path, "r") try: return await self.transactions.fd_read(fd, size, offset) finally: await self.transactions.fd_close(fd)
async def test_sync_by_id_couple(alice_workspace, bob_workspace): alice_entry = alice_workspace.get_workspace_entry() alice_wid = alice_entry.id bob_entry = bob_workspace.get_workspace_entry() bob_wid = bob_entry.id # Create directories await alice_workspace.mkdir("/a") await bob_workspace.mkdir("/b") # Alice sync /a await alice_workspace.sync_by_id(alice_wid) # Bob sync /b and doesn't know about alice /a yet await bob_workspace.sync_by_id(bob_wid, remote_changed=False) # Alice knows about bob /b await alice_workspace.sync_by_id(alice_wid) # Everyone should be up to date by now expected = [FsPath("/a"), FsPath("/b")] assert await bob_workspace.listdir("/") == expected assert await alice_workspace.listdir("/") == expected
async def test_access_not_loaded_entry(alice, bob, alice_entry_transactions): entry_transactions = alice_entry_transactions entry_id = entry_transactions.get_workspace_entry().id manifest = await entry_transactions.local_storage.get_manifest(entry_id) async with entry_transactions.local_storage.lock_entry_id(entry_id): await entry_transactions.local_storage.clear_manifest(entry_id) with pytest.raises(FSRemoteManifestNotFound): await entry_transactions.entry_info(FsPath("/")) async with entry_transactions.local_storage.lock_entry_id(entry_id): await entry_transactions.local_storage.set_manifest(entry_id, manifest) entry_info = await entry_transactions.entry_info(FsPath("/")) assert entry_info == { "type": "folder", "id": entry_id, "created": Pendulum(2000, 1, 1), "updated": Pendulum(2000, 1, 1), "base_version": 0, "is_placeholder": True, "need_sync": True, "children": [], }
async def test_rmdir(alice_workspace): await alice_workspace.mkdir("/foz") await alice_workspace.rmdir("/foz") lst = await alice_workspace.listdir("/") assert lst == [FsPath("/foo")] with pytest.raises(OSError) as context: await alice_workspace.rmdir("/foo") assert context.value.errno == errno.ENOTEMPTY with pytest.raises(NotADirectoryError): await alice_workspace.rmdir("/foo/bar") with pytest.raises(PermissionError): await alice_workspace.rmdir("/")
async def test_file_create(alice_entry_transactions, alice_file_transactions, alice): entry_transactions = alice_entry_transactions file_transactions = alice_file_transactions with freeze_time("2000-01-02"): access_id, fd = await entry_transactions.file_create(FsPath("/foo.txt") ) await file_transactions.fd_close(fd) assert fd == 1 root_stat = await entry_transactions.entry_info(FsPath("/")) assert root_stat == { "type": "folder", "id": entry_transactions.workspace_id, "base_version": 0, "is_placeholder": True, "need_sync": True, "created": datetime(2000, 1, 1), "updated": datetime(2000, 1, 2), "children": ["foo.txt"], "confinement_point": None, } foo_stat = await entry_transactions.entry_info(FsPath("/foo.txt")) assert foo_stat == { "type": "file", "id": access_id, "base_version": 0, "is_placeholder": True, "need_sync": True, "created": datetime(2000, 1, 2), "updated": datetime(2000, 1, 2), "size": 0, "confinement_point": None, }
async def move(self, source: AnyPath, destination: AnyPath) -> None: """ Raises: FSError """ source = FsPath(source) destination = FsPath(destination) real_destination = destination if source.parts == destination.parts[:len(source.parts)]: raise FSInvalidArgumentError( f"Cannot move a directory {source} into itself {destination}") try: if await self.is_dir(destination): real_destination = destination / source.name if await self.exists(real_destination): raise FileExistsError # At this point, real_destination is the target either representing : # - the destination path if it didn't already exist, # - a new entry with the same name as source, but inside the destination directory except FileNotFoundError: pass # Rename if possible if source.parent == real_destination.parent: return await self.rename(source, real_destination) # Copy directory if await self.is_dir(source): await self.copytree(source, real_destination) await self.rmtree(source) return # Copy file await self.copyfile(source, real_destination) await self.unlink(source)