async def _lock_parent_manifest_from_path(
self, path: FsPath
) -> AsyncIterator[Tuple[LocalFolderishManifests,
Optional[BaseLocalManifest]]]:
# This is the most complicated locking scenario.
# It requires locking the parent of the given entry and the entry itself
# if it exists.
# This is done in a two step process:
# - 1. Lock the parent (it must exist). While the parent is locked, no
# children can be added, renamed or removed.
# - 2. Lock the children if exists. It it doesn't, there is nothing to lock
# since the parent lock guarentees that it is not going to be added while
# using the context.
# This double locking is only required for a single use case: the overwriting
# of empty directory during a move. We have to make sure that no one adds
# something to the directory while it is being overwritten.
# If read/write locks were to be implemented, the parent would be write locked
# and the child read locked. This means that despite locking two entries, only
# a single entry is modified at a time.
# Source is root
if path.is_root():
raise FSPermissionError(filename=str(path))
# Loop over attempts
while True:
# Lock parent first
async with self._lock_manifest_from_path(path.parent) as parent:
# Parent is not a directory
if not isinstance(
parent, (LocalFolderManifest, LocalWorkspaceManifest)):
raise FSNotADirectoryError(filename=path.parent)
# Child doesn't exist
if path.name not in parent.children:
yield parent, None
return
# Child exists
entry_id = parent.children[path.name]
try:
async with self.local_storage.lock_manifest(
entry_id) as manifest:
yield parent, manifest
return
# Child is not available
except FSLocalMissError as exc:
assert exc.id == entry_id
# Release the lock and download the child manifest
await self._load_manifest(entry_id)
def open_workspace_file(self, workspace_fs, file_name):
file_name = FsPath("/", file_name) if file_name else FsPath("/")
try:
# The Qt thread should never hit the core directly.
# Synchronous calls can run directly in the job system
# as they won't block the Qt loop for long
path = self.jobs_ctx.run_sync(
self.core.mountpoint_manager.get_path_in_mountpoint,
workspace_fs.workspace_id,
file_name,
workspace_fs.timestamp
if isinstance(workspace_fs, WorkspaceFSTimestamped) else None,
)
except MountpointNotMounted:
# The mountpoint has been umounted in our back, nothing left to do
pass
desktop.open_file(str(path))
def _on_folder_stat_error(self, job):
self.table_files.clear()
if isinstance(job.exc, FSFileNotFoundError):
show_error(self, _("TEXT_FILE_FOLDER_NOT_FOUND"))
self.table_files.add_parent_workspace()
return
if self.current_directory == FsPath("/"):
self.table_files.add_parent_workspace()
else:
self.table_files.add_parent_folder()
async def test_versions_non_existing_file_remove_minimal_synced(
alice_workspace, alice, skip_minimal_sync):
version_lister = alice_workspace.get_version_lister()
versions, version_list_is_complete = await version_lister.list(
FsPath("/moved/renamed"), skip_minimal_sync=skip_minimal_sync)
assert version_list_is_complete is True
assert len(versions) == 1
assert versions[0][1:] == (
2,
_day(9),
_day(11),
alice.device_id,
_day(8),
False,
6,
FsPath("/files/renamed"),
FsPath("/files/renamed"),
)
async def _do_rename(workspace_fs, paths):
new_names = {}
for (old_path, new_path, uuid) in paths:
try:
await workspace_fs.rename(old_path, new_path)
new_names[uuid] = FsPath(new_path).name
except FileExistsError as exc:
raise JobResultError("already-exists",
multi=len(paths) > 1) from exc
except OSError as exc:
raise JobResultError("not-empty", multi=len(paths) > 1) from exc
async def touch(self, path: AnyPath, exist_ok: bool = True) -> None:
"""
Raises:
FSError
"""
path = FsPath(path)
try:
await self.transactions.file_create(path, open=False)
except FileExistsError:
if not exist_ok:
raise
async def exists(self, path: AnyPath) -> bool:
"""
Raises:
FSError
"""
path = FsPath(path)
try:
await self.transactions.entry_info(path)
except (FileNotFoundError, NotADirectoryError):
return False
return True
async def iterdir(self, path: AnyPath) -> AsyncIterator[FsPath]:
"""
Raises:
FSError
"""
path = FsPath(path)
info = await self.transactions.entry_info(path)
if "children" not in info:
raise FSNotADirectoryError(filename=str(path))
for child in cast(Dict[str, EntryID], info["children"]):
yield path / child
def decrypt_file_link_path(
self, addr: BackendOrganizationFileLinkAddr) -> FsPath:
"""
Raises: ValueError
"""
workspace_entry = self.get_workspace_entry()
try:
raw_path = workspace_entry.key.decrypt(addr.encrypted_path)
except CryptoError:
raise ValueError("Cannot decrypt path")
# FsPath raises ValueError, decode() raises UnicodeDecodeError which is a subclass of ValueError
return FsPath(raw_path.decode("utf-8"))
async def test_unlink(alice_workspace):
await alice_workspace.unlink("/foo/bar")
lst = await alice_workspace.listdir("/foo")
assert lst == [FsPath("/foo/baz")]
with pytest.raises(FileNotFoundError):
await alice_workspace.unlink("/foo/bar")
with pytest.raises(IsADirectoryError):
await alice_workspace.unlink("/foo")
# TODO: should this be a `IsADirectoryError`?
with pytest.raises(PermissionError):
await alice_workspace.unlink("/")
async def test_root_entry_info(alice_entry_transactions):
stat = await alice_entry_transactions.entry_info(FsPath("/"))
assert stat == {
"type": "folder",
"id": alice_entry_transactions.workspace_id,
"base_version": 0,
"is_placeholder": True,
"need_sync": True,
"created": Pendulum(2000, 1, 1),
"updated": Pendulum(2000, 1, 1),
"children": [],
}
async def _clear_directory(
workspace_directory_path: FsPath,
local_path: AnyPath,
workspace_fs: WorkspaceFS,
folder_manifest: FolderManifest,
):
local_children_keys = [p.name for p in await local_path.iterdir()]
for name, entry_id in folder_manifest.children.items():
if name not in local_children_keys:
absolute_path = FsPath(workspace_directory_path / name)
print("delete %s" % absolute_path)
await _clear_path(workspace_fs, absolute_path)
async def rmtree(self, path: AnyPath) -> None:
"""
Raises:
FSError
"""
path = FsPath(path)
async for child in self.iterdir(path):
if await self.is_dir(child):
await self.rmtree(child)
else:
await self.unlink(child)
await self.rmdir(path)
async def test_root_manifest_parent(alice_workspace):
workspace_manifest = mock.Mock()
_get_or_create_directory_mock = AsyncMock(spec=mock.Mock)
with mock.patch("parsec.core.cli.rsync._get_or_create_directory",
_get_or_create_directory_mock):
root_manifest, parent = await rsync._root_manifest_parent(
None, alice_workspace, workspace_manifest)
_get_or_create_directory_mock.assert_not_called()
assert root_manifest == workspace_manifest
assert parent == FsPath("/")
workspace_manifest.children = {"test": "id1"}
workspace_test_save_manifest = mock.Mock()
workspace_test_save_manifest.children = {}
workspace_test_manifest = mock.Mock()
_get_or_create_directory_mock.side_effect = [
workspace_test_manifest,
workspace_test_save_manifest,
]
with mock.patch("parsec.core.cli.rsync._get_or_create_directory",
_get_or_create_directory_mock):
root_manifest, parent = await rsync._root_manifest_parent(
FsPath("/path_in_workspace"), alice_workspace, workspace_manifest)
assert root_manifest == workspace_test_manifest
assert parent == FsPath("/path_in_workspace")
_get_or_create_directory_mock.side_effect = [
workspace_test_manifest,
workspace_test_save_manifest,
]
with mock.patch("parsec.core.cli.rsync._get_or_create_directory",
_get_or_create_directory_mock):
root_manifest, parent = await rsync._root_manifest_parent(
FsPath("/path_in_workspace/save"), alice_workspace,
workspace_manifest)
assert root_manifest == workspace_test_save_manifest
assert parent == FsPath("/path_in_workspace/save")
async def test_get_or_create_directory(alice_workspace):
load_manifest_mock = AsyncMock(spec=mock.Mock(),
side_effect=lambda x: "load_manifest_mock")
alice_workspace.remote_loader.load_manifest = load_manifest_mock
_create_path_mock = AsyncMock(spec=mock.Mock(),
side_effect=lambda *x: "_create_path_mock")
with mock.patch("parsec.core.cli.rsync._create_path", _create_path_mock):
entry_id = EntryID()
res = await rsync._get_or_create_directory(
entry_id, alice_workspace, FsPath("/test_directory"),
FsPath("/path_in_workspace"))
load_manifest_mock.assert_called_once_with(entry_id)
_create_path_mock.assert_not_called()
assert res == "load_manifest_mock"
load_manifest_mock.reset_mock()
with mock.patch("parsec.core.cli.rsync._create_path", _create_path_mock):
res = await rsync._get_or_create_directory(
None, alice_workspace, FsPath("/test_directory"),
FsPath("/path_in_workspace"))
load_manifest_mock.assert_not_called()
_create_path_mock.assert_called_once_with(alice_workspace, True,
FsPath("/test_directory"),
FsPath("/path_in_workspace"))
assert res == "_create_path_mock"
async def test_operations_on_file(alice_workspace_t4, alice_workspace_t5):
_, fd4 = await alice_workspace_t4.transactions.file_open(FsPath("/files/content"), "r")
assert isinstance(fd4, int)
transactions_t4 = alice_workspace_t4.transactions
_, fd5 = await alice_workspace_t5.transactions.file_open(FsPath("/files/content"), "r")
assert isinstance(fd5, int)
transactions_t5 = alice_workspace_t5.transactions
data = await transactions_t4.fd_read(fd4, 1, 0)
assert data == b"a"
data = await transactions_t4.fd_read(fd4, 3, 1)
assert data == b"bcd"
data = await transactions_t4.fd_read(fd4, 100, 4)
assert data == b"e"
data = await transactions_t4.fd_read(fd4, 4, 0)
assert data == b"abcd"
data = await transactions_t4.fd_read(fd4, -1, 0)
assert data == b"abcde"
with pytest.raises(FSError): # if removed from local_storage, no write right error?..
await alice_workspace_t4.transactions.fd_write(fd4, b"hello ", 0)
data = await transactions_t5.fd_read(fd5, 100, 0)
assert data == b"fghij"
data = await transactions_t5.fd_read(fd5, 1, 1)
assert data == b"g"
data = await transactions_t4.fd_read(fd4, 1, 2)
assert data == b"c"
await transactions_t5.fd_close(fd5)
with pytest.raises(FSInvalidFileDescriptor):
data = await transactions_t5.fd_read(fd5, 1, 0)
data = await transactions_t4.fd_read(fd4, 1, 3)
assert data == b"d"
_, fd5 = await alice_workspace_t5.transactions.file_open(FsPath("/files/content"), "r")
data = await transactions_t5.fd_read(fd5, 3, 0)
assert data == b"fgh"
async def get_path_at_timestamp(self, entry_id: EntryID,
timestamp: DateTime) -> FsPath:
"""
Find a path for an entry_id at a specific timestamp.
If the path is broken, will raise an EntryNotFound exception. All the other exceptions are
thrown by the ManifestCache.
Raises:
FSError
FSBackendOfflineError
FSWorkspaceInMaintenance
FSBadEncryptionRevision
FSWorkspaceNoAccess
EntryNotFound
"""
# Get first manifest
try:
current_id = entry_id
current_manifest, _ = await self.load(current_id,
timestamp=timestamp)
except FSRemoteManifestNotFound:
raise EntryNotFound(entry_id)
# Loop over parts
parts = []
while not isinstance(current_manifest, WorkspaceManifest):
# Get the manifest
try:
current_manifest = cast(Union[FolderManifest, FileManifest],
current_manifest)
parent_manifest, _ = await self.load(current_manifest.parent,
timestamp=timestamp)
parent_manifest = cast(
Union[FolderManifest, WorkspaceManifest], parent_manifest)
except FSRemoteManifestNotFound:
raise EntryNotFound(entry_id)
# Find the child name
for name, child_id in parent_manifest.children.items():
if child_id == current_id:
parts.append(name)
break
else:
raise EntryNotFound(entry_id)
# Continue until root is found
current_id = current_manifest.parent
current_manifest = parent_manifest
# Return the path
return FsPath("/" + "/".join(reversed(parts)))
def set_workspace_fs(self,
wk_fs,
current_directory=FsPath("/"),
default_selection=None):
self.current_directory = current_directory
self.workspace_fs = wk_fs
ws_entry = self.jobs_ctx.run_sync(
self.workspace_fs.get_workspace_entry)
self.current_user_role = ws_entry.role
# self.label_role.setText(self.ROLES_TEXTS[self.current_user_role])
self.table_files.current_user_role = self.current_user_role
self.clipboard = None
self.reset(default_selection)
def generate_file_link(self,
path: AnyPath) -> BackendOrganizationFileLinkAddr:
"""
Raises: Nothing
"""
workspace_entry = self.get_workspace_entry()
encrypted_path = workspace_entry.key.encrypt(
str(FsPath(path)).encode("utf-8"))
return BackendOrganizationFileLinkAddr.build(
organization_addr=self.device.organization_addr,
workspace_id=workspace_entry.id,
encrypted_path=encrypted_path,
)
async def test_folder_create_delete(alice_entry_transactions,
alice_sync_transactions):
entry_transactions = alice_entry_transactions
sync_transactions = alice_sync_transactions
# Create and delete a foo directory
foo_id = await entry_transactions.folder_create(FsPath("/foo"))
assert await entry_transactions.folder_delete(FsPath("/foo")) == foo_id
# The directory is not synced
manifest = await entry_transactions.local_storage.get_manifest(foo_id)
assert manifest.need_sync
# Create and sync a bar directory
bar_id = await entry_transactions.folder_create(FsPath("/bar"))
remote = await sync_transactions.synchronization_step(bar_id)
assert await sync_transactions.synchronization_step(bar_id, remote) is None
# Remove the bar directory, the manifest is synced
assert await entry_transactions.folder_delete(FsPath("/bar")) == bar_id
manifest = await entry_transactions.local_storage.get_manifest(bar_id)
assert not manifest.need_sync
async def test_create_path(alice_workspace):
mkdir_mock = AsyncMock(spec=mock.Mock)
alice_workspace.mkdir = mkdir_mock
sync_mock = AsyncMock(spec=mock.Mock)
alice_workspace.sync = sync_mock
path_info_mock = AsyncMock(spec=mock.Mock,
side_effect=lambda x: {"id": "mock_id"})
alice_workspace.path_info = path_info_mock
get_manifest_mock = AsyncMock(spec=mock.Mock,
side_effect=lambda x: "mock_manifest")
alice_workspace.local_storage.get_manifest = get_manifest_mock
import_file_mock = AsyncMock(spec=mock.Mock)
with mock.patch("parsec.core.cli.rsync._import_file", import_file_mock):
is_dir = True
res = await rsync._create_path(alice_workspace, is_dir,
FsPath("/test"),
FsPath("/path_in_workspace/test"))
mkdir_mock.assert_called_once_with(FsPath("/path_in_workspace/test"))
sync_mock.assert_called_once_with()
path_info_mock.assert_called_once_with(
FsPath("/path_in_workspace/test"))
get_manifest_mock.assert_called_once_with("mock_id")
import_file_mock.assert_not_called()
assert res == "mock_manifest"
mkdir_mock.reset_mock()
sync_mock.reset_mock()
path_info_mock.reset_mock()
get_manifest_mock.reset_mock()
import_file_mock.reset_mock()
with mock.patch("parsec.core.cli.rsync._import_file", import_file_mock):
is_dir = False
res = await rsync._create_path(alice_workspace, is_dir,
FsPath("/test"),
FsPath("/path_in_workspace/test"))
mkdir_mock.assert_not_called()
path_info_mock.assert_not_called()
get_manifest_mock.assert_not_called()
import_file_mock.assert_called_once_with(
alice_workspace, FsPath("/test"),
FsPath("/path_in_workspace/test"))
sync_mock.assert_called_once_with()
assert res is None
async def _resolve_placeholders_in_path(self, path: FsPath, access: Access,
manifest: LocalManifest) -> bool:
"""
Returns: If an additional sync is needed
"""
# Notes we sync recursively from children to parents, this is more
# efficient given otherwise we would have to do:
# 1) sync the parent
# 2) sync the child
# 3) re-sync the parent with the child
is_placeholder = is_placeholder_manifest(manifest)
if not is_placeholder:
# Cannot have a non-placeholder with a placeholder parent, hence
# we don't have to go any further.
return manifest.need_sync
else:
if is_file_manifest(manifest):
need_more_sync = await self._minimal_sync_file(
path, access, manifest)
else:
need_more_sync = await self._minimal_sync_folder(
path, access, manifest)
# Once the entry is synced, we must sync it parent as well to have
# the entry visible for other clients
if not path.is_root():
try:
parent_access, parent_manifest = self.local_folder_fs.get_entry(
path.parent)
except FSManifestLocalMiss:
# Nothing to do if entry is no present locally
return False
if is_placeholder_manifest(parent_manifest):
await self._resolve_placeholders_in_path(
path.parent, parent_access, parent_manifest)
else:
await self._sync_folder(path.parent,
parent_access,
parent_manifest,
recursive=False)
if not need_more_sync:
self.event_bus.send("fs.entry.synced",
path=str(path),
id=access.id)
return need_more_sync
def test_parse_destination():
mock_workspace1 = mock.Mock()
mock_workspace1.name = "workspace1"
mock_workspace2 = mock.Mock()
mock_workspace2.name = "workspace2"
workspaces_mock = mock.Mock()
workspaces_mock.workspaces = [mock_workspace1, mock_workspace2]
get_user_manifest_mock = mock.Mock(return_value=workspaces_mock)
alice_core = mock.Mock()
alice_core.user_fs.get_user_manifest = get_user_manifest_mock
workspace, path = rsync._parse_destination(alice_core, "workspace1")
assert workspace == mock_workspace1
assert path is None
workspace, path = rsync._parse_destination(alice_core,
"workspace1:/test/save")
assert workspace == mock_workspace1
assert path == FsPath("/test/save")
workspace, path = rsync._parse_destination(alice_core, "workspace1")
assert workspace == mock_workspace1
assert path is None
workspace, path = rsync._parse_destination(alice_core,
"workspace2:/test/save2")
assert workspace == mock_workspace2
assert path == FsPath("/test/save2")
with pytest.raises(SystemExit):
workspace, path = rsync._parse_destination(alice_core,
"unknown_workspace")
with pytest.raises(SystemExit):
workspace, path = rsync._parse_destination(
alice_core, "unknown_workspace:/test/save3")
def get_beacon(self, path: FsPath) -> UUID:
# The beacon is used to notify other clients that we modified an entry.
# We try to use the id of workspace containing the modification as
# beacon. This is not possible when directly modifying the user
# manifest in which case we use the user manifest id as beacon.
try:
_, workspace_name, *_ = path.parts
except ValueError:
return self.root_access.id
access, manifest = self._retrieve_entry_read_only(
FsPath(f"/{workspace_name}"))
assert is_workspace_manifest(manifest)
return access.id
async def read_bytes(self,
path: AnyPath,
size: int = -1,
offset: int = 0) -> bytes:
"""
Raises:
FSError
"""
path = FsPath(path)
_, fd = await self.transactions.file_open(path, "r")
try:
return await self.transactions.fd_read(fd, size, offset)
finally:
await self.transactions.fd_close(fd)
async def test_sync_by_id_couple(alice_workspace, bob_workspace):
alice_entry = alice_workspace.get_workspace_entry()
alice_wid = alice_entry.id
bob_entry = bob_workspace.get_workspace_entry()
bob_wid = bob_entry.id
# Create directories
await alice_workspace.mkdir("/a")
await bob_workspace.mkdir("/b")
# Alice sync /a
await alice_workspace.sync_by_id(alice_wid)
# Bob sync /b and doesn't know about alice /a yet
await bob_workspace.sync_by_id(bob_wid, remote_changed=False)
# Alice knows about bob /b
await alice_workspace.sync_by_id(alice_wid)
# Everyone should be up to date by now
expected = [FsPath("/a"), FsPath("/b")]
assert await bob_workspace.listdir("/") == expected
assert await alice_workspace.listdir("/") == expected
async def test_access_not_loaded_entry(alice, bob, alice_entry_transactions):
entry_transactions = alice_entry_transactions
entry_id = entry_transactions.get_workspace_entry().id
manifest = await entry_transactions.local_storage.get_manifest(entry_id)
async with entry_transactions.local_storage.lock_entry_id(entry_id):
await entry_transactions.local_storage.clear_manifest(entry_id)
with pytest.raises(FSRemoteManifestNotFound):
await entry_transactions.entry_info(FsPath("/"))
async with entry_transactions.local_storage.lock_entry_id(entry_id):
await entry_transactions.local_storage.set_manifest(entry_id, manifest)
entry_info = await entry_transactions.entry_info(FsPath("/"))
assert entry_info == {
"type": "folder",
"id": entry_id,
"created": Pendulum(2000, 1, 1),
"updated": Pendulum(2000, 1, 1),
"base_version": 0,
"is_placeholder": True,
"need_sync": True,
"children": [],
}
async def test_rmdir(alice_workspace):
await alice_workspace.mkdir("/foz")
await alice_workspace.rmdir("/foz")
lst = await alice_workspace.listdir("/")
assert lst == [FsPath("/foo")]
with pytest.raises(OSError) as context:
await alice_workspace.rmdir("/foo")
assert context.value.errno == errno.ENOTEMPTY
with pytest.raises(NotADirectoryError):
await alice_workspace.rmdir("/foo/bar")
with pytest.raises(PermissionError):
await alice_workspace.rmdir("/")
async def test_file_create(alice_entry_transactions, alice_file_transactions,
alice):
entry_transactions = alice_entry_transactions
file_transactions = alice_file_transactions
with freeze_time("2000-01-02"):
access_id, fd = await entry_transactions.file_create(FsPath("/foo.txt")
)
await file_transactions.fd_close(fd)
assert fd == 1
root_stat = await entry_transactions.entry_info(FsPath("/"))
assert root_stat == {
"type": "folder",
"id": entry_transactions.workspace_id,
"base_version": 0,
"is_placeholder": True,
"need_sync": True,
"created": datetime(2000, 1, 1),
"updated": datetime(2000, 1, 2),
"children": ["foo.txt"],
"confinement_point": None,
}
foo_stat = await entry_transactions.entry_info(FsPath("/foo.txt"))
assert foo_stat == {
"type": "file",
"id": access_id,
"base_version": 0,
"is_placeholder": True,
"need_sync": True,
"created": datetime(2000, 1, 2),
"updated": datetime(2000, 1, 2),
"size": 0,
"confinement_point": None,
}
async def move(self, source: AnyPath, destination: AnyPath) -> None:
"""
Raises:
FSError
"""
source = FsPath(source)
destination = FsPath(destination)
real_destination = destination
if source.parts == destination.parts[:len(source.parts)]:
raise FSInvalidArgumentError(
f"Cannot move a directory {source} into itself {destination}")
try:
if await self.is_dir(destination):
real_destination = destination / source.name
if await self.exists(real_destination):
raise FileExistsError
# At this point, real_destination is the target either representing :
# - the destination path if it didn't already exist,
# - a new entry with the same name as source, but inside the destination directory
except FileNotFoundError:
pass
# Rename if possible
if source.parent == real_destination.parent:
return await self.rename(source, real_destination)
# Copy directory
if await self.is_dir(source):
await self.copytree(source, real_destination)
await self.rmtree(source)
return
# Copy file
await self.copyfile(source, real_destination)
await self.unlink(source)
