def _sync_folder_merge_back(
self,
path: FsPath,
access: Access,
base_manifest: LocalFolderManifest,
target_remote_manifest: FolderManifest,
) -> None:
# Merge with the current version of the manifest which may have
# been modified in the meantime
assert is_folderish_manifest(target_remote_manifest)
current_manifest = self.local_folder_fs.get_manifest(access)
assert is_folderish_manifest(current_manifest)
target_manifest = target_remote_manifest.to_local()
final_manifest, conflicts = merge_local_folder_manifests(
base_manifest, current_manifest, target_manifest)
for original_name, original_id, diverged_name, diverged_id in conflicts:
self.event_bus.send(
"fs.entry.name_conflicted",
path=str(path / original_name),
diverged_path=str(path / diverged_name),
original_id=original_id,
diverged_id=diverged_id,
)
self.local_folder_fs.set_manifest(access, final_manifest)
def _delete(self, path: FsPath, expect=None) -> None:
if path.is_root():
raise PermissionError(13, "Permission denied", str(path))
parent_access, parent_manifest = self._retrieve_entry(path.parent)
if not is_folderish_manifest(parent_manifest):
raise NotADirectoryError(20, "Not a directory", str(path.parent))
try:
item_access = parent_manifest.children[path.name]
except KeyError:
raise FileNotFoundError(2, "No such file or directory", str(path))
item_manifest = self.get_manifest(item_access)
if is_folderish_manifest(item_manifest):
if expect == "file":
raise IsADirectoryError(21, "Is a directory", str(path))
if item_manifest.children:
raise OSError(39, "Directory not empty", str(path))
elif expect == "folder":
raise NotADirectoryError(20, "Not a directory", str(path))
parent_manifest = parent_manifest.evolve_children_and_mark_updated(
{path.name: None})
self.set_manifest(parent_access, parent_manifest)
self.event_bus.send("fs.entry.updated", id=parent_access.id)
async def _sync_folder(self, path: FsPath, access: Access,
manifest: LocalFolderManifest,
recursive: bool) -> None:
assert not is_placeholder_manifest(manifest)
assert is_folderish_manifest(manifest)
# Synchronizing a folder is divided into three steps:
# - first synchronizing it children
# - then sychronize itself
# - finally merge the synchronized version with the current one (that
# may have been updated in the meantime)
# Synchronizing children
if recursive:
for child_name, child_access in sorted(manifest.children.items(),
key=lambda x: x[0]):
child_path = path / child_name
try:
await self._sync_nolock(child_path, True)
except FileNotFoundError:
# Concurrent deletion occured, just ignore this child
pass
# The trick here is to retreive the current version of the manifest
# and remove it placeholders (those are the children created since
# the start of our sync)
manifest = self.local_folder_fs.get_manifest(access)
assert is_folderish_manifest(manifest)
manifest = manifest.evolve(
children=self._strip_placeholders(manifest.children))
# Now we can synchronize the folder if needed
if not manifest.need_sync:
target_remote_manifest = await self._sync_folder_look_for_remote_changes(
access, manifest)
# Quick exit if nothing's new
if not target_remote_manifest:
return
event_type = "fs.entry.remote_changed"
else:
target_remote_manifest = await self._sync_folder_actual_sync(
path, access, manifest)
event_type = "fs.entry.synced"
assert is_folderish_manifest(target_remote_manifest)
# Merge the synchronized version with the current one
self._sync_folder_merge_back(path, access, manifest,
target_remote_manifest)
self.event_bus.send(event_type, path=str(path), id=access.id)
async def folder_delete(self, path: FsPath) -> EntryID:
# Check write rights
self.check_write_rights(path)
# Fetch and lock
async with self._lock_parent_manifest_from_path(path) as (parent,
child):
# Entry doesn't exist
if child is None:
raise FSFileNotFoundError(filename=path)
# Not a directory
if not is_folderish_manifest(child):
raise FSNotADirectoryError(filename=path)
# Directory not empty
if child.children:
raise FSDirectoryNotEmptyError(filename=path)
# Create new manifest
new_parent = parent.evolve_children_and_mark_updated(
{path.name: None})
# Atomic change
await self.local_storage.set_manifest(parent.id, new_parent)
# Send event
self._send_event("fs.entry.updated", id=parent.id)
# Return the entry id of the removed folder
return child.id
async def _lock_parent_manifest_from_path(
self, path: FsPath) -> Tuple[LocalManifest, LocalManifest]:
# This is the most complicated locking scenario.
# It requires locking the parent of the given entry and the entry itself
# if it exists.
# This is done in a two step process:
# - 1. Lock the parent (it must exist). While the parent is locked, no
# children can be added, renamed or removed.
# - 2. Lock the children if exists. It it doesn't, there is nothing to lock
# since the parent lock guarentees that it is not going to be added while
# using the context.
# This double locking is only required for a single use case: the overwriting
# of empty directory during a move. We have to make sure that no one adds
# something to the directory while it is being overwritten.
# If read/write locks were to be implemented, the parent would be write locked
# and the child read locked. This means that despite locking two entries, only
# a single entry is modified at a time.
# Source is root
if path.is_root():
raise FSPermissionError(filename=str(path))
# Loop over attempts
while True:
# Lock parent first
async with self._lock_manifest_from_path(path.parent) as parent:
# Parent is not a directory
if not is_folderish_manifest(parent):
raise FSNotADirectoryError(filename=path.parent)
# Child doesn't exist
if path.name not in parent.children:
yield parent, None
return
# Child exists
entry_id = parent.children[path.name]
try:
async with self.local_storage.lock_manifest(
entry_id) as manifest:
yield parent, manifest
return
# Child is not available
except FSLocalMissError as exc:
assert exc.id == entry_id
# Release the lock and download the child manifest
await self._load_manifest(entry_id)
def _recursive_dump(access: Access):
dump_data = {"access": attr.asdict(access)}
try:
manifest = self.get_manifest(access)
except FSManifestLocalMiss:
return dump_data
dump_data.update(attr.asdict(manifest))
if is_folderish_manifest(manifest):
for child_name, child_access in manifest.children.items():
dump_data["children"][child_name] = _recursive_dump(
child_access)
return dump_data
def _recursive_search(access, path):
try:
manifest = self._get_manifest_read_only(access)
except FSManifestLocalMiss:
return
if access.id == entry_id:
return path, access, manifest
if is_folderish_manifest(manifest):
for child_name, child_access in manifest.children.items():
found = _recursive_search(child_access, path / child_name)
if found:
return found
def _recursive_get_local_entries_ids(access):
try:
manifest = self.local_folder_fs.get_manifest(access)
except FSManifestLocalMiss:
# TODO: make the assert true...
# Root should always be loaded
assert access is not self.device.user_manifest_access
return
if is_folderish_manifest(manifest):
for child_access in manifest.children.values():
_recursive_get_local_entries_ids(child_access)
entries.append({
"id": access.id,
"rts": access.rts,
"version": manifest.base_version
})
def _retrieve_entry_read_only(
self,
path: FsPath,
collector=None) -> Tuple[Access, LocalManifest]:
curr_access = self.root_access
curr_manifest = self._get_manifest_read_only(curr_access)
if collector:
collector(curr_access, curr_manifest)
try:
_, *hops, dest = list(path.walk_to_path())
except ValueError:
return curr_access, curr_manifest
for hop in hops:
try:
curr_access = curr_manifest.children[hop.name]
except KeyError:
raise FileNotFoundError(2, "No such file or directory",
str(hop))
curr_manifest = self._get_manifest_read_only(curr_access)
if not is_folderish_manifest(curr_manifest):
raise NotADirectoryError(20, "Not a directory", str(hop))
if collector:
collector(curr_access, curr_manifest)
try:
curr_access = curr_manifest.children[dest.name]
except KeyError:
raise FileNotFoundError(2, "No such file or directory", str(dest))
curr_manifest = self._get_manifest_read_only(curr_access)
if collector:
collector(curr_access, curr_manifest)
return curr_access, curr_manifest
def touch(self, path: FsPath) -> None:
if path.is_root():
raise FileExistsError(17, "File exists", str(path))
if path.parent.is_root():
raise PermissionError(
13, "Permission denied (only workpace allowed at root level)",
str(path))
access, manifest = self._retrieve_entry(path.parent)
if not is_folderish_manifest(manifest):
raise NotADirectoryError(20, "Not a directory", str(path.parent))
if path.name in manifest.children:
raise FileExistsError(17, "File exists", str(path))
child_access = ManifestAccess()
child_manifest = LocalFileManifest(self.local_author)
manifest = manifest.evolve_children_and_mark_updated(
{path.name: child_access})
self.set_manifest(access, manifest)
self.set_manifest(child_access, child_manifest)
self.event_bus.send("fs.entry.updated", id=access.id)
self.event_bus.send("fs.entry.updated", id=child_access.id)
def _recursive_create_copy_map(access, manifest):
copy_map = {"access": access, "manifest": manifest}
manifests_miss = []
if is_folderish_manifest(manifest):
copy_map["children"] = {}
for child_name, child_access in manifest.children.items():
try:
child_manifest = self._get_manifest_read_only(
child_access)
except FSManifestLocalMiss as exc:
manifests_miss.append(exc.access)
else:
try:
copy_map["children"][
child_name] = _recursive_create_copy_map(
child_access, child_manifest)
except FSMultiManifestLocalMiss as exc:
manifests_miss += exc.accesses
if manifests_miss:
raise FSMultiManifestLocalMiss(manifests_miss)
return copy_map
async def _sync_by_id(self,
entry_id: EntryID,
remote_changed: bool = True) -> RemoteManifest:
"""
Synchronize the entry corresponding to a specific ID.
This method keeps performing synchronization steps on the given ID until one of
those two conditions is met:
- there is no more changes to upload
- one upload operation has succeeded and has been acknowledged
This guarantees that any change prior to the call is saved remotely when this
method returns.
"""
# Get the current remote manifest if it has changed
remote_manifest = None
if remote_changed:
try:
remote_manifest = await self.remote_loader.load_manifest(
entry_id)
except FSRemoteManifestNotFound:
pass
# Loop over sync transactions
final = False
while True:
# Protect against race conditions on the entry id
try:
# Perform the sync step transaction
try:
new_remote_manifest = await self.transactions.synchronization_step(
entry_id, remote_manifest, final)
# The entry first requires reshaping
except FSReshapingRequiredError:
await self.transactions.file_reshape(entry_id)
continue
# The manifest doesn't exist locally
except FSLocalMissError:
raise FSNoSynchronizationRequired(entry_id)
# No new manifest to upload, the entry is synced!
if new_remote_manifest is None:
return remote_manifest or (
await self.local_storage.get_manifest(entry_id)).base
# Synchronize placeholder children
if is_folderish_manifest(new_remote_manifest):
await self._synchronize_placeholders(new_remote_manifest)
# Upload blocks
if is_file_manifest(new_remote_manifest):
await self._upload_blocks(new_remote_manifest)
# Restamp the remote manifest
new_remote_manifest = new_remote_manifest.evolve(
timestamp=pendulum_now())
# Upload the new manifest containing the latest changes
try:
await self.remote_loader.upload_manifest(
entry_id, new_remote_manifest)
# The upload has failed: download the latest remote manifest
except FSRemoteSyncError:
remote_manifest = await self.remote_loader.load_manifest(
entry_id)
# The upload has succeeded: loop one last time to acknowledge this new version
else:
final = True
remote_manifest = new_remote_manifest
def _copy(self, src: FsPath, dst: FsPath, delete_src: bool) -> None:
# The idea here is to consider a manifest never move around the fs
# (i.e. a given access always points to the same path). This simplify
# sync notifications handling and avoid ending up with two path
# (possibly from different workspaces !) pointing to the same manifest
# which would be weird for user experience.
# Long story short, when moving/copying manifest, we must recursively
# copy the manifests and create new accesses for them.
parent_src = src.parent
parent_dst = dst.parent
# No matter what, cannot move or overwrite root
if src.is_root():
# Raise FileNotFoundError if parent_dst doesn't exists
_, parent_dst_manifest = self._retrieve_entry(parent_dst)
if not is_folderish_manifest(parent_dst_manifest):
raise NotADirectoryError(20, "Not a directory",
str(parent_dst))
else:
raise PermissionError(13, "Permission denied", str(src),
str(dst))
elif dst.is_root():
# Raise FileNotFoundError if parent_src doesn't exists
_, parent_src_manifest = self._retrieve_entry(src.parent)
if not is_folderish_manifest(parent_src_manifest):
raise NotADirectoryError(20, "Not a directory",
str(src.parent))
else:
raise PermissionError(13, "Permission denied", str(src),
str(dst))
if src == dst:
# Raise FileNotFoundError if doesn't exist
src_access, src_ro_manifest = self._retrieve_entry_read_only(src)
if is_workspace_manifest(src_ro_manifest):
raise PermissionError(
13,
"Permission denied (cannot move/copy workpace, must rename it)",
str(src),
str(dst),
)
return
if parent_src == parent_dst:
parent_access, parent_manifest = self._retrieve_entry(parent_src)
if not is_folderish_manifest(parent_manifest):
raise NotADirectoryError(20, "Not a directory",
str(parent_src))
try:
dst.relative_to(src)
except ValueError:
pass
else:
raise OSError(22, "Invalid argument", str(src), None, str(dst))
try:
src_access = parent_manifest.children[src.name]
except KeyError:
raise FileNotFoundError(2, "No such file or directory",
str(src))
existing_dst_access = parent_manifest.children.get(dst.name)
src_manifest = self.get_manifest(src_access)
if is_workspace_manifest(src_manifest):
raise PermissionError(
13,
"Permission denied (cannot move/copy workpace, must rename it)",
str(src),
str(dst),
)
if existing_dst_access:
existing_dst_manifest = self.get_manifest(existing_dst_access)
if is_folderish_manifest(src_manifest):
if is_file_manifest(existing_dst_manifest):
raise NotADirectoryError(20, "Not a directory",
str(dst))
elif existing_dst_manifest.children:
raise OSError(39, "Directory not empty", str(dst))
else:
if is_folderish_manifest(existing_dst_manifest):
raise IsADirectoryError(21, "Is a directory", str(dst))
moved_access = self._recursive_manifest_copy(
src_access, src_manifest)
if not delete_src:
parent_manifest = parent_manifest.evolve_children(
{dst.name: moved_access})
else:
parent_manifest = parent_manifest.evolve_children({
dst.name:
moved_access,
src.name:
None
})
self.set_manifest(parent_access, parent_manifest)
self.event_bus.send("fs.entry.updated", id=parent_access.id)
else:
parent_src_access, parent_src_manifest = self._retrieve_entry(
parent_src)
if not is_folderish_manifest(parent_src_manifest):
raise NotADirectoryError(20, "Not a directory",
str(parent_src))
parent_dst_access, parent_dst_manifest = self._retrieve_entry(
parent_dst)
if not is_folderish_manifest(parent_dst_manifest):
raise NotADirectoryError(20, "Not a directory",
str(parent_dst))
try:
dst.relative_to(src)
except ValueError:
pass
else:
raise OSError(22, "Invalid argument", str(src), None, str(dst))
try:
src_access = parent_src_manifest.children[src.name]
except KeyError:
raise FileNotFoundError(2, "No such file or directory",
str(src))
existing_dst_access = parent_dst_manifest.children.get(dst.name)
src_manifest = self.get_manifest(src_access)
if is_workspace_manifest(src_manifest):
raise PermissionError(
13,
"Permission denied (cannot move/copy workpace, must rename it)",
str(src),
str(dst),
)
if existing_dst_access:
existing_entry_manifest = self.get_manifest(
existing_dst_access)
if is_folderish_manifest(src_manifest):
if is_file_manifest(existing_entry_manifest):
raise NotADirectoryError(20, "Not a directory",
str(dst))
elif existing_entry_manifest.children:
raise OSError(39, "Directory not empty", str(dst))
else:
if is_folderish_manifest(existing_entry_manifest):
raise IsADirectoryError(21, "Is a directory", str(dst))
moved_access = self._recursive_manifest_copy(
src_access, src_manifest)
parent_dst_manifest = parent_dst_manifest.evolve_children_and_mark_updated(
{dst.name: moved_access})
self.set_manifest(parent_dst_access, parent_dst_manifest)
self.event_bus.send("fs.entry.updated", id=parent_dst_access.id)
if delete_src:
parent_src_manifest = parent_src_manifest.evolve_children_and_mark_updated(
{src.name: None})
self.set_manifest(parent_src_access, parent_src_manifest)
self.event_bus.send("fs.entry.updated",
id=parent_src_access.id)