def _sync_file_merge_back(self, access: Access,
base_manifest: LocalFileManifest,
target_remote_manifest: FileManifest) -> None:
# Merge with the current version of the manifest which may have
# been modified in the meantime
assert is_file_manifest(target_remote_manifest)
current_manifest = self.local_folder_fs.get_manifest(access)
assert is_file_manifest(current_manifest)
final_manifest = fast_forward_file(base_manifest, current_manifest,
target_remote_manifest)
self.local_folder_fs.set_manifest(access, final_manifest)
async def _sync_nolock(self, path: FsPath, recursive: bool) -> None:
# First retrieve a snapshot of the manifest to sync
try:
access, manifest = self.local_folder_fs.get_entry(path)
except FSManifestLocalMiss:
# Nothing to do if entry is no present locally
return
if not manifest.need_sync and not recursive:
return
# In case of placeholder, we must resolve it first (and make sure
# none of it parents are placeholders themselves)
if manifest.is_placeholder:
need_more_sync = await self._resolve_placeholders_in_path(
path, access, manifest)
# If the entry to sync is actually empty the minimal sync was enough
if not need_more_sync and not recursive:
return
# Reload the manifest given the minimal sync has changed it
try:
manifest = self.local_folder_fs.get_manifest(access)
except FSManifestLocalMiss:
# Nothing to do if entry is no present locally
return
# Note from now on it's possible the entry has changed due to
# concurrent access (or even have been totally removed !).
# We will deal with this when merged synced data back into the local fs.
# Now we can do the sync on the entry
if is_file_manifest(manifest):
await self._sync_file(path, access, manifest)
else:
await self._sync_folder(path, access, manifest, recursive)
def flush(self, fd: FileDescriptor) -> None:
cursor = self._get_cursor_from_fd(fd)
manifest = self.local_folder_fs.get_manifest(cursor.access)
assert is_file_manifest(manifest)
hf = self._get_hot_file(cursor.access)
if manifest.size == hf.size and not hf.pending_writes:
return
new_dirty_blocks = []
for pw in hf.pending_writes:
block_access = BlockAccess.from_block(pw.data, pw.start)
self.set_block(block_access, pw.data, False)
new_dirty_blocks.append(block_access)
# TODO: clean overwritten dirty blocks
manifest = manifest.evolve_and_mark_updated(
dirty_blocks=(*manifest.dirty_blocks, *new_dirty_blocks), size=hf.size
)
self.local_folder_fs.set_manifest(cursor.access, manifest)
hf.pending_writes.clear()
self.event_bus.send("fs.entry.updated", id=cursor.access.id)
async def file_delete(self, path: FsPath) -> EntryID:
# Check write rights
self.check_write_rights(path)
# Fetch and lock
async with self._lock_parent_manifest_from_path(path) as (parent,
child):
# Entry doesn't exist
if child is None:
raise FSFileNotFoundError(filename=path)
# Not a file
if not is_file_manifest(child):
raise FSIsADirectoryError(filename=path)
# Create new manifest
new_parent = parent.evolve_children_and_mark_updated(
{path.name: None})
# Atomic change
await self.local_storage.set_manifest(parent.id, new_parent)
# Send event
self._send_event("fs.entry.updated", id=parent.id)
# Return the entry id of the deleted file
return child.id
async def entry_info(self, path: FsPath) -> dict:
# Check read rights
self.check_read_rights(path)
# Fetch data
manifest = await self._get_manifest_from_path(path)
# General stats
stats = {
"id": manifest.id,
"created": manifest.created,
"updated": manifest.updated,
"base_version": manifest.base_version,
"is_placeholder": manifest.is_placeholder,
"need_sync": manifest.need_sync,
}
# File/folder specific stats
if is_file_manifest(manifest):
stats["type"] = "file"
stats["size"] = manifest.size
else:
stats["type"] = "folder"
stats["children"] = sorted(manifest.children.keys())
return stats
def read(self, fd: FileDescriptor, size: int = inf, offset: int = None) -> bytes:
cursor = self._get_cursor_from_fd(fd)
if offset is not None:
self._seek(cursor, offset)
hf = self._get_hot_file(cursor.access)
if cursor.offset > hf.size:
return b""
manifest = self.local_folder_fs.get_manifest(cursor.access)
assert is_file_manifest(manifest)
start = cursor.offset
end = cursor.offset + size
if end > hf.size:
end = hf.size
blocks = self._get_quickly_filtered_blocks(manifest, start, end)
blocks += hf.pending_writes
merged = merge_buffers_with_limits(blocks, start, end)
assert merged.size <= size
assert merged.start == start
missing = []
data = bytearray(end - start)
for cs in merged.spaces:
for bs in cs.buffers:
if isinstance(bs.buffer, DirtyBlockBuffer):
access = bs.buffer.access
try:
buff = self.get_block(access)
except LocalDBMissingEntry as exc:
raise RuntimeError(f"Unknown local block `{access['id']}`") from exc
data[bs.start - cs.start : bs.end - cs.start] = buff[
bs.buffer_slice_start : bs.buffer_slice_end
]
elif isinstance(bs.buffer, BlockBuffer):
access = bs.buffer.access
try:
buff = self.get_block(access)
except LocalDBMissingEntry:
missing.append(access)
continue
data[bs.start - cs.start : bs.end - cs.start] = buff[
bs.buffer_slice_start : bs.buffer_slice_end
]
else:
data[bs.start - start : bs.end - start] = bs.get_data()
if missing:
raise FSBlocksLocalMiss(missing)
cursor.offset += len(data)
return data
def _recursive_process_copy_map(copy_map):
manifest = copy_map["manifest"]
cpy_access = ManifestAccess()
if is_file_manifest(manifest):
cpy_manifest = LocalFileManifest(
author=self.local_author,
size=manifest.size,
blocks=manifest.blocks,
dirty_blocks=manifest.dirty_blocks,
)
else:
cpy_children = {}
for child_name in manifest.children.keys():
child_copy_map = copy_map["children"][child_name]
new_child_access = _recursive_process_copy_map(
child_copy_map)
cpy_children[child_name] = new_child_access
if is_folder_manifest(manifest):
cpy_manifest = LocalFolderManifest(
author=self.local_author, children=cpy_children)
else:
assert is_workspace_manifest(manifest)
cpy_manifest = LocalWorkspaceManifest(
self.local_author, children=cpy_children)
self.set_manifest(cpy_access, cpy_manifest)
return cpy_access
async def _populate_tree_load(
self,
path_level: int,
entry_id: EntryID,
early: Pendulum,
late: Pendulum,
version_number: int,
expected_timestamp: Pendulum,
next_version_number: int,
):
if early > late:
return
manifest = await self.task_list.manifest_cache.load(
entry_id,
version=version_number,
expected_backend_timestamp=expected_timestamp)
data = ManifestDataAndMutablePaths(
ManifestData(
manifest.author,
manifest.updated,
is_folder_manifest(manifest),
None if not is_file_manifest(manifest) else manifest.size,
))
if len(self.target.parts) == path_level:
await data.populate_paths(self.task_list.manifest_cache, entry_id,
early, late)
self.return_dict[TimestampBoundedEntry(
manifest.id, manifest.version, early,
late)] = ManifestDataAndPaths(
data=data.manifest,
source=data.source_path
if data.source_path != data.current_path else None,
destination=data.destination_path
if data.destination_path != data.current_path else None,
)
else:
if not is_file_manifest(
manifest): # If it is a file, just ignores current path
for child_name, child_id in manifest.children.items():
if child_name == self.target.parts[path_level]:
return await self._populate_tree_list_versions(
path_level + 1, child_id, early, late)
async def sync_by_id(self,
entry_id: EntryID,
remote_changed: bool = True,
recursive: bool = True):
"""
Raises:
FSError
"""
# Make sure the corresponding realm exists
await self._create_realm_if_needed()
# Sync parent first
try:
async with self.sync_locks[entry_id]:
manifest = await self._sync_by_id(
entry_id, remote_changed=remote_changed)
# Nothing to synchronize if the manifest does not exist locally
except FSNoSynchronizationRequired:
return
# A file conflict needs to be adressed first
except FSFileConflictError as exc:
local_manifest, remote_manifest = exc.args
# Only file manifest have synchronization conflict
assert is_file_manifest(local_manifest)
await self.transactions.file_conflict(entry_id, local_manifest,
remote_manifest)
return await self.sync_by_id(local_manifest.parent)
# Non-recursive
if not recursive or is_file_manifest(manifest):
return
# Synchronize children
for name, entry_id in manifest.children.items():
await self.sync_by_id(entry_id,
remote_changed=remote_changed,
recursive=True)
def open(self, access: Access) -> FileDescriptor:
cursor = FileCursor(access)
# Sanity check
manifest = self.local_folder_fs.get_manifest(access)
if not is_file_manifest(manifest):
raise IsADirectoryError(21, "Is a directory")
hf = self._ensure_hot_file(access, manifest)
hf.cursors.add(id(cursor))
fd = self._next_fd
self._opened_cursors[fd] = cursor
self._next_fd += 1
return fd
async def _resolve_placeholders_in_path(self, path: FsPath, access: Access,
manifest: LocalManifest) -> bool:
"""
Returns: If an additional sync is needed
"""
# Notes we sync recursively from children to parents, this is more
# efficient given otherwise we would have to do:
# 1) sync the parent
# 2) sync the child
# 3) re-sync the parent with the child
is_placeholder = is_placeholder_manifest(manifest)
if not is_placeholder:
# Cannot have a non-placeholder with a placeholder parent, hence
# we don't have to go any further.
return manifest.need_sync
else:
if is_file_manifest(manifest):
need_more_sync = await self._minimal_sync_file(
path, access, manifest)
else:
need_more_sync = await self._minimal_sync_folder(
path, access, manifest)
# Once the entry is synced, we must sync it parent as well to have
# the entry visible for other clients
if not path.is_root():
try:
parent_access, parent_manifest = self.local_folder_fs.get_entry(
path.parent)
except FSManifestLocalMiss:
# Nothing to do if entry is no present locally
return False
if is_placeholder_manifest(parent_manifest):
await self._resolve_placeholders_in_path(
path.parent, parent_access, parent_manifest)
else:
await self._sync_folder(path.parent,
parent_access,
parent_manifest,
recursive=False)
if not need_more_sync:
self.event_bus.send("fs.entry.synced",
path=str(path),
id=access.id)
return need_more_sync
async def file_resize(self, path: FsPath, length: int) -> EntryID:
# Check write rights
self.check_write_rights(path)
# Lock manifest
async with self._lock_manifest_from_path(path) as manifest:
# Not a file
if not is_file_manifest(manifest):
raise FSIsADirectoryError(filename=path)
# Perform resize
await self._manifest_resize(manifest, length)
# Return entry id
return manifest.id
async def _lock_manifest_from_path(self, path: FsPath) -> LocalManifest:
# Root entry_id and manifest
entry_id = self.workspace_id
# Follow the path
for name in path.parts:
manifest = await self._load_manifest(entry_id)
if is_file_manifest(manifest):
raise FSNotADirectoryError(filename=path)
try:
entry_id = manifest.children[name]
except (AttributeError, KeyError):
raise FSFileNotFoundError(filename=path)
# Lock entry
async with self._load_and_lock_manifest(entry_id) as manifest:
yield manifest
def merge_manifests(
local_author: DeviceID,
local_manifest: LocalManifest,
remote_manifest: Optional[RemoteManifest] = None,
):
# Exctract versions
local_version = local_manifest.base_version
remote_version = local_version if remote_manifest is None else remote_manifest.version
# The remote hasn't changed
if remote_version <= local_version:
return local_manifest
# Only the remote has changed
if not local_manifest.need_sync:
return LocalManifest.from_remote(remote_manifest)
# Both the remote and the local have changed
assert remote_version > local_version and local_manifest.need_sync
# All the local changes have been successfully uploaded
if local_manifest.match_remote(remote_manifest):
return LocalManifest.from_remote(remote_manifest)
# The remote changes are ours, simply acknowledge them and keep our local changes
if remote_manifest.author == local_author:
return local_manifest.evolve(base=remote_manifest)
# The remote has been updated by some other device
assert remote_manifest.author != local_author
# Cannot solve a file conflict directly
if is_file_manifest(local_manifest):
raise FSFileConflictError(local_manifest, remote_manifest)
# Solve the folder conflict
new_children = merge_folder_children(
local_manifest.base.children,
local_manifest.children,
remote_manifest.children,
remote_manifest.author,
)
return local_manifest.evolve_and_mark_updated(base=remote_manifest,
children=new_children)
async def file_open(self,
path: FsPath,
mode="rw") -> Tuple[EntryID, FileDescriptor]:
# Check read and write rights
if "w" in mode:
self.check_write_rights(path)
else:
self.check_read_rights(path)
# Lock path in read mode
async with self._lock_manifest_from_path(path) as manifest:
# Not a file
if not is_file_manifest(manifest):
raise FSIsADirectoryError(filename=path)
# Return the entry id of the open file and the file descriptor
return manifest.id, self.local_storage.create_file_descriptor(
manifest)
async def _sync_file(self, path: FsPath, access: Access,
manifest: LocalFileManifest) -> None:
"""
Raises:
FileSyncConcurrencyError
BackendNotAvailable
"""
assert not is_placeholder_manifest(manifest)
assert is_file_manifest(manifest)
# Now we can synchronize the folder if needed
if not manifest.need_sync:
changed = await self._sync_file_look_for_remote_changes(
path, access, manifest)
else:
await self._sync_file_actual_sync(path, access, manifest)
changed = True
if changed:
self.event_bus.send("fs.entry.synced",
path=str(path),
id=access.id)
def stat(self, path: FsPath) -> dict:
access, manifest = self._retrieve_entry_read_only(path)
if is_file_manifest(manifest):
return {
"type": "file",
"is_folder": False,
"created": manifest.created,
"updated": manifest.updated,
"base_version": manifest.base_version,
"is_placeholder": manifest.is_placeholder,
"need_sync": manifest.need_sync,
"size": manifest.size,
}
elif is_workspace_manifest(manifest):
return {
"type": "workspace",
"is_folder": True,
"created": manifest.created,
"updated": manifest.updated,
"base_version": manifest.base_version,
"is_placeholder": manifest.is_placeholder,
"need_sync": manifest.need_sync,
"children": list(sorted(manifest.children.keys())),
"creator": manifest.creator,
"participants": list(manifest.participants),
}
else:
return {
"type": "root" if path.is_root() else "folder",
"is_folder": True,
"created": manifest.created,
"updated": manifest.updated,
"base_version": manifest.base_version,
"is_placeholder": manifest.is_placeholder,
"need_sync": manifest.need_sync,
"children": list(sorted(manifest.children.keys())),
}
async def synchronization_step(
self,
entry_id: EntryID,
remote_manifest: Optional[RemoteManifest] = None,
final: bool = False,
) -> Optional[RemoteManifest]:
"""Perform a synchronization step.
This step is meant to be called several times until the right state is reached.
It takes the current remote manifest as an argument and returns the new remote
manifest to upload. When the manifest is successfully uploaded, this method has
to be called once again with the new remote manifest as an argument. When there
is no more changes to upload, this method returns None. The `final` argument can
be set to true to indicate that the caller has no intention to upload a new
manifest. This also causes the method to return None.
"""
# Fetch and lock
async with self.local_storage.lock_manifest(entry_id) as local_manifest:
# Sync cannot be performed yet
if not final and is_file_manifest(local_manifest) and not local_manifest.is_reshaped():
# Try a quick reshape (without downloading any block)
missing = await self._manifest_reshape(local_manifest)
# Downloading block is necessary for this reshape
if missing:
raise FSReshapingRequiredError(entry_id)
# The manifest should be reshaped by now
local_manifest = await self.local_storage.get_manifest(entry_id)
assert local_manifest.is_reshaped()
# Merge manifests
new_local_manifest = merge_manifests(self.local_author, local_manifest, remote_manifest)
# Extract authors
base_author = local_manifest.base.author
remote_author = base_author if remote_manifest is None else remote_manifest.author
# Extract versions
base_version = local_manifest.base_version
new_base_version = new_local_manifest.base_version
remote_version = base_version if remote_manifest is None else remote_manifest.version
# Set the new base manifest
if base_version != remote_version or new_local_manifest.need_sync:
await self.local_storage.set_manifest(entry_id, new_local_manifest)
# Send downsynced event
if base_version != new_base_version and remote_author != self.local_author:
self._send_event("fs.entry.downsynced", id=entry_id)
# Send synced event
if local_manifest.need_sync and not new_local_manifest.need_sync:
self._send_event("fs.entry.synced", id=entry_id)
# Nothing new to upload
if final or not new_local_manifest.need_sync:
return None
# Produce the new remote manifest to upload
return new_local_manifest.to_remote(self.local_author, pendulum_now())
async def _sync_by_id(self,
entry_id: EntryID,
remote_changed: bool = True) -> RemoteManifest:
"""
Synchronize the entry corresponding to a specific ID.
This method keeps performing synchronization steps on the given ID until one of
those two conditions is met:
- there is no more changes to upload
- one upload operation has succeeded and has been acknowledged
This guarantees that any change prior to the call is saved remotely when this
method returns.
"""
# Get the current remote manifest if it has changed
remote_manifest = None
if remote_changed:
try:
remote_manifest = await self.remote_loader.load_manifest(
entry_id)
except FSRemoteManifestNotFound:
pass
# Loop over sync transactions
final = False
while True:
# Protect against race conditions on the entry id
try:
# Perform the sync step transaction
try:
new_remote_manifest = await self.transactions.synchronization_step(
entry_id, remote_manifest, final)
# The entry first requires reshaping
except FSReshapingRequiredError:
await self.transactions.file_reshape(entry_id)
continue
# The manifest doesn't exist locally
except FSLocalMissError:
raise FSNoSynchronizationRequired(entry_id)
# No new manifest to upload, the entry is synced!
if new_remote_manifest is None:
return remote_manifest or (
await self.local_storage.get_manifest(entry_id)).base
# Synchronize placeholder children
if is_folderish_manifest(new_remote_manifest):
await self._synchronize_placeholders(new_remote_manifest)
# Upload blocks
if is_file_manifest(new_remote_manifest):
await self._upload_blocks(new_remote_manifest)
# Restamp the remote manifest
new_remote_manifest = new_remote_manifest.evolve(
timestamp=pendulum_now())
# Upload the new manifest containing the latest changes
try:
await self.remote_loader.upload_manifest(
entry_id, new_remote_manifest)
# The upload has failed: download the latest remote manifest
except FSRemoteSyncError:
remote_manifest = await self.remote_loader.load_manifest(
entry_id)
# The upload has succeeded: loop one last time to acknowledge this new version
else:
final = True
remote_manifest = new_remote_manifest
def _copy(self, src: FsPath, dst: FsPath, delete_src: bool) -> None:
# The idea here is to consider a manifest never move around the fs
# (i.e. a given access always points to the same path). This simplify
# sync notifications handling and avoid ending up with two path
# (possibly from different workspaces !) pointing to the same manifest
# which would be weird for user experience.
# Long story short, when moving/copying manifest, we must recursively
# copy the manifests and create new accesses for them.
parent_src = src.parent
parent_dst = dst.parent
# No matter what, cannot move or overwrite root
if src.is_root():
# Raise FileNotFoundError if parent_dst doesn't exists
_, parent_dst_manifest = self._retrieve_entry(parent_dst)
if not is_folderish_manifest(parent_dst_manifest):
raise NotADirectoryError(20, "Not a directory",
str(parent_dst))
else:
raise PermissionError(13, "Permission denied", str(src),
str(dst))
elif dst.is_root():
# Raise FileNotFoundError if parent_src doesn't exists
_, parent_src_manifest = self._retrieve_entry(src.parent)
if not is_folderish_manifest(parent_src_manifest):
raise NotADirectoryError(20, "Not a directory",
str(src.parent))
else:
raise PermissionError(13, "Permission denied", str(src),
str(dst))
if src == dst:
# Raise FileNotFoundError if doesn't exist
src_access, src_ro_manifest = self._retrieve_entry_read_only(src)
if is_workspace_manifest(src_ro_manifest):
raise PermissionError(
13,
"Permission denied (cannot move/copy workpace, must rename it)",
str(src),
str(dst),
)
return
if parent_src == parent_dst:
parent_access, parent_manifest = self._retrieve_entry(parent_src)
if not is_folderish_manifest(parent_manifest):
raise NotADirectoryError(20, "Not a directory",
str(parent_src))
try:
dst.relative_to(src)
except ValueError:
pass
else:
raise OSError(22, "Invalid argument", str(src), None, str(dst))
try:
src_access = parent_manifest.children[src.name]
except KeyError:
raise FileNotFoundError(2, "No such file or directory",
str(src))
existing_dst_access = parent_manifest.children.get(dst.name)
src_manifest = self.get_manifest(src_access)
if is_workspace_manifest(src_manifest):
raise PermissionError(
13,
"Permission denied (cannot move/copy workpace, must rename it)",
str(src),
str(dst),
)
if existing_dst_access:
existing_dst_manifest = self.get_manifest(existing_dst_access)
if is_folderish_manifest(src_manifest):
if is_file_manifest(existing_dst_manifest):
raise NotADirectoryError(20, "Not a directory",
str(dst))
elif existing_dst_manifest.children:
raise OSError(39, "Directory not empty", str(dst))
else:
if is_folderish_manifest(existing_dst_manifest):
raise IsADirectoryError(21, "Is a directory", str(dst))
moved_access = self._recursive_manifest_copy(
src_access, src_manifest)
if not delete_src:
parent_manifest = parent_manifest.evolve_children(
{dst.name: moved_access})
else:
parent_manifest = parent_manifest.evolve_children({
dst.name:
moved_access,
src.name:
None
})
self.set_manifest(parent_access, parent_manifest)
self.event_bus.send("fs.entry.updated", id=parent_access.id)
else:
parent_src_access, parent_src_manifest = self._retrieve_entry(
parent_src)
if not is_folderish_manifest(parent_src_manifest):
raise NotADirectoryError(20, "Not a directory",
str(parent_src))
parent_dst_access, parent_dst_manifest = self._retrieve_entry(
parent_dst)
if not is_folderish_manifest(parent_dst_manifest):
raise NotADirectoryError(20, "Not a directory",
str(parent_dst))
try:
dst.relative_to(src)
except ValueError:
pass
else:
raise OSError(22, "Invalid argument", str(src), None, str(dst))
try:
src_access = parent_src_manifest.children[src.name]
except KeyError:
raise FileNotFoundError(2, "No such file or directory",
str(src))
existing_dst_access = parent_dst_manifest.children.get(dst.name)
src_manifest = self.get_manifest(src_access)
if is_workspace_manifest(src_manifest):
raise PermissionError(
13,
"Permission denied (cannot move/copy workpace, must rename it)",
str(src),
str(dst),
)
if existing_dst_access:
existing_entry_manifest = self.get_manifest(
existing_dst_access)
if is_folderish_manifest(src_manifest):
if is_file_manifest(existing_entry_manifest):
raise NotADirectoryError(20, "Not a directory",
str(dst))
elif existing_entry_manifest.children:
raise OSError(39, "Directory not empty", str(dst))
else:
if is_folderish_manifest(existing_entry_manifest):
raise IsADirectoryError(21, "Is a directory", str(dst))
moved_access = self._recursive_manifest_copy(
src_access, src_manifest)
parent_dst_manifest = parent_dst_manifest.evolve_children_and_mark_updated(
{dst.name: moved_access})
self.set_manifest(parent_dst_access, parent_dst_manifest)
self.event_bus.send("fs.entry.updated", id=parent_dst_access.id)
if delete_src:
parent_src_manifest = parent_src_manifest.evolve_children_and_mark_updated(
{src.name: None})
self.set_manifest(parent_src_access, parent_src_manifest)
self.event_bus.send("fs.entry.updated",
id=parent_src_access.id)
async def entry_rename(self,
source: FsPath,
destination: FsPath,
overwrite: bool = True) -> EntryID:
# Check write rights
self.check_write_rights(source)
# Source is root
if source.is_root():
raise FSPermissionError(filename=source)
# Destination is root
if destination.is_root():
raise FSPermissionError(filename=destination)
# Cross-directory renaming is not supported
if source.parent != destination.parent:
raise FSCrossDeviceError(filename=source, filename2=destination)
# Pre-fetch the source if necessary
if overwrite:
await self._get_manifest_from_path(source)
# Fetch and lock
async with self._lock_parent_manifest_from_path(destination) as (
parent, child):
# Source does not exist
if source.name not in parent.children:
raise FSFileNotFoundError(filename=source)
source_entry_id = parent.children[source.name]
# Source and destination are the same
if source.name == destination.name:
return
# Destination already exists
if not overwrite and child is not None:
raise FSFileExistsError(filename=destination)
# Overwrite logic
if overwrite and child is not None:
source_manifest = await self._get_manifest(source_entry_id)
# Overwrite a file
if is_file_manifest(source_manifest):
# Destination is a folder
if is_folder_manifest(child):
raise FSIsADirectoryError(filename=destination)
# Overwrite a folder
if is_folder_manifest(source_manifest):
# Destination is not a folder
if is_file_manifest(child):
raise FSNotADirectoryError(filename=destination)
# Destination is not empty
if child.children:
raise FSDirectoryNotEmptyError(filename=destination)
# Create new manifest
new_parent = parent.evolve_children_and_mark_updated({
destination.name:
source_entry_id,
source.name:
None
})
# Atomic change
await self.local_storage.set_manifest(parent.id, new_parent)
# Send event
self._send_event("fs.entry.updated", id=parent.id)
# Return the entry id of the renamed entry
return parent.children[source.name]
async def _sync_file_actual_sync(self, path: FsPath, access: Access,
manifest: LocalFileManifest) -> None:
assert is_file_manifest(manifest)
# to_sync_manifest = manifest.to_remote(version=manifest.base_version + 1)
to_sync_manifest = manifest.to_remote()
to_sync_manifest = to_sync_manifest.evolve(
version=manifest.base_version + 1)
# Compute the file's blocks and upload the new ones
blocks = []
sync_map = get_sync_map(manifest, self.block_size)
# Upload the new blocks
spaces = sync_map.spaces
blocks = []
async def _process_spaces():
nonlocal blocks
while spaces:
cs = spaces.pop()
data = await self._build_data_from_contiguous_space(cs)
if not data:
# Already existing blocks taken verbatim
blocks += [bs.buffer.data for bs in cs.buffers]
else:
# Create a new block from existing data
block_access = BlockAccess.from_block(data, cs.start)
await self._backend_block_create(block_access, data)
blocks.append(block_access)
if len(spaces) < 2:
await _process_spaces()
else:
async with trio.open_nursery() as nursery:
nursery.start_soon(_process_spaces)
nursery.start_soon(_process_spaces)
nursery.start_soon(_process_spaces)
nursery.start_soon(_process_spaces)
to_sync_manifest = to_sync_manifest.evolve(
blocks=blocks,
size=sync_map.size # TODO: useful ?
)
# Upload the file manifest as new vlob version
notify_beacons = self.local_folder_fs.get_beacon(path)
try:
if is_placeholder_manifest(manifest):
await self._backend_vlob_create(access, to_sync_manifest,
notify_beacons)
else:
await self._backend_vlob_update(access, to_sync_manifest,
notify_beacons)
except SyncConcurrencyError:
# Placeholder don't have remote version, so concurrency shouldn't
# be possible. However it's possible a previous attempt of
# uploading this manifest succeeded but we didn't receive the
# backend's answer, hence wrongly believing this is still a
# placeholder.
if is_placeholder_manifest(manifest):
logger.warning("Concurrency error while creating vlob",
access_id=access.id)
target_remote_manifest = await self._backend_vlob_read(access)
current_manifest = self.local_folder_fs.get_manifest(access)
# Do a fast-forward to avoid losing block we have uploaded
diverged_manifest = fast_forward_file(manifest, current_manifest,
to_sync_manifest)
self._sync_file_look_resolve_concurrency(path, access,
diverged_manifest,
target_remote_manifest)
# # TODO
# target_local_manifest = remote_to_local_manifest(target_remote_manifest)
# self._sync_file_look_resolve_concurrency(
# path, access, current_manifest, target_local_manifest
# )
# await self._backend_vlob_create(access, to_sync_manifest, notify_beacons)
else:
self._sync_file_merge_back(access, manifest, to_sync_manifest)
return to_sync_manifest
