async def file_delete(self, path: FsPath) -> EntryID:
# Check write rights
self.check_write_rights(path)
# Fetch and lock
async with self._lock_parent_manifest_from_path(path) as (parent,
child):
# Entry doesn't exist
if child is None:
raise FSFileNotFoundError(filename=path)
# Not a file
if not is_file_manifest(child):
raise FSIsADirectoryError(filename=path)
# Create new manifest
new_parent = parent.evolve_children_and_mark_updated(
{path.name: None},
pattern_filter=self.local_storage.get_pattern_filter())
# Atomic change
await self.local_storage.set_manifest(parent.id, new_parent)
# Send event
self._send_event(ClientEvent.FS_ENTRY_UPDATED, id=parent.id)
# Return the entry id of the deleted file
return child.id
def merge_manifests(
local_author: DeviceID,
pattern_filter: Pattern,
local_manifest: BaseLocalManifest,
remote_manifest: Optional[BaseRemoteManifest] = None,
force_filter: Optional[bool] = False,
):
# Start by re-applying filter (idempotent)
if is_folderish_manifest(local_manifest) and force_filter:
local_manifest = cast(LocalFolderishManifests,
local_manifest).apply_filter(pattern_filter)
# The remote hasn't changed
if remote_manifest is None or remote_manifest.version <= local_manifest.base_version:
return local_manifest
remote_manifest = cast(BaseRemoteManifest, remote_manifest)
# Exctract versions
assert remote_manifest is not None
remote_version = remote_manifest.version
local_version = local_manifest.base_version
local_from_remote = BaseLocalManifest.from_remote(remote_manifest,
pattern_filter,
local_manifest)
# Only the remote has changed
if not local_manifest.need_sync:
return local_from_remote
# Both the remote and the local have changed
assert remote_version > local_version and local_manifest.need_sync
# All the local changes have been successfully uploaded
if local_manifest.match_remote(remote_manifest):
return local_from_remote
# The remote changes are ours, simply acknowledge them and keep our local changes
if remote_manifest.author == local_author:
return local_manifest.evolve(base=remote_manifest)
# The remote has been updated by some other device
assert remote_manifest.author != local_author
# Cannot solve a file conflict directly
if is_file_manifest(local_manifest):
raise FSFileConflictError(local_manifest, remote_manifest)
# Solve the folder conflict
new_children = merge_folder_children(
cast(LocalFolderishManifests, local_manifest).base.children,
cast(LocalFolderishManifests, local_manifest).children,
cast(LocalFolderishManifests, local_from_remote).children,
remote_manifest.author,
)
# Mark as updated
return local_from_remote.evolve_and_mark_updated(children=new_children)
async def _populate_tree_load(
self,
path_level: int,
entry_id: EntryID,
early: DateTime,
late: DateTime,
version_number: int,
expected_timestamp: DateTime,
next_version_number: int,
parent: TaskNode,
):
if early > late:
return
manifest = await self.task_list.manifest_cache.load(
entry_id,
version=version_number,
expected_backend_timestamp=expected_timestamp)
data = ManifestDataAndMutablePaths(
ManifestData(
manifest.author,
manifest.updated,
is_folder_manifest(manifest),
None if not is_file_manifest(manifest) else manifest.size,
))
if len(self.target.parts) == path_level:
await data.populate_paths(self.task_list.manifest_cache, entry_id,
early, late)
self.return_dict[TimestampBoundedEntry(
manifest.id, manifest.version, early,
late)] = ManifestDataAndPaths(
data=data.manifest,
source=data.source_path
if data.source_path != data.current_path else None,
destination=data.destination_path
if data.destination_path != data.current_path else None,
)
else:
if not is_file_manifest(
manifest): # If it is a file, just ignores current path
for child_name, child_id in manifest.children.items():
if child_name == self.target.parts[path_level]:
return await self._populate_tree_list_versions(
path_level + 1, child_id, early, late, parent)
async def sync_by_id(self,
entry_id: EntryID,
remote_changed: bool = True,
recursive: bool = True):
"""
Raises:
FSError
"""
# Make sure the corresponding realm exists
await self._create_realm_if_needed()
# Sync parent first
try:
async with self.sync_locks[entry_id]:
manifest = await self._sync_by_id(
entry_id, remote_changed=remote_changed)
# Nothing to synchronize if the manifest does not exist locally
except FSNoSynchronizationRequired:
return
# A file conflict needs to be adressed first
except FSFileConflictError as exc:
local_manifest, remote_manifest = exc.args
# Only file manifest have synchronization conflict
assert is_file_manifest(local_manifest)
await self.transactions.file_conflict(entry_id, local_manifest,
remote_manifest)
return await self.sync_by_id(local_manifest.parent)
# Non-recursive
if not recursive or is_file_manifest(manifest):
return
# Synchronize children
for name, entry_id in cast(RemoteFolderishManifests,
manifest).children.items():
await self.sync_by_id(entry_id,
remote_changed=remote_changed,
recursive=True)
async def file_resize(self, path: FsPath, length: int) -> EntryID:
# Check write rights
self.check_write_rights(path)
# Lock manifest
async with self._lock_manifest_from_path(path) as manifest:
# Not a file
if not is_file_manifest(manifest):
raise FSIsADirectoryError(filename=path)
# Perform resize
await self._manifest_resize(manifest, length)
# Return entry id
return manifest.id
async def _recursive_apply_filter(self, entry_id: EntryID,
pattern_filter: Pattern):
# Load manifest
try:
manifest = await self.local_storage.get_manifest(entry_id)
# Not stored locally, nothing to do
except FSLocalMissError:
return
# A file manifest, nothing to do
if is_file_manifest(manifest):
return
# Apply filter (idempotent)
await self.transactions.apply_filter(entry_id, pattern_filter)
# Synchronize children
for name, child_entry_id in manifest.children.items():
await self._recursive_apply_filter(child_entry_id, pattern_filter)
async def file_open(self,
path: FsPath,
mode="rw") -> Tuple[EntryID, FileDescriptor]:
# Check read and write rights
if "w" in mode:
self.check_write_rights(path)
else:
self.check_read_rights(path)
# Lock path in read mode
async with self._lock_manifest_from_path(path) as manifest:
# Not a file
if not is_file_manifest(manifest):
raise FSIsADirectoryError(filename=path)
# Return the entry id of the open file and the file descriptor
return manifest.id, self.local_storage.create_file_descriptor(
manifest)
async def _entry_id_from_path(self, path: FsPath) -> Tuple[EntryID, bool]:
# Root entry_id and manifest
entry_id = self.workspace_id
confined = False
# Follow the path
for name in path.parts:
manifest = await self._load_manifest(entry_id)
if is_file_manifest(manifest):
raise FSNotADirectoryError(filename=path)
manifest = cast(LocalFolderishManifests, manifest)
try:
entry_id = manifest.children[name]
except (AttributeError, KeyError):
raise FSFileNotFoundError(filename=path)
if entry_id in manifest.confined_entries:
confined = True
# Return both entry_id and confined status
return entry_id, confined
async def synchronization_step(
self,
entry_id: EntryID,
remote_manifest: Optional[BaseRemoteManifest] = None,
final: bool = False,
) -> Optional[BaseRemoteManifest]:
"""Perform a synchronization step.
This step is meant to be called several times until the right state is reached.
It takes the current remote manifest as an argument and returns the new remote
manifest to upload. When the manifest is successfully uploaded, this method has
to be called once again with the new remote manifest as an argument. When there
is no more changes to upload, this method returns None. The `final` argument can
be set to true to indicate that the caller has no intention to upload a new
manifest. This also causes the method to return None.
"""
# Fetch and lock
async with self.local_storage.lock_manifest(
entry_id) as local_manifest:
# Go through the parent chain
current_manifest = local_manifest
while not is_workspace_manifest(current_manifest):
parent_manifest = await self.local_storage.get_manifest(
current_manifest.parent)
parent_manifest = cast(LocalFolderishManifests,
parent_manifest)
# The entry is not confined
if current_manifest.id not in parent_manifest.confined_entries:
current_manifest = parent_manifest
continue
# Send synced event
self._send_event(ClientEvent.FS_ENTRY_CONFINED,
entry_id=entry_id,
cause_id=parent_manifest.id)
return None
# Sync cannot be performed yet
if not final and is_file_manifest(
local_manifest) and not local_manifest.is_reshaped():
# Try a quick reshape (without downloading any block)
missing = await self._manifest_reshape(local_manifest)
# Downloading block is necessary for this reshape
if missing:
raise FSReshapingRequiredError(entry_id)
# The manifest should be reshaped by now
local_manifest = await self.local_storage.get_manifest(entry_id
)
assert local_manifest.is_reshaped()
# Merge manifests
pattern_filter = self.local_storage.get_pattern_filter()
force_filter = not self.local_storage.get_pattern_filter_fully_applied(
)
new_local_manifest = merge_manifests(self.local_author,
pattern_filter,
local_manifest,
remote_manifest, force_filter)
# Extract authors
base_author = local_manifest.base.author
remote_author = base_author if remote_manifest is None else remote_manifest.author
# Extract versions
base_version = local_manifest.base_version
new_base_version = new_local_manifest.base_version
# Set the new base manifest
if new_local_manifest != local_manifest:
await self.local_storage.set_manifest(entry_id,
new_local_manifest)
# Send downsynced event
if base_version != new_base_version and remote_author != self.local_author:
self._send_event(ClientEvent.FS_ENTRY_DOWNSYNCED, id=entry_id)
# Send synced event
if local_manifest.need_sync and not new_local_manifest.need_sync:
self._send_event(ClientEvent.FS_ENTRY_SYNCED, id=entry_id)
# Nothing new to upload
if final or not new_local_manifest.need_sync:
return None
# Produce the new remote manifest to upload
return new_local_manifest.to_remote(self.local_author,
pendulum_now())
async def _sync_by_id(self,
entry_id: EntryID,
remote_changed: bool = True) -> BaseRemoteManifest:
"""
Synchronize the entry corresponding to a specific ID.
This method keeps performing synchronization steps on the given ID until one of
those two conditions is met:
- there is no more changes to upload
- one upload operation has succeeded and has been acknowledged
This guarantees that any change prior to the call is saved remotely when this
method returns.
"""
# Get the current remote manifest if it has changed
remote_manifest = None
if remote_changed:
try:
remote_manifest = await self.remote_loader.load_manifest(
entry_id)
except FSRemoteManifestNotFound:
pass
# Loop over sync transactions
final = False
while True:
# Protect against race conditions on the entry id
try:
# Perform the sync step transaction
try:
new_remote_manifest = await self.transactions.synchronization_step(
entry_id, remote_manifest, final)
# The entry first requires reshaping
except FSReshapingRequiredError:
await self.transactions.file_reshape(entry_id)
continue
# The manifest doesn't exist locally
except FSLocalMissError:
raise FSNoSynchronizationRequired(entry_id)
# No new manifest to upload, the entry is synced!
if new_remote_manifest is None:
return remote_manifest or (
await self.local_storage.get_manifest(entry_id)).base
# Synchronize placeholder children
if is_folderish_manifest(new_remote_manifest):
await self._synchronize_placeholders(
cast(RemoteFolderishManifests, new_remote_manifest))
# Upload blocks
if is_file_manifest(new_remote_manifest):
await self._upload_blocks(
cast(RemoteFileManifest, new_remote_manifest))
# Restamp the remote manifest
new_remote_manifest = new_remote_manifest.evolve(
timestamp=pendulum_now())
# Upload the new manifest containing the latest changes
try:
await self.remote_loader.upload_manifest(
entry_id, new_remote_manifest)
# The upload has failed: download the latest remote manifest
except FSRemoteSyncError:
remote_manifest = await self.remote_loader.load_manifest(
entry_id)
# The upload has succeeded: loop one last time to acknowledge this new version
else:
final = True
remote_manifest = new_remote_manifest
async def entry_rename(self,
source: FsPath,
destination: FsPath,
overwrite: bool = True) -> Optional[EntryID]:
# Check write rights
self.check_write_rights(source)
# Source is root
if source.is_root():
raise FSPermissionError(filename=source)
# Destination is root
if destination.is_root():
raise FSPermissionError(filename=destination)
# Cross-directory renaming is not supported
if source.parent != destination.parent:
raise FSCrossDeviceError(filename=source, filename2=destination)
# Pre-fetch the source if necessary
if overwrite:
await self._get_manifest_from_path(source)
# Fetch and lock
async with self._lock_parent_manifest_from_path(destination) as (
parent, child):
# Source does not exist
if source.name not in parent.children:
raise FSFileNotFoundError(filename=source)
source_entry_id = parent.children[source.name]
# Source and destination are the same
if source.name == destination.name:
return None
# Destination already exists
if not overwrite and child is not None:
raise FSFileExistsError(filename=destination)
# Overwrite logic
if overwrite and child is not None:
source_manifest = await self._get_manifest(source_entry_id)
# Overwrite a file
if is_file_manifest(source_manifest):
# Destination is a folder
if is_folder_manifest(child):
raise FSIsADirectoryError(filename=destination)
# Overwrite a folder
if is_folder_manifest(source_manifest):
# Destination is not a folder
if is_file_manifest(child):
raise FSNotADirectoryError(filename=destination)
# Destination is not empty
if child.children:
raise FSDirectoryNotEmptyError(filename=destination)
# Create new manifest
new_parent = parent.evolve_children_and_mark_updated(
{
destination.name: source_entry_id,
source.name: None
},
pattern_filter=self.local_storage.get_pattern_filter(),
)
# Atomic change
await self.local_storage.set_manifest(parent.id, new_parent)
# Send event
self._send_event(ClientEvent.FS_ENTRY_UPDATED, id=parent.id)
# Return the entry id of the renamed entry
return parent.children[source.name]