def _sync_file_merge_back(self, access: Access, base_manifest: LocalFileManifest, target_remote_manifest: FileManifest) -> None: # Merge with the current version of the manifest which may have # been modified in the meantime assert is_file_manifest(target_remote_manifest) current_manifest = self.local_folder_fs.get_manifest(access) assert is_file_manifest(current_manifest) final_manifest = fast_forward_file(base_manifest, current_manifest, target_remote_manifest) self.local_folder_fs.set_manifest(access, final_manifest)
async def _sync_nolock(self, path: FsPath, recursive: bool) -> None: # First retrieve a snapshot of the manifest to sync try: access, manifest = self.local_folder_fs.get_entry(path) except FSManifestLocalMiss: # Nothing to do if entry is no present locally return if not manifest.need_sync and not recursive: return # In case of placeholder, we must resolve it first (and make sure # none of it parents are placeholders themselves) if manifest.is_placeholder: need_more_sync = await self._resolve_placeholders_in_path( path, access, manifest) # If the entry to sync is actually empty the minimal sync was enough if not need_more_sync and not recursive: return # Reload the manifest given the minimal sync has changed it try: manifest = self.local_folder_fs.get_manifest(access) except FSManifestLocalMiss: # Nothing to do if entry is no present locally return # Note from now on it's possible the entry has changed due to # concurrent access (or even have been totally removed !). # We will deal with this when merged synced data back into the local fs. # Now we can do the sync on the entry if is_file_manifest(manifest): await self._sync_file(path, access, manifest) else: await self._sync_folder(path, access, manifest, recursive)
def flush(self, fd: FileDescriptor) -> None: cursor = self._get_cursor_from_fd(fd) manifest = self.local_folder_fs.get_manifest(cursor.access) assert is_file_manifest(manifest) hf = self._get_hot_file(cursor.access) if manifest.size == hf.size and not hf.pending_writes: return new_dirty_blocks = [] for pw in hf.pending_writes: block_access = BlockAccess.from_block(pw.data, pw.start) self.set_block(block_access, pw.data, False) new_dirty_blocks.append(block_access) # TODO: clean overwritten dirty blocks manifest = manifest.evolve_and_mark_updated( dirty_blocks=(*manifest.dirty_blocks, *new_dirty_blocks), size=hf.size ) self.local_folder_fs.set_manifest(cursor.access, manifest) hf.pending_writes.clear() self.event_bus.send("fs.entry.updated", id=cursor.access.id)
async def file_delete(self, path: FsPath) -> EntryID: # Check write rights self.check_write_rights(path) # Fetch and lock async with self._lock_parent_manifest_from_path(path) as (parent, child): # Entry doesn't exist if child is None: raise FSFileNotFoundError(filename=path) # Not a file if not is_file_manifest(child): raise FSIsADirectoryError(filename=path) # Create new manifest new_parent = parent.evolve_children_and_mark_updated( {path.name: None}) # Atomic change await self.local_storage.set_manifest(parent.id, new_parent) # Send event self._send_event("fs.entry.updated", id=parent.id) # Return the entry id of the deleted file return child.id
async def entry_info(self, path: FsPath) -> dict: # Check read rights self.check_read_rights(path) # Fetch data manifest = await self._get_manifest_from_path(path) # General stats stats = { "id": manifest.id, "created": manifest.created, "updated": manifest.updated, "base_version": manifest.base_version, "is_placeholder": manifest.is_placeholder, "need_sync": manifest.need_sync, } # File/folder specific stats if is_file_manifest(manifest): stats["type"] = "file" stats["size"] = manifest.size else: stats["type"] = "folder" stats["children"] = sorted(manifest.children.keys()) return stats
def read(self, fd: FileDescriptor, size: int = inf, offset: int = None) -> bytes: cursor = self._get_cursor_from_fd(fd) if offset is not None: self._seek(cursor, offset) hf = self._get_hot_file(cursor.access) if cursor.offset > hf.size: return b"" manifest = self.local_folder_fs.get_manifest(cursor.access) assert is_file_manifest(manifest) start = cursor.offset end = cursor.offset + size if end > hf.size: end = hf.size blocks = self._get_quickly_filtered_blocks(manifest, start, end) blocks += hf.pending_writes merged = merge_buffers_with_limits(blocks, start, end) assert merged.size <= size assert merged.start == start missing = [] data = bytearray(end - start) for cs in merged.spaces: for bs in cs.buffers: if isinstance(bs.buffer, DirtyBlockBuffer): access = bs.buffer.access try: buff = self.get_block(access) except LocalDBMissingEntry as exc: raise RuntimeError(f"Unknown local block `{access['id']}`") from exc data[bs.start - cs.start : bs.end - cs.start] = buff[ bs.buffer_slice_start : bs.buffer_slice_end ] elif isinstance(bs.buffer, BlockBuffer): access = bs.buffer.access try: buff = self.get_block(access) except LocalDBMissingEntry: missing.append(access) continue data[bs.start - cs.start : bs.end - cs.start] = buff[ bs.buffer_slice_start : bs.buffer_slice_end ] else: data[bs.start - start : bs.end - start] = bs.get_data() if missing: raise FSBlocksLocalMiss(missing) cursor.offset += len(data) return data
def _recursive_process_copy_map(copy_map): manifest = copy_map["manifest"] cpy_access = ManifestAccess() if is_file_manifest(manifest): cpy_manifest = LocalFileManifest( author=self.local_author, size=manifest.size, blocks=manifest.blocks, dirty_blocks=manifest.dirty_blocks, ) else: cpy_children = {} for child_name in manifest.children.keys(): child_copy_map = copy_map["children"][child_name] new_child_access = _recursive_process_copy_map( child_copy_map) cpy_children[child_name] = new_child_access if is_folder_manifest(manifest): cpy_manifest = LocalFolderManifest( author=self.local_author, children=cpy_children) else: assert is_workspace_manifest(manifest) cpy_manifest = LocalWorkspaceManifest( self.local_author, children=cpy_children) self.set_manifest(cpy_access, cpy_manifest) return cpy_access
async def _populate_tree_load( self, path_level: int, entry_id: EntryID, early: Pendulum, late: Pendulum, version_number: int, expected_timestamp: Pendulum, next_version_number: int, ): if early > late: return manifest = await self.task_list.manifest_cache.load( entry_id, version=version_number, expected_backend_timestamp=expected_timestamp) data = ManifestDataAndMutablePaths( ManifestData( manifest.author, manifest.updated, is_folder_manifest(manifest), None if not is_file_manifest(manifest) else manifest.size, )) if len(self.target.parts) == path_level: await data.populate_paths(self.task_list.manifest_cache, entry_id, early, late) self.return_dict[TimestampBoundedEntry( manifest.id, manifest.version, early, late)] = ManifestDataAndPaths( data=data.manifest, source=data.source_path if data.source_path != data.current_path else None, destination=data.destination_path if data.destination_path != data.current_path else None, ) else: if not is_file_manifest( manifest): # If it is a file, just ignores current path for child_name, child_id in manifest.children.items(): if child_name == self.target.parts[path_level]: return await self._populate_tree_list_versions( path_level + 1, child_id, early, late)
async def sync_by_id(self, entry_id: EntryID, remote_changed: bool = True, recursive: bool = True): """ Raises: FSError """ # Make sure the corresponding realm exists await self._create_realm_if_needed() # Sync parent first try: async with self.sync_locks[entry_id]: manifest = await self._sync_by_id( entry_id, remote_changed=remote_changed) # Nothing to synchronize if the manifest does not exist locally except FSNoSynchronizationRequired: return # A file conflict needs to be adressed first except FSFileConflictError as exc: local_manifest, remote_manifest = exc.args # Only file manifest have synchronization conflict assert is_file_manifest(local_manifest) await self.transactions.file_conflict(entry_id, local_manifest, remote_manifest) return await self.sync_by_id(local_manifest.parent) # Non-recursive if not recursive or is_file_manifest(manifest): return # Synchronize children for name, entry_id in manifest.children.items(): await self.sync_by_id(entry_id, remote_changed=remote_changed, recursive=True)
def open(self, access: Access) -> FileDescriptor: cursor = FileCursor(access) # Sanity check manifest = self.local_folder_fs.get_manifest(access) if not is_file_manifest(manifest): raise IsADirectoryError(21, "Is a directory") hf = self._ensure_hot_file(access, manifest) hf.cursors.add(id(cursor)) fd = self._next_fd self._opened_cursors[fd] = cursor self._next_fd += 1 return fd
async def _resolve_placeholders_in_path(self, path: FsPath, access: Access, manifest: LocalManifest) -> bool: """ Returns: If an additional sync is needed """ # Notes we sync recursively from children to parents, this is more # efficient given otherwise we would have to do: # 1) sync the parent # 2) sync the child # 3) re-sync the parent with the child is_placeholder = is_placeholder_manifest(manifest) if not is_placeholder: # Cannot have a non-placeholder with a placeholder parent, hence # we don't have to go any further. return manifest.need_sync else: if is_file_manifest(manifest): need_more_sync = await self._minimal_sync_file( path, access, manifest) else: need_more_sync = await self._minimal_sync_folder( path, access, manifest) # Once the entry is synced, we must sync it parent as well to have # the entry visible for other clients if not path.is_root(): try: parent_access, parent_manifest = self.local_folder_fs.get_entry( path.parent) except FSManifestLocalMiss: # Nothing to do if entry is no present locally return False if is_placeholder_manifest(parent_manifest): await self._resolve_placeholders_in_path( path.parent, parent_access, parent_manifest) else: await self._sync_folder(path.parent, parent_access, parent_manifest, recursive=False) if not need_more_sync: self.event_bus.send("fs.entry.synced", path=str(path), id=access.id) return need_more_sync
async def file_resize(self, path: FsPath, length: int) -> EntryID: # Check write rights self.check_write_rights(path) # Lock manifest async with self._lock_manifest_from_path(path) as manifest: # Not a file if not is_file_manifest(manifest): raise FSIsADirectoryError(filename=path) # Perform resize await self._manifest_resize(manifest, length) # Return entry id return manifest.id
async def _lock_manifest_from_path(self, path: FsPath) -> LocalManifest: # Root entry_id and manifest entry_id = self.workspace_id # Follow the path for name in path.parts: manifest = await self._load_manifest(entry_id) if is_file_manifest(manifest): raise FSNotADirectoryError(filename=path) try: entry_id = manifest.children[name] except (AttributeError, KeyError): raise FSFileNotFoundError(filename=path) # Lock entry async with self._load_and_lock_manifest(entry_id) as manifest: yield manifest
def merge_manifests( local_author: DeviceID, local_manifest: LocalManifest, remote_manifest: Optional[RemoteManifest] = None, ): # Exctract versions local_version = local_manifest.base_version remote_version = local_version if remote_manifest is None else remote_manifest.version # The remote hasn't changed if remote_version <= local_version: return local_manifest # Only the remote has changed if not local_manifest.need_sync: return LocalManifest.from_remote(remote_manifest) # Both the remote and the local have changed assert remote_version > local_version and local_manifest.need_sync # All the local changes have been successfully uploaded if local_manifest.match_remote(remote_manifest): return LocalManifest.from_remote(remote_manifest) # The remote changes are ours, simply acknowledge them and keep our local changes if remote_manifest.author == local_author: return local_manifest.evolve(base=remote_manifest) # The remote has been updated by some other device assert remote_manifest.author != local_author # Cannot solve a file conflict directly if is_file_manifest(local_manifest): raise FSFileConflictError(local_manifest, remote_manifest) # Solve the folder conflict new_children = merge_folder_children( local_manifest.base.children, local_manifest.children, remote_manifest.children, remote_manifest.author, ) return local_manifest.evolve_and_mark_updated(base=remote_manifest, children=new_children)
async def file_open(self, path: FsPath, mode="rw") -> Tuple[EntryID, FileDescriptor]: # Check read and write rights if "w" in mode: self.check_write_rights(path) else: self.check_read_rights(path) # Lock path in read mode async with self._lock_manifest_from_path(path) as manifest: # Not a file if not is_file_manifest(manifest): raise FSIsADirectoryError(filename=path) # Return the entry id of the open file and the file descriptor return manifest.id, self.local_storage.create_file_descriptor( manifest)
async def _sync_file(self, path: FsPath, access: Access, manifest: LocalFileManifest) -> None: """ Raises: FileSyncConcurrencyError BackendNotAvailable """ assert not is_placeholder_manifest(manifest) assert is_file_manifest(manifest) # Now we can synchronize the folder if needed if not manifest.need_sync: changed = await self._sync_file_look_for_remote_changes( path, access, manifest) else: await self._sync_file_actual_sync(path, access, manifest) changed = True if changed: self.event_bus.send("fs.entry.synced", path=str(path), id=access.id)
def stat(self, path: FsPath) -> dict: access, manifest = self._retrieve_entry_read_only(path) if is_file_manifest(manifest): return { "type": "file", "is_folder": False, "created": manifest.created, "updated": manifest.updated, "base_version": manifest.base_version, "is_placeholder": manifest.is_placeholder, "need_sync": manifest.need_sync, "size": manifest.size, } elif is_workspace_manifest(manifest): return { "type": "workspace", "is_folder": True, "created": manifest.created, "updated": manifest.updated, "base_version": manifest.base_version, "is_placeholder": manifest.is_placeholder, "need_sync": manifest.need_sync, "children": list(sorted(manifest.children.keys())), "creator": manifest.creator, "participants": list(manifest.participants), } else: return { "type": "root" if path.is_root() else "folder", "is_folder": True, "created": manifest.created, "updated": manifest.updated, "base_version": manifest.base_version, "is_placeholder": manifest.is_placeholder, "need_sync": manifest.need_sync, "children": list(sorted(manifest.children.keys())), }
async def synchronization_step( self, entry_id: EntryID, remote_manifest: Optional[RemoteManifest] = None, final: bool = False, ) -> Optional[RemoteManifest]: """Perform a synchronization step. This step is meant to be called several times until the right state is reached. It takes the current remote manifest as an argument and returns the new remote manifest to upload. When the manifest is successfully uploaded, this method has to be called once again with the new remote manifest as an argument. When there is no more changes to upload, this method returns None. The `final` argument can be set to true to indicate that the caller has no intention to upload a new manifest. This also causes the method to return None. """ # Fetch and lock async with self.local_storage.lock_manifest(entry_id) as local_manifest: # Sync cannot be performed yet if not final and is_file_manifest(local_manifest) and not local_manifest.is_reshaped(): # Try a quick reshape (without downloading any block) missing = await self._manifest_reshape(local_manifest) # Downloading block is necessary for this reshape if missing: raise FSReshapingRequiredError(entry_id) # The manifest should be reshaped by now local_manifest = await self.local_storage.get_manifest(entry_id) assert local_manifest.is_reshaped() # Merge manifests new_local_manifest = merge_manifests(self.local_author, local_manifest, remote_manifest) # Extract authors base_author = local_manifest.base.author remote_author = base_author if remote_manifest is None else remote_manifest.author # Extract versions base_version = local_manifest.base_version new_base_version = new_local_manifest.base_version remote_version = base_version if remote_manifest is None else remote_manifest.version # Set the new base manifest if base_version != remote_version or new_local_manifest.need_sync: await self.local_storage.set_manifest(entry_id, new_local_manifest) # Send downsynced event if base_version != new_base_version and remote_author != self.local_author: self._send_event("fs.entry.downsynced", id=entry_id) # Send synced event if local_manifest.need_sync and not new_local_manifest.need_sync: self._send_event("fs.entry.synced", id=entry_id) # Nothing new to upload if final or not new_local_manifest.need_sync: return None # Produce the new remote manifest to upload return new_local_manifest.to_remote(self.local_author, pendulum_now())
async def _sync_by_id(self, entry_id: EntryID, remote_changed: bool = True) -> RemoteManifest: """ Synchronize the entry corresponding to a specific ID. This method keeps performing synchronization steps on the given ID until one of those two conditions is met: - there is no more changes to upload - one upload operation has succeeded and has been acknowledged This guarantees that any change prior to the call is saved remotely when this method returns. """ # Get the current remote manifest if it has changed remote_manifest = None if remote_changed: try: remote_manifest = await self.remote_loader.load_manifest( entry_id) except FSRemoteManifestNotFound: pass # Loop over sync transactions final = False while True: # Protect against race conditions on the entry id try: # Perform the sync step transaction try: new_remote_manifest = await self.transactions.synchronization_step( entry_id, remote_manifest, final) # The entry first requires reshaping except FSReshapingRequiredError: await self.transactions.file_reshape(entry_id) continue # The manifest doesn't exist locally except FSLocalMissError: raise FSNoSynchronizationRequired(entry_id) # No new manifest to upload, the entry is synced! if new_remote_manifest is None: return remote_manifest or ( await self.local_storage.get_manifest(entry_id)).base # Synchronize placeholder children if is_folderish_manifest(new_remote_manifest): await self._synchronize_placeholders(new_remote_manifest) # Upload blocks if is_file_manifest(new_remote_manifest): await self._upload_blocks(new_remote_manifest) # Restamp the remote manifest new_remote_manifest = new_remote_manifest.evolve( timestamp=pendulum_now()) # Upload the new manifest containing the latest changes try: await self.remote_loader.upload_manifest( entry_id, new_remote_manifest) # The upload has failed: download the latest remote manifest except FSRemoteSyncError: remote_manifest = await self.remote_loader.load_manifest( entry_id) # The upload has succeeded: loop one last time to acknowledge this new version else: final = True remote_manifest = new_remote_manifest
def _copy(self, src: FsPath, dst: FsPath, delete_src: bool) -> None: # The idea here is to consider a manifest never move around the fs # (i.e. a given access always points to the same path). This simplify # sync notifications handling and avoid ending up with two path # (possibly from different workspaces !) pointing to the same manifest # which would be weird for user experience. # Long story short, when moving/copying manifest, we must recursively # copy the manifests and create new accesses for them. parent_src = src.parent parent_dst = dst.parent # No matter what, cannot move or overwrite root if src.is_root(): # Raise FileNotFoundError if parent_dst doesn't exists _, parent_dst_manifest = self._retrieve_entry(parent_dst) if not is_folderish_manifest(parent_dst_manifest): raise NotADirectoryError(20, "Not a directory", str(parent_dst)) else: raise PermissionError(13, "Permission denied", str(src), str(dst)) elif dst.is_root(): # Raise FileNotFoundError if parent_src doesn't exists _, parent_src_manifest = self._retrieve_entry(src.parent) if not is_folderish_manifest(parent_src_manifest): raise NotADirectoryError(20, "Not a directory", str(src.parent)) else: raise PermissionError(13, "Permission denied", str(src), str(dst)) if src == dst: # Raise FileNotFoundError if doesn't exist src_access, src_ro_manifest = self._retrieve_entry_read_only(src) if is_workspace_manifest(src_ro_manifest): raise PermissionError( 13, "Permission denied (cannot move/copy workpace, must rename it)", str(src), str(dst), ) return if parent_src == parent_dst: parent_access, parent_manifest = self._retrieve_entry(parent_src) if not is_folderish_manifest(parent_manifest): raise NotADirectoryError(20, "Not a directory", str(parent_src)) try: dst.relative_to(src) except ValueError: pass else: raise OSError(22, "Invalid argument", str(src), None, str(dst)) try: src_access = parent_manifest.children[src.name] except KeyError: raise FileNotFoundError(2, "No such file or directory", str(src)) existing_dst_access = parent_manifest.children.get(dst.name) src_manifest = self.get_manifest(src_access) if is_workspace_manifest(src_manifest): raise PermissionError( 13, "Permission denied (cannot move/copy workpace, must rename it)", str(src), str(dst), ) if existing_dst_access: existing_dst_manifest = self.get_manifest(existing_dst_access) if is_folderish_manifest(src_manifest): if is_file_manifest(existing_dst_manifest): raise NotADirectoryError(20, "Not a directory", str(dst)) elif existing_dst_manifest.children: raise OSError(39, "Directory not empty", str(dst)) else: if is_folderish_manifest(existing_dst_manifest): raise IsADirectoryError(21, "Is a directory", str(dst)) moved_access = self._recursive_manifest_copy( src_access, src_manifest) if not delete_src: parent_manifest = parent_manifest.evolve_children( {dst.name: moved_access}) else: parent_manifest = parent_manifest.evolve_children({ dst.name: moved_access, src.name: None }) self.set_manifest(parent_access, parent_manifest) self.event_bus.send("fs.entry.updated", id=parent_access.id) else: parent_src_access, parent_src_manifest = self._retrieve_entry( parent_src) if not is_folderish_manifest(parent_src_manifest): raise NotADirectoryError(20, "Not a directory", str(parent_src)) parent_dst_access, parent_dst_manifest = self._retrieve_entry( parent_dst) if not is_folderish_manifest(parent_dst_manifest): raise NotADirectoryError(20, "Not a directory", str(parent_dst)) try: dst.relative_to(src) except ValueError: pass else: raise OSError(22, "Invalid argument", str(src), None, str(dst)) try: src_access = parent_src_manifest.children[src.name] except KeyError: raise FileNotFoundError(2, "No such file or directory", str(src)) existing_dst_access = parent_dst_manifest.children.get(dst.name) src_manifest = self.get_manifest(src_access) if is_workspace_manifest(src_manifest): raise PermissionError( 13, "Permission denied (cannot move/copy workpace, must rename it)", str(src), str(dst), ) if existing_dst_access: existing_entry_manifest = self.get_manifest( existing_dst_access) if is_folderish_manifest(src_manifest): if is_file_manifest(existing_entry_manifest): raise NotADirectoryError(20, "Not a directory", str(dst)) elif existing_entry_manifest.children: raise OSError(39, "Directory not empty", str(dst)) else: if is_folderish_manifest(existing_entry_manifest): raise IsADirectoryError(21, "Is a directory", str(dst)) moved_access = self._recursive_manifest_copy( src_access, src_manifest) parent_dst_manifest = parent_dst_manifest.evolve_children_and_mark_updated( {dst.name: moved_access}) self.set_manifest(parent_dst_access, parent_dst_manifest) self.event_bus.send("fs.entry.updated", id=parent_dst_access.id) if delete_src: parent_src_manifest = parent_src_manifest.evolve_children_and_mark_updated( {src.name: None}) self.set_manifest(parent_src_access, parent_src_manifest) self.event_bus.send("fs.entry.updated", id=parent_src_access.id)
async def entry_rename(self, source: FsPath, destination: FsPath, overwrite: bool = True) -> EntryID: # Check write rights self.check_write_rights(source) # Source is root if source.is_root(): raise FSPermissionError(filename=source) # Destination is root if destination.is_root(): raise FSPermissionError(filename=destination) # Cross-directory renaming is not supported if source.parent != destination.parent: raise FSCrossDeviceError(filename=source, filename2=destination) # Pre-fetch the source if necessary if overwrite: await self._get_manifest_from_path(source) # Fetch and lock async with self._lock_parent_manifest_from_path(destination) as ( parent, child): # Source does not exist if source.name not in parent.children: raise FSFileNotFoundError(filename=source) source_entry_id = parent.children[source.name] # Source and destination are the same if source.name == destination.name: return # Destination already exists if not overwrite and child is not None: raise FSFileExistsError(filename=destination) # Overwrite logic if overwrite and child is not None: source_manifest = await self._get_manifest(source_entry_id) # Overwrite a file if is_file_manifest(source_manifest): # Destination is a folder if is_folder_manifest(child): raise FSIsADirectoryError(filename=destination) # Overwrite a folder if is_folder_manifest(source_manifest): # Destination is not a folder if is_file_manifest(child): raise FSNotADirectoryError(filename=destination) # Destination is not empty if child.children: raise FSDirectoryNotEmptyError(filename=destination) # Create new manifest new_parent = parent.evolve_children_and_mark_updated({ destination.name: source_entry_id, source.name: None }) # Atomic change await self.local_storage.set_manifest(parent.id, new_parent) # Send event self._send_event("fs.entry.updated", id=parent.id) # Return the entry id of the renamed entry return parent.children[source.name]
async def _sync_file_actual_sync(self, path: FsPath, access: Access, manifest: LocalFileManifest) -> None: assert is_file_manifest(manifest) # to_sync_manifest = manifest.to_remote(version=manifest.base_version + 1) to_sync_manifest = manifest.to_remote() to_sync_manifest = to_sync_manifest.evolve( version=manifest.base_version + 1) # Compute the file's blocks and upload the new ones blocks = [] sync_map = get_sync_map(manifest, self.block_size) # Upload the new blocks spaces = sync_map.spaces blocks = [] async def _process_spaces(): nonlocal blocks while spaces: cs = spaces.pop() data = await self._build_data_from_contiguous_space(cs) if not data: # Already existing blocks taken verbatim blocks += [bs.buffer.data for bs in cs.buffers] else: # Create a new block from existing data block_access = BlockAccess.from_block(data, cs.start) await self._backend_block_create(block_access, data) blocks.append(block_access) if len(spaces) < 2: await _process_spaces() else: async with trio.open_nursery() as nursery: nursery.start_soon(_process_spaces) nursery.start_soon(_process_spaces) nursery.start_soon(_process_spaces) nursery.start_soon(_process_spaces) to_sync_manifest = to_sync_manifest.evolve( blocks=blocks, size=sync_map.size # TODO: useful ? ) # Upload the file manifest as new vlob version notify_beacons = self.local_folder_fs.get_beacon(path) try: if is_placeholder_manifest(manifest): await self._backend_vlob_create(access, to_sync_manifest, notify_beacons) else: await self._backend_vlob_update(access, to_sync_manifest, notify_beacons) except SyncConcurrencyError: # Placeholder don't have remote version, so concurrency shouldn't # be possible. However it's possible a previous attempt of # uploading this manifest succeeded but we didn't receive the # backend's answer, hence wrongly believing this is still a # placeholder. if is_placeholder_manifest(manifest): logger.warning("Concurrency error while creating vlob", access_id=access.id) target_remote_manifest = await self._backend_vlob_read(access) current_manifest = self.local_folder_fs.get_manifest(access) # Do a fast-forward to avoid losing block we have uploaded diverged_manifest = fast_forward_file(manifest, current_manifest, to_sync_manifest) self._sync_file_look_resolve_concurrency(path, access, diverged_manifest, target_remote_manifest) # # TODO # target_local_manifest = remote_to_local_manifest(target_remote_manifest) # self._sync_file_look_resolve_concurrency( # path, access, current_manifest, target_local_manifest # ) # await self._backend_vlob_create(access, to_sync_manifest, notify_beacons) else: self._sync_file_merge_back(access, manifest, to_sync_manifest) return to_sync_manifest