async def _resolve_placeholders_in_path(self, path: FsPath, access: Access,
manifest: LocalManifest) -> bool:
"""
Returns: If an additional sync is needed
"""
# Notes we sync recursively from children to parents, this is more
# efficient given otherwise we would have to do:
# 1) sync the parent
# 2) sync the child
# 3) re-sync the parent with the child
is_placeholder = is_placeholder_manifest(manifest)
if not is_placeholder:
# Cannot have a non-placeholder with a placeholder parent, hence
# we don't have to go any further.
return manifest.need_sync
else:
if is_file_manifest(manifest):
need_more_sync = await self._minimal_sync_file(
path, access, manifest)
else:
need_more_sync = await self._minimal_sync_folder(
path, access, manifest)
# Once the entry is synced, we must sync it parent as well to have
# the entry visible for other clients
if not path.is_root():
try:
parent_access, parent_manifest = self.local_folder_fs.get_entry(
path.parent)
except FSManifestLocalMiss:
# Nothing to do if entry is no present locally
return False
if is_placeholder_manifest(parent_manifest):
await self._resolve_placeholders_in_path(
path.parent, parent_access, parent_manifest)
else:
await self._sync_folder(path.parent,
parent_access,
parent_manifest,
recursive=False)
if not need_more_sync:
self.event_bus.send("fs.entry.synced",
path=str(path),
id=access.id)
return need_more_sync
async def _minimal_sync_file(self, path: FsPath, access: Access,
manifest: LocalFileManifest) -> bool:
"""
Returns: If additional sync are needed
Raises:
FileSyncConcurrencyError
BackendNotAvailable
"""
if not is_placeholder_manifest(manifest):
return manifest.need_sync
need_more_sync = bool(manifest.dirty_blocks)
# Don't sync the dirty blocks for fast synchronization
try:
last_block = manifest.blocks[-1]
size = last_block.offset + last_block.size
except IndexError:
size = 0
minimal_manifest = manifest.evolve(
updated=manifest.created if need_more_sync else manifest.updated,
size=size,
blocks=manifest.blocks,
dirty_blocks=(),
)
await self._sync_file_actual_sync(path, access, minimal_manifest)
self.event_bus.send("fs.entry.minimal_synced",
path=str(path),
id=access.id)
return need_more_sync
async def _sync_file_look_for_remote_changes(
self, path: FsPath, access: Access,
manifest: LocalFileManifest) -> bool:
# Placeholder means we need synchro !
assert not is_placeholder_manifest(manifest)
# This folder hasn't been modified locally, just download
# last version from the backend if any.
target_remote_manifest = await self._backend_vlob_read(access)
current_manifest = self.local_folder_fs.get_manifest(access)
if target_remote_manifest.version == current_manifest.base_version:
return False
# Remote version has changed...
if current_manifest.need_sync:
# ...and modifications occured on our back, now we have a concurrency error !
self._sync_file_look_resolve_concurrency(path, access,
current_manifest,
target_remote_manifest)
else:
target_local_manifest = target_remote_manifest.to_local()
# Otherwise just fast-forward the local data
self.local_folder_fs.set_manifest(access, target_local_manifest)
return True
async def _sync_folder_look_for_remote_changes(
self, access: Access,
manifest: LocalFolderManifest) -> Optional[FolderManifest]:
# Placeholder means we need synchro !
assert not is_placeholder_manifest(manifest)
# This folder hasn't been modified locally, just download
# last version from the backend if any.
target_remote_manifest = await self._backend_vlob_read(access)
if target_remote_manifest.version == manifest.base_version:
return None
return target_remote_manifest
def _strip_placeholders(self, children: Dict[Access, LocalManifest]):
synced_children = {}
for child_name, child_access in children.items():
try:
child_manifest = self.local_folder_fs.get_manifest(
child_access)
except FSManifestLocalMiss:
# Child not in local, cannot be a placeholder then !
synced_children[child_name] = child_access
else:
if not is_placeholder_manifest(child_manifest):
synced_children[child_name] = child_access
return synced_children
async def _sync_folder(self, path: FsPath, access: Access,
manifest: LocalFolderManifest,
recursive: bool) -> None:
assert not is_placeholder_manifest(manifest)
assert is_folderish_manifest(manifest)
# Synchronizing a folder is divided into three steps:
# - first synchronizing it children
# - then sychronize itself
# - finally merge the synchronized version with the current one (that
# may have been updated in the meantime)
# Synchronizing children
if recursive:
for child_name, child_access in sorted(manifest.children.items(),
key=lambda x: x[0]):
child_path = path / child_name
try:
await self._sync_nolock(child_path, True)
except FileNotFoundError:
# Concurrent deletion occured, just ignore this child
pass
# The trick here is to retreive the current version of the manifest
# and remove it placeholders (those are the children created since
# the start of our sync)
manifest = self.local_folder_fs.get_manifest(access)
assert is_folderish_manifest(manifest)
manifest = manifest.evolve(
children=self._strip_placeholders(manifest.children))
# Now we can synchronize the folder if needed
if not manifest.need_sync:
target_remote_manifest = await self._sync_folder_look_for_remote_changes(
access, manifest)
# Quick exit if nothing's new
if not target_remote_manifest:
return
event_type = "fs.entry.remote_changed"
else:
target_remote_manifest = await self._sync_folder_actual_sync(
path, access, manifest)
event_type = "fs.entry.synced"
assert is_folderish_manifest(target_remote_manifest)
# Merge the synchronized version with the current one
self._sync_folder_merge_back(path, access, manifest,
target_remote_manifest)
self.event_bus.send(event_type, path=str(path), id=access.id)
async def _sync_file(self, path: FsPath, access: Access,
manifest: LocalFileManifest) -> None:
"""
Raises:
FileSyncConcurrencyError
BackendNotAvailable
"""
assert not is_placeholder_manifest(manifest)
assert is_file_manifest(manifest)
# Now we can synchronize the folder if needed
if not manifest.need_sync:
changed = await self._sync_file_look_for_remote_changes(
path, access, manifest)
else:
await self._sync_file_actual_sync(path, access, manifest)
changed = True
if changed:
self.event_bus.send("fs.entry.synced",
path=str(path),
id=access.id)
async def _minimal_sync_folder(self, path: FsPath, access: Access,
manifest: LocalFolderManifest) -> bool:
"""
Returns: If additional sync are needed
Raises:
FileSyncConcurrencyError
BackendNotAvailable
"""
if not is_placeholder_manifest(manifest):
return manifest.need_sync
synced_children = self._strip_placeholders(manifest.children)
need_more_sync = synced_children.keys() != manifest.children.keys()
manifest = manifest.evolve(children=synced_children)
target_remote_manifest = await self._sync_folder_actual_sync(
path, access, manifest)
self._sync_folder_merge_back(path, access, manifest,
target_remote_manifest)
self.event_bus.send("fs.entry.minimal_synced",
path=str(path),
id=access.id)
return need_more_sync
async def _sync_file_actual_sync(self, path: FsPath, access: Access,
manifest: LocalFileManifest) -> None:
assert is_file_manifest(manifest)
# to_sync_manifest = manifest.to_remote(version=manifest.base_version + 1)
to_sync_manifest = manifest.to_remote()
to_sync_manifest = to_sync_manifest.evolve(
version=manifest.base_version + 1)
# Compute the file's blocks and upload the new ones
blocks = []
sync_map = get_sync_map(manifest, self.block_size)
# Upload the new blocks
spaces = sync_map.spaces
blocks = []
async def _process_spaces():
nonlocal blocks
while spaces:
cs = spaces.pop()
data = await self._build_data_from_contiguous_space(cs)
if not data:
# Already existing blocks taken verbatim
blocks += [bs.buffer.data for bs in cs.buffers]
else:
# Create a new block from existing data
block_access = BlockAccess.from_block(data, cs.start)
await self._backend_block_create(block_access, data)
blocks.append(block_access)
if len(spaces) < 2:
await _process_spaces()
else:
async with trio.open_nursery() as nursery:
nursery.start_soon(_process_spaces)
nursery.start_soon(_process_spaces)
nursery.start_soon(_process_spaces)
nursery.start_soon(_process_spaces)
to_sync_manifest = to_sync_manifest.evolve(
blocks=blocks,
size=sync_map.size # TODO: useful ?
)
# Upload the file manifest as new vlob version
notify_beacons = self.local_folder_fs.get_beacon(path)
try:
if is_placeholder_manifest(manifest):
await self._backend_vlob_create(access, to_sync_manifest,
notify_beacons)
else:
await self._backend_vlob_update(access, to_sync_manifest,
notify_beacons)
except SyncConcurrencyError:
# Placeholder don't have remote version, so concurrency shouldn't
# be possible. However it's possible a previous attempt of
# uploading this manifest succeeded but we didn't receive the
# backend's answer, hence wrongly believing this is still a
# placeholder.
if is_placeholder_manifest(manifest):
logger.warning("Concurrency error while creating vlob",
access_id=access.id)
target_remote_manifest = await self._backend_vlob_read(access)
current_manifest = self.local_folder_fs.get_manifest(access)
# Do a fast-forward to avoid losing block we have uploaded
diverged_manifest = fast_forward_file(manifest, current_manifest,
to_sync_manifest)
self._sync_file_look_resolve_concurrency(path, access,
diverged_manifest,
target_remote_manifest)
# # TODO
# target_local_manifest = remote_to_local_manifest(target_remote_manifest)
# self._sync_file_look_resolve_concurrency(
# path, access, current_manifest, target_local_manifest
# )
# await self._backend_vlob_create(access, to_sync_manifest, notify_beacons)
else:
self._sync_file_merge_back(access, manifest, to_sync_manifest)
return to_sync_manifest
async def _sync_folder_actual_sync(
self, path: FsPath, access: Access,
manifest: LocalFolderManifest) -> FolderManifest:
# to_sync_manifest = manifest.to_remote(version=manifest.base_version + 1)
to_sync_manifest = manifest.to_remote()
to_sync_manifest = to_sync_manifest.evolve(
version=manifest.base_version + 1)
# Upload the folder manifest as new vlob version
notify_beacons = self.local_folder_fs.get_beacon(path)
force_update = False
while True:
try:
if is_placeholder_manifest(manifest) and not force_update:
await self._backend_vlob_create(access, to_sync_manifest,
notify_beacons)
else:
await self._backend_vlob_update(access, to_sync_manifest,
notify_beacons)
break
except SyncConcurrencyError:
if is_placeholder_manifest(manifest):
# Placeholder don't have remote version, so concurrency shouldn't
# be possible. However special cases exist:
# - user manifest has it access is shared between devices
# even if it is not yet synced.
# - it's possible a previous attempt of uploading this
# manifest succeeded but we didn't receive the backend's
# answer, hence wrongly believing this is still a placeholder.
# If such cases occured, we just have to pretend we were
# trying to do an update and rely on the generic merge.
if is_user_manifest(manifest):
logger.warning(
"Concurrency error while creating user vlob",
access_id=access.id)
else:
logger.warning("Concurrency error while creating vlob",
access_id=access.id)
base = None
force_update = True
else:
base = await self._backend_vlob_read(
access, to_sync_manifest.version - 1)
# Do a 3-ways merge to fix the concurrency error, first we must
# fetch the base version and the new one present in the backend
# TODO: base should be available locally
target = await self._backend_vlob_read(access)
# 3-ways merge between base, modified and target versions
to_sync_manifest, sync_needed, conflicts = merge_remote_folder_manifests(
base, to_sync_manifest, target)
for original_name, original_id, diverged_name, diverged_id in conflicts:
self.event_bus.send(
"fs.entry.name_conflicted",
path=str(path / original_name),
diverged_path=str(path / diverged_name),
original_id=original_id,
diverged_id=diverged_id,
)
if not sync_needed:
# It maybe possible the changes that cause the concurrency
# error were the same than the one we wanted to make in the
# first place (e.g. when removing the same file)
break
to_sync_manifest = to_sync_manifest.evolve(
version=target.version + 1)
return to_sync_manifest