def merge_manifests(
local_author: DeviceID,
prevent_sync_pattern: Pattern[str],
local_manifest: BaseLocalManifest,
remote_manifest: Optional[BaseRemoteManifest] = None,
force_apply_pattern: Optional[bool] = False,
) -> BaseLocalManifest:
# Start by re-applying pattern (idempotent)
if force_apply_pattern and isinstance(
local_manifest, (LocalFolderManifest, LocalWorkspaceManifest)):
local_manifest = local_manifest.apply_prevent_sync_pattern(
prevent_sync_pattern)
# The remote hasn't changed
if remote_manifest is None or remote_manifest.version <= local_manifest.base_version:
return local_manifest
assert remote_manifest is not None
# Extract versions
remote_version = remote_manifest.version
local_version = local_manifest.base_version
local_from_remote = BaseLocalManifest.from_remote_with_local_context(
remote_manifest, prevent_sync_pattern, local_manifest)
# Only the remote has changed
if not local_manifest.need_sync:
return local_from_remote
# Both the remote and the local have changed
assert remote_version > local_version and local_manifest.need_sync
# All the local changes have been successfully uploaded
if local_manifest.match_remote(remote_manifest):
return local_from_remote
# The remote changes are ours, simply acknowledge them and keep our local changes
if remote_manifest.author == local_author:
return local_manifest.evolve(base=remote_manifest)
# The remote has been updated by some other device
assert remote_manifest.author != local_author
# Cannot solve a file conflict directly
if not isinstance(local_manifest,
(LocalFolderManifest, LocalWorkspaceManifest)):
raise FSFileConflictError(local_manifest, remote_manifest)
# Solve the folder conflict
new_children = merge_folder_children(
local_manifest.base.children,
local_manifest.children,
local_from_remote.children,
remote_manifest.author,
)
# Mark as updated
return local_from_remote.evolve_and_mark_updated(children=new_children)
async def get_manifest(self, entry_id: EntryID) -> BaseLocalManifest:
"""
Raises:
FSLocalMissError
"""
# Look in cache first
try:
return self._cache[entry_id]
except KeyError:
pass
# Look into the database
async with self._open_cursor() as cursor:
cursor.execute("SELECT blob FROM vlobs WHERE vlob_id = ?",
(entry_id.bytes, ))
manifest_row = cursor.fetchone()
# Not found
if not manifest_row:
raise FSLocalMissError(entry_id)
# Safely fill the cache
if entry_id not in self._cache:
self._cache[entry_id] = BaseLocalManifest.decrypt_and_load(
manifest_row[0], key=self.device.local_symkey)
# Always return the cached value
return self._cache[entry_id]
async def _get_manifest(self, entry_id: EntryID) -> BaseLocalManifest:
try:
return await self.local_storage.get_manifest(entry_id)
except FSLocalMissError as exc:
remote_manifest = await self.remote_loader.load_manifest(cast(EntryID, exc.id))
return BaseLocalManifest.from_remote(
remote_manifest, prevent_sync_pattern=self.local_storage.get_prevent_sync_pattern()
)
async def _load_and_lock_manifest(self, entry_id: EntryID) -> AsyncIterator[BaseLocalManifest]:
async with self.local_storage.lock_entry_id(entry_id):
try:
local_manifest = await self.local_storage.get_manifest(entry_id)
except FSLocalMissError as exc:
remote_manifest = await self.remote_loader.load_manifest(cast(EntryID, exc.id))
local_manifest = BaseLocalManifest.from_remote(
remote_manifest,
prevent_sync_pattern=self.local_storage.get_prevent_sync_pattern(),
)
await self.local_storage.set_manifest(entry_id, local_manifest)
yield local_manifest
async def file_conflict(
self,
entry_id: EntryID,
local_manifest: Union[LocalFolderManifest, LocalFileManifest],
remote_manifest: BaseRemoteManifest,
) -> None:
# This is the only transaction that affects more than one manifests
# That's because the local version of the file has to be registered in the
# parent as a new child while the remote version has to be set as the actual
# version. In practice, this should not be an issue.
# Lock parent then child
parent_id = local_manifest.parent
async with self.local_storage.lock_manifest(
parent_id) as parent_manifest:
# Not a folderish manifest
if not isinstance(parent_manifest,
(LocalFolderManifest, LocalWorkspaceManifest)):
raise FSNotADirectoryError(parent_id)
async with self.local_storage.lock_manifest(
entry_id) as current_manifest:
# Not a file manifest
if not isinstance(current_manifest, LocalFileManifest):
raise FSIsADirectoryError(entry_id)
# Make sure the file still exists
filename = get_filename(parent_manifest, entry_id)
if filename is None:
return
# Copy blocks
new_blocks = []
for chunks in current_manifest.blocks:
new_chunks = []
for chunk in chunks:
data = await self.local_storage.get_chunk(chunk.id)
new_chunk = Chunk.new(chunk.start, chunk.stop)
await self.local_storage.set_chunk(new_chunk.id, data)
if len(chunks) == 1:
new_chunk = new_chunk.evolve_as_block(data)
new_chunks.append(chunk)
new_blocks.append(tuple(new_chunks))
# Prepare
prevent_sync_pattern = self.local_storage.get_prevent_sync_pattern(
)
new_name = get_conflict_filename(
filename, list(parent_manifest.children),
remote_manifest.author)
new_manifest = LocalFileManifest.new_placeholder(
self.local_author,
parent=parent_id).evolve(size=current_manifest.size,
blocks=tuple(new_blocks))
new_parent_manifest = parent_manifest.evolve_children_and_mark_updated(
{new_name: new_manifest.id},
prevent_sync_pattern=prevent_sync_pattern)
other_manifest = BaseLocalManifest.from_remote(
remote_manifest, prevent_sync_pattern=prevent_sync_pattern)
# Set manifests
await self.local_storage.set_manifest(new_manifest.id,
new_manifest,
check_lock_status=False)
await self.local_storage.set_manifest(parent_id,
new_parent_manifest)
await self.local_storage.set_manifest(entry_id, other_manifest)
self._send_event(CoreEvent.FS_ENTRY_UPDATED,
id=new_manifest.id)
self._send_event(CoreEvent.FS_ENTRY_UPDATED, id=parent_id)
self._send_event(
CoreEvent.FS_ENTRY_FILE_CONFLICT_RESOLVED,
id=entry_id,
backup_id=new_manifest.id,
)
def merge_manifests(
local_author: DeviceID,
timestamp: DateTime,
prevent_sync_pattern: Pattern[str],
local_manifest: BaseLocalManifest,
remote_manifest: Optional[BaseRemoteManifest] = None,
force_apply_pattern: Optional[bool] = False,
preferred_language: str = "en",
) -> BaseLocalManifest:
# Start by re-applying pattern (idempotent)
if force_apply_pattern and isinstance(
local_manifest, (LocalFolderManifest, LocalWorkspaceManifest)):
local_manifest = local_manifest.apply_prevent_sync_pattern(
prevent_sync_pattern, timestamp)
# The remote hasn't changed
if remote_manifest is None or remote_manifest.version <= local_manifest.base_version:
return local_manifest
assert remote_manifest is not None
# Extract versions
remote_version = remote_manifest.version
local_version = local_manifest.base_version
local_from_remote = BaseLocalManifest.from_remote_with_local_context(
remote_manifest, prevent_sync_pattern, local_manifest, timestamp)
# Only the remote has changed
if not local_manifest.need_sync:
return local_from_remote
# Both the remote and the local have changed
assert remote_version > local_version and local_manifest.need_sync
# All the local changes have been successfully uploaded
if local_manifest.match_remote(remote_manifest):
return local_from_remote
# The remote changes are ours (our current local changes occurs while
# we were uploading previous local changes that became the remote changes),
# simply acknowledge them remote changes and keep our local changes
#
# However speculative manifest can lead to a funny behavior:
# 1) alice has access to the workspace
# 2) alice upload a new remote workspace manifest
# 3) alice gets it local storage removed
# So next time alice tries to access this workspace she will
# creates a speculative workspace manifest.
# This speculative manifest will eventually be synced against
# the previous remote remote manifest which appears to be remote
# changes we know about (given we are the author of it !).
# If the speculative flag is not taken into account, we would
# consider we have willingly removed all entries from the remote,
# hence uploading a new expurged remote manifest.
#
# Of course removing local storage is an unlikely situation, but:
# - it cannot be ruled out and would produce rare&exotic behavior
# that would be considered as bug :/
# - the fixtures and backend data binder system used in the tests
# makes it much more likely
speculative = isinstance(
local_manifest, LocalWorkspaceManifest) and local_manifest.speculative
if remote_manifest.author == local_author and not speculative:
return local_manifest.evolve(base=remote_manifest)
# The remote has been updated by some other device
assert remote_manifest.author != local_author or speculative is True
# Cannot solve a file conflict directly
if isinstance(local_manifest, LocalFileManifest):
raise FSFileConflictError(local_manifest, remote_manifest)
assert isinstance(local_manifest,
(LocalFolderManifest, LocalWorkspaceManifest))
# Solve the folder conflict
new_children = merge_folder_children(
base_children=local_manifest.base.children,
local_children=local_manifest.children,
remote_children=local_from_remote.children,
preferred_language=preferred_language,
)
# Children merge can end up with nothing to sync.
#
# This is typically the case when we sync for the first time a workspace
# shared with us that we didn't modify:
# - the workspace manifest is a speculative placeholder (with arbitrary update&create dates)
# - on sync the update date is different than in the remote, so a merge occurs
# - given we didn't modify the workspace, the children merge is trivial
# So without this check each each user we share the workspace with would
# sync a new workspace manifest version with only it updated date changing :/
#
# Another case where this happen:
# - we have local change on our workspace manifest for removing an entry
# - we rely on a base workspace manifest in version N
# - remote workspace manifest is in version N+1 and already integrate the removal
#
# /!\ Extra attention should be payed here if we want to add new fields
# /!\ with they own sync logic, as this optimization may shadow them !
if new_children == local_from_remote.children:
return local_from_remote
else:
return local_from_remote.evolve_and_mark_updated(children=new_children,
timestamp=timestamp)