async def foo_txt(alice, alice_file_transactions):
local_storage = alice_file_transactions.local_storage
now = Pendulum(2000, 1, 2)
placeholder = LocalFileManifest.new_placeholder(parent=EntryID(), now=now)
remote_v1 = placeholder.to_remote(author=alice.device_id, timestamp=now)
manifest = LocalFileManifest.from_remote(remote_v1)
async with local_storage.lock_entry_id(manifest.id):
await local_storage.set_manifest(manifest.id, manifest)
return File(local_storage, manifest)
def _sync_file_look_resolve_concurrency(
self,
path: FsPath,
access: Access,
diverged_manifest: LocalFileManifest,
target_remote_manifest: FileManifest,
) -> None:
parent_access, parent_manifest = self.local_folder_fs.get_entry(
path.parent)
moved_name = find_conflicting_name_for_child_entry(
path.name, lambda name: name not in parent_manifest.children)
moved_access = ManifestAccess()
parent_manifest = parent_manifest.evolve_children_and_mark_updated(
{moved_name: moved_access})
diverged_manifest = diverged_manifest.evolve(base_version=0,
created=pendulum.now(),
need_sync=True,
is_placeholder=True)
self.local_folder_fs.set_manifest(moved_access, diverged_manifest)
self.local_folder_fs.set_manifest(parent_access, parent_manifest)
target_manifest = target_remote_manifest.to_local()
self.local_folder_fs.set_manifest(access, target_manifest)
self.event_bus.send(
"fs.entry.file_update_conflicted",
path=str(path),
diverged_path=str(path.parent / moved_name),
original_id=access.id,
diverged_id=moved_access.id,
)
self.event_bus.send("fs.entry.updated", id=moved_access.id)
async def file_create(
self, path: FsPath, open: bool = True
) -> Tuple[EntryID, Optional[FileDescriptor]]:
# Check write rights
self.check_write_rights(path)
# Lock parent in write mode
async with self._lock_parent_manifest_from_path(path) as (parent, child):
# Destination already exists
if child is not None:
raise FSFileExistsError(filename=path)
# Create file
child = LocalFileManifest.new_placeholder(self.local_author, parent=parent.id)
# New parent manifest
new_parent = parent.evolve_children_and_mark_updated(
{path.name: child.id},
prevent_sync_pattern=self.local_storage.get_prevent_sync_pattern(),
)
# ~ Atomic change
await self.local_storage.set_manifest(child.id, child, check_lock_status=False)
await self.local_storage.set_manifest(parent.id, new_parent)
fd = self.local_storage.create_file_descriptor(child) if open else None
# Send events
self._send_event(CoreEvent.FS_ENTRY_UPDATED, id=parent.id)
self._send_event(CoreEvent.FS_ENTRY_UPDATED, id=child.id)
# Return the entry id of the created file and the file descriptor
return child.id, fd
def __init__(self) -> None:
super().__init__()
self.oracle = open(tmpdir / "oracle.txt", "w+b")
self.manifest = LocalFileManifest.new_placeholder(DeviceID.new(),
parent=EntryID(),
blocksize=8)
self.storage = Storage()
def prepare_truncate(manifest: LocalFileManifest,
size: int) -> Tuple[LocalFileManifest, Set[BlockID]]:
# Prepare
block, remainder = locate(size, manifest.blocksize)
removed_ids = chunk_id_set(manifest.blocks[block])
# Truncate buffers
blocks = manifest.blocks[:block]
if remainder:
chunks = manifest.blocks[block]
stop_index = index_of_chunk_after_stop(chunks, size)
last_chunk = chunks[stop_index - 1]
chunks = chunks[:stop_index - 1]
chunks += (last_chunk.evolve(stop=size), )
blocks += (chunks, )
removed_ids -= chunk_id_set(chunks)
# Clean up
for chunks in manifest.blocks[block + 1:]:
removed_ids |= chunk_id_set(chunks)
# Craft new manifest
new_manifest = manifest.evolve_and_mark_updated(size=size, blocks=blocks)
# Return truncate result
return new_manifest, removed_ids
def _recursive_process_copy_map(copy_map):
manifest = copy_map["manifest"]
cpy_access = ManifestAccess()
if is_file_manifest(manifest):
cpy_manifest = LocalFileManifest(
author=self.local_author,
size=manifest.size,
blocks=manifest.blocks,
dirty_blocks=manifest.dirty_blocks,
)
else:
cpy_children = {}
for child_name in manifest.children.keys():
child_copy_map = copy_map["children"][child_name]
new_child_access = _recursive_process_copy_map(
child_copy_map)
cpy_children[child_name] = new_child_access
if is_folder_manifest(manifest):
cpy_manifest = LocalFolderManifest(
author=self.local_author, children=cpy_children)
else:
assert is_workspace_manifest(manifest)
cpy_manifest = LocalWorkspaceManifest(
self.local_author, children=cpy_children)
self.set_manifest(cpy_access, cpy_manifest)
return cpy_access
async def init(self):
nonlocal tentative
tentative += 1
await self.reset_all()
await self.start_backend()
self.device = alice
await self.start_transactions()
self.file_transactions = self.transactions_controller.file_transactions
self.local_storage = self.file_transactions.local_storage
self.fresh_manifest = LocalFileManifest.new_placeholder(
alice.device_id,
parent=EntryID.new(),
timestamp=alice.timestamp())
self.entry_id = self.fresh_manifest.id
async with self.local_storage.lock_entry_id(self.entry_id):
await self.local_storage.set_manifest(self.entry_id,
self.fresh_manifest)
self.fd = self.local_storage.create_file_descriptor(
self.fresh_manifest)
self.file_oracle_path = tmpdir / f"oracle-test-{tentative}.txt"
self.file_oracle_fd = os.open(self.file_oracle_path,
os.O_RDWR | os.O_CREAT)
async def _minimal_sync_file(self, path: FsPath, access: Access,
manifest: LocalFileManifest) -> bool:
"""
Returns: If additional sync are needed
Raises:
FileSyncConcurrencyError
BackendNotAvailable
"""
if not is_placeholder_manifest(manifest):
return manifest.need_sync
need_more_sync = bool(manifest.dirty_blocks)
# Don't sync the dirty blocks for fast synchronization
try:
last_block = manifest.blocks[-1]
size = last_block.offset + last_block.size
except IndexError:
size = 0
minimal_manifest = manifest.evolve(
updated=manifest.created if need_more_sync else manifest.updated,
size=size,
blocks=manifest.blocks,
dirty_blocks=(),
)
await self._sync_file_actual_sync(path, access, minimal_manifest)
self.event_bus.send("fs.entry.minimal_synced",
path=str(path),
id=access.id)
return need_more_sync
def test_merge_file_manifests(alice, bob):
timestamp = alice.timestamp()
my_device = alice.device_id
other_device = bob.device_id
parent = EntryID.new()
v1 = LocalFileManifest.new_placeholder(my_device,
parent=parent,
timestamp=timestamp).to_remote(
author=other_device,
timestamp=timestamp)
def evolve(m, n):
chunk = Chunk.new(0, n).evolve_as_block(b"a" * n)
blocks = ((chunk, ), )
return m1.evolve_and_mark_updated(size=n,
blocks=blocks,
timestamp=timestamp)
# Initial base manifest
m1 = LocalFileManifest.from_remote(v1)
assert merge_manifests(my_device, timestamp, empty_pattern, m1) == m1
# Local change
m2 = evolve(m1, 1)
assert merge_manifests(my_device, timestamp, empty_pattern, m2) == m2
# Successful upload
v2 = m2.to_remote(author=my_device, timestamp=timestamp)
m3 = merge_manifests(my_device, timestamp, empty_pattern, m2, v2)
assert m3 == LocalFileManifest.from_remote(v2)
# Two local changes
m4 = evolve(m3, 2)
assert merge_manifests(my_device, timestamp, empty_pattern, m4) == m4
m5 = evolve(m4, 3)
assert merge_manifests(my_device, timestamp, empty_pattern, m4) == m4
# M4 has been successfully uploaded
v3 = m4.to_remote(author=my_device, timestamp=timestamp)
m6 = merge_manifests(my_device, timestamp, empty_pattern, m5, v3)
assert m6 == m5.evolve(base=v3)
# The remote has changed
v4 = v3.evolve(version=4, size=0, author=other_device)
with pytest.raises(FSFileConflictError):
merge_manifests(my_device, timestamp, empty_pattern, m6, v4)
def prepare_write(
manifest: LocalFileManifest, size: int, offset: int, timestamp: DateTime
) -> Tuple[LocalFileManifest, WriteOperationList, ChunkIDSet]:
# Prepare
padding = 0
removed_ids: ChunkIDSet = set()
write_operations: WriteOperationList = []
# Padding
if offset > manifest.size:
padding = offset - manifest.size
size += padding
offset = manifest.size
# Copy buffers
blocks = list(manifest.blocks)
# Loop over blocks
for block, subsize, start, content_offset in split_write(
size, offset, manifest.blocksize):
# Prepare new chunk
new_chunk = Chunk.new(start, start + subsize)
write_operations.append((new_chunk, content_offset - padding))
# Lazy block write
chunks = manifest.get_chunks(block)
new_chunks, more_removed_ids = block_write(chunks, subsize, start,
new_chunk)
# Update data structures
removed_ids |= more_removed_ids
if len(blocks) == block:
blocks.append(new_chunks)
else:
blocks[block] = new_chunks
# Evolve manifest
new_size = max(manifest.size, offset + size)
new_manifest = manifest.evolve_and_mark_updated(size=new_size,
blocks=tuple(blocks),
timestamp=timestamp)
# Return write result
return new_manifest, write_operations, removed_ids
def __init__(self) -> None:
super().__init__()
self.oracle = open(tmpdir / "oracle.txt", "w+b")
self.manifest = LocalFileManifest.new_placeholder(
alice.device_id,
parent=EntryID.new(),
blocksize=8,
timestamp=alice.timestamp())
self.storage = Storage()
async def file_conflict(
self, entry_id: EntryID, local_manifest: LocalManifest, remote_manifest: RemoteManifest
) -> None:
# This is the only transaction that affects more than one manifests
# That's because the local version of the file has to be registered in the
# parent as a new child while the remote version has to be set as the actual
# version. In practice, this should not be an issue.
# Lock parent then child
parent_id = local_manifest.parent
async with self.local_storage.lock_manifest(parent_id) as parent_manifest:
async with self.local_storage.lock_manifest(entry_id) as current_manifest:
# Make sure the file still exists
filename = get_filename(parent_manifest, entry_id)
if filename is None:
return
# Copy blocks
new_blocks = []
for chunks in current_manifest.blocks:
new_chunks = []
for chunk in chunks:
data = await self.local_storage.get_chunk(chunk.id)
new_chunk = Chunk.new(chunk.start, chunk.stop)
await self.local_storage.set_chunk(new_chunk.id, data)
if len(chunks) == 1:
new_chunk = new_chunk.evolve_as_block(data)
new_chunks.append(chunk)
new_blocks.append(tuple(new_chunks))
new_blocks = tuple(new_blocks)
# Prepare
new_name = get_conflict_filename(
filename, list(parent_manifest.children), remote_manifest.author
)
new_manifest = LocalFileManifest.new_placeholder(parent=parent_id).evolve(
size=current_manifest.size, blocks=new_blocks
)
new_parent_manifest = parent_manifest.evolve_children_and_mark_updated(
{new_name: new_manifest.id}
)
other_manifest = LocalManifest.from_remote(remote_manifest)
# Set manifests
await self.local_storage.set_manifest(
new_manifest.id, new_manifest, check_lock_status=False
)
await self.local_storage.set_manifest(parent_id, new_parent_manifest)
await self.local_storage.set_manifest(entry_id, other_manifest)
self._send_event("fs.entry.updated", id=new_manifest.id)
self._send_event("fs.entry.updated", id=parent_id)
self._send_event(
"fs.entry.file_conflict_resolved", id=entry_id, backup_id=new_manifest.id
)
def reshape(self, manifest: LocalFileManifest) -> LocalFileManifest:
for block, source, destination, write_back, removed_ids in prepare_reshape(
manifest):
data = self.build_data(source)
new_chunk = destination.evolve_as_block(data)
if write_back:
self.write_chunk(new_chunk, data)
manifest = manifest.evolve_single_block(block, new_chunk)
for removed_id in removed_ids:
self.clear_chunk_data(removed_id)
return manifest
def prepare_read(manifest: LocalFileManifest, size: int, offset: int) -> Chunks:
# Prepare
chunks: List[Chunk] = []
offset = min(offset, manifest.size)
size = min(size, manifest.size - offset)
# Loop over blocks
for block, length, start in split_read(size, offset, manifest.blocksize):
# Loop over chunks
block_chunks = manifest.get_chunks(block)
chunks += block_read(block_chunks, length, start)
# Return read result
return tuple(chunks)
def fast_forward_file(local_base: LocalFileManifest,
local_current: LocalFileManifest,
remote_target: FileManifest) -> LocalFileManifest:
assert local_base.base_version < remote_target.version
assert local_base.base_version <= local_current.base_version
assert local_current.base_version < remote_target.version
processed_dirty_blocks_ids = [k.id for k in local_base.dirty_blocks]
merged_dirty_blocks = [
k for k in local_current.dirty_blocks
if k.id not in processed_dirty_blocks_ids
]
merged_need_sync = bool(merged_dirty_blocks
or local_current.size != remote_target.size)
return local_current.evolve(
blocks=remote_target.blocks,
dirty_blocks=merged_dirty_blocks,
base_version=remote_target.version,
is_placeholder=False,
need_sync=merged_need_sync,
)
async def _manifest_reshape(self,
manifest: LocalFileManifest,
cache_only: bool = False) -> List[BlockAccess]:
"""This internal helper does not perform any locking."""
# Prepare data structures
missing = []
# Perform operations
for block, source, destination, write_back, removed_ids in prepare_reshape(
manifest):
# Build data block
data, extra_missing = await self._build_data(source)
# Missing data
if extra_missing:
missing += extra_missing
continue
# Write data if necessary
new_chunk = destination.evolve_as_block(data)
if write_back:
await self._write_chunk(new_chunk, data)
# Craft the new manifest
manifest = manifest.evolve_single_block(block, new_chunk)
# Set the new manifest, acting as a checkpoint
await self.local_storage.set_manifest(manifest.id,
manifest,
cache_only=True,
removed_ids=removed_ids)
# Flush if necessary
if not cache_only:
await self.local_storage.ensure_manifest_persistent(manifest.id)
# Return missing block ids
return missing
def touch(self, path: FsPath) -> None:
if path.is_root():
raise FileExistsError(17, "File exists", str(path))
if path.parent.is_root():
raise PermissionError(
13, "Permission denied (only workpace allowed at root level)",
str(path))
access, manifest = self._retrieve_entry(path.parent)
if not is_folderish_manifest(manifest):
raise NotADirectoryError(20, "Not a directory", str(path.parent))
if path.name in manifest.children:
raise FileExistsError(17, "File exists", str(path))
child_access = ManifestAccess()
child_manifest = LocalFileManifest(self.local_author)
manifest = manifest.evolve_children_and_mark_updated(
{path.name: child_access})
self.set_manifest(access, manifest)
self.set_manifest(child_access, child_manifest)
self.event_bus.send("fs.entry.updated", id=access.id)
self.event_bus.send("fs.entry.updated", id=child_access.id)
def test_complete_scenario():
storage = Storage()
with freeze_time("2000-01-01"):
base = manifest = LocalFileManifest.new_placeholder(parent=EntryID(),
blocksize=16)
assert manifest == base.evolve(size=0)
with freeze_time("2000-01-02") as t2:
manifest = storage.write(manifest, b"Hello ", 0)
assert storage.read(manifest, 6, 0) == b"Hello "
(chunk0, ), = manifest.blocks
assert manifest == base.evolve(size=6, blocks=((chunk0, ), ), updated=t2)
assert chunk0 == Chunk(id=chunk0.id,
start=0,
stop=6,
raw_offset=0,
raw_size=6,
access=None)
assert storage[chunk0.id] == b"Hello "
with freeze_time("2000-01-03") as t3:
manifest = storage.write(manifest, b"world !", 6)
assert storage.read(manifest, 13, 0) == b"Hello world !"
(_, chunk1), = manifest.blocks
assert manifest == base.evolve(size=13,
blocks=((chunk0, chunk1), ),
updated=t3)
assert chunk1 == Chunk(id=chunk1.id,
start=6,
stop=13,
raw_offset=6,
raw_size=7,
access=None)
assert storage[chunk1.id] == b"world !"
with freeze_time("2000-01-04") as t4:
manifest = storage.write(manifest, b"\n More kontent", 13)
assert storage.read(manifest, 27, 0) == b"Hello world !\n More kontent"
(_, _, chunk2), (chunk3, ) = manifest.blocks
assert storage[chunk2.id] == b"\n M"
assert storage[chunk3.id] == b"ore kontent"
assert manifest == base.evolve(size=27,
blocks=((chunk0, chunk1, chunk2),
(chunk3, )),
updated=t4)
with freeze_time("2000-01-05") as t5:
manifest = storage.write(manifest, b"c", 20)
assert storage.read(manifest, 27, 0) == b"Hello world !\n More content"
chunk4, chunk5, chunk6 = manifest.blocks[1]
assert chunk3.id == chunk4.id == chunk6.id
assert storage[chunk5.id] == b"c"
assert manifest == base.evolve(size=27,
blocks=((chunk0, chunk1, chunk2),
(chunk4, chunk5, chunk6)),
updated=t5)
with freeze_time("2000-01-06") as t6:
manifest = storage.resize(manifest, 40)
expected = b"Hello world !\n More content" + b"\x00" * 13
assert storage.read(manifest, 40, 0) == expected
(_, _, _, chunk7), (chunk8, ) = manifest.blocks[1:]
assert storage[chunk7.id] == b"\x00" * 5
assert storage[chunk8.id] == b"\x00" * 8
assert manifest == base.evolve(
size=40,
blocks=((chunk0, chunk1, chunk2), (chunk4, chunk5, chunk6, chunk7),
(chunk8, )),
updated=t6,
)
with freeze_time("2000-01-07") as t7:
manifest = storage.resize(manifest, 25)
expected = b"Hello world !\n More conte"
assert storage.read(manifest, 25, 0) == expected
(_, _, chunk9), = manifest.blocks[1:]
assert chunk9.id == chunk6.id
assert manifest == base.evolve(size=25,
blocks=((chunk0, chunk1, chunk2),
(chunk4, chunk5, chunk9)),
updated=t7)
with freeze_time("2000-01-08"):
assert not manifest.is_reshaped()
manifest = storage.reshape(manifest)
expected = b"Hello world !\n More conte"
assert storage.read(manifest, 25, 0) == expected
assert manifest.is_reshaped()
(chunk10, ), (chunk11, ) = manifest.blocks
assert storage[chunk10.id] == b"Hello world !\n M"
assert storage[chunk11.id] == b"ore conte"
assert manifest == base.evolve(size=25,
blocks=((chunk10, ), (chunk11, )),
updated=t7)
def update_manifest(block: int, manifest: LocalFileManifest,
new_chunk: Chunk) -> LocalFileManifest:
blocks = list(manifest.blocks)
blocks[block] = (new_chunk, )
return manifest.evolve(blocks=tuple(blocks))
async def file_conflict(
self,
entry_id: EntryID,
local_manifest: Union[LocalFolderManifest, LocalFileManifest],
remote_manifest: BaseRemoteManifest,
) -> None:
# This is the only transaction that affects more than one manifests
# That's because the local version of the file has to be registered in the
# parent as a new child while the remote version has to be set as the actual
# version. In practice, this should not be an issue.
# Lock parent then child
parent_id = local_manifest.parent
async with self.local_storage.lock_manifest(
parent_id) as parent_manifest:
# Not a folderish manifest
if not isinstance(parent_manifest,
(LocalFolderManifest, LocalWorkspaceManifest)):
raise FSNotADirectoryError(parent_id)
async with self.local_storage.lock_manifest(
entry_id) as current_manifest:
# Not a file manifest
if not isinstance(current_manifest, LocalFileManifest):
raise FSIsADirectoryError(entry_id)
# Make sure the file still exists
filename = get_filename(parent_manifest, entry_id)
if filename is None:
return
# Copy blocks
new_blocks = []
for chunks in current_manifest.blocks:
new_chunks = []
for chunk in chunks:
data = await self.local_storage.get_chunk(chunk.id)
new_chunk = Chunk.new(chunk.start, chunk.stop)
await self.local_storage.set_chunk(new_chunk.id, data)
if len(chunks) == 1:
new_chunk = new_chunk.evolve_as_block(data)
new_chunks.append(chunk)
new_blocks.append(tuple(new_chunks))
# Prepare
prevent_sync_pattern = self.local_storage.get_prevent_sync_pattern(
)
new_name = get_conflict_filename(
filename, list(parent_manifest.children),
remote_manifest.author)
new_manifest = LocalFileManifest.new_placeholder(
self.local_author,
parent=parent_id).evolve(size=current_manifest.size,
blocks=tuple(new_blocks))
new_parent_manifest = parent_manifest.evolve_children_and_mark_updated(
{new_name: new_manifest.id},
prevent_sync_pattern=prevent_sync_pattern)
other_manifest = BaseLocalManifest.from_remote(
remote_manifest, prevent_sync_pattern=prevent_sync_pattern)
# Set manifests
await self.local_storage.set_manifest(new_manifest.id,
new_manifest,
check_lock_status=False)
await self.local_storage.set_manifest(parent_id,
new_parent_manifest)
await self.local_storage.set_manifest(entry_id, other_manifest)
self._send_event(CoreEvent.FS_ENTRY_UPDATED,
id=new_manifest.id)
self._send_event(CoreEvent.FS_ENTRY_UPDATED, id=parent_id)
self._send_event(
CoreEvent.FS_ENTRY_FILE_CONFLICT_RESOLVED,
id=entry_id,
backup_id=new_manifest.id,
)
async def _sync_file_actual_sync(self, path: FsPath, access: Access,
manifest: LocalFileManifest) -> None:
assert is_file_manifest(manifest)
# to_sync_manifest = manifest.to_remote(version=manifest.base_version + 1)
to_sync_manifest = manifest.to_remote()
to_sync_manifest = to_sync_manifest.evolve(
version=manifest.base_version + 1)
# Compute the file's blocks and upload the new ones
blocks = []
sync_map = get_sync_map(manifest, self.block_size)
# Upload the new blocks
spaces = sync_map.spaces
blocks = []
async def _process_spaces():
nonlocal blocks
while spaces:
cs = spaces.pop()
data = await self._build_data_from_contiguous_space(cs)
if not data:
# Already existing blocks taken verbatim
blocks += [bs.buffer.data for bs in cs.buffers]
else:
# Create a new block from existing data
block_access = BlockAccess.from_block(data, cs.start)
await self._backend_block_create(block_access, data)
blocks.append(block_access)
if len(spaces) < 2:
await _process_spaces()
else:
async with trio.open_nursery() as nursery:
nursery.start_soon(_process_spaces)
nursery.start_soon(_process_spaces)
nursery.start_soon(_process_spaces)
nursery.start_soon(_process_spaces)
to_sync_manifest = to_sync_manifest.evolve(
blocks=blocks,
size=sync_map.size # TODO: useful ?
)
# Upload the file manifest as new vlob version
notify_beacons = self.local_folder_fs.get_beacon(path)
try:
if is_placeholder_manifest(manifest):
await self._backend_vlob_create(access, to_sync_manifest,
notify_beacons)
else:
await self._backend_vlob_update(access, to_sync_manifest,
notify_beacons)
except SyncConcurrencyError:
# Placeholder don't have remote version, so concurrency shouldn't
# be possible. However it's possible a previous attempt of
# uploading this manifest succeeded but we didn't receive the
# backend's answer, hence wrongly believing this is still a
# placeholder.
if is_placeholder_manifest(manifest):
logger.warning("Concurrency error while creating vlob",
access_id=access.id)
target_remote_manifest = await self._backend_vlob_read(access)
current_manifest = self.local_folder_fs.get_manifest(access)
# Do a fast-forward to avoid losing block we have uploaded
diverged_manifest = fast_forward_file(manifest, current_manifest,
to_sync_manifest)
self._sync_file_look_resolve_concurrency(path, access,
diverged_manifest,
target_remote_manifest)
# # TODO
# target_local_manifest = remote_to_local_manifest(target_remote_manifest)
# self._sync_file_look_resolve_concurrency(
# path, access, current_manifest, target_local_manifest
# )
# await self._backend_vlob_create(access, to_sync_manifest, notify_beacons)
else:
self._sync_file_merge_back(access, manifest, to_sync_manifest)
return to_sync_manifest
