async def foo_txt(alice, alice_file_transactions): local_storage = alice_file_transactions.local_storage now = Pendulum(2000, 1, 2) placeholder = LocalFileManifest.new_placeholder(parent=EntryID(), now=now) remote_v1 = placeholder.to_remote(author=alice.device_id, timestamp=now) manifest = LocalFileManifest.from_remote(remote_v1) async with local_storage.lock_entry_id(manifest.id): await local_storage.set_manifest(manifest.id, manifest) return File(local_storage, manifest)
def _sync_file_look_resolve_concurrency( self, path: FsPath, access: Access, diverged_manifest: LocalFileManifest, target_remote_manifest: FileManifest, ) -> None: parent_access, parent_manifest = self.local_folder_fs.get_entry( path.parent) moved_name = find_conflicting_name_for_child_entry( path.name, lambda name: name not in parent_manifest.children) moved_access = ManifestAccess() parent_manifest = parent_manifest.evolve_children_and_mark_updated( {moved_name: moved_access}) diverged_manifest = diverged_manifest.evolve(base_version=0, created=pendulum.now(), need_sync=True, is_placeholder=True) self.local_folder_fs.set_manifest(moved_access, diverged_manifest) self.local_folder_fs.set_manifest(parent_access, parent_manifest) target_manifest = target_remote_manifest.to_local() self.local_folder_fs.set_manifest(access, target_manifest) self.event_bus.send( "fs.entry.file_update_conflicted", path=str(path), diverged_path=str(path.parent / moved_name), original_id=access.id, diverged_id=moved_access.id, ) self.event_bus.send("fs.entry.updated", id=moved_access.id)
async def file_create( self, path: FsPath, open: bool = True ) -> Tuple[EntryID, Optional[FileDescriptor]]: # Check write rights self.check_write_rights(path) # Lock parent in write mode async with self._lock_parent_manifest_from_path(path) as (parent, child): # Destination already exists if child is not None: raise FSFileExistsError(filename=path) # Create file child = LocalFileManifest.new_placeholder(self.local_author, parent=parent.id) # New parent manifest new_parent = parent.evolve_children_and_mark_updated( {path.name: child.id}, prevent_sync_pattern=self.local_storage.get_prevent_sync_pattern(), ) # ~ Atomic change await self.local_storage.set_manifest(child.id, child, check_lock_status=False) await self.local_storage.set_manifest(parent.id, new_parent) fd = self.local_storage.create_file_descriptor(child) if open else None # Send events self._send_event(CoreEvent.FS_ENTRY_UPDATED, id=parent.id) self._send_event(CoreEvent.FS_ENTRY_UPDATED, id=child.id) # Return the entry id of the created file and the file descriptor return child.id, fd
def __init__(self) -> None: super().__init__() self.oracle = open(tmpdir / "oracle.txt", "w+b") self.manifest = LocalFileManifest.new_placeholder(DeviceID.new(), parent=EntryID(), blocksize=8) self.storage = Storage()
def prepare_truncate(manifest: LocalFileManifest, size: int) -> Tuple[LocalFileManifest, Set[BlockID]]: # Prepare block, remainder = locate(size, manifest.blocksize) removed_ids = chunk_id_set(manifest.blocks[block]) # Truncate buffers blocks = manifest.blocks[:block] if remainder: chunks = manifest.blocks[block] stop_index = index_of_chunk_after_stop(chunks, size) last_chunk = chunks[stop_index - 1] chunks = chunks[:stop_index - 1] chunks += (last_chunk.evolve(stop=size), ) blocks += (chunks, ) removed_ids -= chunk_id_set(chunks) # Clean up for chunks in manifest.blocks[block + 1:]: removed_ids |= chunk_id_set(chunks) # Craft new manifest new_manifest = manifest.evolve_and_mark_updated(size=size, blocks=blocks) # Return truncate result return new_manifest, removed_ids
def _recursive_process_copy_map(copy_map): manifest = copy_map["manifest"] cpy_access = ManifestAccess() if is_file_manifest(manifest): cpy_manifest = LocalFileManifest( author=self.local_author, size=manifest.size, blocks=manifest.blocks, dirty_blocks=manifest.dirty_blocks, ) else: cpy_children = {} for child_name in manifest.children.keys(): child_copy_map = copy_map["children"][child_name] new_child_access = _recursive_process_copy_map( child_copy_map) cpy_children[child_name] = new_child_access if is_folder_manifest(manifest): cpy_manifest = LocalFolderManifest( author=self.local_author, children=cpy_children) else: assert is_workspace_manifest(manifest) cpy_manifest = LocalWorkspaceManifest( self.local_author, children=cpy_children) self.set_manifest(cpy_access, cpy_manifest) return cpy_access
async def init(self): nonlocal tentative tentative += 1 await self.reset_all() await self.start_backend() self.device = alice await self.start_transactions() self.file_transactions = self.transactions_controller.file_transactions self.local_storage = self.file_transactions.local_storage self.fresh_manifest = LocalFileManifest.new_placeholder( alice.device_id, parent=EntryID.new(), timestamp=alice.timestamp()) self.entry_id = self.fresh_manifest.id async with self.local_storage.lock_entry_id(self.entry_id): await self.local_storage.set_manifest(self.entry_id, self.fresh_manifest) self.fd = self.local_storage.create_file_descriptor( self.fresh_manifest) self.file_oracle_path = tmpdir / f"oracle-test-{tentative}.txt" self.file_oracle_fd = os.open(self.file_oracle_path, os.O_RDWR | os.O_CREAT)
async def _minimal_sync_file(self, path: FsPath, access: Access, manifest: LocalFileManifest) -> bool: """ Returns: If additional sync are needed Raises: FileSyncConcurrencyError BackendNotAvailable """ if not is_placeholder_manifest(manifest): return manifest.need_sync need_more_sync = bool(manifest.dirty_blocks) # Don't sync the dirty blocks for fast synchronization try: last_block = manifest.blocks[-1] size = last_block.offset + last_block.size except IndexError: size = 0 minimal_manifest = manifest.evolve( updated=manifest.created if need_more_sync else manifest.updated, size=size, blocks=manifest.blocks, dirty_blocks=(), ) await self._sync_file_actual_sync(path, access, minimal_manifest) self.event_bus.send("fs.entry.minimal_synced", path=str(path), id=access.id) return need_more_sync
def test_merge_file_manifests(alice, bob): timestamp = alice.timestamp() my_device = alice.device_id other_device = bob.device_id parent = EntryID.new() v1 = LocalFileManifest.new_placeholder(my_device, parent=parent, timestamp=timestamp).to_remote( author=other_device, timestamp=timestamp) def evolve(m, n): chunk = Chunk.new(0, n).evolve_as_block(b"a" * n) blocks = ((chunk, ), ) return m1.evolve_and_mark_updated(size=n, blocks=blocks, timestamp=timestamp) # Initial base manifest m1 = LocalFileManifest.from_remote(v1) assert merge_manifests(my_device, timestamp, empty_pattern, m1) == m1 # Local change m2 = evolve(m1, 1) assert merge_manifests(my_device, timestamp, empty_pattern, m2) == m2 # Successful upload v2 = m2.to_remote(author=my_device, timestamp=timestamp) m3 = merge_manifests(my_device, timestamp, empty_pattern, m2, v2) assert m3 == LocalFileManifest.from_remote(v2) # Two local changes m4 = evolve(m3, 2) assert merge_manifests(my_device, timestamp, empty_pattern, m4) == m4 m5 = evolve(m4, 3) assert merge_manifests(my_device, timestamp, empty_pattern, m4) == m4 # M4 has been successfully uploaded v3 = m4.to_remote(author=my_device, timestamp=timestamp) m6 = merge_manifests(my_device, timestamp, empty_pattern, m5, v3) assert m6 == m5.evolve(base=v3) # The remote has changed v4 = v3.evolve(version=4, size=0, author=other_device) with pytest.raises(FSFileConflictError): merge_manifests(my_device, timestamp, empty_pattern, m6, v4)
def prepare_write( manifest: LocalFileManifest, size: int, offset: int, timestamp: DateTime ) -> Tuple[LocalFileManifest, WriteOperationList, ChunkIDSet]: # Prepare padding = 0 removed_ids: ChunkIDSet = set() write_operations: WriteOperationList = [] # Padding if offset > manifest.size: padding = offset - manifest.size size += padding offset = manifest.size # Copy buffers blocks = list(manifest.blocks) # Loop over blocks for block, subsize, start, content_offset in split_write( size, offset, manifest.blocksize): # Prepare new chunk new_chunk = Chunk.new(start, start + subsize) write_operations.append((new_chunk, content_offset - padding)) # Lazy block write chunks = manifest.get_chunks(block) new_chunks, more_removed_ids = block_write(chunks, subsize, start, new_chunk) # Update data structures removed_ids |= more_removed_ids if len(blocks) == block: blocks.append(new_chunks) else: blocks[block] = new_chunks # Evolve manifest new_size = max(manifest.size, offset + size) new_manifest = manifest.evolve_and_mark_updated(size=new_size, blocks=tuple(blocks), timestamp=timestamp) # Return write result return new_manifest, write_operations, removed_ids
def __init__(self) -> None: super().__init__() self.oracle = open(tmpdir / "oracle.txt", "w+b") self.manifest = LocalFileManifest.new_placeholder( alice.device_id, parent=EntryID.new(), blocksize=8, timestamp=alice.timestamp()) self.storage = Storage()
async def file_conflict( self, entry_id: EntryID, local_manifest: LocalManifest, remote_manifest: RemoteManifest ) -> None: # This is the only transaction that affects more than one manifests # That's because the local version of the file has to be registered in the # parent as a new child while the remote version has to be set as the actual # version. In practice, this should not be an issue. # Lock parent then child parent_id = local_manifest.parent async with self.local_storage.lock_manifest(parent_id) as parent_manifest: async with self.local_storage.lock_manifest(entry_id) as current_manifest: # Make sure the file still exists filename = get_filename(parent_manifest, entry_id) if filename is None: return # Copy blocks new_blocks = [] for chunks in current_manifest.blocks: new_chunks = [] for chunk in chunks: data = await self.local_storage.get_chunk(chunk.id) new_chunk = Chunk.new(chunk.start, chunk.stop) await self.local_storage.set_chunk(new_chunk.id, data) if len(chunks) == 1: new_chunk = new_chunk.evolve_as_block(data) new_chunks.append(chunk) new_blocks.append(tuple(new_chunks)) new_blocks = tuple(new_blocks) # Prepare new_name = get_conflict_filename( filename, list(parent_manifest.children), remote_manifest.author ) new_manifest = LocalFileManifest.new_placeholder(parent=parent_id).evolve( size=current_manifest.size, blocks=new_blocks ) new_parent_manifest = parent_manifest.evolve_children_and_mark_updated( {new_name: new_manifest.id} ) other_manifest = LocalManifest.from_remote(remote_manifest) # Set manifests await self.local_storage.set_manifest( new_manifest.id, new_manifest, check_lock_status=False ) await self.local_storage.set_manifest(parent_id, new_parent_manifest) await self.local_storage.set_manifest(entry_id, other_manifest) self._send_event("fs.entry.updated", id=new_manifest.id) self._send_event("fs.entry.updated", id=parent_id) self._send_event( "fs.entry.file_conflict_resolved", id=entry_id, backup_id=new_manifest.id )
def reshape(self, manifest: LocalFileManifest) -> LocalFileManifest: for block, source, destination, write_back, removed_ids in prepare_reshape( manifest): data = self.build_data(source) new_chunk = destination.evolve_as_block(data) if write_back: self.write_chunk(new_chunk, data) manifest = manifest.evolve_single_block(block, new_chunk) for removed_id in removed_ids: self.clear_chunk_data(removed_id) return manifest
def prepare_read(manifest: LocalFileManifest, size: int, offset: int) -> Chunks: # Prepare chunks: List[Chunk] = [] offset = min(offset, manifest.size) size = min(size, manifest.size - offset) # Loop over blocks for block, length, start in split_read(size, offset, manifest.blocksize): # Loop over chunks block_chunks = manifest.get_chunks(block) chunks += block_read(block_chunks, length, start) # Return read result return tuple(chunks)
def fast_forward_file(local_base: LocalFileManifest, local_current: LocalFileManifest, remote_target: FileManifest) -> LocalFileManifest: assert local_base.base_version < remote_target.version assert local_base.base_version <= local_current.base_version assert local_current.base_version < remote_target.version processed_dirty_blocks_ids = [k.id for k in local_base.dirty_blocks] merged_dirty_blocks = [ k for k in local_current.dirty_blocks if k.id not in processed_dirty_blocks_ids ] merged_need_sync = bool(merged_dirty_blocks or local_current.size != remote_target.size) return local_current.evolve( blocks=remote_target.blocks, dirty_blocks=merged_dirty_blocks, base_version=remote_target.version, is_placeholder=False, need_sync=merged_need_sync, )
async def _manifest_reshape(self, manifest: LocalFileManifest, cache_only: bool = False) -> List[BlockAccess]: """This internal helper does not perform any locking.""" # Prepare data structures missing = [] # Perform operations for block, source, destination, write_back, removed_ids in prepare_reshape( manifest): # Build data block data, extra_missing = await self._build_data(source) # Missing data if extra_missing: missing += extra_missing continue # Write data if necessary new_chunk = destination.evolve_as_block(data) if write_back: await self._write_chunk(new_chunk, data) # Craft the new manifest manifest = manifest.evolve_single_block(block, new_chunk) # Set the new manifest, acting as a checkpoint await self.local_storage.set_manifest(manifest.id, manifest, cache_only=True, removed_ids=removed_ids) # Flush if necessary if not cache_only: await self.local_storage.ensure_manifest_persistent(manifest.id) # Return missing block ids return missing
def touch(self, path: FsPath) -> None: if path.is_root(): raise FileExistsError(17, "File exists", str(path)) if path.parent.is_root(): raise PermissionError( 13, "Permission denied (only workpace allowed at root level)", str(path)) access, manifest = self._retrieve_entry(path.parent) if not is_folderish_manifest(manifest): raise NotADirectoryError(20, "Not a directory", str(path.parent)) if path.name in manifest.children: raise FileExistsError(17, "File exists", str(path)) child_access = ManifestAccess() child_manifest = LocalFileManifest(self.local_author) manifest = manifest.evolve_children_and_mark_updated( {path.name: child_access}) self.set_manifest(access, manifest) self.set_manifest(child_access, child_manifest) self.event_bus.send("fs.entry.updated", id=access.id) self.event_bus.send("fs.entry.updated", id=child_access.id)
def test_complete_scenario(): storage = Storage() with freeze_time("2000-01-01"): base = manifest = LocalFileManifest.new_placeholder(parent=EntryID(), blocksize=16) assert manifest == base.evolve(size=0) with freeze_time("2000-01-02") as t2: manifest = storage.write(manifest, b"Hello ", 0) assert storage.read(manifest, 6, 0) == b"Hello " (chunk0, ), = manifest.blocks assert manifest == base.evolve(size=6, blocks=((chunk0, ), ), updated=t2) assert chunk0 == Chunk(id=chunk0.id, start=0, stop=6, raw_offset=0, raw_size=6, access=None) assert storage[chunk0.id] == b"Hello " with freeze_time("2000-01-03") as t3: manifest = storage.write(manifest, b"world !", 6) assert storage.read(manifest, 13, 0) == b"Hello world !" (_, chunk1), = manifest.blocks assert manifest == base.evolve(size=13, blocks=((chunk0, chunk1), ), updated=t3) assert chunk1 == Chunk(id=chunk1.id, start=6, stop=13, raw_offset=6, raw_size=7, access=None) assert storage[chunk1.id] == b"world !" with freeze_time("2000-01-04") as t4: manifest = storage.write(manifest, b"\n More kontent", 13) assert storage.read(manifest, 27, 0) == b"Hello world !\n More kontent" (_, _, chunk2), (chunk3, ) = manifest.blocks assert storage[chunk2.id] == b"\n M" assert storage[chunk3.id] == b"ore kontent" assert manifest == base.evolve(size=27, blocks=((chunk0, chunk1, chunk2), (chunk3, )), updated=t4) with freeze_time("2000-01-05") as t5: manifest = storage.write(manifest, b"c", 20) assert storage.read(manifest, 27, 0) == b"Hello world !\n More content" chunk4, chunk5, chunk6 = manifest.blocks[1] assert chunk3.id == chunk4.id == chunk6.id assert storage[chunk5.id] == b"c" assert manifest == base.evolve(size=27, blocks=((chunk0, chunk1, chunk2), (chunk4, chunk5, chunk6)), updated=t5) with freeze_time("2000-01-06") as t6: manifest = storage.resize(manifest, 40) expected = b"Hello world !\n More content" + b"\x00" * 13 assert storage.read(manifest, 40, 0) == expected (_, _, _, chunk7), (chunk8, ) = manifest.blocks[1:] assert storage[chunk7.id] == b"\x00" * 5 assert storage[chunk8.id] == b"\x00" * 8 assert manifest == base.evolve( size=40, blocks=((chunk0, chunk1, chunk2), (chunk4, chunk5, chunk6, chunk7), (chunk8, )), updated=t6, ) with freeze_time("2000-01-07") as t7: manifest = storage.resize(manifest, 25) expected = b"Hello world !\n More conte" assert storage.read(manifest, 25, 0) == expected (_, _, chunk9), = manifest.blocks[1:] assert chunk9.id == chunk6.id assert manifest == base.evolve(size=25, blocks=((chunk0, chunk1, chunk2), (chunk4, chunk5, chunk9)), updated=t7) with freeze_time("2000-01-08"): assert not manifest.is_reshaped() manifest = storage.reshape(manifest) expected = b"Hello world !\n More conte" assert storage.read(manifest, 25, 0) == expected assert manifest.is_reshaped() (chunk10, ), (chunk11, ) = manifest.blocks assert storage[chunk10.id] == b"Hello world !\n M" assert storage[chunk11.id] == b"ore conte" assert manifest == base.evolve(size=25, blocks=((chunk10, ), (chunk11, )), updated=t7)
def update_manifest(block: int, manifest: LocalFileManifest, new_chunk: Chunk) -> LocalFileManifest: blocks = list(manifest.blocks) blocks[block] = (new_chunk, ) return manifest.evolve(blocks=tuple(blocks))
async def file_conflict( self, entry_id: EntryID, local_manifest: Union[LocalFolderManifest, LocalFileManifest], remote_manifest: BaseRemoteManifest, ) -> None: # This is the only transaction that affects more than one manifests # That's because the local version of the file has to be registered in the # parent as a new child while the remote version has to be set as the actual # version. In practice, this should not be an issue. # Lock parent then child parent_id = local_manifest.parent async with self.local_storage.lock_manifest( parent_id) as parent_manifest: # Not a folderish manifest if not isinstance(parent_manifest, (LocalFolderManifest, LocalWorkspaceManifest)): raise FSNotADirectoryError(parent_id) async with self.local_storage.lock_manifest( entry_id) as current_manifest: # Not a file manifest if not isinstance(current_manifest, LocalFileManifest): raise FSIsADirectoryError(entry_id) # Make sure the file still exists filename = get_filename(parent_manifest, entry_id) if filename is None: return # Copy blocks new_blocks = [] for chunks in current_manifest.blocks: new_chunks = [] for chunk in chunks: data = await self.local_storage.get_chunk(chunk.id) new_chunk = Chunk.new(chunk.start, chunk.stop) await self.local_storage.set_chunk(new_chunk.id, data) if len(chunks) == 1: new_chunk = new_chunk.evolve_as_block(data) new_chunks.append(chunk) new_blocks.append(tuple(new_chunks)) # Prepare prevent_sync_pattern = self.local_storage.get_prevent_sync_pattern( ) new_name = get_conflict_filename( filename, list(parent_manifest.children), remote_manifest.author) new_manifest = LocalFileManifest.new_placeholder( self.local_author, parent=parent_id).evolve(size=current_manifest.size, blocks=tuple(new_blocks)) new_parent_manifest = parent_manifest.evolve_children_and_mark_updated( {new_name: new_manifest.id}, prevent_sync_pattern=prevent_sync_pattern) other_manifest = BaseLocalManifest.from_remote( remote_manifest, prevent_sync_pattern=prevent_sync_pattern) # Set manifests await self.local_storage.set_manifest(new_manifest.id, new_manifest, check_lock_status=False) await self.local_storage.set_manifest(parent_id, new_parent_manifest) await self.local_storage.set_manifest(entry_id, other_manifest) self._send_event(CoreEvent.FS_ENTRY_UPDATED, id=new_manifest.id) self._send_event(CoreEvent.FS_ENTRY_UPDATED, id=parent_id) self._send_event( CoreEvent.FS_ENTRY_FILE_CONFLICT_RESOLVED, id=entry_id, backup_id=new_manifest.id, )
async def _sync_file_actual_sync(self, path: FsPath, access: Access, manifest: LocalFileManifest) -> None: assert is_file_manifest(manifest) # to_sync_manifest = manifest.to_remote(version=manifest.base_version + 1) to_sync_manifest = manifest.to_remote() to_sync_manifest = to_sync_manifest.evolve( version=manifest.base_version + 1) # Compute the file's blocks and upload the new ones blocks = [] sync_map = get_sync_map(manifest, self.block_size) # Upload the new blocks spaces = sync_map.spaces blocks = [] async def _process_spaces(): nonlocal blocks while spaces: cs = spaces.pop() data = await self._build_data_from_contiguous_space(cs) if not data: # Already existing blocks taken verbatim blocks += [bs.buffer.data for bs in cs.buffers] else: # Create a new block from existing data block_access = BlockAccess.from_block(data, cs.start) await self._backend_block_create(block_access, data) blocks.append(block_access) if len(spaces) < 2: await _process_spaces() else: async with trio.open_nursery() as nursery: nursery.start_soon(_process_spaces) nursery.start_soon(_process_spaces) nursery.start_soon(_process_spaces) nursery.start_soon(_process_spaces) to_sync_manifest = to_sync_manifest.evolve( blocks=blocks, size=sync_map.size # TODO: useful ? ) # Upload the file manifest as new vlob version notify_beacons = self.local_folder_fs.get_beacon(path) try: if is_placeholder_manifest(manifest): await self._backend_vlob_create(access, to_sync_manifest, notify_beacons) else: await self._backend_vlob_update(access, to_sync_manifest, notify_beacons) except SyncConcurrencyError: # Placeholder don't have remote version, so concurrency shouldn't # be possible. However it's possible a previous attempt of # uploading this manifest succeeded but we didn't receive the # backend's answer, hence wrongly believing this is still a # placeholder. if is_placeholder_manifest(manifest): logger.warning("Concurrency error while creating vlob", access_id=access.id) target_remote_manifest = await self._backend_vlob_read(access) current_manifest = self.local_folder_fs.get_manifest(access) # Do a fast-forward to avoid losing block we have uploaded diverged_manifest = fast_forward_file(manifest, current_manifest, to_sync_manifest) self._sync_file_look_resolve_concurrency(path, access, diverged_manifest, target_remote_manifest) # # TODO # target_local_manifest = remote_to_local_manifest(target_remote_manifest) # self._sync_file_look_resolve_concurrency( # path, access, current_manifest, target_local_manifest # ) # await self._backend_vlob_create(access, to_sync_manifest, notify_beacons) else: self._sync_file_merge_back(access, manifest, to_sync_manifest) return to_sync_manifest