async def foo_txt(alice, alice_file_transactions): local_storage = alice_file_transactions.local_storage now = Pendulum(2000, 1, 2) placeholder = LocalFileManifest.new_placeholder(parent=EntryID(), now=now) remote_v1 = placeholder.to_remote(author=alice.device_id, timestamp=now) manifest = LocalFileManifest.from_remote(remote_v1) async with local_storage.lock_entry_id(manifest.id): await local_storage.set_manifest(manifest.id, manifest) return File(local_storage, manifest)
def __init__(self) -> None: super().__init__() self.oracle = open(tmpdir / "oracle.txt", "w+b") self.manifest = LocalFileManifest.new_placeholder( alice.device_id, parent=EntryID.new(), blocksize=8, timestamp=alice.timestamp()) self.storage = Storage()
async def file_conflict( self, entry_id: EntryID, local_manifest: LocalManifest, remote_manifest: RemoteManifest ) -> None: # This is the only transaction that affects more than one manifests # That's because the local version of the file has to be registered in the # parent as a new child while the remote version has to be set as the actual # version. In practice, this should not be an issue. # Lock parent then child parent_id = local_manifest.parent async with self.local_storage.lock_manifest(parent_id) as parent_manifest: async with self.local_storage.lock_manifest(entry_id) as current_manifest: # Make sure the file still exists filename = get_filename(parent_manifest, entry_id) if filename is None: return # Copy blocks new_blocks = [] for chunks in current_manifest.blocks: new_chunks = [] for chunk in chunks: data = await self.local_storage.get_chunk(chunk.id) new_chunk = Chunk.new(chunk.start, chunk.stop) await self.local_storage.set_chunk(new_chunk.id, data) if len(chunks) == 1: new_chunk = new_chunk.evolve_as_block(data) new_chunks.append(chunk) new_blocks.append(tuple(new_chunks)) new_blocks = tuple(new_blocks) # Prepare new_name = get_conflict_filename( filename, list(parent_manifest.children), remote_manifest.author ) new_manifest = LocalFileManifest.new_placeholder(parent=parent_id).evolve( size=current_manifest.size, blocks=new_blocks ) new_parent_manifest = parent_manifest.evolve_children_and_mark_updated( {new_name: new_manifest.id} ) other_manifest = LocalManifest.from_remote(remote_manifest) # Set manifests await self.local_storage.set_manifest( new_manifest.id, new_manifest, check_lock_status=False ) await self.local_storage.set_manifest(parent_id, new_parent_manifest) await self.local_storage.set_manifest(entry_id, other_manifest) self._send_event("fs.entry.updated", id=new_manifest.id) self._send_event("fs.entry.updated", id=parent_id) self._send_event( "fs.entry.file_conflict_resolved", id=entry_id, backup_id=new_manifest.id )
def test_merge_file_manifests(alice, bob): timestamp = alice.timestamp() my_device = alice.device_id other_device = bob.device_id parent = EntryID.new() v1 = LocalFileManifest.new_placeholder(my_device, parent=parent, timestamp=timestamp).to_remote( author=other_device, timestamp=timestamp) def evolve(m, n): chunk = Chunk.new(0, n).evolve_as_block(b"a" * n) blocks = ((chunk, ), ) return m1.evolve_and_mark_updated(size=n, blocks=blocks, timestamp=timestamp) # Initial base manifest m1 = LocalFileManifest.from_remote(v1) assert merge_manifests(my_device, timestamp, empty_pattern, m1) == m1 # Local change m2 = evolve(m1, 1) assert merge_manifests(my_device, timestamp, empty_pattern, m2) == m2 # Successful upload v2 = m2.to_remote(author=my_device, timestamp=timestamp) m3 = merge_manifests(my_device, timestamp, empty_pattern, m2, v2) assert m3 == LocalFileManifest.from_remote(v2) # Two local changes m4 = evolve(m3, 2) assert merge_manifests(my_device, timestamp, empty_pattern, m4) == m4 m5 = evolve(m4, 3) assert merge_manifests(my_device, timestamp, empty_pattern, m4) == m4 # M4 has been successfully uploaded v3 = m4.to_remote(author=my_device, timestamp=timestamp) m6 = merge_manifests(my_device, timestamp, empty_pattern, m5, v3) assert m6 == m5.evolve(base=v3) # The remote has changed v4 = v3.evolve(version=4, size=0, author=other_device) with pytest.raises(FSFileConflictError): merge_manifests(my_device, timestamp, empty_pattern, m6, v4)
async def file_create( self, path: FsPath, open: bool = True) -> Tuple[EntryID, Optional[FileDescriptor]]: # Check write rights self.check_write_rights(path) # Lock parent in write mode async with self._lock_parent_manifest_from_path(path) as (parent, child): # Destination already exists if child is not None: raise FSFileExistsError(filename=path) # Create file timestamp = self.device.timestamp() child = LocalFileManifest.new_placeholder(self.local_author, parent=parent.id, timestamp=timestamp) # New parent manifest new_parent = parent.evolve_children_and_mark_updated( {path.name: child.id}, prevent_sync_pattern=self.local_storage. get_prevent_sync_pattern(), timestamp=self.device.timestamp(), ) # ~ Atomic change await self.local_storage.set_manifest(child.id, child, check_lock_status=False) await self.local_storage.set_manifest(parent.id, new_parent) fd = self.local_storage.create_file_descriptor( child) if open else None # Send events self._send_event(CoreEvent.FS_ENTRY_UPDATED, id=parent.id) self._send_event(CoreEvent.FS_ENTRY_UPDATED, id=child.id) # Return the entry id of the created file and the file descriptor return child.id, fd
async def init(self): nonlocal tentative tentative += 1 await reset_testbed() self.device = alice await self.start_transactions() self.file_transactions = self.transactions_controller.file_transactions self.local_storage = self.file_transactions.local_storage self.fresh_manifest = LocalFileManifest.new_placeholder( parent=EntryID()) self.entry_id = self.fresh_manifest.id async with self.local_storage.lock_entry_id(self.entry_id): await self.local_storage.set_manifest(self.entry_id, self.fresh_manifest) self.fd = self.local_storage.create_file_descriptor( self.fresh_manifest) self.file_oracle_path = tmpdir / f"oracle-test-{tentative}.txt" self.file_oracle_fd = os.open(self.file_oracle_path, os.O_RDWR | os.O_CREAT)
def __init__(self) -> None: super().__init__() self.oracle = open(tmpdir / "oracle.txt", "w+b") self.manifest = LocalFileManifest.new_placeholder(parent=EntryID(), blocksize=8) self.storage = Storage()
def test_complete_scenario(): storage = Storage() with freeze_time("2000-01-01"): base = manifest = LocalFileManifest.new_placeholder(parent=EntryID(), blocksize=16) assert manifest == base.evolve(size=0) with freeze_time("2000-01-02") as t2: manifest = storage.write(manifest, b"Hello ", 0) assert storage.read(manifest, 6, 0) == b"Hello " (chunk0, ), = manifest.blocks assert manifest == base.evolve(size=6, blocks=((chunk0, ), ), updated=t2) assert chunk0 == Chunk(id=chunk0.id, start=0, stop=6, raw_offset=0, raw_size=6, access=None) assert storage[chunk0.id] == b"Hello " with freeze_time("2000-01-03") as t3: manifest = storage.write(manifest, b"world !", 6) assert storage.read(manifest, 13, 0) == b"Hello world !" (_, chunk1), = manifest.blocks assert manifest == base.evolve(size=13, blocks=((chunk0, chunk1), ), updated=t3) assert chunk1 == Chunk(id=chunk1.id, start=6, stop=13, raw_offset=6, raw_size=7, access=None) assert storage[chunk1.id] == b"world !" with freeze_time("2000-01-04") as t4: manifest = storage.write(manifest, b"\n More kontent", 13) assert storage.read(manifest, 27, 0) == b"Hello world !\n More kontent" (_, _, chunk2), (chunk3, ) = manifest.blocks assert storage[chunk2.id] == b"\n M" assert storage[chunk3.id] == b"ore kontent" assert manifest == base.evolve(size=27, blocks=((chunk0, chunk1, chunk2), (chunk3, )), updated=t4) with freeze_time("2000-01-05") as t5: manifest = storage.write(manifest, b"c", 20) assert storage.read(manifest, 27, 0) == b"Hello world !\n More content" chunk4, chunk5, chunk6 = manifest.blocks[1] assert chunk3.id == chunk4.id == chunk6.id assert storage[chunk5.id] == b"c" assert manifest == base.evolve(size=27, blocks=((chunk0, chunk1, chunk2), (chunk4, chunk5, chunk6)), updated=t5) with freeze_time("2000-01-06") as t6: manifest = storage.resize(manifest, 40) expected = b"Hello world !\n More content" + b"\x00" * 13 assert storage.read(manifest, 40, 0) == expected (_, _, _, chunk7), (chunk8, ) = manifest.blocks[1:] assert storage[chunk7.id] == b"\x00" * 5 assert storage[chunk8.id] == b"\x00" * 8 assert manifest == base.evolve( size=40, blocks=((chunk0, chunk1, chunk2), (chunk4, chunk5, chunk6, chunk7), (chunk8, )), updated=t6, ) with freeze_time("2000-01-07") as t7: manifest = storage.resize(manifest, 25) expected = b"Hello world !\n More conte" assert storage.read(manifest, 25, 0) == expected (_, _, chunk9), = manifest.blocks[1:] assert chunk9.id == chunk6.id assert manifest == base.evolve(size=25, blocks=((chunk0, chunk1, chunk2), (chunk4, chunk5, chunk9)), updated=t7) with freeze_time("2000-01-08"): assert not manifest.is_reshaped() manifest = storage.reshape(manifest) expected = b"Hello world !\n More conte" assert storage.read(manifest, 25, 0) == expected assert manifest.is_reshaped() (chunk10, ), (chunk11, ) = manifest.blocks assert storage[chunk10.id] == b"Hello world !\n M" assert storage[chunk11.id] == b"ore conte" assert manifest == base.evolve(size=25, blocks=((chunk10, ), (chunk11, )), updated=t7)
async def file_conflict( self, entry_id: EntryID, local_manifest: Union[LocalFolderManifest, LocalFileManifest], remote_manifest: BaseRemoteManifest, ) -> None: # This is the only transaction that affects more than one manifests # That's because the local version of the file has to be registered in the # parent as a new child while the remote version has to be set as the actual # version. In practice, this should not be an issue. # Lock parent then child parent_id = local_manifest.parent async with self.local_storage.lock_manifest( parent_id) as parent_manifest: # Not a folderish manifest if not isinstance(parent_manifest, (LocalFolderManifest, LocalWorkspaceManifest)): raise FSNotADirectoryError(parent_id) async with self.local_storage.lock_manifest( entry_id) as current_manifest: # Not a file manifest if not isinstance(current_manifest, LocalFileManifest): raise FSIsADirectoryError(entry_id) # Make sure the file still exists filename = get_filename(parent_manifest, entry_id) if filename is None: return # Copy blocks new_blocks = [] for chunks in current_manifest.blocks: new_chunks = [] for chunk in chunks: data = await self.local_storage.get_chunk(chunk.id) new_chunk = Chunk.new(chunk.start, chunk.stop) await self.local_storage.set_chunk(new_chunk.id, data) if len(chunks) == 1: new_chunk = new_chunk.evolve_as_block(data) new_chunks.append(chunk) new_blocks.append(tuple(new_chunks)) # Prepare prevent_sync_pattern = self.local_storage.get_prevent_sync_pattern( ) new_name = get_conflict_filename( filename, list(parent_manifest.children), remote_manifest.author) new_manifest = LocalFileManifest.new_placeholder( self.local_author, parent=parent_id).evolve(size=current_manifest.size, blocks=tuple(new_blocks)) new_parent_manifest = parent_manifest.evolve_children_and_mark_updated( {new_name: new_manifest.id}, prevent_sync_pattern=prevent_sync_pattern) other_manifest = BaseLocalManifest.from_remote( remote_manifest, prevent_sync_pattern=prevent_sync_pattern) # Set manifests await self.local_storage.set_manifest(new_manifest.id, new_manifest, check_lock_status=False) await self.local_storage.set_manifest(parent_id, new_parent_manifest) await self.local_storage.set_manifest(entry_id, other_manifest) self._send_event(CoreEvent.FS_ENTRY_UPDATED, id=new_manifest.id) self._send_event(CoreEvent.FS_ENTRY_UPDATED, id=parent_id) self._send_event( CoreEvent.FS_ENTRY_FILE_CONFLICT_RESOLVED, id=entry_id, backup_id=new_manifest.id, )