async def test_load_block_from_remote(running_backend, alice_file_transactions,
foo_txt):
file_transactions = alice_file_transactions
# Prepare the backend
workspace_id = file_transactions.remote_loader.workspace_id
await file_transactions.remote_loader.create_realm(workspace_id)
foo_manifest = await foo_txt.get_manifest()
chunk1_data = b"a" * 10
chunk2_data = b"b" * 5
chunk1 = Chunk.new(0, 10).evolve_as_block(chunk1_data)
chunk2 = Chunk.new(10, 15).evolve_as_block(chunk2_data)
foo_manifest = foo_manifest.evolve(blocks=((chunk1, chunk2), ), size=15)
await foo_txt.set_manifest(foo_manifest)
fd = foo_txt.open()
await file_transactions.remote_loader.upload_block(chunk1.access,
chunk1_data)
await file_transactions.remote_loader.upload_block(chunk2.access,
chunk2_data)
await file_transactions.local_storage.clear_clean_block(chunk1.access.id)
await file_transactions.local_storage.clear_clean_block(chunk2.access.id)
data = await file_transactions.fd_read(fd, 14, 0)
assert data == chunk1_data + chunk2_data[:4]
async def test_serialize_non_empty_local_file_manifest(tmpdir, alice, workspace_id):
manifest = create_manifest(alice, LocalFileManifest)
chunk1 = Chunk.new(0, 7).evolve_as_block(b"0123456")
chunk2 = Chunk.new(7, 8)
chunk3 = Chunk.new(8, 10)
blocks = (chunk1, chunk2), (chunk3,)
manifest = manifest.evolve_and_mark_updated(blocksize=8, size=10, blocks=blocks)
manifest.assert_integrity()
async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws:
async with aws.lock_entry_id(manifest.id):
await aws.set_manifest(manifest.id, manifest)
async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws2:
assert await aws2.get_manifest(manifest.id) == manifest
async def test_chunk_clearing(alice_workspace_storage, cache_only,
clear_manifest):
aws = alice_workspace_storage
manifest = create_manifest(aws.device, LocalFileManifest)
data1 = b"abc"
chunk1 = Chunk.new(0, 3)
data2 = b"def"
chunk2 = Chunk.new(3, 6)
manifest = manifest.evolve(blocks=((chunk1, chunk2), ), size=6)
async with aws.lock_entry_id(manifest.id):
# Set chunks and manifests
await aws.set_chunk(chunk1.id, data1)
await aws.set_chunk(chunk2.id, data2)
await aws.set_manifest(manifest.id, manifest)
# Set a new version of the manifest without the chunks
removed_ids = {chunk1.id, chunk2.id}
new_manifest = manifest.evolve(blocks=())
await aws.set_manifest(manifest.id,
new_manifest,
cache_only=cache_only,
removed_ids=removed_ids)
# The chunks are still accessible
if cache_only:
assert await aws.get_chunk(chunk1.id) == b"abc"
assert await aws.get_chunk(chunk2.id) == b"def"
# The chunks are gone
else:
with pytest.raises(FSLocalMissError):
await aws.get_chunk(chunk1.id)
with pytest.raises(FSLocalMissError):
await aws.get_chunk(chunk2.id)
# Now flush the manifest
if clear_manifest:
await aws.clear_manifest(manifest.id)
else:
await aws.ensure_manifest_persistent(manifest.id)
# The chunks are gone
with pytest.raises(FSLocalMissError):
await aws.get_chunk(chunk1.id)
with pytest.raises(FSLocalMissError):
await aws.get_chunk(chunk2.id)
# Idempotency
await aws.manifest_storage._ensure_manifest_persistent(manifest.id)
async def test_chunk_interface(alice_workspace_storage):
data = b"0123456"
aws = alice_workspace_storage
chunk = Chunk.new(0, 7)
with pytest.raises(FSLocalMissError):
await aws.get_chunk(chunk.id)
with pytest.raises(FSLocalMissError):
await aws.clear_chunk(chunk.id)
await aws.clear_chunk(chunk.id, miss_ok=True)
assert not await aws.chunk_storage.is_chunk(chunk.id)
assert await aws.chunk_storage.get_total_size() == 0
await aws.set_chunk(chunk.id, data)
assert await aws.get_chunk(chunk.id) == data
assert await aws.chunk_storage.is_chunk(chunk.id)
assert await aws.chunk_storage.get_total_size() >= 7
await aws.clear_chunk(chunk.id)
with pytest.raises(FSLocalMissError):
await aws.get_chunk(chunk.id)
with pytest.raises(FSLocalMissError):
await aws.clear_chunk(chunk.id)
assert not await aws.chunk_storage.is_chunk(chunk.id)
assert await aws.chunk_storage.get_total_size() == 0
await aws.clear_chunk(chunk.id, miss_ok=True)
async def test_block_interface(alice_workspace_storage):
data = b"0123456"
aws = alice_workspace_storage
chunk = Chunk.new(0, 7).evolve_as_block(data)
block_id = chunk.access.id
await aws.clear_clean_block(block_id)
with pytest.raises(FSLocalMissError):
await aws.get_chunk(chunk.id)
assert not await aws.block_storage.is_chunk(chunk.id)
assert await aws.block_storage.get_total_size() == 0
await aws.set_clean_block(block_id, data)
assert await aws.get_chunk(chunk.id) == data
assert await aws.block_storage.is_chunk(chunk.id)
assert await aws.block_storage.get_total_size() >= 7
await aws.clear_clean_block(block_id)
await aws.clear_clean_block(block_id)
with pytest.raises(FSLocalMissError):
await aws.get_chunk(chunk.id)
assert not await aws.block_storage.is_chunk(chunk.id)
assert await aws.block_storage.get_total_size() == 0
await aws.set_chunk(chunk.id, data)
assert await aws.get_dirty_block(block_id) == data
def prepare_reshape(
manifest: LocalFileManifest
) -> Iterator[Tuple[Chunks, Chunk, Callable, Set[BlockID]]]:
# Update manifest
def update_manifest(block: int, manifest: LocalFileManifest,
new_chunk: Chunk) -> LocalFileManifest:
blocks = list(manifest.blocks)
blocks[block] = (new_chunk, )
return manifest.evolve(blocks=tuple(blocks))
# Loop over blocks
for block, chunks in enumerate(manifest.blocks):
# Already a block
if len(chunks) == 1 and chunks[0].is_block:
continue
# Update callback
block_update = partial(update_manifest, block)
# Already a pseudo-block
if len(chunks) == 1 and chunks[0].is_pseudo_block:
yield (chunks, chunks[0], block_update, set())
continue
# Prepare new block
start, stop = chunks[0].start, chunks[-1].stop
new_chunk = Chunk.new(start, stop)
# Cleanup
removed_ids = chunk_id_set(chunks)
# Yield operations
yield (chunks, new_chunk, block_update, removed_ids)
def prepare_reshape(
manifest: LocalFileManifest
) -> Iterator[Tuple[int, Chunks, Chunk, bool, ChunkIDSet]]:
# Loop over blocks
for block, chunks in enumerate(manifest.blocks):
# Already a block
if len(chunks) == 1 and chunks[0].is_block():
continue
# Already a pseudo-block
if len(chunks) == 1 and chunks[0].is_pseudo_block():
yield (block, chunks, chunks[0], False, set())
continue
# Prepare new block
start, stop = chunks[0].start, chunks[-1].stop
new_chunk = Chunk.new(start, stop)
# Cleanup
removed_ids = chunk_id_set(chunks)
# Yield operations
yield (block, chunks, new_chunk, True, removed_ids)
async def test_garbage_collection(tmpdir, alice, workspace_id):
block_size = DEFAULT_BLOCK_SIZE
cache_size = 1 * block_size
data = b"\x00" * block_size
chunk1 = Chunk.new(0, block_size).evolve_as_block(data)
chunk2 = Chunk.new(0, block_size).evolve_as_block(data)
chunk3 = Chunk.new(0, block_size).evolve_as_block(data)
async with WorkspaceStorage.run(alice, tmpdir, workspace_id, cache_size=cache_size) as aws:
assert await aws.block_storage.get_nb_blocks() == 0
await aws.set_clean_block(chunk1.access.id, data)
assert await aws.block_storage.get_nb_blocks() == 1
await aws.set_clean_block(chunk2.access.id, data)
assert await aws.block_storage.get_nb_blocks() == 1
await aws.set_clean_block(chunk3.access.id, data)
assert await aws.block_storage.get_nb_blocks() == 1
await aws.block_storage.clear_all_blocks()
assert await aws.block_storage.get_nb_blocks() == 0
async def file_conflict(
self, entry_id: EntryID, local_manifest: LocalManifest, remote_manifest: RemoteManifest
) -> None:
# This is the only transaction that affects more than one manifests
# That's because the local version of the file has to be registered in the
# parent as a new child while the remote version has to be set as the actual
# version. In practice, this should not be an issue.
# Lock parent then child
parent_id = local_manifest.parent
async with self.local_storage.lock_manifest(parent_id) as parent_manifest:
async with self.local_storage.lock_manifest(entry_id) as current_manifest:
# Make sure the file still exists
filename = get_filename(parent_manifest, entry_id)
if filename is None:
return
# Copy blocks
new_blocks = []
for chunks in current_manifest.blocks:
new_chunks = []
for chunk in chunks:
data = await self.local_storage.get_chunk(chunk.id)
new_chunk = Chunk.new(chunk.start, chunk.stop)
await self.local_storage.set_chunk(new_chunk.id, data)
if len(chunks) == 1:
new_chunk = new_chunk.evolve_as_block(data)
new_chunks.append(chunk)
new_blocks.append(tuple(new_chunks))
new_blocks = tuple(new_blocks)
# Prepare
new_name = get_conflict_filename(
filename, list(parent_manifest.children), remote_manifest.author
)
new_manifest = LocalFileManifest.new_placeholder(parent=parent_id).evolve(
size=current_manifest.size, blocks=new_blocks
)
new_parent_manifest = parent_manifest.evolve_children_and_mark_updated(
{new_name: new_manifest.id}
)
other_manifest = LocalManifest.from_remote(remote_manifest)
# Set manifests
await self.local_storage.set_manifest(
new_manifest.id, new_manifest, check_lock_status=False
)
await self.local_storage.set_manifest(parent_id, new_parent_manifest)
await self.local_storage.set_manifest(entry_id, other_manifest)
self._send_event("fs.entry.updated", id=new_manifest.id)
self._send_event("fs.entry.updated", id=parent_id)
self._send_event(
"fs.entry.file_conflict_resolved", id=entry_id, backup_id=new_manifest.id
)
async def test_block_not_loaded_entry(alice_file_transactions, foo_txt):
file_transactions = alice_file_transactions
foo_manifest = await foo_txt.get_manifest()
chunk1_data = b"a" * 10
chunk2_data = b"b" * 5
chunk1 = Chunk.new(0, 10).evolve_as_block(chunk1_data)
chunk2 = Chunk.new(10, 15).evolve_as_block(chunk2_data)
foo_manifest = foo_manifest.evolve(blocks=((chunk1, chunk2), ), size=15)
async with file_transactions.local_storage.lock_entry_id(
foo_manifest.parent):
await foo_txt.set_manifest(foo_manifest)
fd = foo_txt.open()
with pytest.raises(FSRemoteBlockNotFound):
await file_transactions.fd_read(fd, 14, 0)
await file_transactions.local_storage.set_chunk(chunk1.id, chunk1_data)
await file_transactions.local_storage.set_chunk(chunk2.id, chunk2_data)
data = await file_transactions.fd_read(fd, 14, 0)
assert data == chunk1_data + chunk2_data[:4]
async def test_chunk_many(alice_workspace_storage):
data = b"0123456"
aws = alice_workspace_storage
# More than the sqLite max argument limit to prevent regression
chunks_number = 2000
chunks = []
for i in range(chunks_number):
c = Chunk.new(0, 7)
chunks.append(c.id)
await aws.chunk_storage.set_chunk(c.id, data)
assert len(chunks) == chunks_number
ret = await aws.get_local_chunk_ids(chunks)
for i in range(len(ret)):
assert ret[i]
assert len(ret) == chunks_number
def prepare_write(
manifest: LocalFileManifest, size: int, offset: int, timestamp: DateTime
) -> Tuple[LocalFileManifest, WriteOperationList, ChunkIDSet]:
# Prepare
padding = 0
removed_ids: ChunkIDSet = set()
write_operations: WriteOperationList = []
# Padding
if offset > manifest.size:
padding = offset - manifest.size
size += padding
offset = manifest.size
# Copy buffers
blocks = list(manifest.blocks)
# Loop over blocks
for block, subsize, start, content_offset in split_write(
size, offset, manifest.blocksize):
# Prepare new chunk
new_chunk = Chunk.new(start, start + subsize)
write_operations.append((new_chunk, content_offset - padding))
# Lazy block write
chunks = manifest.get_chunks(block)
new_chunks, more_removed_ids = block_write(chunks, subsize, start,
new_chunk)
# Update data structures
removed_ids |= more_removed_ids
if len(blocks) == block:
blocks.append(new_chunks)
else:
blocks[block] = new_chunks
# Evolve manifest
new_size = max(manifest.size, offset + size)
new_manifest = manifest.evolve_and_mark_updated(size=new_size,
blocks=tuple(blocks),
timestamp=timestamp)
# Return write result
return new_manifest, write_operations, removed_ids
async def test_vacuum(data_base_dir, alice, workspace_id):
data_size = 1 * 1024 * 1024
chunk = Chunk.new(0, data_size)
async with WorkspaceStorage.run(data_base_dir,
alice,
workspace_id,
data_vacuum_threshold=data_size //
2) as aws:
# Make sure the storage is empty
data = b"\x00" * data_size
assert await aws.data_localdb.get_disk_usage() < data_size
# Set and commit a chunk of 1MB
await aws.set_chunk(chunk.id, data)
await aws.data_localdb.commit()
assert await aws.data_localdb.get_disk_usage() > data_size
# Run the vacuum
await aws.run_vacuum()
assert await aws.data_localdb.get_disk_usage() > data_size
# Clear the chunk 1MB
await aws.clear_chunk(chunk.id)
await aws.data_localdb.commit()
assert await aws.data_localdb.get_disk_usage() > data_size
# Run the vacuum
await aws.run_vacuum()
assert await aws.data_localdb.get_disk_usage() < data_size
# Make sure vacuum can run even if a transaction has started
await aws.set_chunk(chunk.id, data)
await aws.run_vacuum()
await aws.clear_chunk(chunk.id)
await aws.run_vacuum()
# Vacuuming the cache storage is no-op
await aws.cache_localdb.run_vacuum()
# Make sure disk usage can be called on a closed storage
assert await aws.data_localdb.get_disk_usage() < data_size
def test_complete_scenario():
storage = Storage()
with freeze_time("2000-01-01"):
base = manifest = LocalFileManifest.new_placeholder(parent=EntryID(),
blocksize=16)
assert manifest == base.evolve(size=0)
with freeze_time("2000-01-02") as t2:
manifest = storage.write(manifest, b"Hello ", 0)
assert storage.read(manifest, 6, 0) == b"Hello "
(chunk0, ), = manifest.blocks
assert manifest == base.evolve(size=6, blocks=((chunk0, ), ), updated=t2)
assert chunk0 == Chunk(id=chunk0.id,
start=0,
stop=6,
raw_offset=0,
raw_size=6,
access=None)
assert storage[chunk0.id] == b"Hello "
with freeze_time("2000-01-03") as t3:
manifest = storage.write(manifest, b"world !", 6)
assert storage.read(manifest, 13, 0) == b"Hello world !"
(_, chunk1), = manifest.blocks
assert manifest == base.evolve(size=13,
blocks=((chunk0, chunk1), ),
updated=t3)
assert chunk1 == Chunk(id=chunk1.id,
start=6,
stop=13,
raw_offset=6,
raw_size=7,
access=None)
assert storage[chunk1.id] == b"world !"
with freeze_time("2000-01-04") as t4:
manifest = storage.write(manifest, b"\n More kontent", 13)
assert storage.read(manifest, 27, 0) == b"Hello world !\n More kontent"
(_, _, chunk2), (chunk3, ) = manifest.blocks
assert storage[chunk2.id] == b"\n M"
assert storage[chunk3.id] == b"ore kontent"
assert manifest == base.evolve(size=27,
blocks=((chunk0, chunk1, chunk2),
(chunk3, )),
updated=t4)
with freeze_time("2000-01-05") as t5:
manifest = storage.write(manifest, b"c", 20)
assert storage.read(manifest, 27, 0) == b"Hello world !\n More content"
chunk4, chunk5, chunk6 = manifest.blocks[1]
assert chunk3.id == chunk4.id == chunk6.id
assert storage[chunk5.id] == b"c"
assert manifest == base.evolve(size=27,
blocks=((chunk0, chunk1, chunk2),
(chunk4, chunk5, chunk6)),
updated=t5)
with freeze_time("2000-01-06") as t6:
manifest = storage.resize(manifest, 40)
expected = b"Hello world !\n More content" + b"\x00" * 13
assert storage.read(manifest, 40, 0) == expected
(_, _, _, chunk7), (chunk8, ) = manifest.blocks[1:]
assert storage[chunk7.id] == b"\x00" * 5
assert storage[chunk8.id] == b"\x00" * 8
assert manifest == base.evolve(
size=40,
blocks=((chunk0, chunk1, chunk2), (chunk4, chunk5, chunk6, chunk7),
(chunk8, )),
updated=t6,
)
with freeze_time("2000-01-07") as t7:
manifest = storage.resize(manifest, 25)
expected = b"Hello world !\n More conte"
assert storage.read(manifest, 25, 0) == expected
(_, _, chunk9), = manifest.blocks[1:]
assert chunk9.id == chunk6.id
assert manifest == base.evolve(size=25,
blocks=((chunk0, chunk1, chunk2),
(chunk4, chunk5, chunk9)),
updated=t7)
with freeze_time("2000-01-08"):
assert not manifest.is_reshaped()
manifest = storage.reshape(manifest)
expected = b"Hello world !\n More conte"
assert storage.read(manifest, 25, 0) == expected
assert manifest.is_reshaped()
(chunk10, ), (chunk11, ) = manifest.blocks
assert storage[chunk10.id] == b"Hello world !\n M"
assert storage[chunk11.id] == b"ore conte"
assert manifest == base.evolve(size=25,
blocks=((chunk10, ), (chunk11, )),
updated=t7)
def evolve(m, n):
chunk = Chunk.new(0, n).evolve_as_block(b"a" * n)
blocks = ((chunk, ), )
return m1.evolve_and_mark_updated(size=n, blocks=blocks)
async def file_conflict(
self,
entry_id: EntryID,
local_manifest: Union[LocalFolderManifest, LocalFileManifest],
remote_manifest: BaseRemoteManifest,
) -> None:
# This is the only transaction that affects more than one manifests
# That's because the local version of the file has to be registered in the
# parent as a new child while the remote version has to be set as the actual
# version. In practice, this should not be an issue.
# Lock parent then child
parent_id = local_manifest.parent
async with self.local_storage.lock_manifest(
parent_id) as parent_manifest:
# Not a folderish manifest
if not isinstance(parent_manifest,
(LocalFolderManifest, LocalWorkspaceManifest)):
raise FSNotADirectoryError(parent_id)
async with self.local_storage.lock_manifest(
entry_id) as current_manifest:
# Not a file manifest
if not isinstance(current_manifest, LocalFileManifest):
raise FSIsADirectoryError(entry_id)
# Make sure the file still exists
filename = get_filename(parent_manifest, entry_id)
if filename is None:
return
# Copy blocks
new_blocks = []
for chunks in current_manifest.blocks:
new_chunks = []
for chunk in chunks:
data = await self.local_storage.get_chunk(chunk.id)
new_chunk = Chunk.new(chunk.start, chunk.stop)
await self.local_storage.set_chunk(new_chunk.id, data)
if len(chunks) == 1:
new_chunk = new_chunk.evolve_as_block(data)
new_chunks.append(chunk)
new_blocks.append(tuple(new_chunks))
# Prepare
prevent_sync_pattern = self.local_storage.get_prevent_sync_pattern(
)
new_name = get_conflict_filename(
filename, list(parent_manifest.children),
remote_manifest.author)
new_manifest = LocalFileManifest.new_placeholder(
self.local_author,
parent=parent_id).evolve(size=current_manifest.size,
blocks=tuple(new_blocks))
new_parent_manifest = parent_manifest.evolve_children_and_mark_updated(
{new_name: new_manifest.id},
prevent_sync_pattern=prevent_sync_pattern)
other_manifest = BaseLocalManifest.from_remote(
remote_manifest, prevent_sync_pattern=prevent_sync_pattern)
# Set manifests
await self.local_storage.set_manifest(new_manifest.id,
new_manifest,
check_lock_status=False)
await self.local_storage.set_manifest(parent_id,
new_parent_manifest)
await self.local_storage.set_manifest(entry_id, other_manifest)
self._send_event(CoreEvent.FS_ENTRY_UPDATED,
id=new_manifest.id)
self._send_event(CoreEvent.FS_ENTRY_UPDATED, id=parent_id)
self._send_event(
CoreEvent.FS_ENTRY_FILE_CONFLICT_RESOLVED,
id=entry_id,
backup_id=new_manifest.id,
)