async def test_basic_set_get_clear(data_base_dir, alice_workspace_storage):
aws = alice_workspace_storage
manifest = create_manifest(aws.device)
async with aws.lock_entry_id(manifest.id):
# 1) No data
with pytest.raises(FSLocalMissError):
await aws.get_manifest(manifest.id)
# 2) Set data
await aws.set_manifest(manifest.id, manifest)
assert await aws.get_manifest(manifest.id) == manifest
# Make sure data are not only stored in cache
async with WorkspaceStorage.run(data_base_dir, aws.device,
aws.workspace_id) as aws2:
assert await aws2.get_manifest(manifest.id) == manifest
# 3) Clear data
await aws.clear_manifest(manifest.id)
with pytest.raises(FSLocalMissError):
await aws.get_manifest(manifest.id)
with pytest.raises(FSLocalMissError):
await aws.clear_manifest(manifest.id)
async with WorkspaceStorage.run(data_base_dir, aws.device,
aws.workspace_id) as aws3:
with pytest.raises(FSLocalMissError):
assert await aws3.get_manifest(manifest.id) == manifest
async def test_cache_set_get(tmpdir, alice, workspace_id):
manifest = create_manifest(alice)
async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws:
async with aws.lock_entry_id(manifest.id):
# 1) Set data
await aws.set_manifest(manifest.id, manifest, cache_only=True)
assert await aws.get_manifest(manifest.id) == manifest
async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws2:
with pytest.raises(FSLocalMissError):
await aws2.get_manifest(manifest.id)
# 2) Clear should work as expected
await aws.clear_manifest(manifest.id)
with pytest.raises(FSLocalMissError):
await aws.get_manifest(manifest.id)
# 3) Re-set data
await aws.set_manifest(manifest.id, manifest, cache_only=True)
assert await aws.get_manifest(manifest.id) == manifest
async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws3:
with pytest.raises(FSLocalMissError):
await aws3.get_manifest(manifest.id)
# 4) Flush data
await aws.ensure_manifest_persistent(manifest.id)
assert await aws.get_manifest(manifest.id) == manifest
async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws4:
assert await aws4.get_manifest(manifest.id) == manifest
# 5) Idempotency
await aws.ensure_manifest_persistent(manifest.id)
async def test_serialize_types(tmpdir, alice, workspace_id, type):
manifest = create_manifest(alice, type)
async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws:
async with aws.lock_entry_id(manifest.id):
await aws.set_manifest(manifest.id, manifest)
async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws2:
assert await aws2.get_manifest(manifest.id) == manifest
async def test_cache_flushed_on_exit(tmpdir, alice, workspace_id):
manifest = create_manifest(alice)
async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws:
async with aws.lock_entry_id(manifest.id):
await aws.set_manifest(manifest.id, manifest, cache_only=True)
async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws2:
assert await aws2.get_manifest(manifest.id) == manifest
async def test_serialize_non_empty_local_file_manifest(tmpdir, alice, workspace_id):
manifest = create_manifest(alice, LocalFileManifest)
chunk1 = Chunk.new(0, 7).evolve_as_block(b"0123456")
chunk2 = Chunk.new(7, 8)
chunk3 = Chunk.new(8, 10)
blocks = (chunk1, chunk2), (chunk3,)
manifest = manifest.evolve_and_mark_updated(blocksize=8, size=10, blocks=blocks)
manifest.assert_integrity()
async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws:
async with aws.lock_entry_id(manifest.id):
await aws.set_manifest(manifest.id, manifest)
async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws2:
assert await aws2.get_manifest(manifest.id) == manifest
async def test_deserialize_legacy_types(tmpdir, alice, workspace_id, type):
# In parsec < 1.15, the author field used to be None for placeholders
# That means those manifests can still exist in the local storage
# However, they should not appear anywhere in the new code bases
# Create legacy manifests to dump and save them in the local storage
manifest = create_manifest(alice, type, use_legacy_none_author=True)
async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws:
async with aws.lock_entry_id(manifest.id):
await aws.set_manifest(manifest.id, manifest)
# Make sure they come out with the author field set to LOCAL_AUTHOR_LEGACY_PLACEHOLDER
expected_base = manifest.base.evolve(author=LOCAL_AUTHOR_LEGACY_PLACEHOLDER)
expected = manifest.evolve(base=expected_base)
async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws2:
assert await aws2.get_manifest(manifest.id) == expected
async def _transactions_controlled_cb(started_cb):
async with WorkspaceStorage.run(alice, Path("/dummy"),
EntryID()) as local_storage:
file_transactions = await file_transactions_factory(
self.device,
alice_backend_cmds,
local_storage=local_storage)
await started_cb(file_transactions=file_transactions)
async def workspace_storage_task(
task_status: TaskStatus[WorkspaceStorage] = trio.
TASK_STATUS_IGNORED
) -> None:
async with WorkspaceStorage.run(self.device, path,
workspace_id) as workspace_storage:
task_status.started(workspace_storage)
await trio.sleep_forever()
async def _transactions_controlled_cb(started_cb):
async with WorkspaceStorage.run(
tmp_path / f"file_operations-{tentative}", alice,
EntryID.new()) as local_storage:
async with file_transactions_factory(
self.device,
local_storage=local_storage) as file_transactions:
await started_cb(file_transactions=file_transactions)
async def test_internal_connections(tmpdir, alice, workspace_id):
async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws:
with pytest.raises(RuntimeError):
await aws.data_localdb._connect()
with pytest.raises(RuntimeError):
await aws.cache_localdb._connect()
# Idempotency
await aws.data_localdb._close()
await aws.cache_localdb._close()
async def test_storage_file_tree(alice, tmpdir, workspace_id):
path = Path(tmpdir)
manifest_sqlite_db = path / "workspace_data-v1.sqlite"
chunk_sqlite_db = path / "workspace_data-v1.sqlite"
block_sqlite_db = path / "workspace_cache-v1.sqlite"
async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws:
assert aws.manifest_storage.path == manifest_sqlite_db
assert aws.chunk_storage.path == chunk_sqlite_db
assert aws.block_storage.path == block_sqlite_db
assert set(path.iterdir()) == {manifest_sqlite_db, chunk_sqlite_db, block_sqlite_db}
async def workspace_storage_task(
task_status: TaskStatus[WorkspaceStorage] = trio.
TASK_STATUS_IGNORED
) -> None:
async with WorkspaceStorage.run(
data_base_dir=self.data_base_dir,
device=self.device,
workspace_id=workspace_id,
cache_size=self.workspace_storage_cache_size,
) as workspace_storage:
task_status.started(workspace_storage)
await trio.sleep_forever()
async def test_lock_manifest(data_base_dir, alice, workspace_id):
manifest = create_manifest(alice, LocalFileManifest)
async with WorkspaceStorage.run(data_base_dir, alice, workspace_id) as aws:
with pytest.raises(FSLocalMissError):
async with aws.lock_manifest(manifest.id):
pass
await aws.set_manifest(manifest.id, manifest, check_lock_status=False)
async with aws.lock_manifest(manifest.id) as m1:
assert m1 == manifest
m2 = manifest.evolve(need_sync=False)
await aws.set_manifest(manifest.id, m2)
assert await aws.get_manifest(manifest.id) == m2
async def test_storage_file_tree(data_base_dir, alice, workspace_id):
manifest_sqlite_db = data_base_dir / alice.slug / str(
workspace_id) / "workspace_data-v1.sqlite"
chunk_sqlite_db = data_base_dir / alice.slug / str(
workspace_id) / "workspace_data-v1.sqlite"
block_sqlite_db = data_base_dir / alice.slug / str(
workspace_id) / "workspace_cache-v1.sqlite"
async with WorkspaceStorage.run(data_base_dir, alice, workspace_id) as aws:
assert aws.manifest_storage.path == manifest_sqlite_db
assert aws.chunk_storage.path == chunk_sqlite_db
assert aws.block_storage.path == block_sqlite_db
assert manifest_sqlite_db.is_file()
assert chunk_sqlite_db.is_file()
assert block_sqlite_db.is_file()
async def test_garbage_collection(tmpdir, alice, workspace_id):
block_size = DEFAULT_BLOCK_SIZE
cache_size = 1 * block_size
data = b"\x00" * block_size
chunk1 = Chunk.new(0, block_size).evolve_as_block(data)
chunk2 = Chunk.new(0, block_size).evolve_as_block(data)
chunk3 = Chunk.new(0, block_size).evolve_as_block(data)
async with WorkspaceStorage.run(alice, tmpdir, workspace_id, cache_size=cache_size) as aws:
assert await aws.block_storage.get_nb_blocks() == 0
await aws.set_clean_block(chunk1.access.id, data)
assert await aws.block_storage.get_nb_blocks() == 1
await aws.set_clean_block(chunk2.access.id, data)
assert await aws.block_storage.get_nb_blocks() == 1
await aws.set_clean_block(chunk3.access.id, data)
assert await aws.block_storage.get_nb_blocks() == 1
await aws.block_storage.clear_all_blocks()
assert await aws.block_storage.get_nb_blocks() == 0
async def test_vacuum(data_base_dir, alice, workspace_id):
data_size = 1 * 1024 * 1024
chunk = Chunk.new(0, data_size)
async with WorkspaceStorage.run(data_base_dir,
alice,
workspace_id,
data_vacuum_threshold=data_size //
2) as aws:
# Make sure the storage is empty
data = b"\x00" * data_size
assert await aws.data_localdb.get_disk_usage() < data_size
# Set and commit a chunk of 1MB
await aws.set_chunk(chunk.id, data)
await aws.data_localdb.commit()
assert await aws.data_localdb.get_disk_usage() > data_size
# Run the vacuum
await aws.run_vacuum()
assert await aws.data_localdb.get_disk_usage() > data_size
# Clear the chunk 1MB
await aws.clear_chunk(chunk.id)
await aws.data_localdb.commit()
assert await aws.data_localdb.get_disk_usage() > data_size
# Run the vacuum
await aws.run_vacuum()
assert await aws.data_localdb.get_disk_usage() < data_size
# Make sure vacuum can run even if a transaction has started
await aws.set_chunk(chunk.id, data)
await aws.run_vacuum()
await aws.clear_chunk(chunk.id)
await aws.run_vacuum()
# Vacuuming the cache storage is no-op
await aws.cache_localdb.run_vacuum()
# Make sure disk usage can be called on a closed storage
assert await aws.data_localdb.get_disk_usage() < data_size
async def alice_workspace_storage(data_base_dir, alice, workspace_id):
async with WorkspaceStorage.run(data_base_dir, alice, workspace_id) as aws:
yield aws
async def alice_workspace_storage(tmpdir, alice, workspace_id):
async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws:
yield aws
async def alice_transaction_local_storage(alice, persistent_mockup):
async with WorkspaceStorage.run(alice, Path("/dummy"),
EntryID.new()) as storage:
yield storage
async def alice_transaction_local_storage(alice, tmp_path):
async with WorkspaceStorage.run(
tmp_path / "alice_transaction_local_storage", alice,
EntryID.new()) as storage:
yield storage