async def test_basic_set_get_clear(data_base_dir, alice_workspace_storage): aws = alice_workspace_storage manifest = create_manifest(aws.device) async with aws.lock_entry_id(manifest.id): # 1) No data with pytest.raises(FSLocalMissError): await aws.get_manifest(manifest.id) # 2) Set data await aws.set_manifest(manifest.id, manifest) assert await aws.get_manifest(manifest.id) == manifest # Make sure data are not only stored in cache async with WorkspaceStorage.run(data_base_dir, aws.device, aws.workspace_id) as aws2: assert await aws2.get_manifest(manifest.id) == manifest # 3) Clear data await aws.clear_manifest(manifest.id) with pytest.raises(FSLocalMissError): await aws.get_manifest(manifest.id) with pytest.raises(FSLocalMissError): await aws.clear_manifest(manifest.id) async with WorkspaceStorage.run(data_base_dir, aws.device, aws.workspace_id) as aws3: with pytest.raises(FSLocalMissError): assert await aws3.get_manifest(manifest.id) == manifest
async def test_cache_set_get(tmpdir, alice, workspace_id): manifest = create_manifest(alice) async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws: async with aws.lock_entry_id(manifest.id): # 1) Set data await aws.set_manifest(manifest.id, manifest, cache_only=True) assert await aws.get_manifest(manifest.id) == manifest async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws2: with pytest.raises(FSLocalMissError): await aws2.get_manifest(manifest.id) # 2) Clear should work as expected await aws.clear_manifest(manifest.id) with pytest.raises(FSLocalMissError): await aws.get_manifest(manifest.id) # 3) Re-set data await aws.set_manifest(manifest.id, manifest, cache_only=True) assert await aws.get_manifest(manifest.id) == manifest async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws3: with pytest.raises(FSLocalMissError): await aws3.get_manifest(manifest.id) # 4) Flush data await aws.ensure_manifest_persistent(manifest.id) assert await aws.get_manifest(manifest.id) == manifest async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws4: assert await aws4.get_manifest(manifest.id) == manifest # 5) Idempotency await aws.ensure_manifest_persistent(manifest.id)
async def test_serialize_types(tmpdir, alice, workspace_id, type): manifest = create_manifest(alice, type) async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws: async with aws.lock_entry_id(manifest.id): await aws.set_manifest(manifest.id, manifest) async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws2: assert await aws2.get_manifest(manifest.id) == manifest
async def test_cache_flushed_on_exit(tmpdir, alice, workspace_id): manifest = create_manifest(alice) async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws: async with aws.lock_entry_id(manifest.id): await aws.set_manifest(manifest.id, manifest, cache_only=True) async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws2: assert await aws2.get_manifest(manifest.id) == manifest
async def test_serialize_non_empty_local_file_manifest(tmpdir, alice, workspace_id): manifest = create_manifest(alice, LocalFileManifest) chunk1 = Chunk.new(0, 7).evolve_as_block(b"0123456") chunk2 = Chunk.new(7, 8) chunk3 = Chunk.new(8, 10) blocks = (chunk1, chunk2), (chunk3,) manifest = manifest.evolve_and_mark_updated(blocksize=8, size=10, blocks=blocks) manifest.assert_integrity() async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws: async with aws.lock_entry_id(manifest.id): await aws.set_manifest(manifest.id, manifest) async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws2: assert await aws2.get_manifest(manifest.id) == manifest
async def test_deserialize_legacy_types(tmpdir, alice, workspace_id, type): # In parsec < 1.15, the author field used to be None for placeholders # That means those manifests can still exist in the local storage # However, they should not appear anywhere in the new code bases # Create legacy manifests to dump and save them in the local storage manifest = create_manifest(alice, type, use_legacy_none_author=True) async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws: async with aws.lock_entry_id(manifest.id): await aws.set_manifest(manifest.id, manifest) # Make sure they come out with the author field set to LOCAL_AUTHOR_LEGACY_PLACEHOLDER expected_base = manifest.base.evolve(author=LOCAL_AUTHOR_LEGACY_PLACEHOLDER) expected = manifest.evolve(base=expected_base) async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws2: assert await aws2.get_manifest(manifest.id) == expected
async def _transactions_controlled_cb(started_cb): async with WorkspaceStorage.run(alice, Path("/dummy"), EntryID()) as local_storage: file_transactions = await file_transactions_factory( self.device, alice_backend_cmds, local_storage=local_storage) await started_cb(file_transactions=file_transactions)
async def workspace_storage_task( task_status: TaskStatus[WorkspaceStorage] = trio. TASK_STATUS_IGNORED ) -> None: async with WorkspaceStorage.run(self.device, path, workspace_id) as workspace_storage: task_status.started(workspace_storage) await trio.sleep_forever()
async def _transactions_controlled_cb(started_cb): async with WorkspaceStorage.run( tmp_path / f"file_operations-{tentative}", alice, EntryID.new()) as local_storage: async with file_transactions_factory( self.device, local_storage=local_storage) as file_transactions: await started_cb(file_transactions=file_transactions)
async def test_internal_connections(tmpdir, alice, workspace_id): async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws: with pytest.raises(RuntimeError): await aws.data_localdb._connect() with pytest.raises(RuntimeError): await aws.cache_localdb._connect() # Idempotency await aws.data_localdb._close() await aws.cache_localdb._close()
async def test_storage_file_tree(alice, tmpdir, workspace_id): path = Path(tmpdir) manifest_sqlite_db = path / "workspace_data-v1.sqlite" chunk_sqlite_db = path / "workspace_data-v1.sqlite" block_sqlite_db = path / "workspace_cache-v1.sqlite" async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws: assert aws.manifest_storage.path == manifest_sqlite_db assert aws.chunk_storage.path == chunk_sqlite_db assert aws.block_storage.path == block_sqlite_db assert set(path.iterdir()) == {manifest_sqlite_db, chunk_sqlite_db, block_sqlite_db}
async def workspace_storage_task( task_status: TaskStatus[WorkspaceStorage] = trio. TASK_STATUS_IGNORED ) -> None: async with WorkspaceStorage.run( data_base_dir=self.data_base_dir, device=self.device, workspace_id=workspace_id, cache_size=self.workspace_storage_cache_size, ) as workspace_storage: task_status.started(workspace_storage) await trio.sleep_forever()
async def test_lock_manifest(data_base_dir, alice, workspace_id): manifest = create_manifest(alice, LocalFileManifest) async with WorkspaceStorage.run(data_base_dir, alice, workspace_id) as aws: with pytest.raises(FSLocalMissError): async with aws.lock_manifest(manifest.id): pass await aws.set_manifest(manifest.id, manifest, check_lock_status=False) async with aws.lock_manifest(manifest.id) as m1: assert m1 == manifest m2 = manifest.evolve(need_sync=False) await aws.set_manifest(manifest.id, m2) assert await aws.get_manifest(manifest.id) == m2
async def test_storage_file_tree(data_base_dir, alice, workspace_id): manifest_sqlite_db = data_base_dir / alice.slug / str( workspace_id) / "workspace_data-v1.sqlite" chunk_sqlite_db = data_base_dir / alice.slug / str( workspace_id) / "workspace_data-v1.sqlite" block_sqlite_db = data_base_dir / alice.slug / str( workspace_id) / "workspace_cache-v1.sqlite" async with WorkspaceStorage.run(data_base_dir, alice, workspace_id) as aws: assert aws.manifest_storage.path == manifest_sqlite_db assert aws.chunk_storage.path == chunk_sqlite_db assert aws.block_storage.path == block_sqlite_db assert manifest_sqlite_db.is_file() assert chunk_sqlite_db.is_file() assert block_sqlite_db.is_file()
async def test_garbage_collection(tmpdir, alice, workspace_id): block_size = DEFAULT_BLOCK_SIZE cache_size = 1 * block_size data = b"\x00" * block_size chunk1 = Chunk.new(0, block_size).evolve_as_block(data) chunk2 = Chunk.new(0, block_size).evolve_as_block(data) chunk3 = Chunk.new(0, block_size).evolve_as_block(data) async with WorkspaceStorage.run(alice, tmpdir, workspace_id, cache_size=cache_size) as aws: assert await aws.block_storage.get_nb_blocks() == 0 await aws.set_clean_block(chunk1.access.id, data) assert await aws.block_storage.get_nb_blocks() == 1 await aws.set_clean_block(chunk2.access.id, data) assert await aws.block_storage.get_nb_blocks() == 1 await aws.set_clean_block(chunk3.access.id, data) assert await aws.block_storage.get_nb_blocks() == 1 await aws.block_storage.clear_all_blocks() assert await aws.block_storage.get_nb_blocks() == 0
async def test_vacuum(data_base_dir, alice, workspace_id): data_size = 1 * 1024 * 1024 chunk = Chunk.new(0, data_size) async with WorkspaceStorage.run(data_base_dir, alice, workspace_id, data_vacuum_threshold=data_size // 2) as aws: # Make sure the storage is empty data = b"\x00" * data_size assert await aws.data_localdb.get_disk_usage() < data_size # Set and commit a chunk of 1MB await aws.set_chunk(chunk.id, data) await aws.data_localdb.commit() assert await aws.data_localdb.get_disk_usage() > data_size # Run the vacuum await aws.run_vacuum() assert await aws.data_localdb.get_disk_usage() > data_size # Clear the chunk 1MB await aws.clear_chunk(chunk.id) await aws.data_localdb.commit() assert await aws.data_localdb.get_disk_usage() > data_size # Run the vacuum await aws.run_vacuum() assert await aws.data_localdb.get_disk_usage() < data_size # Make sure vacuum can run even if a transaction has started await aws.set_chunk(chunk.id, data) await aws.run_vacuum() await aws.clear_chunk(chunk.id) await aws.run_vacuum() # Vacuuming the cache storage is no-op await aws.cache_localdb.run_vacuum() # Make sure disk usage can be called on a closed storage assert await aws.data_localdb.get_disk_usage() < data_size
async def alice_workspace_storage(data_base_dir, alice, workspace_id): async with WorkspaceStorage.run(data_base_dir, alice, workspace_id) as aws: yield aws
async def alice_workspace_storage(tmpdir, alice, workspace_id): async with WorkspaceStorage.run(alice, tmpdir, workspace_id) as aws: yield aws
async def alice_transaction_local_storage(alice, persistent_mockup): async with WorkspaceStorage.run(alice, Path("/dummy"), EntryID.new()) as storage: yield storage
async def alice_transaction_local_storage(alice, tmp_path): async with WorkspaceStorage.run( tmp_path / "alice_transaction_local_storage", alice, EntryID.new()) as storage: yield storage