def test_write_rm_sync(self):
NUM_BLOBS = 15
BLOB_SIZE = 4096
saved_uids = list(self.storage.list_blocks())
self.assertEqual(0, len(saved_uids))
blocks = [
Block(uid=BlockUid(i + 1, i + 100), size=BLOB_SIZE, checksum='0000000000000000') for i in range(NUM_BLOBS)
]
data_by_uid = {}
for block in blocks:
data = self.random_bytes(BLOB_SIZE)
self.assertEqual(BLOB_SIZE, len(data))
self.storage.write_block(block, data)
data_by_uid[block.uid] = data
saved_uids = list(self.storage.list_blocks())
self.assertEqual(NUM_BLOBS, len(saved_uids))
uids_set = set([block.uid for block in blocks])
saved_uids_set = set(saved_uids)
self.assertEqual(NUM_BLOBS, len(uids_set))
self.assertEqual(NUM_BLOBS, len(saved_uids_set))
self.assertEqual(0, len(uids_set.symmetric_difference(saved_uids_set)))
for block in blocks:
data = self.storage.read_block(block)
self.assertEqual(data_by_uid[block.uid], data)
for block in blocks:
self.storage.rm_block(block.uid)
saved_uids = list(self.storage.list_blocks())
self.assertEqual(0, len(saved_uids))
def test_storage_stats(self):
NUM_BLOBS = 15
BLOB_SIZE = 4096
saved_uids = list(self.storage.list_blocks())
self.assertEqual(0, len(saved_uids))
blocks = [
Block(uid=BlockUid(i + 1, i + 100),
size=BLOB_SIZE,
checksum='0000000000000000') for i in range(NUM_BLOBS)
]
for block in blocks:
data = self.random_bytes(BLOB_SIZE)
self.assertEqual(BLOB_SIZE, len(data))
self.storage.write_block(block, data)
objects_count, objects_size = self.storage.storage_stats()
logger.debug(
f'Storage stats: {objects_count} objects using {objects_size} bytes.'
)
self.assertEqual(NUM_BLOBS * 2,
objects_count) # Also counts the metadata objects
self.assertGreater(objects_size, 0)
for block in blocks:
self.storage.rm_block(block.uid)
def read_block_async(self, block: Block, metadata_only: bool = False) -> None:
# We do need to dereference the block outside of the closure otherwise a reference to the block will be held
# inside of the closure leading to database troubles.
# See https://github.com/elemental-lf/benji/issues/61.
block_deref = block.deref()
def job():
return self._read(block_deref, metadata_only)
self._read_executor.submit(job)
def test_not_exists(self):
block = Block(uid=BlockUid(1, 2), size=15, checksum='00000000000000000000')
self.storage.write_block(block, b'test_not_exists')
data = self.storage.read_block(block)
self.assertTrue(len(data) > 0)
self.storage.rm_block(block.uid)
self.assertRaises(BlockNotFoundError, lambda: self.storage.rm_block(block.uid))
self.assertRaises(InvalidBlockException, lambda: self.storage.read_block(block))
def test_write_rm_async(self):
NUM_BLOBS = 15
BLOB_SIZE = 4096
saved_uids = list(self.storage.list_blocks())
self.assertEqual(0, len(saved_uids))
blocks = [
Block(uid=BlockUid(i + 1, i + 100),
size=BLOB_SIZE,
checksum='0000000000000000') for i in range(NUM_BLOBS)
]
data_by_uid = {}
for block in blocks:
data = self.random_bytes(BLOB_SIZE)
self.assertEqual(BLOB_SIZE, len(data))
self.storage.write_block_async(block, data)
data_by_uid[block.uid] = data
self.storage.wait_writes_finished()
for _ in self.storage.write_get_completed(timeout=1):
pass
saved_uids = list(self.storage.list_blocks())
self.assertEqual(NUM_BLOBS, len(saved_uids))
uids_set = {block.uid for block in blocks}
saved_uids_set = set(saved_uids)
self.assertEqual(NUM_BLOBS, len(uids_set))
self.assertEqual(NUM_BLOBS, len(saved_uids_set))
self.assertEqual(0, len(uids_set.symmetric_difference(saved_uids_set)))
for block in blocks:
self.storage.read_block_async(block)
for block, data, metadata in self.storage.read_get_completed(
timeout=5):
self.assertEqual(data_by_uid[block.uid], data)
self.assertEqual(
[],
[future for future in self.storage.read_get_completed(timeout=5)])
for block in blocks:
self.storage.rm_block_async(block.uid)
self.storage.wait_rms_finished()
saved_uids = list(self.storage.list_blocks())
self.assertEqual(0, len(saved_uids))
def read_block(self, block: Block, metadata_only: bool = False) -> Optional[bytes]:
return self._read(block.deref(), metadata_only)[1]