def test_write_rm_sync(self): NUM_BLOBS = 15 BLOB_SIZE = 4096 saved_uids = list(self.storage.list_blocks()) self.assertEqual(0, len(saved_uids)) blocks = [ Block(uid=BlockUid(i + 1, i + 100), size=BLOB_SIZE, checksum='0000000000000000') for i in range(NUM_BLOBS) ] data_by_uid = {} for block in blocks: data = self.random_bytes(BLOB_SIZE) self.assertEqual(BLOB_SIZE, len(data)) self.storage.write_block(block, data) data_by_uid[block.uid] = data saved_uids = list(self.storage.list_blocks()) self.assertEqual(NUM_BLOBS, len(saved_uids)) uids_set = set([block.uid for block in blocks]) saved_uids_set = set(saved_uids) self.assertEqual(NUM_BLOBS, len(uids_set)) self.assertEqual(NUM_BLOBS, len(saved_uids_set)) self.assertEqual(0, len(uids_set.symmetric_difference(saved_uids_set))) for block in blocks: data = self.storage.read_block(block) self.assertEqual(data_by_uid[block.uid], data) for block in blocks: self.storage.rm_block(block.uid) saved_uids = list(self.storage.list_blocks()) self.assertEqual(0, len(saved_uids))
def test_storage_stats(self): NUM_BLOBS = 15 BLOB_SIZE = 4096 saved_uids = list(self.storage.list_blocks()) self.assertEqual(0, len(saved_uids)) blocks = [ Block(uid=BlockUid(i + 1, i + 100), size=BLOB_SIZE, checksum='0000000000000000') for i in range(NUM_BLOBS) ] for block in blocks: data = self.random_bytes(BLOB_SIZE) self.assertEqual(BLOB_SIZE, len(data)) self.storage.write_block(block, data) objects_count, objects_size = self.storage.storage_stats() logger.debug( f'Storage stats: {objects_count} objects using {objects_size} bytes.' ) self.assertEqual(NUM_BLOBS * 2, objects_count) # Also counts the metadata objects self.assertGreater(objects_size, 0) for block in blocks: self.storage.rm_block(block.uid)
def read_block_async(self, block: Block, metadata_only: bool = False) -> None: # We do need to dereference the block outside of the closure otherwise a reference to the block will be held # inside of the closure leading to database troubles. # See https://github.com/elemental-lf/benji/issues/61. block_deref = block.deref() def job(): return self._read(block_deref, metadata_only) self._read_executor.submit(job)
def test_not_exists(self): block = Block(uid=BlockUid(1, 2), size=15, checksum='00000000000000000000') self.storage.write_block(block, b'test_not_exists') data = self.storage.read_block(block) self.assertTrue(len(data) > 0) self.storage.rm_block(block.uid) self.assertRaises(BlockNotFoundError, lambda: self.storage.rm_block(block.uid)) self.assertRaises(InvalidBlockException, lambda: self.storage.read_block(block))
def test_write_rm_async(self): NUM_BLOBS = 15 BLOB_SIZE = 4096 saved_uids = list(self.storage.list_blocks()) self.assertEqual(0, len(saved_uids)) blocks = [ Block(uid=BlockUid(i + 1, i + 100), size=BLOB_SIZE, checksum='0000000000000000') for i in range(NUM_BLOBS) ] data_by_uid = {} for block in blocks: data = self.random_bytes(BLOB_SIZE) self.assertEqual(BLOB_SIZE, len(data)) self.storage.write_block_async(block, data) data_by_uid[block.uid] = data self.storage.wait_writes_finished() for _ in self.storage.write_get_completed(timeout=1): pass saved_uids = list(self.storage.list_blocks()) self.assertEqual(NUM_BLOBS, len(saved_uids)) uids_set = {block.uid for block in blocks} saved_uids_set = set(saved_uids) self.assertEqual(NUM_BLOBS, len(uids_set)) self.assertEqual(NUM_BLOBS, len(saved_uids_set)) self.assertEqual(0, len(uids_set.symmetric_difference(saved_uids_set))) for block in blocks: self.storage.read_block_async(block) for block, data, metadata in self.storage.read_get_completed( timeout=5): self.assertEqual(data_by_uid[block.uid], data) self.assertEqual( [], [future for future in self.storage.read_get_completed(timeout=5)]) for block in blocks: self.storage.rm_block_async(block.uid) self.storage.wait_rms_finished() saved_uids = list(self.storage.list_blocks()) self.assertEqual(0, len(saved_uids))
def read_block(self, block: Block, metadata_only: bool = False) -> Optional[bytes]: return self._read(block.deref(), metadata_only)[1]