コード例 #1
0
ファイル: __init__.py プロジェクト: venjixpazuzu/benji
    def test_write_rm_sync(self):
        NUM_BLOBS = 15
        BLOB_SIZE = 4096

        saved_uids = list(self.storage.list_blocks())
        self.assertEqual(0, len(saved_uids))

        blocks = [
            Block(uid=BlockUid(i + 1, i + 100), size=BLOB_SIZE, checksum='0000000000000000') for i in range(NUM_BLOBS)
        ]
        data_by_uid = {}
        for block in blocks:
            data = self.random_bytes(BLOB_SIZE)
            self.assertEqual(BLOB_SIZE, len(data))
            self.storage.write_block(block, data)
            data_by_uid[block.uid] = data

        saved_uids = list(self.storage.list_blocks())
        self.assertEqual(NUM_BLOBS, len(saved_uids))

        uids_set = set([block.uid for block in blocks])
        saved_uids_set = set(saved_uids)
        self.assertEqual(NUM_BLOBS, len(uids_set))
        self.assertEqual(NUM_BLOBS, len(saved_uids_set))
        self.assertEqual(0, len(uids_set.symmetric_difference(saved_uids_set)))

        for block in blocks:
            data = self.storage.read_block(block)
            self.assertEqual(data_by_uid[block.uid], data)

        for block in blocks:
            self.storage.rm_block(block.uid)
        saved_uids = list(self.storage.list_blocks())
        self.assertEqual(0, len(saved_uids))
コード例 #2
0
    def test_storage_stats(self):
        NUM_BLOBS = 15
        BLOB_SIZE = 4096

        saved_uids = list(self.storage.list_blocks())
        self.assertEqual(0, len(saved_uids))

        blocks = [
            Block(uid=BlockUid(i + 1, i + 100),
                  size=BLOB_SIZE,
                  checksum='0000000000000000') for i in range(NUM_BLOBS)
        ]
        for block in blocks:
            data = self.random_bytes(BLOB_SIZE)
            self.assertEqual(BLOB_SIZE, len(data))
            self.storage.write_block(block, data)

        objects_count, objects_size = self.storage.storage_stats()

        logger.debug(
            f'Storage stats: {objects_count} objects using {objects_size} bytes.'
        )

        self.assertEqual(NUM_BLOBS * 2,
                         objects_count)  # Also counts the metadata objects
        self.assertGreater(objects_size, 0)

        for block in blocks:
            self.storage.rm_block(block.uid)
コード例 #3
0
    def read_block_async(self, block: Block, metadata_only: bool = False) -> None:
        # We do need to dereference the block outside of the closure otherwise a reference to the block will be held
        # inside of the closure leading to database troubles.
        # See https://github.com/elemental-lf/benji/issues/61.
        block_deref = block.deref()

        def job():
            return self._read(block_deref, metadata_only)

        self._read_executor.submit(job)
コード例 #4
0
ファイル: __init__.py プロジェクト: venjixpazuzu/benji
    def test_not_exists(self):
        block = Block(uid=BlockUid(1, 2), size=15, checksum='00000000000000000000')
        self.storage.write_block(block, b'test_not_exists')

        data = self.storage.read_block(block)
        self.assertTrue(len(data) > 0)

        self.storage.rm_block(block.uid)

        self.assertRaises(BlockNotFoundError, lambda: self.storage.rm_block(block.uid))
        self.assertRaises(InvalidBlockException, lambda: self.storage.read_block(block))
コード例 #5
0
    def test_write_rm_async(self):
        NUM_BLOBS = 15
        BLOB_SIZE = 4096

        saved_uids = list(self.storage.list_blocks())
        self.assertEqual(0, len(saved_uids))

        blocks = [
            Block(uid=BlockUid(i + 1, i + 100),
                  size=BLOB_SIZE,
                  checksum='0000000000000000') for i in range(NUM_BLOBS)
        ]
        data_by_uid = {}
        for block in blocks:
            data = self.random_bytes(BLOB_SIZE)
            self.assertEqual(BLOB_SIZE, len(data))
            self.storage.write_block_async(block, data)
            data_by_uid[block.uid] = data

        self.storage.wait_writes_finished()

        for _ in self.storage.write_get_completed(timeout=1):
            pass

        saved_uids = list(self.storage.list_blocks())
        self.assertEqual(NUM_BLOBS, len(saved_uids))

        uids_set = {block.uid for block in blocks}
        saved_uids_set = set(saved_uids)
        self.assertEqual(NUM_BLOBS, len(uids_set))
        self.assertEqual(NUM_BLOBS, len(saved_uids_set))
        self.assertEqual(0, len(uids_set.symmetric_difference(saved_uids_set)))

        for block in blocks:
            self.storage.read_block_async(block)

        for block, data, metadata in self.storage.read_get_completed(
                timeout=5):
            self.assertEqual(data_by_uid[block.uid], data)

        self.assertEqual(
            [],
            [future for future in self.storage.read_get_completed(timeout=5)])

        for block in blocks:
            self.storage.rm_block_async(block.uid)

        self.storage.wait_rms_finished()

        saved_uids = list(self.storage.list_blocks())
        self.assertEqual(0, len(saved_uids))
コード例 #6
0
 def read_block(self, block: Block, metadata_only: bool = False) -> Optional[bytes]:
     return self._read(block.deref(), metadata_only)[1]