def test_set_block_invalid(self):
Storage.sync('s-1', storage_id=1)
versions = []
good_uid = BlockUid(1, 2)
bad_uid = BlockUid(3, 4)
for i in range(6):
version = Version.create(version_uid=VersionUid(f'v{i + 1}'),
volume='backup-name',
snapshot='snapshot-name.{}'.format(i),
size=16 * 1024 * 4096,
storage_id=1,
block_size=4 * 1024 * 4096)
blocks = [{
'idx': 0,
'uid_left': bad_uid.left if i < 3 else good_uid.left,
'uid_right': bad_uid.right if i < 3 else good_uid.right,
'checksum': 'aabbcc',
'size': 4 * 1024 * 4096,
'valid': True,
}]
version.create_blocks(blocks=blocks)
version.set(status=VersionStatus.valid)
versions.append(version)
Version.set_block_invalid(bad_uid)
for i in range(3):
self.assertEqual(VersionStatus.invalid, versions[i].status)
self.assertFalse(list(versions[i].blocks)[0].valid)
for i in range(3, 6):
self.assertEqual(VersionStatus.valid, versions[i].status)
self.assertTrue(list(versions[i].blocks)[0].valid)
def test_write_rm_sync(self):
NUM_BLOBS = 15
BLOB_SIZE = 4096
saved_uids = list(self.storage.list_blocks())
self.assertEqual(0, len(saved_uids))
blocks = [
Block(uid=BlockUid(i + 1, i + 100), size=BLOB_SIZE, checksum='0000000000000000') for i in range(NUM_BLOBS)
]
data_by_uid = {}
for block in blocks:
data = self.random_bytes(BLOB_SIZE)
self.assertEqual(BLOB_SIZE, len(data))
self.storage.write_block(block, data)
data_by_uid[block.uid] = data
saved_uids = list(self.storage.list_blocks())
self.assertEqual(NUM_BLOBS, len(saved_uids))
uids_set = set([block.uid for block in blocks])
saved_uids_set = set(saved_uids)
self.assertEqual(NUM_BLOBS, len(uids_set))
self.assertEqual(NUM_BLOBS, len(saved_uids_set))
self.assertEqual(0, len(uids_set.symmetric_difference(saved_uids_set)))
for block in blocks:
data = self.storage.read_block(block)
self.assertEqual(data_by_uid[block.uid], data)
for block in blocks:
self.storage.rm_block(block.uid)
saved_uids = list(self.storage.list_blocks())
self.assertEqual(0, len(saved_uids))
def test_oom():
history = BlockUidHistory()
for i in range(0, 1000000):
storage_id = random.randint(1, 4)
block = BlockUid(random.randint(1, 2**8), random.randint(1, 2**24))
history.add(storage_id, block)
assert history.seen(storage_id, block)
def test_storage_stats(self):
NUM_BLOBS = 15
BLOB_SIZE = 4096
saved_uids = list(self.storage.list_blocks())
self.assertEqual(0, len(saved_uids))
blocks = [
Block(uid=BlockUid(i + 1, i + 100),
size=BLOB_SIZE,
checksum='0000000000000000') for i in range(NUM_BLOBS)
]
for block in blocks:
data = self.random_bytes(BLOB_SIZE)
self.assertEqual(BLOB_SIZE, len(data))
self.storage.write_block(block, data)
objects_count, objects_size = self.storage.storage_stats()
logger.debug(
f'Storage stats: {objects_count} objects using {objects_size} bytes.'
)
self.assertEqual(NUM_BLOBS * 2,
objects_count) # Also counts the metadata objects
self.assertGreater(objects_size, 0)
for block in blocks:
self.storage.rm_block(block.uid)
def test_block_uid_to_key(self):
for i in range(100):
block_uid = BlockUid(random.randint(1, pow(2, 32) - 1), random.randint(1, pow(2, 32) - 1))
key = block_uid.storage_object_to_path()
block_uid_2 = BlockUid.storage_path_to_object(key)
self.assertEqual(block_uid, block_uid_2)
self.assertEqual(block_uid.left, block_uid_2.left)
self.assertEqual(block_uid.right, block_uid_2.right)
def test_not_exists(self):
block = Block(uid=BlockUid(1, 2), size=15, checksum='00000000000000000000')
self.storage.write_block(block, b'test_not_exists')
data = self.storage.read_block(block)
self.assertTrue(len(data) > 0)
self.storage.rm_block(block.uid)
self.assertRaises(BlockNotFoundError, lambda: self.storage.rm_block(block.uid))
self.assertRaises(InvalidBlockException, lambda: self.storage.read_block(block))
def test_seen(self):
history = BlockUidHistory()
blocks_in_1 = set()
blocks_out_1 = set()
block_exists_1 = set()
blocks_in_2 = set()
blocks_out_2 = set()
block_exists_2 = set()
for i in range(0, 100000):
block = BlockUid(random.randint(1, 2**8),
random.randint(1, 2**64 - 1))
if block in block_exists_1:
continue
block_exists_1.add(block)
if random.randint(1, 100) > 20:
blocks_in_1.add(block)
history.add(1, block)
self.assertTrue(history.seen(1, block))
else:
blocks_out_1.add(block)
self.assertFalse(history.seen(1, block))
for i in range(0, 100000):
block = BlockUid(random.randint(1, 2**8), random.randint(1, 2**20))
if block in block_exists_2:
continue
block_exists_2.add(block)
if random.randint(1, 100) > 20:
blocks_in_2.add(block)
history.add(2, block)
self.assertTrue(history.seen(2, block))
else:
blocks_out_2.add(block)
self.assertFalse(history.seen(2, block))
for block in blocks_in_1:
self.assertTrue(history.seen(1, block))
for block in blocks_out_1:
self.assertFalse(history.seen(1, block))
for block in blocks_in_2:
self.assertTrue(history.seen(2, block))
for block in blocks_out_2:
self.assertFalse(history.seen(2, block))
def test_write_rm_async(self):
NUM_BLOBS = 15
BLOB_SIZE = 4096
saved_uids = list(self.storage.list_blocks())
self.assertEqual(0, len(saved_uids))
blocks = [
Block(uid=BlockUid(i + 1, i + 100),
size=BLOB_SIZE,
checksum='0000000000000000') for i in range(NUM_BLOBS)
]
data_by_uid = {}
for block in blocks:
data = self.random_bytes(BLOB_SIZE)
self.assertEqual(BLOB_SIZE, len(data))
self.storage.write_block_async(block, data)
data_by_uid[block.uid] = data
self.storage.wait_writes_finished()
for _ in self.storage.write_get_completed(timeout=1):
pass
saved_uids = list(self.storage.list_blocks())
self.assertEqual(NUM_BLOBS, len(saved_uids))
uids_set = {block.uid for block in blocks}
saved_uids_set = set(saved_uids)
self.assertEqual(NUM_BLOBS, len(uids_set))
self.assertEqual(NUM_BLOBS, len(saved_uids_set))
self.assertEqual(0, len(uids_set.symmetric_difference(saved_uids_set)))
for block in blocks:
self.storage.read_block_async(block)
for block, data, metadata in self.storage.read_get_completed(
timeout=5):
self.assertEqual(data_by_uid[block.uid], data)
self.assertEqual(
[],
[future for future in self.storage.read_get_completed(timeout=5)])
for block in blocks:
self.storage.rm_block_async(block.uid)
self.storage.wait_rms_finished()
saved_uids = list(self.storage.list_blocks())
self.assertEqual(0, len(saved_uids))
def test_block(self):
version = self.database_backend.create_version(
version_name='name-' + self.random_string(12),
snapshot_name='snapshot-name-' + self.random_string(12),
size=256 * 1024 * 4096,
block_size=1024 * 4096,
storage_id=1)
checksums = []
uids = []
num_blocks = 256
blocks: List[Dict[str, Any]] = []
for id in range(num_blocks):
checksums.append(self.random_hex(64))
uids.append(BlockUid(1, id))
blocks.append({
'id': id,
'version_uid': version.uid,
'uid_left': uids[id].left,
'uid_right': uids[id].right,
'checksum': checksums[id],
'size': 1024 * 4096,
'valid': True
})
self.database_backend.create_blocks(blocks=blocks)
self.database_backend.commit()
for id, checksum in enumerate(checksums):
block = self.database_backend.get_block_by_checksum(checksum, 1)
self.assertEqual(id, block.id)
self.assertEqual(version.uid, block.version_uid)
self.assertEqual(uids[id], block.uid)
self.assertEqual(checksum, block.checksum)
self.assertEqual(1024 * 4096, block.size)
self.assertTrue(block.valid)
for id, uid in enumerate(uids):
block = self.database_backend.get_block(uid)
self.assertEqual(id, block.id)
self.assertEqual(version.uid, block.version_uid)
self.assertEqual(uid, block.uid)
self.assertEqual(checksums[id], block.checksum)
self.assertEqual(1024 * 4096, block.size)
self.assertTrue(block.valid)
for id, uid in enumerate(uids):
block = self.database_backend.get_block_by_id(version.uid, id)
self.assertEqual(id, block.id)
self.assertEqual(version.uid, block.version_uid)
self.assertEqual(uid, block.uid)
self.assertEqual(checksums[id], block.checksum)
self.assertEqual(1024 * 4096, block.size)
self.assertTrue(block.valid)
blocks_iter = self.database_backend.get_blocks_by_version(version.uid)
blocks_count = self.database_backend.get_blocks_count_by_version(
version.uid)
sparse_blocks_count = self.database_backend.get_blocks_count_by_version(
version.uid, sparse_only=True)
self.assertEqual(num_blocks, len(list(blocks_iter)))
self.assertEqual(num_blocks, blocks_count)
self.assertEqual(0, sparse_blocks_count)
blocks_iter = self.database_backend.get_blocks_by_version(version.uid)
for id, block in enumerate(blocks_iter):
self.assertEqual(id, block.id)
self.assertEqual(version.uid, block.version_uid)
self.assertEqual(uids[id], block.uid)
self.assertEqual(checksums[id], block.checksum)
self.assertEqual(1024 * 4096, block.size)
self.assertTrue(block.valid)
blocks_iter = self.database_backend.get_blocks_by_version(version.uid)
for id, block in enumerate(blocks_iter):
dereferenced_block = block.deref()
self.assertEqual(id, dereferenced_block.id)
self.assertEqual(version.uid, dereferenced_block.version_uid)
self.assertEqual(uids[id].left, dereferenced_block.uid.left)
self.assertEqual(uids[id].right, dereferenced_block.uid.right)
self.assertEqual(checksums[id], dereferenced_block.checksum)
self.assertEqual(1024 * 4096, dereferenced_block.size)
self.assertTrue(dereferenced_block.valid)
self.database_backend.rm_version(version.uid)
blocks_iter = self.database_backend.get_blocks_by_version(version.uid)
blocks_count = self.database_backend.get_blocks_count_by_version(
version.uid)
self.assertEqual(0, len(list(blocks_iter)))
self.assertEqual(0, blocks_count)
deleted_count = 0
for uids_deleted in self.database_backend.get_delete_candidates(-1):
for storage in uids_deleted.values():
for uid in storage:
self.assertIn(uid, uids)
deleted_count += 1
self.assertEqual(num_blocks, deleted_count)
def test_block(self):
Storage.sync('s-1', storage_id=1)
version = Version.create(version_uid=VersionUid('v1'),
volume='name-' + self.random_string(12),
snapshot='snapshot-name-' +
self.random_string(12),
size=256 * 1024 * 4096,
block_size=1024 * 4096,
storage_id=1)
checksums = []
uids = []
num_blocks = 256
blocks: List[Dict[str, Any]] = []
for idx in range(num_blocks):
checksums.append(self.random_hex(64))
uids.append(BlockUid(1, idx))
blocks.append({
'idx': idx,
'uid_left': uids[idx].left,
'uid_right': uids[idx].right,
'checksum': checksums[idx],
'size': 1024 * 4096,
'valid': True
})
version.create_blocks(blocks=blocks)
for idx, checksum in enumerate(checksums):
block = version.get_block_by_checksum(checksum)
self.assertEqual(idx, block.idx)
self.assertEqual(version.id, block.version_id)
self.assertEqual(uids[idx], block.uid)
self.assertEqual(checksum, block.checksum)
self.assertEqual(1024 * 4096, block.size)
self.assertTrue(block.valid)
for idx, uid in enumerate(uids):
block = version.get_block_by_idx(idx)
self.assertEqual(idx, block.idx)
self.assertEqual(version.id, block.version_id)
self.assertEqual(uid, block.uid)
self.assertEqual(checksums[idx], block.checksum)
self.assertEqual(1024 * 4096, block.size)
self.assertTrue(block.valid)
self.assertEqual(num_blocks, len(list(version.blocks)))
self.assertEqual(num_blocks, version.blocks_count)
self.assertEqual(0, version.sparse_blocks_count)
for idx, block in enumerate(version.blocks):
self.assertEqual(idx, block.idx)
self.assertEqual(version.id, block.version_id)
self.assertEqual(uids[idx], block.uid)
self.assertEqual(checksums[idx], block.checksum)
self.assertEqual(1024 * 4096, block.size)
self.assertTrue(block.valid)
for idx, block in enumerate(version.blocks):
dereferenced_block = block.deref()
self.assertEqual(idx, dereferenced_block.idx)
self.assertEqual(version.id, dereferenced_block.version_id)
self.assertEqual(uids[idx].left, dereferenced_block.uid.left)
self.assertEqual(uids[idx].right, dereferenced_block.uid.right)
self.assertEqual(checksums[idx], dereferenced_block.checksum)
self.assertEqual(1024 * 4096, dereferenced_block.size)
self.assertTrue(dereferenced_block.valid)
version.remove()
deleted_count = 0
for uids_deleted in DeletedBlock.get_unused_block_uids(-1):
for storage in uids_deleted.values():
for uid in storage:
self.assertIn(uid, uids)
deleted_count += 1
self.assertEqual(num_blocks, deleted_count)
def test_storage_usage(self):
Storage.sync('s-1', storage_id=1)
for i in range(2):
version = Version.create(version_uid=VersionUid(f'v{i + 1}'),
volume='backup-name',
snapshot='snapshot-name.{}'.format(i),
size=32 * 1024 * 4096,
storage_id=1,
block_size=1024 * 4096)
blocks: List[Dict[str, Any]] = []
# shared
for idx in range(0, 6):
uid = BlockUid(1, idx)
blocks.append({
'idx': idx,
'uid_left': uid.left,
'uid_right': uid.right,
'checksum': None,
'size': 1024 * 4096,
'valid': True
})
# sparse
for idx in range(6, 13):
uid = BlockUid(i + 1, idx)
blocks.append({
'idx': idx,
'uid_left': None,
'uid_right': None,
'checksum': None,
'size': 1024 * 4096,
'valid': True
})
# exclusive
for idx in range(13, 25):
uid = BlockUid(i + 1, idx)
blocks.append({
'idx': idx,
'uid_left': uid.left,
'uid_right': uid.right,
'checksum': None,
'size': 1024 * 4096,
'valid': True
})
# exclusive deduplicated
for idx in range(25, 32):
uid = BlockUid(i + 1, idx - 7)
blocks.append({
'idx': idx,
'uid_left': uid.left,
'uid_right': uid.right,
'checksum': None,
'size': 1024 * 4096,
'valid': True
})
version.create_blocks(blocks=blocks)
for uid in ('v1', 'v2'):
usage = Version.storage_usage(f'uid == "{uid}"')
self.assertIsInstance(usage.get('s-1', None), dict)
usage_s_1 = usage['s-1']
check_value_matrix = (
('virtual', 32 * 1024 * 4096),
('shared', 6 * 1024 * 4096),
('sparse', 7 * 1024 * 4096),
('exclusive', 19 * 1024 * 4096),
('deduplicated_exclusive', (19 - 7) * 1024 * 4096),
)
for field, amount in check_value_matrix:
value = usage_s_1.get(field, None)
self.assertIsInstance(value, int)
self.assertEqual(amount, value)
def test_block(self):
version = self.database_backend.create_version(
version_name='name-' + self.random_string(12),
snapshot_name='snapshot-name-' + self.random_string(12),
size=256 * 1024 * 4096,
block_size=1024 * 4096,
storage_id=1)
self.database_backend.commit()
checksums = []
uids = []
num_blocks = 256
for id in range(num_blocks):
checksums.append(self.random_hex(64))
uids.append(BlockUid(1, id))
self.database_backend.set_block(id=id,
version_uid=version.uid,
block_uid=uids[id],
checksum=checksums[id],
size=1024 * 4096,
valid=True)
self.database_backend.commit()
for id, checksum in enumerate(checksums):
block = self.database_backend.get_block_by_checksum(checksum, 1)
self.assertEqual(id, block.id)
self.assertEqual(version.uid, block.version_uid)
self.assertEqual(uids[id], block.uid)
self.assertEqual(checksum, block.checksum)
self.assertEqual(1024 * 4096, block.size)
self.assertTrue(block.valid)
for id, uid in enumerate(uids):
block = self.database_backend.get_block(uid)
self.assertEqual(id, block.id)
self.assertEqual(version.uid, block.version_uid)
self.assertEqual(uid, block.uid)
self.assertEqual(checksums[id], block.checksum)
self.assertEqual(1024 * 4096, block.size)
self.assertTrue(block.valid)
blocks = self.database_backend.get_blocks_by_version(version.uid)
self.assertEqual(num_blocks, len(blocks))
for id, block in enumerate(blocks):
self.assertEqual(id, block.id)
self.assertEqual(version.uid, block.version_uid)
self.assertEqual(uids[id], block.uid)
self.assertEqual(checksums[id], block.checksum)
self.assertEqual(1024 * 4096, block.size)
self.assertTrue(block.valid)
for id, block in enumerate(blocks):
dereferenced_block = block.deref()
self.assertEqual(id, dereferenced_block.id)
self.assertEqual(version.uid, dereferenced_block.version_uid)
self.assertEqual(uids[id].left, dereferenced_block.uid.left)
self.assertEqual(uids[id].right, dereferenced_block.uid.right)
self.assertEqual(checksums[id], dereferenced_block.checksum)
self.assertEqual(1024 * 4096, dereferenced_block.size)
self.assertTrue(dereferenced_block.valid)
self.database_backend.rm_version(version.uid)
self.database_backend.commit()
blocks = self.database_backend.get_blocks_by_version(version.uid)
self.assertEqual(0, len(blocks))
count = 0
for uids_deleted in self.database_backend.get_delete_candidates(-1):
for storage in uids_deleted.values():
for uid in storage:
self.assertIn(uid, uids)
count += 1
self.assertEqual(num_blocks, count)