def test_set_block_invalid(self):
Storage.sync('s-1', storage_id=1)
versions = []
good_uid = BlockUid(1, 2)
bad_uid = BlockUid(3, 4)
for i in range(6):
version = Version.create(version_uid=VersionUid(f'v{i + 1}'),
volume='backup-name',
snapshot='snapshot-name.{}'.format(i),
size=16 * 1024 * 4096,
storage_id=1,
block_size=4 * 1024 * 4096)
blocks = [{
'idx': 0,
'uid_left': bad_uid.left if i < 3 else good_uid.left,
'uid_right': bad_uid.right if i < 3 else good_uid.right,
'checksum': 'aabbcc',
'size': 4 * 1024 * 4096,
'valid': True,
}]
version.create_blocks(blocks=blocks)
version.set(status=VersionStatus.valid)
versions.append(version)
Version.set_block_invalid(bad_uid)
for i in range(3):
self.assertEqual(VersionStatus.invalid, versions[i].status)
self.assertFalse(list(versions[i].blocks)[0].valid)
for i in range(3, 6):
self.assertEqual(VersionStatus.valid, versions[i].status)
self.assertTrue(list(versions[i].blocks)[0].valid)
def test_version_filter_dateparse(self):
Storage.sync('s-1', storage_id=1)
version_uids = set()
for i in range(3):
version = Version.create(version_uid=VersionUid(f'v{i + 1}'),
volume='backup-name',
snapshot='snapshot-name.{}'.format(i),
size=16 * 1024 * 4096,
storage_id=1,
block_size=4 * 1024 * 4096)
self.assertNotIn(version.uid, version_uids)
version_uids.add(version.uid)
# Wait at least one seconds
time.sleep(1)
versions = Version.find_with_filter('date <= "now"')
self.assertEqual(3, len(versions))
versions = Version.find_with_filter('date > "1 month ago"')
self.assertEqual(3, len(versions))
versions = Version.find_with_filter('date > "now"')
self.assertEqual(0, len(versions))
versions = Version.find_with_filter('date < "1 month ago"')
self.assertEqual(0, len(versions))
versions = Version.find_with_filter('"now" >= date')
self.assertEqual(3, len(versions))
versions = Version.find_with_filter('"1 month ago" < date')
self.assertEqual(3, len(versions))
versions = Version.find_with_filter('"now" < date')
self.assertEqual(0, len(versions))
versions = Version.find_with_filter('"1 month ago" > date')
self.assertEqual(0, len(versions))
versions = Version.find_with_filter('date <= "{}"'.format(
datetime.datetime.now(
tz=tz.tzlocal()).strftime("%Y-%m-%dT%H:%M:%S")))
self.assertEqual(3, len(versions))
def test_version_blocks_count(self):
Storage.sync('s-1', storage_id=1)
for i in range(256):
version = Version.create(version_uid=f'v{i + 1}',
volume='backup-name',
snapshot='snapshot-name.{}'.format(i),
size=i * 1111 * 4096,
storage_id=1,
block_size=4 * 1024 * 4096)
self.assertEqual(math.ceil(version.size / version.block_size),
version.blocks_count)
def test_version_filter_issue_9_slowness(self):
Storage.sync('s-1', storage_id=1)
version_uids = set()
for i in range(3):
version = Version.create(version_uid=VersionUid(f'v{i + 1}'),
volume='backup-name',
snapshot='snapshot-name.{}'.format(i),
size=16 * 1024 * 4096,
storage_id=1,
block_size=4 * 1024 * 4096)
self.assertNotIn(version.uid, version_uids)
version_uids.add(version.uid)
t1 = timeit.timeit(lambda: Version.find_with_filter(
'snapshot == "snapshot-name.2" and volume == "backup-name"'),
number=1)
t2 = timeit.timeit(lambda: Version.find_with_filter(
'(snapshot == "snapshot-name.2" and volume == "backup-name")'),
number=1)
logger.debug(
'test_version_filter_issue_9_slowness: t1 {}, t2 {}'.format(
t1, t2))
self.assertLess(t1 - t2, 5)
def test_version_filter_issue_9(self):
Storage.sync('s-1', storage_id=1)
version_uids = set()
for i in range(3):
version = Version.create(version_uid=VersionUid(f'v{i + 1}'),
volume='backup-name',
snapshot='snapshot-name.{}'.format(i),
size=16 * 1024 * 4096,
storage_id=1,
block_size=4 * 1024 * 4096,
status=VersionStatus.valid)
self.assertNotIn(version.uid, version_uids)
version_uids.add(version.uid)
versions = Version.find_with_filter(
'snapshot == "snapshot-name.2" and volume == "backup-name" and status == "valid"'
)
self.assertEqual(1, len(versions))
versions = Version.find_with_filter(
'snapshot == "snapshot-name.2" or volume == "backup-name" or status == "valid"'
)
self.assertEqual(3, len(versions))
def test_block(self):
Storage.sync('s-1', storage_id=1)
version = Version.create(version_uid=VersionUid('v1'),
volume='name-' + self.random_string(12),
snapshot='snapshot-name-' +
self.random_string(12),
size=256 * 1024 * 4096,
block_size=1024 * 4096,
storage_id=1)
checksums = []
uids = []
num_blocks = 256
blocks: List[Dict[str, Any]] = []
for idx in range(num_blocks):
checksums.append(self.random_hex(64))
uids.append(BlockUid(1, idx))
blocks.append({
'idx': idx,
'uid_left': uids[idx].left,
'uid_right': uids[idx].right,
'checksum': checksums[idx],
'size': 1024 * 4096,
'valid': True
})
version.create_blocks(blocks=blocks)
for idx, checksum in enumerate(checksums):
block = version.get_block_by_checksum(checksum)
self.assertEqual(idx, block.idx)
self.assertEqual(version.id, block.version_id)
self.assertEqual(uids[idx], block.uid)
self.assertEqual(checksum, block.checksum)
self.assertEqual(1024 * 4096, block.size)
self.assertTrue(block.valid)
for idx, uid in enumerate(uids):
block = version.get_block_by_idx(idx)
self.assertEqual(idx, block.idx)
self.assertEqual(version.id, block.version_id)
self.assertEqual(uid, block.uid)
self.assertEqual(checksums[idx], block.checksum)
self.assertEqual(1024 * 4096, block.size)
self.assertTrue(block.valid)
self.assertEqual(num_blocks, len(list(version.blocks)))
self.assertEqual(num_blocks, version.blocks_count)
self.assertEqual(0, version.sparse_blocks_count)
for idx, block in enumerate(version.blocks):
self.assertEqual(idx, block.idx)
self.assertEqual(version.id, block.version_id)
self.assertEqual(uids[idx], block.uid)
self.assertEqual(checksums[idx], block.checksum)
self.assertEqual(1024 * 4096, block.size)
self.assertTrue(block.valid)
for idx, block in enumerate(version.blocks):
dereferenced_block = block.deref()
self.assertEqual(idx, dereferenced_block.idx)
self.assertEqual(version.id, dereferenced_block.version_id)
self.assertEqual(uids[idx].left, dereferenced_block.uid.left)
self.assertEqual(uids[idx].right, dereferenced_block.uid.right)
self.assertEqual(checksums[idx], dereferenced_block.checksum)
self.assertEqual(1024 * 4096, dereferenced_block.size)
self.assertTrue(dereferenced_block.valid)
version.remove()
deleted_count = 0
for uids_deleted in DeletedBlock.get_unused_block_uids(-1):
for storage in uids_deleted.values():
for uid in storage:
self.assertIn(uid, uids)
deleted_count += 1
self.assertEqual(num_blocks, deleted_count)
def test_storage_usage(self):
Storage.sync('s-1', storage_id=1)
for i in range(2):
version = Version.create(version_uid=VersionUid(f'v{i + 1}'),
volume='backup-name',
snapshot='snapshot-name.{}'.format(i),
size=32 * 1024 * 4096,
storage_id=1,
block_size=1024 * 4096)
blocks: List[Dict[str, Any]] = []
# shared
for idx in range(0, 6):
uid = BlockUid(1, idx)
blocks.append({
'idx': idx,
'uid_left': uid.left,
'uid_right': uid.right,
'checksum': None,
'size': 1024 * 4096,
'valid': True
})
# sparse
for idx in range(6, 13):
uid = BlockUid(i + 1, idx)
blocks.append({
'idx': idx,
'uid_left': None,
'uid_right': None,
'checksum': None,
'size': 1024 * 4096,
'valid': True
})
# exclusive
for idx in range(13, 25):
uid = BlockUid(i + 1, idx)
blocks.append({
'idx': idx,
'uid_left': uid.left,
'uid_right': uid.right,
'checksum': None,
'size': 1024 * 4096,
'valid': True
})
# exclusive deduplicated
for idx in range(25, 32):
uid = BlockUid(i + 1, idx - 7)
blocks.append({
'idx': idx,
'uid_left': uid.left,
'uid_right': uid.right,
'checksum': None,
'size': 1024 * 4096,
'valid': True
})
version.create_blocks(blocks=blocks)
for uid in ('v1', 'v2'):
usage = Version.storage_usage(f'uid == "{uid}"')
self.assertIsInstance(usage.get('s-1', None), dict)
usage_s_1 = usage['s-1']
check_value_matrix = (
('virtual', 32 * 1024 * 4096),
('shared', 6 * 1024 * 4096),
('sparse', 7 * 1024 * 4096),
('exclusive', 19 * 1024 * 4096),
('deduplicated_exclusive', (19 - 7) * 1024 * 4096),
)
for field, amount in check_value_matrix:
value = usage_s_1.get(field, None)
self.assertIsInstance(value, int)
self.assertEqual(amount, value)
def test_version_filter(self):
Storage.sync('s-1', storage_id=1)
for i in range(256):
version = Version.create(version_uid=VersionUid(f'v{i + 1}'),
volume='backup-name',
snapshot='snapshot-name.{}'.format(i),
size=16 * 1024 * 4096,
storage_id=1,
block_size=4 * 1024 * 4096,
status=VersionStatus.valid)
version = Version.get_by_uid(version.uid)
self.assertEqual(0, len(version.labels))
version.add_label('label-key', 'label-value')
version.add_label('label-key-2', str(i))
version.add_label('label-key-3', '')
if i > 127:
version.add_label('label-key-4', '')
self.assertEqual(4, len(version.labels))
else:
self.assertEqual(3, len(version.labels))
versions = Version.find_with_filter()
self.assertEqual(256, len(versions))
versions = Version.find_with_filter(
'labels["label-key"] == "label-value"')
self.assertEqual(256, len(versions))
versions = Version.find_with_filter(
'"label-value" == labels["label-key"]')
self.assertEqual(256, len(versions))
self.assertRaises(
UsageError, lambda: Version.find_with_filter(
'labels["label-key"] and "label-value"'))
self.assertRaises(UsageError, lambda: Version.find_with_filter('True'))
self.assertRaises(UsageError, lambda: Version.find_with_filter('10'))
self.assertRaises(
UsageError, lambda: Version.find_with_filter(
'labels["label-key"] == "label-value" and True'))
self.assertRaises(
UsageError, lambda: Version.find_with_filter(
'labels["label-key"] == "label-value" and False'))
self.assertRaises(UsageError,
lambda: Version.find_with_filter('"hallo" == "hey"'))
# volume is always true because it is never empty
versions = Version.find_with_filter('volume')
self.assertEqual(256, len(versions))
versions = Version.find_with_filter('status == "valid"')
self.assertEqual(256, len(versions))
self.assertRaises(UsageError,
lambda: Version.find_with_filter('status == wrong'))
versions = Version.find_with_filter('labels["label-key-3"] == ""')
self.assertEqual(256, len(versions))
versions = Version.find_with_filter(
'labels["label-key"] != "label-value"')
self.assertEqual(0, len(versions))
versions = Version.find_with_filter('labels["label-key-2"] == 9')
self.assertEqual(1, len(versions))
versions = Version.find_with_filter('snapshot == "snapshot-name.1"')
self.assertEqual(1, len(versions))
self.assertEqual(VersionUid('v2'), versions[0].uid)
versions = Version.find_with_filter(
'snapshot == "snapshot-name.1" and labels["label-key-2"] == 1')
self.assertEqual(1, len(versions))
self.assertEqual(VersionUid('v2'), versions[0].uid)
versions = Version.find_with_filter(
'snapshot == "snapshot-name.1" and labels["label-key-2"] == "2"')
self.assertEqual(0, len(versions))
versions = Version.find_with_filter(
'snapshot == "snapshot-name.1" or labels["label-key-2"] == 2')
self.assertEqual(2, len(versions))
self.assertSetEqual(
{VersionUid('v2'), VersionUid('v3')},
{version.uid
for version in versions})
versions = Version.find_with_filter(
'volume == "backup-name" and snapshot == "snapshot-name.1"')
self.assertEqual(1, len(versions))
self.assertEqual(VersionUid('v2'), versions[0].uid)
versions = Version.find_with_filter(
'volume == "backup-name" and (snapshot == "snapshot-name.1" or snapshot == "snapshot-name.2")'
)
self.assertEqual(2, len(versions))
self.assertSetEqual(
{VersionUid('v2'), VersionUid('v3')},
{version.uid
for version in versions})
versions = Version.find_with_filter('uid == "v1" or uid == "v12"')
self.assertEqual(2, len(versions))
self.assertSetEqual(
{VersionUid('v1'), VersionUid('v12')},
{version.uid
for version in versions})
versions = Version.find_with_filter('uid == "v1" and uid == "v12"')
self.assertEqual(0, len(versions))
versions = Version.find_with_filter('not labels["not-exists"]')
self.assertEqual(256, len(versions))
versions = Version.find_with_filter('labels["label-key-4"]')
self.assertEqual(128, len(versions))
versions = Version.find_with_filter('labels["label-key-4"] and volume')
self.assertEqual(128, len(versions))
def test_version(self):
Storage.sync('s-1', storage_id=1)
version = Version.create(version_uid=VersionUid('v1'),
volume='backup-name',
snapshot='snapshot-name',
size=16 * 1024 * 4096,
storage_id=1,
block_size=4 * 1024 * 4096)
version = Version.get_by_uid(version.uid)
self.assertEqual('backup-name', version.volume)
self.assertEqual('snapshot-name', version.snapshot)
self.assertEqual(16 * 1024 * 4096, version.size)
self.assertEqual(4 * 1024 * 4096, version.block_size)
self.assertEqual(version.status, VersionStatus.incomplete)
self.assertFalse(version.protected)
version.set(status=VersionStatus.valid)
version = Version.get_by_uid(version.uid)
self.assertEqual(version.status, VersionStatus.valid)
version.set(status=VersionStatus.invalid)
version = Version.get_by_uid(version.uid)
self.assertEqual(version.status, VersionStatus.invalid)
version.set(protected=True)
version = Version.get_by_uid(version.uid)
self.assertTrue(version.protected)
version.set(protected=False)
version = Version.get_by_uid(version.uid)
self.assertFalse(version.protected)
version.add_label('label-1', 'bla')
version.add_label('label-2', '')
version = Version.get_by_uid(version.uid)
self.assertEqual(2, len(version.labels))
self.assertEqual(version.id, version.labels['label-1'].version_id)
self.assertEqual('label-1', version.labels['label-1'].name)
self.assertEqual('bla', version.labels['label-1'].value)
self.assertEqual(version.id, version.labels['label-2'].version_id)
self.assertEqual('label-2', version.labels['label-2'].name)
self.assertEqual('', version.labels['label-2'].value)
version.add_label('label-2', 'test123')
version = Version.get_by_uid(version.uid)
self.assertEqual(version.id, version.labels['label-2'].version_id)
self.assertEqual('label-2', version.labels['label-2'].name)
self.assertEqual('test123', version.labels['label-2'].value)
version.rm_label('label-1')
version = Version.get_by_uid(version.uid)
self.assertEqual(1, len(version.labels))
version.rm_label('label-2')
version = Version.get_by_uid(version.uid)
self.assertEqual(0, len(version.labels))
version.rm_label('label-3')
version = Version.get_by_uid(version.uid)
self.assertEqual(0, len(version.labels))
def test(self):
testpath = self.testpath.path
base_version_uid = None
version_uids = []
old_size = 0
init_database = True
image_filename = os.path.join(testpath, 'image')
block_size = random.sample({512, 1024, 2048, 4096}, 1)[0]
scrub_history = BlockUidHistory()
deep_scrub_history = BlockUidHistory()
storage_name = 's1'
for i in range(1, 40):
logger.debug('Run {}'.format(i + 1))
hints = []
if not os.path.exists(image_filename):
open(image_filename, 'wb').close()
if old_size and random.randint(
0, 10
) == 0: # every 10th time or so do not apply any changes.
size = old_size
else:
size = 32 * 4 * kB + random.randint(-4 * kB, 4 * kB)
for j in range(random.randint(0, 10)): # up to 10 changes
if random.randint(0, 1):
patch_size = random.randint(0, 4 * kB)
data = self.random_bytes(patch_size)
exists = "true"
else:
patch_size = random.randint(
0, 4 * 4 * kB) # we want full blocks sometimes
data = b'\0' * patch_size
exists = "false"
offset = random.randint(0, size - patch_size - 1)
logger.debug(
'Applied change at {}({}):{}, exists {}'.format(
offset, int(offset / 4096), patch_size, exists))
self.patch(image_filename, offset, data)
hints.append({
'offset': offset,
'length': patch_size,
'exists': exists
})
# truncate?
with open(image_filename, 'r+b') as f:
f.truncate(size)
if old_size and size > old_size:
patch_size = size - old_size + 1
offset = old_size - 1
logger.debug('Image got bigger at {}({}):{}'.format(
offset, int(offset / 4096), patch_size))
hints.append({
'offset': offset,
'length': patch_size,
'exists': 'true'
})
old_size = size
copyfile(image_filename, '{}.{}'.format(image_filename, i + 1))
logger.debug('Applied {} changes, size is {}.'.format(
len(hints), size))
with open(os.path.join(testpath, 'hints'), 'w') as f:
f.write(json.dumps(hints))
benji_obj = self.benji_open(init_database=init_database)
init_database = False
with open(os.path.join(testpath, 'hints')) as hints:
version = benji_obj.backup(
version_uid=VersionUid(str(uuid.uuid4())),
volume='data-backup',
snapshot='snapshot-name',
source='file:' + image_filename,
hints=hints_from_rbd_diff(hints.read())
if base_version_uid else None,
base_version_uid=base_version_uid,
storage_name=storage_name,
block_size=block_size)
# Don't keep a reference to version because we're closing the SQLAlchemy session
version_uid = version.uid
benji_obj.close()
version_uids.append(version_uid)
logger.debug('Backup successful')
benji_obj = self.benji_open()
benji_obj.add_label(version_uid, 'label-1', 'value-1')
benji_obj.add_label(version_uid, 'label-2', 'value-2')
benji_obj.close()
logger.debug('Labeling of version successful')
benji_obj = self.benji_open()
benji_obj.rm(version_uid, force=True, keep_metadata_backup=True)
benji_obj.close()
logger.debug('Removal of version successful')
benji_obj = self.benji_open()
benji_obj.metadata_restore([version_uid], storage_name)
benji_obj.close()
logger.debug('Metadata restore of version successful')
benji_obj = self.benji_open()
version = Version.get_by_uid(version_uid)
blocks = list(version.blocks)
self.assertEqual(list(range(len(blocks))),
sorted([block.idx for block in blocks]))
self.assertTrue(len(blocks) > 0)
if len(blocks) > 1:
self.assertTrue(
reduce(and_,
[block.size == block_size
for block in blocks[:-1]]))
benji_obj.close()
logger.debug('Block list successful')
benji_obj = self.benji_open()
versions = benji_obj.find_versions_with_filter()
self.assertEqual(set(),
{version.uid
for version in versions} ^ set(version_uids))
self.assertTrue(
reduce(
and_,
[version.volume == 'data-backup' for version in versions]))
self.assertTrue(
reduce(and_, [
version.snapshot == 'snapshot-name' for version in versions
]))
self.assertTrue(
reduce(
and_,
[version.block_size == block_size
for version in versions]))
self.assertTrue(
reduce(and_, [version.size > 0 for version in versions]))
benji_obj.close()
logger.debug('Version list successful')
benji_obj = self.benji_open()
benji_obj.scrub(version_uid)
benji_obj.close()
logger.debug('Scrub successful')
benji_obj = self.benji_open()
benji_obj.deep_scrub(version_uid)
benji_obj.close()
logger.debug('Deep scrub successful')
benji_obj = self.benji_open()
benji_obj.deep_scrub(version_uid, 'file:' + image_filename)
benji_obj.close()
logger.debug('Deep scrub with source successful')
benji_obj = self.benji_open()
benji_obj.scrub(version_uid, history=scrub_history)
benji_obj.close()
logger.debug('Scrub with history successful')
benji_obj = self.benji_open()
benji_obj.deep_scrub(version_uid, history=deep_scrub_history)
benji_obj.close()
logger.debug('Deep scrub with history successful')
benji_obj = self.benji_open()
benji_obj.batch_scrub('uid == "{}"'.format(version_uid), 100, 100)
benji_obj.close()
logger.debug('Batch scrub with history successful')
benji_obj = self.benji_open()
benji_obj.batch_deep_scrub('uid == "{}"'.format(version_uid), 100,
100)
benji_obj.close()
logger.debug('Batch deep scrub with history successful')
restore_filename = os.path.join(testpath,
'restore.{}'.format(i + 1))
restore_filename_mdl = os.path.join(testpath,
'restore-mdl.{}'.format(i + 1))
restore_filename_sparse = os.path.join(
testpath, 'restore-sparse.{}'.format(i + 1))
benji_obj = self.benji_open()
benji_obj.restore(version_uid,
'file:' + restore_filename,
sparse=False,
force=False)
benji_obj.close()
self.assertTrue(self.same(image_filename, restore_filename))
logger.debug('Restore successful')
benji_obj = self.benji_open(in_memory_database=True)
benji_obj.metadata_restore([version_uid], storage_name)
benji_obj.restore(version_uid,
'file:' + restore_filename_mdl,
sparse=False,
force=False)
benji_obj.close()
self.assertTrue(self.same(image_filename, restore_filename_mdl))
logger.debug('Database-less non-sparse restore successful')
benji_obj = self.benji_open()
benji_obj.restore(version_uid,
'file:' + restore_filename_sparse,
sparse=True,
force=False)
benji_obj.close()
self.assertTrue(self.same(image_filename, restore_filename_sparse))
logger.debug('Sparse restore successful')
benji_obj = self.benji_open()
objects_count, objects_size = benji_obj.storage_stats(storage_name)
benji_obj.close()
self.assertGreater(objects_count, 0)
self.assertGreater(objects_size, 0)
logger.debug(
f'Storage stats: {objects_count} objects using {objects_size} bytes.'
)
base_version_uid = version_uid
# delete old versions
if len(version_uids) > 10:
benji_obj = self.benji_open()
dismissed_versions = benji_obj.enforce_retention_policy(
'volume == "data-backup"', 'latest10,hours24,days30')
for dismissed_version in dismissed_versions:
version_uids.remove(dismissed_version.uid)
benji_obj.close()
if (i % 7) == 0:
benji_obj = self.benji_open()
benji_obj.cleanup(dt=0)
benji_obj.close()
if (i % 13) == 0:
scrub_history = BlockUidHistory()
deep_scrub_history = BlockUidHistory()
if (i % 7) == 0:
base_version_uid = None
if storage_name == 's1':
storage_name = 's2'
else:
storage_name = 's1'