def test_version(self): Storage.sync('s-1', storage_id=1) version = Version.create(version_uid=VersionUid('v1'), volume='backup-name', snapshot='snapshot-name', size=16 * 1024 * 4096, storage_id=1, block_size=4 * 1024 * 4096) version = Version.get_by_uid(version.uid) self.assertEqual('backup-name', version.volume) self.assertEqual('snapshot-name', version.snapshot) self.assertEqual(16 * 1024 * 4096, version.size) self.assertEqual(4 * 1024 * 4096, version.block_size) self.assertEqual(version.status, VersionStatus.incomplete) self.assertFalse(version.protected) version.set(status=VersionStatus.valid) version = Version.get_by_uid(version.uid) self.assertEqual(version.status, VersionStatus.valid) version.set(status=VersionStatus.invalid) version = Version.get_by_uid(version.uid) self.assertEqual(version.status, VersionStatus.invalid) version.set(protected=True) version = Version.get_by_uid(version.uid) self.assertTrue(version.protected) version.set(protected=False) version = Version.get_by_uid(version.uid) self.assertFalse(version.protected) version.add_label('label-1', 'bla') version.add_label('label-2', '') version = Version.get_by_uid(version.uid) self.assertEqual(2, len(version.labels)) self.assertEqual(version.id, version.labels['label-1'].version_id) self.assertEqual('label-1', version.labels['label-1'].name) self.assertEqual('bla', version.labels['label-1'].value) self.assertEqual(version.id, version.labels['label-2'].version_id) self.assertEqual('label-2', version.labels['label-2'].name) self.assertEqual('', version.labels['label-2'].value) version.add_label('label-2', 'test123') version = Version.get_by_uid(version.uid) self.assertEqual(version.id, version.labels['label-2'].version_id) self.assertEqual('label-2', version.labels['label-2'].name) self.assertEqual('test123', version.labels['label-2'].value) version.rm_label('label-1') version = Version.get_by_uid(version.uid) self.assertEqual(1, len(version.labels)) version.rm_label('label-2') version = Version.get_by_uid(version.uid) self.assertEqual(0, len(version.labels)) version.rm_label('label-3') version = Version.get_by_uid(version.uid) self.assertEqual(0, len(version.labels))
def test_version_filter(self): Storage.sync('s-1', storage_id=1) for i in range(256): version = Version.create(version_uid=VersionUid(f'v{i + 1}'), volume='backup-name', snapshot='snapshot-name.{}'.format(i), size=16 * 1024 * 4096, storage_id=1, block_size=4 * 1024 * 4096, status=VersionStatus.valid) version = Version.get_by_uid(version.uid) self.assertEqual(0, len(version.labels)) version.add_label('label-key', 'label-value') version.add_label('label-key-2', str(i)) version.add_label('label-key-3', '') if i > 127: version.add_label('label-key-4', '') self.assertEqual(4, len(version.labels)) else: self.assertEqual(3, len(version.labels)) versions = Version.find_with_filter() self.assertEqual(256, len(versions)) versions = Version.find_with_filter( 'labels["label-key"] == "label-value"') self.assertEqual(256, len(versions)) versions = Version.find_with_filter( '"label-value" == labels["label-key"]') self.assertEqual(256, len(versions)) self.assertRaises( UsageError, lambda: Version.find_with_filter( 'labels["label-key"] and "label-value"')) self.assertRaises(UsageError, lambda: Version.find_with_filter('True')) self.assertRaises(UsageError, lambda: Version.find_with_filter('10')) self.assertRaises( UsageError, lambda: Version.find_with_filter( 'labels["label-key"] == "label-value" and True')) self.assertRaises( UsageError, lambda: Version.find_with_filter( 'labels["label-key"] == "label-value" and False')) self.assertRaises(UsageError, lambda: Version.find_with_filter('"hallo" == "hey"')) # volume is always true because it is never empty versions = Version.find_with_filter('volume') self.assertEqual(256, len(versions)) versions = Version.find_with_filter('status == "valid"') self.assertEqual(256, len(versions)) self.assertRaises(UsageError, lambda: Version.find_with_filter('status == wrong')) versions = Version.find_with_filter('labels["label-key-3"] == ""') self.assertEqual(256, len(versions)) versions = Version.find_with_filter( 'labels["label-key"] != "label-value"') self.assertEqual(0, len(versions)) versions = Version.find_with_filter('labels["label-key-2"] == 9') self.assertEqual(1, len(versions)) versions = Version.find_with_filter('snapshot == "snapshot-name.1"') self.assertEqual(1, len(versions)) self.assertEqual(VersionUid('v2'), versions[0].uid) versions = Version.find_with_filter( 'snapshot == "snapshot-name.1" and labels["label-key-2"] == 1') self.assertEqual(1, len(versions)) self.assertEqual(VersionUid('v2'), versions[0].uid) versions = Version.find_with_filter( 'snapshot == "snapshot-name.1" and labels["label-key-2"] == "2"') self.assertEqual(0, len(versions)) versions = Version.find_with_filter( 'snapshot == "snapshot-name.1" or labels["label-key-2"] == 2') self.assertEqual(2, len(versions)) self.assertSetEqual( {VersionUid('v2'), VersionUid('v3')}, {version.uid for version in versions}) versions = Version.find_with_filter( 'volume == "backup-name" and snapshot == "snapshot-name.1"') self.assertEqual(1, len(versions)) self.assertEqual(VersionUid('v2'), versions[0].uid) versions = Version.find_with_filter( 'volume == "backup-name" and (snapshot == "snapshot-name.1" or snapshot == "snapshot-name.2")' ) self.assertEqual(2, len(versions)) self.assertSetEqual( {VersionUid('v2'), VersionUid('v3')}, {version.uid for version in versions}) versions = Version.find_with_filter('uid == "v1" or uid == "v12"') self.assertEqual(2, len(versions)) self.assertSetEqual( {VersionUid('v1'), VersionUid('v12')}, {version.uid for version in versions}) versions = Version.find_with_filter('uid == "v1" and uid == "v12"') self.assertEqual(0, len(versions)) versions = Version.find_with_filter('not labels["not-exists"]') self.assertEqual(256, len(versions)) versions = Version.find_with_filter('labels["label-key-4"]') self.assertEqual(128, len(versions)) versions = Version.find_with_filter('labels["label-key-4"] and volume') self.assertEqual(128, len(versions))
def test(self): testpath = self.testpath.path base_version_uid = None version_uids = [] old_size = 0 init_database = True image_filename = os.path.join(testpath, 'image') block_size = random.sample({512, 1024, 2048, 4096}, 1)[0] scrub_history = BlockUidHistory() deep_scrub_history = BlockUidHistory() storage_name = 's1' for i in range(1, 40): logger.debug('Run {}'.format(i + 1)) hints = [] if not os.path.exists(image_filename): open(image_filename, 'wb').close() if old_size and random.randint( 0, 10 ) == 0: # every 10th time or so do not apply any changes. size = old_size else: size = 32 * 4 * kB + random.randint(-4 * kB, 4 * kB) for j in range(random.randint(0, 10)): # up to 10 changes if random.randint(0, 1): patch_size = random.randint(0, 4 * kB) data = self.random_bytes(patch_size) exists = "true" else: patch_size = random.randint( 0, 4 * 4 * kB) # we want full blocks sometimes data = b'\0' * patch_size exists = "false" offset = random.randint(0, size - patch_size - 1) logger.debug( 'Applied change at {}({}):{}, exists {}'.format( offset, int(offset / 4096), patch_size, exists)) self.patch(image_filename, offset, data) hints.append({ 'offset': offset, 'length': patch_size, 'exists': exists }) # truncate? with open(image_filename, 'r+b') as f: f.truncate(size) if old_size and size > old_size: patch_size = size - old_size + 1 offset = old_size - 1 logger.debug('Image got bigger at {}({}):{}'.format( offset, int(offset / 4096), patch_size)) hints.append({ 'offset': offset, 'length': patch_size, 'exists': 'true' }) old_size = size copyfile(image_filename, '{}.{}'.format(image_filename, i + 1)) logger.debug('Applied {} changes, size is {}.'.format( len(hints), size)) with open(os.path.join(testpath, 'hints'), 'w') as f: f.write(json.dumps(hints)) benji_obj = self.benji_open(init_database=init_database) init_database = False with open(os.path.join(testpath, 'hints')) as hints: version = benji_obj.backup( version_uid=VersionUid(str(uuid.uuid4())), volume='data-backup', snapshot='snapshot-name', source='file:' + image_filename, hints=hints_from_rbd_diff(hints.read()) if base_version_uid else None, base_version_uid=base_version_uid, storage_name=storage_name, block_size=block_size) # Don't keep a reference to version because we're closing the SQLAlchemy session version_uid = version.uid benji_obj.close() version_uids.append(version_uid) logger.debug('Backup successful') benji_obj = self.benji_open() benji_obj.add_label(version_uid, 'label-1', 'value-1') benji_obj.add_label(version_uid, 'label-2', 'value-2') benji_obj.close() logger.debug('Labeling of version successful') benji_obj = self.benji_open() benji_obj.rm(version_uid, force=True, keep_metadata_backup=True) benji_obj.close() logger.debug('Removal of version successful') benji_obj = self.benji_open() benji_obj.metadata_restore([version_uid], storage_name) benji_obj.close() logger.debug('Metadata restore of version successful') benji_obj = self.benji_open() version = Version.get_by_uid(version_uid) blocks = list(version.blocks) self.assertEqual(list(range(len(blocks))), sorted([block.idx for block in blocks])) self.assertTrue(len(blocks) > 0) if len(blocks) > 1: self.assertTrue( reduce(and_, [block.size == block_size for block in blocks[:-1]])) benji_obj.close() logger.debug('Block list successful') benji_obj = self.benji_open() versions = benji_obj.find_versions_with_filter() self.assertEqual(set(), {version.uid for version in versions} ^ set(version_uids)) self.assertTrue( reduce( and_, [version.volume == 'data-backup' for version in versions])) self.assertTrue( reduce(and_, [ version.snapshot == 'snapshot-name' for version in versions ])) self.assertTrue( reduce( and_, [version.block_size == block_size for version in versions])) self.assertTrue( reduce(and_, [version.size > 0 for version in versions])) benji_obj.close() logger.debug('Version list successful') benji_obj = self.benji_open() benji_obj.scrub(version_uid) benji_obj.close() logger.debug('Scrub successful') benji_obj = self.benji_open() benji_obj.deep_scrub(version_uid) benji_obj.close() logger.debug('Deep scrub successful') benji_obj = self.benji_open() benji_obj.deep_scrub(version_uid, 'file:' + image_filename) benji_obj.close() logger.debug('Deep scrub with source successful') benji_obj = self.benji_open() benji_obj.scrub(version_uid, history=scrub_history) benji_obj.close() logger.debug('Scrub with history successful') benji_obj = self.benji_open() benji_obj.deep_scrub(version_uid, history=deep_scrub_history) benji_obj.close() logger.debug('Deep scrub with history successful') benji_obj = self.benji_open() benji_obj.batch_scrub('uid == "{}"'.format(version_uid), 100, 100) benji_obj.close() logger.debug('Batch scrub with history successful') benji_obj = self.benji_open() benji_obj.batch_deep_scrub('uid == "{}"'.format(version_uid), 100, 100) benji_obj.close() logger.debug('Batch deep scrub with history successful') restore_filename = os.path.join(testpath, 'restore.{}'.format(i + 1)) restore_filename_mdl = os.path.join(testpath, 'restore-mdl.{}'.format(i + 1)) restore_filename_sparse = os.path.join( testpath, 'restore-sparse.{}'.format(i + 1)) benji_obj = self.benji_open() benji_obj.restore(version_uid, 'file:' + restore_filename, sparse=False, force=False) benji_obj.close() self.assertTrue(self.same(image_filename, restore_filename)) logger.debug('Restore successful') benji_obj = self.benji_open(in_memory_database=True) benji_obj.metadata_restore([version_uid], storage_name) benji_obj.restore(version_uid, 'file:' + restore_filename_mdl, sparse=False, force=False) benji_obj.close() self.assertTrue(self.same(image_filename, restore_filename_mdl)) logger.debug('Database-less non-sparse restore successful') benji_obj = self.benji_open() benji_obj.restore(version_uid, 'file:' + restore_filename_sparse, sparse=True, force=False) benji_obj.close() self.assertTrue(self.same(image_filename, restore_filename_sparse)) logger.debug('Sparse restore successful') benji_obj = self.benji_open() objects_count, objects_size = benji_obj.storage_stats(storage_name) benji_obj.close() self.assertGreater(objects_count, 0) self.assertGreater(objects_size, 0) logger.debug( f'Storage stats: {objects_count} objects using {objects_size} bytes.' ) base_version_uid = version_uid # delete old versions if len(version_uids) > 10: benji_obj = self.benji_open() dismissed_versions = benji_obj.enforce_retention_policy( 'volume == "data-backup"', 'latest10,hours24,days30') for dismissed_version in dismissed_versions: version_uids.remove(dismissed_version.uid) benji_obj.close() if (i % 7) == 0: benji_obj = self.benji_open() benji_obj.cleanup(dt=0) benji_obj.close() if (i % 13) == 0: scrub_history = BlockUidHistory() deep_scrub_history = BlockUidHistory() if (i % 7) == 0: base_version_uid = None if storage_name == 's1': storage_name = 's2' else: storage_name = 's1'