def test_lock_version_context_manager(self): with Locking.with_version_lock(VersionUid('v1'), reason='locking test'): with self.assertRaises(InternalError): Locking.lock_version(VersionUid('v1'), reason='locking test') Locking.lock_version(VersionUid('v1'), reason='locking test') Locking.unlock_version(VersionUid('v1'))
def test_lock_version_context_manager(self): locking = self.database_backend.locking() with locking.with_version_lock(VersionUid(1), reason='locking test'): with self.assertRaises(InternalError): locking.lock_version(VersionUid(1), reason='locking test') locking.lock_version(VersionUid(1), reason='locking test') locking.unlock_version(VersionUid(1))
def test_lock_version(self): locking = self.database_backend.locking() locking.lock_version(VersionUid(1), reason='locking test') self.assertRaises( InternalError, lambda: locking.lock_version(VersionUid(1), reason='locking test')) locking.unlock_version(VersionUid(1))
def test_import(self): benji_obj = self.benjiOpen(init_database=True) benji_obj.metadata_import(StringIO(self.IMPORT)) version = benji_obj.ls(version_uid=VersionUid(1))[0] self.assertTrue(isinstance(version.uid, VersionUid)) self.assertEqual(1, version.uid) self.assertEqual('data-backup', version.name) self.assertEqual('snapshot-name', version.snapshot_name) self.assertEqual(4194304, version.block_size) self.assertEqual(version.status, VersionStatus.valid) self.assertFalse(version.protected) self.assertIsInstance(version.blocks, list) self.assertIsInstance(version.labels, list) self.assertEqual( datetime.datetime.strptime('2018-12-19T20:28:18.123456', '%Y-%m-%dT%H:%M:%S.%f'), version.date) blocks = benji_obj._database_backend.get_blocks_by_version( VersionUid(1)) self.assertTrue(len(blocks) > 0) block = blocks[0] self.assertEqual(VersionUid(1), block.version_uid) self.assertEqual(0, block.id) self.assertEqual(670293, block.size) self.assertTrue(block.valid) benji_obj.close()
def test_version_uid_to_key(self): for i in range(100): version_uid = VersionUid('v{}'.format( random.randint(1, pow(2, 32) - 1))) key = version_uid.storage_object_to_path() version_uid_2 = VersionUid.storage_path_to_object(key) self.assertEqual(version_uid, version_uid_2)
def list_versions(self) -> Iterable[VersionUid]: keys = self._list_objects(VersionUid.storage_prefix()) for key in keys: assert isinstance(key, str) if key.endswith(self._META_SUFFIX): continue try: yield cast(VersionUid, VersionUid.storage_path_to_object(key)) except (RuntimeError, ValueError): # Ignore any keys which don't match our pattern to account for stray objects/files pass
def list_versions(self) -> List[VersionUid]: keys = self._list_objects(VersionUid.storage_prefix()) version_uids: List[VersionUid] = [] for key in keys: if key.endswith(self._META_SUFFIX): continue try: version_uids.append( cast(VersionUid, VersionUid.storage_path_to_object(key))) except (RuntimeError, ValueError): # Ignore any keys which don't match our pattern to account for stray objects/files pass return version_uids
def _api_v1_versions_patch( self, version_uid: str, protected: fields.Bool(missing=None), labels: fields.DelimitedList(fields.Str(), missing=None) ) -> StringIO: version_uid_obj = VersionUid(version_uid) if labels is not None: label_add, label_remove = InputValidation.parse_and_validate_labels( labels) else: label_add, label_remove = [], [] result = StringIO() with Benji(self._config) as benji_obj: try: if protected is not None: benji_obj.protect(version_uid_obj, protected=protected) for name, value in label_add: benji_obj.add_label(version_uid_obj, name, value) for name in label_remove: benji_obj.rm_label(version_uid_obj, name) benji_obj.export_any( { 'versions': [ benji_obj.get_version_by_uid( version_uid=version_uid_obj) ] }, result, ignore_relationships=[((Version, ), ('blocks', ))]) except KeyError: response.status = f'410 Version {version_uid} not found.' return result
def test_set_block_invalid(self): Storage.sync('s-1', storage_id=1) versions = [] good_uid = BlockUid(1, 2) bad_uid = BlockUid(3, 4) for i in range(6): version = Version.create(version_uid=VersionUid(f'v{i + 1}'), volume='backup-name', snapshot='snapshot-name.{}'.format(i), size=16 * 1024 * 4096, storage_id=1, block_size=4 * 1024 * 4096) blocks = [{ 'idx': 0, 'uid_left': bad_uid.left if i < 3 else good_uid.left, 'uid_right': bad_uid.right if i < 3 else good_uid.right, 'checksum': 'aabbcc', 'size': 4 * 1024 * 4096, 'valid': True, }] version.create_blocks(blocks=blocks) version.set(status=VersionStatus.valid) versions.append(version) Version.set_block_invalid(bad_uid) for i in range(3): self.assertEqual(VersionStatus.invalid, versions[i].status) self.assertFalse(list(versions[i].blocks)[0].valid) for i in range(3, 6): self.assertEqual(VersionStatus.valid, versions[i].status) self.assertTrue(list(versions[i].blocks)[0].valid)
def test_import_1_0_0(self): benji_obj = self.benji_open(init_database=True) version_uid = VersionUid('V0000000001') benji_obj.metadata_import(StringIO(self.IMPORT_1_0_0)) version = benji_obj.get_version_by_uid(version_uid=version_uid) self.assertTrue(isinstance(version.uid, VersionUid)) self.assertEqual(version_uid, version.uid) self.assertEqual('data-backup', version.volume) self.assertEqual('snapshot-name', version.snapshot) self.assertEqual(4194304, version.block_size) self.assertEqual(version.status, VersionStatus.valid) self.assertFalse(version.protected) self.assertIsInstance(version.blocks, Iterable) self.assertIsInstance(version.labels, dict) self.assertEqual( datetime.datetime.strptime('2018-12-19T20:28:18.123456', '%Y-%m-%dT%H:%M:%S.%f'), version.date) self.assertIsNone(version.bytes_read) self.assertIsNone(version.bytes_written) self.assertIsNone(version.bytes_deduplicated) self.assertIsNone(version.bytes_sparse) self.assertIsNone(version.duration) self.assertTrue(len(list(version.blocks)) > 0) block = list(version.blocks)[0] self.assertEqual(version.id, block.version_id) self.assertEqual(0, block.idx) self.assertEqual(670293, block.size) self.assertTrue(block.valid) benji_obj.close()
def unprotect(self, version_uids: List[str]) -> None: version_uid_objs = [ VersionUid(version_uid) for version_uid in version_uids ] with Benji(self.config) as benji_obj: for version_uid in version_uid_objs: benji_obj.protect(version_uid, protected=False)
def scrub(self, version_uid: str, block_percentage: int) -> None: version_uid_obj = VersionUid(version_uid) with Benji(self.config) as benji_obj: try: benji_obj.scrub(version_uid_obj, block_percentage=block_percentage) except benji.exception.ScrubbingError: assert benji_obj is not None if self.machine_output: benji_obj.export_any( { 'versions': [ benji_obj.get_version_by_uid( version_uid=version_uid_obj) ], 'errors': [ benji_obj.get_version_by_uid( version_uid=version_uid_obj) ] }, sys.stdout, ignore_relationships=(((Version, ), ('blocks', )), )) raise else: if self.machine_output: benji_obj.export_any( { 'versions': [ benji_obj.get_version_by_uid( version_uid=version_uid_obj) ], 'errors': [] }, sys.stdout, ignore_relationships=(((Version, ), ('blocks', )), ))
def deep_scrub( self, version_uid: str, source: fields.Str(missing=None), block_percentage: fields.Int(missing=100) ) -> StringIO: version_uid_obj = VersionUid(version_uid) result = StringIO() benji_obj = None try: benji_obj = Benji(self._config) benji_obj.deep_scrub(version_uid_obj, source=source, block_percentage=block_percentage) except benji.exception.ScrubbingError: assert benji_obj is not None benji_obj.export_any( { 'versions': benji_obj.ls(version_uid=version_uid_obj), 'errors': benji_obj.ls(version_uid=version_uid_obj) }, result, ignore_relationships=[((Version, ), ('blocks', ))]) else: benji_obj.export_any( { 'versions': benji_obj.ls(version_uid=version_uid_obj), 'errors': [] }, result, ignore_relationships=[((Version, ), ('blocks', ))]) finally: if benji_obj: benji_obj.close() return result
def _rm( self, version_uid: str, force: fields.Bool(missing=False), keep_metadata_backup: fields.Bool(missing=False), override_lock: fields.Bool(missing=False) ) -> None: version_uid_obj = VersionUid(version_uid) disallow_rm_when_younger_than_days = self._config.get( 'disallowRemoveWhenYounger', types=int) benji_obj = None try: benji_obj = Benji(self._config) result = StringIO() # Do this before deleting the version benji_obj.export_any({'versions': [version_uid_obj]}, result, ignore_relationships=[((Version, ), ('blocks', ))]) benji_obj.rm(version_uid_obj, force=force, disallow_rm_when_younger_than_days= disallow_rm_when_younger_than_days, keep_metadata_backup=keep_metadata_backup, override_lock=override_lock) return result finally: if benji_obj: benji_obj.close()
def _api_v1_versions_delete( self, version_uid: str, force: fields.Bool(missing=False), keep_metadata_backup: fields.Bool(missing=False), override_lock: fields.Bool(missing=False) ) -> StringIO: version_uid_obj = VersionUid(version_uid) result = StringIO() with Benji(self._config) as benji_obj: try: # Do this before deleting the version benji_obj.export_any( { 'versions': [ benji_obj.get_version_by_uid( version_uid=version_uid_obj) ] }, result, ignore_relationships=[((Version, ), ('blocks', ))]) benji_obj.rm(version_uid_obj, force=force, keep_metadata_backup=keep_metadata_backup, override_lock=override_lock) except KeyError: response.status = f'410 Version {version_uid} not found.' return result
def _protect( self, version_uid: str, protected: fields.Bool(missing=None), labels: fields.DelimitedList(fields.Str(), missing=None) ) -> StringIO: version_uid_obj = VersionUid(version_uid) if labels is not None: label_add, label_remove = InputValidation.parse_and_validate_labels( labels) else: label_add, label_remove = [], [] benji_obj = None try: benji_obj = Benji(self._config) if protected is True: benji_obj.protect(version_uid_obj) elif protected is False: benji_obj.unprotect(version_uid_obj) for name, value in label_add: benji_obj.add_label(version_uid_obj, name, value) for name in label_remove: benji_obj.rm_label(version_uid_obj, name) result = StringIO() benji_obj.export_any( {'versions': [benji_obj.ls(version_uid=version_uid_obj)]}, result, ignore_relationships=[((Version, ), ('blocks', ))]) return result finally: if benji_obj: benji_obj.close()
def deep_scrub(self, version_uid: str, source: str, block_percentage: int) -> None: version_uid_obj = VersionUid(version_uid) benji_obj = None try: benji_obj = Benji(self.config) benji_obj.deep_scrub(version_uid_obj, source=source, block_percentage=block_percentage) except benji.exception.ScrubbingError: assert benji_obj is not None if self.machine_output: benji_obj.export_any( { 'versions': benji_obj.ls(version_uid=version_uid_obj), 'errors': benji_obj.ls(version_uid=version_uid_obj) }, sys.stdout, ignore_relationships=[((Version,), ('blocks',))]) raise else: if self.machine_output: benji_obj.export_any({ 'versions': benji_obj.ls(version_uid=version_uid_obj), 'errors': [] }, sys.stdout, ignore_relationships=[((Version,), ('blocks',))]) finally: if benji_obj: benji_obj.close()
def write_version(self, version_uid: VersionUid, data: str, overwrite: Optional[bool] = False) -> None: key = version_uid.storage_object_to_path() metadata_key = key + self._META_SUFFIX if not overwrite: try: self._read_object(key) except FileNotFoundError: pass else: raise FileExistsError('Version {} already exists in storage.'.format(version_uid.v_string)) data_bytes = data.encode('utf-8') size = len(data_bytes) data_bytes, transforms_metadata = self._encapsulate(data_bytes) metadata, metadata_json = self._build_metadata( size=size, object_size=len(data_bytes), transforms_metadata=transforms_metadata) try: self._write_object(key, data_bytes) self._write_object(metadata_key, metadata_json) except: try: self._rm_object(key) self._rm_object(metadata_key) except FileNotFoundError: pass raise if self._consistency_check_writes: self._check_write(key=key, metadata_key=metadata_key, data_expected=data_bytes)
def backup(self, version_uid: str, volume: str, snapshot: str, source: str, rbd_hints: str, base_version_uid: str, block_size: int, labels: List[str], storage: str) -> None: if version_uid is None: version_uid = '{}-{}'.format(volume[:248], random_string(6)) version_uid_obj = VersionUid(version_uid) base_version_uid_obj = VersionUid( base_version_uid) if base_version_uid else None if labels: label_add, label_remove = InputValidation.parse_and_validate_labels( labels) with Benji(self.config) as benji_obj: hints = None if rbd_hints: logger.debug(f'Loading RBD hints from file {rbd_hints}.') with open(rbd_hints, 'r') as f: hints = hints_from_rbd_diff(f.read()) backup_version = benji_obj.backup( version_uid=version_uid_obj, volume=volume, snapshot=snapshot, source=source, hints=hints, base_version_uid=base_version_uid_obj, storage_name=storage, block_size=block_size) if labels: for key, value in label_add: benji_obj.add_label(backup_version.uid, key, value) for key in label_remove: benji_obj.rm_label(backup_version.uid, key) if label_add: logger.info('Added label(s) to version {}: {}.'.format( backup_version.uid, ', '.join('{}={}'.format(name, value) for name, value in label_add))) if label_remove: logger.info('Removed label(s) from version {}: {}.'.format( backup_version.uid, ', '.join(label_remove))) if self.machine_output: benji_obj.export_any({'versions': [backup_version]}, sys.stdout, ignore_relationships=(((Version, ), ('blocks', )), ))
def _make_version(uid: str, date: datetime.datetime) -> Version: version = MagicMock(spec=('uid', 'date', '__repr__')) version.uid = VersionUid(uid) version.date = date version.__repr__ = Mock() version.__repr__.return_value = '{} - {}'.format( version.uid, version.date.isoformat(timespec='seconds')) return version
def metadata_restore(self, version_uids: List[str], storage: str = None) -> None: version_uid_objs = [ VersionUid(version_uid) for version_uid in version_uids ] with Benji(self.config) as benji_obj: benji_obj.metadata_restore(version_uid_objs, storage)
def test_lock_override(self): locking = self.database_backend.locking() locking.lock_version(VersionUid(1), reason='locking test') self.assertRaises( InternalError, lambda: locking.lock_version(VersionUid(1), reason='locking test')) old_uuid = locking._uuid new_uuid = uuid.uuid1().hex # This fakes the appearance of another instance locking._uuid = new_uuid self.assertRaises( AlreadyLocked, lambda: locking.lock_version(VersionUid(1), reason='locking test')) locking.lock_version(VersionUid(1), reason='locking test', override_lock=True) self.assertRaises( InternalError, lambda: locking.lock_version(VersionUid(1), reason='locking test')) locking._uuid = old_uuid self.assertRaises( AlreadyLocked, lambda: locking.lock_version(VersionUid(1), reason='locking test')) locking._uuid = new_uuid locking.unlock_version(VersionUid(1))
def metadata_restore(self, version_uids: List[str], storage: str = None) -> None: version_uid_objs = [VersionUid(version_uid) for version_uid in version_uids] benji_obj = None try: benji_obj = Benji(self.config) benji_obj.metadata_restore(version_uid_objs, storage) finally: if benji_obj: benji_obj.close()
def generate_versions(self, testpath): version_uids = [] old_size = 0 init_database = True image_filename = os.path.join(testpath, 'image') for i in range(self.VERSIONS): logger.debug('Run {}'.format(i + 1)) hints = [] if old_size and random.randint( 0, 10 ) == 0: # every 10th time or so do not apply any changes. size = old_size else: size = 32 * 4 * kB + random.randint(-4 * kB, 4 * kB) old_size = size for j in range(random.randint(0, 10)): # up to 10 changes if random.randint(0, 1): patch_size = random.randint(0, 4 * kB) data = self.random_bytes(patch_size) exists = "true" else: patch_size = random.randint( 0, 4 * 4 * kB) # we want full blocks sometimes data = b'\0' * patch_size exists = "false" offset = random.randint(0, size - 1 - patch_size) logger.debug('Applied change at {}:{}, exists {}'.format( offset, patch_size, exists)) self.patch(image_filename, offset, data) hints.append({ 'offset': offset, 'length': patch_size, 'exists': exists }) # truncate? if not os.path.exists(image_filename): open(image_filename, 'wb').close() with open(image_filename, 'r+b') as f: f.truncate(size) logger.debug('Applied {} changes, size is {}.'.format( len(hints), size)) with open(os.path.join(testpath, 'hints'), 'w') as f: f.write(json.dumps(hints)) benji_obj = self.benji_open(init_database=init_database) init_database = False with open(os.path.join(testpath, 'hints')) as hints: version = benji_obj.backup( version_uid=VersionUid(str(uuid.uuid4())), volume='data-backup', snapshot='snapshot-name', source='file:' + image_filename, hints=hints_from_rbd_diff(hints.read())) version_uids.append((version.uid, size)) benji_obj.close() return version_uids
def _api_v1_versions_metadata_restore_create( self, version_uids: fields.DelimitedList(fields.Str, required=True), storage_name: fields.Str(missing=None) ) -> None: version_uid_objs = [ VersionUid(version_uid) for version_uid in version_uids ] with Benji(self._config) as benji_obj: benji_obj.metadata_restore(version_uid_objs, storage_name)
def unprotect(self, version_uids: List[str]) -> None: version_uid_objs = [VersionUid(version_uid) for version_uid in version_uids] benji_obj = None try: benji_obj = Benji(self.config) for version_uid in version_uid_objs: benji_obj.unprotect(version_uid) finally: if benji_obj: benji_obj.close()
def rm_version(self, version_uid: VersionUid) -> None: key = version_uid.storage_object_to_path() metadata_key = key + self._META_SUFFIX try: self._rm_object(key) finally: try: self._rm_object(metadata_key) except FileNotFoundError: pass
def test_version(self): version_uid = VersionUid(1) self.storage.write_version(version_uid, 'Hallo') data = self.storage.read_version(version_uid) self.assertEqual('Hallo', data) version_uids = list(self.storage.list_versions()) self.assertTrue(len(version_uids) == 1) self.assertEqual(version_uid, version_uids[0]) self.storage.rm_version(version_uid) version_uids = list(self.storage.list_versions()) self.assertTrue(len(version_uids) == 0)
def restore(self, version_uid: str, destination: str, sparse: bool, force: bool, database_less: bool, storage: str) -> None: if not database_less and storage is not None: raise benji.exception.UsageError( 'Specifying a storage location is only supported for database-less restores.' ) version_uid_obj = VersionUid(version_uid) with Benji(self.config, in_memory_database=database_less) as benji_obj: if database_less: benji_obj.metadata_restore([version_uid_obj], storage) benji_obj.restore(version_uid_obj, destination, sparse, force)
def _test_import_2_and_3(self, import_source: str) -> None: benji_obj = self.benji_open(init_database=True) version_uid = VersionUid('V0000000001') benji_obj.metadata_import(StringIO(import_source)) version = benji_obj.get_version_by_uid(version_uid=version_uid) self.assertTrue(isinstance(version.uid, VersionUid)) self.assertEqual(version_uid, version.uid) self.assertEqual('data-backup', version.volume) self.assertEqual('snapshot-name', version.snapshot) self.assertEqual(4864597, version.size) self.assertEqual(4194304, version.block_size) self.assertEqual(version.status, VersionStatus.valid) self.assertFalse(version.protected) self.assertIsInstance(version.blocks, Iterable) self.assertIsInstance(version.labels, dict) self.assertEqual( datetime.datetime.strptime('2018-12-19T20:28:18.123456', '%Y-%m-%dT%H:%M:%S.%f'), version.date) self.assertEqual(1, version.bytes_read) self.assertEqual(2, version.bytes_written) self.assertEqual(3, version.bytes_deduplicated) self.assertEqual(4, version.bytes_sparse) self.assertEqual(5, version.duration) for label in version.labels.values(): self.assertIsInstance(label, Label) self.assertSetEqual({'label-1', 'label-2'}, set(version.labels.keys())) self.assertEqual('label-1', version.labels['label-1'].name) self.assertEqual('label-2', version.labels['label-2'].name) self.assertEqual('bla', version.labels['label-1'].value) self.assertEqual('blub', version.labels['label-2'].value) self.assertEqual(version.id, version.labels['label-1'].version_id) self.assertEqual(version.id, version.labels['label-2'].version_id) self.assertEqual(2, len(list(version.blocks))) block = list(version.blocks)[0] self.assertIsInstance(block, Block) self.assertEqual(version.id, block.version_id) self.assertEqual(0, block.idx) self.assertEqual(4194304, block.size) self.assertTrue(block.valid) block = list(version.blocks)[1] self.assertIsInstance(block, Block) self.assertEqual(version.id, block.version_id) self.assertEqual(1, block.idx) self.assertEqual(670293, block.size) self.assertTrue(block.valid) self.assertEqual(None, block.checksum) benji_obj.close()