def test_lock_version_context_manager(self):
with Locking.with_version_lock(VersionUid('v1'),
reason='locking test'):
with self.assertRaises(InternalError):
Locking.lock_version(VersionUid('v1'), reason='locking test')
Locking.lock_version(VersionUid('v1'), reason='locking test')
Locking.unlock_version(VersionUid('v1'))
def test_lock_version_context_manager(self):
locking = self.database_backend.locking()
with locking.with_version_lock(VersionUid(1), reason='locking test'):
with self.assertRaises(InternalError):
locking.lock_version(VersionUid(1), reason='locking test')
locking.lock_version(VersionUid(1), reason='locking test')
locking.unlock_version(VersionUid(1))
def test_lock_version(self):
locking = self.database_backend.locking()
locking.lock_version(VersionUid(1), reason='locking test')
self.assertRaises(
InternalError,
lambda: locking.lock_version(VersionUid(1), reason='locking test'))
locking.unlock_version(VersionUid(1))
def test_import(self):
benji_obj = self.benjiOpen(init_database=True)
benji_obj.metadata_import(StringIO(self.IMPORT))
version = benji_obj.ls(version_uid=VersionUid(1))[0]
self.assertTrue(isinstance(version.uid, VersionUid))
self.assertEqual(1, version.uid)
self.assertEqual('data-backup', version.name)
self.assertEqual('snapshot-name', version.snapshot_name)
self.assertEqual(4194304, version.block_size)
self.assertEqual(version.status, VersionStatus.valid)
self.assertFalse(version.protected)
self.assertIsInstance(version.blocks, list)
self.assertIsInstance(version.labels, list)
self.assertEqual(
datetime.datetime.strptime('2018-12-19T20:28:18.123456',
'%Y-%m-%dT%H:%M:%S.%f'), version.date)
blocks = benji_obj._database_backend.get_blocks_by_version(
VersionUid(1))
self.assertTrue(len(blocks) > 0)
block = blocks[0]
self.assertEqual(VersionUid(1), block.version_uid)
self.assertEqual(0, block.id)
self.assertEqual(670293, block.size)
self.assertTrue(block.valid)
benji_obj.close()
def test_version_uid_to_key(self):
for i in range(100):
version_uid = VersionUid('v{}'.format(
random.randint(1,
pow(2, 32) - 1)))
key = version_uid.storage_object_to_path()
version_uid_2 = VersionUid.storage_path_to_object(key)
self.assertEqual(version_uid, version_uid_2)
def list_versions(self) -> Iterable[VersionUid]:
keys = self._list_objects(VersionUid.storage_prefix())
for key in keys:
assert isinstance(key, str)
if key.endswith(self._META_SUFFIX):
continue
try:
yield cast(VersionUid, VersionUid.storage_path_to_object(key))
except (RuntimeError, ValueError):
# Ignore any keys which don't match our pattern to account for stray objects/files
pass
def list_versions(self) -> List[VersionUid]:
keys = self._list_objects(VersionUid.storage_prefix())
version_uids: List[VersionUid] = []
for key in keys:
if key.endswith(self._META_SUFFIX):
continue
try:
version_uids.append(
cast(VersionUid, VersionUid.storage_path_to_object(key)))
except (RuntimeError, ValueError):
# Ignore any keys which don't match our pattern to account for stray objects/files
pass
return version_uids
def _api_v1_versions_patch(
self, version_uid: str, protected: fields.Bool(missing=None),
labels: fields.DelimitedList(fields.Str(), missing=None)
) -> StringIO:
version_uid_obj = VersionUid(version_uid)
if labels is not None:
label_add, label_remove = InputValidation.parse_and_validate_labels(
labels)
else:
label_add, label_remove = [], []
result = StringIO()
with Benji(self._config) as benji_obj:
try:
if protected is not None:
benji_obj.protect(version_uid_obj, protected=protected)
for name, value in label_add:
benji_obj.add_label(version_uid_obj, name, value)
for name in label_remove:
benji_obj.rm_label(version_uid_obj, name)
benji_obj.export_any(
{
'versions': [
benji_obj.get_version_by_uid(
version_uid=version_uid_obj)
]
},
result,
ignore_relationships=[((Version, ), ('blocks', ))])
except KeyError:
response.status = f'410 Version {version_uid} not found.'
return result
def test_set_block_invalid(self):
Storage.sync('s-1', storage_id=1)
versions = []
good_uid = BlockUid(1, 2)
bad_uid = BlockUid(3, 4)
for i in range(6):
version = Version.create(version_uid=VersionUid(f'v{i + 1}'),
volume='backup-name',
snapshot='snapshot-name.{}'.format(i),
size=16 * 1024 * 4096,
storage_id=1,
block_size=4 * 1024 * 4096)
blocks = [{
'idx': 0,
'uid_left': bad_uid.left if i < 3 else good_uid.left,
'uid_right': bad_uid.right if i < 3 else good_uid.right,
'checksum': 'aabbcc',
'size': 4 * 1024 * 4096,
'valid': True,
}]
version.create_blocks(blocks=blocks)
version.set(status=VersionStatus.valid)
versions.append(version)
Version.set_block_invalid(bad_uid)
for i in range(3):
self.assertEqual(VersionStatus.invalid, versions[i].status)
self.assertFalse(list(versions[i].blocks)[0].valid)
for i in range(3, 6):
self.assertEqual(VersionStatus.valid, versions[i].status)
self.assertTrue(list(versions[i].blocks)[0].valid)
def test_import_1_0_0(self):
benji_obj = self.benji_open(init_database=True)
version_uid = VersionUid('V0000000001')
benji_obj.metadata_import(StringIO(self.IMPORT_1_0_0))
version = benji_obj.get_version_by_uid(version_uid=version_uid)
self.assertTrue(isinstance(version.uid, VersionUid))
self.assertEqual(version_uid, version.uid)
self.assertEqual('data-backup', version.volume)
self.assertEqual('snapshot-name', version.snapshot)
self.assertEqual(4194304, version.block_size)
self.assertEqual(version.status, VersionStatus.valid)
self.assertFalse(version.protected)
self.assertIsInstance(version.blocks, Iterable)
self.assertIsInstance(version.labels, dict)
self.assertEqual(
datetime.datetime.strptime('2018-12-19T20:28:18.123456',
'%Y-%m-%dT%H:%M:%S.%f'), version.date)
self.assertIsNone(version.bytes_read)
self.assertIsNone(version.bytes_written)
self.assertIsNone(version.bytes_deduplicated)
self.assertIsNone(version.bytes_sparse)
self.assertIsNone(version.duration)
self.assertTrue(len(list(version.blocks)) > 0)
block = list(version.blocks)[0]
self.assertEqual(version.id, block.version_id)
self.assertEqual(0, block.idx)
self.assertEqual(670293, block.size)
self.assertTrue(block.valid)
benji_obj.close()
def unprotect(self, version_uids: List[str]) -> None:
version_uid_objs = [
VersionUid(version_uid) for version_uid in version_uids
]
with Benji(self.config) as benji_obj:
for version_uid in version_uid_objs:
benji_obj.protect(version_uid, protected=False)
def scrub(self, version_uid: str, block_percentage: int) -> None:
version_uid_obj = VersionUid(version_uid)
with Benji(self.config) as benji_obj:
try:
benji_obj.scrub(version_uid_obj,
block_percentage=block_percentage)
except benji.exception.ScrubbingError:
assert benji_obj is not None
if self.machine_output:
benji_obj.export_any(
{
'versions': [
benji_obj.get_version_by_uid(
version_uid=version_uid_obj)
],
'errors': [
benji_obj.get_version_by_uid(
version_uid=version_uid_obj)
]
},
sys.stdout,
ignore_relationships=(((Version, ), ('blocks', )), ))
raise
else:
if self.machine_output:
benji_obj.export_any(
{
'versions': [
benji_obj.get_version_by_uid(
version_uid=version_uid_obj)
],
'errors': []
},
sys.stdout,
ignore_relationships=(((Version, ), ('blocks', )), ))
def deep_scrub(
self, version_uid: str, source: fields.Str(missing=None),
block_percentage: fields.Int(missing=100)
) -> StringIO:
version_uid_obj = VersionUid(version_uid)
result = StringIO()
benji_obj = None
try:
benji_obj = Benji(self._config)
benji_obj.deep_scrub(version_uid_obj,
source=source,
block_percentage=block_percentage)
except benji.exception.ScrubbingError:
assert benji_obj is not None
benji_obj.export_any(
{
'versions': benji_obj.ls(version_uid=version_uid_obj),
'errors': benji_obj.ls(version_uid=version_uid_obj)
},
result,
ignore_relationships=[((Version, ), ('blocks', ))])
else:
benji_obj.export_any(
{
'versions': benji_obj.ls(version_uid=version_uid_obj),
'errors': []
},
result,
ignore_relationships=[((Version, ), ('blocks', ))])
finally:
if benji_obj:
benji_obj.close()
return result
def _rm(
self, version_uid: str, force: fields.Bool(missing=False),
keep_metadata_backup: fields.Bool(missing=False),
override_lock: fields.Bool(missing=False)
) -> None:
version_uid_obj = VersionUid(version_uid)
disallow_rm_when_younger_than_days = self._config.get(
'disallowRemoveWhenYounger', types=int)
benji_obj = None
try:
benji_obj = Benji(self._config)
result = StringIO()
# Do this before deleting the version
benji_obj.export_any({'versions': [version_uid_obj]},
result,
ignore_relationships=[((Version, ),
('blocks', ))])
benji_obj.rm(version_uid_obj,
force=force,
disallow_rm_when_younger_than_days=
disallow_rm_when_younger_than_days,
keep_metadata_backup=keep_metadata_backup,
override_lock=override_lock)
return result
finally:
if benji_obj:
benji_obj.close()
def _api_v1_versions_delete(
self, version_uid: str, force: fields.Bool(missing=False),
keep_metadata_backup: fields.Bool(missing=False),
override_lock: fields.Bool(missing=False)
) -> StringIO:
version_uid_obj = VersionUid(version_uid)
result = StringIO()
with Benji(self._config) as benji_obj:
try:
# Do this before deleting the version
benji_obj.export_any(
{
'versions': [
benji_obj.get_version_by_uid(
version_uid=version_uid_obj)
]
},
result,
ignore_relationships=[((Version, ), ('blocks', ))])
benji_obj.rm(version_uid_obj,
force=force,
keep_metadata_backup=keep_metadata_backup,
override_lock=override_lock)
except KeyError:
response.status = f'410 Version {version_uid} not found.'
return result
def _protect(
self, version_uid: str, protected: fields.Bool(missing=None),
labels: fields.DelimitedList(fields.Str(), missing=None)
) -> StringIO:
version_uid_obj = VersionUid(version_uid)
if labels is not None:
label_add, label_remove = InputValidation.parse_and_validate_labels(
labels)
else:
label_add, label_remove = [], []
benji_obj = None
try:
benji_obj = Benji(self._config)
if protected is True:
benji_obj.protect(version_uid_obj)
elif protected is False:
benji_obj.unprotect(version_uid_obj)
for name, value in label_add:
benji_obj.add_label(version_uid_obj, name, value)
for name in label_remove:
benji_obj.rm_label(version_uid_obj, name)
result = StringIO()
benji_obj.export_any(
{'versions': [benji_obj.ls(version_uid=version_uid_obj)]},
result,
ignore_relationships=[((Version, ), ('blocks', ))])
return result
finally:
if benji_obj:
benji_obj.close()
def deep_scrub(self, version_uid: str, source: str, block_percentage: int) -> None:
version_uid_obj = VersionUid(version_uid)
benji_obj = None
try:
benji_obj = Benji(self.config)
benji_obj.deep_scrub(version_uid_obj, source=source, block_percentage=block_percentage)
except benji.exception.ScrubbingError:
assert benji_obj is not None
if self.machine_output:
benji_obj.export_any(
{
'versions': benji_obj.ls(version_uid=version_uid_obj),
'errors': benji_obj.ls(version_uid=version_uid_obj)
},
sys.stdout,
ignore_relationships=[((Version,), ('blocks',))])
raise
else:
if self.machine_output:
benji_obj.export_any({
'versions': benji_obj.ls(version_uid=version_uid_obj),
'errors': []
},
sys.stdout,
ignore_relationships=[((Version,), ('blocks',))])
finally:
if benji_obj:
benji_obj.close()
def write_version(self, version_uid: VersionUid, data: str, overwrite: Optional[bool] = False) -> None:
key = version_uid.storage_object_to_path()
metadata_key = key + self._META_SUFFIX
if not overwrite:
try:
self._read_object(key)
except FileNotFoundError:
pass
else:
raise FileExistsError('Version {} already exists in storage.'.format(version_uid.v_string))
data_bytes = data.encode('utf-8')
size = len(data_bytes)
data_bytes, transforms_metadata = self._encapsulate(data_bytes)
metadata, metadata_json = self._build_metadata(
size=size, object_size=len(data_bytes), transforms_metadata=transforms_metadata)
try:
self._write_object(key, data_bytes)
self._write_object(metadata_key, metadata_json)
except:
try:
self._rm_object(key)
self._rm_object(metadata_key)
except FileNotFoundError:
pass
raise
if self._consistency_check_writes:
self._check_write(key=key, metadata_key=metadata_key, data_expected=data_bytes)
def backup(self, version_uid: str, volume: str, snapshot: str, source: str,
rbd_hints: str, base_version_uid: str, block_size: int,
labels: List[str], storage: str) -> None:
if version_uid is None:
version_uid = '{}-{}'.format(volume[:248], random_string(6))
version_uid_obj = VersionUid(version_uid)
base_version_uid_obj = VersionUid(
base_version_uid) if base_version_uid else None
if labels:
label_add, label_remove = InputValidation.parse_and_validate_labels(
labels)
with Benji(self.config) as benji_obj:
hints = None
if rbd_hints:
logger.debug(f'Loading RBD hints from file {rbd_hints}.')
with open(rbd_hints, 'r') as f:
hints = hints_from_rbd_diff(f.read())
backup_version = benji_obj.backup(
version_uid=version_uid_obj,
volume=volume,
snapshot=snapshot,
source=source,
hints=hints,
base_version_uid=base_version_uid_obj,
storage_name=storage,
block_size=block_size)
if labels:
for key, value in label_add:
benji_obj.add_label(backup_version.uid, key, value)
for key in label_remove:
benji_obj.rm_label(backup_version.uid, key)
if label_add:
logger.info('Added label(s) to version {}: {}.'.format(
backup_version.uid,
', '.join('{}={}'.format(name, value)
for name, value in label_add)))
if label_remove:
logger.info('Removed label(s) from version {}: {}.'.format(
backup_version.uid, ', '.join(label_remove)))
if self.machine_output:
benji_obj.export_any({'versions': [backup_version]},
sys.stdout,
ignore_relationships=(((Version, ),
('blocks', )), ))
def _make_version(uid: str, date: datetime.datetime) -> Version:
version = MagicMock(spec=('uid', 'date', '__repr__'))
version.uid = VersionUid(uid)
version.date = date
version.__repr__ = Mock()
version.__repr__.return_value = '{} - {}'.format(
version.uid, version.date.isoformat(timespec='seconds'))
return version
def metadata_restore(self,
version_uids: List[str],
storage: str = None) -> None:
version_uid_objs = [
VersionUid(version_uid) for version_uid in version_uids
]
with Benji(self.config) as benji_obj:
benji_obj.metadata_restore(version_uid_objs, storage)
def test_lock_override(self):
locking = self.database_backend.locking()
locking.lock_version(VersionUid(1), reason='locking test')
self.assertRaises(
InternalError,
lambda: locking.lock_version(VersionUid(1), reason='locking test'))
old_uuid = locking._uuid
new_uuid = uuid.uuid1().hex
# This fakes the appearance of another instance
locking._uuid = new_uuid
self.assertRaises(
AlreadyLocked,
lambda: locking.lock_version(VersionUid(1), reason='locking test'))
locking.lock_version(VersionUid(1),
reason='locking test',
override_lock=True)
self.assertRaises(
InternalError,
lambda: locking.lock_version(VersionUid(1), reason='locking test'))
locking._uuid = old_uuid
self.assertRaises(
AlreadyLocked,
lambda: locking.lock_version(VersionUid(1), reason='locking test'))
locking._uuid = new_uuid
locking.unlock_version(VersionUid(1))
def metadata_restore(self, version_uids: List[str], storage: str = None) -> None:
version_uid_objs = [VersionUid(version_uid) for version_uid in version_uids]
benji_obj = None
try:
benji_obj = Benji(self.config)
benji_obj.metadata_restore(version_uid_objs, storage)
finally:
if benji_obj:
benji_obj.close()
def generate_versions(self, testpath):
version_uids = []
old_size = 0
init_database = True
image_filename = os.path.join(testpath, 'image')
for i in range(self.VERSIONS):
logger.debug('Run {}'.format(i + 1))
hints = []
if old_size and random.randint(
0, 10
) == 0: # every 10th time or so do not apply any changes.
size = old_size
else:
size = 32 * 4 * kB + random.randint(-4 * kB, 4 * kB)
old_size = size
for j in range(random.randint(0, 10)): # up to 10 changes
if random.randint(0, 1):
patch_size = random.randint(0, 4 * kB)
data = self.random_bytes(patch_size)
exists = "true"
else:
patch_size = random.randint(
0, 4 * 4 * kB) # we want full blocks sometimes
data = b'\0' * patch_size
exists = "false"
offset = random.randint(0, size - 1 - patch_size)
logger.debug('Applied change at {}:{}, exists {}'.format(
offset, patch_size, exists))
self.patch(image_filename, offset, data)
hints.append({
'offset': offset,
'length': patch_size,
'exists': exists
})
# truncate?
if not os.path.exists(image_filename):
open(image_filename, 'wb').close()
with open(image_filename, 'r+b') as f:
f.truncate(size)
logger.debug('Applied {} changes, size is {}.'.format(
len(hints), size))
with open(os.path.join(testpath, 'hints'), 'w') as f:
f.write(json.dumps(hints))
benji_obj = self.benji_open(init_database=init_database)
init_database = False
with open(os.path.join(testpath, 'hints')) as hints:
version = benji_obj.backup(
version_uid=VersionUid(str(uuid.uuid4())),
volume='data-backup',
snapshot='snapshot-name',
source='file:' + image_filename,
hints=hints_from_rbd_diff(hints.read()))
version_uids.append((version.uid, size))
benji_obj.close()
return version_uids
def _api_v1_versions_metadata_restore_create(
self, version_uids: fields.DelimitedList(fields.Str, required=True),
storage_name: fields.Str(missing=None)
) -> None:
version_uid_objs = [
VersionUid(version_uid) for version_uid in version_uids
]
with Benji(self._config) as benji_obj:
benji_obj.metadata_restore(version_uid_objs, storage_name)
def unprotect(self, version_uids: List[str]) -> None:
version_uid_objs = [VersionUid(version_uid) for version_uid in version_uids]
benji_obj = None
try:
benji_obj = Benji(self.config)
for version_uid in version_uid_objs:
benji_obj.unprotect(version_uid)
finally:
if benji_obj:
benji_obj.close()
def rm_version(self, version_uid: VersionUid) -> None:
key = version_uid.storage_object_to_path()
metadata_key = key + self._META_SUFFIX
try:
self._rm_object(key)
finally:
try:
self._rm_object(metadata_key)
except FileNotFoundError:
pass
def test_version(self):
version_uid = VersionUid(1)
self.storage.write_version(version_uid, 'Hallo')
data = self.storage.read_version(version_uid)
self.assertEqual('Hallo', data)
version_uids = list(self.storage.list_versions())
self.assertTrue(len(version_uids) == 1)
self.assertEqual(version_uid, version_uids[0])
self.storage.rm_version(version_uid)
version_uids = list(self.storage.list_versions())
self.assertTrue(len(version_uids) == 0)
def restore(self, version_uid: str, destination: str, sparse: bool,
force: bool, database_less: bool, storage: str) -> None:
if not database_less and storage is not None:
raise benji.exception.UsageError(
'Specifying a storage location is only supported for database-less restores.'
)
version_uid_obj = VersionUid(version_uid)
with Benji(self.config, in_memory_database=database_less) as benji_obj:
if database_less:
benji_obj.metadata_restore([version_uid_obj], storage)
benji_obj.restore(version_uid_obj, destination, sparse, force)
def _test_import_2_and_3(self, import_source: str) -> None:
benji_obj = self.benji_open(init_database=True)
version_uid = VersionUid('V0000000001')
benji_obj.metadata_import(StringIO(import_source))
version = benji_obj.get_version_by_uid(version_uid=version_uid)
self.assertTrue(isinstance(version.uid, VersionUid))
self.assertEqual(version_uid, version.uid)
self.assertEqual('data-backup', version.volume)
self.assertEqual('snapshot-name', version.snapshot)
self.assertEqual(4864597, version.size)
self.assertEqual(4194304, version.block_size)
self.assertEqual(version.status, VersionStatus.valid)
self.assertFalse(version.protected)
self.assertIsInstance(version.blocks, Iterable)
self.assertIsInstance(version.labels, dict)
self.assertEqual(
datetime.datetime.strptime('2018-12-19T20:28:18.123456',
'%Y-%m-%dT%H:%M:%S.%f'), version.date)
self.assertEqual(1, version.bytes_read)
self.assertEqual(2, version.bytes_written)
self.assertEqual(3, version.bytes_deduplicated)
self.assertEqual(4, version.bytes_sparse)
self.assertEqual(5, version.duration)
for label in version.labels.values():
self.assertIsInstance(label, Label)
self.assertSetEqual({'label-1', 'label-2'}, set(version.labels.keys()))
self.assertEqual('label-1', version.labels['label-1'].name)
self.assertEqual('label-2', version.labels['label-2'].name)
self.assertEqual('bla', version.labels['label-1'].value)
self.assertEqual('blub', version.labels['label-2'].value)
self.assertEqual(version.id, version.labels['label-1'].version_id)
self.assertEqual(version.id, version.labels['label-2'].version_id)
self.assertEqual(2, len(list(version.blocks)))
block = list(version.blocks)[0]
self.assertIsInstance(block, Block)
self.assertEqual(version.id, block.version_id)
self.assertEqual(0, block.idx)
self.assertEqual(4194304, block.size)
self.assertTrue(block.valid)
block = list(version.blocks)[1]
self.assertIsInstance(block, Block)
self.assertEqual(version.id, block.version_id)
self.assertEqual(1, block.idx)
self.assertEqual(670293, block.size)
self.assertTrue(block.valid)
self.assertEqual(None, block.checksum)
benji_obj.close()
