def backup(self,
name,
snapshot_name,
source,
rbd,
base_version_uid,
block_size=None,
tags=None):
base_version_uid = VersionUid.create_from_readables(base_version_uid)
benji_obj = None
try:
benji_obj = Benji(self.config, block_size=block_size)
hints = None
if rbd:
data = ''.join(
[line for line in fileinput.input(rbd).readline()])
hints = hints_from_rbd_diff(data)
backup_version_uid = benji_obj.backup(name, snapshot_name, source,
hints, base_version_uid,
tags)
if self.machine_output:
benji_obj.export_any(
{'versions': benji_obj.ls(version_uid=backup_version_uid)},
sys.stdout,
ignore_relationships=[((Version, ), ('blocks', ))])
finally:
if benji_obj:
benji_obj.close()
def generate_versions(self, testpath):
version_uids = []
old_size = 0
init_database = True
image_filename = os.path.join(testpath, 'image')
for i in range(self.VERSIONS):
logger.debug('Run {}'.format(i + 1))
hints = []
if old_size and random.randint(
0, 10
) == 0: # every 10th time or so do not apply any changes.
size = old_size
else:
size = 32 * 4 * kB + random.randint(-4 * kB, 4 * kB)
old_size = size
for j in range(random.randint(0, 10)): # up to 10 changes
if random.randint(0, 1):
patch_size = random.randint(0, 4 * kB)
data = self.random_bytes(patch_size)
exists = "true"
else:
patch_size = random.randint(
0, 4 * 4 * kB) # we want full blocks sometimes
data = b'\0' * patch_size
exists = "false"
offset = random.randint(0, size - 1 - patch_size)
logger.debug('Applied change at {}:{}, exists {}'.format(
offset, patch_size, exists))
self.patch(image_filename, offset, data)
hints.append({
'offset': offset,
'length': patch_size,
'exists': exists
})
# truncate?
if not os.path.exists(image_filename):
open(image_filename, 'wb').close()
with open(image_filename, 'r+b') as f:
f.truncate(size)
logger.debug('Applied {} changes, size is {}.'.format(
len(hints), size))
with open(os.path.join(testpath, 'hints'), 'w') as f:
f.write(json.dumps(hints))
benji_obj = self.benji_open(init_database=init_database)
init_database = False
with open(os.path.join(testpath, 'hints')) as hints:
version = benji_obj.backup(
version_uid=VersionUid(str(uuid.uuid4())),
volume='data-backup',
snapshot='snapshot-name',
source='file:' + image_filename,
hints=hints_from_rbd_diff(hints.read()))
version_uids.append((version.uid, size))
benji_obj.close()
return version_uids
def backup(self, version_name: str, snapshot_name: str, source: str,
rbd_hints: str, base_version_uid: str, block_size: int,
labels: List[str], storage) -> None:
# Validate version_name and snapshot_name
if not InputValidation.is_backup_name(version_name):
raise benji.exception.UsageError(
'Version name {} is invalid.'.format(version_name))
if not InputValidation.is_snapshot_name(snapshot_name):
raise benji.exception.UsageError(
'Snapshot name {} is invalid.'.format(snapshot_name))
base_version_uid_obj = VersionUid(
base_version_uid) if base_version_uid else None
if labels:
label_add, label_remove = self._parse_labels(labels)
if label_remove:
raise benji.exception.UsageError(
'Wanting to delete labels on a new version is senseless.')
benji_obj = None
try:
benji_obj = Benji(self.config, block_size=block_size)
hints = None
if rbd_hints:
data = ''.join(
[line for line in fileinput.input(rbd_hints).readline()])
hints = hints_from_rbd_diff(data)
backup_version = benji_obj.backup(version_name, snapshot_name,
source, hints,
base_version_uid_obj, storage)
if labels:
for key, value in label_add:
benji_obj.add_label(backup_version.uid, key, value)
for key in label_remove:
benji_obj.rm_label(backup_version.uid, key)
if label_add:
logger.info('Added label(s) to version {}: {}.'.format(
backup_version.uid.v_string, ', '.join([
'{}={}'.format(name, value)
for name, value in label_add
])))
if label_remove:
logger.info('Removed label(s) from version {}: {}.'.format(
backup_version.uid.v_string, ', '.join(label_remove)))
if self.machine_output:
benji_obj.export_any({'versions': [backup_version]},
sys.stdout,
ignore_relationships=[((Version, ),
('blocks', ))])
finally:
if benji_obj:
benji_obj.close()
def backup(self, version_uid: str, volume: str, snapshot: str, source: str,
rbd_hints: str, base_version_uid: str, block_size: int,
labels: List[str], storage: str) -> None:
if version_uid is None:
version_uid = '{}-{}'.format(volume[:248], random_string(6))
version_uid_obj = VersionUid(version_uid)
base_version_uid_obj = VersionUid(
base_version_uid) if base_version_uid else None
if labels:
label_add, label_remove = InputValidation.parse_and_validate_labels(
labels)
with Benji(self.config) as benji_obj:
hints = None
if rbd_hints:
logger.debug(f'Loading RBD hints from file {rbd_hints}.')
with open(rbd_hints, 'r') as f:
hints = hints_from_rbd_diff(f.read())
backup_version = benji_obj.backup(
version_uid=version_uid_obj,
volume=volume,
snapshot=snapshot,
source=source,
hints=hints,
base_version_uid=base_version_uid_obj,
storage_name=storage,
block_size=block_size)
if labels:
for key, value in label_add:
benji_obj.add_label(backup_version.uid, key, value)
for key in label_remove:
benji_obj.rm_label(backup_version.uid, key)
if label_add:
logger.info('Added label(s) to version {}: {}.'.format(
backup_version.uid,
', '.join('{}={}'.format(name, value)
for name, value in label_add)))
if label_remove:
logger.info('Removed label(s) from version {}: {}.'.format(
backup_version.uid, ', '.join(label_remove)))
if self.machine_output:
benji_obj.export_any({'versions': [backup_version]},
sys.stdout,
ignore_relationships=(((Version, ),
('blocks', )), ))
def _backup(
self, version_name: fields.Str(required=True),
snapshot_name: fields.Str(required=True),
source: fields.Str(required=True), rbd_hints: fields.Str(missing=None),
base_version_uid: fields.Str(missing=None),
block_size: fields.Int(missing=None), labels: fields.DelimitedList(
fields.Str(), missing=None), storage_name: fields.Str(missing=None)
) -> str:
# Validate version_name and snapshot_name
if not InputValidation.is_backup_name(version_name):
raise benji.exception.UsageError(
'Version name {} is invalid.'.format(version_name))
if not InputValidation.is_snapshot_name(snapshot_name):
raise benji.exception.UsageError(
'Snapshot name {} is invalid.'.format(snapshot_name))
base_version_uid_obj = VersionUid(
base_version_uid) if base_version_uid else None
if labels:
label_add, label_remove = self._parse_labels(labels)
benji_obj = None
try:
benji_obj = Benji(self._config, block_size=block_size)
hints = None
if rbd_hints:
with open(rbd_hints, 'r') as f:
hints = hints_from_rbd_diff(f.read())
backup_version = benji_obj.backup(version_name, snapshot_name,
source, hints,
base_version_uid_obj,
storage_name)
result = StringIO()
benji_obj.export_any({'versions': [backup_version]},
result,
ignore_relationships=[((Version, ),
('blocks', ))])
return result
finally:
if benji_obj:
benji_obj.close()
def _api_v1_versions_create(
self, version_uid: fields.Str(missing=None),
volume: fields.Str(required=True), snapshot: fields.Str(required=True),
source: fields.Str(required=True), rbd_hints: fields.Str(missing=None),
base_version_uid: fields.Str(missing=None),
block_size: fields.Int(missing=None),
storage_name: fields.Str(missing=None)
) -> StringIO:
if version_uid is None:
version_uid = '{}-{}'.format(volume[:248], random_string(6))
version_uid_obj = VersionUid(version_uid)
base_version_uid_obj = VersionUid(
base_version_uid) if base_version_uid else None
result = StringIO()
with Benji(self._config) as benji_obj:
hints = None
if rbd_hints:
with open(rbd_hints, 'r') as f:
hints = hints_from_rbd_diff(f.read())
backup_version = benji_obj.backup(
version_uid=version_uid_obj,
volume=volume,
snapshot=snapshot,
source=source,
hints=hints,
base_version_uid=base_version_uid_obj,
storage_name=storage_name,
block_size=block_size)
benji_obj.export_any({'versions': [backup_version]},
result,
ignore_relationships=[((Version, ),
('blocks', ))])
response.status = 201
response.set_header('Location',
f'{request.path}/{backup_version.uid}')
return result
def test(self):
testpath = self.testpath.path
base_version_uid = None
version_uids = []
old_size = 0
init_database = True
image_filename = os.path.join(testpath, 'image')
block_size = random.sample({512, 1024, 2048, 4096}, 1)[0]
scrub_history = BlockUidHistory()
deep_scrub_history = BlockUidHistory()
storage_name = 's1'
for i in range(1, 40):
logger.debug('Run {}'.format(i + 1))
hints = []
if not os.path.exists(image_filename):
open(image_filename, 'wb').close()
if old_size and random.randint(0, 10) == 0: # every 10th time or so do not apply any changes.
size = old_size
else:
size = 32 * 4 * kB + random.randint(-4 * kB, 4 * kB)
for j in range(random.randint(0, 10)): # up to 10 changes
if random.randint(0, 1):
patch_size = random.randint(0, 4 * kB)
data = self.random_bytes(patch_size)
exists = "true"
else:
patch_size = random.randint(0, 4 * 4 * kB) # we want full blocks sometimes
data = b'\0' * patch_size
exists = "false"
offset = random.randint(0, size - patch_size - 1)
logger.debug('Applied change at {}({}):{}, exists {}'.format(offset, int(offset / 4096), patch_size,
exists))
self.patch(image_filename, offset, data)
hints.append({'offset': offset, 'length': patch_size, 'exists': exists})
# truncate?
with open(image_filename, 'r+b') as f:
f.truncate(size)
if old_size and size > old_size:
patch_size = size - old_size + 1
offset = old_size - 1
logger.debug('Image got bigger at {}({}):{}'.format(offset, int(offset / 4096), patch_size))
hints.append({'offset': offset, 'length': patch_size, 'exists': 'true'})
old_size = size
copyfile(image_filename, '{}.{}'.format(image_filename, i + 1))
logger.debug('Applied {} changes, size is {}.'.format(len(hints), size))
with open(os.path.join(testpath, 'hints'), 'w') as f:
f.write(json.dumps(hints))
benji_obj = self.benjiOpen(init_database=init_database, block_size=block_size)
init_database = False
with open(os.path.join(testpath, 'hints')) as hints:
version = benji_obj.backup(version_name='data-backup',
version_snapshot_name='snapshot-name',
source='file:' + image_filename,
hints=hints_from_rbd_diff(hints.read()) if base_version_uid else None,
base_version_uid=base_version_uid,
storage_name=storage_name)
# Don't keep a reference to version because we're closing the SQLAlchemy session
version_uid = version.uid
benji_obj.close()
version_uids.append(version_uid)
logger.debug('Backup successful')
benji_obj = self.benjiOpen()
benji_obj.add_label(version_uid, 'label-1', 'value-1')
benji_obj.add_label(version_uid, 'label-2', 'value-2')
benji_obj.close()
logger.debug('Labeling of version successful')
benji_obj = self.benjiOpen()
benji_obj.rm(version_uid, force=True, keep_metadata_backup=True)
benji_obj.close()
logger.debug('Removal of version successful')
benji_obj = self.benjiOpen()
benji_obj.metadata_restore([version_uid], storage_name)
benji_obj.close()
logger.debug('Restore of version successful')
benji_obj = self.benjiOpen()
blocks = list(benji_obj._database_backend.get_blocks_by_version(version_uid))
self.assertEqual(list(range(len(blocks))), sorted([block.id for block in blocks]))
self.assertTrue(len(blocks) > 0)
if len(blocks) > 1:
self.assertTrue(reduce(and_, [block.size == block_size for block in blocks[:-1]]))
benji_obj.close()
logger.debug('Block list successful')
benji_obj = self.benjiOpen()
versions = benji_obj.ls()
self.assertEqual(set(), set([version.uid for version in versions]) ^ set(version_uids))
self.assertTrue(reduce(and_, [version.name == 'data-backup' for version in versions]))
self.assertTrue(reduce(and_, [version.snapshot_name == 'snapshot-name' for version in versions]))
self.assertTrue(reduce(and_, [version.block_size == block_size for version in versions]))
self.assertTrue(reduce(and_, [version.size > 0 for version in versions]))
benji_obj.close()
logger.debug('Version list successful')
benji_obj = self.benjiOpen()
benji_obj.scrub(version_uid)
benji_obj.close()
logger.debug('Scrub successful')
benji_obj = self.benjiOpen()
benji_obj.deep_scrub(version_uid)
benji_obj.close()
logger.debug('Deep scrub successful')
benji_obj = self.benjiOpen()
benji_obj.deep_scrub(version_uid, 'file:' + image_filename)
benji_obj.close()
logger.debug('Deep scrub with source successful')
benji_obj = self.benjiOpen()
benji_obj.scrub(version_uid, history=scrub_history)
benji_obj.close()
logger.debug('Scrub with history successful')
benji_obj = self.benjiOpen()
benji_obj.deep_scrub(version_uid, history=deep_scrub_history)
benji_obj.close()
logger.debug('Deep scrub with history successful')
benji_obj = self.benjiOpen()
benji_obj.batch_scrub('uid == {}'.format(version_uid.integer), 100, 100)
benji_obj.close()
logger.debug('Batch scrub with history successful')
benji_obj = self.benjiOpen()
benji_obj.batch_deep_scrub('uid == {}'.format(version_uid.integer), 100, 100)
benji_obj.close()
logger.debug('Batch deep scrub with history successful')
restore_filename = os.path.join(testpath, 'restore.{}'.format(i + 1))
restore_filename_mdl = os.path.join(testpath, 'restore-mdl.{}'.format(i + 1))
restore_filename_sparse = os.path.join(testpath, 'restore-sparse.{}'.format(i + 1))
benji_obj = self.benjiOpen()
benji_obj.restore(version_uid, 'file:' + restore_filename, sparse=False, force=False)
benji_obj.close()
self.assertTrue(self.same(image_filename, restore_filename))
logger.debug('Restore successful')
benji_obj = self.benjiOpen(in_memory_database=True)
benji_obj.metadata_restore([version_uid], storage_name)
benji_obj.restore(version_uid, 'file:' + restore_filename_mdl, sparse=False, force=False)
benji_obj.close()
self.assertTrue(self.same(image_filename, restore_filename_mdl))
logger.debug('Database-less restore successful')
benji_obj = self.benjiOpen(in_memory_database=True)
benji_obj.metadata_restore([version_uid], storage_name)
benji_obj.restore(version_uid, 'file:' + restore_filename_sparse, sparse=True, force=False)
benji_obj.close()
self.assertTrue(self.same(image_filename, restore_filename_sparse))
logger.debug('Sparse restore successful')
benji_obj = self.benjiOpen()
objects_count, objects_size = benji_obj.storage_stats(storage_name)
benji_obj.close()
self.assertGreater(objects_count, 0)
self.assertGreater(objects_size, 0)
logger.debug(f'Storage stats: {objects_count} objects using {objects_size} bytes.')
base_version_uid = version_uid
# delete old versions
if len(version_uids) > 10:
benji_obj = self.benjiOpen()
dismissed_versions = benji_obj.enforce_retention_policy('name=="data-backup"', 'latest10,hours24,days30')
for dismissed_version in dismissed_versions:
version_uids.remove(dismissed_version.uid)
benji_obj.close()
if (i % 7) == 0:
benji_obj = self.benjiOpen()
benji_obj.cleanup(dt=0)
benji_obj.close()
if (i % 13) == 0:
scrub_history = BlockUidHistory()
deep_scrub_history = BlockUidHistory()
if (i % 7) == 0:
base_version_uid = None
if storage_name == 's1':
storage_name = 's2'
else:
storage_name = 's1'