def metadata_ls(self) -> List[str]:
benji_obj = None
try:
benji_obj = Benji(self._config)
return benji_obj.list_storages()
finally:
if benji_obj:
benji_obj.close()
def metadata_ls(self, storage_name: str = None) -> List[str]:
benji_obj = None
try:
benji_obj = Benji(self._config)
version_uids = benji_obj.metadata_ls(storage_name)
return [version_uid.v_string for version_uid in version_uids]
finally:
if benji_obj:
benji_obj.close()
def nbd(self, bind_address: str, bind_port: str, read_only: bool) -> None:
benji_obj = None
try:
benji_obj = Benji(self.config)
store = BenjiStore(benji_obj)
addr = (bind_address, bind_port)
server = NbdServer(addr, store, read_only)
logger.info("Starting to serve NBD on %s:%s" % (addr[0], addr[1]))
server.serve_forever()
finally:
if benji_obj:
benji_obj.close()
def enforce_retention_policy(self, rules_spec, version_names, dry_run,
keep_backend_metadata):
benji_obj = None
try:
benji_obj = Benji(self.config)
dismissed_version_uids = []
for version_name in version_names:
dismissed_version_uids.extend(
benji_obj.enforce_retention_policy(
version_name=version_name,
rules_spec=rules_spec,
dry_run=dry_run,
keep_backend_metadata=keep_backend_metadata))
if self.machine_output:
benji_obj.export_any(
{
'versions': [
benji_obj.ls(version_uid=version_uid)[0]
for version_uid in dismissed_version_uids
]
},
sys.stdout,
ignore_relationships=[((Version, ), ('blocks', ))])
finally:
if benji_obj:
benji_obj.close()
def _batch_scrub(self, method: str, filter_expression: Optional[str], version_percentage: int,
block_percentage: int, group_label: Optional[str]) -> None:
benji_obj = None
try:
benji_obj = Benji(self.config)
versions, errors = getattr(benji_obj, method)(filter_expression, version_percentage, block_percentage,
group_label)
if errors:
if self.machine_output:
benji_obj.export_any({
'versions': versions,
'errors': errors,
},
sys.stdout,
ignore_relationships=[((Version,), ('blocks',))])
raise benji.exception.ScrubbingError('One or more version had scrubbing errors: {}.'.format(', '.join(
[version.uid.v_string for version in errors])))
else:
if self.machine_output:
benji_obj.export_any({
'versions': versions,
'errors': []
},
sys.stdout,
ignore_relationships=[((Version,), ('blocks',))])
finally:
if benji_obj:
benji_obj.close()
def backup(self,
name,
snapshot_name,
source,
rbd,
base_version_uid,
block_size=None,
tags=None):
base_version_uid = VersionUid.create_from_readables(base_version_uid)
benji_obj = None
try:
benji_obj = Benji(self.config, block_size=block_size)
hints = None
if rbd:
data = ''.join(
[line for line in fileinput.input(rbd).readline()])
hints = hints_from_rbd_diff(data)
backup_version_uid = benji_obj.backup(name, snapshot_name, source,
hints, base_version_uid,
tags)
if self.machine_output:
benji_obj.export_any(
{'versions': benji_obj.ls(version_uid=backup_version_uid)},
sys.stdout,
ignore_relationships=[((Version, ), ('blocks', ))])
finally:
if benji_obj:
benji_obj.close()
def _rm(
self, version_uid: str, force: fields.Bool(missing=False),
keep_metadata_backup: fields.Bool(missing=False),
override_lock: fields.Bool(missing=False)
) -> None:
version_uid_obj = VersionUid(version_uid)
disallow_rm_when_younger_than_days = self._config.get(
'disallowRemoveWhenYounger', types=int)
benji_obj = None
try:
benji_obj = Benji(self._config)
result = StringIO()
# Do this before deleting the version
benji_obj.export_any({'versions': [version_uid_obj]},
result,
ignore_relationships=[((Version, ),
('blocks', ))])
benji_obj.rm(version_uid_obj,
force=force,
disallow_rm_when_younger_than_days=
disallow_rm_when_younger_than_days,
keep_metadata_backup=keep_metadata_backup,
override_lock=override_lock)
return result
finally:
if benji_obj:
benji_obj.close()
def _storage_stats(self, storage_name: str) -> str:
benji_obj = None
try:
benji_obj = Benji(self._config)
objects_count, objects_size = benji_obj.storage_stats(storage_name)
result = {
'objects_count': objects_count,
'objects_size': objects_size,
}
return result
finally:
if benji_obj:
benji_obj.close()
def _api_v1_versions_delete(
self, version_uid: str, force: fields.Bool(missing=False),
keep_metadata_backup: fields.Bool(missing=False),
override_lock: fields.Bool(missing=False)
) -> StringIO:
version_uid_obj = VersionUid(version_uid)
result = StringIO()
with Benji(self._config) as benji_obj:
try:
# Do this before deleting the version
benji_obj.export_any(
{
'versions': [
benji_obj.get_version_by_uid(
version_uid=version_uid_obj)
]
},
result,
ignore_relationships=[((Version, ), ('blocks', ))])
benji_obj.rm(version_uid_obj,
force=force,
keep_metadata_backup=keep_metadata_backup,
override_lock=override_lock)
except KeyError:
response.status = f'410 Version {version_uid} not found.'
return result
def _api_v1_versions_patch(
self, version_uid: str, protected: fields.Bool(missing=None),
labels: fields.DelimitedList(fields.Str(), missing=None)
) -> StringIO:
version_uid_obj = VersionUid(version_uid)
if labels is not None:
label_add, label_remove = InputValidation.parse_and_validate_labels(
labels)
else:
label_add, label_remove = [], []
result = StringIO()
with Benji(self._config) as benji_obj:
try:
if protected is not None:
benji_obj.protect(version_uid_obj, protected=protected)
for name, value in label_add:
benji_obj.add_label(version_uid_obj, name, value)
for name in label_remove:
benji_obj.rm_label(version_uid_obj, name)
benji_obj.export_any(
{
'versions': [
benji_obj.get_version_by_uid(
version_uid=version_uid_obj)
]
},
result,
ignore_relationships=[((Version, ), ('blocks', ))])
except KeyError:
response.status = f'410 Version {version_uid} not found.'
return result
def unprotect(self, version_uids: List[str]) -> None:
version_uid_objs = [
VersionUid(version_uid) for version_uid in version_uids
]
with Benji(self.config) as benji_obj:
for version_uid in version_uid_objs:
benji_obj.protect(version_uid, protected=False)
def storage_usage(self, filter_expression: str):
with Benji(self.config) as benji_obj:
usage = benji_obj.storage_usage(filter_expression)
if self.machine_output:
print(json.dumps(usage, indent=4))
else:
self._storage_usage_table_output(usage)
def metadata_import(self, input_file: str = None) -> None:
with Benji(self.config) as benji_obj:
if input_file is None:
benji_obj.metadata_import(sys.stdin)
else:
with open(input_file, 'r') as f:
benji_obj.metadata_import(f)
def metadata_ls(self, storage: str = None) -> None:
benji_obj = None
try:
benji_obj = Benji(self.config)
version_uids = benji_obj.metadata_ls(storage)
if self.machine_output:
json.dump(
[version_uid.v_string for version_uid in version_uids],
sys.stdout,
indent=2,
)
else:
self._metadata_ls_table_output(version_uids)
finally:
if benji_obj:
benji_obj.close()
def scrub(self, version_uid: str, block_percentage: int) -> None:
version_uid_obj = VersionUid(version_uid)
with Benji(self.config) as benji_obj:
try:
benji_obj.scrub(version_uid_obj,
block_percentage=block_percentage)
except benji.exception.ScrubbingError:
assert benji_obj is not None
if self.machine_output:
benji_obj.export_any(
{
'versions': [
benji_obj.get_version_by_uid(
version_uid=version_uid_obj)
],
'errors': [
benji_obj.get_version_by_uid(
version_uid=version_uid_obj)
]
},
sys.stdout,
ignore_relationships=(((Version, ), ('blocks', )), ))
raise
else:
if self.machine_output:
benji_obj.export_any(
{
'versions': [
benji_obj.get_version_by_uid(
version_uid=version_uid_obj)
],
'errors': []
},
sys.stdout,
ignore_relationships=(((Version, ), ('blocks', )), ))
def metadata_restore(self,
version_uids: List[str],
storage: str = None) -> None:
version_uid_objs = [
VersionUid(version_uid) for version_uid in version_uids
]
with Benji(self.config) as benji_obj:
benji_obj.metadata_restore(version_uid_objs, storage)
def storage_stats(self, storage_name: str = None) -> None:
benji_obj = None
try:
benji_obj = Benji(self.config)
objects_count, objects_size = benji_obj.storage_stats(storage_name)
if self.machine_output:
result = {
'objects_count': objects_count,
'objects_size': objects_size,
}
print(json.dumps(result, indent=4))
else:
self._ls_storage_stats_table_output(objects_count, objects_size)
finally:
if benji_obj:
benji_obj.close()
def benjiOpen(self, initdb=False, block_size=None, in_memory=False):
self.benji = Benji(self.config,
initdb=initdb,
_destroydb=initdb,
_migratedb=False,
block_size=block_size,
in_memory=in_memory)
return self.benji
def _enforce_retention_policy(
self, rules_spec: fields.Str(required=True),
filter_expression: fields.Str(missing=None),
dry_run: fields.Bool(missing=False),
keep_metadata_backup: fields.Bool(missing=False),
group_label: fields.Str(missing=None)
) -> None:
benji_obj = None
try:
benji_obj = Benji(self._config)
dismissed_versions = benji_obj.enforce_retention_policy(
filter_expression=filter_expression,
rules_spec=rules_spec,
dry_run=dry_run,
keep_metadata_backup=keep_metadata_backup,
group_label=group_label)
result = StringIO()
benji_obj.export_any(
{'versions': dismissed_versions},
result,
ignore_relationships=[((Version, ), ('blocks', ))],
)
return result
finally:
if benji_obj:
benji_obj.close()
def stats(self, version_uid, limit=None):
version_uid = VersionUid.create_from_readables(version_uid)
if limit:
limit = int(limit)
if limit <= 0:
raise benji.exception.UsageError(
'Limit has to be a positive integer.')
benji_obj = None
try:
benji_obj = Benji(self.config)
stats = benji_obj.stats(version_uid, limit)
if self.machine_output:
stats = list(
stats) # resolve iterator, otherwise it's not serializable
benji_obj.export_any(
{'stats': stats},
sys.stdout,
)
else:
self._stats_tbl_output(stats)
finally:
if benji_obj:
benji_obj.close()
def metadata_backup(self,
filter_expression: str,
force: bool = False) -> None:
with Benji(self.config) as benji_obj:
version_uid_objs = [
version.uid for version in benji_obj.find_versions_with_filter(
filter_expression)
]
benji_obj.metadata_backup(version_uid_objs, overwrite=force)
def _api_v1_versions_metadata_restore_create(
self, version_uids: fields.DelimitedList(fields.Str, required=True),
storage_name: fields.Str(missing=None)
) -> None:
version_uid_objs = [
VersionUid(version_uid) for version_uid in version_uids
]
with Benji(self._config) as benji_obj:
benji_obj.metadata_restore(version_uid_objs, storage_name)
def cleanup(self, full):
benji_obj = None
try:
benji_obj = Benji(self.config)
if full:
benji_obj.cleanup_full()
else:
benji_obj.cleanup_fast()
finally:
if benji_obj:
benji_obj.close()
def _api_v1_storages_read(self, storage_name: str) -> Dict[str, int]:
with Benji(self._config) as benji_obj:
objects_count, objects_size = benji_obj.storage_stats(storage_name)
result = {
'objects_count': objects_count,
'objects_size': objects_size,
}
return result
def metadata_import(self, input_file: str = None) -> None:
benji_obj = None
try:
benji_obj = Benji(self.config)
if input_file is None:
benji_obj.metadata_import(sys.stdin)
else:
with open(input_file, 'r') as f:
benji_obj.metadata_import(f)
finally:
if benji_obj:
benji_obj.close()
def _api_v1_versions_metadata_backup_create(
self, filter_expression: fields.Str(missing=None),
force: fields.Bool(missing=False)
) -> None:
with Benji(self._config) as benji_obj:
version_uid_objs = [
version.uid for version in benji_obj.find_versions_with_filter(
filter_expression)
]
benji_obj.metadata_backup(version_uid_objs, overwrite=force)
def import_(self, input_file=None):
benji_obj = None
try:
benji_obj = Benji(self.config)
if input_file is None:
benji_obj.import_(sys.stdin)
else:
with open(input_file, 'r') as f:
benji_obj.import_(f)
finally:
if benji_obj:
benji_obj.close()
def benjiOpen(self,
init_database=False,
block_size=None,
in_memory_database=False):
self.benji = Benji(self.config,
block_size=block_size,
init_database=init_database,
in_memory_database=in_memory_database,
_destroy_database=init_database)
return self.benji
def metadata_export(self, filter_expression: Optional[str], output_file: Optional[str], force: bool) -> None:
benji_obj = None
try:
benji_obj = Benji(self.config)
version_uid_objs = [version.uid for version in benji_obj.ls_with_filter(filter_expression)]
if output_file is None:
benji_obj.metadata_export(version_uid_objs, sys.stdout)
else:
if os.path.exists(output_file) and not force:
raise FileExistsError('The output file already exists.')
with open(output_file, 'w') as f:
benji_obj.metadata_export(version_uid_objs, f)
finally:
if benji_obj:
benji_obj.close()
def metadata_backup(self, filter_expression: str, force: bool = False) -> None:
benji_obj = None
try:
benji_obj = Benji(self.config)
version_uid_objs = [version.uid for version in benji_obj.ls_with_filter(filter_expression)]
benji_obj.metadata_backup(version_uid_objs, overwrite=force)
finally:
if benji_obj:
benji_obj.close()