def __init__(self, *, config: Config, name: str, storage_id: int, module_configuration: ConfigDict) -> None:
read_cache_directory = Config.get_from_dict(module_configuration, 'readCache.directory', None, types=str)
read_cache_maximum_size = Config.get_from_dict(module_configuration, 'readCache.maximumSize', None, types=int)
read_cache_shards = Config.get_from_dict(module_configuration, 'readCache.shards', None, types=int)
if read_cache_directory and read_cache_maximum_size:
os.makedirs(read_cache_directory, exist_ok=True)
try:
self._read_cache = FanoutCache(
read_cache_directory,
size_limit=read_cache_maximum_size,
shards=read_cache_shards,
eviction_policy='least-frequently-used',
statistics=1,
)
except Exception:
logger.warning('Unable to enable disk based read caching. Continuing without it.')
self._read_cache = None
else:
logger.debug('Disk based read caching instantiated (cache size {}, shards {}).'.format(
read_cache_maximum_size, read_cache_shards))
else:
self._read_cache = None
self._use_read_cache = True
# Start reader and write threads after the disk cached is created, so that they see it.
super().__init__(config=config, name=name, storage_id=storage_id, module_configuration=module_configuration)
def encapsulate(self, *,
data: bytes) -> Tuple[Optional[bytes], Optional[Dict]]:
if self._ecc_key.has_private():
logger.warning(
'ECC key loaded from config includes private key data, which is not needed for encryption.'
)
return super().encapsulate(data=data)
def close(self):
self._log_compression_statistics()
if len(self._read_futures) > 0:
logger.warning(
'Data backend closed with {} outstanding read jobs, cancelling them.'
.format(len(self._read_futures)))
for future in self._read_futures:
future.cancel()
logger.debug('Data backend cancelled all outstanding read jobs.')
# Get all jobs so that the semaphore gets released and still waiting jobs can complete
for future in self.read_get_completed():
pass
logger.debug(
'Data backend read results from all outstanding read jobs.')
if len(self._write_futures) > 0:
logger.warning(
'Data backend closed with {} outstanding write jobs, cancelling them.'
.format(len(self._write_futures)))
for future in self._write_futures:
future.cancel()
logger.debug('Data backend cancelled all outstanding write jobs.')
# Write jobs release their semaphore at completion so we don't need to collect the results
self._write_futures = []
self._write_executor.shutdown()
self._read_executor.shutdown()
def close(self) -> None:
if len(self._read_queue) > 0:
logger.warning('Closing IO module with {} read outstanding jobs.'.format(self._name, len(self._read_queue)))
self._read_queue = []
if self._outstanding_write is not None:
logger.warning('Closing IO module with one outstanding write.'.format(self._name))
self._outstanding_write = None
def filter(self, versions: Sequence[Version]) -> List[Version]:
# Category labels without latest
categories = [
category for category in self.rules.keys() if category != 'latest'
]
for category in categories:
setattr(self, '_{}_dict'.format(category), defaultdict(list))
# Make our own copy
versions = list(versions)
# Sort from youngest to oldest
versions.sort(key=lambda version: version.date.timestamp(),
reverse=True)
# Remove latest versions from consideration if configured
if 'latest' in self.rules:
logger.debug('Keeping {} latest versions.'.format(
self.rules['latest']))
del versions[:self.rules['latest']]
dismissed_versions = []
for version in versions:
try:
td = _Timedelta(version.date.timestamp(), self.reference_time)
except _TimedeltaError as exception:
# Err on the safe side, ignore this versions (i.e. it won't be dismissed)
logger.warning('Version {}: {}'.format(version.uid.v_string,
exception))
continue
logger.debug(
'Time and time delta for version {} are {} and {}.'.format(
version.uid.v_string, version.date, td))
for category in categories:
timecount = getattr(td, category)
if timecount <= self.rules[category]:
logger.debug(
'Found matching category {}, timecount {}.'.format(
category, timecount))
getattr(
self,
'_{}_dict'.format(category))[timecount].append(version)
break
else:
# For loop did not break: The item doesn't fit into any category,
# it's too old
dismissed_versions.append(version)
logger.debug(
'Dismissing version, it doesn\'t fit into any category.')
for category in categories:
category_dict = getattr(self, '_{}_dict'.format(category))
for timecount in category_dict:
# Keep the oldest of each category, reject the rest
dismissed_versions.extend(category_dict[timecount][:-1])
return dismissed_versions
def close(self) -> None:
if len(self._read_queue) > 0:
logger.warning(
'Closing IO module with {} outstanding read jobs.'.format(
len(self._read_queue)))
self._read_queue = []
if self._outstanding_write is not None:
logger.warning('Closing IO module with one outstanding write.')
self._outstanding_write = None
self._iscsi_context = None
def shutdown(self) -> None:
if len(self._futures) > 0:
logger.warning('Job executor "{}" is being shutdown with {} outstanding jobs, cancelling them.'.format(
self._name, len(self._futures)))
for future in self._futures:
future.cancel()
logger.debug('Job executor "{}" cancelled all outstanding jobs.'.format(self._name))
if not self._blocking_submit:
# Get all jobs so that the semaphore gets released and still waiting jobs can complete
for _ in self.get_completed():
pass
logger.debug('Job executor "{}" read results for all outstanding jobs.'.format(self._name))
self._executor.shutdown()
def protect(self, version_uids):
version_uids = VersionUid.create_from_readables(version_uids)
benji_obj = None
try:
benji_obj = Benji(self.config)
for version_uid in version_uids:
try:
benji_obj.protect(version_uid)
except benji.exception.NoChange:
logger.warning('Version {} already was protected.'.format(
version_uid))
finally:
if benji_obj:
benji_obj.close()
def rm_tag(self, version_uid, names):
version_uid = VersionUid.create_from_readables(version_uid)
benji_obj = None
try:
benji_obj = Benji(self.config)
for name in names:
try:
benji_obj.rm_tag(version_uid, name)
except benji.exception.NoChange:
logger.warning('Version {} has no tag {}.'.format(
version_uid, name))
finally:
if benji_obj:
benji_obj.close()
def future_results_as_completed(futures: List[Future], semaphore=None, timeout: int = None) -> Iterator[Any]:
if sys.version_info < (3, 6, 4):
logger.warning('Large backup jobs are likely to fail because of excessive memory usage. ' +
'Upgrade your Python to at least 3.6.4.')
for future in concurrent.futures.as_completed(futures, timeout=timeout):
futures.remove(future)
if semaphore and not future.cancelled():
semaphore.release()
try:
result = future.result()
except Exception as exception:
result = exception
del future
yield result
def _write_object(self, key: str, data: bytes) -> None:
for i in range(self._write_object_attempts):
try:
self.bucket.upload_bytes(data, key)
except (B2Error, B2ConnectionError):
if i + 1 < self._write_object_attempts:
sleep_time = (2
**(i + 1)) + (random.randint(0, 1000) / 1000)
logger.warning(
'Upload of object with key {} to B2 failed repeatedly, will try again in {:.2f} seconds.'
.format(key, sleep_time))
time.sleep(sleep_time)
continue
raise
else:
break
def _write_object(self, key: str, data: bytes) -> None:
for i in range(self._write_object_attempts):
try:
self.bucket.upload_bytes(data, key)
# This is overly broad!
except B2Error as exception:
if i + 1 < self._write_object_attempts:
sleep_time = (2
**(i + 1)) + (random.randint(0, 1000) / 1000)
logger.warning(
'Upload of object with key {} to B2 failed repeatedly, will try again in {:.2f} seconds. Exception thrown was {}'
.format(key, sleep_time, str(exception)))
time.sleep(sleep_time)
continue
raise
else:
break
def close(self):
""" Close the io
"""
if self._read_executor:
if len(self._read_futures) > 0:
logger.warning(
'IO backend closed with {} outstanding read jobs, cancelling them.'
.format(len(self._read_futures)))
for future in self._read_futures:
future.cancel()
logger.debug('IO backend cancelled all outstanding read jobs.')
# Get all jobs so that the semaphore gets released and still waiting jobs can complete
for future in self.read_get_completed():
pass
logger.debug(
'IO backend read results from all outstanding read jobs.')
self._read_executor.shutdown()
def _read_object_length(self, key: str) -> int:
for i in range(self._read_object_attempts):
try:
file_version_info = self._file_info(key)
except (B2Error, B2ConnectionError) as exception:
if isinstance(exception, FileNotPresent):
raise FileNotFoundError(
'Object {} not found.'.format(key)) from None
else:
if i + 1 < self._read_object_attempts:
sleep_time = (2**(i + 1)) + (random.randint(0, 1000) /
1000)
logger.warning(
'Object length request for key {} to B2 failed, will try again in {:.2f} seconds.'
.format(key, sleep_time))
time.sleep(sleep_time)
continue
raise
else:
break
return file_version_info.size
def _read_object_length(self, key: str) -> int:
for i in range(self._read_object_attempts):
try:
file_version_info = self.bucket.get_file_info_by_name(key)
# This is overly broad!
except B2Error as exception:
if isinstance(exception, FileNotPresent):
raise FileNotFoundError(
'Object {} not found.'.format(key)) from None
else:
if i + 1 < self._read_object_attempts:
sleep_time = (2**(i + 1)) + (random.randint(0, 1000) /
1000)
logger.warning(
'Object length request for key {} to B2 failed, will try again in {:.2f} seconds. Exception thrown was {}'
.format(key, sleep_time, str(exception)))
time.sleep(sleep_time)
continue
raise
else:
break
return int(file_version_info.size)
def _read_object(self, key: str) -> bytes:
for i in range(self._read_object_attempts):
data_io = DownloadDestBytes()
try:
self.bucket.download_file_by_name(key, data_io)
except (B2Error, B2ConnectionError) as exception:
if isinstance(exception, FileNotPresent):
raise FileNotFoundError(
'Object {} not found.'.format(key)) from None
else:
if i + 1 < self._read_object_attempts:
sleep_time = (2**(i + 1)) + (random.randint(0, 1000) /
1000)
logger.warning(
'Download of object with key {} to B2 failed, will try again in {:.2f} seconds.'
.format(key, sleep_time))
time.sleep(sleep_time)
continue
raise
else:
break
return data_io.get_bytes_written()
def __init__(self, config):
read_cache_directory = config.get('dataBackend.readCache.directory',
None,
types=str)
read_cache_maximum_size = config.get(
'dataBackend.readCache.maximumSize', None, types=int)
if read_cache_directory and not read_cache_maximum_size or not read_cache_directory and read_cache_maximum_size:
raise ConfigurationError(
'Both dataBackend.readCache.directory and dataBackend.readCache.maximumSize need to be set '
+ 'to enable disk based caching.')
if read_cache_directory and read_cache_maximum_size:
os.makedirs(read_cache_directory, exist_ok=True)
try:
self._read_cache = Cache(
read_cache_directory,
size_limit=read_cache_maximum_size,
eviction_policy='least-frequently-used',
statistics=1,
)
except Exception:
logger.warning(
'Unable to enable disk based read caching. Continuing without it.'
)
self._read_cache = None
else:
logger.debug(
'Disk based read caching instantiated (cache size {}).'.
format(read_cache_maximum_size))
else:
self._read_cache = None
self._use_read_cache = True
# Start reader and write threads after the disk cached is created, so that they see it.
super().__init__(config)
def main():
if sys.hexversion < 0x030600F0:
raise InternalError('Benji only supports Python 3.6 or above.')
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
allow_abbrev=False)
parser.add_argument('-c',
'--config-file',
default=None,
type=str,
help='Specify a non-default configuration file')
parser.add_argument('-m',
'--machine-output',
action='store_true',
default=False,
help='Enable machine-readable JSON output')
parser.add_argument(
'--log-level',
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'],
default='INFO',
help='Only log messages of this level or above on the console')
parser.add_argument('--no-color',
action='store_true',
default=False,
help='Disable colorization of console logging')
subparsers_root = parser.add_subparsers(title='commands')
# BACKUP
p = subparsers_root.add_parser('backup', help='Perform a backup')
p.add_argument('-s',
'--snapshot-name',
default='',
help='Snapshot name (e.g. the name of the RBD snapshot)')
p.add_argument('-r',
'--rbd-hints',
default=None,
help='Hints in rbd diff JSON format')
p.add_argument('-f',
'--base-version',
dest='base_version_uid',
default=None,
help='Base version UID')
p.add_argument('-b',
'--block-size',
type=int,
default=None,
help='Block size in bytes')
p.add_argument('-l',
'--label',
action='append',
dest='labels',
metavar='label',
default=None,
help='Labels for this version (can be repeated)')
p.add_argument(
'-S',
'--storage',
default='',
help='Destination storage (if unspecified the default is used)')
p.add_argument('source', help='Source URL')
p.add_argument('version_name',
help='Backup version name (e.g. the hostname)')
p.set_defaults(func='backup')
# BATCH-DEEP-SCRUB
p = subparsers_root.add_parser(
'batch-deep-scrub',
help='Check data and metadata integrity of multiple versions at once',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
p.add_argument('-p',
'--block-percentage',
type=partial(integer_range, 1, 100),
default=100,
help='Check only a certain percentage of blocks')
p.add_argument('-P',
'--version-percentage',
type=partial(integer_range, 1, 100),
default=100,
help='Check only a certain percentage of versions')
p.add_argument('-g',
'--group_label',
default=None,
help='Label to find related versions')
p.add_argument('filter_expression',
nargs='?',
default=None,
help='Version filter expression')
p.set_defaults(func='batch_deep_scrub')
# BATCH-SCRUB
p = subparsers_root.add_parser(
'batch-scrub',
help=
'Check block existence and metadata integrity of multiple versions at once',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
p.add_argument('-p',
'--block-percentage',
type=partial(integer_range, 1, 100),
default=100,
help='Check only a certain percentage of blocks')
p.add_argument('-P',
'--version-percentage',
type=partial(integer_range, 1, 100),
default=100,
help='Check only a certain percentage of versions')
p.add_argument('-g',
'--group_label',
default=None,
help='Label to find related versions')
p.add_argument('filter_expression',
nargs='?',
default=None,
help='Version filter expression')
p.set_defaults(func='batch_scrub')
# CLEANUP
p = subparsers_root.add_parser('cleanup',
help='Cleanup no longer referenced blocks')
p.add_argument('--override-lock',
action='store_true',
help='Override and release any held lock (dangerous)')
p.set_defaults(func='cleanup')
# COMPLETION
p = subparsers_root.add_parser('completion',
help='Emit autocompletion script')
p.add_argument('shell', choices=['bash', 'tcsh'], help='Shell')
p.set_defaults(func='completion')
# DATABASE-INIT
p = subparsers_root.add_parser(
'database-init',
help='Initialize the database (will not delete existing tables or data)'
)
p.set_defaults(func='database_init')
# DATABASE-MIGRATE
p = subparsers_root.add_parser(
'database-migrate',
help='Migrate an existing database to a new schema revision')
p.set_defaults(func='database_migrate')
# DEEP-SCRUB
p = subparsers_root.add_parser(
'deep-scrub',
help='Check a version\'s data and metadata integrity',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
p.add_argument('-s',
'--source',
default=None,
help='Additionally compare version against source URL')
p.add_argument('-p',
'--block-percentage',
type=partial(integer_range, 1, 100),
default=100,
help='Check only a certain percentage of blocks')
p.add_argument('version_uid', help='Version UID')
p.set_defaults(func='deep_scrub')
# ENFORCE
p = subparsers_root.add_parser('enforce',
help="Enforce a retention policy ")
p.add_argument('--dry-run',
action='store_true',
help='Only show which versions would be removed')
p.add_argument('-k',
'--keep-metadata-backup',
action='store_true',
help='Keep version metadata backup')
p.add_argument('-g',
'--group_label',
default=None,
help='Label to find related versions to remove')
p.add_argument('rules_spec', help='Retention rules specification')
p.add_argument('filter_expression',
nargs='?',
default=None,
help='Version filter expression')
p.set_defaults(func='enforce_retention_policy')
# LABEL
p = subparsers_root.add_parser('label', help='Add labels to a version')
p.add_argument('version_uid')
p.add_argument('labels', nargs='+')
p.set_defaults(func='label')
# LS
p = subparsers_root.add_parser('ls', help='List versions')
p.add_argument('filter_expression',
nargs='?',
default=None,
help='Version filter expression')
p.add_argument('-l',
'--include-labels',
action='store_true',
help='Include labels in output')
p.add_argument('-s',
'--include-stats',
action='store_true',
help='Include statistics in output')
p.set_defaults(func='ls')
# METADATA-BACKUP
p = subparsers_root.add_parser(
'metadata-backup', help='Back up the metadata of one or more versions')
p.add_argument('filter_expression', help="Version filter expression")
p.add_argument('-f',
'--force',
action='store_true',
help='Overwrite existing metadata backups')
p.set_defaults(func='metadata_backup')
# METADATA EXPORT
p = subparsers_root.add_parser(
'metadata-export',
help=
'Export the metadata of one or more versions to a file or standard output'
)
p.add_argument('filter_expression',
nargs='?',
default=None,
help="Version filter expression")
p.add_argument('-f',
'--force',
action='store_true',
help='Overwrite an existing output file')
p.add_argument('-o',
'--output-file',
default=None,
help='Output file (standard output if missing)')
p.set_defaults(func='metadata_export')
# METADATA-IMPORT
p = subparsers_root.add_parser(
'metadata-import',
help=
'Import the metadata of one or more versions from a file or standard input'
)
p.add_argument('-i',
'--input-file',
default=None,
help='Input file (standard input if missing)')
p.set_defaults(func='metadata_import')
# METADATA-LS
p = subparsers_root.add_parser('metadata-ls',
help='List the version metadata backup')
p.add_argument('-S',
'--storage',
default=None,
help='Source storage (if unspecified the default is used)')
p.set_defaults(func='metadata_ls')
# METADATA-RESTORE
p = subparsers_root.add_parser(
'metadata-restore',
help='Restore the metadata of one ore more versions')
p.add_argument('-S',
'--storage',
default=None,
help='Source storage (if unspecified the default is used)')
p.add_argument('version_uids',
metavar='VERSION_UID',
nargs='+',
help="Version UID")
p.set_defaults(func='metadata_restore')
# NBD
p = subparsers_root.add_parser(
'nbd',
help='Start an NBD server',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
p.add_argument('-a',
'--bind-address',
default='127.0.0.1',
help='Bind to the specified IP address')
p.add_argument('-p',
'--bind-port',
default=10809,
help='Bind to the specified port')
p.add_argument('-r',
'--read-only',
action='store_true',
default=False,
help='NBD device is read-only')
p.set_defaults(func='nbd')
# PROTECT
p = subparsers_root.add_parser('protect',
help='Protect one or more versions')
p.add_argument('version_uids',
metavar='version_uid',
nargs='+',
help="Version UID")
p.set_defaults(func='protect')
# RESTORE
p = subparsers_root.add_parser('restore', help='Restore a backup')
p.add_argument('-s',
'--sparse',
action='store_true',
help='Restore only existing blocks')
p.add_argument('-f',
'--force',
action='store_true',
help='Overwrite an existing file, device or image')
p.add_argument('-d',
'--database-backend-less',
action='store_true',
help='Restore without requiring the database backend')
p.add_argument('version_uid', help='Version UID to restore')
p.add_argument('destination', help='Destination URL')
p.set_defaults(func='restore')
# RM
p = subparsers_root.add_parser('rm', help='Remove one or more versions')
p.add_argument(
'-f',
'--force',
action='store_true',
help='Force removal (overrides protection of recent versions)')
p.add_argument('-k',
'--keep-metadata-backup',
action='store_true',
help='Keep version metadata backup')
p.add_argument('--override-lock',
action='store_true',
help='Override and release any held locks (dangerous)')
p.add_argument('version_uids',
metavar='version_uid',
nargs='+',
help='Version UID')
p.set_defaults(func='rm')
# SCRUB
p = subparsers_root.add_parser(
'scrub',
help='Check a version\'s block existence and metadata integrity',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
p.add_argument('-p',
'--block-percentage',
type=partial(integer_range, 1, 100),
default=100,
help='Check only a certain percentage of blocks')
p.add_argument('version_uid', help='Version UID')
p.set_defaults(func='scrub')
# STORAGE-STATS
p = subparsers_root.add_parser('storage-stats',
help='Show storage statistics')
p.add_argument('storage_name', nargs='?', default=None, help='Storage')
p.set_defaults(func='storage_stats')
# UNPROTECT
p = subparsers_root.add_parser('unprotect',
help='Unprotect one or more versions')
p.add_argument('version_uids',
metavar='version_uid',
nargs='+',
help='Version UID')
p.set_defaults(func='unprotect')
# VERSION-INFO
p = subparsers_root.add_parser('version-info',
help='Program version information')
p.set_defaults(func='version_info')
argcomplete.autocomplete(parser)
args = parser.parse_args()
if not hasattr(args, 'func'):
parser.print_usage()
sys.exit(os.EX_USAGE)
if args.func == 'completion':
completion(args.shell)
sys.exit(os.EX_OK)
from benji.config import Config
from benji.logging import logger, init_logging
if args.config_file is not None and args.config_file != '':
try:
cfg = open(args.config_file, 'r', encoding='utf-8').read()
except FileNotFoundError:
logger.error('File {} not found.'.format(args.config_file))
sys.exit(os.EX_USAGE)
config = Config(ad_hoc_config=cfg)
else:
config = Config()
init_logging(config.get('logFile', types=(str, type(None))),
console_level=args.log_level,
console_formatter='console-plain'
if args.no_color else 'console-colored')
if sys.hexversion < 0x030604F0:
logger.warning(
'The installed Python version will use excessive amounts of memory when used with Benji. Upgrade Python to at least 3.6.4.'
)
import benji.commands
commands = benji.commands.Commands(args.machine_output, config)
func = getattr(commands, args.func)
# Pass over to function
func_args = dict(args._get_kwargs())
del func_args['config_file']
del func_args['func']
del func_args['log_level']
del func_args['machine_output']
del func_args['no_color']
# From most specific to least specific
exception_mappings = [
_ExceptionMapping(exception=benji.exception.UsageError,
exit_code=os.EX_USAGE),
_ExceptionMapping(exception=benji.exception.AlreadyLocked,
exit_code=os.EX_NOPERM),
_ExceptionMapping(exception=benji.exception.InternalError,
exit_code=os.EX_SOFTWARE),
_ExceptionMapping(exception=benji.exception.ConfigurationError,
exit_code=os.EX_CONFIG),
_ExceptionMapping(exception=benji.exception.InputDataError,
exit_code=os.EX_DATAERR),
_ExceptionMapping(exception=benji.exception.ScrubbingError,
exit_code=os.EX_DATAERR),
_ExceptionMapping(exception=PermissionError, exit_code=os.EX_NOPERM),
_ExceptionMapping(exception=FileExistsError,
exit_code=os.EX_CANTCREAT),
_ExceptionMapping(exception=FileNotFoundError,
exit_code=os.EX_NOINPUT),
_ExceptionMapping(exception=EOFError, exit_code=os.EX_IOERR),
_ExceptionMapping(exception=IOError, exit_code=os.EX_IOERR),
_ExceptionMapping(exception=OSError, exit_code=os.EX_OSERR),
_ExceptionMapping(exception=ConnectionError, exit_code=os.EX_IOERR),
_ExceptionMapping(exception=LookupError, exit_code=os.EX_NOINPUT),
_ExceptionMapping(exception=KeyboardInterrupt,
exit_code=os.EX_NOINPUT),
_ExceptionMapping(exception=BaseException, exit_code=os.EX_SOFTWARE),
]
try:
logger.debug('commands.{0}(**{1!r})'.format(args.func, func_args))
func(**func_args)
sys.exit(os.EX_OK)
except SystemExit:
raise
except BaseException as exception:
for case in exception_mappings:
if isinstance(exception, case.exception):
message = str(exception)
if message:
message = '{}: {}'.format(exception.__class__.__name__,
message)
else:
message = '{} exception occurred.'.format(
exception.__class__.__name__)
logger.debug(message, exc_info=True)
logger.error(message)
sys.exit(case.exit_code)
def _filter(
self, versions: Union[Sequence[Version], Set[Version]]
) -> Tuple[List[Version], Dict[str, Dict[int, List[Version]]]]:
# Category labels without latest
categories = [
category for category in self.rules.keys() if category != 'latest'
]
versions_by_category: Dict[str, Dict[int, List[Version]]] = {}
versions_by_category_remaining: Dict[str, Dict[int,
List[Version]]] = {}
for category in categories:
versions_by_category[category] = defaultdict(list)
versions_by_category_remaining[category] = {}
# Make our own copy
versions = list(versions)
# Sort from youngest to oldest
versions.sort(key=lambda version: version.date, reverse=True)
# Remove latest versions from consideration if configured
if 'latest' in self.rules:
logger.debug('Keeping {} latest versions.'.format(
self.rules['latest']))
versions_by_category_remaining['latest'] = {
0: versions[:self.rules['latest']]
}
del versions[:self.rules['latest']]
dismissed_versions = []
for version in versions:
try:
# version.date is naive and in UTC, attach time zone to make it time zone aware.
td = _Timedelta(
version.date.replace(tzinfo=datetime.timezone.utc),
self.reference_time,
tz=self.tz)
except ValueError as exception:
# Err on the safe side, ignore this versions (i.e. it won't be dismissed)
logger.warning('Version {}: {}.'.format(
version.uid.v_string, exception))
continue
logger.debug(
'Time and time delta for version {} are {} and {}.'.format(
version.uid.v_string,
version.date.isoformat(timespec='seconds'), td))
for category in categories:
timecount = getattr(td, category)
if timecount <= self.rules[category]:
logger.debug(
'Found matching category {}, timecount {}.'.format(
category, timecount))
versions_by_category[category][timecount].append(version)
break
else:
# For loop did not break: The item doesn't fit into any category, it's too old.
dismissed_versions.append(version)
logger.debug(
'Dismissing version, it doesn\'t fit into any category.')
for category in categories:
for timecount in versions_by_category[category]:
# Keep the oldest of each category, reject the rest
dismissed_versions.extend(
versions_by_category[category][timecount][:-1])
versions_by_category_remaining[category][
timecount] = versions_by_category[category][timecount][-1:]
return dismissed_versions, versions_by_category_remaining
def __init__(self, *, config: Config, name: str,
module_configuration: ConfigDict):
super().__init__(config=config,
name=name,
module_configuration=module_configuration)
account_id = Config.get_from_dict(module_configuration,
'accountId',
None,
types=str)
if account_id is None:
account_id_file = Config.get_from_dict(module_configuration,
'accountIdFile',
types=str)
with open(account_id_file, 'r') as f:
account_id = f.read().rstrip()
application_key = Config.get_from_dict(module_configuration,
'applicationKey',
None,
types=str)
if application_key is None:
application_key_file = Config.get_from_dict(module_configuration,
'applicationKeyFile',
types=str)
with open(application_key_file, 'r') as f:
application_key = f.read().rstrip()
bucket_name = Config.get_from_dict(module_configuration,
'bucketName',
types=str)
account_info_file = Config.get_from_dict(module_configuration,
'accountInfoFile',
None,
types=str)
if account_info_file is not None:
account_info = SqliteAccountInfo(file_name=account_info_file)
else:
account_info = InMemoryAccountInfo()
b2sdk.bucket.Bucket.MAX_UPLOAD_ATTEMPTS = Config.get_from_dict(
module_configuration, 'uploadAttempts', types=int)
self._write_object_attempts = Config.get_from_dict(
module_configuration, 'writeObjectAttempts', types=int)
self._read_object_attempts = Config.get_from_dict(module_configuration,
'readObjectAttempts',
types=int)
self.service = b2sdk.api.B2Api(account_info)
if account_info_file is not None:
try:
# This temporarily disables all logging as the b2 library does some very verbose logging
# of the exception we're trying to catch here...
logging.disable(logging.ERROR)
_ = self.service.get_account_id()
logging.disable(logging.NOTSET)
except MissingAccountData:
self.service.authorize_account('production', account_id,
application_key)
else:
self.service.authorize_account('production', account_id,
application_key)
self.bucket = self.service.get_bucket_by_name(bucket_name)
# Check bucket configuration
bucket_type = self.bucket.type_
if bucket_type != 'allPrivate':
logger.warning(
f'The type of bucket {bucket_name} is {bucket_type}. '
'It is strongly recommended to set it to allPrivate.')
