def __init__(self, *, config: Config, name: str, storage_id: int, module_configuration: ConfigDict) -> None: read_cache_directory = Config.get_from_dict(module_configuration, 'readCache.directory', None, types=str) read_cache_maximum_size = Config.get_from_dict(module_configuration, 'readCache.maximumSize', None, types=int) read_cache_shards = Config.get_from_dict(module_configuration, 'readCache.shards', None, types=int) if read_cache_directory and read_cache_maximum_size: os.makedirs(read_cache_directory, exist_ok=True) try: self._read_cache = FanoutCache( read_cache_directory, size_limit=read_cache_maximum_size, shards=read_cache_shards, eviction_policy='least-frequently-used', statistics=1, ) except Exception: logger.warning('Unable to enable disk based read caching. Continuing without it.') self._read_cache = None else: logger.debug('Disk based read caching instantiated (cache size {}, shards {}).'.format( read_cache_maximum_size, read_cache_shards)) else: self._read_cache = None self._use_read_cache = True # Start reader and write threads after the disk cached is created, so that they see it. super().__init__(config=config, name=name, storage_id=storage_id, module_configuration=module_configuration)
def encapsulate(self, *, data: bytes) -> Tuple[Optional[bytes], Optional[Dict]]: if self._ecc_key.has_private(): logger.warning( 'ECC key loaded from config includes private key data, which is not needed for encryption.' ) return super().encapsulate(data=data)
def close(self): self._log_compression_statistics() if len(self._read_futures) > 0: logger.warning( 'Data backend closed with {} outstanding read jobs, cancelling them.' .format(len(self._read_futures))) for future in self._read_futures: future.cancel() logger.debug('Data backend cancelled all outstanding read jobs.') # Get all jobs so that the semaphore gets released and still waiting jobs can complete for future in self.read_get_completed(): pass logger.debug( 'Data backend read results from all outstanding read jobs.') if len(self._write_futures) > 0: logger.warning( 'Data backend closed with {} outstanding write jobs, cancelling them.' .format(len(self._write_futures))) for future in self._write_futures: future.cancel() logger.debug('Data backend cancelled all outstanding write jobs.') # Write jobs release their semaphore at completion so we don't need to collect the results self._write_futures = [] self._write_executor.shutdown() self._read_executor.shutdown()
def close(self) -> None: if len(self._read_queue) > 0: logger.warning('Closing IO module with {} read outstanding jobs.'.format(self._name, len(self._read_queue))) self._read_queue = [] if self._outstanding_write is not None: logger.warning('Closing IO module with one outstanding write.'.format(self._name)) self._outstanding_write = None
def filter(self, versions: Sequence[Version]) -> List[Version]: # Category labels without latest categories = [ category for category in self.rules.keys() if category != 'latest' ] for category in categories: setattr(self, '_{}_dict'.format(category), defaultdict(list)) # Make our own copy versions = list(versions) # Sort from youngest to oldest versions.sort(key=lambda version: version.date.timestamp(), reverse=True) # Remove latest versions from consideration if configured if 'latest' in self.rules: logger.debug('Keeping {} latest versions.'.format( self.rules['latest'])) del versions[:self.rules['latest']] dismissed_versions = [] for version in versions: try: td = _Timedelta(version.date.timestamp(), self.reference_time) except _TimedeltaError as exception: # Err on the safe side, ignore this versions (i.e. it won't be dismissed) logger.warning('Version {}: {}'.format(version.uid.v_string, exception)) continue logger.debug( 'Time and time delta for version {} are {} and {}.'.format( version.uid.v_string, version.date, td)) for category in categories: timecount = getattr(td, category) if timecount <= self.rules[category]: logger.debug( 'Found matching category {}, timecount {}.'.format( category, timecount)) getattr( self, '_{}_dict'.format(category))[timecount].append(version) break else: # For loop did not break: The item doesn't fit into any category, # it's too old dismissed_versions.append(version) logger.debug( 'Dismissing version, it doesn\'t fit into any category.') for category in categories: category_dict = getattr(self, '_{}_dict'.format(category)) for timecount in category_dict: # Keep the oldest of each category, reject the rest dismissed_versions.extend(category_dict[timecount][:-1]) return dismissed_versions
def close(self) -> None: if len(self._read_queue) > 0: logger.warning( 'Closing IO module with {} outstanding read jobs.'.format( len(self._read_queue))) self._read_queue = [] if self._outstanding_write is not None: logger.warning('Closing IO module with one outstanding write.') self._outstanding_write = None self._iscsi_context = None
def shutdown(self) -> None: if len(self._futures) > 0: logger.warning('Job executor "{}" is being shutdown with {} outstanding jobs, cancelling them.'.format( self._name, len(self._futures))) for future in self._futures: future.cancel() logger.debug('Job executor "{}" cancelled all outstanding jobs.'.format(self._name)) if not self._blocking_submit: # Get all jobs so that the semaphore gets released and still waiting jobs can complete for _ in self.get_completed(): pass logger.debug('Job executor "{}" read results for all outstanding jobs.'.format(self._name)) self._executor.shutdown()
def protect(self, version_uids): version_uids = VersionUid.create_from_readables(version_uids) benji_obj = None try: benji_obj = Benji(self.config) for version_uid in version_uids: try: benji_obj.protect(version_uid) except benji.exception.NoChange: logger.warning('Version {} already was protected.'.format( version_uid)) finally: if benji_obj: benji_obj.close()
def rm_tag(self, version_uid, names): version_uid = VersionUid.create_from_readables(version_uid) benji_obj = None try: benji_obj = Benji(self.config) for name in names: try: benji_obj.rm_tag(version_uid, name) except benji.exception.NoChange: logger.warning('Version {} has no tag {}.'.format( version_uid, name)) finally: if benji_obj: benji_obj.close()
def future_results_as_completed(futures: List[Future], semaphore=None, timeout: int = None) -> Iterator[Any]: if sys.version_info < (3, 6, 4): logger.warning('Large backup jobs are likely to fail because of excessive memory usage. ' + 'Upgrade your Python to at least 3.6.4.') for future in concurrent.futures.as_completed(futures, timeout=timeout): futures.remove(future) if semaphore and not future.cancelled(): semaphore.release() try: result = future.result() except Exception as exception: result = exception del future yield result
def _write_object(self, key: str, data: bytes) -> None: for i in range(self._write_object_attempts): try: self.bucket.upload_bytes(data, key) except (B2Error, B2ConnectionError): if i + 1 < self._write_object_attempts: sleep_time = (2 **(i + 1)) + (random.randint(0, 1000) / 1000) logger.warning( 'Upload of object with key {} to B2 failed repeatedly, will try again in {:.2f} seconds.' .format(key, sleep_time)) time.sleep(sleep_time) continue raise else: break
def _write_object(self, key: str, data: bytes) -> None: for i in range(self._write_object_attempts): try: self.bucket.upload_bytes(data, key) # This is overly broad! except B2Error as exception: if i + 1 < self._write_object_attempts: sleep_time = (2 **(i + 1)) + (random.randint(0, 1000) / 1000) logger.warning( 'Upload of object with key {} to B2 failed repeatedly, will try again in {:.2f} seconds. Exception thrown was {}' .format(key, sleep_time, str(exception))) time.sleep(sleep_time) continue raise else: break
def close(self): """ Close the io """ if self._read_executor: if len(self._read_futures) > 0: logger.warning( 'IO backend closed with {} outstanding read jobs, cancelling them.' .format(len(self._read_futures))) for future in self._read_futures: future.cancel() logger.debug('IO backend cancelled all outstanding read jobs.') # Get all jobs so that the semaphore gets released and still waiting jobs can complete for future in self.read_get_completed(): pass logger.debug( 'IO backend read results from all outstanding read jobs.') self._read_executor.shutdown()
def _read_object_length(self, key: str) -> int: for i in range(self._read_object_attempts): try: file_version_info = self._file_info(key) except (B2Error, B2ConnectionError) as exception: if isinstance(exception, FileNotPresent): raise FileNotFoundError( 'Object {} not found.'.format(key)) from None else: if i + 1 < self._read_object_attempts: sleep_time = (2**(i + 1)) + (random.randint(0, 1000) / 1000) logger.warning( 'Object length request for key {} to B2 failed, will try again in {:.2f} seconds.' .format(key, sleep_time)) time.sleep(sleep_time) continue raise else: break return file_version_info.size
def _read_object_length(self, key: str) -> int: for i in range(self._read_object_attempts): try: file_version_info = self.bucket.get_file_info_by_name(key) # This is overly broad! except B2Error as exception: if isinstance(exception, FileNotPresent): raise FileNotFoundError( 'Object {} not found.'.format(key)) from None else: if i + 1 < self._read_object_attempts: sleep_time = (2**(i + 1)) + (random.randint(0, 1000) / 1000) logger.warning( 'Object length request for key {} to B2 failed, will try again in {:.2f} seconds. Exception thrown was {}' .format(key, sleep_time, str(exception))) time.sleep(sleep_time) continue raise else: break return int(file_version_info.size)
def _read_object(self, key: str) -> bytes: for i in range(self._read_object_attempts): data_io = DownloadDestBytes() try: self.bucket.download_file_by_name(key, data_io) except (B2Error, B2ConnectionError) as exception: if isinstance(exception, FileNotPresent): raise FileNotFoundError( 'Object {} not found.'.format(key)) from None else: if i + 1 < self._read_object_attempts: sleep_time = (2**(i + 1)) + (random.randint(0, 1000) / 1000) logger.warning( 'Download of object with key {} to B2 failed, will try again in {:.2f} seconds.' .format(key, sleep_time)) time.sleep(sleep_time) continue raise else: break return data_io.get_bytes_written()
def __init__(self, config): read_cache_directory = config.get('dataBackend.readCache.directory', None, types=str) read_cache_maximum_size = config.get( 'dataBackend.readCache.maximumSize', None, types=int) if read_cache_directory and not read_cache_maximum_size or not read_cache_directory and read_cache_maximum_size: raise ConfigurationError( 'Both dataBackend.readCache.directory and dataBackend.readCache.maximumSize need to be set ' + 'to enable disk based caching.') if read_cache_directory and read_cache_maximum_size: os.makedirs(read_cache_directory, exist_ok=True) try: self._read_cache = Cache( read_cache_directory, size_limit=read_cache_maximum_size, eviction_policy='least-frequently-used', statistics=1, ) except Exception: logger.warning( 'Unable to enable disk based read caching. Continuing without it.' ) self._read_cache = None else: logger.debug( 'Disk based read caching instantiated (cache size {}).'. format(read_cache_maximum_size)) else: self._read_cache = None self._use_read_cache = True # Start reader and write threads after the disk cached is created, so that they see it. super().__init__(config)
def main(): if sys.hexversion < 0x030600F0: raise InternalError('Benji only supports Python 3.6 or above.') parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, allow_abbrev=False) parser.add_argument('-c', '--config-file', default=None, type=str, help='Specify a non-default configuration file') parser.add_argument('-m', '--machine-output', action='store_true', default=False, help='Enable machine-readable JSON output') parser.add_argument( '--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'], default='INFO', help='Only log messages of this level or above on the console') parser.add_argument('--no-color', action='store_true', default=False, help='Disable colorization of console logging') subparsers_root = parser.add_subparsers(title='commands') # BACKUP p = subparsers_root.add_parser('backup', help='Perform a backup') p.add_argument('-s', '--snapshot-name', default='', help='Snapshot name (e.g. the name of the RBD snapshot)') p.add_argument('-r', '--rbd-hints', default=None, help='Hints in rbd diff JSON format') p.add_argument('-f', '--base-version', dest='base_version_uid', default=None, help='Base version UID') p.add_argument('-b', '--block-size', type=int, default=None, help='Block size in bytes') p.add_argument('-l', '--label', action='append', dest='labels', metavar='label', default=None, help='Labels for this version (can be repeated)') p.add_argument( '-S', '--storage', default='', help='Destination storage (if unspecified the default is used)') p.add_argument('source', help='Source URL') p.add_argument('version_name', help='Backup version name (e.g. the hostname)') p.set_defaults(func='backup') # BATCH-DEEP-SCRUB p = subparsers_root.add_parser( 'batch-deep-scrub', help='Check data and metadata integrity of multiple versions at once', formatter_class=argparse.ArgumentDefaultsHelpFormatter) p.add_argument('-p', '--block-percentage', type=partial(integer_range, 1, 100), default=100, help='Check only a certain percentage of blocks') p.add_argument('-P', '--version-percentage', type=partial(integer_range, 1, 100), default=100, help='Check only a certain percentage of versions') p.add_argument('-g', '--group_label', default=None, help='Label to find related versions') p.add_argument('filter_expression', nargs='?', default=None, help='Version filter expression') p.set_defaults(func='batch_deep_scrub') # BATCH-SCRUB p = subparsers_root.add_parser( 'batch-scrub', help= 'Check block existence and metadata integrity of multiple versions at once', formatter_class=argparse.ArgumentDefaultsHelpFormatter) p.add_argument('-p', '--block-percentage', type=partial(integer_range, 1, 100), default=100, help='Check only a certain percentage of blocks') p.add_argument('-P', '--version-percentage', type=partial(integer_range, 1, 100), default=100, help='Check only a certain percentage of versions') p.add_argument('-g', '--group_label', default=None, help='Label to find related versions') p.add_argument('filter_expression', nargs='?', default=None, help='Version filter expression') p.set_defaults(func='batch_scrub') # CLEANUP p = subparsers_root.add_parser('cleanup', help='Cleanup no longer referenced blocks') p.add_argument('--override-lock', action='store_true', help='Override and release any held lock (dangerous)') p.set_defaults(func='cleanup') # COMPLETION p = subparsers_root.add_parser('completion', help='Emit autocompletion script') p.add_argument('shell', choices=['bash', 'tcsh'], help='Shell') p.set_defaults(func='completion') # DATABASE-INIT p = subparsers_root.add_parser( 'database-init', help='Initialize the database (will not delete existing tables or data)' ) p.set_defaults(func='database_init') # DATABASE-MIGRATE p = subparsers_root.add_parser( 'database-migrate', help='Migrate an existing database to a new schema revision') p.set_defaults(func='database_migrate') # DEEP-SCRUB p = subparsers_root.add_parser( 'deep-scrub', help='Check a version\'s data and metadata integrity', formatter_class=argparse.ArgumentDefaultsHelpFormatter) p.add_argument('-s', '--source', default=None, help='Additionally compare version against source URL') p.add_argument('-p', '--block-percentage', type=partial(integer_range, 1, 100), default=100, help='Check only a certain percentage of blocks') p.add_argument('version_uid', help='Version UID') p.set_defaults(func='deep_scrub') # ENFORCE p = subparsers_root.add_parser('enforce', help="Enforce a retention policy ") p.add_argument('--dry-run', action='store_true', help='Only show which versions would be removed') p.add_argument('-k', '--keep-metadata-backup', action='store_true', help='Keep version metadata backup') p.add_argument('-g', '--group_label', default=None, help='Label to find related versions to remove') p.add_argument('rules_spec', help='Retention rules specification') p.add_argument('filter_expression', nargs='?', default=None, help='Version filter expression') p.set_defaults(func='enforce_retention_policy') # LABEL p = subparsers_root.add_parser('label', help='Add labels to a version') p.add_argument('version_uid') p.add_argument('labels', nargs='+') p.set_defaults(func='label') # LS p = subparsers_root.add_parser('ls', help='List versions') p.add_argument('filter_expression', nargs='?', default=None, help='Version filter expression') p.add_argument('-l', '--include-labels', action='store_true', help='Include labels in output') p.add_argument('-s', '--include-stats', action='store_true', help='Include statistics in output') p.set_defaults(func='ls') # METADATA-BACKUP p = subparsers_root.add_parser( 'metadata-backup', help='Back up the metadata of one or more versions') p.add_argument('filter_expression', help="Version filter expression") p.add_argument('-f', '--force', action='store_true', help='Overwrite existing metadata backups') p.set_defaults(func='metadata_backup') # METADATA EXPORT p = subparsers_root.add_parser( 'metadata-export', help= 'Export the metadata of one or more versions to a file or standard output' ) p.add_argument('filter_expression', nargs='?', default=None, help="Version filter expression") p.add_argument('-f', '--force', action='store_true', help='Overwrite an existing output file') p.add_argument('-o', '--output-file', default=None, help='Output file (standard output if missing)') p.set_defaults(func='metadata_export') # METADATA-IMPORT p = subparsers_root.add_parser( 'metadata-import', help= 'Import the metadata of one or more versions from a file or standard input' ) p.add_argument('-i', '--input-file', default=None, help='Input file (standard input if missing)') p.set_defaults(func='metadata_import') # METADATA-LS p = subparsers_root.add_parser('metadata-ls', help='List the version metadata backup') p.add_argument('-S', '--storage', default=None, help='Source storage (if unspecified the default is used)') p.set_defaults(func='metadata_ls') # METADATA-RESTORE p = subparsers_root.add_parser( 'metadata-restore', help='Restore the metadata of one ore more versions') p.add_argument('-S', '--storage', default=None, help='Source storage (if unspecified the default is used)') p.add_argument('version_uids', metavar='VERSION_UID', nargs='+', help="Version UID") p.set_defaults(func='metadata_restore') # NBD p = subparsers_root.add_parser( 'nbd', help='Start an NBD server', formatter_class=argparse.ArgumentDefaultsHelpFormatter) p.add_argument('-a', '--bind-address', default='127.0.0.1', help='Bind to the specified IP address') p.add_argument('-p', '--bind-port', default=10809, help='Bind to the specified port') p.add_argument('-r', '--read-only', action='store_true', default=False, help='NBD device is read-only') p.set_defaults(func='nbd') # PROTECT p = subparsers_root.add_parser('protect', help='Protect one or more versions') p.add_argument('version_uids', metavar='version_uid', nargs='+', help="Version UID") p.set_defaults(func='protect') # RESTORE p = subparsers_root.add_parser('restore', help='Restore a backup') p.add_argument('-s', '--sparse', action='store_true', help='Restore only existing blocks') p.add_argument('-f', '--force', action='store_true', help='Overwrite an existing file, device or image') p.add_argument('-d', '--database-backend-less', action='store_true', help='Restore without requiring the database backend') p.add_argument('version_uid', help='Version UID to restore') p.add_argument('destination', help='Destination URL') p.set_defaults(func='restore') # RM p = subparsers_root.add_parser('rm', help='Remove one or more versions') p.add_argument( '-f', '--force', action='store_true', help='Force removal (overrides protection of recent versions)') p.add_argument('-k', '--keep-metadata-backup', action='store_true', help='Keep version metadata backup') p.add_argument('--override-lock', action='store_true', help='Override and release any held locks (dangerous)') p.add_argument('version_uids', metavar='version_uid', nargs='+', help='Version UID') p.set_defaults(func='rm') # SCRUB p = subparsers_root.add_parser( 'scrub', help='Check a version\'s block existence and metadata integrity', formatter_class=argparse.ArgumentDefaultsHelpFormatter) p.add_argument('-p', '--block-percentage', type=partial(integer_range, 1, 100), default=100, help='Check only a certain percentage of blocks') p.add_argument('version_uid', help='Version UID') p.set_defaults(func='scrub') # STORAGE-STATS p = subparsers_root.add_parser('storage-stats', help='Show storage statistics') p.add_argument('storage_name', nargs='?', default=None, help='Storage') p.set_defaults(func='storage_stats') # UNPROTECT p = subparsers_root.add_parser('unprotect', help='Unprotect one or more versions') p.add_argument('version_uids', metavar='version_uid', nargs='+', help='Version UID') p.set_defaults(func='unprotect') # VERSION-INFO p = subparsers_root.add_parser('version-info', help='Program version information') p.set_defaults(func='version_info') argcomplete.autocomplete(parser) args = parser.parse_args() if not hasattr(args, 'func'): parser.print_usage() sys.exit(os.EX_USAGE) if args.func == 'completion': completion(args.shell) sys.exit(os.EX_OK) from benji.config import Config from benji.logging import logger, init_logging if args.config_file is not None and args.config_file != '': try: cfg = open(args.config_file, 'r', encoding='utf-8').read() except FileNotFoundError: logger.error('File {} not found.'.format(args.config_file)) sys.exit(os.EX_USAGE) config = Config(ad_hoc_config=cfg) else: config = Config() init_logging(config.get('logFile', types=(str, type(None))), console_level=args.log_level, console_formatter='console-plain' if args.no_color else 'console-colored') if sys.hexversion < 0x030604F0: logger.warning( 'The installed Python version will use excessive amounts of memory when used with Benji. Upgrade Python to at least 3.6.4.' ) import benji.commands commands = benji.commands.Commands(args.machine_output, config) func = getattr(commands, args.func) # Pass over to function func_args = dict(args._get_kwargs()) del func_args['config_file'] del func_args['func'] del func_args['log_level'] del func_args['machine_output'] del func_args['no_color'] # From most specific to least specific exception_mappings = [ _ExceptionMapping(exception=benji.exception.UsageError, exit_code=os.EX_USAGE), _ExceptionMapping(exception=benji.exception.AlreadyLocked, exit_code=os.EX_NOPERM), _ExceptionMapping(exception=benji.exception.InternalError, exit_code=os.EX_SOFTWARE), _ExceptionMapping(exception=benji.exception.ConfigurationError, exit_code=os.EX_CONFIG), _ExceptionMapping(exception=benji.exception.InputDataError, exit_code=os.EX_DATAERR), _ExceptionMapping(exception=benji.exception.ScrubbingError, exit_code=os.EX_DATAERR), _ExceptionMapping(exception=PermissionError, exit_code=os.EX_NOPERM), _ExceptionMapping(exception=FileExistsError, exit_code=os.EX_CANTCREAT), _ExceptionMapping(exception=FileNotFoundError, exit_code=os.EX_NOINPUT), _ExceptionMapping(exception=EOFError, exit_code=os.EX_IOERR), _ExceptionMapping(exception=IOError, exit_code=os.EX_IOERR), _ExceptionMapping(exception=OSError, exit_code=os.EX_OSERR), _ExceptionMapping(exception=ConnectionError, exit_code=os.EX_IOERR), _ExceptionMapping(exception=LookupError, exit_code=os.EX_NOINPUT), _ExceptionMapping(exception=KeyboardInterrupt, exit_code=os.EX_NOINPUT), _ExceptionMapping(exception=BaseException, exit_code=os.EX_SOFTWARE), ] try: logger.debug('commands.{0}(**{1!r})'.format(args.func, func_args)) func(**func_args) sys.exit(os.EX_OK) except SystemExit: raise except BaseException as exception: for case in exception_mappings: if isinstance(exception, case.exception): message = str(exception) if message: message = '{}: {}'.format(exception.__class__.__name__, message) else: message = '{} exception occurred.'.format( exception.__class__.__name__) logger.debug(message, exc_info=True) logger.error(message) sys.exit(case.exit_code)
def _filter( self, versions: Union[Sequence[Version], Set[Version]] ) -> Tuple[List[Version], Dict[str, Dict[int, List[Version]]]]: # Category labels without latest categories = [ category for category in self.rules.keys() if category != 'latest' ] versions_by_category: Dict[str, Dict[int, List[Version]]] = {} versions_by_category_remaining: Dict[str, Dict[int, List[Version]]] = {} for category in categories: versions_by_category[category] = defaultdict(list) versions_by_category_remaining[category] = {} # Make our own copy versions = list(versions) # Sort from youngest to oldest versions.sort(key=lambda version: version.date, reverse=True) # Remove latest versions from consideration if configured if 'latest' in self.rules: logger.debug('Keeping {} latest versions.'.format( self.rules['latest'])) versions_by_category_remaining['latest'] = { 0: versions[:self.rules['latest']] } del versions[:self.rules['latest']] dismissed_versions = [] for version in versions: try: # version.date is naive and in UTC, attach time zone to make it time zone aware. td = _Timedelta( version.date.replace(tzinfo=datetime.timezone.utc), self.reference_time, tz=self.tz) except ValueError as exception: # Err on the safe side, ignore this versions (i.e. it won't be dismissed) logger.warning('Version {}: {}.'.format( version.uid.v_string, exception)) continue logger.debug( 'Time and time delta for version {} are {} and {}.'.format( version.uid.v_string, version.date.isoformat(timespec='seconds'), td)) for category in categories: timecount = getattr(td, category) if timecount <= self.rules[category]: logger.debug( 'Found matching category {}, timecount {}.'.format( category, timecount)) versions_by_category[category][timecount].append(version) break else: # For loop did not break: The item doesn't fit into any category, it's too old. dismissed_versions.append(version) logger.debug( 'Dismissing version, it doesn\'t fit into any category.') for category in categories: for timecount in versions_by_category[category]: # Keep the oldest of each category, reject the rest dismissed_versions.extend( versions_by_category[category][timecount][:-1]) versions_by_category_remaining[category][ timecount] = versions_by_category[category][timecount][-1:] return dismissed_versions, versions_by_category_remaining
def __init__(self, *, config: Config, name: str, module_configuration: ConfigDict): super().__init__(config=config, name=name, module_configuration=module_configuration) account_id = Config.get_from_dict(module_configuration, 'accountId', None, types=str) if account_id is None: account_id_file = Config.get_from_dict(module_configuration, 'accountIdFile', types=str) with open(account_id_file, 'r') as f: account_id = f.read().rstrip() application_key = Config.get_from_dict(module_configuration, 'applicationKey', None, types=str) if application_key is None: application_key_file = Config.get_from_dict(module_configuration, 'applicationKeyFile', types=str) with open(application_key_file, 'r') as f: application_key = f.read().rstrip() bucket_name = Config.get_from_dict(module_configuration, 'bucketName', types=str) account_info_file = Config.get_from_dict(module_configuration, 'accountInfoFile', None, types=str) if account_info_file is not None: account_info = SqliteAccountInfo(file_name=account_info_file) else: account_info = InMemoryAccountInfo() b2sdk.bucket.Bucket.MAX_UPLOAD_ATTEMPTS = Config.get_from_dict( module_configuration, 'uploadAttempts', types=int) self._write_object_attempts = Config.get_from_dict( module_configuration, 'writeObjectAttempts', types=int) self._read_object_attempts = Config.get_from_dict(module_configuration, 'readObjectAttempts', types=int) self.service = b2sdk.api.B2Api(account_info) if account_info_file is not None: try: # This temporarily disables all logging as the b2 library does some very verbose logging # of the exception we're trying to catch here... logging.disable(logging.ERROR) _ = self.service.get_account_id() logging.disable(logging.NOTSET) except MissingAccountData: self.service.authorize_account('production', account_id, application_key) else: self.service.authorize_account('production', account_id, application_key) self.bucket = self.service.get_bucket_by_name(bucket_name) # Check bucket configuration bucket_type = self.bucket.type_ if bucket_type != 'allPrivate': logger.warning( f'The type of bucket {bucket_name} is {bucket_type}. ' 'It is strongly recommended to set it to allPrivate.')