def __init__(self, *, config: Config, name: str,
module_configuration: ConfigDict) -> None:
super().__init__(config=config,
name=name,
module_configuration=module_configuration)
master_key_encoded: Optional[str] = Config.get_from_dict(
module_configuration, 'masterKey', None, types=str)
if master_key_encoded is not None:
master_key = base64.b64decode(master_key_encoded)
if len(master_key) != 32:
raise ValueError(
'Key masterKey has the wrong length. It must be 32 bytes long and encoded as BASE64.'
)
self._master_key = master_key
else:
kdf_salt: bytes = base64.b64decode(
Config.get_from_dict(module_configuration,
'kdfSalt',
types=str))
kdf_iterations: int = Config.get_from_dict(module_configuration,
'kdfIterations',
types=int)
password: str = Config.get_from_dict(module_configuration,
'password',
types=str)
self._master_key = derive_key(salt=kdf_salt,
iterations=kdf_iterations,
key_length=32,
password=password)
def __init__(self, *, config: Config, name: str, storage_id: int, module_configuration: ConfigDict) -> None:
read_cache_directory = Config.get_from_dict(module_configuration, 'readCache.directory', None, types=str)
read_cache_maximum_size = Config.get_from_dict(module_configuration, 'readCache.maximumSize', None, types=int)
read_cache_shards = Config.get_from_dict(module_configuration, 'readCache.shards', None, types=int)
if read_cache_directory and read_cache_maximum_size:
os.makedirs(read_cache_directory, exist_ok=True)
try:
self._read_cache = FanoutCache(
read_cache_directory,
size_limit=read_cache_maximum_size,
shards=read_cache_shards,
eviction_policy='least-frequently-used',
statistics=1,
)
except Exception:
logger.warning('Unable to enable disk based read caching. Continuing without it.')
self._read_cache = None
else:
logger.debug('Disk based read caching instantiated (cache size {}, shards {}).'.format(
read_cache_maximum_size, read_cache_shards))
else:
self._read_cache = None
self._use_read_cache = True
# Start reader and write threads after the disk cached is created, so that they see it.
super().__init__(config=config, name=name, storage_id=storage_id, module_configuration=module_configuration)
def __init__(self, *, identifier, materials):
master_key = Config.get_from_dict(materials,
'masterKey',
None,
types=bytes)
if master_key is not None:
if len(master_key) != 32:
raise ValueError(
'Key masterKey has the wrong length. It must be 32 bytes long.'
)
self._master_key = master_key
else:
kdfSalt = Config.get_from_dict(materials, 'kdfSalt', types=bytes)
kdfIterations = Config.get_from_dict(materials,
'kdfIterations',
types=int)
password = Config.get_from_dict(materials, 'password', types=str)
self._master_key = derive_key(salt=kdfSalt,
iterations=kdfIterations,
key_length=32,
password=password)
self._identifier = identifier
def test_lists(self):
config = Config(cfg=self.CONFIG, merge_defaults=False)
self.assertTrue(type(config.get('io.rbd.newImageFeatures')) is list)
self.assertRaises(TypeError,
config.get('io.rbd.newImageFeatures', types=list))
self.assertEqual('RBD_FEATURE_EXCLUSIVE_LOCK',
config.get('io.rbd.newImageFeatures')[1])
def __init__(self, *, config: Config, name: str,
module_configuration: ConfigDict, path: str,
block_size: int) -> None:
super().__init__(config=config,
name=name,
module_configuration=module_configuration,
path=path,
block_size=block_size)
ceph_config_file = config.get_from_dict(module_configuration,
'cephConfigFile',
types=str)
client_identifier = config.get_from_dict(module_configuration,
'clientIdentifier',
types=str)
self._cluster = rados.Rados(conffile=ceph_config_file,
rados_id=client_identifier)
self._cluster.connect()
# create a bitwise or'd list of the configured features
self._new_image_features = 0
for feature in config.get_from_dict(module_configuration,
'newImageFeatures',
types=list):
try:
self._new_image_features = self._new_image_features | getattr(
rbd, feature)
except AttributeError:
raise ConfigurationError(
'{}: Unknown image feature {}.'.format(
module_configuration.full_name, feature))
self._pool_name = None
self._image_name = None
self._snapshot_name = None
def __init__(self, *, config: Config, name: str,
module_configuration: ConfigDict) -> None:
ecc_key_der: str = Config.get_from_dict(module_configuration,
'eccKey',
types=str)
ecc_curve: Optional[str] = Config.get_from_dict(module_configuration,
'eccCurve',
'NIST P-384',
types=str)
ecc_key = self._unpack_envelope_key(base64.b64decode(ecc_key_der))
if ecc_key.curve != ecc_curve:
raise ValueError(
f'Key eccKey does not match the eccCurve setting (found: {ecc_key.curve}, expected: {ecc_curve}).'
)
self._ecc_key = ecc_key
self._ecc_curve = ecc_key.curve
point_q_len = self._ecc_key.pointQ.size_in_bytes()
if point_q_len < self.AES_KEY_LEN:
raise ValueError(
f'Size of point Q is smaller than the AES key length, which reduces security ({point_q_len} < {self.AES_KEY_LEN}).'
)
# Note: We don't actually have a "master" aes key, because the key is derived from the ECC key
# and set before calling the parent's encapsulate/decapsulate method.
aes_config = module_configuration.copy()
aes_config['masterKey'] = base64.b64encode(
b'\x00' * self.AES_KEY_LEN).decode('ascii')
super().__init__(config=config,
name=name,
module_configuration=aes_config)
def __init__(self, *, config: Config, name: str,
module_configuration: ConfigDict) -> None:
super().__init__(config=config,
name=name,
module_configuration=module_configuration)
self.level: str = Config.get_from_dict(
module_configuration,
'level',
types=int,
check_func=lambda v: v >= 1 and v <= zstandard.
MAX_COMPRESSION_LEVEL,
check_message='Option level must be between 1 and {} (inclusive)'.
format(zstandard.MAX_COMPRESSION_LEVEL))
dict_data_file: str = Config.get_from_dict(module_configuration,
'dictDataFile',
None,
types=str)
if dict_data_file:
with open(dict_data_file, 'rb') as f:
dict_data_content = f.read()
self._dict_data = zstandard.ZstdCompressionDict(
dict_data_content, dict_type=zstandard.DICT_TYPE_FULLDICT)
self._dict_data.precompute_compress(self.level)
else:
self._dict_data = None
self._local = threading.local()
def _import_modules(cls, config: Config, modules: ConfigList) -> None:
for index, module_dict in enumerate(modules):
module = Config.get_from_dict(module_dict,
'module',
types=str,
full_name_override=modules.full_name,
index=index)
name = Config.get_from_dict(module_dict,
'name',
types=str,
full_name_override=modules.full_name,
index=index)
configuration = Config.get_from_dict(module_dict,
'configuration',
None,
types=dict,
full_name_override=modules.full_name,
index=index)
if name in cls._modules:
raise ConfigurationError('Duplicate name "{}" in list {}.'.format(name, modules.full_name))
module = importlib.import_module('{}.{}'.format(__package__, module))
try:
configuration = config.validate(module=module.__name__, config=configuration)
except ConfigurationError as exception:
raise ConfigurationError('Configuration for IO {} is invalid.'.format(name)) from exception
cls._modules[name] = _IOFactoryModule(module=module,
arguments={
'config': config,
'name': name,
'module_configuration': configuration
})
def __init__(self, *, config: Config, name: str, module_configuration: ConfigDict, url: str,
block_size: int) -> None:
super().__init__(config=config,
name=name,
module_configuration=module_configuration,
url=url,
block_size=block_size)
if self.parsed_url.username or self.parsed_url.password or self.parsed_url.hostname or self.parsed_url.port \
or self.parsed_url.params or self.parsed_url.fragment or self.parsed_url.query:
raise UsageError('The supplied URL {} is invalid.'.format(self.url))
ceph_config_file = config.get_from_dict(module_configuration, 'cephConfigFile', types=str)
client_identifier = config.get_from_dict(module_configuration, 'clientIdentifier', types=str)
self._cluster = rados.Rados(conffile=ceph_config_file, rados_id=client_identifier)
self._cluster.connect()
# create a bitwise or'd list of the configured features
self._new_image_features = 0
for feature in config.get_from_dict(module_configuration, 'newImageFeatures', types=list):
try:
self._new_image_features = self._new_image_features | getattr(rbd, feature)
except AttributeError:
raise ConfigurationError('{}: Unknown image feature {}.'.format(module_configuration.full_name, feature))
self._pool_name = None
self._image_name = None
self._snapshot_name = None
self._simultaneous_reads = config.get_from_dict(module_configuration, 'simultaneousReads', types=int)
self._simultaneous_writes = config.get_from_dict(module_configuration, 'simultaneousWrites', types=int)
self._read_executor: Optional[JobExecutor] = None
self._write_executor: Optional[JobExecutor] = None
def upgrade():
op.create_table(
'storages',
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_storages')),
sa.UniqueConstraint('name', name=op.f('uq_storages_name')))
benji_config = op.get_context().config.attributes.get('benji_config', None)
if benji_config is not None:
storages = sa.Table('storages',
sa.MetaData(),
autoload_with=op.get_bind())
storages_list = benji_config.get('storages', types=list)
for index, storage in enumerate(storages_list):
name = Config.get_from_dict(storage,
'name',
types=str,
index=index)
storage_id = Config.get_from_dict(storage,
'storageId',
None,
types=int,
index=index)
op.execute(storages.insert().values(name=name, id=storage_id))
with op.batch_alter_table('versions', schema=None) as batch_op:
batch_op.create_foreign_key(
batch_op.f('fk_versions_storage_id_storages'), 'storages',
['storage_id'], ['id'])
with op.batch_alter_table('deleted_blocks', schema=None) as batch_op:
batch_op.create_foreign_key(
batch_op.f('fk_deleted_blocks_storage_id_storages'), 'storages',
['storage_id'], ['id'])
def __init__(self, *, config: Config, name: str,
module_configuration: ConfigDict, url: str,
block_size: int) -> None:
super().__init__(config=config,
name=name,
module_configuration=module_configuration,
url=url,
block_size=block_size)
if self.parsed_url.username or self.parsed_url.password or self.parsed_url.hostname or self.parsed_url.port \
or self.parsed_url.params or self.parsed_url.fragment or self.parsed_url.query:
raise UsageError('The supplied URL {} is invalid.'.format(
self.url))
ceph_config_file = config.get_from_dict(module_configuration,
'cephConfigFile',
types=str)
client_identifier = config.get_from_dict(module_configuration,
'clientIdentifier',
types=str)
self._cluster = rados.Rados(conffile=ceph_config_file,
rados_id=client_identifier)
self._cluster.connect()
# create a bitwise or'd list of the configured features
self._new_image_features = 0
for feature in config.get_from_dict(module_configuration,
'newImageFeatures',
types=list):
try:
self._new_image_features = self._new_image_features | getattr(
rbd, feature)
except AttributeError:
raise ConfigurationError(
'{}: Unknown image feature {}.'.format(
module_configuration.full_name, feature))
self._pool_name = None
self._image_name = None
self._snapshot_name = None
self._rbd_image = None
self._simultaneous_reads = config.get_from_dict(module_configuration,
'simultaneousReads',
types=int)
self._simultaneous_writes = config.get_from_dict(module_configuration,
'simultaneousWrites',
types=int)
self._read_queue: Deque[DereferencedBlock] = deque()
self._write_queue: Deque[Tuple[DereferencedBlock, bytes]] = deque()
self._outstanding_aio_reads = 0
self._outstanding_aio_writes = 0
self._submitted_aio_writes = threading.BoundedSemaphore(
self._simultaneous_writes)
self._read_completion_queue: queue.Queue[Tuple[rbd.Completion, float,
float,
DereferencedBlock,
bytes]] = queue.Queue()
self._write_completion_queue: queue.Queue[Tuple[
rbd.Completion, float, float, DereferencedBlock]] = queue.Queue()
def __init__(self, *, config: Config, name: str, module_configuration: ConfigDict, url: str,
block_size: int) -> None:
super().__init__(
config=config, name=name, module_configuration=module_configuration, url=url, block_size=block_size)
self._simultaneous_reads = config.get_from_dict(module_configuration, 'simultaneousReads', types=int)
self._simultaneous_writes = config.get_from_dict(module_configuration, 'simultaneousWrites', types=int)
self._read_executor: Optional[JobExecutor] = None
self._write_executor: Optional[JobExecutor] = None
def test_default_overwrite(self):
config = Config(cfg="""
configurationVersion: '{}'
dataBackend:
simultaneousReads: 12345678
""".format(Config.CONFIG_VERSION),
merge_defaults=True)
self.assertEqual(12345678, config.get('dataBackend.simultaneousReads'))
self.assertEqual(1, config.get('io.rbd.simultaneousReads'))
def _import_modules(cls, config: Config, modules: ConfigList) -> None:
for index, module_dict in enumerate(modules):
module = Config.get_from_dict(module_dict,
'module',
types=str,
full_name_override=modules.full_name,
index=index)
name = Config.get_from_dict(module_dict,
'name',
types=str,
full_name_override=modules.full_name,
index=index)
storage_id = Config.get_from_dict(
module_dict,
'storageId',
types=int,
full_name_override=modules.full_name,
index=index)
configuration = Config.get_from_dict(
module_dict,
'configuration',
None,
types=dict,
full_name_override=modules.full_name,
index=index)
if name in cls._name_to_storage_id:
raise ConfigurationError(
'Duplicate name "{}" in list {}.'.format(
name, modules.full_name))
if storage_id in cls._storage_id_to_name:
raise ConfigurationError('Duplicate id {} in list {}.'.format(
storage_id, modules.full_name))
module = importlib.import_module('{}.{}.{}'.format(
__package__, cls._MODULE, module))
try:
configuration = config.validate(module=module.__name__,
config=configuration)
except ConfigurationError as exception:
raise ConfigurationError(
'Configuration for storage {} is invalid.'.format(
name)) from exception
cls._modules[storage_id] = _StorageFactoryModule(
module=module,
arguments={
'config': config,
'name': name,
'storage_id': storage_id,
'module_configuration': configuration
})
cls._name_to_storage_id[name] = storage_id
cls._storage_id_to_name[storage_id] = name
def __init__(self, *, config: Config, name: str,
module_configuration: ConfigDict, path: str,
block_size: int) -> None:
self._name = name
self._path = path
self._block_size = block_size
self._simultaneous_reads = config.get_from_dict(module_configuration,
'simultaneousReads',
types=int)
self._simultaneous_writes = config.get_from_dict(module_configuration,
'simultaneousWrites',
types=int)
self._read_executor: Optional[JobExecutor] = None
self._write_executor: Optional[JobExecutor] = None
def setUp(self):
self.testpath = _TestPath()
init_logging(console_level=logging.WARN if os.environ.get(
'UNITTEST_QUIET', False) else logging.DEBUG,
console_formatter='console-plain')
self.config = Config(ad_hoc_config=self.CONFIG.format(
testpath=self.testpath.path))
def setUp(self):
self.testpath = self.TestPath()
init_logging(None, logging.DEBUG)
self.config = Config(
cfg=self.CONFIG.format(testpath=self.testpath.path),
merge_defaults=False)
def __init__(self, *, config: Config, name: str, module_configuration: ConfigDict, url: str,
block_size: int) -> None:
super().__init__(config=config,
name=name,
module_configuration=module_configuration,
url=url,
block_size=block_size)
if self.parsed_url.username or self.parsed_url.password or self.parsed_url.hostname or self.parsed_url.port \
or self.parsed_url.params or self.parsed_url.fragment or self.parsed_url.query:
raise UsageError('The supplied URL {} is invalid.'.format(self.url))
self._simultaneous_reads = config.get_from_dict(module_configuration, 'simultaneousReads', types=int)
self._simultaneous_writes = config.get_from_dict(module_configuration, 'simultaneousWrites', types=int)
self._read_executor: Optional[JobExecutor] = None
self._write_executor: Optional[JobExecutor] = None
def test_validation(self):
config = Config(ad_hoc_config=self.CONFIG)
module_configuration = {'path': '/var/tmp'}
self.assertEqual(
{
'bandwidthRead': 0,
'bandwidthWrite': 0,
'consistencyCheckWrites': False,
'path': '/var/tmp',
'simultaneousReads': 3,
'simultaneousWrites': 3,
'simultaneousRemovals': 5,
},
config.validate(module='benji.storage.file',
config=module_configuration))
module_configuration = {'asdasdas': 'dasdasd'}
self.assertRaises(
ConfigurationError,
lambda: config.validate(module='benji.storage.file',
config=module_configuration))
module_configuration = {}
self.assertRaises(
ConfigurationError,
lambda: config.validate(module='benji.storage.file',
config=module_configuration))
module_configuration = {'path': '/var/tmp', 'bandwidthRead': -1}
self.assertRaises(
ConfigurationError,
lambda: config.validate(module='benji.storage.file',
config=module_configuration))
module_configuration = {'path': [1, 2, 3]}
self.assertRaises(
ConfigurationError,
lambda: config.validate(module='benji.storage.file',
config=module_configuration))
def __init__(self, *, materials):
self.level = Config.get_from_dict(
materials,
'level',
self.DEFAULT_LEVEL,
types=int,
check_func=lambda v: v >= 1 and v <= zstandard.
MAX_COMPRESSION_LEVEL,
check_message='Option level must be between 1 and {} (inclusive)'.
format(zstandard.MAX_COMPRESSION_LEVEL))
self.compressors = {}
self.decompressors = {}
def test_validation_io_rbd(self):
config = Config(ad_hoc_config=self.CONFIG)
module_configuration = config.get('ios')[0]['configuration']
self.assertEqual(
{
'cephConfigFile':
'/etc/ceph/ceph.conf',
'clientIdentifier':
'admin',
'newImageFeatures':
['RBD_FEATURE_LAYERING', 'RBD_FEATURE_EXCLUSIVE_LOCK'],
'simultaneousReads':
10,
'simultaneousWrites':
10,
},
config.validate(module='benji.io.rbd',
config=module_configuration))
module_configuration['newImageFeatures'] = ['ASASA', 'DDASAD']
self.assertRaises(
ConfigurationError,
lambda: config.validate(module='benji.io.rbd',
config=module_configuration))
def setUp(self):
self.testpath = self.TestPath()
init_logging(
None, logging.WARN
if os.environ.get('UNITTEST_QUIET', False) else logging.DEBUG)
# This disables ResourceWarnings from boto3 which are normal
# See: https://github.com/boto/boto3/issues/454
warnings.filterwarnings(
"ignore",
category=ResourceWarning,
message=r'unclosed.*<(?:ssl.SSLSocket|socket\.socket).*>')
self.config = Config(ad_hoc_config=self.CONFIG.format(
testpath=self.testpath.path))
def __init__(self, *, config: Config, name: str,
module_configuration: ConfigDict, url: str,
block_size: int) -> None:
super().__init__(config=config,
name=name,
module_configuration=module_configuration,
url=url,
block_size=block_size)
if self.parsed_url.params or self.parsed_url.fragment:
raise UsageError('The supplied URL {} is invalid.'.format(
self.url))
self._read_queue: List[DereferencedBlock] = []
self._outstanding_write: Optional[Tuple[DereferencedBlock,
bytes]] = None
self._username = config.get_from_dict(module_configuration,
'username',
None,
types=str)
self._password = config.get_from_dict(module_configuration,
'password',
None,
types=str)
self._target_username = config.get_from_dict(module_configuration,
'targetUsername',
None,
types=str)
self._target_password = config.get_from_dict(module_configuration,
'targetPassword',
None,
types=str)
header_digest = config.get_from_dict(module_configuration,
'headerDigest',
types=str)
header_digest_attr_name = 'ISCSI_HEADER_DIGEST_{}'.format(
header_digest)
if hasattr(libiscsi, header_digest_attr_name):
self._header_digest = getattr(libiscsi, header_digest_attr_name)
else:
raise ConfigurationError(
'Unknown header digest setting {}.'.format(header_digest))
self._initiator_name = config.get_from_dict(module_configuration,
'initiatorName',
types=str)
self._timeout = config.get_from_dict(module_configuration,
'timeout',
None,
types=int)
self._iscsi_context: Any = None
def __init__(self, *, config: Config, name: str,
module_configuration: ConfigDict):
super().__init__(config=config,
name=name,
module_configuration=module_configuration)
if os.sep != '/':
raise RuntimeError(
'This module only works with / as a path separator.')
self.path = Config.get_from_dict(module_configuration,
'path',
types=str)
# Ensure that self.path ends in os.path.sep
if not self.path.endswith(os.path.sep):
self.path = os.path.join(self.path, '')
def test_lists(self):
config = Config(ad_hoc_config=self.CONFIG)
ios = config.get('ios', types=list)
self.assertIsInstance(
Config.get_from_dict(ios[0], 'configuration.newImageFeatures'),
ConfigList)
self.assertRaises(
TypeError, lambda: Config.get_from_dict(
ios[0], 'configuration.newImageFeatures', types=int))
self.assertEqual(
'RBD_FEATURE_EXCLUSIVE_LOCK',
Config.get_from_dict(ios[0], 'configuration.newImageFeatures')[1])
def __init__(self, *, config: Config, name: str, storage_id: int,
module_configuration: ConfigDict):
super().__init__(config=config,
name=name,
storage_id=storage_id,
module_configuration=module_configuration)
if os.sep != '/':
raise RuntimeError(
'This module only works with / as a path separator.')
self.path = Config.get_from_dict(module_configuration,
'path',
types=str)
# Ensure that self.path ends in a slash
if not self.path.endswith('/'):
self.path = self.path + '/'
def main():
if sys.hexversion < 0x030600F0:
raise InternalError('Benji only supports Python 3.6 or above.')
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
allow_abbrev=False)
parser.add_argument('-c',
'--config-file',
default=None,
type=str,
help='Specify a non-default configuration file')
parser.add_argument('-m',
'--machine-output',
action='store_true',
default=False,
help='Enable machine-readable JSON output')
parser.add_argument(
'--log-level',
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'],
default='INFO',
help='Only log messages of this level or above on the console')
parser.add_argument('--no-color',
action='store_true',
default=False,
help='Disable colorization of console logging')
subparsers_root = parser.add_subparsers(title='commands')
# BACKUP
p = subparsers_root.add_parser('backup', help='Perform a backup')
p.add_argument('-s',
'--snapshot-name',
default='',
help='Snapshot name (e.g. the name of the RBD snapshot)')
p.add_argument('-r',
'--rbd-hints',
default=None,
help='Hints in rbd diff JSON format')
p.add_argument('-f',
'--base-version',
dest='base_version_uid',
default=None,
help='Base version UID')
p.add_argument('-b',
'--block-size',
type=int,
default=None,
help='Block size in bytes')
p.add_argument('-l',
'--label',
action='append',
dest='labels',
metavar='label',
default=None,
help='Labels for this version (can be repeated)')
p.add_argument(
'-S',
'--storage',
default='',
help='Destination storage (if unspecified the default is used)')
p.add_argument('source', help='Source URL')
p.add_argument('version_name',
help='Backup version name (e.g. the hostname)')
p.set_defaults(func='backup')
# BATCH-DEEP-SCRUB
p = subparsers_root.add_parser(
'batch-deep-scrub',
help='Check data and metadata integrity of multiple versions at once',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
p.add_argument('-p',
'--block-percentage',
type=partial(integer_range, 1, 100),
default=100,
help='Check only a certain percentage of blocks')
p.add_argument('-P',
'--version-percentage',
type=partial(integer_range, 1, 100),
default=100,
help='Check only a certain percentage of versions')
p.add_argument('-g',
'--group_label',
default=None,
help='Label to find related versions')
p.add_argument('filter_expression',
nargs='?',
default=None,
help='Version filter expression')
p.set_defaults(func='batch_deep_scrub')
# BATCH-SCRUB
p = subparsers_root.add_parser(
'batch-scrub',
help=
'Check block existence and metadata integrity of multiple versions at once',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
p.add_argument('-p',
'--block-percentage',
type=partial(integer_range, 1, 100),
default=100,
help='Check only a certain percentage of blocks')
p.add_argument('-P',
'--version-percentage',
type=partial(integer_range, 1, 100),
default=100,
help='Check only a certain percentage of versions')
p.add_argument('-g',
'--group_label',
default=None,
help='Label to find related versions')
p.add_argument('filter_expression',
nargs='?',
default=None,
help='Version filter expression')
p.set_defaults(func='batch_scrub')
# CLEANUP
p = subparsers_root.add_parser('cleanup',
help='Cleanup no longer referenced blocks')
p.add_argument('--override-lock',
action='store_true',
help='Override and release any held lock (dangerous)')
p.set_defaults(func='cleanup')
# COMPLETION
p = subparsers_root.add_parser('completion',
help='Emit autocompletion script')
p.add_argument('shell', choices=['bash', 'tcsh'], help='Shell')
p.set_defaults(func='completion')
# DATABASE-INIT
p = subparsers_root.add_parser(
'database-init',
help='Initialize the database (will not delete existing tables or data)'
)
p.set_defaults(func='database_init')
# DATABASE-MIGRATE
p = subparsers_root.add_parser(
'database-migrate',
help='Migrate an existing database to a new schema revision')
p.set_defaults(func='database_migrate')
# DEEP-SCRUB
p = subparsers_root.add_parser(
'deep-scrub',
help='Check a version\'s data and metadata integrity',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
p.add_argument('-s',
'--source',
default=None,
help='Additionally compare version against source URL')
p.add_argument('-p',
'--block-percentage',
type=partial(integer_range, 1, 100),
default=100,
help='Check only a certain percentage of blocks')
p.add_argument('version_uid', help='Version UID')
p.set_defaults(func='deep_scrub')
# ENFORCE
p = subparsers_root.add_parser('enforce',
help="Enforce a retention policy ")
p.add_argument('--dry-run',
action='store_true',
help='Only show which versions would be removed')
p.add_argument('-k',
'--keep-metadata-backup',
action='store_true',
help='Keep version metadata backup')
p.add_argument('-g',
'--group_label',
default=None,
help='Label to find related versions to remove')
p.add_argument('rules_spec', help='Retention rules specification')
p.add_argument('filter_expression',
nargs='?',
default=None,
help='Version filter expression')
p.set_defaults(func='enforce_retention_policy')
# LABEL
p = subparsers_root.add_parser('label', help='Add labels to a version')
p.add_argument('version_uid')
p.add_argument('labels', nargs='+')
p.set_defaults(func='label')
# LS
p = subparsers_root.add_parser('ls', help='List versions')
p.add_argument('filter_expression',
nargs='?',
default=None,
help='Version filter expression')
p.add_argument('-l',
'--include-labels',
action='store_true',
help='Include labels in output')
p.add_argument('-s',
'--include-stats',
action='store_true',
help='Include statistics in output')
p.set_defaults(func='ls')
# METADATA-BACKUP
p = subparsers_root.add_parser(
'metadata-backup', help='Back up the metadata of one or more versions')
p.add_argument('filter_expression', help="Version filter expression")
p.add_argument('-f',
'--force',
action='store_true',
help='Overwrite existing metadata backups')
p.set_defaults(func='metadata_backup')
# METADATA EXPORT
p = subparsers_root.add_parser(
'metadata-export',
help=
'Export the metadata of one or more versions to a file or standard output'
)
p.add_argument('filter_expression',
nargs='?',
default=None,
help="Version filter expression")
p.add_argument('-f',
'--force',
action='store_true',
help='Overwrite an existing output file')
p.add_argument('-o',
'--output-file',
default=None,
help='Output file (standard output if missing)')
p.set_defaults(func='metadata_export')
# METADATA-IMPORT
p = subparsers_root.add_parser(
'metadata-import',
help=
'Import the metadata of one or more versions from a file or standard input'
)
p.add_argument('-i',
'--input-file',
default=None,
help='Input file (standard input if missing)')
p.set_defaults(func='metadata_import')
# METADATA-LS
p = subparsers_root.add_parser('metadata-ls',
help='List the version metadata backup')
p.add_argument('-S',
'--storage',
default=None,
help='Source storage (if unspecified the default is used)')
p.set_defaults(func='metadata_ls')
# METADATA-RESTORE
p = subparsers_root.add_parser(
'metadata-restore',
help='Restore the metadata of one ore more versions')
p.add_argument('-S',
'--storage',
default=None,
help='Source storage (if unspecified the default is used)')
p.add_argument('version_uids',
metavar='VERSION_UID',
nargs='+',
help="Version UID")
p.set_defaults(func='metadata_restore')
# NBD
p = subparsers_root.add_parser(
'nbd',
help='Start an NBD server',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
p.add_argument('-a',
'--bind-address',
default='127.0.0.1',
help='Bind to the specified IP address')
p.add_argument('-p',
'--bind-port',
default=10809,
help='Bind to the specified port')
p.add_argument('-r',
'--read-only',
action='store_true',
default=False,
help='NBD device is read-only')
p.set_defaults(func='nbd')
# PROTECT
p = subparsers_root.add_parser('protect',
help='Protect one or more versions')
p.add_argument('version_uids',
metavar='version_uid',
nargs='+',
help="Version UID")
p.set_defaults(func='protect')
# RESTORE
p = subparsers_root.add_parser('restore', help='Restore a backup')
p.add_argument('-s',
'--sparse',
action='store_true',
help='Restore only existing blocks')
p.add_argument('-f',
'--force',
action='store_true',
help='Overwrite an existing file, device or image')
p.add_argument('-d',
'--database-backend-less',
action='store_true',
help='Restore without requiring the database backend')
p.add_argument('version_uid', help='Version UID to restore')
p.add_argument('destination', help='Destination URL')
p.set_defaults(func='restore')
# RM
p = subparsers_root.add_parser('rm', help='Remove one or more versions')
p.add_argument(
'-f',
'--force',
action='store_true',
help='Force removal (overrides protection of recent versions)')
p.add_argument('-k',
'--keep-metadata-backup',
action='store_true',
help='Keep version metadata backup')
p.add_argument('--override-lock',
action='store_true',
help='Override and release any held locks (dangerous)')
p.add_argument('version_uids',
metavar='version_uid',
nargs='+',
help='Version UID')
p.set_defaults(func='rm')
# SCRUB
p = subparsers_root.add_parser(
'scrub',
help='Check a version\'s block existence and metadata integrity',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
p.add_argument('-p',
'--block-percentage',
type=partial(integer_range, 1, 100),
default=100,
help='Check only a certain percentage of blocks')
p.add_argument('version_uid', help='Version UID')
p.set_defaults(func='scrub')
# STORAGE-STATS
p = subparsers_root.add_parser('storage-stats',
help='Show storage statistics')
p.add_argument('storage_name', nargs='?', default=None, help='Storage')
p.set_defaults(func='storage_stats')
# UNPROTECT
p = subparsers_root.add_parser('unprotect',
help='Unprotect one or more versions')
p.add_argument('version_uids',
metavar='version_uid',
nargs='+',
help='Version UID')
p.set_defaults(func='unprotect')
# VERSION-INFO
p = subparsers_root.add_parser('version-info',
help='Program version information')
p.set_defaults(func='version_info')
argcomplete.autocomplete(parser)
args = parser.parse_args()
if not hasattr(args, 'func'):
parser.print_usage()
sys.exit(os.EX_USAGE)
if args.func == 'completion':
completion(args.shell)
sys.exit(os.EX_OK)
from benji.config import Config
from benji.logging import logger, init_logging
if args.config_file is not None and args.config_file != '':
try:
cfg = open(args.config_file, 'r', encoding='utf-8').read()
except FileNotFoundError:
logger.error('File {} not found.'.format(args.config_file))
sys.exit(os.EX_USAGE)
config = Config(ad_hoc_config=cfg)
else:
config = Config()
init_logging(config.get('logFile', types=(str, type(None))),
console_level=args.log_level,
console_formatter='console-plain'
if args.no_color else 'console-colored')
if sys.hexversion < 0x030604F0:
logger.warning(
'The installed Python version will use excessive amounts of memory when used with Benji. Upgrade Python to at least 3.6.4.'
)
import benji.commands
commands = benji.commands.Commands(args.machine_output, config)
func = getattr(commands, args.func)
# Pass over to function
func_args = dict(args._get_kwargs())
del func_args['config_file']
del func_args['func']
del func_args['log_level']
del func_args['machine_output']
del func_args['no_color']
# From most specific to least specific
exception_mappings = [
_ExceptionMapping(exception=benji.exception.UsageError,
exit_code=os.EX_USAGE),
_ExceptionMapping(exception=benji.exception.AlreadyLocked,
exit_code=os.EX_NOPERM),
_ExceptionMapping(exception=benji.exception.InternalError,
exit_code=os.EX_SOFTWARE),
_ExceptionMapping(exception=benji.exception.ConfigurationError,
exit_code=os.EX_CONFIG),
_ExceptionMapping(exception=benji.exception.InputDataError,
exit_code=os.EX_DATAERR),
_ExceptionMapping(exception=benji.exception.ScrubbingError,
exit_code=os.EX_DATAERR),
_ExceptionMapping(exception=PermissionError, exit_code=os.EX_NOPERM),
_ExceptionMapping(exception=FileExistsError,
exit_code=os.EX_CANTCREAT),
_ExceptionMapping(exception=FileNotFoundError,
exit_code=os.EX_NOINPUT),
_ExceptionMapping(exception=EOFError, exit_code=os.EX_IOERR),
_ExceptionMapping(exception=IOError, exit_code=os.EX_IOERR),
_ExceptionMapping(exception=OSError, exit_code=os.EX_OSERR),
_ExceptionMapping(exception=ConnectionError, exit_code=os.EX_IOERR),
_ExceptionMapping(exception=LookupError, exit_code=os.EX_NOINPUT),
_ExceptionMapping(exception=KeyboardInterrupt,
exit_code=os.EX_NOINPUT),
_ExceptionMapping(exception=BaseException, exit_code=os.EX_SOFTWARE),
]
try:
logger.debug('commands.{0}(**{1!r})'.format(args.func, func_args))
func(**func_args)
sys.exit(os.EX_OK)
except SystemExit:
raise
except BaseException as exception:
for case in exception_mappings:
if isinstance(exception, case.exception):
message = str(exception)
if message:
message = '{}: {}'.format(exception.__class__.__name__,
message)
else:
message = '{} exception occurred.'.format(
exception.__class__.__name__)
logger.debug(message, exc_info=True)
logger.error(message)
sys.exit(case.exit_code)
def __init__(self, *, config: Config, name: str,
module_configuration: ConfigDict):
super().__init__(config=config,
name=name,
module_configuration=module_configuration)
account_id = Config.get_from_dict(module_configuration,
'accountId',
None,
types=str)
if account_id is None:
account_id_file = Config.get_from_dict(module_configuration,
'accountIdFile',
types=str)
with open(account_id_file, 'r') as f:
account_id = f.read().rstrip()
application_key = Config.get_from_dict(module_configuration,
'applicationKey',
None,
types=str)
if application_key is None:
application_key_file = Config.get_from_dict(module_configuration,
'applicationKeyFile',
types=str)
with open(application_key_file, 'r') as f:
application_key = f.read().rstrip()
bucket_name = Config.get_from_dict(module_configuration,
'bucketName',
types=str)
account_info_file = Config.get_from_dict(module_configuration,
'accountInfoFile',
None,
types=str)
if account_info_file is not None:
account_info = SqliteAccountInfo(file_name=account_info_file)
else:
account_info = InMemoryAccountInfo()
b2.bucket.Bucket.MAX_UPLOAD_ATTEMPTS = Config.get_from_dict(
module_configuration, 'uploadAttempts', types=int)
self._write_object_attempts = Config.get_from_dict(
module_configuration, 'writeObjectAttempts', types=int)
self._read_object_attempts = Config.get_from_dict(module_configuration,
'readObjectAttempts',
types=int)
self.service = b2.api.B2Api(account_info)
if account_info_file is not None:
try:
# This temporarily disables all logging as the b2 library does some very verbose logging
# of the exception we're trying to catch here...
logging.disable(logging.ERROR)
_ = self.service.get_account_id()
logging.disable(logging.NOTSET)
except MissingAccountData:
self.service.authorize_account('production', account_id,
application_key)
else:
self.service.authorize_account('production', account_id,
application_key)
self.bucket = self.service.get_bucket_by_name(bucket_name)
def __init__(self, *, config: Config, name: str, storage_id: int, module_configuration: ConfigDict):
aws_access_key_id = Config.get_from_dict(module_configuration, 'awsAccessKeyId', None, types=str)
if aws_access_key_id is None:
aws_access_key_id_file = Config.get_from_dict(module_configuration, 'awsAccessKeyIdFile', types=str)
with open(aws_access_key_id_file, 'r') as f:
aws_access_key_id = f.read().rstrip()
aws_secret_access_key = Config.get_from_dict(module_configuration, 'awsSecretAccessKey', None, types=str)
if aws_secret_access_key is None:
aws_secret_access_key_file = Config.get_from_dict(module_configuration, 'awsSecretAccessKeyFile', types=str)
with open(aws_secret_access_key_file, 'r') as f:
aws_secret_access_key = f.read().rstrip()
region_name = Config.get_from_dict(module_configuration, 'regionName', None, types=str)
endpoint_url = Config.get_from_dict(module_configuration, 'endpointUrl', None, types=str)
use_ssl = Config.get_from_dict(module_configuration, 'useSsl', None, types=bool)
addressing_style = Config.get_from_dict(module_configuration, 'addressingStyle', None, types=str)
signature_version = Config.get_from_dict(module_configuration, 'signatureVersion', None, types=str)
self._bucket_name = Config.get_from_dict(module_configuration, 'bucketName', types=str)
self._disable_encoding_type = Config.get_from_dict(module_configuration, 'disableEncodingType', types=bool)
self._resource_config = {
'aws_access_key_id': aws_access_key_id,
'aws_secret_access_key': aws_secret_access_key,
}
if region_name:
self._resource_config['region_name'] = region_name
if endpoint_url:
self._resource_config['endpoint_url'] = endpoint_url
if use_ssl:
self._resource_config['use_ssl'] = use_ssl
resource_config = {}
if addressing_style:
resource_config['s3'] = {'addressing_style': addressing_style}
if signature_version:
resource_config['signature_version'] = signature_version
self._resource_config['config'] = BotoCoreClientConfig(**resource_config)
self._local = threading.local()
self._init_connection()
self._local.bucket = self._local.resource.Bucket(self._bucket_name)
super().__init__(config=config, name=name, storage_id=storage_id, module_configuration=module_configuration)
def initialize(cls, config: Config) -> None:
TransformFactory.initialize(config)
cls._modules = {}
storages: ConfigList = config.get('storages', types=list)
cls._import_modules(config, storages)
