def __init__(self,
meta0_client,
conscience_client,
replicas=3,
digits=None,
logger=None,
min_dist=1,
**kwargs):
"""
:param replicas: number of services to allocate to manage a base
:type replicas: strictly positive `int`
:param digits: number of digits used to name the database files
:type digits: `int` between 0 and 4 (inclusive)
"""
self.cs = conscience_client
self.m0 = meta0_client
self._admin = None
self.raw_svc_by_base = dict()
self.svc_by_base = dict()
self.services = dict()
self.logger = logger
self.replicas = int(replicas)
if self.replicas < 1:
raise ConfigurationException("replicas must be >= 1")
if digits is None:
digits = 4
self.digits = int(digits)
if self.digits < 0:
raise ConfigurationException("meta_digits must be >= 0")
if self.digits > 4:
raise ConfigurationException("meta_digits must be <= 4")
self.min_dist = min_dist
self.reset()
def __init__(self, meta0_client, replicas=3,
digits=None, min_dist=1, **kwargs):
"""
:param replicas: number of services to allocate to manage a base
:type replicas: strictly positive `int`
:param digits: number of digits used to name the database files
:type digits: `int` between 0 and 4 (inclusive)
"""
self.m0 = meta0_client
self.replicas = int(replicas)
if self.replicas < 1:
raise ConfigurationException("replicas must be >= 1")
if digits is None:
digits = 4
self.digits = int(digits)
if self.digits < 0:
raise ConfigurationException("meta_digits must be >= 0")
if self.digits > 4:
raise ConfigurationException("meta_digits must be <= 4")
self.min_dist = min_dist
if self.min_dist < 1:
raise ConfigurationException("min_dist must be >= 1")
if self.min_dist > 4:
raise ConfigurationException("min_dist must be <= 4")
super(Meta0PrefixMapping, self).__init__(
meta0_client.conf, ['meta1'], **kwargs)
def filter_factory(global_conf, **local_config):
conf = global_conf.copy()
conf.update(local_config)
ns = conf.get('sds_namespace')
acct = conf.get('sds_default_account')
proxy = conf.get('sds_proxy_url')
if ns is None:
raise ConfigurationException('No OIO-SDS namespace configured')
if acct is None:
raise ConfigurationException('No OIO-SDS account configured')
if proxy is None:
raise ConfigurationException('No OIO-SDS proxy URL configured')
strip_v1 = config_true_value(local_config.pop('strip_v1', False))
account_first = config_true_value(local_config.pop('account_first', False))
skip_metadata = config_true_value(local_config.pop('skip_metadata', False))
def factory(app):
return HashedContainerMiddleware(app,
ns,
acct,
proxy,
strip_v1=strip_v1,
account_first=account_first,
skip_metadata=skip_metadata,
**local_config)
return factory
def __init__(self,
meta0_client,
conscience_client,
replicas=3,
digits=None,
logger=None,
**kwargs):
"""
Parameters:
- replicas: strictily positive integer
- digits: integer between 0 and 4 (default: 4)
"""
self.cs = conscience_client
self.m0 = meta0_client
self.svc_by_pfx = dict()
self.services = dict()
self.logger = logger
assert (replicas > 0)
self.replicas = replicas
if digits is None:
digits = 4
self.digits = int(digits)
if self.digits < 0:
raise ConfigurationException("meta_digits must be >= 0")
if self.digits > 4:
raise ConfigurationException("meta_digits must be <= 4")
for svc in self.cs.all_services("meta1"):
self.services[svc["addr"]] = svc
def _loc(svc):
"""Extract the location of a service and ensure it is well formed."""
loc = svc.get("tags", {}).get("tag.loc", None)
if not loc:
raise ConfigurationException("No location on {0}".format(svc))
tokens = loc.split('.')
if len(tokens) > 4:
raise ConfigurationException("Malformed location for {0}"
.format(svc))
return tokens
def __init__(self, conf, logger=None, **kwargs):
self.conf = conf
self.logger = logger or get_logger(conf)
volume = conf.get('volume')
if not volume:
raise ConfigurationException('No volume specified for converter')
self.volume = volume
self.namespace, self.volume_id = check_volume(self.volume)
# cache
self.name_by_cid = CacheDict()
self.content_id_by_name = CacheDict()
# client
self.container_client = ContainerClient(conf, **kwargs)
self.content_factory = ContentFactory(conf,
self.container_client,
logger=self.logger)
# stats/logs
self.errors = 0
self.passes = 0
self.total_chunks_processed = 0
self.start_time = 0
self.last_reported = 0
self.report_interval = int_value(conf.get('report_interval'), 3600)
# speed
self.chunks_run_time = 0
self.max_chunks_per_second = int_value(conf.get('chunks_per_second'),
30)
# backup
self.no_backup = true_value(conf.get('no_backup', False))
self.backup_dir = conf.get('backup_dir') or tempfile.gettempdir()
self.backup_name = 'backup_%s_%f' \
% (self.volume_id, time.time())
# dry run
self.dry_run = true_value(conf.get('dry_run', False))
def filter_factory(global_conf, **local_config):
conf = global_conf.copy()
conf.update(local_config)
global LOG
LOG = get_logger(conf)
LOG.warn(
'%s middleware is proof-of-concept '
'and not suitable for production use!', MIDDLEWARE_NAME)
acct = conf.get('sds_default_account')
if acct is None:
raise ConfigurationException('No OIO-SDS account configured')
account_first = config_true_value(local_config.get('account_first'))
swift3_compat = config_true_value(local_config.get('swift3_compat'))
strip_v1 = config_true_value(local_config.get('strip_v1'))
create_dir_placeholders = config_true_value(
local_config.get('create_dir_placeholders'))
recursive_placeholders = config_true_value(
local_config.get('recursive_placeholders'))
def factory(app):
return ContainerHierarchyMiddleware(
app,
global_conf,
acct,
strip_v1=strip_v1,
account_first=account_first,
swift3_compat=swift3_compat,
create_dir_placeholders=create_dir_placeholders,
recursive_placeholders=recursive_placeholders)
return factory
def filter_factory(global_conf, **local_config):
conf = global_conf.copy()
conf.update(local_config)
acct = conf.get('sds_default_account')
if acct is None:
raise ConfigurationException('No OIO-SDS account configured')
account_first = config_true_value(local_config.get('account_first'))
failsafe = config_true_value(local_config.get('failsafe'))
swift3_compat = config_true_value(local_config.get('swift3_compat'))
strip_v1 = config_true_value(local_config.get('strip_v1'))
# By default this is enabled, to be compatible with openio-sds < 4.2.
stop_at_first_match = config_true_value(
local_config.get('stop_at_first_match', True))
pattern_dict = {k: v for k, v in local_config.items()
if k.startswith("pattern")}
def factory(app):
patterns = [pattern_dict[k] for k in sorted(pattern_dict.keys())]
logger = get_logger(conf)
logger.info("Using patterns %s", patterns)
return RegexContainerMiddleware(
app, acct, patterns,
strip_v1=strip_v1, account_first=account_first,
swift3_compat=swift3_compat,
stop_at_first_match=stop_at_first_match,
failsafe=failsafe)
return factory
def __init__(self, conf, logger, volume, distributed_addr, **kwargs):
super(DistributedBlobRebuilder, self).__init__(conf, logger, volume,
**kwargs)
if not self.beanstalkd_listener:
raise ConfigurationException(
"No beanstalkd to fetch responses from")
distributed_tube = conf.get('distributed_tube', DEFAULT_REBUILDER_TUBE)
if self.beanstalkd_listener.tube == distributed_tube:
raise ConfigurationException(
"The beanstalkd tubes must be different")
self.distributed = True
self.beanstalkd_senders = dict()
for addr in distributed_addr.split(';'):
sender = BeanstalkdSender(addr, distributed_tube, self.logger,
**kwargs)
self.beanstalkd_senders[sender.addr] = sender
self.sending = False
self.rebuilder_id = str(uuid.uuid4())
def _fetch_chunks(self, **kwargs):
if self.input_file:
return self._fetch_chunks_from_file(**kwargs)
if self.beanstalkd_listener and not self.distributed:
return self._fetch_events_from_beanstalk(**kwargs)
if self.volume:
return self.rdir_client.chunk_fetch(self.volume,
limit=self.rdir_fetch_limit,
rebuild=True,
**kwargs)
raise ConfigurationException('No source to fetch chunks from')
def get_flatns_manager(self):
# TODO(jfs): this is also needed in oio-swift, need to factorize in
# oiopy
self.info()
options = self._nsinfo['options']
bitlength, offset, size = None, 0, 0
try:
bitlength = int(options['flat_bitlength'])
except:
raise ConfigurationException(
"Namespace not configured for autocontainers")
try:
if 'flat_hash_offset' in options:
offset = int(options['flat_hash_offset'])
if 'flat_hash_size' in options:
size = int(options['flat_hash_size'])
except:
raise Exception("Invalid autocontainer config: offset/size")
return HashedContainerBuilder(offset=offset, size=size, bits=bitlength)
def filter_factory(global_conf, **local_config):
conf = global_conf.copy()
conf.update(local_config)
global LOG
LOG = get_logger(conf)
acct = conf.get('sds_default_account')
if acct is None:
raise ConfigurationException('No OIO-SDS account configured')
account_first = config_true_value(local_config.get('account_first'))
swift3_compat = config_true_value(local_config.get('swift3_compat'))
strip_v1 = config_true_value(local_config.get('strip_v1'))
beurk = local_config.get('support_listing_versioning')
if beurk is not None:
LOG.warn('"support_listing_versioning" is a deprecated alias for '
'"mask_empty_prefixes".')
mask_empty_prefixes = config_true_value(
local_config.get('mask_empty_prefixes', beurk))
redis_conf = {k[6:]: v for k, v in conf.items() if k.startswith("redis_")}
if redis_conf.get('sentinel_hosts') is None:
# TODO(adu): Delete when it will no longer be used
redis_conf['sentinel_hosts'] = local_config.get('sentinel_hosts')
if redis_conf.get('sentinel_name') is None:
# TODO(adu): Delete when it will no longer be used
redis_conf['sentinel_name'] = local_config.get('sentinel_name')
def factory(app):
return ContainerHierarchyMiddleware(
app,
global_conf,
acct,
strip_v1=strip_v1,
account_first=account_first,
swift3_compat=swift3_compat,
mask_empty_prefixes=mask_empty_prefixes,
redis_conf=redis_conf)
return factory
def flatns_manager(self):
if self._flatns_manager is not None:
return self._flatns_manager
from oio.common.autocontainer import HashedContainerBuilder
options = self.nsinfo['options']
bitlength, offset, size = None, 0, None
try:
bitlength = int(self._flatns_bits or options['flat_bitlength'])
except Exception:
from oio.common.exceptions import ConfigurationException
raise ConfigurationException(
"Namespace not configured for autocontainers")
try:
if 'flat_hash_offset' in options:
offset = int(options['flat_hash_offset'])
if 'flat_hash_size' in options:
size = int(options['flat_hash_size'])
except Exception:
raise Exception("Invalid autocontainer config: offset/size")
self._flatns_manager = HashedContainerBuilder(
offset=offset, size=size, bits=bitlength)
return self._flatns_manager
def get_flatns_manager(self):
if self._flatns_manager:
return self._flatns_manager
self.info()
options = self._nsinfo['options']
bitlength, offset, size = None, 0, 0
try:
bitlength = int(options['flat_bitlength'])
except:
raise ConfigurationException(
"Namespace not configured for autocontainers")
try:
if 'flat_hash_offset' in options:
offset = int(options['flat_hash_offset'])
if 'flat_hash_size' in options:
size = int(options['flat_hash_size'])
except:
raise Exception("Invalid autocontainer config: offset/size")
self._flatns_manager = HashedContainerBuilder(offset=offset,
size=size,
bits=bitlength)
return self._flatns_manager