def __init__(self, meta0_client, conscience_client, replicas=3, digits=None, logger=None, min_dist=1, **kwargs): """ :param replicas: number of services to allocate to manage a base :type replicas: strictly positive `int` :param digits: number of digits used to name the database files :type digits: `int` between 0 and 4 (inclusive) """ self.cs = conscience_client self.m0 = meta0_client self._admin = None self.raw_svc_by_base = dict() self.svc_by_base = dict() self.services = dict() self.logger = logger self.replicas = int(replicas) if self.replicas < 1: raise ConfigurationException("replicas must be >= 1") if digits is None: digits = 4 self.digits = int(digits) if self.digits < 0: raise ConfigurationException("meta_digits must be >= 0") if self.digits > 4: raise ConfigurationException("meta_digits must be <= 4") self.min_dist = min_dist self.reset()
def __init__(self, meta0_client, replicas=3, digits=None, min_dist=1, **kwargs): """ :param replicas: number of services to allocate to manage a base :type replicas: strictly positive `int` :param digits: number of digits used to name the database files :type digits: `int` between 0 and 4 (inclusive) """ self.m0 = meta0_client self.replicas = int(replicas) if self.replicas < 1: raise ConfigurationException("replicas must be >= 1") if digits is None: digits = 4 self.digits = int(digits) if self.digits < 0: raise ConfigurationException("meta_digits must be >= 0") if self.digits > 4: raise ConfigurationException("meta_digits must be <= 4") self.min_dist = min_dist if self.min_dist < 1: raise ConfigurationException("min_dist must be >= 1") if self.min_dist > 4: raise ConfigurationException("min_dist must be <= 4") super(Meta0PrefixMapping, self).__init__( meta0_client.conf, ['meta1'], **kwargs)
def filter_factory(global_conf, **local_config): conf = global_conf.copy() conf.update(local_config) ns = conf.get('sds_namespace') acct = conf.get('sds_default_account') proxy = conf.get('sds_proxy_url') if ns is None: raise ConfigurationException('No OIO-SDS namespace configured') if acct is None: raise ConfigurationException('No OIO-SDS account configured') if proxy is None: raise ConfigurationException('No OIO-SDS proxy URL configured') strip_v1 = config_true_value(local_config.pop('strip_v1', False)) account_first = config_true_value(local_config.pop('account_first', False)) skip_metadata = config_true_value(local_config.pop('skip_metadata', False)) def factory(app): return HashedContainerMiddleware(app, ns, acct, proxy, strip_v1=strip_v1, account_first=account_first, skip_metadata=skip_metadata, **local_config) return factory
def __init__(self, meta0_client, conscience_client, replicas=3, digits=None, logger=None, **kwargs): """ Parameters: - replicas: strictily positive integer - digits: integer between 0 and 4 (default: 4) """ self.cs = conscience_client self.m0 = meta0_client self.svc_by_pfx = dict() self.services = dict() self.logger = logger assert (replicas > 0) self.replicas = replicas if digits is None: digits = 4 self.digits = int(digits) if self.digits < 0: raise ConfigurationException("meta_digits must be >= 0") if self.digits > 4: raise ConfigurationException("meta_digits must be <= 4") for svc in self.cs.all_services("meta1"): self.services[svc["addr"]] = svc
def _loc(svc): """Extract the location of a service and ensure it is well formed.""" loc = svc.get("tags", {}).get("tag.loc", None) if not loc: raise ConfigurationException("No location on {0}".format(svc)) tokens = loc.split('.') if len(tokens) > 4: raise ConfigurationException("Malformed location for {0}" .format(svc)) return tokens
def __init__(self, conf, logger=None, **kwargs): self.conf = conf self.logger = logger or get_logger(conf) volume = conf.get('volume') if not volume: raise ConfigurationException('No volume specified for converter') self.volume = volume self.namespace, self.volume_id = check_volume(self.volume) # cache self.name_by_cid = CacheDict() self.content_id_by_name = CacheDict() # client self.container_client = ContainerClient(conf, **kwargs) self.content_factory = ContentFactory(conf, self.container_client, logger=self.logger) # stats/logs self.errors = 0 self.passes = 0 self.total_chunks_processed = 0 self.start_time = 0 self.last_reported = 0 self.report_interval = int_value(conf.get('report_interval'), 3600) # speed self.chunks_run_time = 0 self.max_chunks_per_second = int_value(conf.get('chunks_per_second'), 30) # backup self.no_backup = true_value(conf.get('no_backup', False)) self.backup_dir = conf.get('backup_dir') or tempfile.gettempdir() self.backup_name = 'backup_%s_%f' \ % (self.volume_id, time.time()) # dry run self.dry_run = true_value(conf.get('dry_run', False))
def filter_factory(global_conf, **local_config): conf = global_conf.copy() conf.update(local_config) global LOG LOG = get_logger(conf) LOG.warn( '%s middleware is proof-of-concept ' 'and not suitable for production use!', MIDDLEWARE_NAME) acct = conf.get('sds_default_account') if acct is None: raise ConfigurationException('No OIO-SDS account configured') account_first = config_true_value(local_config.get('account_first')) swift3_compat = config_true_value(local_config.get('swift3_compat')) strip_v1 = config_true_value(local_config.get('strip_v1')) create_dir_placeholders = config_true_value( local_config.get('create_dir_placeholders')) recursive_placeholders = config_true_value( local_config.get('recursive_placeholders')) def factory(app): return ContainerHierarchyMiddleware( app, global_conf, acct, strip_v1=strip_v1, account_first=account_first, swift3_compat=swift3_compat, create_dir_placeholders=create_dir_placeholders, recursive_placeholders=recursive_placeholders) return factory
def filter_factory(global_conf, **local_config): conf = global_conf.copy() conf.update(local_config) acct = conf.get('sds_default_account') if acct is None: raise ConfigurationException('No OIO-SDS account configured') account_first = config_true_value(local_config.get('account_first')) failsafe = config_true_value(local_config.get('failsafe')) swift3_compat = config_true_value(local_config.get('swift3_compat')) strip_v1 = config_true_value(local_config.get('strip_v1')) # By default this is enabled, to be compatible with openio-sds < 4.2. stop_at_first_match = config_true_value( local_config.get('stop_at_first_match', True)) pattern_dict = {k: v for k, v in local_config.items() if k.startswith("pattern")} def factory(app): patterns = [pattern_dict[k] for k in sorted(pattern_dict.keys())] logger = get_logger(conf) logger.info("Using patterns %s", patterns) return RegexContainerMiddleware( app, acct, patterns, strip_v1=strip_v1, account_first=account_first, swift3_compat=swift3_compat, stop_at_first_match=stop_at_first_match, failsafe=failsafe) return factory
def __init__(self, conf, logger, volume, distributed_addr, **kwargs): super(DistributedBlobRebuilder, self).__init__(conf, logger, volume, **kwargs) if not self.beanstalkd_listener: raise ConfigurationException( "No beanstalkd to fetch responses from") distributed_tube = conf.get('distributed_tube', DEFAULT_REBUILDER_TUBE) if self.beanstalkd_listener.tube == distributed_tube: raise ConfigurationException( "The beanstalkd tubes must be different") self.distributed = True self.beanstalkd_senders = dict() for addr in distributed_addr.split(';'): sender = BeanstalkdSender(addr, distributed_tube, self.logger, **kwargs) self.beanstalkd_senders[sender.addr] = sender self.sending = False self.rebuilder_id = str(uuid.uuid4())
def _fetch_chunks(self, **kwargs): if self.input_file: return self._fetch_chunks_from_file(**kwargs) if self.beanstalkd_listener and not self.distributed: return self._fetch_events_from_beanstalk(**kwargs) if self.volume: return self.rdir_client.chunk_fetch(self.volume, limit=self.rdir_fetch_limit, rebuild=True, **kwargs) raise ConfigurationException('No source to fetch chunks from')
def get_flatns_manager(self): # TODO(jfs): this is also needed in oio-swift, need to factorize in # oiopy self.info() options = self._nsinfo['options'] bitlength, offset, size = None, 0, 0 try: bitlength = int(options['flat_bitlength']) except: raise ConfigurationException( "Namespace not configured for autocontainers") try: if 'flat_hash_offset' in options: offset = int(options['flat_hash_offset']) if 'flat_hash_size' in options: size = int(options['flat_hash_size']) except: raise Exception("Invalid autocontainer config: offset/size") return HashedContainerBuilder(offset=offset, size=size, bits=bitlength)
def filter_factory(global_conf, **local_config): conf = global_conf.copy() conf.update(local_config) global LOG LOG = get_logger(conf) acct = conf.get('sds_default_account') if acct is None: raise ConfigurationException('No OIO-SDS account configured') account_first = config_true_value(local_config.get('account_first')) swift3_compat = config_true_value(local_config.get('swift3_compat')) strip_v1 = config_true_value(local_config.get('strip_v1')) beurk = local_config.get('support_listing_versioning') if beurk is not None: LOG.warn('"support_listing_versioning" is a deprecated alias for ' '"mask_empty_prefixes".') mask_empty_prefixes = config_true_value( local_config.get('mask_empty_prefixes', beurk)) redis_conf = {k[6:]: v for k, v in conf.items() if k.startswith("redis_")} if redis_conf.get('sentinel_hosts') is None: # TODO(adu): Delete when it will no longer be used redis_conf['sentinel_hosts'] = local_config.get('sentinel_hosts') if redis_conf.get('sentinel_name') is None: # TODO(adu): Delete when it will no longer be used redis_conf['sentinel_name'] = local_config.get('sentinel_name') def factory(app): return ContainerHierarchyMiddleware( app, global_conf, acct, strip_v1=strip_v1, account_first=account_first, swift3_compat=swift3_compat, mask_empty_prefixes=mask_empty_prefixes, redis_conf=redis_conf) return factory
def flatns_manager(self): if self._flatns_manager is not None: return self._flatns_manager from oio.common.autocontainer import HashedContainerBuilder options = self.nsinfo['options'] bitlength, offset, size = None, 0, None try: bitlength = int(self._flatns_bits or options['flat_bitlength']) except Exception: from oio.common.exceptions import ConfigurationException raise ConfigurationException( "Namespace not configured for autocontainers") try: if 'flat_hash_offset' in options: offset = int(options['flat_hash_offset']) if 'flat_hash_size' in options: size = int(options['flat_hash_size']) except Exception: raise Exception("Invalid autocontainer config: offset/size") self._flatns_manager = HashedContainerBuilder( offset=offset, size=size, bits=bitlength) return self._flatns_manager
def get_flatns_manager(self): if self._flatns_manager: return self._flatns_manager self.info() options = self._nsinfo['options'] bitlength, offset, size = None, 0, 0 try: bitlength = int(options['flat_bitlength']) except: raise ConfigurationException( "Namespace not configured for autocontainers") try: if 'flat_hash_offset' in options: offset = int(options['flat_hash_offset']) if 'flat_hash_size' in options: size = int(options['flat_hash_size']) except: raise Exception("Invalid autocontainer config: offset/size") self._flatns_manager = HashedContainerBuilder(offset=offset, size=size, bits=bitlength) return self._flatns_manager