Exemplo n.º 1
0
 def __init__(self,
              meta0_client,
              conscience_client,
              replicas=3,
              digits=None,
              logger=None,
              min_dist=1,
              **kwargs):
     """
     :param replicas: number of services to allocate to manage a base
     :type replicas: strictly positive `int`
     :param digits: number of digits used to name the database files
     :type digits: `int` between 0 and 4 (inclusive)
     """
     self.cs = conscience_client
     self.m0 = meta0_client
     self._admin = None
     self.raw_svc_by_base = dict()
     self.svc_by_base = dict()
     self.services = dict()
     self.logger = logger
     self.replicas = int(replicas)
     if self.replicas < 1:
         raise ConfigurationException("replicas must be >= 1")
     if digits is None:
         digits = 4
     self.digits = int(digits)
     if self.digits < 0:
         raise ConfigurationException("meta_digits must be >= 0")
     if self.digits > 4:
         raise ConfigurationException("meta_digits must be <= 4")
     self.min_dist = min_dist
     self.reset()
Exemplo n.º 2
0
 def __init__(self, meta0_client, replicas=3,
              digits=None, min_dist=1, **kwargs):
     """
     :param replicas: number of services to allocate to manage a base
     :type replicas: strictly positive `int`
     :param digits: number of digits used to name the database files
     :type digits: `int` between 0 and 4 (inclusive)
     """
     self.m0 = meta0_client
     self.replicas = int(replicas)
     if self.replicas < 1:
         raise ConfigurationException("replicas must be >= 1")
     if digits is None:
         digits = 4
     self.digits = int(digits)
     if self.digits < 0:
         raise ConfigurationException("meta_digits must be >= 0")
     if self.digits > 4:
         raise ConfigurationException("meta_digits must be <= 4")
     self.min_dist = min_dist
     if self.min_dist < 1:
         raise ConfigurationException("min_dist must be >= 1")
     if self.min_dist > 4:
         raise ConfigurationException("min_dist must be <= 4")
     super(Meta0PrefixMapping, self).__init__(
         meta0_client.conf, ['meta1'], **kwargs)
Exemplo n.º 3
0
def filter_factory(global_conf, **local_config):
    conf = global_conf.copy()
    conf.update(local_config)

    ns = conf.get('sds_namespace')
    acct = conf.get('sds_default_account')
    proxy = conf.get('sds_proxy_url')

    if ns is None:
        raise ConfigurationException('No OIO-SDS namespace configured')
    if acct is None:
        raise ConfigurationException('No OIO-SDS account configured')
    if proxy is None:
        raise ConfigurationException('No OIO-SDS proxy URL configured')

    strip_v1 = config_true_value(local_config.pop('strip_v1', False))
    account_first = config_true_value(local_config.pop('account_first', False))
    skip_metadata = config_true_value(local_config.pop('skip_metadata', False))

    def factory(app):
        return HashedContainerMiddleware(app,
                                         ns,
                                         acct,
                                         proxy,
                                         strip_v1=strip_v1,
                                         account_first=account_first,
                                         skip_metadata=skip_metadata,
                                         **local_config)

    return factory
Exemplo n.º 4
0
 def __init__(self,
              meta0_client,
              conscience_client,
              replicas=3,
              digits=None,
              logger=None,
              **kwargs):
     """
     Parameters:
     - replicas: strictily positive integer
     - digits: integer between 0 and 4 (default: 4)
     """
     self.cs = conscience_client
     self.m0 = meta0_client
     self.svc_by_pfx = dict()
     self.services = dict()
     self.logger = logger
     assert (replicas > 0)
     self.replicas = replicas
     if digits is None:
         digits = 4
     self.digits = int(digits)
     if self.digits < 0:
         raise ConfigurationException("meta_digits must be >= 0")
     if self.digits > 4:
         raise ConfigurationException("meta_digits must be <= 4")
     for svc in self.cs.all_services("meta1"):
         self.services[svc["addr"]] = svc
Exemplo n.º 5
0
def _loc(svc):
    """Extract the location of a service and ensure it is well formed."""
    loc = svc.get("tags", {}).get("tag.loc", None)
    if not loc:
        raise ConfigurationException("No location on {0}".format(svc))
    tokens = loc.split('.')
    if len(tokens) > 4:
        raise ConfigurationException("Malformed location for {0}"
                                     .format(svc))
    return tokens
Exemplo n.º 6
0
 def __init__(self, conf, logger=None, **kwargs):
     self.conf = conf
     self.logger = logger or get_logger(conf)
     volume = conf.get('volume')
     if not volume:
         raise ConfigurationException('No volume specified for converter')
     self.volume = volume
     self.namespace, self.volume_id = check_volume(self.volume)
     # cache
     self.name_by_cid = CacheDict()
     self.content_id_by_name = CacheDict()
     # client
     self.container_client = ContainerClient(conf, **kwargs)
     self.content_factory = ContentFactory(conf,
                                           self.container_client,
                                           logger=self.logger)
     # stats/logs
     self.errors = 0
     self.passes = 0
     self.total_chunks_processed = 0
     self.start_time = 0
     self.last_reported = 0
     self.report_interval = int_value(conf.get('report_interval'), 3600)
     # speed
     self.chunks_run_time = 0
     self.max_chunks_per_second = int_value(conf.get('chunks_per_second'),
                                            30)
     # backup
     self.no_backup = true_value(conf.get('no_backup', False))
     self.backup_dir = conf.get('backup_dir') or tempfile.gettempdir()
     self.backup_name = 'backup_%s_%f' \
         % (self.volume_id, time.time())
     # dry run
     self.dry_run = true_value(conf.get('dry_run', False))
def filter_factory(global_conf, **local_config):
    conf = global_conf.copy()
    conf.update(local_config)
    global LOG
    LOG = get_logger(conf)
    LOG.warn(
        '%s middleware is proof-of-concept '
        'and not suitable for production use!', MIDDLEWARE_NAME)

    acct = conf.get('sds_default_account')

    if acct is None:
        raise ConfigurationException('No OIO-SDS account configured')

    account_first = config_true_value(local_config.get('account_first'))
    swift3_compat = config_true_value(local_config.get('swift3_compat'))
    strip_v1 = config_true_value(local_config.get('strip_v1'))
    create_dir_placeholders = config_true_value(
        local_config.get('create_dir_placeholders'))
    recursive_placeholders = config_true_value(
        local_config.get('recursive_placeholders'))

    def factory(app):
        return ContainerHierarchyMiddleware(
            app,
            global_conf,
            acct,
            strip_v1=strip_v1,
            account_first=account_first,
            swift3_compat=swift3_compat,
            create_dir_placeholders=create_dir_placeholders,
            recursive_placeholders=recursive_placeholders)

    return factory
Exemplo n.º 8
0
def filter_factory(global_conf, **local_config):
    conf = global_conf.copy()
    conf.update(local_config)

    acct = conf.get('sds_default_account')

    if acct is None:
        raise ConfigurationException('No OIO-SDS account configured')

    account_first = config_true_value(local_config.get('account_first'))
    failsafe = config_true_value(local_config.get('failsafe'))
    swift3_compat = config_true_value(local_config.get('swift3_compat'))
    strip_v1 = config_true_value(local_config.get('strip_v1'))
    # By default this is enabled, to be compatible with openio-sds < 4.2.
    stop_at_first_match = config_true_value(
        local_config.get('stop_at_first_match', True))
    pattern_dict = {k: v for k, v in local_config.items()
                    if k.startswith("pattern")}

    def factory(app):
        patterns = [pattern_dict[k] for k in sorted(pattern_dict.keys())]
        logger = get_logger(conf)
        logger.info("Using patterns %s", patterns)
        return RegexContainerMiddleware(
            app, acct, patterns,
            strip_v1=strip_v1, account_first=account_first,
            swift3_compat=swift3_compat,
            stop_at_first_match=stop_at_first_match,
            failsafe=failsafe)
    return factory
Exemplo n.º 9
0
    def __init__(self, conf, logger, volume, distributed_addr, **kwargs):
        super(DistributedBlobRebuilder, self).__init__(conf, logger, volume,
                                                       **kwargs)

        if not self.beanstalkd_listener:
            raise ConfigurationException(
                "No beanstalkd to fetch responses from")
        distributed_tube = conf.get('distributed_tube', DEFAULT_REBUILDER_TUBE)
        if self.beanstalkd_listener.tube == distributed_tube:
            raise ConfigurationException(
                "The beanstalkd tubes must be different")

        self.distributed = True
        self.beanstalkd_senders = dict()
        for addr in distributed_addr.split(';'):
            sender = BeanstalkdSender(addr, distributed_tube, self.logger,
                                      **kwargs)
            self.beanstalkd_senders[sender.addr] = sender
        self.sending = False
        self.rebuilder_id = str(uuid.uuid4())
Exemplo n.º 10
0
 def _fetch_chunks(self, **kwargs):
     if self.input_file:
         return self._fetch_chunks_from_file(**kwargs)
     if self.beanstalkd_listener and not self.distributed:
         return self._fetch_events_from_beanstalk(**kwargs)
     if self.volume:
         return self.rdir_client.chunk_fetch(self.volume,
                                             limit=self.rdir_fetch_limit,
                                             rebuild=True,
                                             **kwargs)
     raise ConfigurationException('No source to fetch chunks from')
Exemplo n.º 11
0
 def get_flatns_manager(self):
     # TODO(jfs): this is also needed in oio-swift, need to factorize in
     #            oiopy
     self.info()
     options = self._nsinfo['options']
     bitlength, offset, size = None, 0, 0
     try:
         bitlength = int(options['flat_bitlength'])
     except:
         raise ConfigurationException(
             "Namespace not configured for autocontainers")
     try:
         if 'flat_hash_offset' in options:
             offset = int(options['flat_hash_offset'])
         if 'flat_hash_size' in options:
             size = int(options['flat_hash_size'])
     except:
         raise Exception("Invalid autocontainer config: offset/size")
     return HashedContainerBuilder(offset=offset, size=size, bits=bitlength)
Exemplo n.º 12
0
def filter_factory(global_conf, **local_config):
    conf = global_conf.copy()
    conf.update(local_config)
    global LOG
    LOG = get_logger(conf)

    acct = conf.get('sds_default_account')

    if acct is None:
        raise ConfigurationException('No OIO-SDS account configured')

    account_first = config_true_value(local_config.get('account_first'))
    swift3_compat = config_true_value(local_config.get('swift3_compat'))
    strip_v1 = config_true_value(local_config.get('strip_v1'))
    beurk = local_config.get('support_listing_versioning')
    if beurk is not None:
        LOG.warn('"support_listing_versioning" is a deprecated alias for '
                 '"mask_empty_prefixes".')
    mask_empty_prefixes = config_true_value(
        local_config.get('mask_empty_prefixes', beurk))
    redis_conf = {k[6:]: v for k, v in conf.items() if k.startswith("redis_")}
    if redis_conf.get('sentinel_hosts') is None:
        # TODO(adu): Delete when it will no longer be used
        redis_conf['sentinel_hosts'] = local_config.get('sentinel_hosts')
    if redis_conf.get('sentinel_name') is None:
        # TODO(adu): Delete when it will no longer be used
        redis_conf['sentinel_name'] = local_config.get('sentinel_name')

    def factory(app):
        return ContainerHierarchyMiddleware(
            app,
            global_conf,
            acct,
            strip_v1=strip_v1,
            account_first=account_first,
            swift3_compat=swift3_compat,
            mask_empty_prefixes=mask_empty_prefixes,
            redis_conf=redis_conf)

    return factory
Exemplo n.º 13
0
 def flatns_manager(self):
     if self._flatns_manager is not None:
         return self._flatns_manager
     from oio.common.autocontainer import HashedContainerBuilder
     options = self.nsinfo['options']
     bitlength, offset, size = None, 0, None
     try:
         bitlength = int(self._flatns_bits or options['flat_bitlength'])
     except Exception:
         from oio.common.exceptions import ConfigurationException
         raise ConfigurationException(
                 "Namespace not configured for autocontainers")
     try:
         if 'flat_hash_offset' in options:
             offset = int(options['flat_hash_offset'])
         if 'flat_hash_size' in options:
             size = int(options['flat_hash_size'])
     except Exception:
         raise Exception("Invalid autocontainer config: offset/size")
     self._flatns_manager = HashedContainerBuilder(
         offset=offset, size=size, bits=bitlength)
     return self._flatns_manager
Exemplo n.º 14
0
 def get_flatns_manager(self):
     if self._flatns_manager:
         return self._flatns_manager
     self.info()
     options = self._nsinfo['options']
     bitlength, offset, size = None, 0, 0
     try:
         bitlength = int(options['flat_bitlength'])
     except:
         raise ConfigurationException(
             "Namespace not configured for autocontainers")
     try:
         if 'flat_hash_offset' in options:
             offset = int(options['flat_hash_offset'])
         if 'flat_hash_size' in options:
             size = int(options['flat_hash_size'])
     except:
         raise Exception("Invalid autocontainer config: offset/size")
     self._flatns_manager = HashedContainerBuilder(offset=offset,
                                                   size=size,
                                                   bits=bitlength)
     return self._flatns_manager