def take_action(self, parsed_args): from oio.common.easy_value import convert_size self.log.debug('take_action(%s)', parsed_args) account = self.app.client_manager.account self.take_action_container(parsed_args) # The command is named 'show' but we must call # container_get_properties() because container_show() does # not return system properties (and we need them). data = self.app.client_manager.storage.container_get_properties( account, parsed_args.container, cid=parsed_args.cid, admin_mode=True) sys = data['system'] ctime = float(sys[M2_PROP_CTIME]) / 1000000. bytes_usage = sys.get(M2_PROP_USAGE, 0) objects = sys.get(M2_PROP_OBJECTS, 0) damaged_objects = sys.get(M2_PROP_DAMAGED_OBJECTS, 0) missing_chunks = sys.get(M2_PROP_MISSING_CHUNKS, 0) if parsed_args.formatter == 'table': ctime = int(ctime) bytes_usage = convert_size(int(bytes_usage), unit="B") objects = convert_size(int(objects)) info = { 'account': sys['sys.account'], 'base_name': sys['sys.name'], 'container': sys['sys.user.name'], 'ctime': ctime, 'bytes_usage': bytes_usage, 'quota': sys.get(M2_PROP_QUOTA, "Namespace default"), 'objects': objects, 'damaged_objects': damaged_objects, 'missing_chunks': missing_chunks, 'storage_policy': sys.get(M2_PROP_STORAGE_POLICY, "Namespace default"), 'max_versions': sys.get(M2_PROP_VERSIONING_POLICY, "Namespace default"), 'status': OIO_DB_STATUS_NAME.get(sys.get('sys.status'), "Unknown"), } for k in ('stats.page_count', 'stats.freelist_count', 'stats.page_size'): info[k] = sys.get(k) wasted = (float_value(info['stats.freelist_count'], 0) / float_value(info['stats.page_count'], 1)) wasted_bytes = (int_value(info['stats.freelist_count'], 0) * int_value(info['stats.page_size'], 0)) info['stats.space_wasted'] = "%5.2f%% (est. %s)" % \ (wasted * 100, convert_size(wasted_bytes)) bucket = sys.get(M2_PROP_BUCKET_NAME, None) if bucket is not None: info['bucket'] = bucket delete_exceeding = sys.get(M2_PROP_DEL_EXC_VERSIONS, None) if delete_exceeding is not None: info['delete_exceeding_versions'] = delete_exceeding != '0' for k, v in iteritems(data['properties']): info['meta.' + k] = v return list(zip(*sorted(info.items())))
def sanitize_params(cls, job_params): sanitized_job_params, _ = super(RawxDecommissionJob, cls).sanitize_params(job_params) # specific configuration service_id = job_params.get('service_id') if not service_id: raise ValueError('Missing service ID') sanitized_job_params['service_id'] = service_id sanitized_job_params['rawx_timeout'] = float_value( job_params.get('rawx_timeout'), cls.DEFAULT_RAWX_TIMEOUT) sanitized_job_params['min_chunk_size'] = int_value( job_params.get('min_chunk_size'), cls.DEFAULT_MIN_CHUNK_SIZE) sanitized_job_params['max_chunk_size'] = int_value( job_params.get('max_chunk_size'), cls.DEFAULT_MAX_CHUNK_SIZE) excluded_rawx = job_params.get('excluded_rawx') if excluded_rawx: excluded_rawx = excluded_rawx.split(',') else: excluded_rawx = list() sanitized_job_params['excluded_rawx'] = excluded_rawx sanitized_job_params['usage_target'] = int_value( job_params.get('usage_target'), cls.DEFAULT_USAGE_TARGET) sanitized_job_params['usage_check_interval'] = float_value( job_params.get('usage_check_interval'), cls.DEFAULT_USAGE_CHECK_INTERVAL) return sanitized_job_params, 'rawx/%s' % service_id
def init(self): super(ReplicateFilter, self).init() self.account = self.app_env['account_client'] self.cache_duration = float_value(self.conf.get('cache_duration'), CACHE_DURATION) self.cache_size = int_value(self.conf.get('cache_size'), CACHE_SIZE) self.cache = CacheDict(self.cache_size) self.check_account = boolean_value( self.conf.get('check_replication_enabled'), False) self.connection_timeout = float_value( self.conf.get('connection_timeout'), CONNECTION_TIMEOUT) self.read_timeout = float_value(self.conf.get('read_timeout'), READ_TIMEOUT)
def __init__(self, conf, endpoint=None, proxy_endpoint=None, refresh_delay=3600.0, logger=None, **kwargs): """ Initialize a client for the account service. :param conf: dictionary with at least the namespace name :type conf: `dict` :param endpoint: URL of an account service :param proxy_endpoint: URL of the proxy :param refresh_interval: time between refreshes of the account service endpoint (if not provided at instantiation) :type refresh_interval: `float` seconds """ super(AccountClient, self).__init__(endpoint=endpoint, service_type='account-service', **kwargs) self.logger = logger or get_logger(conf) self.cs = ConscienceClient(conf, endpoint=proxy_endpoint, logger=self.logger, **kwargs) self._global_kwargs = { tok: float_value(tov, None) for tok, tov in kwargs.items() if tok in TIMEOUT_KEYS } self._refresh_delay = refresh_delay if not self.endpoint else -1.0 self._last_refresh = 0.0
def __init__(self, namespace, logger=None, **kwargs): """ Initialize the object storage API. :param namespace: name of the namespace to interract with :type namespace: `str` :keyword connection_timeout: connection timeout towards rawx services :type connection_timeout: `float` seconds :keyword read_timeout: timeout for rawx responses and data reads from the caller (when uploading) :type read_timeout: `float` seconds :keyword write_timeout: timeout for rawx write requests :type write_timeout: `float` seconds :keyword pool_manager: a pooled connection manager that will be used for all HTTP based APIs (except rawx) :type pool_manager: `urllib3.PoolManager` """ self.namespace = namespace conf = {"namespace": self.namespace} self.logger = logger or get_logger(conf) self.timeouts = {tok: float_value(tov, None) for tok, tov in kwargs.items() if tok in self.__class__.TIMEOUT_KEYS} from oio.account.client import AccountClient from oio.container.client import ContainerClient from oio.directory.client import DirectoryClient self.directory = DirectoryClient(conf, logger=self.logger, **kwargs) self.container = ContainerClient(conf, logger=self.logger, **kwargs) # In AccountClient, "endpoint" is the account service, not the proxy acct_kwargs = kwargs.copy() acct_kwargs["proxy_endpoint"] = acct_kwargs.pop("endpoint", None) self.account = AccountClient(conf, logger=self.logger, **acct_kwargs)
def list_containers(self, account_id, limit=1000, marker=None, end_marker=None, prefix=None, delimiter=None): raw_list = self._raw_listing(account_id, limit=limit, marker=marker, end_marker=end_marker, prefix=prefix, delimiter=delimiter) pipeline = self.conn.pipeline(True) # skip prefix for container in [entry for entry in raw_list if not entry[3]]: pipeline.hmget(AccountBackend.ckey(account_id, container[0]), 'objects', 'bytes', 'mtime') res = pipeline.execute() i = 0 for container in raw_list: if not container[3]: container[1] = int_value(res[i][0], 0) container[2] = int_value(res[i][1], 0) container[4] = float_value(res[i][2], 0.0) i += 1 return raw_list
def list_containers(self, account_id, limit=1000, marker=None, end_marker=None, prefix=None, delimiter=None, s3_buckets_only=False): raw_list, _next_marker = self._raw_listing( self.clistkey(account_id), limit=limit, marker=marker, end_marker=end_marker, prefix=prefix, delimiter=delimiter, s3_buckets_only=s3_buckets_only) pipeline = self.conn_slave.pipeline(True) # skip prefix for container in [entry for entry in raw_list if not entry[3]]: pipeline.hmget(AccountBackend.ckey(account_id, container[0]), 'objects', 'bytes', 'mtime') res = pipeline.execute() i = 0 for container in raw_list: if not container[3]: # FIXME(adu) Convert to dict container[1] = int_value(res[i][0], 0) container[2] = int_value(res[i][1], 0) container[4] = float_value(res[i][2], 0.0) i += 1 return raw_list
def __init__(self, conf, service, **kwargs): self.conf = conf self.running = False for k in ['host', 'port', 'type']: if k not in service: raise Exception('Missing field "%s" in service configuration' % k) self.name = '%s|%s|%s' % \ (service['type'], service['host'], service['port']) self.service = service self.rise = int_value(self._load_item_config('rise'), 1) self.fall = int_value(self._load_item_config('fall'), 1) self.check_interval = float_value( self._load_item_config('check_interval'), 1) self.deregister_on_exit = true_value( self._load_item_config('deregister_on_exit', False)) self.logger = get_logger(self.conf) self.pool_manager = get_pool_manager() self.cs = ConscienceClient(self.conf, pool_manager=self.pool_manager, logger=self.logger) # FIXME: explain that self.client = ProxyClient(self.conf, pool_manager=self.pool_manager, no_ns_in_url=True, logger=self.logger) self.last_status = False self.status = False self.failed = False self.service_definition = { 'ns': self.conf['namespace'], 'type': self.service['type'], 'addr': '%s:%s' % (self.service['host'], self.service['port']), 'score': 0, 'tags': {} } if self.service.get('slots', None): self.service_definition['tags']['tag.slots'] = \ ','.join(self.service['slots']) for name, tag in (('location', 'tag.loc'), ('service_id', 'tag.service_id'), ('tls', 'tag.tls')): if self.service.get(name): self.service_definition['tags'][tag] = \ self.service[name] self.service_checks = list() self.service_stats = list() self.init_checkers(service) self.init_stats(service)
def __init__(self, agent, checker_conf, logger): self.agent = agent self.checker_conf = checker_conf self.logger = logger self.timeout = float_value(checker_conf.get('timeout'), 5.0) self.rise = checker_conf['rise'] self.fall = checker_conf['fall'] self.results = RingBuffer(max([self.rise, self.fall])) self.name = checker_conf.get('name') self.srv_type = agent.service['type'] self.last_result = None self.configure()
def configure(self): self.parser = self.stat_conf.get('parser', 'lines') self.path = self.stat_conf['path'].lstrip('/') self.host = self.stat_conf['host'] self.port = self.stat_conf['port'] self.url = '%s:%s/%s' % (self.host, self.port, self.path) if self.parser == 'json': # use json parser (account and rdir style) self._parse_func = self._parse_stats_json else: # default to lines parser (rawx style) self._parse_func = self._parse_stats_lines self.timeout = float_value(self.stat_conf.get('timeout'), 10.0)
def __init__(self, conf, input_file=None, service_id=None, **kwargs): super(BlobRebuilder, self).__init__(conf, **kwargs) # counters self.bytes_processed = 0 self.total_bytes_processed = 0 # input self.input_file = input_file self.rawx_id = service_id # rawx/rdir self.rdir_client = RdirClient(self.conf, logger=self.logger) self.rdir_fetch_limit = int_value(self.conf.get('rdir_fetch_limit'), self.DEFAULT_RDIR_FETCH_LIMIT) self.rdir_timeout = float_value(conf.get('rdir_timeout'), self.DEFAULT_RDIR_TIMEOUT)
def list_buckets(self, account_id, limit=1000, marker=None, end_marker=None, prefix=None, **kwargs): """ Get the list of buckets of the specified account. :returns: the list of buckets (with metadata), and the next marker (in case the list is truncated). """ raw_list, next_marker = self._raw_listing(self.blistkey(account_id), limit=limit, marker=marker, end_marker=end_marker, prefix=prefix, **kwargs) conn = self.get_slave_conn(**kwargs) pipeline = conn.pipeline(True) for entry in raw_list: # For real buckets (not prefixes), fetch metadata. if not entry[3]: pipeline.hmget(self.bkey(entry[0]), 'objects', 'bytes', 'mtime') res = pipeline.execute() output = list() i = 0 for bucket in raw_list: if not bucket[3]: bdict = { 'name': bucket[0], 'objects': int_value(res[i][0], 0), 'bytes': int_value(res[i][1], 0), 'mtime': float_value(res[i][2], 0.0), } i += 1 else: bdict = {'prefix': bucket} output.append(bdict) return output, next_marker
def sanitize_params(cls, job_params): sanitized_job_params, _ = super(RawxRebuildJob, cls).sanitize_params(job_params) # specific configuration service_id = job_params.get('service_id') if not service_id: raise ValueError('Missing service ID') sanitized_job_params['service_id'] = service_id sanitized_job_params['rawx_timeout'] = float_value( job_params.get('rawx_timeout'), cls.DEFAULT_RAWX_TIMEOUT) sanitized_job_params['dry_run'] = boolean_value( job_params.get('dry_run'), cls.DEFAULT_DRY_RUN) sanitized_job_params['allow_same_rawx'] = boolean_value( job_params.get('allow_same_rawx'), cls.DEFAULT_ALLOW_SAME_RAWX) sanitized_job_params['try_chunk_delete'] = boolean_value( job_params.get('try_chunk_delete'), cls.DEFAULT_TRY_CHUNK_DELETE) sanitized_job_params['allow_frozen_container'] = boolean_value( job_params.get('allow_frozen_container'), cls.DEFAULT_ALLOW_FROZEN_CT) set_specific_incident_date = int_value( job_params.get('set_specific_incident_date'), None) if set_specific_incident_date is None: set_incident_date = boolean_value( job_params.get('set_incident_date'), cls.DEFAULT_DECLARE_INCIDENT_DATE) if set_incident_date: set_specific_incident_date = int(time.time()) else: set_incident_date = True sanitized_job_params['set_incident_date'] = set_incident_date sanitized_job_params['set_specific_incident_date'] = \ set_specific_incident_date return sanitized_job_params, 'rawx/%s' % service_id
def __init__(self, agent, checker_conf, logger): self.agent = agent self.checker_conf = checker_conf self.logger = logger self.timeout = float_value(checker_conf.get('timeout'), 5.0) self.rise = checker_conf['rise'] self.fall = checker_conf['fall'] self.results = RingBuffer(max([self.rise, self.fall])) self.name = checker_conf.get('name') self.srv_type = agent.service['type'] self.last_result = None for k in ('host', 'port'): if k not in self.checker_conf: raise exc.ConfigurationException( 'Missing field "%s" in configuration' % k) self.host = self.checker_conf['host'] self.port = self.checker_conf['port'] self.name = '%s|%s|%s|%s' % \ (self.srv_type, self.checker_type, self.host, self.port) self.last_check_success = True self._configure()