def take_action(self, parsed_args):
from oio.common.easy_value import convert_size
self.log.debug('take_action(%s)', parsed_args)
account = self.app.client_manager.account
self.take_action_container(parsed_args)
# The command is named 'show' but we must call
# container_get_properties() because container_show() does
# not return system properties (and we need them).
data = self.app.client_manager.storage.container_get_properties(
account,
parsed_args.container,
cid=parsed_args.cid,
admin_mode=True)
sys = data['system']
ctime = float(sys[M2_PROP_CTIME]) / 1000000.
bytes_usage = sys.get(M2_PROP_USAGE, 0)
objects = sys.get(M2_PROP_OBJECTS, 0)
damaged_objects = sys.get(M2_PROP_DAMAGED_OBJECTS, 0)
missing_chunks = sys.get(M2_PROP_MISSING_CHUNKS, 0)
if parsed_args.formatter == 'table':
ctime = int(ctime)
bytes_usage = convert_size(int(bytes_usage), unit="B")
objects = convert_size(int(objects))
info = {
'account': sys['sys.account'],
'base_name': sys['sys.name'],
'container': sys['sys.user.name'],
'ctime': ctime,
'bytes_usage': bytes_usage,
'quota': sys.get(M2_PROP_QUOTA, "Namespace default"),
'objects': objects,
'damaged_objects': damaged_objects,
'missing_chunks': missing_chunks,
'storage_policy': sys.get(M2_PROP_STORAGE_POLICY,
"Namespace default"),
'max_versions': sys.get(M2_PROP_VERSIONING_POLICY,
"Namespace default"),
'status': OIO_DB_STATUS_NAME.get(sys.get('sys.status'), "Unknown"),
}
for k in ('stats.page_count', 'stats.freelist_count',
'stats.page_size'):
info[k] = sys.get(k)
wasted = (float_value(info['stats.freelist_count'], 0) /
float_value(info['stats.page_count'], 1))
wasted_bytes = (int_value(info['stats.freelist_count'], 0) *
int_value(info['stats.page_size'], 0))
info['stats.space_wasted'] = "%5.2f%% (est. %s)" % \
(wasted * 100, convert_size(wasted_bytes))
bucket = sys.get(M2_PROP_BUCKET_NAME, None)
if bucket is not None:
info['bucket'] = bucket
delete_exceeding = sys.get(M2_PROP_DEL_EXC_VERSIONS, None)
if delete_exceeding is not None:
info['delete_exceeding_versions'] = delete_exceeding != '0'
for k, v in iteritems(data['properties']):
info['meta.' + k] = v
return list(zip(*sorted(info.items())))
def sanitize_params(cls, job_params):
sanitized_job_params, _ = super(RawxDecommissionJob,
cls).sanitize_params(job_params)
# specific configuration
service_id = job_params.get('service_id')
if not service_id:
raise ValueError('Missing service ID')
sanitized_job_params['service_id'] = service_id
sanitized_job_params['rawx_timeout'] = float_value(
job_params.get('rawx_timeout'), cls.DEFAULT_RAWX_TIMEOUT)
sanitized_job_params['min_chunk_size'] = int_value(
job_params.get('min_chunk_size'), cls.DEFAULT_MIN_CHUNK_SIZE)
sanitized_job_params['max_chunk_size'] = int_value(
job_params.get('max_chunk_size'), cls.DEFAULT_MAX_CHUNK_SIZE)
excluded_rawx = job_params.get('excluded_rawx')
if excluded_rawx:
excluded_rawx = excluded_rawx.split(',')
else:
excluded_rawx = list()
sanitized_job_params['excluded_rawx'] = excluded_rawx
sanitized_job_params['usage_target'] = int_value(
job_params.get('usage_target'), cls.DEFAULT_USAGE_TARGET)
sanitized_job_params['usage_check_interval'] = float_value(
job_params.get('usage_check_interval'),
cls.DEFAULT_USAGE_CHECK_INTERVAL)
return sanitized_job_params, 'rawx/%s' % service_id
def init(self):
super(ReplicateFilter, self).init()
self.account = self.app_env['account_client']
self.cache_duration = float_value(self.conf.get('cache_duration'),
CACHE_DURATION)
self.cache_size = int_value(self.conf.get('cache_size'), CACHE_SIZE)
self.cache = CacheDict(self.cache_size)
self.check_account = boolean_value(
self.conf.get('check_replication_enabled'), False)
self.connection_timeout = float_value(
self.conf.get('connection_timeout'), CONNECTION_TIMEOUT)
self.read_timeout = float_value(self.conf.get('read_timeout'),
READ_TIMEOUT)
def __init__(self,
conf,
endpoint=None,
proxy_endpoint=None,
refresh_delay=3600.0,
logger=None,
**kwargs):
"""
Initialize a client for the account service.
:param conf: dictionary with at least the namespace name
:type conf: `dict`
:param endpoint: URL of an account service
:param proxy_endpoint: URL of the proxy
:param refresh_interval: time between refreshes of the
account service endpoint (if not provided at instantiation)
:type refresh_interval: `float` seconds
"""
super(AccountClient, self).__init__(endpoint=endpoint,
service_type='account-service',
**kwargs)
self.logger = logger or get_logger(conf)
self.cs = ConscienceClient(conf,
endpoint=proxy_endpoint,
logger=self.logger,
**kwargs)
self._global_kwargs = {
tok: float_value(tov, None)
for tok, tov in kwargs.items() if tok in TIMEOUT_KEYS
}
self._refresh_delay = refresh_delay if not self.endpoint else -1.0
self._last_refresh = 0.0
def __init__(self, namespace, logger=None, **kwargs):
"""
Initialize the object storage API.
:param namespace: name of the namespace to interract with
:type namespace: `str`
:keyword connection_timeout: connection timeout towards rawx services
:type connection_timeout: `float` seconds
:keyword read_timeout: timeout for rawx responses and data reads from
the caller (when uploading)
:type read_timeout: `float` seconds
:keyword write_timeout: timeout for rawx write requests
:type write_timeout: `float` seconds
:keyword pool_manager: a pooled connection manager that will be used
for all HTTP based APIs (except rawx)
:type pool_manager: `urllib3.PoolManager`
"""
self.namespace = namespace
conf = {"namespace": self.namespace}
self.logger = logger or get_logger(conf)
self.timeouts = {tok: float_value(tov, None)
for tok, tov in kwargs.items()
if tok in self.__class__.TIMEOUT_KEYS}
from oio.account.client import AccountClient
from oio.container.client import ContainerClient
from oio.directory.client import DirectoryClient
self.directory = DirectoryClient(conf, logger=self.logger, **kwargs)
self.container = ContainerClient(conf, logger=self.logger, **kwargs)
# In AccountClient, "endpoint" is the account service, not the proxy
acct_kwargs = kwargs.copy()
acct_kwargs["proxy_endpoint"] = acct_kwargs.pop("endpoint", None)
self.account = AccountClient(conf, logger=self.logger, **acct_kwargs)
def list_containers(self,
account_id,
limit=1000,
marker=None,
end_marker=None,
prefix=None,
delimiter=None):
raw_list = self._raw_listing(account_id,
limit=limit,
marker=marker,
end_marker=end_marker,
prefix=prefix,
delimiter=delimiter)
pipeline = self.conn.pipeline(True)
# skip prefix
for container in [entry for entry in raw_list if not entry[3]]:
pipeline.hmget(AccountBackend.ckey(account_id, container[0]),
'objects', 'bytes', 'mtime')
res = pipeline.execute()
i = 0
for container in raw_list:
if not container[3]:
container[1] = int_value(res[i][0], 0)
container[2] = int_value(res[i][1], 0)
container[4] = float_value(res[i][2], 0.0)
i += 1
return raw_list
def list_containers(self,
account_id,
limit=1000,
marker=None,
end_marker=None,
prefix=None,
delimiter=None,
s3_buckets_only=False):
raw_list, _next_marker = self._raw_listing(
self.clistkey(account_id),
limit=limit,
marker=marker,
end_marker=end_marker,
prefix=prefix,
delimiter=delimiter,
s3_buckets_only=s3_buckets_only)
pipeline = self.conn_slave.pipeline(True)
# skip prefix
for container in [entry for entry in raw_list if not entry[3]]:
pipeline.hmget(AccountBackend.ckey(account_id, container[0]),
'objects', 'bytes', 'mtime')
res = pipeline.execute()
i = 0
for container in raw_list:
if not container[3]:
# FIXME(adu) Convert to dict
container[1] = int_value(res[i][0], 0)
container[2] = int_value(res[i][1], 0)
container[4] = float_value(res[i][2], 0.0)
i += 1
return raw_list
def __init__(self, conf, service, **kwargs):
self.conf = conf
self.running = False
for k in ['host', 'port', 'type']:
if k not in service:
raise Exception('Missing field "%s" in service configuration' %
k)
self.name = '%s|%s|%s' % \
(service['type'], service['host'], service['port'])
self.service = service
self.rise = int_value(self._load_item_config('rise'), 1)
self.fall = int_value(self._load_item_config('fall'), 1)
self.check_interval = float_value(
self._load_item_config('check_interval'), 1)
self.deregister_on_exit = true_value(
self._load_item_config('deregister_on_exit', False))
self.logger = get_logger(self.conf)
self.pool_manager = get_pool_manager()
self.cs = ConscienceClient(self.conf,
pool_manager=self.pool_manager,
logger=self.logger)
# FIXME: explain that
self.client = ProxyClient(self.conf,
pool_manager=self.pool_manager,
no_ns_in_url=True,
logger=self.logger)
self.last_status = False
self.status = False
self.failed = False
self.service_definition = {
'ns': self.conf['namespace'],
'type': self.service['type'],
'addr': '%s:%s' % (self.service['host'], self.service['port']),
'score': 0,
'tags': {}
}
if self.service.get('slots', None):
self.service_definition['tags']['tag.slots'] = \
','.join(self.service['slots'])
for name, tag in (('location', 'tag.loc'),
('service_id', 'tag.service_id'), ('tls',
'tag.tls')):
if self.service.get(name):
self.service_definition['tags'][tag] = \
self.service[name]
self.service_checks = list()
self.service_stats = list()
self.init_checkers(service)
self.init_stats(service)
def __init__(self, agent, checker_conf, logger):
self.agent = agent
self.checker_conf = checker_conf
self.logger = logger
self.timeout = float_value(checker_conf.get('timeout'), 5.0)
self.rise = checker_conf['rise']
self.fall = checker_conf['fall']
self.results = RingBuffer(max([self.rise, self.fall]))
self.name = checker_conf.get('name')
self.srv_type = agent.service['type']
self.last_result = None
self.configure()
def configure(self):
self.parser = self.stat_conf.get('parser', 'lines')
self.path = self.stat_conf['path'].lstrip('/')
self.host = self.stat_conf['host']
self.port = self.stat_conf['port']
self.url = '%s:%s/%s' % (self.host, self.port, self.path)
if self.parser == 'json':
# use json parser (account and rdir style)
self._parse_func = self._parse_stats_json
else:
# default to lines parser (rawx style)
self._parse_func = self._parse_stats_lines
self.timeout = float_value(self.stat_conf.get('timeout'), 10.0)
def __init__(self, conf, input_file=None, service_id=None, **kwargs):
super(BlobRebuilder, self).__init__(conf, **kwargs)
# counters
self.bytes_processed = 0
self.total_bytes_processed = 0
# input
self.input_file = input_file
self.rawx_id = service_id
# rawx/rdir
self.rdir_client = RdirClient(self.conf, logger=self.logger)
self.rdir_fetch_limit = int_value(self.conf.get('rdir_fetch_limit'),
self.DEFAULT_RDIR_FETCH_LIMIT)
self.rdir_timeout = float_value(conf.get('rdir_timeout'),
self.DEFAULT_RDIR_TIMEOUT)
def list_buckets(self,
account_id,
limit=1000,
marker=None,
end_marker=None,
prefix=None,
**kwargs):
"""
Get the list of buckets of the specified account.
:returns: the list of buckets (with metadata), and the next
marker (in case the list is truncated).
"""
raw_list, next_marker = self._raw_listing(self.blistkey(account_id),
limit=limit,
marker=marker,
end_marker=end_marker,
prefix=prefix,
**kwargs)
conn = self.get_slave_conn(**kwargs)
pipeline = conn.pipeline(True)
for entry in raw_list:
# For real buckets (not prefixes), fetch metadata.
if not entry[3]:
pipeline.hmget(self.bkey(entry[0]), 'objects', 'bytes',
'mtime')
res = pipeline.execute()
output = list()
i = 0
for bucket in raw_list:
if not bucket[3]:
bdict = {
'name': bucket[0],
'objects': int_value(res[i][0], 0),
'bytes': int_value(res[i][1], 0),
'mtime': float_value(res[i][2], 0.0),
}
i += 1
else:
bdict = {'prefix': bucket}
output.append(bdict)
return output, next_marker
def sanitize_params(cls, job_params):
sanitized_job_params, _ = super(RawxRebuildJob,
cls).sanitize_params(job_params)
# specific configuration
service_id = job_params.get('service_id')
if not service_id:
raise ValueError('Missing service ID')
sanitized_job_params['service_id'] = service_id
sanitized_job_params['rawx_timeout'] = float_value(
job_params.get('rawx_timeout'), cls.DEFAULT_RAWX_TIMEOUT)
sanitized_job_params['dry_run'] = boolean_value(
job_params.get('dry_run'), cls.DEFAULT_DRY_RUN)
sanitized_job_params['allow_same_rawx'] = boolean_value(
job_params.get('allow_same_rawx'), cls.DEFAULT_ALLOW_SAME_RAWX)
sanitized_job_params['try_chunk_delete'] = boolean_value(
job_params.get('try_chunk_delete'), cls.DEFAULT_TRY_CHUNK_DELETE)
sanitized_job_params['allow_frozen_container'] = boolean_value(
job_params.get('allow_frozen_container'),
cls.DEFAULT_ALLOW_FROZEN_CT)
set_specific_incident_date = int_value(
job_params.get('set_specific_incident_date'), None)
if set_specific_incident_date is None:
set_incident_date = boolean_value(
job_params.get('set_incident_date'),
cls.DEFAULT_DECLARE_INCIDENT_DATE)
if set_incident_date:
set_specific_incident_date = int(time.time())
else:
set_incident_date = True
sanitized_job_params['set_incident_date'] = set_incident_date
sanitized_job_params['set_specific_incident_date'] = \
set_specific_incident_date
return sanitized_job_params, 'rawx/%s' % service_id
def __init__(self, agent, checker_conf, logger):
self.agent = agent
self.checker_conf = checker_conf
self.logger = logger
self.timeout = float_value(checker_conf.get('timeout'), 5.0)
self.rise = checker_conf['rise']
self.fall = checker_conf['fall']
self.results = RingBuffer(max([self.rise, self.fall]))
self.name = checker_conf.get('name')
self.srv_type = agent.service['type']
self.last_result = None
for k in ('host', 'port'):
if k not in self.checker_conf:
raise exc.ConfigurationException(
'Missing field "%s" in configuration' % k)
self.host = self.checker_conf['host']
self.port = self.checker_conf['port']
self.name = '%s|%s|%s|%s' % \
(self.srv_type, self.checker_type, self.host, self.port)
self.last_check_success = True
self._configure()
