def take_action(self, parsed_args):
self.log.debug('take_action(%s)', parsed_args)
results = []
srv_types = parsed_args.srv_types
local_scores = boolean_value(
self.app.client_manager.sds_conf.get('proxy.quirk.local_scores'),
False)
if not local_scores:
self.log.warn("'proxy.quirk.local_scores' not set, "
"scores won't be realistic.")
data = self.app.client_manager.conscience.local_services()
for srv in data:
tags = srv['tags']
location = tags.get('tag.loc', 'n/a')
slots = tags.get('tag.slots', 'n/a')
volume = tags.get('tag.vol', 'n/a')
service_id = tags.get('tag.service_id', 'n/a')
addr = srv['addr']
up = tags.get('tag.up', 'n/a')
score = srv['score']
locked = boolean_value(tags.get('tag.lock'), False)
srv_type = srv['type']
if not srv_types or srv_type in srv_types:
results.append((srv_type, addr, service_id, volume, location,
slots, up, score, locked))
columns = ('Type', 'Addr', 'Service Id', 'Volume', 'Location', 'Slots',
'Up', 'Score', 'Locked')
result_gen = (r for r in results)
return columns, result_gen
def init(self):
super(ReplicateFilter, self).init()
self.account = self.app_env['account_client']
self.cache_duration = float_value(self.conf.get('cache_duration'),
CACHE_DURATION)
self.cache_size = int_value(self.conf.get('cache_size'), CACHE_SIZE)
self.cache = CacheDict(self.cache_size)
self.check_account = boolean_value(
self.conf.get('check_replication_enabled'), False)
self.connection_timeout = float_value(
self.conf.get('connection_timeout'), CONNECTION_TIMEOUT)
self.force_master = boolean_value(self.conf.get('force_master'), False)
self.read_timeout = float_value(self.conf.get('read_timeout'),
READ_TIMEOUT)
def _list_services(self, parsed_args):
if not parsed_args.srv_types:
parsed_args.srv_types = \
self.app.client_manager.conscience.service_types()
for srv_type in parsed_args.srv_types:
try:
data = self.app.client_manager.conscience.all_services(
srv_type, parsed_args.stats)
except OioException as exc:
self.success = False
self.log.error("Failed to list services of type %s: %s",
srv_type, exc)
continue
for srv in data:
tags = srv['tags']
location = tags.get('tag.loc', 'n/a')
slots = tags.get('tag.slots', 'n/a')
volume = tags.get('tag.vol', 'n/a')
service_id = tags.get('tag.service_id', 'n/a')
addr = srv['addr']
locked = boolean_value(tags.get('tag.lock', False))
up = tags.get('tag.up', 'n/a')
score = srv['score']
if parsed_args.stats:
stats = [
"%s=%s" % (k, v) for k, v in tags.items()
if k.startswith('stat.')
]
values = (srv_type, addr, service_id, volume, location,
slots, up, score, locked, " ".join(stats))
else:
values = (srv_type, addr, service_id, volume, location,
slots, up, score, locked)
yield values
def container_flush(self,
account=None,
reference=None,
cid=None,
**kwargs):
params = self._make_params(account, reference, cid=cid)
resp, _ = self._request('POST', '/flush', params=params, **kwargs)
return {
'truncated': boolean_value(resp.getheader('x-oio-truncated',
False))
}
def sanitize_params(cls, job_params):
sanitized_job_params, _ = super(RawxRebuildJob,
cls).sanitize_params(job_params)
# specific configuration
service_id = job_params.get('service_id')
if not service_id:
raise ValueError('Missing service ID')
sanitized_job_params['service_id'] = service_id
sanitized_job_params['rawx_timeout'] = float_value(
job_params.get('rawx_timeout'), cls.DEFAULT_RAWX_TIMEOUT)
sanitized_job_params['dry_run'] = boolean_value(
job_params.get('dry_run'), cls.DEFAULT_DRY_RUN)
sanitized_job_params['allow_same_rawx'] = boolean_value(
job_params.get('allow_same_rawx'), cls.DEFAULT_ALLOW_SAME_RAWX)
sanitized_job_params['try_chunk_delete'] = boolean_value(
job_params.get('try_chunk_delete'), cls.DEFAULT_TRY_CHUNK_DELETE)
sanitized_job_params['allow_frozen_container'] = boolean_value(
job_params.get('allow_frozen_container'),
cls.DEFAULT_ALLOW_FROZEN_CT)
set_specific_incident_date = int_value(
job_params.get('set_specific_incident_date'), None)
if set_specific_incident_date is None:
set_incident_date = boolean_value(
job_params.get('set_incident_date'),
cls.DEFAULT_DECLARE_INCIDENT_DATE)
if set_incident_date:
set_specific_incident_date = int(time.time())
else:
set_incident_date = True
sanitized_job_params['set_incident_date'] = set_incident_date
sanitized_job_params['set_specific_incident_date'] = \
set_specific_incident_date
return sanitized_job_params, 'rawx/%s' % service_id
def __init__(self, conf):
self.conf = conf
redis_conf = {
k[6:]: v
for k, v in self.conf.items() if k.startswith("redis_")
}
redis_host = redis_conf.pop('host', None)
if redis_host:
parsed = urlparse('http://' + redis_host)
if parsed.port is None:
redis_host = '%s:%s' % (redis_host,
redis_conf.pop('port', '6379'))
redis_sentinel_hosts = redis_conf.pop(
'sentinel_hosts',
# TODO(adu): Delete when it will no longer be used
self.conf.get('sentinel_hosts'))
redis_sentinel_name = redis_conf.pop(
'sentinel_name',
# TODO(adu): Delete when it will no longer be used
self.conf.get('sentinel_master_name'))
super(AccountBackend,
self).__init__(host=redis_host,
sentinel_hosts=redis_sentinel_hosts,
sentinel_name=redis_sentinel_name,
**redis_conf)
self.autocreate = boolean_value(conf.get('autocreate'), True)
self._account_prefix = conf.get('account_prefix', ACCOUNT_KEY_PREFIX)
self._bucket_prefix = conf.get('bucket_prefix', BUCKET_KEY_PREFIX)
self._bucket_list_prefix = conf.get('bucket_list_prefix',
BUCKET_LIST_PREFIX)
self._container_list_prefix = conf.get('container_list_prefix',
CONTAINER_LIST_PREFIX)
self._bucket_lock_prefix = conf.get('bucket_lock_prefix',
BUCKET_LOCK_KEY_PREFIX)
update_container_patched = self.lua_update_container % {
'bucket_list_prefix': self._bucket_list_prefix
}
self.script_update_container = self.register_script(
update_container_patched)
self.script_refresh_bucket = self.register_script(
self.lua_refresh_bucket_batch)
self.script_refresh_account = self.register_script(
self.lua_refresh_account)
self.script_flush_account = self.register_script(
self.lua_flush_account)
self.script_get_container_info = self.register_script(
self.lua_get_extended_container_info)
self.script_get_lock_bucket = self.register_script(
self.lua_lock_bucket)
def __init__(self, volume_path, conf, pool_manager=None):
"""
Initializes an Indexing worker for indexing meta2 databases.
Possible values of conf relating to this worker are:
- interval: (int) in sec time between two full scans. Default: half an
hour.
- report_interval: (int) in sec, time between two reports: Default: 300
- scanned_per_second: (int) maximum number of indexed databases /s.
- try_removing_faulty_indexes : In the event where we encounter a
database that's not supposed to be handled by this volume, attempt
to remove it from this volume rdir index if it exists
WARNING: The decision is based off of a proxy response, that could
be affected by cache inconsistencies for example, use at your own
risk. Default: False
:param volume_path: The volume path to be indexed
:param conf: The configuration to be passed to the needed services
:param pool_manager: A connection pool manager. If none is given, a
new one with a default size of 10 will be created.
"""
self.logger = get_logger(conf)
self._stop = False
self.volume = volume_path
self.success_nb = 0
self.failed_nb = 0
self.full_scan_nb = 0
self.last_report_time = 0
self.last_scan_time = 0
self.last_index_time = 0
self.start_time = 0
self.indexed_since_last_report = 0
self.scans_interval = int_value(
conf.get('interval'), 1800)
self.report_interval = int_value(
conf.get('report_interval'), 300)
self.max_indexed_per_second = int_value(
conf.get('scanned_per_second'), 3000)
self.namespace, self.volume_id = check_volume_for_service_type(
self.volume, "meta2")
self.attempt_bad_index_removal = boolean_value(
conf.get('try_removing_faulty_indexes', False)
)
if not pool_manager:
pool_manager = get_pool_manager(pool_connections=10)
self.index_client = RdirClient(conf, logger=self.logger,
pool_manager=pool_manager)
self.dir_client = DirectoryClient(conf, logger=self.logger,
pool_manager=pool_manager)
def cast_fields(self, info):
"""
Cast dict entries to the type they are supposed to be.
"""
for what in (b'bytes', b'objects'):
try:
info[what] = int_value(info.get(what), 0)
except (TypeError, ValueError):
pass
for what in (BUCKET_PROP_REPLI_ENABLED.encode('utf-8'), ):
try:
val = info.get(what)
decoded = val.decode('utf-8') if val is not None else None
info[what] = boolean_value(decoded)
except (TypeError, ValueError):
pass
def cast_fields(self, info):
"""
Cast dict entries to the type they are supposed to be.
"""
for what in (b'bytes', b'objects', b'damaged_objects',
b'missing_chunks'):
try:
info[what] = int_value(info.get(what), 0)
except (TypeError, ValueError):
pass
for what in (BUCKET_PROP_REPLI_ENABLED.encode('utf-8'), ):
try:
decoded = info.get(what, b'').decode('utf-8')
info[what] = boolean_value(decoded)
except (TypeError, ValueError):
pass
def flush_and_check(truncated=False, objects=0, usage=0):
resp = self.request('POST',
self.url_container('flush'),
params=params)
self.assertEqual(204, resp.status)
self.assertEqual(truncated,
boolean_value(resp.getheader('x-oio-truncated')))
self._wait_account_meta2()
resp = self.request('POST',
self.url_container('get_properties'),
params=params)
data = self.json_loads(resp.data)
self.assertEqual(data['system']['sys.m2.objects'], str(objects))
self.assertEqual(data['system']['sys.m2.usage'], str(usage))
resp = self.request('GET',
self.url_container('list'),
params=params)
data = self.json_loads(resp.data)
self.assertEqual(len(data['objects']), objects)
