def __init__(self, tool, queue_workers, queue_reply):
super(ContentRepairerWorker, self).__init__(tool, queue_workers,
queue_reply)
self.chunk_operator = ChunkOperator(self.conf, logger=self.logger)
self.blob_client = BlobClient(self.conf)
self.container_client = ContainerClient(self.conf, logger=self.logger)
def __init__(self, conf, job_params, logger=None):
super(RawxRebuildTask, self).__init__(conf, job_params, logger=logger)
self.service_id = job_params['service_id']
self.rawx_timeout = job_params['rawx_timeout']
self.allow_frozen_container = job_params['allow_frozen_container']
self.allow_same_rawx = job_params['allow_same_rawx']
self.try_chunk_delete = job_params['try_chunk_delete']
self.dry_run = job_params['dry_run']
self.chunk_operator = ChunkOperator(self.conf, logger=self.logger)
def __init__(self, tool, queue_workers, queue_reply):
super(BlobRebuilderWorker, self).__init__(
tool, queue_workers, queue_reply)
self.allow_frozen_container = true_value(self.tool.conf.get(
'allow_frozen_container', self.tool.DEFAULT_ALLOW_FROZEN_CT))
self.allow_same_rawx = true_value(self.tool.conf.get(
'allow_same_rawx', self.tool.DEFAULT_ALLOW_SAME_RAWX))
self.try_chunk_delete = true_value(self.tool.conf.get(
'try_chunk_delete', self.tool.DEFAULT_TRY_CHUNK_DELETE))
self.dry_run = true_value(self.tool.conf.get(
'dry_run', self.tool.DEFAULT_DRY_RUN))
self.chunk_operator = ChunkOperator(self.conf, logger=self.logger)
class BlobRebuilderWorker(ToolWorker):
def __init__(self, tool, queue_workers, queue_reply):
super(BlobRebuilderWorker, self).__init__(tool, queue_workers,
queue_reply)
self.allow_same_rawx = true_value(
self.tool.conf.get('allow_same_rawx',
self.tool.DEFAULT_ALLOW_SAME_RAWX))
self.try_chunk_delete = true_value(
self.tool.conf.get('try_chunk_delete',
self.tool.DEFAULT_TRY_CHUNK_DELETE))
self.dry_run = true_value(
self.tool.conf.get('dry_run', self.tool.DEFAULT_DRY_RUN))
self.chunk_operator = ChunkOperator(self.conf, logger=self.logger)
def _process_item(self, item):
namespace, container_id, content_id, chunk_id_or_pos = item
if namespace != self.tool.namespace:
raise ValueError('Invalid namespace (actual=%s, expected=%s)' %
(namespace, self.tool.namespace))
log_rebuilding = 'Rebuilding %s' % self.tool.string_from_item(item)
if self.dry_run:
self.logger.debug('[dryrun] %s', log_rebuilding)
return None
self.logger.debug(log_rebuilding)
return self.chunk_operator.rebuild(
container_id,
content_id,
chunk_id_or_pos,
rawx_id=self.tool.rawx_id,
try_chunk_delete=self.try_chunk_delete,
allow_same_rawx=self.allow_same_rawx)
class RawxRebuildTask(XcuteTask):
def __init__(self, conf, job_params, logger=None):
super(RawxRebuildTask, self).__init__(conf, job_params, logger=logger)
self.service_id = job_params['service_id']
self.rawx_timeout = job_params['rawx_timeout']
self.allow_frozen_container = job_params['allow_frozen_container']
self.allow_same_rawx = job_params['allow_same_rawx']
self.try_chunk_delete = job_params['try_chunk_delete']
self.dry_run = job_params['dry_run']
self.chunk_operator = ChunkOperator(self.conf, logger=self.logger)
def process(self, task_id, task_payload, reqid=None):
container_id = task_payload['container_id']
content_id = task_payload['content_id']
chunk_id = task_payload['chunk_id']
if self.dry_run:
self.logger.debug('[reqid=%s] [dryrun] Rebuilding %s', reqid,
chunk_id)
return {'skipped_chunks': 1}
# Start rebuilding the chunk
self.logger.debug('[reqid=%s] Rebuilding %s', reqid, chunk_id)
try:
chunk_size = self.chunk_operator.rebuild(
container_id,
content_id,
chunk_id,
rawx_id=self.service_id,
try_chunk_delete=self.try_chunk_delete,
allow_frozen_container=self.allow_frozen_container,
allow_same_rawx=self.allow_same_rawx)
except (ContentNotFound, OrphanChunk):
return {'orphan_chunks': 1}
return {'rebuilt_chunks': 1, 'rebuilt_bytes': chunk_size}
class ContentRepairerWorker(ToolWorker):
def __init__(self, tool, queue_workers, queue_reply):
super(ContentRepairerWorker, self).__init__(tool, queue_workers,
queue_reply)
self.chunk_operator = ChunkOperator(self.conf, logger=self.logger)
self.blob_client = BlobClient(self.conf)
self.container_client = ContainerClient(self.conf, logger=self.logger)
def _safe_chunk_rebuild(self, item, content_id, chunk_id_or_pos, **kwargs):
_, account, container, _, _ = item
try:
container_id = cid_from_name(account, container)
self.chunk_operator.rebuild(container_id, content_id,
chunk_id_or_pos, **kwargs)
except Exception as exc: # pylint: disable=broad-except
self.logger.error('ERROR when rebuilding chunk %s (%s): %s',
self.tool.string_from_item(item),
chunk_id_or_pos, exc)
return exc
def _repair_metachunk(self, item, content_id, stg_met, pos, chunks):
"""
Check that a metachunk has the right number of chunks.
:returns: the list (generator) of missing chunks
"""
exceptions = list()
required = stg_met.expected_chunks
if len(chunks) < required:
if stg_met.ec:
subs = {x['num'] for x in chunks}
for sub in range(required):
if sub not in subs:
exc = self._safe_chunk_rebuild(item, content_id,
"%d.%d" % (pos, sub))
if exc:
exceptions.append(exc)
else:
missing_chunks = required - len(chunks)
for _ in range(missing_chunks):
exc = self._safe_chunk_rebuild(item, content_id, pos)
if exc:
exceptions.append(exc)
for chunk in chunks:
try:
self.blob_client.chunk_head(chunk['url'],
xattr=True,
check_hash=True)
except (NotFound, ClientPreconditionFailed) as e:
kwargs = {
'try_chunk_delete': isinstance(e, ClientPreconditionFailed)
}
exc = self._safe_chunk_rebuild(item, content_id, chunk['url'],
**kwargs)
if exc:
exceptions.append(exc)
except Exception as exc: # pylint: disable=broad-except
self.logger.error('ERROR when checking chunk %s (%s): %s',
self.tool.string_from_item(item),
chunk['url'], exc)
exceptions.append(exc)
return exceptions
def _process_item(self, item):
namespace, account, container, obj_name, version = item
if namespace != self.tool.namespace:
raise ValueError('Invalid namespace (actual=%s, expected=%s)' %
(namespace, self.tool.namespace))
obj_meta, chunks = self.container_client.content_locate(
account=account,
reference=container,
path=obj_name,
version=version,
properties=False)
content_id = obj_meta['id']
exceptions = list()
stg_met = STORAGE_METHODS.load(obj_meta['chunk_method'])
chunks_by_pos = _sort_chunks(chunks, stg_met.ec)
for pos, chunks in iteritems(chunks_by_pos):
try:
exceptions += self._repair_metachunk(item, content_id, stg_met,
pos, chunks)
except Exception as exc: # pylint: disable=broad-except
self.logger.error('ERROR when repair metachunk %s (%d): %s',
self.tool.string_from_item(item), pos, exc)
exceptions.append(exc)
if exceptions:
raise Exception(exceptions)
self.container_client.content_touch(account=account,
reference=container,
path=obj_name,
version=version)