def _generate_fake_excluded_chunks(self):
conscience_client = ConscienceClient(self.conf, logger=self.logger)
fake_excluded_chunks = list()
fake_chunk_id = '0' * 64
for service_id in self.excluded_rawx:
service_addr = conscience_client.resolve_service_id(
'rawx', service_id)
chunk = dict()
chunk['hash'] = '0000000000000000000000000000000000'
chunk['pos'] = '0'
chunk['size'] = 1
chunk['score'] = 1
chunk['url'] = 'http://' + service_id + '/' + fake_chunk_id
chunk['real_url'] = 'http://' + service_addr + '/' + fake_chunk_id
fake_excluded_chunks.append(chunk)
return fake_excluded_chunks
class RawxDecommissionTask(XcuteTask):
def __init__(self, conf, job_params, logger=None):
super(RawxDecommissionTask, self).__init__(conf,
job_params,
logger=logger)
self.service_id = job_params['service_id']
self.rawx_timeout = job_params['rawx_timeout']
self.min_chunk_size = job_params['min_chunk_size']
self.max_chunk_size = job_params['max_chunk_size']
self.excluded_rawx = job_params['excluded_rawx']
self.blob_client = BlobClient(self.conf, logger=self.logger)
self.content_factory = ContentFactory(self.conf)
self.conscience_client = ConscienceClient(self.conf,
logger=self.logger)
self.fake_excluded_chunks = self._generate_fake_excluded_chunks(
self.excluded_rawx)
def _generate_fake_excluded_chunks(self, excluded_rawx):
fake_excluded_chunks = list()
fake_chunk_id = '0' * 64
for service_id in excluded_rawx:
service_addr = self.conscience_client.resolve_service_id(
'rawx', service_id)
chunk = dict()
chunk['hash'] = '0000000000000000000000000000000000'
chunk['pos'] = '0'
chunk['size'] = 1
chunk['score'] = 1
chunk['url'] = 'http://{}/{}'.format(service_id, fake_chunk_id)
chunk['real_url'] = 'http://{}/{}'.format(service_addr,
fake_chunk_id)
fake_excluded_chunks.append(chunk)
return fake_excluded_chunks
def process(self, task_id, task_payload, reqid=None):
container_id = task_payload['container_id']
content_id = task_payload['content_id']
chunk_id = task_payload['chunk_id']
chunk_url = 'http://{}/{}'.format(self.service_id, chunk_id)
try:
meta = self.blob_client.chunk_head(chunk_url,
timeout=self.rawx_timeout,
reqid=reqid)
except NotFound:
# The chunk is still present in the rdir,
# but the chunk no longer exists in the rawx.
# We ignore it because there is nothing to move.
return {'skipped_chunks_no_longer_exist': 1}
if container_id != meta['container_id']:
raise ValueError('Mismatch container ID: %s != %s', container_id,
meta['container_id'])
if content_id != meta['content_id']:
raise ValueError('Mismatch content ID: %s != %s', content_id,
meta['content_id'])
chunk_size = int(meta['chunk_size'])
# Maybe skip the chunk because it doesn't match the size constaint
if chunk_size < self.min_chunk_size:
self.logger.debug('[reqid=%s] SKIP %s too small', reqid, chunk_url)
return {'skipped_chunks_too_small': 1}
if self.max_chunk_size > 0 and chunk_size > self.max_chunk_size:
self.logger.debug('[reqid=%s] SKIP %s too big', reqid, chunk_url)
return {'skipped_chunks_too_big': 1}
# Start moving the chunk
try:
content = self.content_factory.get(container_id,
content_id,
reqid=reqid)
content.move_chunk(chunk_id,
fake_excluded_chunks=self.fake_excluded_chunks,
reqid=reqid)
except (ContentNotFound, OrphanChunk):
return {'orphan_chunks': 1}
return {'moved_chunks': 1, 'moved_bytes': chunk_size}
