def rebuild_chunk(self, chunk_id, allow_same_rawx=False, chunk_pos=None):
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None and chunk_pos is None:
raise OrphanChunk("Chunk not found in content")
elif current_chunk is None:
chunk = {"pos": chunk_pos, "url": ""}
current_chunk = Chunk(chunk)
chunks = self.chunks.filter(metapos=current_chunk.metapos)\
.exclude(id=chunk_id)
if chunk_id is None:
current_chunk.size = chunks[0].size
current_chunk.checksum = chunks[0].checksum
broken_list = list()
if not allow_same_rawx:
broken_list.append(current_chunk)
spare_url = self._get_spare_chunk(chunks.all(), broken_list)
handler = ECRebuildHandler(chunks.raw(), current_chunk.subpos,
self.storage_method)
new_chunk = {'pos': current_chunk.pos, 'url': spare_url[0]}
new_chunk = Chunk(new_chunk)
stream = handler.rebuild()
meta = {}
meta['chunk_id'] = new_chunk.id
meta['chunk_pos'] = current_chunk.pos
meta['container_id'] = self.container_id
# FIXME: should be 'content_chunkmethod' everywhere
# but sadly it isn't
meta['chunk_method'] = self.chunk_method
# FIXME: should be 'content_id' everywhere
# but sadly it isn't
meta['id'] = self.content_id
meta['content_path'] = self.path
# FIXME: should be 'content_policy' everywhere
# but sadly it isn't
meta['policy'] = self.policy
# FIXME: should be 'content_version' everywhere
# but sadly it isn't
meta['version'] = self.version
meta['metachunk_hash'] = current_chunk.checksum
meta['metachunk_size'] = current_chunk.size
meta['full_path'] = self.full_path
meta['oio_version'] = OIO_VERSION
self.blob_client.chunk_put(spare_url[0], meta, GeneratorIO(stream))
if chunk_id is None:
self._add_raw_chunk(current_chunk, spare_url[0])
else:
self._update_spare_chunk(current_chunk, spare_url[0])
def rebuild_chunk(self, chunk_id):
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None:
raise OrphanChunk("Chunk not found in content")
chunks = self.chunks.filter(metapos=current_chunk.metapos)\
.exclude(id=chunk_id)
spare_url = self._get_spare_chunk(chunks.all(), [current_chunk])
handler = ECRebuildHandler(
chunks.raw(), current_chunk.subpos, self.storage_method)
new_chunk = {'pos': current_chunk.pos, 'url': spare_url[0]}
new_chunk = Chunk(new_chunk)
stream = handler.rebuild()
meta = {}
meta['chunk_id'] = new_chunk.id
meta['chunk_pos'] = current_chunk.pos
meta['container_id'] = self.container_id
meta['content_chunkmethod'] = self.chunk_method
meta['content_id'] = self.content_id
meta['content_path'] = self.path
meta['content_policy'] = self.stgpol
meta['content_version'] = self.version
meta['metachunk_hash'] = current_chunk.checksum
meta['metachunk_size'] = current_chunk.size
self.blob_client.chunk_put(spare_url[0], meta, stream)
self._update_spare_chunk(current_chunk, spare_url[0])
def rebuild_chunk(self, chunk_id):
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None:
raise OrphanChunk("Chunk not found in content")
chunks = self.chunks.filter(metapos=current_chunk.metapos)\
.exclude(id=chunk_id)
spare_url = self._get_spare_chunk(chunks.all(), [current_chunk])
handler = ECRebuildHandler(
chunks.raw(), current_chunk.subpos, self.storage_method)
new_chunk = {'pos': current_chunk.pos, 'url': spare_url[0]}
new_chunk = Chunk(new_chunk)
stream = handler.rebuild()
meta = {}
meta['chunk_id'] = new_chunk.id
meta['chunk_pos'] = current_chunk.pos
meta['container_id'] = self.container_id
meta['content_chunkmethod'] = self.chunk_method
meta['content_id'] = self.content_id
meta['content_path'] = self.path
meta['content_policy'] = self.stgpol
meta['content_version'] = self.version
meta['metachunk_hash'] = current_chunk.checksum
meta['metachunk_size'] = current_chunk.size
self.blob_client.chunk_put(spare_url[0], meta, stream)
self._update_spare_chunk(current_chunk, spare_url[0])
def test_rebuild_with_wrong_chunk_size(self):
test_data = (b'1234' * self.storage_method.ec_segment_size)[:-777]
ec_chunks = self._make_ec_chunks(test_data)
missing_chunk_body = ec_chunks.pop(1)
meta_chunk = self.meta_chunk()
missing_chunk = meta_chunk.pop(1)
responses = list()
for i, ec_chunk in enumerate(ec_chunks):
chunk_size = len(ec_chunk)
if i < self.storage_method.ec_nb_parity - 1:
# Change the chunk size for the first chunks
chunk_size = random.randrange(chunk_size)
headers = {'x-oio-chunk-meta-chunk-size': chunk_size}
responses.append(FakeResponse(200, ec_chunk[:chunk_size], headers))
def get_response(req):
return responses.pop(0) if responses else FakeResponse(404)
missing = missing_chunk['num']
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
with set_http_requests(get_response) as conn_record:
handler = ECRebuildHandler(meta_chunk, missing,
self.storage_method)
expected_chunk_size, stream = handler.rebuild()
if expected_chunk_size is not None:
self.assertEqual(expected_chunk_size, len(missing_chunk_body))
result = b''.join(stream)
self.assertEqual(len(result), len(missing_chunk_body))
self.assertEqual(
self.checksum(result).hexdigest(),
self.checksum(missing_chunk_body).hexdigest())
self.assertEqual(len(conn_record), nb - 1)
def test_rebuild_failure(self):
meta_chunk = self.meta_chunk()
missing_chunk = meta_chunk.pop(1)
nb = self.storage_method.ec_nb_data +\
self.storage_method.ec_nb_parity
# add errors on other chunks
errors = [Timeout(), 404, Exception('failure')]
responses = [
FakeResponse(random.choice(errors), b'', {}) for i in range(nb - 1)
]
def get_response(req):
return responses.pop(0) if responses else FakeResponse(404)
missing = missing_chunk['num']
nb = self.storage_method.ec_nb_data +\
self.storage_method.ec_nb_parity
with set_http_requests(get_response) as conn_record:
handler = ECRebuildHandler(meta_chunk, missing,
self.storage_method)
# TODO use specialized exception
self.assertRaises(exc.OioException, handler.rebuild)
self.assertEqual(len(conn_record), nb - 1)
def rebuild_chunk(self, chunk_id):
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None:
raise OrphanChunk("Chunk not found in content")
chunks = self.chunks.filter(metapos=current_chunk.metapos).exclude(id=chunk_id)
spare_url = self._get_spare_chunk(chunks.all(), [current_chunk])
handler = ECRebuildHandler(chunks.raw(), current_chunk.subpos, self.storage_method)
new_chunk = {"pos": current_chunk.pos, "url": spare_url[0]}
new_chunk = Chunk(new_chunk)
stream = handler.rebuild()
meta = {}
meta["chunk_id"] = new_chunk.id
meta["chunk_pos"] = current_chunk.pos
meta["container_id"] = self.container_id
# FIXME: should be 'content_chunkmethod' everywhere
# but sadly it isn't
meta["chunk_method"] = self.chunk_method
# FIXME: should be 'content_id' everywhere
# but sadly it isn't
meta["id"] = self.content_id
meta["content_path"] = self.path
# FIXME: should be 'content_policy' everywhere
# but sadly it isn't
meta["policy"] = self.stgpol
# FIXME: should be 'content_version' everywhere
# but sadly it isn't
meta["version"] = self.version
meta["metachunk_hash"] = current_chunk.checksum
meta["metachunk_size"] = current_chunk.size
self.blob_client.chunk_put(spare_url[0], meta, GeneratorReader(stream))
self._update_spare_chunk(current_chunk, spare_url[0])
def test_rebuild_parity_errors(self):
test_data = (b'1234' * self.storage_method.ec_segment_size)[:-777]
ec_chunks = self._make_ec_chunks(test_data)
# break one parity chunk
missing_chunk_body = ec_chunks.pop(-1)
meta_chunk = self.meta_chunk()
missing_chunk = meta_chunk.pop(-1)
# add also error on another chunk
for error in (Timeout(), 404, Exception('failure')):
headers = {}
base_responses = list()
for ec_chunk in ec_chunks:
base_responses.append(FakeResponse(200, ec_chunk, headers))
responses = base_responses
error_idx = random.randint(0, len(responses) - 1)
responses[error_idx] = FakeResponse(error, b'', {})
def get_response(req):
return responses.pop(0) if responses else FakeResponse(404)
missing = missing_chunk['num']
nb = self.storage_method.ec_nb_data +\
self.storage_method.ec_nb_parity
with set_http_requests(get_response) as conn_record:
handler = ECRebuildHandler(meta_chunk, missing,
self.storage_method)
expected_chunk_size, stream = handler.rebuild()
if expected_chunk_size is not None:
self.assertEqual(expected_chunk_size,
len(missing_chunk_body))
result = b''.join(stream)
self.assertEqual(len(result), len(missing_chunk_body))
self.assertEqual(
self.checksum(result).hexdigest(),
self.checksum(missing_chunk_body).hexdigest())
self.assertEqual(len(conn_record), nb - 1)
def test_rebuild(self):
test_data = (b'1234' * self.storage_method.ec_segment_size)[:-777]
ec_chunks = self._make_ec_chunks(test_data)
missing_chunk_body = ec_chunks.pop(1)
meta_chunk = self.meta_chunk()
missing_chunk = meta_chunk.pop(1)
headers = {}
responses = [
FakeResponse(200, ec_chunks[0], headers),
FakeResponse(200, ec_chunks[1], headers),
FakeResponse(200, ec_chunks[2], headers),
FakeResponse(200, ec_chunks[3], headers),
FakeResponse(200, ec_chunks[4], headers),
FakeResponse(200, ec_chunks[5], headers),
FakeResponse(200, ec_chunks[6], headers),
]
def get_response(req):
return responses.pop(0) if responses else FakeResponse(404)
missing = missing_chunk['num']
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
with set_http_requests(get_response) as conn_record:
handler = ECRebuildHandler(meta_chunk, missing,
self.storage_method)
expected_chunk_size, stream = handler.rebuild()
if expected_chunk_size is not None:
self.assertEqual(expected_chunk_size, len(missing_chunk_body))
result = b''.join(stream)
self.assertEqual(len(result), len(missing_chunk_body))
self.assertEqual(
self.checksum(result).hexdigest(),
self.checksum(missing_chunk_body).hexdigest())
self.assertEqual(len(conn_record), nb - 1)
def test_rebuild_parity_errors(self):
test_data = ('1234' * self.storage_method.ec_segment_size)[:-777]
ec_chunks = self._make_ec_chunks(test_data)
# break one parity chunk
missing_chunk_body = ec_chunks.pop(-1)
meta_chunk = self.meta_chunk()
missing_chunk = meta_chunk.pop(-1)
# add also error on another chunk
for error in (Timeout(), 404, Exception('failure')):
headers = {}
base_responses = list()
for ec_chunk in ec_chunks:
base_responses.append(FakeResponse(200, ec_chunk, headers))
responses = base_responses
error_idx = random.randint(0, len(responses) - 1)
responses[error_idx] = FakeResponse(error, '', {})
def get_response(req):
return responses.pop(0) if responses else FakeResponse(404)
missing = missing_chunk['num']
nb = self.storage_method.ec_nb_data +\
self.storage_method.ec_nb_parity
with set_http_requests(get_response) as conn_record:
handler = ECRebuildHandler(
meta_chunk, missing, self.storage_method)
stream = handler.rebuild()
result = ''.join(stream)
self.assertEqual(len(result), len(missing_chunk_body))
self.assertEqual(self.checksum(result).hexdigest(),
self.checksum(missing_chunk_body).hexdigest())
self.assertEqual(len(conn_record), nb - 1)
def test_rebuild(self):
test_data = ('1234' * self.storage_method.ec_segment_size)[:-777]
ec_chunks = self._make_ec_chunks(test_data)
missing_chunk_body = ec_chunks.pop(1)
meta_chunk = self.meta_chunk()
missing_chunk = meta_chunk.pop(1)
headers = {}
responses = [
FakeResponse(200, ec_chunks[0], headers),
FakeResponse(200, ec_chunks[1], headers),
FakeResponse(200, ec_chunks[2], headers),
FakeResponse(200, ec_chunks[3], headers),
FakeResponse(200, ec_chunks[4], headers),
FakeResponse(200, ec_chunks[5], headers),
FakeResponse(200, ec_chunks[6], headers),
]
def get_response(req):
return responses.pop(0) if responses else FakeResponse(404)
missing = missing_chunk['num']
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
with set_http_requests(get_response) as conn_record:
handler = ECRebuildHandler(
meta_chunk, missing, self.storage_method)
stream = handler.rebuild()
result = ''.join(stream)
self.assertEqual(len(result), len(missing_chunk_body))
self.assertEqual(self.checksum(result).hexdigest(),
self.checksum(missing_chunk_body).hexdigest())
self.assertEqual(len(conn_record), nb - 1)
def rebuild_chunk(self,
chunk_id,
allow_same_rawx=False,
chunk_pos=None,
allow_frozen_container=False):
# Identify the chunk to rebuild
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None and chunk_pos is None:
raise OrphanChunk("Chunk not found in content")
elif current_chunk is None:
current_chunk = self.chunks.filter(pos=chunk_pos).one()
if current_chunk is None:
chunk = {'pos': chunk_pos, 'url': ''}
current_chunk = Chunk(chunk)
else:
chunk_id = current_chunk.id
self.logger.debug('Chunk at pos %s has id %s', chunk_pos,
chunk_id)
chunks = self.chunks.filter(metapos=current_chunk.metapos)\
.exclude(id=chunk_id, pos=chunk_pos)
if chunk_id is None:
current_chunk.size = chunks[0].size
current_chunk.checksum = chunks[0].checksum
# Find a spare chunk address
broken_list = list()
if not allow_same_rawx and chunk_id is not None:
broken_list.append(current_chunk)
spare_url, _quals = self._get_spare_chunk(chunks.all(), broken_list)
new_chunk = Chunk({'pos': current_chunk.pos, 'url': spare_url[0]})
# Regenerate the lost chunk's data, from existing chunks
handler = ECRebuildHandler(chunks.raw(), current_chunk.subpos,
self.storage_method)
stream = handler.rebuild()
# Actually create the spare chunk
meta = {}
meta['chunk_id'] = new_chunk.id
meta['chunk_pos'] = current_chunk.pos
meta['container_id'] = self.container_id
# FIXME: should be 'content_chunkmethod' everywhere
# but sadly it isn't
meta['chunk_method'] = self.chunk_method
# FIXME: should be 'content_id' everywhere
# but sadly it isn't
meta['id'] = self.content_id
meta['content_path'] = self.path
# FIXME: should be 'content_policy' everywhere
# but sadly it isn't
meta['policy'] = self.policy
# FIXME: should be 'content_version' everywhere
# but sadly it isn't
meta['version'] = self.version
meta['metachunk_hash'] = current_chunk.checksum
meta['metachunk_size'] = current_chunk.size
meta['full_path'] = self.full_path
meta['oio_version'] = OIO_VERSION
self.blob_client.chunk_put(spare_url[0], meta, GeneratorIO(stream))
# Register the spare chunk in object's metadata
if chunk_id is None:
self._add_raw_chunk(current_chunk,
spare_url[0],
frozen=allow_frozen_container)
else:
self._update_spare_chunk(current_chunk,
spare_url[0],
frozen=allow_frozen_container)
self.logger.debug('Chunk %s repaired in %s', chunk_id or chunk_pos,
spare_url[0])
def rebuild_chunk(self,
chunk_id,
service_id=None,
allow_same_rawx=False,
chunk_pos=None,
allow_frozen_container=False):
# Identify the chunk to rebuild
candidates = self.chunks.filter(id=chunk_id)
if service_id is not None:
candidates = candidates.filter(host=service_id)
current_chunk = candidates.one()
if current_chunk is None and chunk_pos is None:
raise OrphanChunk("Chunk not found in content")
if current_chunk is None:
current_chunk = self.chunks.filter(pos=chunk_pos).one()
if current_chunk is None:
chunk = {'pos': chunk_pos, 'url': ''}
current_chunk = Chunk(chunk)
else:
chunk_id = current_chunk.id
self.logger.debug('Chunk at pos %s has id %s', chunk_pos,
chunk_id)
# Sort chunks by score to try to rebuild with higher score.
# When scores are close together (e.g. [95, 94, 94, 93, 50]),
# don't always start with the highest element.
chunks = self.chunks \
.filter(metapos=current_chunk.metapos) \
.exclude(id=chunk_id, pos=chunk_pos) \
.sort(key=lambda chunk: _get_weighted_random_score(chunk.raw()),
reverse=True)
if chunk_id is None:
current_chunk.size = chunks[0].size
current_chunk.checksum = chunks[0].checksum
# Find a spare chunk address
broken_list = list()
if not allow_same_rawx and chunk_id is not None:
broken_list.append(current_chunk)
spare_url, _quals = self._get_spare_chunk(chunks.all(),
broken_list,
position=current_chunk.pos)
new_chunk = Chunk({'pos': current_chunk.pos, 'url': spare_url[0]})
# Regenerate the lost chunk's data, from existing chunks
handler = ECRebuildHandler(chunks.raw(), current_chunk.subpos,
self.storage_method)
expected_chunk_size, stream = handler.rebuild()
# Actually create the spare chunk
meta = {}
meta['chunk_id'] = new_chunk.id
meta['chunk_pos'] = current_chunk.pos
meta['container_id'] = self.container_id
# FIXME: should be 'content_chunkmethod' everywhere
# but sadly it isn't
meta['chunk_method'] = self.chunk_method
# FIXME: should be 'content_id' everywhere
# but sadly it isn't
meta['id'] = self.content_id
meta['content_path'] = self.path
# FIXME: should be 'content_policy' everywhere
# but sadly it isn't
meta['policy'] = self.policy
# FIXME: should be 'content_version' everywhere
# but sadly it isn't
meta['version'] = self.version
meta['metachunk_hash'] = current_chunk.checksum
meta['metachunk_size'] = current_chunk.size
meta['full_path'] = self.full_path
meta['oio_version'] = OIO_VERSION
bytes_transferred, _ = self.blob_client.chunk_put(
spare_url[0], meta, GeneratorIO(stream, sub_generator=PY2))
if expected_chunk_size is not None \
and bytes_transferred != expected_chunk_size:
try:
self.blob_client.chunk_delete(spare_url[0])
except Exception as exc:
self.logger.warning(
'Failed to rollback the rebuild of the chunk: %s', exc)
raise ChunkException('The rebuilt chunk is not the correct size')
# Register the spare chunk in object's metadata
if chunk_id is None:
self._add_raw_chunk(current_chunk,
spare_url[0],
frozen=allow_frozen_container)
else:
self._update_spare_chunk(current_chunk,
spare_url[0],
frozen=allow_frozen_container)
self.logger.debug('Chunk %s repaired in %s', chunk_id or chunk_pos,
spare_url[0])
return bytes_transferred
