def rebuild_chunk(self, chunk_id):
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None:
raise exc.OrphanChunk("Chunk not found in content")
duplicate_chunks = self.chunks.filter(pos=current_chunk.pos).exclude(
id=chunk_id).all()
if len(duplicate_chunks) == 0:
raise UnrecoverableContent("No copy of missing chunk")
spare_urls = self._get_spare_chunk(duplicate_chunks, [current_chunk])
uploaded = False
for src in duplicate_chunks:
try:
self.blob_client.chunk_copy(src.url, spare_urls[0])
self.logger.debug("copy chunk from %s to %s", src.url,
spare_urls[0])
uploaded = True
break
except Exception as e:
self.logger.warn("Failed to copy chunk from %s to %s: %s",
src.url, spare_urls[0], str(e.message))
if not uploaded:
raise UnrecoverableContent("No copy available of missing chunk")
self._update_spare_chunk(current_chunk, spare_urls[0])
def rebuild_chunk(self, chunk_id, allow_same_rawx=False, chunk_pos=None):
# Identify the chunk to rebuild
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None and chunk_pos is None:
raise exc.OrphanChunk("Chunk not found in content")
elif chunk_pos is None:
chunk_pos = current_chunk.pos
duplicate_chunks = self.chunks.filter(pos=chunk_pos).exclude(
id=chunk_id).all()
if len(duplicate_chunks) == 0:
raise UnrecoverableContent("No copy of missing chunk")
if current_chunk is None:
chunk = {}
chunk['hash'] = duplicate_chunks[0].checksum
chunk['size'] = duplicate_chunks[0].size
chunk['url'] = ''
chunk['pos'] = chunk_pos
current_chunk = Chunk(chunk)
# Find a spare chunk address
broken_list = list()
if not allow_same_rawx and chunk_id is not None:
broken_list.append(current_chunk)
spare_urls, _quals = self._get_spare_chunk(duplicate_chunks,
broken_list)
spare_url = spare_urls[0]
# Actually create the spare chunk, by duplicating a good one
uploaded = False
for src in duplicate_chunks:
try:
self.blob_client.chunk_copy(src.url,
spare_url,
chunk_id=chunk_id,
fullpath=self.full_path,
cid=self.container_id,
path=self.path,
version=self.version,
content_id=self.content_id)
self.logger.debug('Chunk copied from %s to %s, registering it',
src.url, spare_url)
uploaded = True
break
except Exception as err:
self.logger.warn("Failed to copy chunk from %s to %s: %s %s",
src.url, spare_url, type(err),
str(err.message))
if not uploaded:
raise UnrecoverableContent("No copy available of missing chunk")
# Register the spare chunk in object's metadata
if chunk_id is None:
self._add_raw_chunk(current_chunk, spare_url)
else:
self._update_spare_chunk(current_chunk, spare_url)
self.logger.debug('Chunk %s repaired in %s', chunk_id or chunk_pos,
spare_url)
def rebuild_chunk(self, chunk_id, allow_same_rawx=False, chunk_pos=None):
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None and chunk_pos is None:
raise exc.OrphanChunk("Chunk not found in content")
elif chunk_pos is None:
chunk_pos = current_chunk.pos
duplicate_chunks = self.chunks.filter(
pos=chunk_pos).exclude(id=chunk_id).all()
if len(duplicate_chunks) == 0:
raise UnrecoverableContent("No copy of missing chunk")
if current_chunk is None:
chunk = {}
chunk['hash'] = duplicate_chunks[0].checksum
chunk['size'] = duplicate_chunks[0].size
chunk['url'] = ''
chunk['pos'] = chunk_pos
current_chunk = Chunk(chunk)
broken_list = list()
if not allow_same_rawx and chunk_id is not None:
broken_list.append(current_chunk)
spare_urls = self._get_spare_chunk(
duplicate_chunks, broken_list)
uploaded = False
for src in duplicate_chunks:
try:
self.blob_client.chunk_copy(src.url, spare_urls[0])
self.logger.debug('Chunk copied from %s to %s, registering it',
src.url, spare_urls[0])
uploaded = True
break
except Exception as err:
self.logger.warn(
"Failed to copy chunk from %s to %s: %s", src.url,
spare_urls[0], str(err.message))
if not uploaded:
raise UnrecoverableContent("No copy available of missing chunk")
if chunk_id is None:
self._add_raw_chunk(current_chunk, spare_urls[0])
else:
self._update_spare_chunk(current_chunk, spare_urls[0])
self.logger.info('Chunk %s repaired in %s',
chunk_id or chunk_pos, spare_urls[0])
def _fetch_stream(self, chunks, storage_method, headers):
meta_ranges = get_meta_ranges([(None, None)], chunks)
for pos, meta_range in meta_ranges.iteritems():
meta_start, meta_end = meta_range
reader = io.ChunkReader(iter(chunks[pos]), io.READ_CHUNK_SIZE,
headers)
it = reader.get_iter()
if not it:
raise UnrecoverableContent("Error while downloading")
for part in it:
for d in part['iter']:
yield d
def _fetch_stream(self, chunks, storage_method, headers):
meta_range_list = get_meta_ranges([(None, None)], chunks)
for meta_range_dict in meta_range_list:
for pos, meta_range in meta_range_dict.iteritems():
meta_start, meta_end = meta_range
reader = io.ChunkReader(iter(chunks[pos]), io.READ_CHUNK_SIZE,
headers)
try:
it = reader.get_iter()
except Exception as err:
raise UnrecoverableContent("Error while downloading: %s" %
err)
for part in it:
for d in part['iter']:
yield d
def _download_chunk(self, pos):
stream = None
for c in self.chunks.filter(pos=pos):
try:
meta, stream = self.blob_client.chunk_get(c.url)
break
except NotFound:
self.logger.debug("Chunk %s not found" % c.url)
continue
if stream is None:
raise UnrecoverableContent("No chunk found at pos %d" % pos)
for data in stream:
yield data
def move_chunk(self,
chunk_id,
service_id=None,
check_quality=False,
dry_run=False,
max_attempts=3,
**kwargs):
"""
Move a chunk to another place. Optionally ensure that the
new place is an improvement over the current one.
"""
if isinstance(chunk_id, Chunk):
current_chunk = chunk_id
chunk_id = current_chunk.id
service_id = current_chunk.host
else:
candidates = self.chunks.filter(id=chunk_id)
if len(candidates) > 1:
if service_id is None:
raise exc.ChunkException(
"Several chunks with ID %s and no service ID" %
(chunk_id, ))
candidates = candidates.filter(host=service_id)
current_chunk = candidates.one()
if current_chunk is None or current_chunk not in self.chunks:
raise exc.OrphanChunk("Chunk not found in content")
if service_id:
other_chunks = self.chunks.filter(
metapos=current_chunk.metapos).exclude(host=service_id).all()
else:
other_chunks = self.chunks.filter(
metapos=current_chunk.metapos).exclude(id=chunk_id).all()
spare_urls, qualities = self._get_spare_chunk(
other_chunks, [current_chunk],
position=current_chunk.pos,
check_quality=check_quality,
max_attempts=max_attempts,
**kwargs)
# Sort chunks by score to try to copy with higher score.
# When scores are close together (e.g. [95, 94, 94, 93, 50]),
# don't always start with the highest element.
duplicate_chunks = self.chunks \
.filter(pos=current_chunk.pos) \
.sort(key=lambda chunk: _get_weighted_random_score(chunk.raw()),
reverse=True) \
.all()
if dry_run:
self.logger.info('Dry-run: would copy chunk from %s to %s',
duplicate_chunks[0].url, spare_urls[0])
else:
# To reduce the load on the rawx to decommission,
# use one of the rawx with a copy of the chunk to move.
for src in duplicate_chunks:
try:
self.logger.info('Copying chunk from %s to %s', src.url,
spare_urls[0])
# TODO(FVE): retry to copy (max_attempts times)
self.blob_client.chunk_copy(src.url,
spare_urls[0],
chunk_id=chunk_id,
fullpath=self.full_path,
cid=self.container_id,
path=self.path,
version=self.version,
content_id=self.content_id,
**kwargs)
break
except Exception as err:
self.logger.warn('Failed to copy chunk from %s to %s: %s',
src.url, spare_urls[0], err)
if len(duplicate_chunks) == 1:
raise
else:
raise UnrecoverableContent(
'No copy available of chunk to move')
self._update_spare_chunk(current_chunk, spare_urls[0])
try:
self.blob_client.chunk_delete(current_chunk.url, **kwargs)
except Exception as err:
self.logger.warn("Failed to delete chunk %s: %s",
current_chunk.url, err)
current_chunk.url = spare_urls[0]
current_chunk.quality = qualities[current_chunk.url]
return current_chunk.raw()
def rebuild_metachunk(self, metapos, force_broken_chunk=None,
on_the_fly=False):
def _encode_sparerawxlist(broken_chunks, spare_urls):
res = []
for i, bc in enumerate(broken_chunks):
if bc.is_parity:
broken_idx = self.k + int(bc.paritypos)
else:
broken_idx = int(bc.subpos)
spare_url = spare_urls[i].split('/', 2)[2] # remove http//
res.append("%s|%d|%s" % (spare_url, broken_idx, bc.hash))
return ';'.join(res)
current_chunks = self.chunks.filter(metapos=metapos)
broken_chunks = []
notin_chunks = []
for c in current_chunks:
if force_broken_chunk is not None \
and force_broken_chunk.id == c.id:
broken_chunks.append(c)
continue
try:
self.blob_client.chunk_head(c.url)
except Exception as e:
self.logger.debug("Failed to download chunk %s: %s"
% (c.url, e.message))
broken_chunks.append(c)
continue
notin_chunks.append(c)
if len(broken_chunks) > self.m:
raise UnrecoverableContent(
"Not enough chunks to rebuild the metachunk")
spare_urls = self._meta2_get_spare_chunk(notin_chunks, broken_chunks)
headers = {}
headers["X-oio-chunk-meta-content-storage-policy"] = self.stgpol_name
headers["X-oio-chunk-meta-rawxlist"] = \
self._encode_rawxlist(current_chunks)
headers["X-oio-chunk-meta-sparerawxlist"] = \
_encode_sparerawxlist(broken_chunks, spare_urls)
headers[chunk_headers["content_id"]] = self.content_id
headers[chunk_headers["content_version"]] = self.version
headers[chunk_headers["content_path"]] = self.path
headers[chunk_headers["content_size"]] = self.length
headers[chunk_headers["content_chunksnb"]] = \
self._get_metachunk_nb()
headers[chunk_headers["content_cid"]] = self.container_id
headers[chunk_headers["chunk_pos"]] = metapos
headers["X-oio-chunk-meta-chunk-size"] = \
self._get_metachunk_size(metapos)
headers[chunk_headers["content_mimetype"]] = self.mime_type
headers[chunk_headers["content_chunkmethod"]] = self.chunk_method
resp = self.session.get(self._get_rain_addr(on_the_fly),
headers=headers, stream=True)
resp.raise_for_status()
if on_the_fly:
return resp.iter_content(READ_CHUNK_SIZE)
resp.close()
for i, bc in enumerate(broken_chunks):
# TODO send only one request with all chunks modifications
self._meta2_update_spare_chunk(bc, spare_urls[i])
bc.url = spare_urls[i] # update current content
def rebuild_chunk(self,
chunk_id,
allow_same_rawx=False,
chunk_pos=None,
allow_frozen_container=False):
# Identify the chunk to rebuild
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None and chunk_pos is None:
raise exc.OrphanChunk("Chunk not found in content")
elif chunk_pos is None:
chunk_pos = current_chunk.pos
# Sort chunks by score to try to copy with higher score.
# When scores are close together (e.g. [95, 94, 94, 93, 50]),
# don't always start with the highest element.
duplicate_chunks = self.chunks \
.filter(pos=chunk_pos) \
.exclude(id=chunk_id) \
.sort(key=lambda chunk: _get_weighted_random_score(chunk.raw()),
reverse=True) \
.all()
if len(duplicate_chunks) == 0:
raise UnrecoverableContent("No copy of missing chunk")
if current_chunk is None:
chunk = {}
chunk['hash'] = duplicate_chunks[0].checksum
chunk['size'] = duplicate_chunks[0].size
chunk['url'] = ''
chunk['pos'] = chunk_pos
current_chunk = Chunk(chunk)
# Find a spare chunk address
broken_list = list()
if not allow_same_rawx and chunk_id is not None:
broken_list.append(current_chunk)
spare_urls, _quals = self._get_spare_chunk(duplicate_chunks,
broken_list)
spare_url = spare_urls[0]
# Actually create the spare chunk, by duplicating a good one
for src in duplicate_chunks:
try:
self.blob_client.chunk_copy(src.url,
spare_url,
chunk_id=chunk_id,
fullpath=self.full_path,
cid=self.container_id,
path=self.path,
version=self.version,
content_id=self.content_id)
self.logger.debug('Chunk copied from %s to %s, registering it',
src.url, spare_url)
break
except Exception as err:
self.logger.warn("Failed to copy chunk from %s to %s: %s %s",
src.url, spare_url, type(err), err)
else:
raise UnrecoverableContent("No copy available of missing chunk")
# Register the spare chunk in object's metadata
if chunk_id is None:
self._add_raw_chunk(current_chunk,
spare_url,
frozen=allow_frozen_container)
else:
self._update_spare_chunk(current_chunk,
spare_url,
frozen=allow_frozen_container)
self.logger.debug('Chunk %s repaired in %s', chunk_id or chunk_pos,
spare_url)
return current_chunk.size