def _get_spare_chunk(self,
chunks_notin,
chunks_broken,
position,
max_attempts=3,
check_quality=False,
fake_excluded_chunks=None,
**kwargs):
notin = ChunksHelper(chunks_notin, False).raw()
broken = ChunksHelper(chunks_broken, False).raw()
if fake_excluded_chunks:
for fake_excluded_chunk in fake_excluded_chunks:
chunk = fake_excluded_chunk.copy()
chunk['hash'] = broken[0]['hash']
chunk['pos'] = broken[0]['pos']
chunk['size'] = broken[0]['size']
broken.append(chunk)
spare_data = {"notin": notin, "broken": broken}
last_exc = None
bal = 0
for attempt in range(max_attempts):
try:
spare_resp = self.container_client.content_spare(
cid=self.container_id,
path=self.path,
version=self.version,
data=spare_data,
stgpol=self.policy,
position=position,
**kwargs)
quals = extract_chunk_qualities(spare_resp.get(
'properties', {}),
raw=True)
if check_quality:
bal = ensure_better_chunk_qualities(chunks_broken, quals)
break
except (exc.ClientException, exc.SpareChunkException) as err:
self.logger.info(
"Failed to find spare chunk (attempt %d/%d): %s",
attempt + 1, max_attempts, err)
last_exc = err
# TODO(FVE): exponential backoff?
else:
if isinstance(last_exc, exc.SpareChunkException):
exc.reraise(exc.SpareChunkException, last_exc)
raise exc.SpareChunkException("No spare chunk: %s" % str(last_exc))
url_list = []
for chunk in spare_resp["chunks"]:
url_list.append(chunk["id"])
if check_quality:
self.logger.info(
"Found %d spare chunks, that will improve "
"metachunk quality by %d", len(url_list), bal)
return url_list, quals
def ensure_better_chunk_qualities(current_chunks, candidates, threshold=1):
"""
Ensure that the set of spare chunks is really an improvement over
the set of current chunks, raise SpareChunkException if it is not.
"""
balance = 0
for current, candidate in zip(current_chunks, candidates.keys()):
balance += compare_chunk_quality(current.quality,
candidates[candidate])
if balance < threshold:
raise exc.SpareChunkException(
"the spare chunks found would not improve the quality "
"(balance=%d, threshold=%d)" % (balance, threshold))
return balance
def _get_spare_chunk(self, chunks_notin, chunks_broken):
spare_data = {
"notin": ChunksHelper(chunks_notin, False).raw(),
"broken": ChunksHelper(chunks_broken, False).raw()
}
try:
spare_resp = self.container_client.content_spare(
cid=self.container_id, path=self.content_id,
data=spare_data, stgpol=self.policy)
except ClientException as e:
raise exc.SpareChunkException("No spare chunk (%s)" % e.message)
url_list = []
for c in spare_resp["chunks"]:
url_list.append(c["id"])
return url_list
