def _get_spare_chunk(self, chunks_notin, chunks_broken, position, max_attempts=3, check_quality=False, fake_excluded_chunks=None, **kwargs): notin = ChunksHelper(chunks_notin, False).raw() broken = ChunksHelper(chunks_broken, False).raw() if fake_excluded_chunks: for fake_excluded_chunk in fake_excluded_chunks: chunk = fake_excluded_chunk.copy() chunk['hash'] = broken[0]['hash'] chunk['pos'] = broken[0]['pos'] chunk['size'] = broken[0]['size'] broken.append(chunk) spare_data = {"notin": notin, "broken": broken} last_exc = None bal = 0 for attempt in range(max_attempts): try: spare_resp = self.container_client.content_spare( cid=self.container_id, path=self.path, version=self.version, data=spare_data, stgpol=self.policy, position=position, **kwargs) quals = extract_chunk_qualities(spare_resp.get( 'properties', {}), raw=True) if check_quality: bal = ensure_better_chunk_qualities(chunks_broken, quals) break except (exc.ClientException, exc.SpareChunkException) as err: self.logger.info( "Failed to find spare chunk (attempt %d/%d): %s", attempt + 1, max_attempts, err) last_exc = err # TODO(FVE): exponential backoff? else: if isinstance(last_exc, exc.SpareChunkException): exc.reraise(exc.SpareChunkException, last_exc) raise exc.SpareChunkException("No spare chunk: %s" % str(last_exc)) url_list = [] for chunk in spare_resp["chunks"]: url_list.append(chunk["id"]) if check_quality: self.logger.info( "Found %d spare chunks, that will improve " "metachunk quality by %d", len(url_list), bal) return url_list, quals
def ensure_better_chunk_qualities(current_chunks, candidates, threshold=1): """ Ensure that the set of spare chunks is really an improvement over the set of current chunks, raise SpareChunkException if it is not. """ balance = 0 for current, candidate in zip(current_chunks, candidates.keys()): balance += compare_chunk_quality(current.quality, candidates[candidate]) if balance < threshold: raise exc.SpareChunkException( "the spare chunks found would not improve the quality " "(balance=%d, threshold=%d)" % (balance, threshold)) return balance
def _get_spare_chunk(self, chunks_notin, chunks_broken): spare_data = { "notin": ChunksHelper(chunks_notin, False).raw(), "broken": ChunksHelper(chunks_broken, False).raw() } try: spare_resp = self.container_client.content_spare( cid=self.container_id, path=self.content_id, data=spare_data, stgpol=self.policy) except ClientException as e: raise exc.SpareChunkException("No spare chunk (%s)" % e.message) url_list = [] for c in spare_resp["chunks"]: url_list.append(c["id"]) return url_list