def fetch_stream(chunks, ranges, storage_method, headers=None, **kwargs): ranges = ranges or [(None, None)] meta_range_list = get_meta_ranges(ranges, chunks) for meta_range_dict in meta_range_list: for pos in sorted(meta_range_dict.keys()): meta_start, meta_end = meta_range_dict[pos] if meta_start is not None and meta_end is not None: headers['Range'] = http_header_from_ranges( (meta_range_dict[pos], )) reader = ChunkReader(iter(chunks[pos]), READ_CHUNK_SIZE, headers=headers, **kwargs) try: it = reader.get_iter() except exc.NotFound as err: raise exc.UnrecoverableContent( "Cannot download position %d: %s" % (pos, err)) except Exception as err: raise exc.ServiceUnavailable( "Error while downloading position %d: %s" % (pos, err)) for part in it: for dat in part['iter']: yield dat
def rebuild(self): pile = GreenPile(len(self.meta_chunk)) nb_data = self.storage_method.ec_nb_data headers = {} for chunk in self.meta_chunk: pile.spawn(self._get_response, chunk, headers) # Sort all responses according to the chunk size total_resps = 0 resps_by_size = dict() resps_without_chunk_size = list() for resp in pile: if not resp: continue chunk_size = int_value( resp.getheader(CHUNK_HEADERS['chunk_size'], None), None) if chunk_size is None: self.logger.warning('Missing chunk size') resps_without_chunk_size.append(resp) continue total_resps += 1 resps_by_size.setdefault(chunk_size, list()).append(resp) # Select the chunk with the majority chunk size resps = None max_resps = 0 assumed_chunk_size = None for chunk_size, resps in resps_by_size.items(): nb_resp = len(resps) if nb_resp > max_resps: max_resps = nb_resp assumed_chunk_size = chunk_size if assumed_chunk_size is None: self.logger.warning( 'No chunk available with chunk size information') resps = list() else: resps = resps_by_size[assumed_chunk_size] if max_resps != total_resps: self.logger.warning( '%d/%d chunks are not the same size as others (%d), ' 'they should be removed', total_resps - max_resps, total_resps, assumed_chunk_size) # Check the number of chunks available if max_resps < nb_data: # Add the chunks without size information # assuming they are the correct size resps = resps + resps_without_chunk_size if len(resps) < nb_data: self.logger.error( 'Unable to read enough valid sources to rebuild') raise exceptions.UnrecoverableContent( 'Not enough valid sources to rebuild') self.logger.warning( 'Use chunk(s) without size information to rebuild a chunk') rebuild_iter = self._make_rebuild_iter(resps[:nb_data]) return assumed_chunk_size, rebuild_iter
def rebuild(self): pile = GreenPile(len(self.meta_chunk)) nb_data = self.storage_method.ec_nb_data headers = {} for chunk in self.meta_chunk: pile.spawn(self._get_response, chunk, headers) resps = [] for resp in pile: if not resp: continue resps.append(resp) if len(resps) >= self.storage_method.ec_nb_data: break else: logger.error('Unable to read enough valid sources to rebuild') raise exc.UnrecoverableContent('Unable to rebuild chunk') rebuild_iter = self._make_rebuild_iter(resps[:nb_data]) return rebuild_iter
def fetch_stream_ec(chunks, ranges, storage_method, **kwargs): ranges = ranges or [(None, None)] meta_range_list = get_meta_ranges(ranges, chunks) for meta_range_dict in meta_range_list: for pos in sorted(meta_range_dict.keys()): meta_start, meta_end = meta_range_dict[pos] handler = ECChunkDownloadHandler(storage_method, chunks[pos], meta_start, meta_end, **kwargs) try: stream = handler.get_stream() except exc.NotFound as err: raise exc.UnrecoverableContent( "Cannot download position %d: %s" % (pos, err)) except Exception as err: raise exc.ServiceUnavailable( "Error while downloading position %d: %s" % (pos, err)) try: for part_info in stream: for dat in part_info['iter']: yield dat finally: # This must be done in a finally block to handle the case # when the reader does not read until the end of the stream. stream.close()