def get_stream(self):
range_infos = self._get_range_infos()
chunk_iter = iter(self.chunks)
# we use eventlet GreenPool to manage readers
with green.ContextPool(self.storage_method.ec_nb_data) as pool:
pile = GreenPile(pool)
# we use eventlet GreenPile to spawn readers
for _j in range(self.storage_method.ec_nb_data):
pile.spawn(self._get_fragment, chunk_iter, range_infos,
self.storage_method)
readers = []
for reader, parts_iter in pile:
if reader.status in (200, 206):
readers.append((reader, parts_iter))
# TODO log failures?
# with EC we need at least ec_nb_data valid readers
if len(readers) >= self.storage_method.ec_nb_data:
# all readers should return the same Content-Length
# so just take the headers from one of them
resp_headers = HeadersDict(readers[0][0].headers)
fragment_length = int(resp_headers.get('Content-Length'))
read_iterators = [it for _, it in readers]
stream = ECStream(self.storage_method, read_iterators, range_infos,
self.meta_length, fragment_length)
# start the stream
stream.start()
return stream
else:
raise exceptions.ServiceUnavailable(
'Not enough valid sources to read (%d/%d)' % (
len(readers), self.storage_method.ec_nb_data))
def fetch_stream(chunks, ranges, storage_method, headers=None, **kwargs):
ranges = ranges or [(None, None)]
meta_range_list = get_meta_ranges(ranges, chunks)
for meta_range_dict in meta_range_list:
for pos in sorted(meta_range_dict.keys()):
meta_start, meta_end = meta_range_dict[pos]
if meta_start is not None and meta_end is not None:
headers['Range'] = http_header_from_ranges(
(meta_range_dict[pos], ))
reader = ChunkReader(iter(chunks[pos]),
READ_CHUNK_SIZE,
headers=headers,
**kwargs)
try:
it = reader.get_iter()
except exc.NotFound as err:
raise exc.UnrecoverableContent(
"Cannot download position %d: %s" % (pos, err))
except Exception as err:
raise exc.ServiceUnavailable(
"Error while downloading position %d: %s" % (pos, err))
for part in it:
for dat in part['iter']:
yield dat
def get_iter(self):
source, chunk = self._get_source()
if source:
return self._get_iter(chunk, source)
errors = group_chunk_errors(self._resp_by_chunk.items())
if len(errors) == 1:
# All errors are of the same type, group them
status, chunks = errors.popitem()
raise exc.from_status(status[0], "%s %s" % (status[1], chunks))
raise exc.ServiceUnavailable("unavailable chunks: %s" %
self._resp_by_chunk)
def test_GET_service_unavailable(self):
req = Request.blank('/v1/a/c/o')
ret_value = ({
'hash': 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
'mtime': 0,
'length': 10,
'deleted': False,
'version': 42,
}, fake_stream(10, exc.ServiceUnavailable('missing chunks')))
self.storage.object_fetch = Mock(return_value=ret_value)
resp = req.get_response(self.app)
# Everything seems ok,
self.assertEqual(resp.status_int, 200)
# but an exception is raised when trying to read response data.
try:
for _ in resp.app_iter:
pass
except swob.HTTPException as err:
self.assertEqual(503, err.status_int)
def fetch_stream_ec(chunks, ranges, storage_method, **kwargs):
ranges = ranges or [(None, None)]
meta_range_list = get_meta_ranges(ranges, chunks)
for meta_range_dict in meta_range_list:
for pos in sorted(meta_range_dict.keys()):
meta_start, meta_end = meta_range_dict[pos]
handler = ECChunkDownloadHandler(storage_method, chunks[pos],
meta_start, meta_end, **kwargs)
try:
stream = handler.get_stream()
except exc.NotFound as err:
raise exc.UnrecoverableContent(
"Cannot download position %d: %s" % (pos, err))
except Exception as err:
raise exc.ServiceUnavailable(
"Error while downloading position %d: %s" % (pos, err))
try:
for part_info in stream:
for dat in part_info['iter']:
yield dat
finally:
# This must be done in a finally block to handle the case
# when the reader does not read until the end of the stream.
stream.close()
