def fetch_stream(chunks, ranges, storage_method, headers=None, **kwargs):
ranges = ranges or [(None, None)]
meta_range_list = get_meta_ranges(ranges, chunks)
for meta_range_dict in meta_range_list:
for pos in sorted(meta_range_dict.keys()):
meta_start, meta_end = meta_range_dict[pos]
if meta_start is not None and meta_end is not None:
headers['Range'] = http_header_from_ranges(
(meta_range_dict[pos], ))
reader = ChunkReader(iter(chunks[pos]),
READ_CHUNK_SIZE,
headers=headers,
**kwargs)
try:
it = reader.get_iter()
except exc.NotFound as err:
raise exc.UnrecoverableContent(
"Cannot download position %d: %s" % (pos, err))
except Exception as err:
raise exc.ServiceUnavailable(
"Error while downloading position %d: %s" % (pos, err))
for part in it:
for dat in part['iter']:
yield dat
def fill_ranges(self, start, end, length):
"""
Fill the request ranges.
"""
if length == 0:
return
if self.align and self.buf_size:
# discard bytes
# so we only yield complete EC segments
self.discard_bytes = discard_bytes(self.buf_size, start)
# change headers for efficient recovery
if 'Range' in self.request_headers:
try:
orig_ranges = ranges_from_http_header(
self.request_headers['Range'])
new_ranges = [(start, end)] + orig_ranges[1:]
except ValueError:
new_ranges = [(start, end)]
else:
new_ranges = [(start, end)]
self.request_headers['Range'] = http_header_from_ranges(
new_ranges)
def _fetch_stream(self, meta, chunks, ranges, storage_method, headers):
total_bytes = 0
headers = headers or {}
ranges = ranges or [(None, None)]
meta_range_list = get_meta_ranges(ranges, chunks)
for meta_range_dict in meta_range_list:
for pos, meta_range in meta_range_dict.iteritems():
meta_start, meta_end = meta_range
if meta_start is not None and meta_end is not None:
headers['Range'] = http_header_from_ranges([meta_range])
reader = io.ChunkReader(
iter(chunks[pos]),
io.READ_CHUNK_SIZE,
headers,
connection_timeout=self.connection_timeout,
response_timeout=self.read_timeout,
read_timeout=self.read_timeout)
try:
it = reader.get_iter()
except Exception as err:
raise exc.OioException(
"Error while downloading position %d: %s" % (pos, err))
for part in it:
for d in part['iter']:
total_bytes += len(d)
yield d
def recover(self, nb_bytes):
"""
Recover the request.
:param nb_bytes: number of bytes already consumed that we need to
discard if we perform a recovery from another source.
:raises `ValueError`: if range header is not valid
:raises `oio.common.exceptions.UnsatisfiableRange`:
:raises `oio.common.exceptions.EmptyByteRange`:
"""
if 'Range' in self.request_headers:
request_range = ranges_from_http_header(
self.request_headers['Range'])
start, end = request_range[0]
if start is None:
# suffix byte range
end -= nb_bytes
else:
start += nb_bytes
if end is not None:
if start == end + 1:
# no more bytes to serve in the requested byte range
raise exc.EmptyByteRange()
if start > end:
# invalid range
raise exc.UnsatisfiableRange()
if end and start:
# full byte range
request_range = [(start, end)] + request_range[1:]
else:
# suffix byte range
request_range = [(None, end)] + request_range[1:]
else:
# prefix byte range
request_range = [(start, None)] + request_range[1:]
self.request_headers['Range'] = http_header_from_ranges(
request_range)
else:
# just add an offset to the request
self.request_headers['Range'] = 'bytes=%d-' % nb_bytes
def _fetch_stream(self, meta, chunks, ranges, storage_method, headers):
total_bytes = 0
headers = headers or {}
ranges = ranges or [(None, None)]
meta_range_list = get_meta_ranges(ranges, chunks)
for meta_range_dict in meta_range_list:
for pos, meta_range in meta_range_dict.iteritems():
meta_start, meta_end = meta_range
if meta_start is not None and meta_end is not None:
headers["Range"] = http_header_from_ranges([meta_range])
reader = io.ChunkReader(iter(chunks[pos]), io.READ_CHUNK_SIZE, headers)
it = reader.get_iter()
if not it:
raise exc.OioException("Error while downloading")
for part in it:
for d in part["iter"]:
total_bytes += len(d)
yield d
def _fetch_stream(self, meta, chunks, ranges, storage_method, headers):
total_bytes = 0
headers = headers or {}
ranges = ranges or [(None, None)]
meta_range_list = get_meta_ranges(ranges, chunks)
for meta_range_dict in meta_range_list:
for pos, meta_range in meta_range_dict.iteritems():
meta_start, meta_end = meta_range
if meta_start is not None and meta_end is not None:
headers['Range'] = http_header_from_ranges([meta_range])
reader = io.ChunkReader(iter(chunks[pos]), io.READ_CHUNK_SIZE,
headers)
it = reader.get_iter()
if not it:
raise exc.OioException("Error while downloading")
for part in it:
for d in part['iter']:
total_bytes += len(d)
yield d
def recover(self, nb_bytes):
"""
Recover the request.
:params nb_bytes: number of bytes already consumed that we need to
discard if we perform a recovery from another source.
:raises ValueError: if range header is not valid
:raises UnsatisfiableRange
:raises EmptyByteRange
"""
if "Range" in self.request_headers:
request_range = ranges_from_http_header(self.request_headers["Range"])
start, end = request_range[0]
if start is None:
# suffix byte range
end -= nb_bytes
else:
start += nb_bytes
if end is not None:
if start == end + 1:
# no more bytes to serve in the requested byte range
raise exc.EmptyByteRange()
if start > end:
# invalid range
raise exc.UnsatisfiableRange()
if end and start:
# full byte range
request_range = [(start, end)] + request_range[1:]
else:
# suffix byte range
request_range = [(None, end)] + request_range[1:]
else:
# prefix byte range
request_range = [(start, None)] + request_range[1:]
self.request_headers["Range"] = http_header_from_ranges(request_range)
else:
# just add an offset to the request
self.request_headers["Range"] = "bytes=%d-" % nb_bytes
def fill_ranges(self, start, end, length):
"""
Fill the request ranges.
"""
if length == 0:
return
if self.buf_size:
# discard bytes
# so we only yield complete EC segments
self.discard_bytes = discard_bytes(self.buf_size, start)
# change headers for efficient recovery
if "Range" in self.request_headers:
try:
orig_ranges = ranges_from_http_header(self.request_headers["Range"])
new_ranges = [(start, end)] + orig_ranges[1:]
except ValueError:
new_ranges = [(start, end)]
else:
new_ranges = [(start, end)]
self.request_headers["Range"] = http_header_from_ranges(new_ranges)