def get_stream(self):
range_infos = self._get_range_infos()
chunk_iter = iter(self.chunks)
# we use eventlet GreenPool to manage readers
with green.ContextPool(self.storage_method.ec_nb_data) as pool:
pile = GreenPile(pool)
# we use eventlet GreenPile to spawn readers
for _j in range(self.storage_method.ec_nb_data):
pile.spawn(self._get_fragment, chunk_iter, range_infos,
self.storage_method)
readers = []
for reader, parts_iter in pile:
if reader.status in (200, 206):
readers.append((reader, parts_iter))
# TODO log failures?
# with EC we need at least ec_nb_data valid readers
if len(readers) >= self.storage_method.ec_nb_data:
# all readers should return the same Content-Length
# so just take the headers from one of them
resp_headers = HeadersDict(readers[0][0].headers)
fragment_length = int(resp_headers.get('Content-Length'))
read_iterators = [it for _, it in readers]
stream = ECStream(self.storage_method, read_iterators, range_infos,
self.meta_length, fragment_length)
# start the stream
stream.start()
return stream
else:
raise exceptions.ServiceUnavailable(
'Not enough valid sources to read (%d/%d)' % (
len(readers), self.storage_method.ec_nb_data))
def get_stream(self):
range_infos = self._get_range_infos()
chunk_iter = iter(self.chunks)
# we use eventlet GreenPool to manage readers
with utils.ContextPool(self.storage_method.ec_nb_data) as pool:
pile = GreenPile(pool)
# we use eventlet GreenPile to spawn readers
for _j in range(self.storage_method.ec_nb_data):
pile.spawn(self._get_fragment, chunk_iter, self.storage_method)
readers = []
for reader, parts_iter in pile:
if reader.status in (200, 206):
readers.append((reader, parts_iter))
# TODO log failures?
# with EC we need at least ec_nb_data valid readers
if len(readers) >= self.storage_method.ec_nb_data:
# all readers should return the same Content-Length
# so just take the headers from one of them
resp_headers = HeadersDict(readers[0][0].headers)
fragment_length = int(resp_headers.get('Content-Length'))
r = [it for reader, it in readers]
stream = ECStream(self.storage_method, r, range_infos,
self.meta_length, fragment_length)
# start the stream
stream.start()
return stream
else:
raise exc.OioException("Not enough valid sources to read")
def get_stream(self):
range_infos = self._get_range_infos()
# the meta chunk length
# (the amount of actual data stored into the meta chunk)
meta_length = self.chunks[0]['size']
chunk_iter = iter(self.chunks)
# we use eventlet GreenPool to manage readers
with utils.ContextPool(self.storage_method.ec_nb_data) as pool:
pile = GreenPile(pool)
# we use eventlet GreenPile to spawn readers
for _j in range(self.storage_method.ec_nb_data):
pile.spawn(self._get_fragment, chunk_iter, self.storage_method)
readers = []
for reader, parts_iter in pile:
if reader.status in (200, 206):
readers.append((reader, parts_iter))
# TODO log failures?
# with EC we need at least ec_nb_data valid readers
if len(readers) >= self.storage_method.ec_nb_data:
# all readers should return the same Content-Length
# so just take the headers from one of them
resp_headers = HeadersDict(readers[0][0].headers)
fragment_length = int(resp_headers.get('Content-Length'))
r = [it for reader, it in readers]
stream = ECStream(self.storage_method, r, range_infos, meta_length,
fragment_length)
# start the stream
stream.start()
return stream
else:
raise exc.OioException("Not enough valid sources to read")
class FakeResponse(object):
def __init__(self, status, body='', headers=None, slow=0):
self.status = status
self.body = body
self.headers = HeadersDict(headers)
self.stream = BytesIO(body)
self.slow = slow
def getheader(self, name, default=None):
return self.headers.get(name, default)
def getheaders(self):
if 'Content-Length' not in self.headers:
self.headers['Content-Length'] = len(self.body)
return self.headers.items()
def _slow(self):
sleep(self.slow)
def read(self, amt=0):
if self.slow:
self._slow()
return self.stream.read(amt)
def __repr__(self):
return 'FakeResponse(status=%s)' % self.status
def reason(self):
return str(self.status)
class FakeResponse(object):
def __init__(self, status, body='', headers=None, slow=0):
self.status = status
self.body = body
self.headers = HeadersDict(headers)
self.stream = BytesIO(body)
self.slow = slow
def getheader(self, name, default=None):
return self.headers.get(name, default)
def getheaders(self):
if 'Content-Length' not in self.headers:
self.headers['Content-Length'] = len(self.body)
return self.headers.items()
def _slow(self):
sleep(self.slow)
def read(self, amt=0):
if self.slow:
self._slow()
return self.stream.read(amt)
def __repr__(self):
return 'FakeResponse(status=%s)' % self.status
