def range_iter():
results = {}
while True:
next_range = self._next()
headers, fragment_iters = next_range
content_range = headers.get('Content-Range')
if content_range is not None:
fragment_start, fragment_end, fragment_length = \
parse_content_range(content_range)
elif self.fragment_length <= 0:
fragment_start = None
fragment_end = None
fragment_length = 0
else:
fragment_start = 0
fragment_end = self.fragment_length - 1
fragment_length = self.fragment_length
self._add_ranges_for_fragment(fragment_length, range_infos)
satisfiable = False
for range_info in range_infos:
satisfiable |= range_info['satisfiable']
k = (range_info['resp_fragment_start'],
range_info['resp_fragment_end'])
results.setdefault(k, []).append(range_info)
try:
range_info = results[(fragment_start, fragment_end)].pop(0)
except KeyError:
self.logger.error(
"Invalid range: %s, available: %s (reqid=%s)",
repr((fragment_start, fragment_end)), results.keys(),
self.reqid)
raise
if self.perfdata is not None:
ec_start = monotonic_time()
segment_iter = self._decode_segments(fragment_iters)
if self.perfdata is not None:
ec_end = monotonic_time()
rawx_perfdata = self.perfdata.setdefault('rawx', dict())
rawx_perfdata['ec'] = rawx_perfdata.get('ec', 0.0) \
+ ec_end - ec_start
if not range_info['satisfiable']:
io.consume(segment_iter)
continue
byterange_iter = self._iter_range(range_info, segment_iter)
result = {
'start': range_info['resp_meta_start'],
'end': range_info['resp_meta_end'],
'iter': byterange_iter
}
yield result
def connect(cls, chunk, sysmeta, reqid=None,
connection_timeout=None, write_timeout=None, **kwargs):
raw_url = chunk.get("real_url", chunk["url"])
parsed = urlparse(raw_url)
chunk_path = parsed.path.split('/')[-1]
hdrs = headers_from_object_metadata(sysmeta)
if reqid:
hdrs[REQID_HEADER] = reqid
hdrs[CHUNK_HEADERS["chunk_pos"]] = chunk["pos"]
hdrs[CHUNK_HEADERS["chunk_id"]] = chunk_path
# in the trailer
# metachunk_size & metachunk_hash
trailers = (CHUNK_HEADERS["metachunk_size"],
CHUNK_HEADERS["metachunk_hash"])
if kwargs.get('chunk_checksum_algo'):
trailers = trailers + (CHUNK_HEADERS["chunk_hash"], )
hdrs["Trailer"] = ', '.join(trailers)
with ConnectionTimeout(
connection_timeout or io.CONNECTION_TIMEOUT):
perfdata = kwargs.get('perfdata', None)
if perfdata is not None:
connect_start = monotonic_time()
conn = io.http_connect(
parsed.netloc, 'PUT', parsed.path, hdrs, scheme=parsed.scheme)
conn.set_cork(True)
if perfdata is not None:
connect_end = monotonic_time()
perfdata_rawx = perfdata.setdefault('rawx', dict())
perfdata_rawx['connect.' + chunk['url']] = \
connect_end - connect_start
conn.chunk = chunk
return cls(chunk, conn, write_timeout=write_timeout,
reqid=reqid, **kwargs)
def send(data):
self.checksum.update(data)
self.global_checksum.update(data)
# get the encoded fragments
if self.perfdata is not None:
ec_start = monotonic_time()
fragments = ec_stream.send(data)
if self.perfdata is not None:
ec_end = monotonic_time()
rawx_perfdata = self.perfdata.setdefault('rawx', dict())
rawx_perfdata['ec'] = rawx_perfdata.get('ec', 0.0) \
+ ec_end - ec_start
if fragments is None:
# not enough data given
return
current_writers = list(writers)
failed_chunks = list()
for writer in current_writers:
fragment = fragments[chunk_index[writer]]
if not writer.failed:
if writer.checksum:
writer.checksum.update(fragment)
writer.send(fragment)
else:
current_writers.remove(writer)
failed_chunks.append(writer.chunk)
sleep(0)
self.quorum_or_fail([w.chunk for w in current_writers],
failed_chunks)
def _send_data(self, conn):
"""
Send data to an open connection, taking data blocks from `conn.queue`.
"""
conn.upload_start = None
while True:
data = conn.queue.get()
if isinstance(data, text_type):
data = data.encode('utf-8')
if not conn.failed:
try:
with green.ChunkWriteTimeout(self.write_timeout):
if self.perfdata is not None \
and conn.upload_start is None:
conn.upload_start = monotonic_time()
conn.send(b'%x\r\n' % len(data))
conn.send(data)
conn.send(b'\r\n')
if not data:
if self.perfdata is not None:
fin_start = monotonic_time()
# Last segment sent, disable TCP_CORK to flush buffers
conn.set_cork(False)
if self.perfdata is not None:
fin_end = monotonic_time()
rawx_perfdata = self.perfdata.setdefault(
'rawx', dict())
chunk_url = conn.chunk['url']
rawx_perfdata['upload_finish.' + chunk_url] = \
fin_end - fin_start
green.eventlet_yield()
except (Exception, green.ChunkWriteTimeout) as err:
conn.failed = True
conn.chunk['error'] = str(err)
def test_deadline_to_timeout(self):
self.assertRaises(DeadlineReached, deadline_to_timeout,
monotonic_time() - 0.001, True)
self.assertRaises(DeadlineReached, deadline_to_timeout,
monotonic_time(), True)
deadline = monotonic_time() + 1.0
to = deadline_to_timeout(deadline, True)
self.assertLessEqual(to, 1.0)
self.assertGreater(to, 0.9)
def _update_rawx_perfdata(self, *args, **kwargs):
perfdata = kwargs.get('perfdata') or self.perfdata
if perfdata is not None:
req_start = utils.monotonic_time()
res = func(self, *args, **kwargs)
if perfdata is not None:
req_end = utils.monotonic_time()
val = perfdata.get('rawx', 0.0) + req_end - req_start
perfdata['rawx'] = val
return res
def _update_rawx_perfdata(self, *args, **kwargs):
perfdata = kwargs.get('perfdata') or self.perfdata
if perfdata is not None:
req_start = utils.monotonic_time()
res = func(self, *args, **kwargs)
if perfdata is not None:
req_end = utils.monotonic_time()
perfdata_rawx = perfdata.setdefault('rawx', dict())
overall = perfdata_rawx.get('overall', 0.0) + req_end - req_start
perfdata_rawx['overall'] = overall
return res
def test_object_list_empty_container(self):
"""
Ensure object listing of an empty container takes less than 35ms.
"""
container = self.__class__.__name__ + random_str(8)
self.api.container_create(self.account, container)
self.containers.add(container)
for _ in range(8):
start = monotonic_time()
self.api.object_list(self.account, container)
duration = monotonic_time() - start
logging.info("Object list took %.6fs", duration)
self.assertLess(duration, 0.035)
def _get_request(self, chunk):
"""
Connect to a chunk, fetch headers but don't read data.
Save the response object in `self.sources` list.
"""
try:
with green.ConnectionTimeout(self.connection_timeout):
raw_url = chunk.get("real_url", chunk["url"])
parsed = urlparse(raw_url)
if self.perfdata is not None:
connect_start = monotonic_time()
conn = http_connect(parsed.netloc,
'GET',
parsed.path,
self.request_headers,
scheme=parsed.scheme)
if self.perfdata is not None:
connect_end = monotonic_time()
rawx_perfdata = self.perfdata.setdefault('rawx', dict())
rawx_perfdata['connect.' + chunk['url']] = \
connect_end - connect_start
with green.OioTimeout(self.read_timeout):
source = conn.getresponse()
source.conn = conn
if self.perfdata is not None:
source.download_start = monotonic_time()
except (SocketError, Timeout) as err:
self.logger.error('Connection failed to %s (reqid=%s): %s', chunk,
self.reqid, err)
self._resp_by_chunk[chunk["url"]] = (0, text_type(err))
return False
except Exception as err:
self.logger.exception('Connection failed to %s (reqid=%s)', chunk,
self.reqid)
self._resp_by_chunk[chunk["url"]] = (0, text_type(err))
return False
if source.status in (200, 206):
self.status = source.status
self._headers = [(k.lower(), v) for k, v in source.getheaders()]
self.sources.append((source, chunk))
return True
else:
self.logger.warn("Invalid response from %s (reqid=%s): %d %s",
chunk, self.reqid, source.status, source.reason)
self._resp_by_chunk[chunk["url"]] = (source.status,
text_type(source.reason))
close_source(source, self.logger)
return False
def _get_response(self, conn):
"""
Wait for server response.
:returns: a tuple with `conn` and the reponse object or an exception.
"""
try:
with green.ChunkWriteTimeout(self.write_timeout):
resp = conn.getresponse()
if self.perfdata is not None:
upload_end = monotonic_time()
perfdata_rawx = self.perfdata.setdefault('rawx', dict())
url_chunk = conn.chunk['url']
perfdata_rawx[url_chunk] = \
perfdata_rawx.get(url_chunk, 0.0) \
+ upload_end - conn.upload_start
except Timeout as err:
resp = err
logger.error('Failed to read response from %s (reqid=%s): %s',
conn.chunk, self.reqid, err)
except Exception as err:
resp = err
logger.exception("Failed to read response from %s (reqid=%s)",
conn.chunk, self.reqid)
return (conn, resp)
def _send_data(self, conn):
"""
Send data to an open connection, taking data blocks from `conn.queue`.
"""
conn.upload_start = None
while True:
data = conn.queue.get()
if isinstance(data, text_type):
data = data.encode('utf-8')
if not conn.failed:
try:
with green.ChunkWriteTimeout(self.write_timeout):
if self.perfdata is not None \
and conn.upload_start is None:
conn.upload_start = monotonic_time()
conn.send(b'%x\r\n' % len(data))
conn.send(data)
conn.send(b'\r\n')
if not data:
# Last segment sent, disable TCP_CORK to flush buffers
conn.set_cork(False)
sleep(0)
except (Exception, green.ChunkWriteTimeout) as err:
conn.failed = True
conn.chunk['error'] = str(err)
def _send(self):
"""Send coroutine loop"""
self.conn.upload_start = None
while not self.failed:
# fetch input data from the queue
data = self.queue.get()
# use HTTP transfer encoding chunked
# to write data to RAWX
try:
with ChunkWriteTimeout(self.write_timeout):
if self.perfdata is not None \
and self.conn.upload_start is None:
self.conn.upload_start = monotonic_time()
self.conn.send("%x\r\n" % len(data))
self.conn.send(data)
self.conn.send("\r\n")
self.bytes_transferred += len(data)
eventlet_yield()
except (Exception, ChunkWriteTimeout) as exc:
self.failed = True
msg = str(exc)
self.logger.warn("Failed to write to %s (%s, reqid=%s)",
self.chunk, msg, self.reqid)
self.chunk['error'] = 'write: %s' % msg
# Drain the queue before quitting
while True:
try:
self.queue.get_nowait()
except Empty:
break
def test_deadline_from_read_timeout(self):
kwargs = dict()
set_deadline_from_read_timeout(kwargs)
# nothing changed
self.assertFalse(kwargs)
now = monotonic_time()
kwargs['read_timeout'] = 10.0
set_deadline_from_read_timeout(kwargs)
# deadline is computed from read timeout
self.assertIn('deadline', kwargs)
self.assertLessEqual(kwargs['deadline'],
now + kwargs['read_timeout'] + 0.1)
prev_deadline = kwargs['deadline']
set_deadline_from_read_timeout(kwargs)
# deadline is not recomputed
self.assertIn('deadline', kwargs)
self.assertEqual(prev_deadline, kwargs['deadline'])
prev_deadline = kwargs['read_timeout']
set_deadline_from_read_timeout(kwargs, force=True)
# deadline is recomputed
self.assertIn('deadline', kwargs)
self.assertNotEqual(prev_deadline, kwargs['deadline'])
def finish(self, metachunk_size, metachunk_hash):
"""
Send metachunk_size and metachunk_hash as trailers.
:returns: the chunk object if the upload has failed, else None
"""
self.wait()
if self.failed:
self.logger.debug("NOT sending end marker and trailers to %s, "
"because upload has failed", self.chunk['url'])
return self.chunk
self.logger.debug("Sending end marker and trailers to %s",
self.chunk['url'])
parts = [
'0\r\n',
'%s: %s\r\n' % (CHUNK_HEADERS['metachunk_size'],
metachunk_size),
'%s: %s\r\n' % (CHUNK_HEADERS['metachunk_hash'],
metachunk_hash)
]
if self.checksum:
parts.append('%s: %s\r\n' % (CHUNK_HEADERS['chunk_hash'],
self.checksum.hexdigest()))
parts.append('\r\n')
to_send = ''.join(parts).encode('utf-8')
if self.perfdata is not None:
fin_start = monotonic_time()
try:
with ChunkWriteTimeout(self.write_timeout):
self.conn.send(to_send)
# Last segment sent, disable TCP_CORK to flush buffers
self.conn.set_cork(False)
except (Exception, ChunkWriteTimeout) as exc:
self.failed = True
msg = text_type(exc)
self.logger.warn("Failed to finish %s (%s, reqid=%s)",
self.chunk, msg, self.reqid)
self.chunk['error'] = 'finish: %s' % msg
return self.chunk
finally:
if self.perfdata is not None:
fin_end = monotonic_time()
rawx_perfdata = self.perfdata.setdefault('rawx', dict())
chunk_url = self.conn.chunk['url']
rawx_perfdata['upload_finish.' + chunk_url] = \
fin_end - fin_start
return None
def _connect_put(self, chunk):
"""
Create a connection in order to PUT `chunk`.
:returns: a tuple with the connection object and `chunk`
"""
raw_url = chunk.get("real_url", chunk["url"])
parsed = urlparse(raw_url)
try:
chunk_path = parsed.path.split('/')[-1]
hdrs = headers_from_object_metadata(self.sysmeta)
hdrs[CHUNK_HEADERS["chunk_pos"]] = chunk["pos"]
hdrs[CHUNK_HEADERS["chunk_id"]] = chunk_path
hdrs.update(self.headers)
hdrs = encode(hdrs)
with green.ConnectionTimeout(self.connection_timeout):
if self.perfdata is not None:
connect_start = monotonic_time()
conn = io.http_connect(parsed.netloc,
'PUT',
parsed.path,
hdrs,
scheme=parsed.scheme)
conn.set_cork(True)
if self.perfdata is not None:
connect_end = monotonic_time()
perfdata_rawx = self.perfdata.setdefault('rawx', dict())
perfdata_rawx['connect.' + chunk['url']] = \
connect_end - connect_start
conn.chunk = chunk
return conn, chunk
except (SocketError, Timeout) as err:
msg = str(err)
self.logger.warn("Failed to connect to %s (reqid=%s): %s", chunk,
self.reqid, err)
except Exception as err:
msg = str(err)
self.logger.exception("Failed to connect to %s (reqid=%s)", chunk,
self.reqid)
chunk['error'] = msg
return None, chunk
def encode_and_send(self, ec_stream, data, writers):
"""
Encode a buffer of data through `ec_stream`,
and dispatch the encoded data to the chunk writers.
:returns: the list of writers that are still writing
"""
current_writers = list(writers)
self.checksum.update(data)
self.global_checksum.update(data)
# get the encoded fragments
if self.perfdata is not None:
ec_start = monotonic_time()
fragments = ec_stream.send(data)
if self.perfdata is not None:
ec_end = monotonic_time()
rawx_perfdata = self.perfdata.setdefault('rawx', dict())
rawx_perfdata['ec'] = rawx_perfdata.get('ec', 0.0) \
+ ec_end - ec_start
if fragments is None:
# not enough data given
return current_writers
for writer in writers:
fragment = fragments[writer.chunk['num']]
if not writer.failed:
if writer.checksum:
writer.checksum.update(fragment)
writer.send(fragment)
else:
current_writers.remove(writer)
self.failed_chunks.append(writer.chunk)
eventlet_yield()
self.quorum_or_fail([w.chunk for w in current_writers],
self.failed_chunks)
return current_writers
def getresponse(self):
"""Read the HTTP response from the connection"""
# As the server may buffer data before writing it to non-volatile
# storage, we don't know if we have to wait while sending data or
# while reading response, thus we apply the same timeout to both.
with Timeout(self.write_timeout):
resp = self.conn.getresponse()
if self.perfdata is not None:
perfdata_rawx = self.perfdata.setdefault('rawx', dict())
url_chunk = self.conn.chunk['url']
upload_end = monotonic_time()
perfdata_rawx[url_chunk] = \
perfdata_rawx.get(url_chunk, 0.0) \
+ upload_end - self.conn.upload_start
return resp
def _should_notify(self, account, container):
if not self.check_account:
return True
now = monotonic_time()
enabled, last_update = self.cache.get((account, container), (None, 0))
if now - last_update > self.cache_duration:
ctinfo = self.account.container_show(
account,
container,
connection_timeout=self.connection_timeout,
read_timeout=self.read_timeout,
reqid=request_id('ev-repl-'))
enabled = ctinfo.get(BUCKET_PROP_REPLI_ENABLED, False)
self.cache[(account, container)] = (enabled, now)
return enabled
def _should_notify(self, account, container):
if not self.check_account:
return True
now = monotonic_time()
enabled, last_update = self.cache.get((account, container), (None, 0))
if now - last_update > self.cache_duration:
try:
ctinfo = self.account.container_show(
account,
container,
force_master=self.force_master,
connection_timeout=self.connection_timeout,
read_timeout=self.read_timeout,
reqid=request_id('ev-repl-'))
enabled = ctinfo.get(BUCKET_PROP_REPLI_ENABLED, False)
self.cache[(account, container)] = (enabled, now)
except Exception:
self.logger.exception(
"Not updating the cached value %s=%s for %s/%s",
BUCKET_PROP_REPLI_ENABLED, enabled, account, container)
return enabled
def _decode_segments(self, fragment_iterators):
"""
Reads from fragments and yield full segments
"""
# we use eventlet Queue to read fragments
queues = []
# each iterators has its queue
for _j in range(len(fragment_iterators)):
queues.append(LightQueue(1))
def put_in_queue(fragment_iterator, queue):
"""
Coroutine to read the fragments from the iterator
"""
try:
for fragment in fragment_iterator:
# put the read fragment in the queue
queue.put(fragment)
# the queues are of size 1 so this coroutine blocks
# until we decode a full segment
except GreenletExit:
# ignore
pass
except ChunkReadTimeout as err:
self.logger.error('%s (reqid=%s)', err, self.reqid)
except Exception:
self.logger.exception("Exception on reading (reqid=%s)",
self.reqid)
finally:
queue.resize(2)
# put None to indicate the decoding loop
# this is over
queue.put(None)
# close the iterator
fragment_iterator.close()
# we use eventlet GreenPool to manage the read of fragments
with ContextPool(len(fragment_iterators)) as pool:
# spawn coroutines to read the fragments
for fragment_iterator, queue in zip(fragment_iterators, queues):
pool.spawn(put_in_queue, fragment_iterator, queue)
# main decoding loop
while True:
data = []
# get the fragments from the queues
for queue in queues:
fragment = queue.get()
data.append(fragment)
if not all(data):
# one of the readers returned None
# impossible to read segment
break
# actually decode the fragments into a segment
if self.perfdata is not None:
ec_start = monotonic_time()
try:
segment = self.storage_method.driver.decode(data)
except exceptions.ECError:
# something terrible happened
self.logger.exception(
"ERROR decoding fragments (reqid=%s)", self.reqid)
raise
finally:
if self.perfdata is not None:
ec_end = monotonic_time()
rawx_pdata = self.perfdata.setdefault('rawx', dict())
rawx_pdata['ec'] = rawx_pdata.get('ec', 0.0) \
+ ec_end - ec_start
yield segment
def _direct_request(self, method, url, headers=None, data=None, json=None,
params=None, admin_mode=False, pool_manager=None,
force_master=False, **kwargs):
"""
Make an HTTP request.
:param method: HTTP method to use (e.g. "GET")
:type method: `str`
:param url: URL to request
:type url: `str`
:keyword admin_mode: allow operations on slave or worm namespaces
:type admin_mode: `bool`
:keyword deadline: deadline for the request, in monotonic time.
Supersedes `read_timeout`.
:type deadline: `float` seconds
:keyword timeout: optional timeout for the request (in seconds).
May be a `urllib3.Timeout(connect=connection_timeout,
read=read_timeout)`.
This method also accepts `connection_timeout` and `read_timeout`
as separate arguments.
:type timeout: `float` or `urllib3.Timeout`
:keyword headers: optional headers to add to the request
:type headers: `dict`
:keyword force_master: request will run on master service only.
:type force_master: `bool`
:raise oio.common.exceptions.OioTimeout: in case of read, write
or connection timeout
:raise oio.common.exceptions.OioNetworkException: in case of
connection error
:raise oio.common.exceptions.OioException: in other case of HTTP error
:raise oio.common.exceptions.ClientException: in case of HTTP status
code >= 400
"""
out_kwargs = dict(kwargs)
# Ensure headers are all strings
if headers:
out_headers = {k: str(v) for k, v in headers.items()}
else:
out_headers = dict()
if self.admin_mode or admin_mode:
out_headers[ADMIN_HEADER] = '1'
if self.force_master or force_master:
out_headers[FORCEMASTER_HEADER] = '1'
# Look for a request deadline, deduce the timeout from it.
if kwargs.get('deadline', None) is not None:
to = deadline_to_timeout(kwargs['deadline'], True)
to = min(to, kwargs.get('read_timeout', to))
out_kwargs['timeout'] = urllib3.Timeout(
connect=kwargs.get('connection_timeout', CONNECTION_TIMEOUT),
read=to)
# Shorten the deadline by 1% to compensate for the time spent
# connecting and reading response.
out_headers[TIMEOUT_HEADER] = int(to * 990000.0)
# Ensure there is a timeout
if 'timeout' not in out_kwargs:
out_kwargs['timeout'] = urllib3.Timeout(
connect=kwargs.get('connection_timeout', CONNECTION_TIMEOUT),
read=kwargs.get('read_timeout', READ_TIMEOUT))
if TIMEOUT_HEADER not in out_headers:
to = out_kwargs['timeout']
if isinstance(to, urllib3.Timeout):
to = to.read_timeout
else:
to = float(to)
out_headers[TIMEOUT_HEADER] = int(to * 1000000.0)
# Look for a request ID
if 'reqid' in kwargs:
out_headers[REQID_HEADER] = str(kwargs['reqid'])
if len(out_headers.get(REQID_HEADER, '')) > STRLEN_REQID:
out_headers[REQID_HEADER] = \
out_headers[REQID_HEADER][:STRLEN_REQID]
self.__logger().warn('Request ID truncated to %d characters',
STRLEN_REQID)
# Convert json and add Content-Type
if json:
out_headers["Content-Type"] = "application/json"
data = jsonlib.dumps(json)
# Trigger performance measurments
perfdata = kwargs.get('perfdata', None)
if perfdata is not None:
out_headers[PERFDATA_HEADER] = 'enabled'
# Explicitly keep or close the connection
if 'Connection' not in out_headers:
out_headers['Connection'] = self.connection
out_kwargs['headers'] = out_headers
out_kwargs['body'] = data
# Add query string
if params:
out_param = []
for k, v in params.items():
if v is not None:
if isinstance(v, unicode):
v = unicode(v).encode('utf-8')
out_param.append((k, v))
encoded_args = urlencode(out_param)
url += '?' + encoded_args
if not pool_manager:
pool_manager = self.pool_manager
try:
if perfdata is not None:
request_start = monotonic_time()
resp = pool_manager.request(method, url, **out_kwargs)
if perfdata is not None:
request_end = monotonic_time()
service_perfdata = perfdata.setdefault(
self.service_type, dict())
service_perfdata['total'] = service_perfdata.get(
'total', 0.0) + request_end - request_start
body = resp.data
if body:
try:
body = jsonlib.loads(body)
except ValueError:
pass
if perfdata is not None and PERFDATA_HEADER in resp.headers:
service_perfdata = perfdata[self.service_type]
for header_val in resp.headers[PERFDATA_HEADER].split(','):
kv = header_val.split('=', 1)
service_perfdata[kv[0]] = service_perfdata.get(
kv[0], 0.0) + float(kv[1]) / 1000000.0
except urllib3.exceptions.HTTPError as exc:
oio_exception_from_httperror(exc,
reqid=out_headers.get(REQID_HEADER),
url=url)
if resp.status >= 400:
raise exceptions.from_response(resp, body)
return resp, body
def iter_from_resp(self, source, parts_iter, part, chunk):
bytes_consumed = 0
count = 0
buf = b''
if self.perfdata is not None:
rawx_perfdata = self.perfdata.setdefault('rawx', dict())
chunk_url = chunk['url']
while True:
try:
with green.ChunkReadTimeout(self.read_timeout):
data = part.read(READ_CHUNK_SIZE)
count += 1
buf += data
except (green.ChunkReadTimeout, IOError) as crto:
try:
self.recover(bytes_consumed)
except (exc.UnsatisfiableRange, ValueError):
raise
except exc.EmptyByteRange:
# we are done already
break
buf = b''
# find a new source to perform recovery
new_source, new_chunk = self._get_source()
if new_source:
self.logger.warn(
"Failed to read from %s (%s), "
"retrying from %s (reqid=%s)", chunk, crto, new_chunk,
self.reqid)
close_source(source[0], self.logger)
# switch source
source[0] = new_source
chunk = new_chunk
parts_iter[0] = make_iter_from_resp(source[0])
try:
_j, _j, _j, _j, part = \
self.get_next_part(parts_iter)
except StopIteration:
# failed to recover
# we did our best
return
else:
self.logger.warn("Failed to read from %s (%s, reqid=%s)",
chunk, crto, self.reqid)
# no valid source found to recover
raise
else:
# discard bytes
if buf and self.discard_bytes:
if self.discard_bytes < len(buf):
buf = buf[self.discard_bytes:]
bytes_consumed += self.discard_bytes
self.discard_bytes = 0
else:
self.discard_bytes -= len(buf)
bytes_consumed += len(buf)
buf = b''
# no data returned
# flush out buffer
if not data:
if buf:
bytes_consumed += len(buf)
yield buf
buf = b''
break
# If buf_size is defined, yield bounded data buffers
if self.buf_size is not None:
while len(buf) >= self.buf_size:
read_d = buf[:self.buf_size]
buf = buf[self.buf_size:]
yield read_d
bytes_consumed += len(read_d)
else:
yield buf
bytes_consumed += len(buf)
buf = b''
# avoid starvation by yielding
# every once in a while
if count % 10 == 0:
eventlet_yield()
if self.perfdata is not None:
download_end = monotonic_time()
key = 'download.' + chunk_url
rawx_perfdata[key] = rawx_perfdata.get(key, 0.0) \
+ download_end - source[0].download_start
