def _send(self):
"""Send coroutine loop"""
self.conn.upload_start = None
while not self.failed:
# fetch input data from the queue
data = self.queue.get()
# use HTTP transfer encoding chunked
# to write data to RAWX
try:
with ChunkWriteTimeout(self.write_timeout):
if self.perfdata is not None \
and self.conn.upload_start is None:
self.conn.upload_start = monotonic_time()
self.conn.send("%x\r\n" % len(data))
self.conn.send(data)
self.conn.send("\r\n")
self.bytes_transferred += len(data)
eventlet_yield()
except (Exception, ChunkWriteTimeout) as exc:
self.failed = True
msg = str(exc)
self.logger.warn("Failed to write to %s (%s, reqid=%s)",
self.chunk, msg, self.reqid)
self.chunk['error'] = 'write: %s' % msg
# Drain the queue before quitting
while True:
try:
self.queue.get_nowait()
except Empty:
break
def _send_data(self, conn):
"""
Send data to an open connection, taking data blocks from `conn.queue`.
"""
conn.upload_start = None
while True:
data = conn.queue.get()
if isinstance(data, text_type):
data = data.encode('utf-8')
if not conn.failed:
try:
with green.ChunkWriteTimeout(self.write_timeout):
if self.perfdata is not None \
and conn.upload_start is None:
conn.upload_start = monotonic_time()
conn.send(b'%x\r\n' % len(data))
conn.send(data)
conn.send(b'\r\n')
if not data:
if self.perfdata is not None:
fin_start = monotonic_time()
# Last segment sent, disable TCP_CORK to flush buffers
conn.set_cork(False)
if self.perfdata is not None:
fin_end = monotonic_time()
rawx_perfdata = self.perfdata.setdefault(
'rawx', dict())
chunk_url = conn.chunk['url']
rawx_perfdata['upload_finish.' + chunk_url] = \
fin_end - fin_start
green.eventlet_yield()
except (Exception, green.ChunkWriteTimeout) as err:
conn.failed = True
conn.chunk['error'] = str(err)
def wait(self):
"""
Wait until all data in the queue
has been processed by the send coroutine
"""
self.logger.debug("Waiting for %s to receive data", self.chunk['url'])
while self.queue.qsize() and not self.failed:
eventlet_yield()
def encode_and_send(self, ec_stream, data, writers):
"""
Encode a buffer of data through `ec_stream`,
and dispatch the encoded data to the chunk writers.
:returns: the list of writers that are still writing
"""
current_writers = list(writers)
self.checksum.update(data)
self.global_checksum.update(data)
# get the encoded fragments
if self.perfdata is not None:
ec_start = monotonic_time()
fragments = ec_stream.send(data)
if self.perfdata is not None:
ec_end = monotonic_time()
rawx_perfdata = self.perfdata.setdefault('rawx', dict())
rawx_perfdata['ec'] = rawx_perfdata.get('ec', 0.0) \
+ ec_end - ec_start
if fragments is None:
# not enough data given
return current_writers
for writer in writers:
fragment = fragments[writer.chunk['num']]
if not writer.failed:
if writer.checksum:
writer.checksum.update(fragment)
writer.send(fragment)
else:
current_writers.remove(writer)
self.failed_chunks.append(writer.chunk)
eventlet_yield()
self.quorum_or_fail([w.chunk for w in current_writers],
self.failed_chunks)
return current_writers
def iter_from_resp(self, source, parts_iter, part, chunk):
bytes_consumed = 0
count = 0
buf = b''
if self.perfdata is not None:
rawx_perfdata = self.perfdata.setdefault('rawx', dict())
chunk_url = chunk['url']
while True:
try:
with green.ChunkReadTimeout(self.read_timeout):
data = part.read(READ_CHUNK_SIZE)
count += 1
buf += data
except (green.ChunkReadTimeout, IOError) as crto:
try:
self.recover(bytes_consumed)
except (exc.UnsatisfiableRange, ValueError):
raise
except exc.EmptyByteRange:
# we are done already
break
buf = b''
# find a new source to perform recovery
new_source, new_chunk = self._get_source()
if new_source:
self.logger.warn(
"Failed to read from %s (%s), "
"retrying from %s (reqid=%s)", chunk, crto, new_chunk,
self.reqid)
close_source(source[0], self.logger)
# switch source
source[0] = new_source
chunk = new_chunk
parts_iter[0] = make_iter_from_resp(source[0])
try:
_j, _j, _j, _j, part = \
self.get_next_part(parts_iter)
except StopIteration:
# failed to recover
# we did our best
return
else:
self.logger.warn("Failed to read from %s (%s, reqid=%s)",
chunk, crto, self.reqid)
# no valid source found to recover
raise
else:
# discard bytes
if buf and self.discard_bytes:
if self.discard_bytes < len(buf):
buf = buf[self.discard_bytes:]
bytes_consumed += self.discard_bytes
self.discard_bytes = 0
else:
self.discard_bytes -= len(buf)
bytes_consumed += len(buf)
buf = b''
# no data returned
# flush out buffer
if not data:
if buf:
bytes_consumed += len(buf)
yield buf
buf = b''
break
# If buf_size is defined, yield bounded data buffers
if self.buf_size is not None:
while len(buf) >= self.buf_size:
read_d = buf[:self.buf_size]
buf = buf[self.buf_size:]
yield read_d
bytes_consumed += len(read_d)
else:
yield buf
bytes_consumed += len(buf)
buf = b''
# avoid starvation by yielding
# every once in a while
if count % 10 == 0:
eventlet_yield()
if self.perfdata is not None:
download_end = monotonic_time()
key = 'download.' + chunk_url
rawx_perfdata[key] = rawx_perfdata.get(key, 0.0) \
+ download_end - source[0].download_start
def stream(self, source, size):
bytes_transferred = 0
meta_chunk = self.meta_chunk
if self.chunk_checksum_algo:
meta_checksum = hashlib.new(self.chunk_checksum_algo)
else:
meta_checksum = None
pile = GreenPile(len(meta_chunk))
failed_chunks = []
current_conns = []
for chunk in meta_chunk:
pile.spawn(self._connect_put, chunk)
for conn, chunk in pile:
if not conn:
failed_chunks.append(chunk)
else:
current_conns.append(conn)
self.quorum_or_fail([co.chunk for co in current_conns], failed_chunks)
bytes_transferred = 0
try:
with green.ContextPool(len(meta_chunk)) as pool:
for conn in current_conns:
conn.failed = False
conn.queue = LightQueue(io.PUT_QUEUE_DEPTH)
pool.spawn(self._send_data, conn)
while True:
buffer_size = self.buffer_size()
if size is not None:
remaining_bytes = size - bytes_transferred
if buffer_size < remaining_bytes:
read_size = buffer_size
else:
read_size = remaining_bytes
else:
read_size = buffer_size
with green.SourceReadTimeout(self.read_timeout):
try:
data = source.read(read_size)
except (ValueError, IOError) as err:
raise SourceReadError(str(err))
if len(data) == 0:
for conn in current_conns:
if not conn.failed:
conn.queue.put(b'')
break
self.checksum.update(data)
if meta_checksum:
meta_checksum.update(data)
bytes_transferred += len(data)
# copy current_conns to be able to remove a failed conn
for conn in current_conns[:]:
if not conn.failed:
conn.queue.put(data)
else:
current_conns.remove(conn)
failed_chunks.append(conn.chunk)
self.quorum_or_fail([co.chunk for co in current_conns],
failed_chunks)
for conn in current_conns:
while conn.queue.qsize():
green.eventlet_yield()
except green.SourceReadTimeout as err:
self.logger.warn('Source read timeout (reqid=%s): %s', self.reqid,
err)
raise SourceReadTimeout(err)
except SourceReadError as err:
self.logger.warn('Source read error (reqid=%s): %s', self.reqid,
err)
raise
except Timeout as to:
self.logger.warn('Timeout writing data (reqid=%s): %s', self.reqid,
to)
raise OioTimeout(to)
except Exception:
self.logger.exception('Exception writing data (reqid=%s)',
self.reqid)
raise
success_chunks = []
for conn in current_conns:
if conn.failed:
failed_chunks.append(conn.chunk)
continue
pile.spawn(self._get_response, conn)
for (conn, resp) in pile:
if resp:
self._handle_resp(
conn, resp,
meta_checksum.hexdigest() if meta_checksum else None,
success_chunks, failed_chunks)
self.quorum_or_fail(success_chunks, failed_chunks)
for chunk in success_chunks:
chunk["size"] = bytes_transferred
return bytes_transferred, success_chunks[0]['hash'], success_chunks
