def _send(self): """Send coroutine loop""" self.conn.upload_start = None while not self.failed: # fetch input data from the queue data = self.queue.get() # use HTTP transfer encoding chunked # to write data to RAWX try: with ChunkWriteTimeout(self.write_timeout): if self.perfdata is not None \ and self.conn.upload_start is None: self.conn.upload_start = monotonic_time() self.conn.send("%x\r\n" % len(data)) self.conn.send(data) self.conn.send("\r\n") self.bytes_transferred += len(data) eventlet_yield() except (Exception, ChunkWriteTimeout) as exc: self.failed = True msg = str(exc) self.logger.warn("Failed to write to %s (%s, reqid=%s)", self.chunk, msg, self.reqid) self.chunk['error'] = 'write: %s' % msg # Drain the queue before quitting while True: try: self.queue.get_nowait() except Empty: break
def _send_data(self, conn): """ Send data to an open connection, taking data blocks from `conn.queue`. """ conn.upload_start = None while True: data = conn.queue.get() if isinstance(data, text_type): data = data.encode('utf-8') if not conn.failed: try: with green.ChunkWriteTimeout(self.write_timeout): if self.perfdata is not None \ and conn.upload_start is None: conn.upload_start = monotonic_time() conn.send(b'%x\r\n' % len(data)) conn.send(data) conn.send(b'\r\n') if not data: if self.perfdata is not None: fin_start = monotonic_time() # Last segment sent, disable TCP_CORK to flush buffers conn.set_cork(False) if self.perfdata is not None: fin_end = monotonic_time() rawx_perfdata = self.perfdata.setdefault( 'rawx', dict()) chunk_url = conn.chunk['url'] rawx_perfdata['upload_finish.' + chunk_url] = \ fin_end - fin_start green.eventlet_yield() except (Exception, green.ChunkWriteTimeout) as err: conn.failed = True conn.chunk['error'] = str(err)
def wait(self): """ Wait until all data in the queue has been processed by the send coroutine """ self.logger.debug("Waiting for %s to receive data", self.chunk['url']) while self.queue.qsize() and not self.failed: eventlet_yield()
def encode_and_send(self, ec_stream, data, writers): """ Encode a buffer of data through `ec_stream`, and dispatch the encoded data to the chunk writers. :returns: the list of writers that are still writing """ current_writers = list(writers) self.checksum.update(data) self.global_checksum.update(data) # get the encoded fragments if self.perfdata is not None: ec_start = monotonic_time() fragments = ec_stream.send(data) if self.perfdata is not None: ec_end = monotonic_time() rawx_perfdata = self.perfdata.setdefault('rawx', dict()) rawx_perfdata['ec'] = rawx_perfdata.get('ec', 0.0) \ + ec_end - ec_start if fragments is None: # not enough data given return current_writers for writer in writers: fragment = fragments[writer.chunk['num']] if not writer.failed: if writer.checksum: writer.checksum.update(fragment) writer.send(fragment) else: current_writers.remove(writer) self.failed_chunks.append(writer.chunk) eventlet_yield() self.quorum_or_fail([w.chunk for w in current_writers], self.failed_chunks) return current_writers
def iter_from_resp(self, source, parts_iter, part, chunk): bytes_consumed = 0 count = 0 buf = b'' if self.perfdata is not None: rawx_perfdata = self.perfdata.setdefault('rawx', dict()) chunk_url = chunk['url'] while True: try: with green.ChunkReadTimeout(self.read_timeout): data = part.read(READ_CHUNK_SIZE) count += 1 buf += data except (green.ChunkReadTimeout, IOError) as crto: try: self.recover(bytes_consumed) except (exc.UnsatisfiableRange, ValueError): raise except exc.EmptyByteRange: # we are done already break buf = b'' # find a new source to perform recovery new_source, new_chunk = self._get_source() if new_source: self.logger.warn( "Failed to read from %s (%s), " "retrying from %s (reqid=%s)", chunk, crto, new_chunk, self.reqid) close_source(source[0], self.logger) # switch source source[0] = new_source chunk = new_chunk parts_iter[0] = make_iter_from_resp(source[0]) try: _j, _j, _j, _j, part = \ self.get_next_part(parts_iter) except StopIteration: # failed to recover # we did our best return else: self.logger.warn("Failed to read from %s (%s, reqid=%s)", chunk, crto, self.reqid) # no valid source found to recover raise else: # discard bytes if buf and self.discard_bytes: if self.discard_bytes < len(buf): buf = buf[self.discard_bytes:] bytes_consumed += self.discard_bytes self.discard_bytes = 0 else: self.discard_bytes -= len(buf) bytes_consumed += len(buf) buf = b'' # no data returned # flush out buffer if not data: if buf: bytes_consumed += len(buf) yield buf buf = b'' break # If buf_size is defined, yield bounded data buffers if self.buf_size is not None: while len(buf) >= self.buf_size: read_d = buf[:self.buf_size] buf = buf[self.buf_size:] yield read_d bytes_consumed += len(read_d) else: yield buf bytes_consumed += len(buf) buf = b'' # avoid starvation by yielding # every once in a while if count % 10 == 0: eventlet_yield() if self.perfdata is not None: download_end = monotonic_time() key = 'download.' + chunk_url rawx_perfdata[key] = rawx_perfdata.get(key, 0.0) \ + download_end - source[0].download_start
def stream(self, source, size): bytes_transferred = 0 meta_chunk = self.meta_chunk if self.chunk_checksum_algo: meta_checksum = hashlib.new(self.chunk_checksum_algo) else: meta_checksum = None pile = GreenPile(len(meta_chunk)) failed_chunks = [] current_conns = [] for chunk in meta_chunk: pile.spawn(self._connect_put, chunk) for conn, chunk in pile: if not conn: failed_chunks.append(chunk) else: current_conns.append(conn) self.quorum_or_fail([co.chunk for co in current_conns], failed_chunks) bytes_transferred = 0 try: with green.ContextPool(len(meta_chunk)) as pool: for conn in current_conns: conn.failed = False conn.queue = LightQueue(io.PUT_QUEUE_DEPTH) pool.spawn(self._send_data, conn) while True: buffer_size = self.buffer_size() if size is not None: remaining_bytes = size - bytes_transferred if buffer_size < remaining_bytes: read_size = buffer_size else: read_size = remaining_bytes else: read_size = buffer_size with green.SourceReadTimeout(self.read_timeout): try: data = source.read(read_size) except (ValueError, IOError) as err: raise SourceReadError(str(err)) if len(data) == 0: for conn in current_conns: if not conn.failed: conn.queue.put(b'') break self.checksum.update(data) if meta_checksum: meta_checksum.update(data) bytes_transferred += len(data) # copy current_conns to be able to remove a failed conn for conn in current_conns[:]: if not conn.failed: conn.queue.put(data) else: current_conns.remove(conn) failed_chunks.append(conn.chunk) self.quorum_or_fail([co.chunk for co in current_conns], failed_chunks) for conn in current_conns: while conn.queue.qsize(): green.eventlet_yield() except green.SourceReadTimeout as err: self.logger.warn('Source read timeout (reqid=%s): %s', self.reqid, err) raise SourceReadTimeout(err) except SourceReadError as err: self.logger.warn('Source read error (reqid=%s): %s', self.reqid, err) raise except Timeout as to: self.logger.warn('Timeout writing data (reqid=%s): %s', self.reqid, to) raise OioTimeout(to) except Exception: self.logger.exception('Exception writing data (reqid=%s)', self.reqid) raise success_chunks = [] for conn in current_conns: if conn.failed: failed_chunks.append(conn.chunk) continue pile.spawn(self._get_response, conn) for (conn, resp) in pile: if resp: self._handle_resp( conn, resp, meta_checksum.hexdigest() if meta_checksum else None, success_chunks, failed_chunks) self.quorum_or_fail(success_chunks, failed_chunks) for chunk in success_chunks: chunk["size"] = bytes_transferred return bytes_transferred, success_chunks[0]['hash'], success_chunks