def _make_app_iter(self, node, source): """ Returns an iterator over the contents of the source (via its read func). There is also quite a bit of cleanup to ensure garbage collection works and the underlying socket of the source is closed. :param source: The httplib.Response object this iterator should read from. :param node: The node the source is reading from, for logging purposes. """ try: nchunks = 0 while True: with ChunkReadTimeout(self.app.node_timeout): chunk = source.read(self.app.object_chunk_size) nchunks += 1 if not chunk: break with ChunkWriteTimeout(self.app.client_timeout): yield chunk # This is for fairness; if the network is outpacing the CPU, # we'll always be able to read and write data without # encountering an EWOULDBLOCK, and so eventlet will not switch # greenthreads on its own. We do it manually so that clients # don't starve. # # The number 5 here was chosen by making stuff up. It's not # every single chunk, but it's not too big either, so it seemed # like it would probably be an okay choice. # # Note that we may trampoline to other greenthreads more often # than once every 5 chunks, depending on how blocking our # network IO is; the explicit sleep here simply provides a # lower bound on the rate of trampolining. if nchunks % 5 == 0: sleep() except ChunkReadTimeout: self.exception_occurred(node, _('Object'), _('Trying to read during GET')) raise except ChunkWriteTimeout: self.app.logger.warn( _('Client did not read from proxy within %ss') % self.app.client_timeout) self.app.logger.increment('client_timeouts') except GeneratorExit: self.app.logger.warn(_('Client disconnected on read')) except Exception: self.app.logger.exception(_('Trying to send to client')) raise finally: # Close-out the connection as best as possible. if getattr(source, 'swift_conn', None): self.close_swift_conn(source)
def _send_file(self, conn, path): """Method for a file PUT coro""" while True: chunk = conn.queue.get() if not conn.failed: try: with ChunkWriteTimeout(self.node_timeout): conn.send(chunk) self.logger.debug('Sending...') except (Exception, ChunkWriteTimeout): conn.failed = True self.logger.debug('Trying to write to %s' % path) conn.queue.task_done()
def _send_file(self, conn, path): """Method for a file PUT coro""" while True: chunk = conn.queue.get() if not conn.failed: try: with ChunkWriteTimeout(self.app.node_timeout): conn.send(chunk) except (Exception, ChunkWriteTimeout): conn.failed = True self.exception_occurred(conn.node, _('Object'), _('Trying to write to %s') % path) conn.queue.task_done()