def read_until_close(self, callback, streaming_callback=None): """Reads all data from the socket until it is closed. If a ``streaming_callback`` is given, it will be called with chunks of data as they become available, and the argument to the final ``callback`` will be empty. Subject to ``max_buffer_size`` limit from `IOStream` constructor if a ``streaming_callback`` is not used. """ assert not self._read_callback, "Already reading" if self.closed(): self._run_callback(callback, self._consume(self._read_buffer_size)) return self._read_until_close = True self._read_callback = stack_context.wrap(callback) self._streaming_callback = stack_context.wrap(streaming_callback) self._add_io_state(self.io_loop.READ)
def read_bytes(self, num_bytes, callback, streaming_callback=None): """Call callback when we read the given number of bytes. If a ``streaming_callback`` is given, it will be called with chunks of data as they become available, and the argument to the final ``callback`` will be empty. """ assert not self._read_callback, "Already reading" assert isinstance(num_bytes, (int, long)) self._read_bytes = num_bytes self._read_callback = stack_context.wrap(callback) self._streaming_callback = stack_context.wrap(streaming_callback) while True: if self._read_from_buffer(): return if self._check_closed() or self._read_to_buffer() == 0: break self._add_io_state(self.io_loop.READ)
def fetch(self, request, callback, **kwargs): if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) if not isinstance(request.headers, HTTPHeaders): request.headers = HTTPHeaders(request.headers) callback = stack_context.wrap(callback) self.queue.append((request, callback)) self._process_queue() if self.queue: logging.debug("max_clients limit reached, request queued. " "%d active, %d queued requests." % ( len(self.active), len(self.queue)))
def read_until_regex(self, regex, callback): """Call callback when we read the given regex pattern.""" assert not self._read_callback, "Already reading" self._read_regex = re.compile(regex) self._read_callback = stack_context.wrap(callback) while True: # See if we've already got the data from a previous read if self._read_from_buffer(): return if self._check_closed() or self._read_to_buffer() == 0: break self._add_io_state(self.io_loop.READ)
def read_until(self, delimiter, callback): """Call callback when we read the given delimiter.""" assert not self._read_callback, "Already reading" self._read_delimiter = delimiter self._read_callback = stack_context.wrap(callback) while True: # See if we've already got the data from a previous read if self._read_from_buffer(): return if self._check_closed() or self._read_to_buffer() == 0: break self._add_io_state(self.io_loop.READ)
def fetch(self, request, callback, **kwargs): if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) if not isinstance(request.headers, HTTPHeaders): request.headers = HTTPHeaders(request.headers) callback = stack_context.wrap(callback) self.queue.append((request, callback)) self._process_queue() if self.queue: logging.debug("max_clients limit reached, request queued. " "%d active, %d queued requests." % (len(self.active), len(self.queue)))
def add_timeout(self, deadline, callback): """Calls the given callback at the time deadline from the I/O loop. Returns a handle that may be passed to remove_timeout to cancel. ``deadline`` may be a number denoting a unix timestamp (as returned by ``time.time()`` or a ``datetime.timedelta`` object for a deadline relative to the current time. """ timeout = _Timeout(deadline, stack_context.wrap(callback)) heapq.heappush(self._timeouts, timeout) return timeout
def add_callback(self, callback): """Calls the given callback on the next I/O loop iteration. It is safe to call this method from any thread at any time. Note that this is the *only* method in IOLoop that makes this guarantee; all other interaction with the IOLoop must be done from that IOLoop's thread. add_callback() may be used to transfer control from other threads to the IOLoop's thread. """ with self._callback_lock: list_empty = not self._callbacks self._callbacks.append(stack_context.wrap(callback)) if list_empty and thread.get_ident() != self._thread_ident: # If we're in the IOLoop's thread, we know it's not currently # polling. If we're not, and we added the first callback to an # empty list, we may need to wake it up (it may wake up on its # own, but an occasional extra wake is harmless). Waking # up a polling IOLoop is relatively expensive, so we try to # avoid it when we can. self._waker.wake()
def write(self, data, callback=None): """Write the given data to this stream. If callback is given, we call it when all of the buffered write data has been successfully written to the stream. If there was previously buffered write data and an old write callback, that callback is simply overwritten with this new callback. """ assert isinstance(data, bytes_type) self._closed_by_our_side = False self._check_closed() if data: # We use bool(_write_buffer) as a proxy for write_buffer_size>0, # so never put empty strings in the buffer. self._write_buffer.append(data) self._write_callback = stack_context.wrap(callback) self._handle_write() if self._write_buffer: self._add_io_state(self.io_loop.WRITE) self._maybe_add_error_listener()
def __init__(self, stream, address, request_callback, no_keep_alive=False, xheaders=False): self.stream = stream if self.stream.socket.family not in (socket.AF_INET, socket.AF_INET6): # Unix (or other) socket; fake the remote address address = ('0.0.0.0', 0) self.address = address self.request_callback = request_callback self.no_keep_alive = no_keep_alive self.xheaders = xheaders self._request = None self._request_finished = False # Save stack context here, outside of any request. This keeps # contexts from one request from leaking into the next. self._header_callback = stack_context.wrap(self._on_headers) self.stream.read_until(b("\r\n\r\n"), self._header_callback) self._write_callback = None
def add_handler(self, fd, handler, events): """Registers the given handler to receive the given events for fd.""" self._handlers[fd] = stack_context.wrap(handler) self._impl.register(fd, events | self.ERROR)
def set_close_callback(self, callback): """Call the given callback when the stream is closed.""" self._close_callback = stack_context.wrap(callback)
def write(self, chunk, callback=None): """Writes a chunk of output to the stream.""" assert self._request, "Request closed" if not self.stream.closed(): self._write_callback = stack_context.wrap(callback) self.stream.write(chunk, self._on_write_complete)
def fetch(self, request, callback, **kwargs): if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) self._requests.append((request, stack_context.wrap(callback))) self._process_queue() self._set_timeout(0)
def library_function(callback): # capture the caller's context before introducing our own callback = wrap(callback) with StackContext(functools.partial(self.context, 'library')): self.io_loop.add_callback( functools.partial(library_inner_callback, callback))
class IOStream(object): r"""A utility class to write to and read from a non-blocking socket. We support a non-blocking ``write()`` and a family of ``read_*()`` methods. All of the methods take callbacks (since writing and reading are non-blocking and asynchronous). The socket parameter may either be connected or unconnected. For server operations the socket is the result of calling socket.accept(). For client operations the socket is created with socket.socket(), and may either be connected before passing it to the IOStream or connected with IOStream.connect. A very simple (and broken) HTTP client using this class:: from anzu import ioloop from anzu import iostream import socket def send_request(): stream.write("GET / HTTP/1.0\r\nHost: friendfeed.com\r\n\r\n") stream.read_until("\r\n\r\n", on_headers) def on_headers(data): headers = {} for line in data.split("\r\n"): parts = line.split(":") if len(parts) == 2: headers[parts[0].strip()] = parts[1].strip() stream.read_bytes(int(headers["Content-Length"]), on_body) def on_body(data): print data stream.close() ioloop.IOLoop.instance().stop() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) stream = iostream.IOStream(s) stream.connect(("friendfeed.com", 80), send_request) ioloop.IOLoop.instance().start() """ def __init__(self, socket, io_loop=None, max_buffer_size=104857600, read_chunk_size=4096): self.socket = socket self.socket.setblocking(False) self.io_loop = io_loop or ioloop.IOLoop.instance() self.max_buffer_size = max_buffer_size self.read_chunk_size = read_chunk_size self._read_buffer = collections.deque() self._write_buffer = collections.deque() self._read_buffer_size = 0 self._write_buffer_frozen = False self._read_delimiter = None self._read_regex = None self._read_bytes = None self._read_until_close = False self._read_callback = None self._streaming_callback = None self._write_callback = None self._close_callback = None self._closed_by_our_side = False self._connect_callback = None self._connecting = False self._state = None self._pending_callbacks = 0 def connect(self, address, callback=None): """Connects the socket to a remote address without blocking. May only be called if the socket passed to the constructor was not previously connected. The address parameter is in the same format as for socket.connect, i.e. a (host, port) tuple. If callback is specified, it will be called when the connection is completed. Note that it is safe to call IOStream.write while the connection is pending, in which case the data will be written as soon as the connection is ready. Calling IOStream read methods before the socket is connected works on some platforms but is non-portable. """ self._connecting = True try: self.socket.connect(address) except socket.error, e: # In non-blocking mode we expect connect() to raise an # exception with EINPROGRESS or EWOULDBLOCK. # # On freebsd, other errors such as ECONNREFUSED may be # returned immediately when attempting to connect to # localhost, so handle them the same way as an error # reported later in _handle_connect. if e.args[0] not in (errno.EINPROGRESS, errno.EWOULDBLOCK): logging.warning("Connect error on fd %d: %s", self.socket.fileno(), e) self.close() return self._connect_callback = stack_context.wrap(callback) self._add_io_state(self.io_loop.WRITE)
def library_function(callback): # capture the caller's context before introducing our own callback = wrap(callback) with StackContext(functools.partial(self.context, "library")): self.io_loop.add_callback(functools.partial(library_inner_callback, callback))