def read_until_close(self, callback, streaming_callback=None): """Reads all data from the socket until it is closed. If a ``streaming_callback`` is given, it will be called with chunks of data as they become available, and the argument to the final ``callback`` will be empty. Otherwise, the ``callback`` gets the data as an argument. Subject to ``max_buffer_size`` limit from `IOStream` constructor if a ``streaming_callback`` is not used. """ self._set_read_callback(callback) self._streaming_callback = stack_context.wrap(streaming_callback) if self.closed(): if self._streaming_callback is not None: self._run_callback(self._streaming_callback, self._consume(self._read_buffer_size)) self._run_callback(self._read_callback, self._consume(self._read_buffer_size)) self._streaming_callback = None self._read_callback = None return self._read_until_close = True self._streaming_callback = stack_context.wrap(streaming_callback) self._try_inline_read()
def add_callback(self, callback, *args, **kwargs): if self.closing: raise RuntimeError("IOLoop is closing") if kwargs: self.asyncio_loop.call_soon_threadsafe( functools.partial(self._run_callback, stack_context.wrap(callback), *args, **kwargs) ) else: self.asyncio_loop.call_soon_threadsafe(self._run_callback, stack_context.wrap(callback), *args)
def write(self, data, callback=None): """Write the given data to this stream. If ``callback`` is given, we call it when all of the buffered write data has been successfully written to the stream. If there was previously buffered write data and an old write callback, that callback is simply overwritten with this new callback. """ assert isinstance(data, bytes_type) self._check_closed() # We use bool(_write_buffer) as a proxy for write_buffer_size>0, # so never put empty strings in the buffer. if data: # Break up large contiguous strings before inserting them in the # write buffer, so we don't have to recopy the entire thing # as we slice off pieces to send to the socket. WRITE_BUFFER_CHUNK_SIZE = 128 * 1024 if len(data) > WRITE_BUFFER_CHUNK_SIZE: for i in range(0, len(data), WRITE_BUFFER_CHUNK_SIZE): self._write_buffer.append(data[i:i + WRITE_BUFFER_CHUNK_SIZE]) else: self._write_buffer.append(data) self._write_callback = stack_context.wrap(callback) if not self._connecting: self._handle_write() if self._write_buffer: self._add_io_state(self.io_loop.WRITE) self._maybe_add_error_listener()
def add_timeout(self, deadline, callback): if isinstance(deadline, (int, long, float)): delay = max(deadline - self.time(), 0) elif isinstance(deadline, datetime.timedelta): delay = webalchemy.tornado.ioloop._Timeout.timedelta_to_seconds(deadline) else: raise TypeError("Unsupported deadline %r") return self.reactor.callLater(delay, self._run_callback, wrap(callback))
def add_timeout(self, deadline, callback): if isinstance(deadline, (int, float)): delay = max(deadline - self.time(), 0) elif isinstance(deadline, datetime.timedelta): delay = deadline.total_seconds() else: raise TypeError("Unsupported deadline %r", deadline) return self.asyncio_loop.call_later(delay, self._run_callback, stack_context.wrap(callback))
def set_close_callback(self, callback): """Sets a callback that will be run when the connection is closed. Use this instead of accessing `HTTPConnection.stream.set_close_callback <.BaseIOStream.set_close_callback>` directly (which was the recommended approach prior to Tornado 3.0). """ self._close_callback = stack_context.wrap(callback)
def add_handler(self, fd, handler, events): if fd in self.handlers: raise ValueError("fd %d added twice" % fd) self.handlers[fd] = stack_context.wrap(handler) if events & IOLoop.READ: self.asyncio_loop.add_reader(fd, self._handle_events, fd, IOLoop.READ) self.readers.add(fd) if events & IOLoop.WRITE: self.asyncio_loop.add_writer(fd, self._handle_events, fd, IOLoop.WRITE) self.writers.add(fd)
def result_callback(self, key): def inner(*args, **kwargs): if kwargs or len(args) > 1: result = Arguments(args, kwargs) elif args: result = args[0] else: result = None self.set_result(key, result) return wrap(inner)
def add_handler(self, fd, handler, events): if fd in self.fds: raise ValueError('fd %d added twice' % fd) self.fds[fd] = _FD(fd, wrap(handler)) if events & webalchemy.tornado.ioloop.IOLoop.READ: self.fds[fd].reading = True self.reactor.addReader(self.fds[fd]) if events & webalchemy.tornado.ioloop.IOLoop.WRITE: self.fds[fd].writing = True self.reactor.addWriter(self.fds[fd])
def capitalize(self, request_data, callback=None): logging.info("capitalize") self.request_data = request_data self.stream = IOStream(socket.socket(), io_loop=self.io_loop) self.stream.connect(('127.0.0.1', self.port), callback=self.handle_connect) self.future = Future() if callback is not None: self.future.add_done_callback( stack_context.wrap(lambda future: callback(future.result()))) return self.future
def add_future(self, future, callback): """Schedules a callback on the ``IOLoop`` when the given `.Future` is finished. The callback is invoked with one argument, the `.Future`. """ assert isinstance(future, Future) callback = stack_context.wrap(callback) future.add_done_callback( lambda future: self.add_callback(callback, future))
def make_wrapped_function(): """Wraps a function in three stack contexts, and returns the function along with the deactivation functions. """ # Remove the test's stack context to make sure we can cover # the case where the last context is deactivated. with NullContext(): partial = functools.partial with StackContext(partial(self.context, 'c0')) as c0: with StackContext(partial(self.context, 'c1')) as c1: with StackContext(partial(self.context, 'c2')) as c2: return (wrap(check_contexts), [c0, c1, c2])
def read_bytes(self, num_bytes, callback, streaming_callback=None): """Run callback when we read the given number of bytes. If a ``streaming_callback`` is given, it will be called with chunks of data as they become available, and the argument to the final ``callback`` will be empty. Otherwise, the ``callback`` gets the data as an argument. """ self._set_read_callback(callback) assert isinstance(num_bytes, numbers.Integral) self._read_bytes = num_bytes self._streaming_callback = stack_context.wrap(streaming_callback) self._try_inline_read()
def __init__(self, io_loop, client, request, release_callback, final_callback, max_buffer_size, resolver): self.start_time = io_loop.time() self.io_loop = io_loop self.client = client self.request = request self.release_callback = release_callback self.final_callback = final_callback self.max_buffer_size = max_buffer_size self.resolver = resolver self.code = None self.headers = None self.chunks = None self._decompressor = None # Timeout handle returned by IOLoop.add_timeout self._timeout = None with stack_context.ExceptionStackContext(self._handle_exception): self.parsed = urlparse.urlsplit(_unicode(self.request.url)) if self.parsed.scheme not in ("http", "https"): raise ValueError("Unsupported url scheme: %s" % self.request.url) # urlsplit results have hostname and port results, but they # didn't support ipv6 literals until python 2.7. netloc = self.parsed.netloc if "@" in netloc: userpass, _, netloc = netloc.rpartition("@") match = re.match(r'^(.+):(\d+)$', netloc) if match: host = match.group(1) port = int(match.group(2)) else: host = netloc port = 443 if self.parsed.scheme == "https" else 80 if re.match(r'^\[.*\]$', host): # raw ipv6 addresses in urls are enclosed in brackets host = host[1:-1] self.parsed_hostname = host # save final host for _on_connect if request.allow_ipv6: af = socket.AF_UNSPEC else: # We only try the first IP we get from getaddrinfo, # so restrict to ipv4 by default. af = socket.AF_INET timeout = min(self.request.connect_timeout, self.request.request_timeout) if timeout: self._timeout = self.io_loop.add_timeout( self.start_time + timeout, stack_context.wrap(self._on_timeout)) self.resolver.resolve(host, port, af, callback=self._on_resolve)
def test_pre_wrap(self): # A pre-wrapped callback is run in the context in which it was # wrapped, not when it was added to the IOLoop. def f1(): self.assertIn('c1', self.active_contexts) self.assertNotIn('c2', self.active_contexts) self.stop() with StackContext(functools.partial(self.context, 'c1')): wrapped = wrap(f1) with StackContext(functools.partial(self.context, 'c2')): self.add_callback(wrapped) self.wait()
def add_callback(self, callback, *args, **kwargs): with self._callback_lock: if self._closing: raise RuntimeError("IOLoop is closing") list_empty = not self._callbacks self._callbacks.append(functools.partial( stack_context.wrap(callback), *args, **kwargs)) if list_empty and thread.get_ident() != self._thread_ident: # If we're in the IOLoop's thread, we know it's not currently # polling. If we're not, and we added the first callback to an # empty list, we may need to wake it up (it may wake up on its # own, but an occasional extra wake is harmless). Waking # up a polling IOLoop is relatively expensive, so we try to # avoid it when we can. self._waker.wake()
def fetch(self, request, callback=None, **kwargs): """Executes a request, asynchronously returning an `HTTPResponse`. The request may be either a string URL or an `HTTPRequest` object. If it is a string, we construct an `HTTPRequest` using any additional kwargs: ``HTTPRequest(request, **kwargs)`` This method returns a `.Future` whose result is an `HTTPResponse`. The ``Future`` wil raise an `HTTPError` if the request returned a non-200 response code. If a ``callback`` is given, it will be invoked with the `HTTPResponse`. In the callback interface, `HTTPError` is not automatically raised. Instead, you must check the response's ``error`` attribute or call its `~HTTPResponse.rethrow` method. """ if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) # We may modify this (to add Host, Accept-Encoding, etc), # so make sure we don't modify the caller's object. This is also # where normal dicts get converted to HTTPHeaders objects. request.headers = httputil.HTTPHeaders(request.headers) request = _RequestProxy(request, self.defaults) future = TracebackFuture() if callback is not None: callback = stack_context.wrap(callback) def handle_future(future): exc = future.exception() if isinstance(exc, HTTPError) and exc.response is not None: response = exc.response elif exc is not None: response = HTTPResponse( request, 599, error=exc, request_time=time.time() - request.start_time) else: response = future.result() self.io_loop.add_callback(callback, response) future.add_done_callback(handle_future) def handle_response(response): if response.error: future.set_exception(response.error) else: future.set_result(response) self.fetch_impl(request, handle_response) return future
def add_callback_from_signal(self, callback, *args, **kwargs): with stack_context.NullContext(): if thread.get_ident() != self._thread_ident: # if the signal is handled on another thread, we can add # it normally (modulo the NullContext) self.add_callback(callback, *args, **kwargs) else: # If we're on the IOLoop's thread, we cannot use # the regular add_callback because it may deadlock on # _callback_lock. Blindly insert into self._callbacks. # This is safe because the GIL makes list.append atomic. # One subtlety is that if the signal interrupted the # _callback_lock block in IOLoop.start, we may modify # either the old or new version of self._callbacks, # but either way will work. self._callbacks.append(functools.partial( stack_context.wrap(callback), *args, **kwargs))
def test_pre_wrap_with_args(self): # Same as test_pre_wrap, but the function takes arguments. # Implementation note: The function must not be wrapped in a # functools.partial until after it has been passed through # stack_context.wrap def f1(foo, bar): self.assertIn('c1', self.active_contexts) self.assertNotIn('c2', self.active_contexts) self.stop((foo, bar)) with StackContext(functools.partial(self.context, 'c1')): wrapped = wrap(f1) with StackContext(functools.partial(self.context, 'c2')): self.add_callback(wrapped, 1, bar=2) result = self.wait() self.assertEqual(result, (1, 2))
def __init__(self, stream, address, request_callback, no_keep_alive=False, xheaders=False, protocol=None): self.stream = stream self.address = address # Save the socket's address family now so we know how to # interpret self.address even after the stream is closed # and its socket attribute replaced with None. self.address_family = stream.socket.family self.request_callback = request_callback self.no_keep_alive = no_keep_alive self.xheaders = xheaders self.protocol = protocol self._clear_request_state() # Save stack context here, outside of any request. This keeps # contexts from one request from leaking into the next. self._header_callback = stack_context.wrap(self._on_headers) self.stream.set_close_callback(self._on_connection_close) self.stream.read_until(b"\r\n\r\n", self._header_callback)
def set_exit_callback(self, callback): """Runs ``callback`` when this process exits. The callback takes one argument, the return code of the process. This method uses a ``SIGCHILD`` handler, which is a global setting and may conflict if you have other libraries trying to handle the same signal. If you are using more than one ``IOLoop`` it may be necessary to call `Subprocess.initialize` first to designate one ``IOLoop`` to run the signal handlers. In many cases a close callback on the stdout or stderr streams can be used as an alternative to an exit callback if the signal handler is causing a problem. """ self._exit_callback = stack_context.wrap(callback) Subprocess.initialize(self.io_loop) Subprocess._waiting[self.pid] = self Subprocess._try_cleanup_process(self.pid)
def wrapper(*args, **kwargs): future = TracebackFuture() callback, args, kwargs = replacer.replace( lambda value=_NO_RESULT: future.set_result(value), args, kwargs) def handle_error(typ, value, tb): future.set_exc_info((typ, value, tb)) return True exc_info = None with ExceptionStackContext(handle_error): try: result = f(*args, **kwargs) if result is not None: raise ReturnValueIgnoredError( "@return_future should not be used with functions " "that return values") except: exc_info = sys.exc_info() raise if exc_info is not None: # If the initial synchronous part of f() raised an exception, # go ahead and raise it to the caller directly without waiting # for them to inspect the Future. raise_exc_info(exc_info) # If the caller passed in a callback, schedule it to be called # when the future resolves. It is important that this happens # just before we return the future, or else we risk confusing # stack contexts with multiple exceptions (one here with the # immediate exception, and again when the future resolves and # the callback triggers its exception by calling future.result()). if callback is not None: def run_callback(future): result = future.result() if result is _NO_RESULT: callback() else: callback(future.result()) future.add_done_callback(wrap(run_callback)) return future
def connect(self, address, callback=None, server_hostname=None): """Connects the socket to a remote address without blocking. May only be called if the socket passed to the constructor was not previously connected. The address parameter is in the same format as for `socket.connect <socket.socket.connect>`, i.e. a ``(host, port)`` tuple. If ``callback`` is specified, it will be called when the connection is completed. If specified, the ``server_hostname`` parameter will be used in SSL connections for certificate validation (if requested in the ``ssl_options``) and SNI (if supported; requires Python 3.2+). Note that it is safe to call `IOStream.write <BaseIOStream.write>` while the connection is pending, in which case the data will be written as soon as the connection is ready. Calling `IOStream` read methods before the socket is connected works on some platforms but is non-portable. """ self._connecting = True try: self.socket.connect(address) except socket.error as e: # In non-blocking mode we expect connect() to raise an # exception with EINPROGRESS or EWOULDBLOCK. # # On freebsd, other errors such as ECONNREFUSED may be # returned immediately when attempting to connect to # localhost, so handle them the same way as an error # reported later in _handle_connect. if (e.args[0] != errno.EINPROGRESS and e.args[0] not in _ERRNO_WOULDBLOCK): gen_log.warning("Connect error on fd %d: %s", self.socket.fileno(), e) self.close(exc_info=True) return self._connect_callback = stack_context.wrap(callback) self._add_io_state(self.io_loop.WRITE)
def add_timeout(self, deadline, callback): timeout = _Timeout(deadline, stack_context.wrap(callback), self) heapq.heappush(self._timeouts, timeout) return timeout
def connect(self, address, callback=None, server_hostname=None): # Save the user's callback and run it after the ssl handshake # has completed. self._ssl_connect_callback = stack_context.wrap(callback) self._server_hostname = server_hostname super(SSLIOStream, self).connect(address, callback=None)
def add_handler(self, fd, handler, events): self._handlers[fd] = stack_context.wrap(handler) self._impl.register(fd, events | self.ERROR)
def set_close_callback(self, callback): """Call the given callback when the stream is closed.""" self._close_callback = stack_context.wrap(callback)
def library_function(callback): # capture the caller's context before introducing our own callback = wrap(callback) with StackContext(functools.partial(self.context, 'library')): self.io_loop.add_callback( functools.partial(library_inner_callback, callback))
def add_callback(self, callback, *args, **kwargs): self.reactor.callFromThread(self._run_callback, wrap(callback), *args, **kwargs)
def _set_read_callback(self, callback): assert not self._read_callback, "Already reading" self._read_callback = stack_context.wrap(callback)