def add_future( self, future: "Union[Future[_T], concurrent.futures.Future[_T]]", callback: Callable[["Future[_T]"], None], ) -> None: """Schedules a callback on the ``IOLoop`` when the given `.Future` is finished. The callback is invoked with one argument, the `.Future`. This method only accepts `.Future` objects and not other awaitables (unlike most of Tornado where the two are interchangeable). """ if isinstance(future, Future): # Note that we specifically do not want the inline behavior of # tornado.concurrent.future_add_done_callback. We always want # this callback scheduled on the next IOLoop iteration (which # asyncio.Future always does). # # Wrap the callback in self._run_callback so we control # the error logging (i.e. it goes to tornado.log.app_log # instead of asyncio's log). future.add_done_callback(lambda f: self._run_callback( functools.partial(callback, future))) else: assert is_future(future) # For concurrent futures, we use self.add_callback, so # it's fine if future_add_done_callback inlines that call. future_add_done_callback( future, lambda f: self.add_callback(callback, future))
def finish(self) -> None: """Implements `.HTTPConnection.finish`.""" if (self._expected_content_remaining is not None and self._expected_content_remaining != 0 and not self.stream.closed()): self.stream.close() raise httputil.HTTPOutputError( "Tried to write %d bytes less than Content-Length" % self._expected_content_remaining) if self._chunking_output: if not self.stream.closed(): self._pending_write = self.stream.write(b"0\r\n\r\n") self._pending_write.add_done_callback(self._on_write_complete) self._write_finished = True # If the app finished the request while we're still reading, # divert any remaining data away from the delegate and # close the connection when we're done sending our response. # Closing the connection is the only way to avoid reading the # whole input body. if not self._read_finished: self._disconnect_on_finish = True # No more data is coming, so instruct TCP to send any remaining # data immediately instead of waiting for a full packet or ack. self.stream.set_nodelay(True) if self._pending_write is None: self._finish_request(None) else: future_add_done_callback(self._pending_write, self._finish_request)
def try_connect( self, addrs: Iterator[Tuple[socket.AddressFamily, Tuple]]) -> None: try: af, addr = next(addrs) except StopIteration: # We've reached the end of our queue, but the other queue # might still be working. Send a final error on the future # only when both queues are finished. if self.remaining == 0 and not self.future.done(): self.future.set_exception(self.last_error or IOError("connection failed")) return stream, future = self.connect(af, addr) self.streams.add(stream) future_add_done_callback( future, functools.partial(self.on_connect_done, addrs, af, addr))
def write(self, chunk: bytes) -> "Future[None]": """Implements `.HTTPConnection.write`. For backwards compatibility it is allowed but deprecated to skip `write_headers` and instead call `write()` with a pre-encoded header block. """ future = None if self.stream.closed(): future = self._write_future = Future() self._write_future.set_exception(iostream.StreamClosedError()) self._write_future.exception() else: future = self._write_future = Future() self._pending_write = self.stream.write(self._format_chunk(chunk)) future_add_done_callback(self._pending_write, self._on_write_complete) return future
def __init__(self, *args: Future, **kwargs: Future) -> None: if args and kwargs: raise ValueError("You must provide args or kwargs, not both") if kwargs: self._unfinished = dict((f, k) for (k, f) in kwargs.items()) futures = list(kwargs.values()) # type: Sequence[Future] else: self._unfinished = dict((f, i) for (i, f) in enumerate(args)) futures = args self._finished = collections.deque() # type: Deque[Future] self.current_index = None # type: Optional[Union[str, int]] self.current_future = None # type: Optional[Future] self._running_future = None # type: Optional[Future] for future in futures: future_add_done_callback(future, self._done_callback)
def write_headers( self, start_line: Union[httputil.RequestStartLine, httputil.ResponseStartLine], headers: httputil.HTTPHeaders, chunk: Optional[bytes] = None, ) -> "Future[None]": """Implements `.HTTPConnection.write_headers`.""" lines = [] if self.is_client: assert isinstance(start_line, httputil.RequestStartLine) self._request_start_line = start_line lines.append( utf8("%s %s HTTP/1.1" % (start_line[0], start_line[1]))) # Client requests with a non-empty body must have either a # Content-Length or a Transfer-Encoding. self._chunking_output = ( start_line.method in ("POST", "PUT", "PATCH") and "Content-Length" not in headers and ("Transfer-Encoding" not in headers or headers["Transfer-Encoding"] == "chunked")) else: assert isinstance(start_line, httputil.ResponseStartLine) assert self._request_start_line is not None assert self._request_headers is not None self._response_start_line = start_line lines.append( utf8("HTTP/1.1 %d %s" % (start_line[1], start_line[2]))) self._chunking_output = ( # TODO: should this use # self._request_start_line.version or # start_line.version? self._request_start_line.version == "HTTP/1.1" # 1xx, 204 and 304 responses have no body (not even a zero-length # body), and so should not have either Content-Length or # Transfer-Encoding headers. and start_line.code not in (204, 304) and (start_line.code < 100 or start_line.code >= 200) # No need to chunk the output if a Content-Length is specified. and "Content-Length" not in headers # Applications are discouraged from touching Transfer-Encoding, # but if they do, leave it alone. and "Transfer-Encoding" not in headers) # If connection to a 1.1 client will be closed, inform client if (self._request_start_line.version == "HTTP/1.1" and self._disconnect_on_finish): headers["Connection"] = "close" # If a 1.0 client asked for keep-alive, add the header. if (self._request_start_line.version == "HTTP/1.0" and self._request_headers.get("Connection", "").lower() == "keep-alive"): headers["Connection"] = "Keep-Alive" if self._chunking_output: headers["Transfer-Encoding"] = "chunked" if not self.is_client and (self._request_start_line.method == "HEAD" or cast(httputil.ResponseStartLine, start_line).code == 304): self._expected_content_remaining = 0 elif "Content-Length" in headers: self._expected_content_remaining = int(headers["Content-Length"]) else: self._expected_content_remaining = None # TODO: headers are supposed to be of type str, but we still have some # cases that let bytes slip through. Remove these native_str calls when those # are fixed. header_lines = (native_str(n) + ": " + native_str(v) for n, v in headers.get_all()) lines.extend(l.encode("latin1") for l in header_lines) for line in lines: if b"\n" in line: raise ValueError("Newline in header: " + repr(line)) future = None if self.stream.closed(): future = self._write_future = Future() future.set_exception(iostream.StreamClosedError()) future.exception() else: future = self._write_future = Future() data = b"\r\n".join(lines) + b"\r\n\r\n" if chunk: data += self._format_chunk(chunk) self._pending_write = self.stream.write(data) future_add_done_callback(self._pending_write, self._on_write_complete) return future
def timeout_callback() -> None: if not result.done(): result.set_exception(TimeoutError("Timeout")) # In case the wrapped future goes on to fail, log it. future_add_done_callback(future_converted, error_callback)
def with_timeout( timeout: Union[float, datetime.timedelta], future: _Yieldable, quiet_exceptions: "Union[Type[Exception], Tuple[Type[Exception], ...]]" = (), ) -> Future: """Wraps a `.Future` (or other yieldable object) in a timeout. Raises `tornado.util.TimeoutError` if the input future does not complete before ``timeout``, which may be specified in any form allowed by `.IOLoop.add_timeout` (i.e. a `datetime.timedelta` or an absolute time relative to `.IOLoop.time`) If the wrapped `.Future` fails after it has timed out, the exception will be logged unless it is either of a type contained in ``quiet_exceptions`` (which may be an exception type or a sequence of types), or an ``asyncio.CancelledError``. The wrapped `.Future` is not canceled when the timeout expires, permitting it to be reused. `asyncio.wait_for` is similar to this function but it does cancel the wrapped `.Future` on timeout. .. versionadded:: 4.0 .. versionchanged:: 4.1 Added the ``quiet_exceptions`` argument and the logging of unhandled exceptions. .. versionchanged:: 4.4 Added support for yieldable objects other than `.Future`. .. versionchanged:: 6.0.3 ``asyncio.CancelledError`` is now always considered "quiet". """ # It's tempting to optimize this by cancelling the input future on timeout # instead of creating a new one, but A) we can't know if we are the only # one waiting on the input future, so cancelling it might disrupt other # callers and B) concurrent futures can only be cancelled while they are # in the queue, so cancellation cannot reliably bound our waiting time. future_converted = convert_yielded(future) result = _create_future() chain_future(future_converted, result) io_loop = IOLoop.current() def error_callback(future: Future) -> None: try: future.result() except asyncio.CancelledError: pass except Exception as e: if not isinstance(e, quiet_exceptions): app_log.error( "Exception in Future %r after timeout", future, exc_info=True ) def timeout_callback() -> None: if not result.done(): result.set_exception(TimeoutError("Timeout")) # In case the wrapped future goes on to fail, log it. future_add_done_callback(future_converted, error_callback) timeout_handle = io_loop.add_timeout(timeout, timeout_callback) if isinstance(future_converted, Future): # We know this future will resolve on the IOLoop, so we don't # need the extra thread-safety of IOLoop.add_future (and we also # don't care about StackContext here. future_add_done_callback( future_converted, lambda future: io_loop.remove_timeout(timeout_handle) ) else: # concurrent.futures.Futures may resolve on any thread, so we # need to route them back to the IOLoop. io_loop.add_future( future_converted, lambda future: io_loop.remove_timeout(timeout_handle) ) return result
def multi_future( children: Union[List[_Yieldable], Dict[Any, _Yieldable]], quiet_exceptions: "Union[Type[Exception], Tuple[Type[Exception], ...]]" = (), ) -> "Union[Future[List], Future[Dict]]": """Wait for multiple asynchronous futures in parallel. Since Tornado 6.0, this function is exactly the same as `multi`. .. versionadded:: 4.0 .. versionchanged:: 4.2 If multiple ``Futures`` fail, any exceptions after the first (which is raised) will be logged. Added the ``quiet_exceptions`` argument to suppress this logging for selected exception types. .. deprecated:: 4.3 Use `multi` instead. """ if isinstance(children, dict): keys = list(children.keys()) # type: Optional[List] children_seq = children.values() # type: Iterable else: keys = None children_seq = children children_futs = list(map(convert_yielded, children_seq)) assert all(is_future(i) or isinstance(i, _NullFuture) for i in children_futs) unfinished_children = set(children_futs) future = _create_future() if not children_futs: future_set_result_unless_cancelled(future, {} if keys is not None else []) def callback(fut: Future) -> None: unfinished_children.remove(fut) if not unfinished_children: result_list = [] for f in children_futs: try: result_list.append(f.result()) except Exception as e: if future.done(): if not isinstance(e, quiet_exceptions): app_log.error( "Multiple exceptions in yield list", exc_info=True ) else: future_set_exc_info(future, sys.exc_info()) if not future.done(): if keys is not None: future_set_result_unless_cancelled( future, dict(zip(keys, result_list)) ) else: future_set_result_unless_cancelled(future, result_list) listening = set() # type: Set[Future] for f in children_futs: if f not in listening: listening.add(f) future_add_done_callback(f, callback) return future