def wait(self, timeout: Union[float, datetime.timedelta] = None) -> "Future[None]": """Block until the internal flag is true. Returns a Future, which raises `tornado.util.TimeoutError` after a timeout. """ fut = Future() # type: Future[None] if self._value: fut.set_result(None) return fut self._waiters.add(fut) fut.add_done_callback(lambda fut: self._waiters.remove(fut)) if timeout is None: return fut else: timeout_fut = gen.with_timeout( timeout, fut, quiet_exceptions=(CancelledError,) ) # This is a slightly clumsy workaround for the fact that # gen.with_timeout doesn't cancel its futures. Cancelling # fut will remove it from the waiters list. timeout_fut.add_done_callback( lambda tf: fut.cancel() if not fut.done() else None ) return timeout_fut
class RPC2Connection(RPCConnection): # main implementation of RPC2.0 def __init__(self, stream, is_client=False, timeout=None): self.stream = stream self.is_client = is_client self.timeout = timeout self._finish_future = Future() def read_response(self, delegate): """Read a single RPC response. Typical client-mode usage is to write a request using `write_headers`, `write`, and `finish`, and then call ``read_response``. :arg delegate: a `.RPCMessageDelegate` Returns a `.Future` that resolves to None after the full response has been read. """ return self._read_message(delegate) @gen.coroutine def _read_message(self, delegate): try: while True: message_future = self.stream.read_until(b"\r\n") if self.timeout is None: message_data = yield message_future else: try: message_data = yield gen.with_timeout( self.stream.io_loop.time() + self.timeout, message_future, io_loop=self.stream.io_loop, quiet_exceptions=iostream.StreamClosedError) except gen.TimeoutError: self.close() raise gen.Return(False) with _ExceptionLoggingContext(app_log): data = delegate.data_received(message_data) self.stream.write(data) finally: self._clear_callbacks() def close(self): if self.stream is not None: self.stream.close() self._clear_callbacks() if not self._finish_future.done(): self._finish_future.set_result(None) def _clear_callbacks(self): """Clears the callback attributes. This allows the request handler to be garbage collected more quickly in CPython by breaking up reference cycles. """ self._write_callback = None self._write_future = None self._close_callback = None if self.stream is not None: self.stream.set_close_callback(None)
def test_coroutine(): td = ThingDescription.from_thing(exposed_thing.thing) event_name = next(six.iterkeys(td.events)) future_conn = Future() future_event = Future() payload = Faker().sentence() def on_next(ev): if not future_conn.done(): future_conn.set_result(True) return if ev.data == payload: future_event.set_result(True) subscription = subscribe_func(event_name, on_next) while not future_conn.done(): yield tornado.gen.sleep(0) exposed_thing.emit_event(event_name, Faker().sentence()) exposed_thing.emit_event(event_name, payload) yield future_event assert future_event.result() subscription.dispose()
def test_coroutine(): prop_name = Faker().pystr() exposed_thing.add_property(prop_name, prop_init_non_observable) observable_prop = exposed_thing.on_property_change(prop_name) future_next = Future() future_error = Future() def on_next(item): future_next.set_result(item) def on_error(err): future_error.set_exception(err) subscription = observable_prop.subscribe(on_next=on_next, on_error=on_error) yield exposed_thing.write_property(prop_name, Faker().pystr()) with pytest.raises(Exception): future_error.result() assert not future_next.done() subscription.dispose()
def test_coroutine(): td = ThingDescription.from_thing(exposed_thing.thing) prop_name = next(six.iterkeys(td.properties)) future_conn = Future() future_change = Future() prop_value = Faker().sentence() def on_next(ev): if not future_conn.done(): future_conn.set_result(True) return if ev.data.value == prop_value: future_change.set_result(True) subscription = subscribe_func(prop_name, on_next) while not future_conn.done(): yield tornado.gen.sleep(0) yield exposed_thing.write_property(prop_name, Faker().sentence()) yield exposed_thing.write_property(prop_name, prop_value) yield future_change assert future_change.result() subscription.dispose()
def test_handle_response_none(self): transport = FAsyncTransport() ctx = FContext() future = Future() transport._futures[str(ctx._get_op_id())] = future yield transport.handle_response(None) self.assertFalse(future.done())
def test_handle_response_unregistered_op_id(self): transport = FAsyncTransport() ctx1 = FContext() ctx2 = FContext() future = Future() transport._futures[str(ctx1._get_op_id())] = future yield transport.handle_response(utils.mock_frame(ctx2)) self.assertFalse(future.done())
class KernelGatewayWSClient(LoggingConfigurable): '''Proxy web socket connection to a kernel gateway.''' def __init__(self): self.ws = None self.ws_future = Future() @gen.coroutine def _connect(self, kernel_id): ws_url = url_path_join(KG_URL.replace('http', 'ws'), '/api/kernels', url_escape(kernel_id), 'channels') self.log.info('Connecting to {}'.format(ws_url)) request = HTTPRequest(ws_url, headers=KG_HEADERS, validate_cert=VALIDATE_KG_CERT) self.ws_future = websocket_connect(request) self.ws = yield self.ws_future # TODO: handle connection errors/timeout def _disconnect(self): if self.ws is not None: # Close connection self.ws.close() elif not self.ws_future.done(): # Cancel pending connection self.ws_future.cancel() @gen.coroutine def _read_messages(self, callback): '''Read messages from server.''' while True: message = yield self.ws.read_message() if message is None: break # TODO: handle socket close callback(message) def on_open(self, kernel_id, message_callback, **kwargs): '''Web socket connection open.''' self._connect(kernel_id) loop = IOLoop.current() loop.add_future(self.ws_future, lambda future: self._read_messages(message_callback)) def on_message(self, message): '''Send message to server.''' if self.ws is None: loop = IOLoop.current() loop.add_future(self.ws_future, lambda future: self._write_message(message)) else: self._write_message(message) def _write_message(self, message): '''Send message to server.''' self.ws.write_message(message) def on_close(self): '''Web socket closed event.''' self._disconnect()
class DBusConnection: def __init__(self, bus_addr): self.auth_parser = SASLParser() self.parser = Parser() self.router = Router(Future) self.authentication = Future() self.unique_name = None self._sock = socket.socket(family=socket.AF_UNIX) self.stream = IOStream(self._sock, read_chunk_size=4096) def connected(): self.stream.write(b'\0' + make_auth_external()) self.stream.connect(bus_addr, connected) self.stream.read_until_close(streaming_callback=self.data_received) def _authenticated(self): self.stream.write(BEGIN) self.authentication.set_result(True) self.data_received_post_auth(self.auth_parser.buffer) def data_received(self, data): if self.authentication.done(): return self.data_received_post_auth(data) self.auth_parser.feed(data) if self.auth_parser.authenticated: self._authenticated() elif self.auth_parser.error: self.authentication.set_exception(AuthenticationError(self.auth_parser.error)) def data_received_post_auth(self, data): for msg in self.parser.feed(data): self.router.incoming(msg) def send_message(self, message): if not self.authentication.done(): raise RuntimeError("Wait for authentication before sending messages") future = self.router.outgoing(message) data = message.serialise() self.stream.write(data) return future
class Event(object): """An event blocks coroutines until its internal flag is set to True. Similar to `threading.Event`. """ def __init__(self): self._future = Future() def __repr__(self): return '<%s %s>' % ( self.__class__.__name__, 'set' if self.is_set() else 'clear') def is_set(self): """Return ``True`` if the internal flag is true.""" return self._future.done() def set(self): """Set the internal flag to ``True``. All waiters are awakened. Calling `.wait` once the flag is set will not block. """ if not self._future.done(): self._future.set_result(None) def clear(self): """Reset the internal flag to ``False``. Calls to `.wait` will block until `.set` is called. """ if self._future.done(): self._future = Future() def wait(self, timeout=None): """Block until the internal flag is true. Returns a Future, which raises `tornado.gen.TimeoutError` after a timeout. """ if timeout is None: return self._future else: return gen.with_timeout(timeout, self._future)
class Event(object): """An event blocks coroutines until its internal flag is set to True. Similar to `threading.Event`. """ def __init__(self): self._future = Future() def __repr__(self): return '<%s %s>' % (self.__class__.__name__, 'set' if self.is_set() else 'clear') def is_set(self): """Return ``True`` if the internal flag is true.""" return self._future.done() def set(self): """Set the internal flag to ``True``. All waiters are awakened. Calling `.wait` once the flag is set will not block. """ if not self._future.done(): self._future.set_result(None) def clear(self): """Reset the internal flag to ``False``. Calls to `.wait` will block until `.set` is called. """ if self._future.done(): self._future = Future() def wait(self, timeout=None): """Block until the internal flag is true. Returns a Future, which raises `tornado.gen.TimeoutError` after a timeout. """ if timeout is None: return self._future else: return gen.with_timeout(timeout, self._future)
class KernelGatewayWSClient(LoggingConfigurable): """Proxy web socket connection to a kernel gateway.""" def __init__(self): self.ws = None self.ws_future = Future() @gen.coroutine def _connect(self, kernel_id): ws_url = url_path_join(KG_URL.replace("http", "ws"), "/api/kernels", url_escape(kernel_id), "channels") self.log.info("Connecting to {}".format(ws_url)) request = HTTPRequest(ws_url, headers=KG_HEADERS, validate_cert=VALIDATE_KG_CERT) self.ws_future = websocket_connect(request) self.ws = yield self.ws_future # TODO: handle connection errors/timeout def _disconnect(self): if self.ws is not None: # Close connection self.ws.close() elif not self.ws_future.done(): # Cancel pending connection self.ws_future.cancel() @gen.coroutine def _read_messages(self, callback): """Read messages from server.""" while True: message = yield self.ws.read_message() if message is None: break # TODO: handle socket close callback(message) def on_open(self, kernel_id, message_callback, **kwargs): """Web socket connection open.""" self._connect(kernel_id) loop = IOLoop.current() loop.add_future(self.ws_future, lambda future: self._read_messages(message_callback)) def on_message(self, message): """Send message to server.""" if self.ws is None: loop = IOLoop.current() loop.add_future(self.ws_future, lambda future: self._write_message(message)) else: self._write_message(message) def _write_message(self, message): """Send message to server.""" self.ws.write_message(message) def on_close(self): """Web socket closed event.""" self._disconnect()
def test_notify_waiter_only_correct_brewhouse(self): brewhouse1 = Brewhouse.objects.create(name="Foo") brewhouse2 = Brewhouse.objects.create(name="Baz") recipe = Recipe.objects.create(name="Bar") instance = RecipeInstance.objects.create(recipe=recipe) future1 = Future() future2 = Future() recipe_instance.RecipeInstanceHandler.waiters[brewhouse1.pk] = [future1] recipe_instance.RecipeInstanceHandler.waiters[brewhouse2.pk] = [future2] recipe_instance.RecipeInstanceHandler.notify(brewhouse1.pk, instance.pk) self.assertEquals(future1.result(), {'recipe_instance': instance.pk}) self.assertFalse(future2.done())
def warmup(self, managed_client): exc = managed_client.get_running_stats(self, 'warmup_exception') if exc is not None: raise exc warmup_future = managed_client.get_running_stats(self, 'warmup_future') if warmup_future is None: warmup_future = Future() managed_client.set_running_stats(self, 'warmup_future', warmup_future) if self.warmup_code == "": warmup_future.set_result("") else: app_log.debug("Running warmup code: %s", self.warmup_code) with (yield managed_client.lock.acquire()): try: yield managed_client.execute_code(self.warmup_code) warmup_future.done() except Exception as exc: app_log.exception(exc) managed_client.set_running_stats(self, 'warmup_exception', exc) raise exc return warmup_future
class NetConnection(LineReceived): # main implementation def __init__(self, stream, timeout=None): self.stream = stream self.timeout = timeout self._finish_future = Future() def read_response(self, delegate): return self._read_message(delegate) @gen.coroutine def _read_message(self, delegate): try: while True: message_future = self.stream.read_until(b"\r\n") if self.timeout is None: message_data = yield message_future else: try: message_data = yield gen.with_timeout( self.stream.io_loop.time() + self.timeout, message_future, io_loop=self.stream.io_loop, quiet_exceptions=iostream.StreamClosedError) except gen.TimeoutError: self.close() raise gen.Return(False) with _ExceptionLoggingContext(app_log): data = delegate.data_received(message_data) self.stream.write(data) finally: self._clear_callbacks() def close(self): if self.stream is not None: self.stream.close() self._clear_callbacks() if not self._finish_future.done(): self._finish_future.set_result(None) def _clear_callbacks(self): """Clears the callback attributes. This allows the request handler to be garbage collected more quickly in CPython by breaking up reference cycles. """ self._write_callback = None self._write_future = None self._close_callback = None if self.stream is not None: self.stream.set_close_callback(None)
def test_await_future(self): f = Future() tf = TornadoFuture() def finish_later(): time.sleep(0.1) f.set_result('future') tf.set_result('tornado') Thread(target=finish_later).start() assert self.client.wait([f, tf]) assert f.done() assert tf.done() assert f.result() == 'future' assert tf.result() == 'tornado'
def test_await_future(self): f = Future() tf = TornadoFuture() def finish_later(): time.sleep(0.1) f.set_result('future') tf.set_result('tornado') Thread(target=finish_later).start() assert self.client.wait([f, tf]) assert f.done() assert tf.done() assert f.result() == 'future' assert tf.result() == 'tornado'
def test_notify_waiter_only_correct_brewhouse(self): brewhouse1 = Brewhouse.objects.create(name="Foo") brewhouse2 = Brewhouse.objects.create(name="Baz") recipe = Recipe.objects.create(name="Bar") instance = RecipeInstance.objects.create(recipe=recipe) future1 = Future() future2 = Future() recipe_instance.RecipeInstanceHandler.waiters[brewhouse1.pk] = [ future1 ] recipe_instance.RecipeInstanceHandler.waiters[brewhouse2.pk] = [ future2 ] recipe_instance.RecipeInstanceHandler.notify(brewhouse1.pk, instance.pk) self.assertEquals(future1.result(), {'recipe_instance': instance.pk}) self.assertFalse(future2.done())
def generate(): decode = decode_unicode if self.encoding is None: decode = False if decode: decoder = codecs.getincrementaldecoder( self.encoding)(errors='replace') if self.raw.stream: content_remain = {'': ''} while content_remain: future = Future() def callback(status): chunk = self.raw.body.getvalue() self.raw.body.truncate(0) self.raw.body.seek(0) if decode: chunk = decoder.decode(chunk) if not status: content_remain.clear() future.set_result(chunk) self.raw.connection.read_stream_body(self.raw, chunk_size, callback=callback) yield future while not future.done(): yield future else: if self.raw.body: self.raw.body.seek(0) while True: chunk = self.raw.body.read(chunk_size) if decode: chunk = decoder.decode(chunk) if not chunk: break else: yield chunk self._content_consumed = True
def generate(): decode = decode_unicode if self.encoding is None: decode = False if decode: decoder = codecs.getincrementaldecoder( self.encoding)(errors='replace') if self.raw.stream: for chunk in self.raw.connection.iter_read_body( self.raw, chunk_size): future = Future() def callback(status): # this will reraise exception v = chunk.result() if decode: v = decoder.decode(v) future.set_result(v) chunk.add_done_callback(callback) yield future while not future.done(): yield future else: if self.raw.body: self.raw.body.seek(0) while True: chunk = self.raw.body.read(chunk_size) if decode: chunk = decoder.decode(chunk) if not chunk: break else: yield chunk self._content_consumed = True
class Runner(object): """Internal implementation of `tornado.gen.coroutine`. Maintains information about pending callbacks and their results. The results of the generator are stored in ``result_future`` (a `.Future`) """ def __init__( self, gen: "Generator[_Yieldable, Any, _T]", result_future: "Future[_T]", first_yielded: _Yieldable, ) -> None: self.gen = gen self.result_future = result_future self.future = _null_future # type: Union[None, Future] self.running = False self.finished = False self.io_loop = IOLoop.current() if self.handle_yield(first_yielded): gen = result_future = first_yielded = None # type: ignore self.run() def run(self) -> None: """Starts or resumes the generator, running until it reaches a yield point that is not ready. """ if self.running or self.finished: return try: self.running = True while True: future = self.future if future is None: raise Exception("No pending future") if not future.done(): return self.future = None try: exc_info = None try: value = future.result() except Exception: exc_info = sys.exc_info() future = None if exc_info is not None: try: yielded = self.gen.throw(*exc_info) # type: ignore finally: # Break up a reference to itself # for faster GC on CPython. exc_info = None else: yielded = self.gen.send(value) except (StopIteration, Return) as e: self.finished = True self.future = _null_future future_set_result_unless_cancelled( self.result_future, _value_from_stopiteration(e)) self.result_future = None # type: ignore return except Exception: self.finished = True self.future = _null_future future_set_exc_info(self.result_future, sys.exc_info()) self.result_future = None # type: ignore return if not self.handle_yield(yielded): return yielded = None finally: self.running = False def handle_yield(self, yielded: _Yieldable) -> bool: try: self.future = convert_yielded(yielded) except BadYieldError: self.future = Future() future_set_exc_info(self.future, sys.exc_info()) if self.future is moment: self.io_loop.add_callback(self.run) return False elif self.future is None: raise Exception("no pending future") elif not self.future.done(): def inner(f: Any) -> None: # Break a reference cycle to speed GC. f = None # noqa: F841 self.run() self.io_loop.add_future(self.future, inner) return False return True def handle_exception(self, typ: Type[Exception], value: Exception, tb: types.TracebackType) -> bool: if not self.running and not self.finished: self.future = Future() future_set_exc_info(self.future, (typ, value, tb)) self.run() return True else: return False
class Runner(object): """Internal implementation of `tornado.gen.coroutine`. Maintains information about pending callbacks and their results. The results of the generator are stored in ``result_future`` (a `.Future`) """ def __init__( self, gen: "Generator[_Yieldable, Any, _T]", result_future: "Future[_T]", first_yielded: _Yieldable, ) -> None: self.gen = gen self.result_future = result_future self.future = _null_future # type: Union[None, Future] self.running = False self.finished = False self.io_loop = IOLoop.current() if self.handle_yield(first_yielded): gen = result_future = first_yielded = None # type: ignore self.run() def run(self) -> None: """Starts or resumes the generator, running until it reaches a yield point that is not ready. """ if self.running or self.finished: return try: self.running = True while True: future = self.future if future is None: raise Exception("No pending future") if not future.done(): return self.future = None try: exc_info = None try: value = future.result() except Exception: exc_info = sys.exc_info() future = None if exc_info is not None: try: yielded = self.gen.throw(*exc_info) # type: ignore finally: # Break up a reference to itself # for faster GC on CPython. exc_info = None else: yielded = self.gen.send(value) except (StopIteration, Return) as e: self.finished = True self.future = _null_future future_set_result_unless_cancelled( self.result_future, _value_from_stopiteration(e) ) self.result_future = None # type: ignore return except Exception: self.finished = True self.future = _null_future future_set_exc_info(self.result_future, sys.exc_info()) self.result_future = None # type: ignore return if not self.handle_yield(yielded): return yielded = None finally: self.running = False def handle_yield(self, yielded: _Yieldable) -> bool: try: self.future = convert_yielded(yielded) except BadYieldError: self.future = Future() future_set_exc_info(self.future, sys.exc_info()) if self.future is moment: self.io_loop.add_callback(self.run) return False elif self.future is None: raise Exception("no pending future") elif not self.future.done(): def inner(f: Any) -> None: # Break a reference cycle to speed GC. f = None # noqa: F841 self.run() self.io_loop.add_future(self.future, inner) return False return True def handle_exception( self, typ: Type[Exception], value: Exception, tb: types.TracebackType ) -> bool: if not self.running and not self.finished: self.future = Future() future_set_exc_info(self.future, (typ, value, tb)) self.run() return True else: return False
class WebSocketClientConnection(simple_httpclient._HTTPConnection): """WebSocket client connection.""" def __init__(self, io_loop, request): self.connect_future = Future() self.read_future = None self.read_queue = collections.deque() self.key = base64.b64encode(os.urandom(16)) scheme, sep, rest = request.url.partition(':') scheme = {'ws': 'http', 'wss': 'https'}[scheme] request.url = scheme + sep + rest request.headers.update({ 'Upgrade': 'websocket', 'Connection': 'Upgrade', 'Sec-WebSocket-Key': self.key, 'Sec-WebSocket-Version': '13', }) super(WebSocketClientConnection, self).__init__( io_loop, None, request, lambda: None, self._on_http_response, 104857600, Resolver(io_loop=io_loop)) def _on_close(self): self.on_message(None) def _on_http_response(self, response): if not self.connect_future.done(): if response.error: self.connect_future.set_exception(response.error) else: self.connect_future.set_exception(WebSocketError( "Non-websocket response")) def _handle_1xx(self, code): assert code == 101 assert self.headers['Upgrade'].lower() == 'websocket' assert self.headers['Connection'].lower() == 'upgrade' accept = WebSocketProtocol13.compute_accept_value(self.key) assert self.headers['Sec-Websocket-Accept'] == accept self.protocol = WebSocketProtocol13(self, mask_outgoing=True) self.protocol._receive_frame() if self._timeout is not None: self.io_loop.remove_timeout(self._timeout) self._timeout = None self.connect_future.set_result(self) def write_message(self, message, binary=False): """Sends a message to the WebSocket server.""" self.protocol.write_message(message, binary) def read_message(self, callback=None): """Reads a message from the WebSocket server. Returns a future whose result is the message, or None if the connection is closed. If a callback argument is given it will be called with the future when it is ready. """ assert self.read_future is None future = Future() if self.read_queue: future.set_result(self.read_queue.popleft()) else: self.read_future = future if callback is not None: self.io_loop.add_future(future, callback) return future def on_message(self, message): if self.read_future is not None: self.read_future.set_result(message) self.read_future = None else: self.read_queue.append(message) def on_pong(self, data): pass
class WebSocketClientConnection(simple_httpclient._HTTPConnection): """WebSocket client connection. This class should not be instantiated directly; use the `websocket_connect` function instead. """ def __init__(self, request, on_message_callback=None, compression_options=None, ping_interval=None, ping_timeout=None, max_message_size=None): self.compression_options = compression_options self.connect_future = Future() self.protocol = None self.read_future = None self.read_queue = collections.deque() self.key = base64.b64encode(os.urandom(16)) self._on_message_callback = on_message_callback self.close_code = self.close_reason = None self.ping_interval = ping_interval self.ping_timeout = ping_timeout self.max_message_size = max_message_size scheme, sep, rest = request.url.partition(':') scheme = {'ws': 'http', 'wss': 'https'}[scheme] request.url = scheme + sep + rest request.headers.update({ 'Upgrade': 'websocket', 'Connection': 'Upgrade', 'Sec-WebSocket-Key': self.key, 'Sec-WebSocket-Version': '13', }) if self.compression_options is not None: # Always offer to let the server set our max_wbits (and even though # we don't offer it, we will accept a client_no_context_takeover # from the server). # TODO: set server parameters for deflate extension # if requested in self.compression_options. request.headers['Sec-WebSocket-Extensions'] = ( 'permessage-deflate; client_max_window_bits') self.tcp_client = TCPClient() super(WebSocketClientConnection, self).__init__( None, request, lambda: None, self._on_http_response, 104857600, self.tcp_client, 65536, 104857600) def close(self, code=None, reason=None): """Closes the websocket connection. ``code`` and ``reason`` are documented under `WebSocketHandler.close`. .. versionadded:: 3.2 .. versionchanged:: 4.0 Added the ``code`` and ``reason`` arguments. """ if self.protocol is not None: self.protocol.close(code, reason) self.protocol = None def on_connection_close(self): if not self.connect_future.done(): self.connect_future.set_exception(StreamClosedError()) self.on_message(None) self.tcp_client.close() super(WebSocketClientConnection, self).on_connection_close() def _on_http_response(self, response): if not self.connect_future.done(): if response.error: self.connect_future.set_exception(response.error) else: self.connect_future.set_exception(WebSocketError( "Non-websocket response")) def headers_received(self, start_line, headers): if start_line.code != 101: return super(WebSocketClientConnection, self).headers_received( start_line, headers) self.headers = headers self.protocol = self.get_websocket_protocol() self.protocol._process_server_headers(self.key, self.headers) self.protocol.start_pinging() self.protocol._receive_frame() if self._timeout is not None: self.io_loop.remove_timeout(self._timeout) self._timeout = None self.stream = self.connection.detach() self.stream.set_close_callback(self.on_connection_close) # Once we've taken over the connection, clear the final callback # we set on the http request. This deactivates the error handling # in simple_httpclient that would otherwise interfere with our # ability to see exceptions. self.final_callback = None future_set_result_unless_cancelled(self.connect_future, self) def write_message(self, message, binary=False): """Sends a message to the WebSocket server.""" return self.protocol.write_message(message, binary) def read_message(self, callback=None): """Reads a message from the WebSocket server. If on_message_callback was specified at WebSocket initialization, this function will never return messages Returns a future whose result is the message, or None if the connection is closed. If a callback argument is given it will be called with the future when it is ready. """ assert self.read_future is None future = Future() if self.read_queue: future_set_result_unless_cancelled(future, self.read_queue.popleft()) else: self.read_future = future if callback is not None: self.io_loop.add_future(future, callback) return future def on_message(self, message): if self._on_message_callback: self._on_message_callback(message) elif self.read_future is not None: future_set_result_unless_cancelled(self.read_future, message) self.read_future = None else: self.read_queue.append(message) def on_pong(self, data): pass def on_ping(self, data): pass def get_websocket_protocol(self): return WebSocketProtocol13(self, mask_outgoing=True, compression_options=self.compression_options)
class WaitIterator(object): """Provides an iterator to yield the results of futures as they finish. Yielding a set of futures like this: ``results = yield [future1, future2]`` pauses the coroutine until both ``future1`` and ``future2`` return, and then restarts the coroutine with the results of both futures. If either future is an exception, the expression will raise that exception and all the results will be lost. If you need to get the result of each future as soon as possible, or if you need the result of some futures even if others produce errors, you can use ``WaitIterator``:: wait_iterator = gen.WaitIterator(future1, future2) while not wait_iterator.done(): try: result = yield wait_iterator.next() except Exception as e: print("Error {} from {}".format(e, wait_iterator.current_future)) else: print("Result {} received from {} at {}".format( result, wait_iterator.current_future, wait_iterator.current_index)) Because results are returned as soon as they are available the output from the iterator *will not be in the same order as the input arguments*. If you need to know which future produced the current result, you can use the attributes ``WaitIterator.current_future``, or ``WaitIterator.current_index`` to get the index of the future from the input list. (if keyword arguments were used in the construction of the `WaitIterator`, ``current_index`` will use the corresponding keyword). On Python 3.5, `WaitIterator` implements the async iterator protocol, so it can be used with the ``async for`` statement (note that in this version the entire iteration is aborted if any value raises an exception, while the previous example can continue past individual errors):: async for result in gen.WaitIterator(future1, future2): print("Result {} received from {} at {}".format( result, wait_iterator.current_future, wait_iterator.current_index)) .. versionadded:: 4.1 .. versionchanged:: 4.3 Added ``async for`` support in Python 3.5. """ _unfinished = {} # type: Dict[Future, Union[int, str]] def __init__(self, *args: Future, **kwargs: Future) -> None: if args and kwargs: raise ValueError("You must provide args or kwargs, not both") if kwargs: self._unfinished = dict((f, k) for (k, f) in kwargs.items()) futures = list(kwargs.values()) # type: Sequence[Future] else: self._unfinished = dict((f, i) for (i, f) in enumerate(args)) futures = args self._finished = collections.deque() # type: Deque[Future] self.current_index = None # type: Optional[Union[str, int]] self.current_future = None # type: Optional[Future] self._running_future = None # type: Optional[Future] for future in futures: future_add_done_callback(future, self._done_callback) def done(self) -> bool: """Returns True if this iterator has no more results.""" if self._finished or self._unfinished: return False # Clear the 'current' values when iteration is done. self.current_index = self.current_future = None return True def next(self) -> Future: """Returns a `.Future` that will yield the next available result. Note that this `.Future` will not be the same object as any of the inputs. """ self._running_future = Future() if self._finished: self._return_result(self._finished.popleft()) return self._running_future def _done_callback(self, done: Future) -> None: if self._running_future and not self._running_future.done(): self._return_result(done) else: self._finished.append(done) def _return_result(self, done: Future) -> None: """Called set the returned future's state that of the future we yielded, and set the current future for the iterator. """ if self._running_future is None: raise Exception("no future is running") chain_future(done, self._running_future) self.current_future = done self.current_index = self._unfinished.pop(done) def __aiter__(self) -> typing.AsyncIterator: return self def __anext__(self) -> Future: if self.done(): # Lookup by name to silence pyflakes on older versions. raise getattr(builtins, "StopAsyncIteration")() return self.next()
class KernelGatewayWSClient(LoggingConfigurable): """Proxy web socket connection to a kernel/enterprise gateway.""" def __init__(self, **kwargs): super(KernelGatewayWSClient, self).__init__(**kwargs) self.kernel_id = None self.ws = None self.ws_future = Future() self.ws_future_cancelled = False @gen.coroutine def _connect(self, kernel_id): self.kernel_id = kernel_id ws_url = url_path_join( os.getenv('KG_WS_URL', KG_URL.replace('http', 'ws')), '/api/kernels', url_escape(kernel_id), 'channels') self.log.info('Connecting to {}'.format(ws_url)) parameters = { "headers": KG_HEADERS, "validate_cert": VALIDATE_KG_CERT, "connect_timeout": KG_CONNECT_TIMEOUT, "request_timeout": KG_REQUEST_TIMEOUT } if KG_HTTP_USER: parameters["auth_username"] = KG_HTTP_USER if KG_HTTP_PASS: parameters["auth_password"] = KG_HTTP_PASS if KG_CLIENT_KEY: parameters["client_key"] = KG_CLIENT_KEY parameters["client_cert"] = KG_CLIENT_CERT if KG_CLIENT_CA: parameters["ca_certs"] = KG_CLIENT_CA request = HTTPRequest(ws_url, **parameters) self.ws_future = websocket_connect(request) self.ws_future.add_done_callback(self._connection_done) def _connection_done(self, fut): if not self.ws_future_cancelled: # prevent concurrent.futures._base.CancelledError self.ws = fut.result() self.log.debug("Connection is ready: ws: {}".format(self.ws)) else: self.log.warning( "Websocket connection has been cancelled via client disconnect before its establishment. " "Kernel with ID '{}' may not be terminated on Gateway: {}". format(self.kernel_id, KG_URL)) def _disconnect(self): if self.ws is not None: # Close connection self.ws.close() elif not self.ws_future.done(): # Cancel pending connection. Since future.cancel() is a noop on tornado, we'll track cancellation locally self.ws_future.cancel() self.ws_future_cancelled = True self.log.debug("_disconnect: ws_future_cancelled: {}".format( self.ws_future_cancelled)) @gen.coroutine def _read_messages(self, callback): """Read messages from gateway server.""" while True: message = None if not self.ws_future_cancelled: try: message = yield self.ws.read_message() except Exception as e: self.log.error( "Exception reading message from websocket: {}".format( e)) # , exc_info=True) if message is None: break callback( message ) # pass back to notebook client (see self.on_open and WebSocketChannelsHandler.open) else: # ws cancelled - stop reading break def on_open(self, kernel_id, message_callback, **kwargs): """Web socket connection open against gateway server.""" self._connect(kernel_id) loop = IOLoop.current() loop.add_future(self.ws_future, lambda future: self._read_messages(message_callback)) def on_message(self, message): """Send message to gateway server.""" if self.ws is None: loop = IOLoop.current() loop.add_future(self.ws_future, lambda future: self._write_message(message)) else: self._write_message(message) def _write_message(self, message): """Send message to gateway server.""" try: if not self.ws_future_cancelled: self.ws.write_message(message) except Exception as e: self.log.error("Exception writing message to websocket: {}".format( e)) # , exc_info=True) def on_close(self): """Web socket closed event.""" self._disconnect()
class Queue(object): """An unbounded, thread-safe asynchronous queue.""" __slots__ = ('_get', '_put', '_lock') # How this works: # # _get and _put are futures maintaining pointers to a linked list of # futures. The linked list is implemented as Node objects holding the # value and the next future. # # Node # +---+---+ +---+---+ E: Empty future # | 1 | F-|-->| 2 | E | F: Filled future # +---+---+ +---+---+ # ^ ^ # +---+ | +---+ | # | F-|-+ | F-|-+ # +---+ +---+ # _get _put # # When there's a put, we fill the current empty future with a Node # containing the value and a pointer to the next, newly created empty # future. # # +---+---+ +---+---+ +---+---+ # | 1 | F-|-->| 2 | F-|-->| 3 | E | # +---+---+ +---+---+ +---+---+ # ^ ^ # +---+ | +---+ | # | F-|-+ | F-|-+ # +---+ +---+ # _get _put # # When there's a get, we read the value from the current Node, and move # _get to the next future. # # +---+---+ +---+---+ # | 2 | F-|-->| 3 | E | # +---+---+ +---+---+ # ^ ^ # +---+ | +---+ | # | F-|-+ | F-|-+ # +---+ +---+ # _get _put def __init__(self): self._lock = threading.Lock() # Space for the next Node. hole = Future() # Pointer to the Future that will contain the next Node. self._get = Future() self._get.set_result(hole) # Pointer to the next empty Future that should be filled with a Node. self._put = Future() self._put.set_result(hole) def put(self, value): """Puts an item into the queue. Returns a Future that resolves to None once the value has been accepted by the queue. """ io_loop = IOLoop.current() new_hole = Future() new_put = Future() new_put.set_result(new_hole) with self._lock: self._put, put = new_put, self._put answer = Future() def _on_put(future): if future.exception(): # pragma: no cover (never happens) return answer.set_exc_info(future.exc_info()) old_hole = put.result() old_hole.set_result(Node(value, new_hole)) answer.set_result(None) io_loop.add_future(put, _on_put) return answer def get_nowait(self): """Returns a value from the queue without waiting. Raises ``QueueEmpty`` if no values are available right now. """ new_get = Future() with self._lock: if not self._get.done(): raise QueueEmpty get, self._get = self._get, new_get hole = get.result() if not hole.done(): # Restore the unfinished hole. new_get.set_result(hole) raise QueueEmpty value, new_hole = hole.result() new_get.set_result(new_hole) return value def get(self): """Gets the next item from the queue. Returns a Future that resolves to the next item once it is available. """ io_loop = IOLoop.current() new_get = Future() with self._lock: get, self._get = self._get, new_get answer = Future() def _on_node(future): if future.exception(): # pragma: no cover (never happens) return answer.set_exc_info(future.exc_info()) value, new_hole = future.result() new_get.set_result(new_hole) answer.set_result(value) def _on_get(future): if future.exception(): # pragma: no cover (never happens) return answer.set_exc_info(future.exc_info()) hole = future.result() io_loop.add_future(hole, _on_node) io_loop.add_future(get, _on_get) return answer
class Event(object): """一个阻塞协程的事件直到它的内部标识设置为True. 类似于 `threading.Event`. 协程可以等待一个事件被设置. 一旦它被设置, 调用 ``yield event.wait()`` 将不会被阻塞除非该事件已经被清除: .. testcode:: from tornado import gen from tornado.ioloop import IOLoop from tornado.locks import Event event = Event() @gen.coroutine def waiter(): print("Waiting for event") yield event.wait() print("Not waiting this time") yield event.wait() print("Done") @gen.coroutine def setter(): print("About to set the event") event.set() @gen.coroutine def runner(): yield [waiter(), setter()] IOLoop.current().run_sync(runner) .. testoutput:: Waiting for event About to set the event Not waiting this time Done """ def __init__(self): self._future = Future() def __repr__(self): return '<%s %s>' % ( self.__class__.__name__, 'set' if self.is_set() else 'clear') def is_set(self): """如果内部标识是true将返回 ``True`` .""" return self._future.done() def set(self): """设置内部标识为 ``True``. 所有的等待者(waiters)都被唤醒. 一旦该标识被设置调用 `.wait` 将不会阻塞. """ if not self._future.done(): self._future.set_result(None) def clear(self): """重置内部标识为 ``False``. 调用 `.wait` 将阻塞直到 `.set` 被调用. """ if self._future.done(): self._future = Future() def wait(self, timeout=None): """阻塞直到内部标识为true. 返回一个Future对象, 在超时之后会抛出一个 `tornado.gen.TimeoutError` 异常. """ if timeout is None: return self._future else: return gen.with_timeout(timeout, self._future)
class TestTornadoHeroku(AsyncHTTPTestCase): def get_app(self): con = self.io_loop.run_sync(connect_to_amqp) return tornado.web.Application([(r"/heroku/.*", HerokuHandler, dict(amqp_con=con))]) def setUp(self): super(TestTornadoHeroku, self).setUp() def test_h2l_split_error(self): payload = b"50 <40>1 2017-06-14T13:52:29+00:00 host app web.3" \ b" - State changed from starting to up\n119 <40>1 " \ b"2017-06-14T13:53:26+00:00 host app web.3 - " \ b"Starting process with command `bundle exec rackup config.ru -p 24405`" response = self.fetch('/heroku/v1/toto', method='POST', body=payload) self.assertEqual(response.code, 500) self.assertEqual(len(response.body), 0) @gen_test def test_h2l_heroku_push_to_amqp_success(self): """ Message is forwarded to amqp without errors :return: """ consumer = AMQPConnection() yield consumer.connect(self.io_loop) self.futureMsg = Future() yield consumer.subscribe("heroku.v1.integration.toto", "heroku_queue", self.on_message) payload = b"123 <40>1 2017-06-21T17:02:55+00:00 host ponzi web.1 - " \ b"Lorem ipsum dolor sit amet, consecteteur adipiscing elit b'quis' b'ad'.\n" response = self.http_client.fetch(self.get_url('/heroku/v1/integration/toto'), method='POST', body=payload) res = yield self.futureMsg json_res = json.loads(res.decode('utf-8')) self.assertEqual(json_res['app'], 'toto') self.assertEqual(json_res['env'], 'integration') self.assertEqual(json_res['type'], 'heroku') self.assertEqual(json_res['http_content_length'], 122) self.assertEqual(json_res['parser_ver'], 'v1') self.assertEqual(json_res['message'], '<40>1 2017-06-21T17:02:55+00:00 host ponzi web.1' ' - Lorem ipsum dolor sit amet, consecteteur' ' adipiscing elit b\'quis\' b\'ad\'.') value = yield response self.assertEqual(value.code, 200) self.assertEqual(len(value.body), 0) yield consumer.disconnect() def on_message(self, channel, basic_deliver, properties, body): if not self.futureMsg.done(): self.futureMsg.set_result(body) channel.basic_ack(basic_deliver.delivery_tag) @gen_test def test_h2l_heroku_push_to_amqp_success_no_routing(self): """ Message is forwarded to amqp without errors :return: """ payload = b"123 <40>1 2017-06-21T17:02:55+00:00 host ponzi web.1 - " \ b"Lorem ipsum dolor sit amet, consecteteur adipiscing elit b'quis' b'ad'.\n" response = self.http_client.fetch(self.get_url('/heroku/v1/integration/toto2'), method='POST', body=payload) value = yield response self.assertEqual(value.code, 200) self.assertEqual(len(value.body), 0)
class HTTP1Connection(httputil.HTTPConnection): """Implements the HTTP/1.x protocol. This class can be on its own for clients, or via `HTTP1ServerConnection` for servers. """ def __init__(self, stream, is_client, params=None, context=None): """ :arg stream: an `.IOStream` :arg bool is_client: client or server :arg params: a `.HTTP1ConnectionParameters` instance or ``None`` :arg context: an opaque application-defined object that can be accessed as ``connection.context``. """ self.is_client = is_client self.stream = stream if params is None: params = HTTP1ConnectionParameters() self.params = params self.context = context self.no_keep_alive = params.no_keep_alive # The body limits can be altered by the delegate, so save them # here instead of just referencing self.params later. self._max_body_size = (self.params.max_body_size or self.stream.max_buffer_size) self._body_timeout = self.params.body_timeout # _write_finished is set to True when finish() has been called, # i.e. there will be no more data sent. Data may still be in the # stream's write buffer. self._write_finished = False # True when we have read the entire incoming body. self._read_finished = False # _finish_future resolves when all data has been written and flushed # to the IOStream. self._finish_future = Future() # If true, the connection should be closed after this request # (after the response has been written in the server side, # and after it has been read in the client) self._disconnect_on_finish = False self._clear_callbacks() # Save the start lines after we read or write them; they # affect later processing (e.g. 304 responses and HEAD methods # have content-length but no bodies) self._request_start_line = None self._response_start_line = None self._request_headers = None # True if we are writing output with chunked encoding. self._chunking_output = None # While reading a body with a content-length, this is the # amount left to read. self._expected_content_remaining = None # A Future for our outgoing writes, returned by IOStream.write. self._pending_write = None def read_response(self, delegate): """Read a single HTTP response. Typical client-mode usage is to write a request using `write_headers`, `write`, and `finish`, and then call ``read_response``. :arg delegate: a `.HTTPMessageDelegate` Returns a `.Future` that resolves to None after the full response has been read. """ if self.params.decompress: delegate = _GzipMessageDelegate(delegate, self.params.chunk_size) return self._read_message(delegate) @gen.coroutine def _read_message(self, delegate): need_delegate_close = False try: # 消息头与消息体之间由一个空行分开 header_future = self.stream.read_until_regex( b"\r?\n\r?\n", max_bytes=self.params.max_header_size) if self.params.header_timeout is None: header_data = yield header_future else: try: header_data = yield gen.with_timeout( self.stream.io_loop.time() + self.params.header_timeout, header_future, io_loop=self.stream.io_loop) except gen.TimeoutError: self.close() raise gen.Return(False) # 解析消息头,分离头字段和首行(request-line/status-line) start_line, headers = self._parse_headers(header_data) # 作为 client 解析的是 server 的 response,作为 server 解析的是 client 的 request。 # response 与 request 的 start_line(status-line/request-line) 的字段内容不同: # 1. response's status-line: HTTP-Version SP Status-Code SP Reason-Phrase CRLF # 2. request's request-line:Method SP Request-URI SP HTTP-Version CRLF # start_line 的值是一个 namedtuple。 if self.is_client: start_line = httputil.parse_response_start_line(start_line) self._response_start_line = start_line else: start_line = httputil.parse_request_start_line(start_line) self._request_start_line = start_line self._request_headers = headers # 非 keep-alive 的请求或响应处理完成后要关闭连接。 self._disconnect_on_finish = not self._can_keep_alive( start_line, headers) need_delegate_close = True with _ExceptionLoggingContext(app_log): header_future = delegate.headers_received(start_line, headers) if header_future is not None: # 如果 header_future 是一个 `Future` 实例,则要等到完成才读取 body。 yield header_future # websocket ??? if self.stream is None: # We've been detached. need_delegate_close = False raise gen.Return(False) skip_body = False if self.is_client: # 作为 client 如果发起的是 HEAD 请求,那么 server response 应该无消息体 if (self._request_start_line is not None and self._request_start_line.method == 'HEAD'): skip_body = True code = start_line.code if code == 304: # 304 responses may include the content-length header # but do not actually have a body. # http://tools.ietf.org/html/rfc7230#section-3.3 skip_body = True if code >= 100 and code < 200: # 1xx responses should never indicate the presence of # a body. if ('Content-Length' in headers or 'Transfer-Encoding' in headers): raise httputil.HTTPInputError( "Response code %d cannot have body" % code) # TODO: client delegates will get headers_received twice # in the case of a 100-continue. Document or change? yield self._read_message(delegate) else: # 100-continue 这个状态码是在 HTTP/1.1 中为了提高传输效率而设置的。当 # client 需要 POST 较大数据给 WebServer 时,可以在发送 HTTP 请求时带上 # Expect: 100-continue,WebServer 如果接受这个请求则应答一个 # ``HTTP/1.1 100 (Continue)``,那么 client 就继续传输 request body, # 否则应答 ``HTTP/1.1 417 Expectation Failed`` client 就放弃传输剩余 # 的数据。(注:Expect 头部域,用于指出客户端要求的特殊服务器行为采用扩展语法 # 定义,以方便扩展。) if (headers.get("Expect") == "100-continue" and not self._write_finished): self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n") if not skip_body: body_future = self._read_body( start_line.code if self.is_client else 0, headers, delegate) if body_future is not None: if self._body_timeout is None: yield body_future else: try: yield gen.with_timeout( self.stream.io_loop.time() + self._body_timeout, body_future, self.stream.io_loop) except gen.TimeoutError: gen_log.info("Timeout reading body from %s", self.context) self.stream.close() raise gen.Return(False) self._read_finished = True # 对 client mode ,response 解析完成就调用 HTTPMessageDelegate.finish() 方法是合适的; # 对 server mode ,_write_finished 表示 response 是否发送完成,未完成前调用 # HTTPMessageDelegate.finish() 方法让 delegate 执行请求响应; if not self._write_finished or self.is_client: need_delegate_close = False with _ExceptionLoggingContext(app_log): delegate.finish() # If we're waiting for the application to produce an asynchronous # response, and we're not detached, register a close callback # on the stream (we didn't need one while we were reading) # # NOTE:_finish_future resolves when all data has been written and flushed # to the IOStream. # # hold 住执行流程,直到异步响应完成,所有数据都写入 fd,才继续后续处理,通常调用方执行 `finish` 方法 # 设置 `_finish_future` 完成,详细见 `finish` 和 `_finish_request` 方法实现。 if (not self._finish_future.done() and self.stream is not None and not self.stream.closed()): self.stream.set_close_callback(self._on_connection_close) yield self._finish_future # 对于 client mode,处理完响应后如果不是 keep-alive 就断开连接。 # 对于 server mode,需要在 response 完成后才断开连接,详细见 _finish_request/finish 方法实现。 if self.is_client and self._disconnect_on_finish: self.close() if self.stream is None: raise gen.Return(False) except httputil.HTTPInputError as e: gen_log.info("Malformed HTTP message from %s: %s", self.context, e) self.close() raise gen.Return(False) finally: # 连接 “关闭” 前还没能结束处理请求(call HTTPMessageDelegate.finish()),则 # call HTTPMessageDelegate.on_connection_close() if need_delegate_close: with _ExceptionLoggingContext(app_log): delegate.on_connection_close() self._clear_callbacks() raise gen.Return(True) def _clear_callbacks(self): """Clears the callback attributes. This allows the request handler to be garbage collected more quickly in CPython by breaking up reference cycles. """ self._write_callback = None self._write_future = None self._close_callback = None if self.stream is not None: self.stream.set_close_callback(None) def set_close_callback(self, callback): """Sets a callback that will be run when the connection is closed. .. deprecated:: 4.0 Use `.HTTPMessageDelegate.on_connection_close` instead. """ self._close_callback = stack_context.wrap(callback) def _on_connection_close(self): # Note that this callback is only registered on the IOStream # when we have finished reading the request and are waiting for # the application to produce its response. if self._close_callback is not None: callback = self._close_callback self._close_callback = None callback() if not self._finish_future.done(): self._finish_future.set_result(None) self._clear_callbacks() def close(self): if self.stream is not None: self.stream.close() self._clear_callbacks() if not self._finish_future.done(): self._finish_future.set_result(None) def detach(self): """Take control of the underlying stream. Returns the underlying `.IOStream` object and stops all further HTTP processing. May only be called during `.HTTPMessageDelegate.headers_received`. Intended for implementing protocols like websockets that tunnel over an HTTP handshake. """ self._clear_callbacks() stream = self.stream self.stream = None return stream def set_body_timeout(self, timeout): """Sets the body timeout for a single request. Overrides the value from `.HTTP1ConnectionParameters`. """ self._body_timeout = timeout def set_max_body_size(self, max_body_size): """Sets the body size limit for a single request. Overrides the value from `.HTTP1ConnectionParameters`. """ self._max_body_size = max_body_size def write_headers(self, start_line, headers, chunk=None, callback=None): """Implements `.HTTPConnection.write_headers`.""" if self.is_client: self._request_start_line = start_line # Client requests with a non-empty body must have either a # Content-Length or a Transfer-Encoding. # 不检查是否 Http/1.0 是不完备的。 self._chunking_output = ( start_line.method in ('POST', 'PUT', 'PATCH') and 'Content-Length' not in headers and 'Transfer-Encoding' not in headers) else: self._response_start_line = start_line # 对于 HTTP/1.0 ``self._chunking_output=False``,不支持分块传输编码。 self._chunking_output = ( # TODO: should this use # self._request_start_line.version or # start_line.version? self._request_start_line.version == 'HTTP/1.1' and # 304 responses have no body (not even a zero-length body), and so # should not have either Content-Length or Transfer-Encoding. # headers. start_line.code != 304 and # No need to chunk the output if a Content-Length is specified. 'Content-Length' not in headers and # Applications are discouraged from touching Transfer-Encoding, # but if they do, leave it alone. 'Transfer-Encoding' not in headers) # If a 1.0 client asked for keep-alive, add the header. # HTTP/1.1 默认就是持久化连接,不需要单独指定。 # 假设客户端请求使用 HTTP/1.0 和 `Connection:Keep-Alive`,服务端响应时没有指定 # `Content-Length` (比如在 handler 中多次调用 flush 方法),那么响应数据就无法 # 判断边界,代码中应该对这个条件做特别处理。 if (self._request_start_line.version == 'HTTP/1.0' and (self._request_headers.get('Connection', '').lower() == 'keep-alive')): headers['Connection'] = 'Keep-Alive' if self._chunking_output: headers['Transfer-Encoding'] = 'chunked' # 服务端响应 `HEAD` 或者 304 时不需要 body 数据。 if (not self.is_client and (self._request_start_line.method == 'HEAD' or start_line.code == 304)): self._expected_content_remaining = 0 elif 'Content-Length' in headers: self._expected_content_remaining = int(headers['Content-Length']) else: self._expected_content_remaining = None lines = [utf8("%s %s %s" % start_line)] # 通过 add 添加的响应头会输出多个,比如:“Set-Cookie” 响应头。 lines.extend([utf8(n) + b": " + utf8(v) for n, v in headers.get_all()]) for line in lines: if b'\n' in line: raise ValueError('Newline in header: ' + repr(line)) future = None if self.stream.closed(): future = self._write_future = Future() future.set_exception(iostream.StreamClosedError()) else: # "写回调" 是一个实例字段 `_write_callback`,当上一次写操作还没有回调时就再次执行 # 写操作,那么上一次写操作的回调将被放弃(callback is not None) if callback is not None: self._write_callback = stack_context.wrap(callback) else: # 没有 callback 时,返回 Future(self._write_future) future = self._write_future = Future() # Headers data = b"\r\n".join(lines) + b"\r\n\r\n" # message-body if chunk: data += self._format_chunk(chunk) self._pending_write = self.stream.write(data) self._pending_write.add_done_callback(self._on_write_complete) return future def _format_chunk(self, chunk): if self._expected_content_remaining is not None: self._expected_content_remaining -= len(chunk) if self._expected_content_remaining < 0: # Close the stream now to stop further framing errors. self.stream.close() raise httputil.HTTPOutputError( "Tried to write more data than Content-Length") if self._chunking_output and chunk: # Don't write out empty chunks because that means END-OF-STREAM # with chunked encoding # # Each chunk: the number of octets of the data(hex number) + CRLF + chunk data + CRLF return utf8("%x" % len(chunk)) + b"\r\n" + chunk + b"\r\n" else: return chunk def write(self, chunk, callback=None): """Implements `.HTTPConnection.write`. For backwards compatibility is is allowed but deprecated to skip `write_headers` and instead call `write()` with a pre-encoded header block. """ future = None if self.stream.closed(): future = self._write_future = Future() self._write_future.set_exception(iostream.StreamClosedError()) else: if callback is not None: self._write_callback = stack_context.wrap(callback) else: future = self._write_future = Future() self._pending_write = self.stream.write(self._format_chunk(chunk)) self._pending_write.add_done_callback(self._on_write_complete) return future def finish(self): """Implements `.HTTPConnection.finish`.""" if (self._expected_content_remaining is not None and self._expected_content_remaining != 0 and not self.stream.closed()): self.stream.close() raise httputil.HTTPOutputError( "Tried to write %d bytes less than Content-Length" % self._expected_content_remaining) if self._chunking_output: if not self.stream.closed(): # `Transfer-Encoding:chunked`: The terminating chunk is a # regular chunk, with the exception that its length is zero. self._pending_write = self.stream.write(b"0\r\n\r\n") self._pending_write.add_done_callback(self._on_write_complete) self._write_finished = True # If the app finished the request while we're still reading, # divert any remaining data away from the delegate and # close the connection when we're done sending our response. # Closing the connection is the only way to avoid reading the # whole input body. if not self._read_finished: self._disconnect_on_finish = True # No more data is coming, so instruct TCP to send any remaining # data immediately instead of waiting for a full packet or ack. # 关闭 Nagle 算法,效果相当于让 socket 立即 flush 数据到客户端,随后将在 # `_finish_request` 中恢复 Nagle 算法。 self.stream.set_nodelay(True) if self._pending_write is None: self._finish_request(None) else: # 最后一次挂起的写操作完成后回调 `_finish_request` 方法。 self._pending_write.add_done_callback(self._finish_request) def _on_write_complete(self, future): if self._write_callback is not None: callback = self._write_callback self._write_callback = None self.stream.io_loop.add_callback(callback) if self._write_future is not None: future = self._write_future self._write_future = None future.set_result(None) def _can_keep_alive(self, start_line, headers): if self.params.no_keep_alive: return False # 在 HTTP/1.0 中没有官方的 keep-alive 操作,HTTP/1.1 所有连接都是默认持 # 久化,除非特殊声明不支持。通常是在现有协议上添加一个指数。 # client 若支持 keep-alive,会在请求头中添加:Connection: Keep-Alive, # server 响应的时候也要在响应头中增加: Connection: Keep-Alive。参见: # https://zh.wikipedia.org/wiki/HTTP%E6%8C%81%E4%B9%85%E8%BF%9E%E6%8E%A5 connection_header = headers.get("Connection") if connection_header is not None: connection_header = connection_header.lower() # 不论是否 is_client mode, start_line 中都包含 version 字段。 # ***************************** NOTE *********************************** # 1. 假设使用的是 HTTP/1.0 协议,对 is_client=True,start_line 不包含 method 字段, # 而当 response empty headers(eg.100-continue) 时,start_line.method 会报错。避 # 免这种情况只能在 is_client mode 时设置 no_keep_alive=True,自行实现客户端时要注意。 # 2. 如果 POST 数据时使用 'Connection: Keep-Alive' 和 'Transfer-Encoding: Chunked', # 这个时候由于没有 'Content-Length' 字段,所以检查不到 keep-alive 而关闭连接。需要增加: # or headers.get("Transfer-Encoding", "").lower() == "chunked" # *************************** NOTE END ********************************* if start_line.version == "HTTP/1.1": return connection_header != "close" elif ("Content-Length" in headers or start_line.method in ("HEAD", "GET")): # HTTP/1.0 要支持持久化连接需要能够知道 request body 的大小,才能区分 # 不同的 HTTP 请求。没有 "Content-Length" 字段,表示没有 request body。 return connection_header == "keep-alive" return False def _finish_request(self, future): # ``close`` 中还会执行一次,调整到后面执行更好 self._clear_callbacks() # 服务端不需要支持长连接时,执行关闭操作 if not self.is_client and self._disconnect_on_finish: self.close() return # Turn Nagle's algorithm back on, leaving the stream in its # default state for the next request. self.stream.set_nodelay(False) if not self._finish_future.done(): self._finish_future.set_result(None) def _parse_headers(self, data): # HTTP 消息包括 Request(c2s)和 Response(s2c),消息格式为: # 起始行(Request-Line/Status-Line) + 0 个或多个头域(((general-header | # (request-header | response-header) | entity-header)CRLF)) + # 头域结束行(\r\n,CRLF) + 可选的消息体(message-body),所以读取消息头时以 # r"\r\n\r\n" 作为匹配字符。 # 每个头域由一个头域名称(name) + 冒号(:) + 域值(value), 三部分组成, # name 是大小写无关的,value 前可以添加任何数量的空格符,头域可以被扩展为 # 多行,在每行开始处,使用至少一个空格或制表符。相关 RFC: # 1. Request:http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5 # 2. Response:http://www.w3.org/Protocols/rfc2616/rfc2616-sec6.html#sec6 data = native_str(data.decode('latin1')) eol = data.find("\r\n") start_line = data[:eol] try: headers = httputil.HTTPHeaders.parse(data[eol:]) except ValueError: # probably form split() if there was no ':' in the line raise httputil.HTTPInputError("Malformed HTTP headers: %r" % data[eol:100]) return start_line, headers def _read_body(self, code, headers, delegate): if "Content-Length" in headers: if "," in headers["Content-Length"]: # Proxies sometimes cause Content-Length headers to get # duplicated. If all the values are identical then we can # use them but if they differ it's an error. pieces = re.split(r',\s*', headers["Content-Length"]) if any(i != pieces[0] for i in pieces): raise httputil.HTTPInputError( "Multiple unequal Content-Lengths: %r" % headers["Content-Length"]) headers["Content-Length"] = pieces[0] content_length = int(headers["Content-Length"]) if content_length > self._max_body_size: raise httputil.HTTPInputError("Content-Length too long") else: content_length = None # 204 No Content,表示服务器已经完成了请求,但是返回的信息不包括 message-body,但是可以通过 # header fields 返回一些用于更新的元数据。 if code == 204: # This response code is not allowed to have a non-empty body, # and has an implicit length of zero instead of read-until-close. # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3 if ("Transfer-Encoding" in headers or content_length not in (None, 0)): raise httputil.HTTPInputError( "Response with code %d should not have body" % code) content_length = 0 # 持久连接: Content-Length or Transfer-Encoding if content_length is not None: return self._read_fixed_body(content_length, delegate) if headers.get("Transfer-Encoding") == "chunked": return self._read_chunked_body(delegate) # 非持久连接 if self.is_client: return self._read_body_until_close(delegate) return None @gen.coroutine def _read_fixed_body(self, content_length, delegate): while content_length > 0: body = yield self.stream.read_bytes( min(self.params.chunk_size, content_length), partial=True) content_length -= len(body) if not self._write_finished or self.is_client: with _ExceptionLoggingContext(app_log): yield gen.maybe_future(delegate.data_received(body)) @gen.coroutine def _read_chunked_body(self, delegate): # TODO: "chunk extensions" http://tools.ietf.org/html/rfc2616#section-3.6.1 # # *************************** chunk extensions ************************* # 使用分块传输编码(chunked transfer encoding)时,消息体由数量未定的块组成,并以最 # 后一个大小为 0 的块结束。 # 1. 每一个非空的块都以该块包含数据的字节数(十六进制表示)开始,跟随一个 CRLF,然后是数 # 据本身,最后跟 CRLF 结束。在一些实现中,块大小和 CRLF 之间填充有白空格(0x20)。 # 2. 最后一块由块大小(0),一些可选的填充白空格,以及 CRLF。最后一块不包含任何数据,但 # 是可以发送包含消息头字段的可选尾部(注:以下代码实现不支持可选尾部),最后以 CRLF 结尾。 # ----------------------------eg. start-------------------------------- # HTTP/1.1 200 OK\r\n # Content-Type: text/plain\r\n # Transfer-Encoding: chunked\r\n # \r\n # 25\r\n # This is the data in the first chunk\r\n # 1C\r\n # and this is the second one\r\n # 3\r\n # con\r\n # 8\r\n # sequence\r\n # 0\r\n # \r\n # ----------------------------eg. end-------------------------------- # ********************************************************************** total_size = 0 while True: chunk_len = yield self.stream.read_until(b"\r\n", max_bytes=64) chunk_len = int(chunk_len.strip(), 16) if chunk_len == 0: return total_size += chunk_len if total_size > self._max_body_size: raise httputil.HTTPInputError("chunked body too large") bytes_to_read = chunk_len while bytes_to_read: chunk = yield self.stream.read_bytes( min(bytes_to_read, self.params.chunk_size), partial=True) bytes_to_read -= len(chunk) if not self._write_finished or self.is_client: with _ExceptionLoggingContext(app_log): yield gen.maybe_future(delegate.data_received(chunk)) # chunk ends with \r\n crlf = yield self.stream.read_bytes(2) # 如果最后一个 chunk 中包含了可选的尾部,断言会失败。可选尾部由 Trailer 头域支持, # 参考:http://tools.ietf.org/html/rfc2616#section-14.40。 # 目前 tornado 中的实现不支持这个可选尾部,如果发生异常的话,可尝试判断是否是 last chunk, # 然后吞掉可选尾部。 # eg. # if bytes_to_read == 0 and crlf != b"\r\n": # yield self.stream.read_until(b"\r\n", max_bytes=self._max_body_size - total_size) # else: # assert crlf == b"\r\n" assert crlf == b"\r\n" @gen.coroutine def _read_body_until_close(self, delegate): body = yield self.stream.read_until_close() if not self._write_finished or self.is_client: with _ExceptionLoggingContext(app_log): delegate.data_received(body)
class GatewayWebSocketClient(LoggingConfigurable): """Proxy web socket connection to a kernel/enterprise gateway.""" def __init__(self, **kwargs): super(GatewayWebSocketClient, self).__init__(**kwargs) self.kernel_id = None self.ws = None self.ws_future = Future() self.ws_future_cancelled = False @gen.coroutine def _connect(self, kernel_id): self.kernel_id = kernel_id ws_url = url_path_join( GatewayClient.instance().ws_url, GatewayClient.instance().kernels_endpoint, url_escape(kernel_id), 'channels' ) self.log.info('Connecting to {}'.format(ws_url)) kwargs = {} kwargs = GatewayClient.instance().load_connection_args(**kwargs) request = HTTPRequest(ws_url, **kwargs) self.ws_future = websocket_connect(request) self.ws_future.add_done_callback(self._connection_done) def _connection_done(self, fut): if not self.ws_future_cancelled: # prevent concurrent.futures._base.CancelledError self.ws = fut.result() self.log.debug("Connection is ready: ws: {}".format(self.ws)) else: self.log.warning("Websocket connection has been cancelled via client disconnect before its establishment. " "Kernel with ID '{}' may not be terminated on GatewayClient: {}". format(self.kernel_id, GatewayClient.instance().url)) def _disconnect(self): if self.ws is not None: # Close connection self.ws.close() elif not self.ws_future.done(): # Cancel pending connection. Since future.cancel() is a noop on tornado, we'll track cancellation locally self.ws_future.cancel() self.ws_future_cancelled = True self.log.debug("_disconnect: ws_future_cancelled: {}".format(self.ws_future_cancelled)) @gen.coroutine def _read_messages(self, callback): """Read messages from gateway server.""" while True: message = None if not self.ws_future_cancelled: try: message = yield self.ws.read_message() except Exception as e: self.log.error("Exception reading message from websocket: {}".format(e)) # , exc_info=True) if message is None: break callback(message) # pass back to notebook client (see self.on_open and WebSocketChannelsHandler.open) else: # ws cancelled - stop reading break def on_open(self, kernel_id, message_callback, **kwargs): """Web socket connection open against gateway server.""" self._connect(kernel_id) loop = IOLoop.current() loop.add_future( self.ws_future, lambda future: self._read_messages(message_callback) ) def on_message(self, message): """Send message to gateway server.""" if self.ws is None: loop = IOLoop.current() loop.add_future( self.ws_future, lambda future: self._write_message(message) ) else: self._write_message(message) def _write_message(self, message): """Send message to gateway server.""" try: if not self.ws_future_cancelled: self.ws.write_message(message) except Exception as e: self.log.error("Exception writing message to websocket: {}".format(e)) # , exc_info=True) def on_close(self): """Web socket closed event.""" self._disconnect()
class DMTPConnection(httputil.HTTPConnection): """Implements the HTTP/1.x protocol. This class can be on its own for clients, or via `HTTP1ServerConnection` for servers. """ def __init__(self, stream, is_client, params=None, context=None): """ :arg stream: an `.IOStream` :arg bool is_client: client or server :arg params: a `.HTTP1ConnectionParameters` instance or ``None`` :arg context: an opaque application-defined object that can be accessed as ``connection.context``. """ self.is_client = is_client self.stream = stream if params is None: params = HTTP1ConnectionParameters() self.params = params self.context = context self.no_keep_alive = params.no_keep_alive # The body limits can be altered by the delegate, so save them # here instead of just referencing self.params later. self._max_body_size = (self.params.max_body_size or self.stream.max_buffer_size) self._body_timeout = self.params.body_timeout # _write_finished is set to True when finish() has been called, # i.e. there will be no more data sent. Data may still be in the # stream's write buffer. self._write_finished = False # True when we have read the entire incoming body. self._read_finished = False # _finish_future resolves when all data has been written and flushed # to the IOStream. self._finish_future = Future() # If true, the connection should be closed after this request # (after the response has been written in the server side, # and after it has been read in the client) self._disconnect_on_finish = False self._clear_callbacks() # Save the start lines after we read or write them; they # affect later processing (e.g. 304 responses and HEAD methods # have content-length but no bodies) self._request_start_line = None self._response_start_line = None self._request_headers = None # True if we are writing output with chunked encoding. self._chunking_output = None # While reading a body with a content-length, this is the # amount left to read. self._expected_content_remaining = None # A Future for our outgoing writes, returned by IOStream.write. self._pending_write = None def read_response(self, delegate): """Read a single HTTP response. Typical client-mode usage is to write a request using `write_headers`, `write`, and `finish`, and then call ``read_response``. :arg delegate: a `.HTTPMessageDelegate` Returns a `.Future` that resolves to None after the full response has been read. """ return self._read_message(delegate) @gen.coroutine def _read_message(self, delegate): need_delegate_close = False try: '''header_future = self.stream.read_until_regex( "\x00", max_bytes=3) if self.params.header_timeout is None: header_data = yield header_future else: try: header_data = yield gen.with_timeout( self.stream.io_loop.time() + self.params.header_timeout, header_future, io_loop=self.stream.io_loop, quiet_exceptions=iostream.StreamClosedError) except gen.TimeoutError: self.close() raise gen.Return(False)''' """start_line, headers = self._parse_headers(header_data) if self.is_client: start_line = httputil.parse_response_start_line(start_line) self._response_start_line = start_line else: start_line = httputil.parse_request_start_line(start_line) self._request_start_line = start_line self._request_headers = headers self._disconnect_on_finish = not self._can_keep_alive( start_line, headers) """ yield self._read_body_until_close(delegate) need_delegate_close = True '''with _ExceptionLoggingContext(app_log): header_future = delegate.headers_received(start_line, headers) if header_future is not None: yield header_future if self.stream is None: # We've been detached. need_delegate_close = False raise gen.Return(False) skip_body = False if self.is_client: if (self._request_start_line is not None and self._request_start_line.method == 'HEAD'): skip_body = True code = start_line.code if code == 304: # 304 responses may include the content-length header # but do not actually have a body. # http://tools.ietf.org/html/rfc7230#section-3.3 skip_body = True if code >= 100 and code < 200: # 1xx responses should never indicate the presence of # a body. if ('Content-Length' in headers or 'Transfer-Encoding' in headers): raise httputil.HTTPInputError( "Response code %d cannot have body" % code) # TODO: client delegates will get headers_received twice # in the case of a 100-continue. Document or change? yield self._read_message(delegate) else: if (headers.get("Expect") == "100-continue" and not self._write_finished): self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n") if not skip_body: body_future = self._read_body( start_line.code if self.is_client else 0, headers, delegate) if body_future is not None: if self._body_timeout is None: yield body_future else: try: yield gen.with_timeout( self.stream.io_loop.time() + self._body_timeout, body_future, self.stream.io_loop, quiet_exceptions=iostream.StreamClosedError) except gen.TimeoutError: gen_log.info("Timeout reading body from %s", self.context) self.stream.close() raise gen.Return(False)''' #self._read_finished = True if not self._write_finished or self.is_client: need_delegate_close = False with _ExceptionLoggingContext(app_log): delegate.finish() # If we're waiting for the application to produce an asynchronous # response, and we're not detached, register a close callback # on the stream (we didn't need one while we were reading) if (not self._finish_future.done() and self.stream is not None and not self.stream.closed()): self.stream.set_close_callback(self._on_connection_close) yield self._finish_future if self.is_client and self._disconnect_on_finish: self.close() if self.stream is None: raise gen.Return(False) except httputil.HTTPInputError as e: gen_log.info("Malformed HTTP message from %s: %s", self.context, e) self.close() raise gen.Return(False) finally: if need_delegate_close: with _ExceptionLoggingContext(app_log): delegate.on_connection_close() self._clear_callbacks() raise gen.Return(True) def _clear_callbacks(self): """Clears the callback attributes. This allows the request handler to be garbage collected more quickly in CPython by breaking up reference cycles. """ self._write_callback = None self._write_future = None self._close_callback = None if self.stream is not None: self.stream.set_close_callback(None) def set_close_callback(self, callback): """Sets a callback that will be run when the connection is closed. .. deprecated:: 4.0 Use `.HTTPMessageDelegate.on_connection_close` instead. """ self._close_callback = stack_context.wrap(callback) def _on_connection_close(self): # Note that this callback is only registered on the IOStream # when we have finished reading the request and are waiting for # the application to produce its response. if self._close_callback is not None: callback = self._close_callback self._close_callback = None callback() if not self._finish_future.done(): self._finish_future.set_result(None) self._clear_callbacks() def close(self): if self.stream is not None: self.stream.close() self._clear_callbacks() if not self._finish_future.done(): self._finish_future.set_result(None) def detach(self): """Take control of the underlying stream. Returns the underlying `.IOStream` object and stops all further HTTP processing. May only be called during `.HTTPMessageDelegate.headers_received`. Intended for implementing protocols like websockets that tunnel over an HTTP handshake. """ self._clear_callbacks() stream = self.stream self.stream = None if not self._finish_future.done(): self._finish_future.set_result(None) return stream def set_body_timeout(self, timeout): """Sets the body timeout for a single request. Overrides the value from `.HTTP1ConnectionParameters`. """ self._body_timeout = timeout def set_max_body_size(self, max_body_size): """Sets the body size limit for a single request. Overrides the value from `.HTTP1ConnectionParameters`. """ self._max_body_size = max_body_size def write(self, chunk, callback=None): """Implements `.HTTPConnection.write`. For backwards compatibility is is allowed but deprecated to skip `write_headers` and instead call `write()` with a pre-encoded header block. """ future = None if self.stream.closed(): future = self._write_future = Future() self._write_future.set_exception(iostream.StreamClosedError()) self._write_future.exception() else: if callback is not None: self._write_callback = stack_context.wrap(callback) else: future = self._write_future = Future() self._pending_write = self.stream.write(chunk) self._pending_write.add_done_callback(self._on_write_complete) return future def finish(self): if not self.stream.closed(): #self._pending_write = self.stream.write("\xe0\x00\x00") self._pending_write.add_done_callback(self._on_write_complete) self._write_finished = True # If the app finished the request while we're still reading, # divert any remaining data away from the delegate and # close the connection when we're done sending our response. # Closing the connection is the only way to avoid reading the # whole input body. if not self._read_finished: self._disconnect_on_finish = True # No more data is coming, so instruct TCP to send any remaining # data immediately instead of waiting for a full packet or ack. self.stream.set_nodelay(True) if self._pending_write is None: self._finish_request(None) else: self._pending_write.add_done_callback(self._finish_request) def _on_write_complete(self, future): exc = future.exception() if exc is not None and not isinstance(exc, iostream.StreamClosedError): future.result() if self._write_callback is not None: callback = self._write_callback self._write_callback = None self.stream.io_loop.add_callback(callback) if self._write_future is not None: future = self._write_future self._write_future = None future.set_result(None) def _finish_request(self, future): self._clear_callbacks() if not self.is_client and self._disconnect_on_finish: self.close() return # Turn Nagle's algorithm back on, leaving the stream in its # default state for the next request. self.stream.set_nodelay(False) if not self._finish_future.done(): self._finish_future.set_result(None) @gen.coroutine def _read_body_until_close(self, delegate): body = yield self.stream.read_until_close() self._read_finished = True if not self._write_finished or self.is_client: with _ExceptionLoggingContext(app_log): delegate.data_received(body)
class _Connector(object): """A stateless implementation of the "Happy Eyeballs" algorithm. "Happy Eyeballs" is documented in RFC6555 as the recommended practice for when both IPv4 and IPv6 addresses are available. In this implementation, we partition the addresses by family, and make the first connection attempt to whichever address was returned first by ``getaddrinfo``. If that connection fails or times out, we begin a connection in parallel to the first address of the other family. If there are additional failures we retry with other addresses, keeping one connection attempt per family in flight at a time. http://tools.ietf.org/html/rfc6555 """ def __init__(self, addrinfo, io_loop, connect): self.io_loop = io_loop self.connect = connect self.future = Future() self.timeout = None self.last_error = None self.remaining = len(addrinfo) self.primary_addrs, self.secondary_addrs = self.split(addrinfo) @staticmethod def split(addrinfo): """Partition the ``addrinfo`` list by address family. Returns two lists. The first list contains the first entry from ``addrinfo`` and all others with the same family, and the second list contains all other addresses (normally one list will be AF_INET and the other AF_INET6, although non-standard resolvers may return additional families). """ primary = [] secondary = [] primary_af = addrinfo[0][0] for af, addr in addrinfo: if af == primary_af: primary.append((af, addr)) else: secondary.append((af, addr)) return primary, secondary def start(self, timeout=_INITIAL_CONNECT_TIMEOUT): self.try_connect(iter(self.primary_addrs)) self.set_timout(timeout) return self.future def try_connect(self, addrs): try: af, addr = next(addrs) except StopIteration: # We've reached the end of our queue, but the other queue # might still be working. Send a final error on the future # only when both queues are finished. if self.remaining == 0 and not self.future.done(): self.future.set_exception(self.last_error or IOError("connection failed")) return future = self.connect(af, addr) future.add_done_callback(functools.partial(self.on_connect_done, addrs, af, addr)) def on_connect_done(self, addrs, af, addr, future): self.remaining -= 1 try: stream = future.result() except Exception as e: if self.future.done(): return # Error: try again (but remember what happened so we have an # error to raise in the end) self.last_error = e self.try_connect(addrs) if self.timeout is not None: # If the first attempt failed, don't wait for the # timeout to try an address from the secondary queue. self.on_timeout() return self.clear_timeout() if self.future.done(): # This is a late arrival; just drop it. stream.close() else: self.future.set_result((af, addr, stream)) def set_timout(self, timeout): self.timeout = self.io_loop.add_timeout(self.io_loop.time() + timeout, self.on_timeout) def on_timeout(self): self.timeout = None self.try_connect(iter(self.secondary_addrs)) def clear_timeout(self): if self.timeout is not None: self.io_loop.remove_timeout(self.timeout)
class WaitIterator(object): """Provides an iterator to yield the results of futures as they finish. Yielding a set of futures like this: ``results = yield [future1, future2]`` pauses the coroutine until both ``future1`` and ``future2`` return, and then restarts the coroutine with the results of both futures. If either future is an exception, the expression will raise that exception and all the results will be lost. If you need to get the result of each future as soon as possible, or if you need the result of some futures even if others produce errors, you can use ``WaitIterator``:: wait_iterator = gen.WaitIterator(future1, future2) while not wait_iterator.done(): try: result = yield wait_iterator.next() except Exception as e: print("Error {} from {}".format(e, wait_iterator.current_future)) else: print("Result {} received from {} at {}".format( result, wait_iterator.current_future, wait_iterator.current_index)) Because results are returned as soon as they are available the output from the iterator *will not be in the same order as the input arguments*. If you need to know which future produced the current result, you can use the attributes ``WaitIterator.current_future``, or ``WaitIterator.current_index`` to get the index of the future from the input list. (if keyword arguments were used in the construction of the `WaitIterator`, ``current_index`` will use the corresponding keyword). On Python 3.5, `WaitIterator` implements the async iterator protocol, so it can be used with the ``async for`` statement (note that in this version the entire iteration is aborted if any value raises an exception, while the previous example can continue past individual errors):: async for result in gen.WaitIterator(future1, future2): print("Result {} received from {} at {}".format( result, wait_iterator.current_future, wait_iterator.current_index)) .. versionadded:: 4.1 .. versionchanged:: 4.3 Added ``async for`` support in Python 3.5. """ def __init__(self, *args, **kwargs): if args and kwargs: raise ValueError("You must provide args or kwargs, not both") if kwargs: self._unfinished = dict((f, k) for (k, f) in kwargs.items()) futures = list(kwargs.values()) else: self._unfinished = dict((f, i) for (i, f) in enumerate(args)) futures = args self._finished = collections.deque() self.current_index = self.current_future = None self._running_future = None for future in futures: future_add_done_callback(future, self._done_callback) def done(self): """Returns True if this iterator has no more results.""" if self._finished or self._unfinished: return False # Clear the 'current' values when iteration is done. self.current_index = self.current_future = None return True def next(self): """Returns a `.Future` that will yield the next available result. Note that this `.Future` will not be the same object as any of the inputs. """ self._running_future = Future() if self._finished: self._return_result(self._finished.popleft()) return self._running_future def _done_callback(self, done): if self._running_future and not self._running_future.done(): self._return_result(done) else: self._finished.append(done) def _return_result(self, done): """Called set the returned future's state that of the future we yielded, and set the current future for the iterator. """ chain_future(done, self._running_future) self.current_future = done self.current_index = self._unfinished.pop(done) def __aiter__(self): return self def __anext__(self): if self.done(): # Lookup by name to silence pyflakes on older versions. raise getattr(builtins, 'StopAsyncIteration')() return self.next()
class Event(object): """An event blocks coroutines until its internal flag is set to True. Similar to `threading.Event`. A coroutine can wait for an event to be set. Once it is set, calls to ``yield event.wait()`` will not block unless the event has been cleared: .. testcode:: from tornado import gen from tornado.ioloop import IOLoop from tornado.locks import Event event = Event() @gen.coroutine def waiter(): print("Waiting for event") yield event.wait() print("Not waiting this time") yield event.wait() print("Done") @gen.coroutine def setter(): print("About to set the event") event.set() @gen.coroutine def runner(): yield [waiter(), setter()] IOLoop.current().run_sync(runner) .. testoutput:: Waiting for event About to set the event Not waiting this time Done """ def __init__(self): self._future = Future() def __repr__(self): return '<%s %s>' % ( self.__class__.__name__, 'set' if self.is_set() else 'clear') def is_set(self): """Return ``True`` if the internal flag is true.""" return self._future.done() def set(self): """Set the internal flag to ``True``. All waiters are awakened. Calling `.wait` once the flag is set will not block. """ if not self._future.done(): self._future.set_result(None) def clear(self): """Reset the internal flag to ``False``. Calls to `.wait` will block until `.set` is called. """ if self._future.done(): self._future = Future() def wait(self, timeout=None): """Block until the internal flag is true. Returns a Future, which raises `tornado.gen.TimeoutError` after a timeout. """ if timeout is None: return self._future else: return gen.with_timeout(timeout, self._future)
def coroutine_func(): """ Concrete call to main function. """ port = random.randint(9000, 9999) while is_port_opened(port, HOSTNAME): port = random.randint(9000, 9999) nb_human_players = 1 if nb_daide_clients < 7 else 0 server.start(port=port) server_game = ServerGame(map_name='standard', n_controls=nb_daide_clients + nb_human_players, rules=rules, server=server) # Register game on server. game_id = server_game.game_id server.add_new_game(server_game) server.start_new_daide_server(game_id) # Creating human player human_username = '******' human_password = '******' # Creating bot player to play for dummy powers bot_username = constants.PRIVATE_BOT_USERNAME bot_password = constants.PRIVATE_BOT_PASSWORD # Connecting connection = yield connect(HOSTNAME, port) human_channel = yield connection.authenticate(human_username, human_password) bot_channel = yield connection.authenticate(bot_username, bot_password) # Joining human to game channels = {BOT_KEYWORD: bot_channel} if nb_human_players: yield human_channel.join_game(game_id=game_id, power_name='AUSTRIA') channels['AUSTRIA'] = human_channel comms_simulator = ClientsCommsSimulator(nb_daide_clients, csv_file, game_id, channels) yield comms_simulator.retrieve_game_port(HOSTNAME, port) # done_future is only used to prevent pylint E1101 errors on daide_future done_future = Future() daide_future = comms_simulator.execute() chain_future(daide_future, done_future) for _ in range(3 + nb_daide_clients): if done_future.done() or server_game.count_controlled_powers() >= ( nb_daide_clients + nb_human_players): break yield gen.sleep(2.5) else: raise TimeoutError() # Waiting for process to finish while not done_future.done() and server_game.status == strings.ACTIVE: yield gen.sleep(2.5) yield daide_future
class AsyncGroup: """ Grouping of several async requests and final callback in such way that final callback is invoked after the last request is finished. If any callback throws an exception, all pending callbacks would be aborted and finish_cb would not be automatically called. """ def __init__(self, finish_cb, name=None): self._counter = 0 self._finish_cb = finish_cb self._finished = False self._name = name self._future = Future() self._start_time = time.time() def abort(self): async_logger.info('aborting %s', self) self._finished = True if not self._future.done(): self._future.set_exception(AbortAsyncGroup()) def finish(self): if self._finished: async_logger.warning('trying to finish already finished %s', self) return self._finished = True self._future.set_result(None) try: self._finish_cb() finally: # prevent possible cycle references self._finish_cb = None def try_finish(self): if self._counter == 0: self.finish() def try_finish_async(self): """Executes finish_cb in next IOLoop iteration""" if self._counter == 0: IOLoop.current().add_callback(self.finish) def _inc(self): if self._finished: async_logger.info('ignoring adding callback in %s', self) raise AbortAsyncGroup() self._counter += 1 def _dec(self): self._counter -= 1 def add(self, intermediate_cb): self._inc() @wraps(intermediate_cb) def new_cb(*args, **kwargs): if self._finished: async_logger.info('ignoring executing callback in %s', self) return try: self._dec() intermediate_cb(*args, **kwargs) except Exception: self.abort() raise self.try_finish() return new_cb def add_notification(self): self._inc() def new_cb(*args, **kwargs): self._dec() self.try_finish() return new_cb def add_future(self, future): IOLoop.current().add_future(future, self.add_notification()) return future def get_finish_future(self): return self._future def __str__(self): return f'AsyncGroup(name={self._name}, finished={self._finished})'
class WebSocketClientConnection(simple_httpclient._HTTPConnection): """WebSocket client connection.""" def __init__(self, io_loop, request): self.connect_future = Future() self.read_future = None self.read_queue = collections.deque() self.key = base64.b64encode(os.urandom(16)) scheme, sep, rest = request.url.partition(':') scheme = {'ws': 'http', 'wss': 'https'}[scheme] request.url = scheme + sep + rest request.headers.update({ 'Upgrade': 'websocket', 'Connection': 'Upgrade', 'Sec-WebSocket-Key': self.key, 'Sec-WebSocket-Version': '13', }) self.resolver = Resolver(io_loop=io_loop) super(WebSocketClientConnection, self).__init__( io_loop, None, request, lambda: None, self._on_http_response, 104857600, self.resolver) def _on_close(self): self.on_message(None) self.resolver.close() def _on_http_response(self, response): if not self.connect_future.done(): if response.error: self.connect_future.set_exception(response.error) else: self.connect_future.set_exception(WebSocketError( "Non-websocket response")) def _handle_1xx(self, code): assert code == 101 assert self.headers['Upgrade'].lower() == 'websocket' assert self.headers['Connection'].lower() == 'upgrade' accept = WebSocketProtocol13.compute_accept_value(self.key) assert self.headers['Sec-Websocket-Accept'] == accept self.protocol = WebSocketProtocol13(self, mask_outgoing=True) self.protocol._receive_frame() if self._timeout is not None: self.io_loop.remove_timeout(self._timeout) self._timeout = None self.connect_future.set_result(self) def write_message(self, message, binary=False): """Sends a message to the WebSocket server.""" self.protocol.write_message(message, binary) def read_message(self, callback=None): """Reads a message from the WebSocket server. Returns a future whose result is the message, or None if the connection is closed. If a callback argument is given it will be called with the future when it is ready. """ assert self.read_future is None future = Future() if self.read_queue: future.set_result(self.read_queue.popleft()) else: self.read_future = future if callback is not None: self.io_loop.add_future(future, callback) return future def on_message(self, message): if self.read_future is not None: self.read_future.set_result(message) self.read_future = None else: self.read_queue.append(message) def on_pong(self, data): pass
class HTTP1Connection(httputil.HTTPConnection): """Implements the HTTP/1.x protocol. This class can be on its own for clients, or via `HTTP1ServerConnection` for servers. """ def __init__(self, stream, is_client, params=None, context=None): """ :arg stream: an `.IOStream` :arg bool is_client: client or server :arg params: a `.HTTP1ConnectionParameters` instance or ``None`` :arg context: an opaque application-defined object that can be accessed as ``connection.context``. """ self.is_client = is_client self.stream = stream if params is None: params = HTTP1ConnectionParameters() self.params = params self.context = context self.no_keep_alive = params.no_keep_alive # The body limits can be altered by the delegate, so save them # here instead of just referencing self.params later. self._max_body_size = (self.params.max_body_size or self.stream.max_buffer_size) self._body_timeout = self.params.body_timeout # _write_finished is set to True when finish() has been called, # i.e. there will be no more data sent. Data may still be in the # stream's write buffer. self._write_finished = False # True when we have read the entire incoming body. self._read_finished = False # _finish_future resolves when all data has been written and flushed # to the IOStream. self._finish_future = Future() # If true, the connection should be closed after this request # (after the response has been written in the server side, # and after it has been read in the client) self._disconnect_on_finish = False self._clear_callbacks() # Save the start lines after we read or write them; they # affect later processing (e.g. 304 responses and HEAD methods # have content-length but no bodies) self._request_start_line = None self._response_start_line = None self._request_headers = None # True if we are writing output with chunked encoding. self._chunking_output = None # While reading a body with a content-length, this is the # amount left to read. self._expected_content_remaining = None # A Future for our outgoing writes, returned by IOStream.write. self._pending_write = None def read_response(self, delegate): """Read a single HTTP response. Typical client-mode usage is to write a request using `write_headers`, `write`, and `finish`, and then call ``read_response``. :arg delegate: a `.HTTPMessageDelegate` Returns a `.Future` that resolves to None after the full response has been read. """ if self.params.decompress: delegate = _GzipMessageDelegate(delegate, self.params.chunk_size) return self._read_message(delegate) @gen.coroutine def _read_message(self, delegate): need_delegate_close = False try: header_future = self.stream.read_until_regex( b"\r?\n\r?\n", max_bytes=self.params.max_header_size) if self.params.header_timeout is None: header_data = yield header_future else: try: header_data = yield gen.with_timeout( self.stream.io_loop.time() + self.params.header_timeout, header_future, quiet_exceptions=iostream.StreamClosedError) except gen.TimeoutError: self.close() raise gen.Return(False) start_line, headers = self._parse_headers(header_data) if self.is_client: start_line = httputil.parse_response_start_line(start_line) self._response_start_line = start_line else: start_line = httputil.parse_request_start_line(start_line) self._request_start_line = start_line self._request_headers = headers self._disconnect_on_finish = not self._can_keep_alive( start_line, headers) need_delegate_close = True with _ExceptionLoggingContext(app_log): header_future = delegate.headers_received(start_line, headers) if header_future is not None: yield header_future if self.stream is None: # We've been detached. need_delegate_close = False raise gen.Return(False) skip_body = False if self.is_client: if (self._request_start_line is not None and self._request_start_line.method == 'HEAD'): skip_body = True code = start_line.code if code == 304: # 304 responses may include the content-length header # but do not actually have a body. # http://tools.ietf.org/html/rfc7230#section-3.3 skip_body = True if code >= 100 and code < 200: # 1xx responses should never indicate the presence of # a body. if ('Content-Length' in headers or 'Transfer-Encoding' in headers): raise httputil.HTTPInputError( "Response code %d cannot have body" % code) # TODO: client delegates will get headers_received twice # in the case of a 100-continue. Document or change? yield self._read_message(delegate) else: if (headers.get("Expect") == "100-continue" and not self._write_finished): self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n") if not skip_body: body_future = self._read_body( start_line.code if self.is_client else 0, headers, delegate) if body_future is not None: if self._body_timeout is None: yield body_future else: try: yield gen.with_timeout( self.stream.io_loop.time() + self._body_timeout, body_future, quiet_exceptions=iostream.StreamClosedError) except gen.TimeoutError: gen_log.info("Timeout reading body from %s", self.context) self.stream.close() raise gen.Return(False) self._read_finished = True if not self._write_finished or self.is_client: need_delegate_close = False with _ExceptionLoggingContext(app_log): delegate.finish() # If we're waiting for the application to produce an asynchronous # response, and we're not detached, register a close callback # on the stream (we didn't need one while we were reading) if (not self._finish_future.done() and self.stream is not None and not self.stream.closed()): self.stream.set_close_callback(self._on_connection_close) yield self._finish_future if self.is_client and self._disconnect_on_finish: self.close() if self.stream is None: raise gen.Return(False) except httputil.HTTPInputError as e: gen_log.info("Malformed HTTP message from %s: %s", self.context, e) if not self.is_client: yield self.stream.write(b'HTTP/1.1 400 Bad Request\r\n\r\n') self.close() raise gen.Return(False) finally: if need_delegate_close: with _ExceptionLoggingContext(app_log): delegate.on_connection_close() header_future = None self._clear_callbacks() raise gen.Return(True) def _clear_callbacks(self): """Clears the callback attributes. This allows the request handler to be garbage collected more quickly in CPython by breaking up reference cycles. """ self._write_callback = None self._write_future = None self._close_callback = None if self.stream is not None: self.stream.set_close_callback(None) def set_close_callback(self, callback): """Sets a callback that will be run when the connection is closed. Note that this callback is slightly different from `.HTTPMessageDelegate.on_connection_close`: The `.HTTPMessageDelegate` method is called when the connection is closed while recieving a message. This callback is used when there is not an active delegate (for example, on the server side this callback is used if the client closes the connection after sending its request but before receiving all the response. """ self._close_callback = stack_context.wrap(callback) def _on_connection_close(self): # Note that this callback is only registered on the IOStream # when we have finished reading the request and are waiting for # the application to produce its response. if self._close_callback is not None: callback = self._close_callback self._close_callback = None callback() if not self._finish_future.done(): future_set_result_unless_cancelled(self._finish_future, None) self._clear_callbacks() def close(self): if self.stream is not None: self.stream.close() self._clear_callbacks() if not self._finish_future.done(): future_set_result_unless_cancelled(self._finish_future, None) def detach(self): """Take control of the underlying stream. Returns the underlying `.IOStream` object and stops all further HTTP processing. May only be called during `.HTTPMessageDelegate.headers_received`. Intended for implementing protocols like websockets that tunnel over an HTTP handshake. """ self._clear_callbacks() stream = self.stream self.stream = None if not self._finish_future.done(): future_set_result_unless_cancelled(self._finish_future, None) return stream def set_body_timeout(self, timeout): """Sets the body timeout for a single request. Overrides the value from `.HTTP1ConnectionParameters`. """ self._body_timeout = timeout def set_max_body_size(self, max_body_size): """Sets the body size limit for a single request. Overrides the value from `.HTTP1ConnectionParameters`. """ self._max_body_size = max_body_size def write_headers(self, start_line, headers, chunk=None, callback=None): """Implements `.HTTPConnection.write_headers`.""" lines = [] if self.is_client: self._request_start_line = start_line lines.append(utf8('%s %s HTTP/1.1' % (start_line[0], start_line[1]))) # Client requests with a non-empty body must have either a # Content-Length or a Transfer-Encoding. self._chunking_output = ( start_line.method in ('POST', 'PUT', 'PATCH') and 'Content-Length' not in headers and 'Transfer-Encoding' not in headers) else: self._response_start_line = start_line lines.append(utf8('HTTP/1.1 %d %s' % (start_line[1], start_line[2]))) self._chunking_output = ( # TODO: should this use # self._request_start_line.version or # start_line.version? self._request_start_line.version == 'HTTP/1.1' and # 1xx, 204 and 304 responses have no body (not even a zero-length # body), and so should not have either Content-Length or # Transfer-Encoding headers. start_line.code not in (204, 304) and (start_line.code < 100 or start_line.code >= 200) and # No need to chunk the output if a Content-Length is specified. 'Content-Length' not in headers and # Applications are discouraged from touching Transfer-Encoding, # but if they do, leave it alone. 'Transfer-Encoding' not in headers) # If connection to a 1.1 client will be closed, inform client if (self._request_start_line.version == 'HTTP/1.1' and self._disconnect_on_finish): headers['Connection'] = 'close' # If a 1.0 client asked for keep-alive, add the header. if (self._request_start_line.version == 'HTTP/1.0' and self._request_headers.get('Connection', '').lower() == 'keep-alive'): headers['Connection'] = 'Keep-Alive' if self._chunking_output: headers['Transfer-Encoding'] = 'chunked' if (not self.is_client and (self._request_start_line.method == 'HEAD' or start_line.code == 304)): self._expected_content_remaining = 0 elif 'Content-Length' in headers: self._expected_content_remaining = int(headers['Content-Length']) else: self._expected_content_remaining = None # TODO: headers are supposed to be of type str, but we still have some # cases that let bytes slip through. Remove these native_str calls when those # are fixed. header_lines = (native_str(n) + ": " + native_str(v) for n, v in headers.get_all()) if PY3: lines.extend(l.encode('latin1') for l in header_lines) else: lines.extend(header_lines) for line in lines: if b'\n' in line: raise ValueError('Newline in header: ' + repr(line)) future = None if self.stream.closed(): future = self._write_future = Future() future.set_exception(iostream.StreamClosedError()) future.exception() else: if callback is not None: warnings.warn("callback argument is deprecated, use returned Future instead", DeprecationWarning) self._write_callback = stack_context.wrap(callback) else: future = self._write_future = Future() data = b"\r\n".join(lines) + b"\r\n\r\n" if chunk: data += self._format_chunk(chunk) self._pending_write = self.stream.write(data) future_add_done_callback(self._pending_write, self._on_write_complete) return future def _format_chunk(self, chunk): if self._expected_content_remaining is not None: self._expected_content_remaining -= len(chunk) if self._expected_content_remaining < 0: # Close the stream now to stop further framing errors. self.stream.close() raise httputil.HTTPOutputError( "Tried to write more data than Content-Length") if self._chunking_output and chunk: # Don't write out empty chunks because that means END-OF-STREAM # with chunked encoding return utf8("%x" % len(chunk)) + b"\r\n" + chunk + b"\r\n" else: return chunk def write(self, chunk, callback=None): """Implements `.HTTPConnection.write`. For backwards compatibility it is allowed but deprecated to skip `write_headers` and instead call `write()` with a pre-encoded header block. """ future = None if self.stream.closed(): future = self._write_future = Future() self._write_future.set_exception(iostream.StreamClosedError()) self._write_future.exception() else: if callback is not None: warnings.warn("callback argument is deprecated, use returned Future instead", DeprecationWarning) self._write_callback = stack_context.wrap(callback) else: future = self._write_future = Future() self._pending_write = self.stream.write(self._format_chunk(chunk)) self._pending_write.add_done_callback(self._on_write_complete) return future def finish(self): """Implements `.HTTPConnection.finish`.""" if (self._expected_content_remaining is not None and self._expected_content_remaining != 0 and not self.stream.closed()): self.stream.close() raise httputil.HTTPOutputError( "Tried to write %d bytes less than Content-Length" % self._expected_content_remaining) if self._chunking_output: if not self.stream.closed(): self._pending_write = self.stream.write(b"0\r\n\r\n") self._pending_write.add_done_callback(self._on_write_complete) self._write_finished = True # If the app finished the request while we're still reading, # divert any remaining data away from the delegate and # close the connection when we're done sending our response. # Closing the connection is the only way to avoid reading the # whole input body. if not self._read_finished: self._disconnect_on_finish = True # No more data is coming, so instruct TCP to send any remaining # data immediately instead of waiting for a full packet or ack. self.stream.set_nodelay(True) if self._pending_write is None: self._finish_request(None) else: future_add_done_callback(self._pending_write, self._finish_request) def _on_write_complete(self, future): exc = future.exception() if exc is not None and not isinstance(exc, iostream.StreamClosedError): future.result() if self._write_callback is not None: callback = self._write_callback self._write_callback = None self.stream.io_loop.add_callback(callback) if self._write_future is not None: future = self._write_future self._write_future = None future_set_result_unless_cancelled(future, None) def _can_keep_alive(self, start_line, headers): if self.params.no_keep_alive: return False connection_header = headers.get("Connection") if connection_header is not None: connection_header = connection_header.lower() if start_line.version == "HTTP/1.1": return connection_header != "close" elif ("Content-Length" in headers or headers.get("Transfer-Encoding", "").lower() == "chunked" or getattr(start_line, 'method', None) in ("HEAD", "GET")): # start_line may be a request or response start line; only # the former has a method attribute. return connection_header == "keep-alive" return False def _finish_request(self, future): self._clear_callbacks() if not self.is_client and self._disconnect_on_finish: self.close() return # Turn Nagle's algorithm back on, leaving the stream in its # default state for the next request. self.stream.set_nodelay(False) if not self._finish_future.done(): future_set_result_unless_cancelled(self._finish_future, None) def _parse_headers(self, data): # The lstrip removes newlines that some implementations sometimes # insert between messages of a reused connection. Per RFC 7230, # we SHOULD ignore at least one empty line before the request. # http://tools.ietf.org/html/rfc7230#section-3.5 data = native_str(data.decode('latin1')).lstrip("\r\n") # RFC 7230 section allows for both CRLF and bare LF. eol = data.find("\n") start_line = data[:eol].rstrip("\r") headers = httputil.HTTPHeaders.parse(data[eol:]) return start_line, headers def _read_body(self, code, headers, delegate): if "Content-Length" in headers: if "Transfer-Encoding" in headers: # Response cannot contain both Content-Length and # Transfer-Encoding headers. # http://tools.ietf.org/html/rfc7230#section-3.3.3 raise httputil.HTTPInputError( "Response with both Transfer-Encoding and Content-Length") if "," in headers["Content-Length"]: # Proxies sometimes cause Content-Length headers to get # duplicated. If all the values are identical then we can # use them but if they differ it's an error. pieces = re.split(r',\s*', headers["Content-Length"]) if any(i != pieces[0] for i in pieces): raise httputil.HTTPInputError( "Multiple unequal Content-Lengths: %r" % headers["Content-Length"]) headers["Content-Length"] = pieces[0] try: content_length = int(headers["Content-Length"]) except ValueError: # Handles non-integer Content-Length value. raise httputil.HTTPInputError( "Only integer Content-Length is allowed: %s" % headers["Content-Length"]) if content_length > self._max_body_size: raise httputil.HTTPInputError("Content-Length too long") else: content_length = None if code == 204: # This response code is not allowed to have a non-empty body, # and has an implicit length of zero instead of read-until-close. # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3 if ("Transfer-Encoding" in headers or content_length not in (None, 0)): raise httputil.HTTPInputError( "Response with code %d should not have body" % code) content_length = 0 if content_length is not None: return self._read_fixed_body(content_length, delegate) if headers.get("Transfer-Encoding", "").lower() == "chunked": return self._read_chunked_body(delegate) if self.is_client: return self._read_body_until_close(delegate) return None @gen.coroutine def _read_fixed_body(self, content_length, delegate): while content_length > 0: body = yield self.stream.read_bytes( min(self.params.chunk_size, content_length), partial=True) content_length -= len(body) if not self._write_finished or self.is_client: with _ExceptionLoggingContext(app_log): ret = delegate.data_received(body) if ret is not None: yield ret @gen.coroutine def _read_chunked_body(self, delegate): # TODO: "chunk extensions" http://tools.ietf.org/html/rfc2616#section-3.6.1 total_size = 0 while True: chunk_len = yield self.stream.read_until(b"\r\n", max_bytes=64) chunk_len = int(chunk_len.strip(), 16) if chunk_len == 0: crlf = yield self.stream.read_bytes(2) if crlf != b'\r\n': raise httputil.HTTPInputError("improperly terminated chunked request") return total_size += chunk_len if total_size > self._max_body_size: raise httputil.HTTPInputError("chunked body too large") bytes_to_read = chunk_len while bytes_to_read: chunk = yield self.stream.read_bytes( min(bytes_to_read, self.params.chunk_size), partial=True) bytes_to_read -= len(chunk) if not self._write_finished or self.is_client: with _ExceptionLoggingContext(app_log): ret = delegate.data_received(chunk) if ret is not None: yield ret # chunk ends with \r\n crlf = yield self.stream.read_bytes(2) assert crlf == b"\r\n" @gen.coroutine def _read_body_until_close(self, delegate): body = yield self.stream.read_until_close() if not self._write_finished or self.is_client: with _ExceptionLoggingContext(app_log): delegate.data_received(body)
class Runner(object): """Internal implementation of `tornado.gen.engine`. Maintains information about pending callbacks and their results. The results of the generator are stored in ``result_future`` (a `.Future`) """ def __init__(self, gen, result_future, first_yielded): self.gen = gen self.result_future = result_future self.future = _null_future self.yield_point = None self.pending_callbacks = None self.results = None self.running = False self.finished = False self.had_exception = False self.io_loop = IOLoop.current() # For efficiency, we do not create a stack context until we # reach a YieldPoint (stack contexts are required for the historical # semantics of YieldPoints, but not for Futures). When we have # done so, this field will be set and must be called at the end # of the coroutine. self.stack_context_deactivate = None if self.handle_yield(first_yielded): gen = result_future = first_yielded = None self.run() def register_callback(self, key): """Adds ``key`` to the list of callbacks.""" if self.pending_callbacks is None: # Lazily initialize the old-style YieldPoint data structures. self.pending_callbacks = set() self.results = {} if key in self.pending_callbacks: raise KeyReuseError("key %r is already pending" % (key,)) self.pending_callbacks.add(key) def is_ready(self, key): """Returns true if a result is available for ``key``.""" if self.pending_callbacks is None or key not in self.pending_callbacks: raise UnknownKeyError("key %r is not pending" % (key,)) return key in self.results def set_result(self, key, result): """Sets the result for ``key`` and attempts to resume the generator.""" self.results[key] = result if self.yield_point is not None and self.yield_point.is_ready(): try: future_set_result_unless_cancelled(self.future, self.yield_point.get_result()) except: future_set_exc_info(self.future, sys.exc_info()) self.yield_point = None self.run() def pop_result(self, key): """Returns the result for ``key`` and unregisters it.""" self.pending_callbacks.remove(key) return self.results.pop(key) def run(self): """Starts or resumes the generator, running until it reaches a yield point that is not ready. """ if self.running or self.finished: return try: self.running = True while True: future = self.future if not future.done(): return self.future = None try: orig_stack_contexts = stack_context._state.contexts exc_info = None try: value = future.result() except Exception: self.had_exception = True exc_info = sys.exc_info() future = None if exc_info is not None: try: yielded = self.gen.throw(*exc_info) finally: # Break up a reference to itself # for faster GC on CPython. exc_info = None else: yielded = self.gen.send(value) if stack_context._state.contexts is not orig_stack_contexts: self.gen.throw( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)')) except (StopIteration, Return) as e: self.finished = True self.future = _null_future if self.pending_callbacks and not self.had_exception: # If we ran cleanly without waiting on all callbacks # raise an error (really more of a warning). If we # had an exception then some callbacks may have been # orphaned, so skip the check in that case. raise LeakedCallbackError( "finished without waiting for callbacks %r" % self.pending_callbacks) future_set_result_unless_cancelled(self.result_future, _value_from_stopiteration(e)) self.result_future = None self._deactivate_stack_context() return except Exception: self.finished = True self.future = _null_future future_set_exc_info(self.result_future, sys.exc_info()) self.result_future = None self._deactivate_stack_context() return if not self.handle_yield(yielded): return yielded = None finally: self.running = False def handle_yield(self, yielded): # Lists containing YieldPoints require stack contexts; # other lists are handled in convert_yielded. if _contains_yieldpoint(yielded): yielded = multi(yielded) if isinstance(yielded, YieldPoint): # YieldPoints are too closely coupled to the Runner to go # through the generic convert_yielded mechanism. self.future = Future() def start_yield_point(): try: yielded.start(self) if yielded.is_ready(): future_set_result_unless_cancelled(self.future, yielded.get_result()) else: self.yield_point = yielded except Exception: self.future = Future() future_set_exc_info(self.future, sys.exc_info()) if self.stack_context_deactivate is None: # Start a stack context if this is the first # YieldPoint we've seen. with stack_context.ExceptionStackContext( self.handle_exception) as deactivate: self.stack_context_deactivate = deactivate def cb(): start_yield_point() self.run() self.io_loop.add_callback(cb) return False else: start_yield_point() else: try: self.future = convert_yielded(yielded) except BadYieldError: self.future = Future() future_set_exc_info(self.future, sys.exc_info()) if self.future is moment: self.io_loop.add_callback(self.run) return False elif not self.future.done(): def inner(f): # Break a reference cycle to speed GC. f = None # noqa self.run() self.io_loop.add_future( self.future, inner) return False return True def result_callback(self, key): return stack_context.wrap(_argument_adapter( functools.partial(self.set_result, key))) def handle_exception(self, typ, value, tb): if not self.running and not self.finished: self.future = Future() future_set_exc_info(self.future, (typ, value, tb)) self.run() return True else: return False def _deactivate_stack_context(self): if self.stack_context_deactivate is not None: self.stack_context_deactivate() self.stack_context_deactivate = None
class Runner(object): """Internal implementation of `tornado.gen.engine`. Maintains information about pending callbacks and their results. The results of the generator are stored in ``result_future`` (a `.Future`) """ def __init__(self, gen, result_future, first_yielded): self.gen = gen self.result_future = result_future self.future = _null_future self.yield_point = None self.pending_callbacks = None self.results = None self.running = False self.finished = False self.had_exception = False self.io_loop = IOLoop.current() # For efficiency, we do not create a stack context until we # reach a YieldPoint (stack contexts are required for the historical # semantics of YieldPoints, but not for Futures). When we have # done so, this field will be set and must be called at the end # of the coroutine. self.stack_context_deactivate = None if self.handle_yield(first_yielded): gen = result_future = first_yielded = None self.run() def register_callback(self, key): """Adds ``key`` to the list of callbacks.""" if self.pending_callbacks is None: # Lazily initialize the old-style YieldPoint data structures. self.pending_callbacks = set() self.results = {} if key in self.pending_callbacks: raise KeyReuseError("key %r is already pending" % (key, )) self.pending_callbacks.add(key) def is_ready(self, key): """Returns true if a result is available for ``key``.""" if self.pending_callbacks is None or key not in self.pending_callbacks: raise UnknownKeyError("key %r is not pending" % (key, )) return key in self.results def set_result(self, key, result): """Sets the result for ``key`` and attempts to resume the generator.""" self.results[key] = result if self.yield_point is not None and self.yield_point.is_ready(): try: future_set_result_unless_cancelled( self.future, self.yield_point.get_result()) except: future_set_exc_info(self.future, sys.exc_info()) self.yield_point = None self.run() def pop_result(self, key): """Returns the result for ``key`` and unregisters it.""" self.pending_callbacks.remove(key) return self.results.pop(key) def run(self): """Starts or resumes the generator, running until it reaches a yield point that is not ready. """ if self.running or self.finished: return try: self.running = True while True: future = self.future if not future.done(): return self.future = None try: orig_stack_contexts = stack_context._state.contexts exc_info = None try: value = future.result() except Exception: self.had_exception = True exc_info = sys.exc_info() future = None if exc_info is not None: try: yielded = self.gen.throw(*exc_info) finally: # Break up a reference to itself # for faster GC on CPython. exc_info = None else: yielded = self.gen.send(value) if stack_context._state.contexts is not orig_stack_contexts: self.gen.throw( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)') ) except (StopIteration, Return) as e: self.finished = True self.future = _null_future if self.pending_callbacks and not self.had_exception: # If we ran cleanly without waiting on all callbacks # raise an error (really more of a warning). If we # had an exception then some callbacks may have been # orphaned, so skip the check in that case. raise LeakedCallbackError( "finished without waiting for callbacks %r" % self.pending_callbacks) future_set_result_unless_cancelled( self.result_future, _value_from_stopiteration(e)) self.result_future = None self._deactivate_stack_context() return except Exception: self.finished = True self.future = _null_future future_set_exc_info(self.result_future, sys.exc_info()) self.result_future = None self._deactivate_stack_context() return if not self.handle_yield(yielded): return yielded = None finally: self.running = False def handle_yield(self, yielded): # Lists containing YieldPoints require stack contexts; # other lists are handled in convert_yielded. if _contains_yieldpoint(yielded): yielded = multi(yielded) if isinstance(yielded, YieldPoint): # YieldPoints are too closely coupled to the Runner to go # through the generic convert_yielded mechanism. self.future = Future() def start_yield_point(): try: yielded.start(self) if yielded.is_ready(): future_set_result_unless_cancelled( self.future, yielded.get_result()) else: self.yield_point = yielded except Exception: self.future = Future() future_set_exc_info(self.future, sys.exc_info()) if self.stack_context_deactivate is None: # Start a stack context if this is the first # YieldPoint we've seen. with stack_context.ExceptionStackContext( self.handle_exception) as deactivate: self.stack_context_deactivate = deactivate def cb(): start_yield_point() self.run() self.io_loop.add_callback(cb) return False else: start_yield_point() else: try: self.future = convert_yielded(yielded) except BadYieldError: self.future = Future() future_set_exc_info(self.future, sys.exc_info()) if self.future is moment: self.io_loop.add_callback(self.run) return False elif not self.future.done(): def inner(f): # Break a reference cycle to speed GC. f = None # noqa self.run() self.io_loop.add_future(self.future, inner) return False return True def result_callback(self, key): return stack_context.wrap( _argument_adapter(functools.partial(self.set_result, key))) def handle_exception(self, typ, value, tb): if not self.running and not self.finished: self.future = Future() future_set_exc_info(self.future, (typ, value, tb)) self.run() return True else: return False def _deactivate_stack_context(self): if self.stack_context_deactivate is not None: self.stack_context_deactivate() self.stack_context_deactivate = None
class HTTP1Connection(httputil.HTTPConnection): """Implements the HTTP/1.x protocol. This class can be on its own for clients, or via `HTTP1ServerConnection` for servers. """ def __init__( self, stream: iostream.IOStream, is_client: bool, params: HTTP1ConnectionParameters = None, context: object = None, ) -> None: """ :arg stream: an `.IOStream` :arg bool is_client: client or server :arg params: a `.HTTP1ConnectionParameters` instance or ``None`` :arg context: an opaque application-defined object that can be accessed as ``connection.context``. """ self.is_client = is_client self.stream = stream if params is None: params = HTTP1ConnectionParameters() self.params = params self.context = context self.no_keep_alive = params.no_keep_alive # The body limits can be altered by the delegate, so save them # here instead of just referencing self.params later. self._max_body_size = self.params.max_body_size or self.stream.max_buffer_size self._body_timeout = self.params.body_timeout # _write_finished is set to True when finish() has been called, # i.e. there will be no more data sent. Data may still be in the # stream's write buffer. self._write_finished = False # True when we have read the entire incoming body. self._read_finished = False # _finish_future resolves when all data has been written and flushed # to the IOStream. self._finish_future = Future() # type: Future[None] # If true, the connection should be closed after this request # (after the response has been written in the server side, # and after it has been read in the client) self._disconnect_on_finish = False self._clear_callbacks() # Save the start lines after we read or write them; they # affect later processing (e.g. 304 responses and HEAD methods # have content-length but no bodies) self._request_start_line = None # type: Optional[httputil.RequestStartLine] self._response_start_line = None # type: Optional[httputil.ResponseStartLine] self._request_headers = None # type: Optional[httputil.HTTPHeaders] # True if we are writing output with chunked encoding. self._chunking_output = False # While reading a body with a content-length, this is the # amount left to read. self._expected_content_remaining = None # type: Optional[int] # A Future for our outgoing writes, returned by IOStream.write. self._pending_write = None # type: Optional[Future[None]] def read_response( self, delegate: httputil.HTTPMessageDelegate) -> Awaitable[bool]: """Read a single HTTP response. Typical client-mode usage is to write a request using `write_headers`, `write`, and `finish`, and then call ``read_response``. :arg delegate: a `.HTTPMessageDelegate` Returns a `.Future` that resolves to a bool after the full response has been read. The result is true if the stream is still open. """ if self.params.decompress: delegate = _GzipMessageDelegate(delegate, self.params.chunk_size) return self._read_message(delegate) async def _read_message(self, delegate: httputil.HTTPMessageDelegate) -> bool: need_delegate_close = False try: header_future = self.stream.read_until_regex( b"\r?\n\r?\n", max_bytes=self.params.max_header_size) if self.params.header_timeout is None: header_data = await header_future else: try: header_data = await gen.with_timeout( self.stream.io_loop.time() + self.params.header_timeout, header_future, quiet_exceptions=iostream.StreamClosedError, ) except gen.TimeoutError: self.close() return False start_line_str, headers = self._parse_headers(header_data) if self.is_client: resp_start_line = httputil.parse_response_start_line( start_line_str) self._response_start_line = resp_start_line start_line = ( resp_start_line ) # type: Union[httputil.RequestStartLine, httputil.ResponseStartLine] # TODO: this will need to change to support client-side keepalive self._disconnect_on_finish = False else: req_start_line = httputil.parse_request_start_line( start_line_str) self._request_start_line = req_start_line self._request_headers = headers start_line = req_start_line self._disconnect_on_finish = not self._can_keep_alive( req_start_line, headers) need_delegate_close = True with _ExceptionLoggingContext(app_log): header_recv_future = delegate.headers_received( start_line, headers) if header_recv_future is not None: await header_recv_future if self.stream is None: # We've been detached. need_delegate_close = False return False skip_body = False if self.is_client: assert isinstance(start_line, httputil.ResponseStartLine) if (self._request_start_line is not None and self._request_start_line.method == "HEAD"): skip_body = True code = start_line.code if code == 304: # 304 responses may include the content-length header # but do not actually have a body. # http://tools.ietf.org/html/rfc7230#section-3.3 skip_body = True if code >= 100 and code < 200: # 1xx responses should never indicate the presence of # a body. if "Content-Length" in headers or "Transfer-Encoding" in headers: raise httputil.HTTPInputError( "Response code %d cannot have body" % code) # TODO: client delegates will get headers_received twice # in the case of a 100-continue. Document or change? await self._read_message(delegate) else: if headers.get( "Expect" ) == "100-continue" and not self._write_finished: self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n") if not skip_body: body_future = self._read_body( resp_start_line.code if self.is_client else 0, headers, delegate) if body_future is not None: if self._body_timeout is None: await body_future else: try: await gen.with_timeout( self.stream.io_loop.time() + self._body_timeout, body_future, quiet_exceptions=iostream.StreamClosedError, ) except gen.TimeoutError: gen_log.info("Timeout reading body from %s", self.context) self.stream.close() return False self._read_finished = True if not self._write_finished or self.is_client: need_delegate_close = False with _ExceptionLoggingContext(app_log): delegate.finish() # If we're waiting for the application to produce an asynchronous # response, and we're not detached, register a close callback # on the stream (we didn't need one while we were reading) if (not self._finish_future.done() and self.stream is not None and not self.stream.closed()): self.stream.set_close_callback(self._on_connection_close) await self._finish_future if self.is_client and self._disconnect_on_finish: self.close() if self.stream is None: return False except httputil.HTTPInputError as e: gen_log.info("Malformed HTTP message from %s: %s", self.context, e) if not self.is_client: await self.stream.write(b"HTTP/1.1 400 Bad Request\r\n\r\n") self.close() return False finally: if need_delegate_close: with _ExceptionLoggingContext(app_log): delegate.on_connection_close() header_future = None # type: ignore self._clear_callbacks() return True def _clear_callbacks(self) -> None: """Clears the callback attributes. This allows the request handler to be garbage collected more quickly in CPython by breaking up reference cycles. """ self._write_callback = None self._write_future = None # type: Optional[Future[None]] self._close_callback = None # type: Optional[Callable[[], None]] if self.stream is not None: self.stream.set_close_callback(None) def set_close_callback(self, callback: Optional[Callable[[], None]]) -> None: """Sets a callback that will be run when the connection is closed. Note that this callback is slightly different from `.HTTPMessageDelegate.on_connection_close`: The `.HTTPMessageDelegate` method is called when the connection is closed while recieving a message. This callback is used when there is not an active delegate (for example, on the server side this callback is used if the client closes the connection after sending its request but before receiving all the response. """ self._close_callback = callback def _on_connection_close(self) -> None: # Note that this callback is only registered on the IOStream # when we have finished reading the request and are waiting for # the application to produce its response. if self._close_callback is not None: callback = self._close_callback self._close_callback = None callback() if not self._finish_future.done(): future_set_result_unless_cancelled(self._finish_future, None) self._clear_callbacks() def close(self) -> None: if self.stream is not None: self.stream.close() self._clear_callbacks() if not self._finish_future.done(): future_set_result_unless_cancelled(self._finish_future, None) def detach(self) -> iostream.IOStream: """Take control of the underlying stream. Returns the underlying `.IOStream` object and stops all further HTTP processing. May only be called during `.HTTPMessageDelegate.headers_received`. Intended for implementing protocols like websockets that tunnel over an HTTP handshake. """ self._clear_callbacks() stream = self.stream self.stream = None # type: ignore if not self._finish_future.done(): future_set_result_unless_cancelled(self._finish_future, None) return stream def set_body_timeout(self, timeout: float) -> None: """Sets the body timeout for a single request. Overrides the value from `.HTTP1ConnectionParameters`. """ self._body_timeout = timeout def set_max_body_size(self, max_body_size: int) -> None: """Sets the body size limit for a single request. Overrides the value from `.HTTP1ConnectionParameters`. """ self._max_body_size = max_body_size def write_headers( self, start_line: Union[httputil.RequestStartLine, httputil.ResponseStartLine], headers: httputil.HTTPHeaders, chunk: bytes = None, ) -> "Future[None]": """Implements `.HTTPConnection.write_headers`.""" lines = [] if self.is_client: assert isinstance(start_line, httputil.RequestStartLine) self._request_start_line = start_line lines.append( utf8("%s %s HTTP/1.1" % (start_line[0], start_line[1]))) # Client requests with a non-empty body must have either a # Content-Length or a Transfer-Encoding. self._chunking_output = ( start_line.method in ("POST", "PUT", "PATCH") and "Content-Length" not in headers and ("Transfer-Encoding" not in headers or headers["Transfer-Encoding"] == "chunked")) else: assert isinstance(start_line, httputil.ResponseStartLine) assert self._request_start_line is not None assert self._request_headers is not None self._response_start_line = start_line lines.append( utf8("HTTP/1.1 %d %s" % (start_line[1], start_line[2]))) self._chunking_output = ( # TODO: should this use # self._request_start_line.version or # start_line.version? self._request_start_line.version == "HTTP/1.1" # 1xx, 204 and 304 responses have no body (not even a zero-length # body), and so should not have either Content-Length or # Transfer-Encoding headers. and start_line.code not in (204, 304) and (start_line.code < 100 or start_line.code >= 200) # No need to chunk the output if a Content-Length is specified. and "Content-Length" not in headers # Applications are discouraged from touching Transfer-Encoding, # but if they do, leave it alone. and "Transfer-Encoding" not in headers) # If connection to a 1.1 client will be closed, inform client if (self._request_start_line.version == "HTTP/1.1" and self._disconnect_on_finish): headers["Connection"] = "close" # If a 1.0 client asked for keep-alive, add the header. if (self._request_start_line.version == "HTTP/1.0" and self._request_headers.get("Connection", "").lower() == "keep-alive"): headers["Connection"] = "Keep-Alive" if self._chunking_output: headers["Transfer-Encoding"] = "chunked" if not self.is_client and (self._request_start_line.method == "HEAD" or cast(httputil.ResponseStartLine, start_line).code == 304): self._expected_content_remaining = 0 elif "Content-Length" in headers: self._expected_content_remaining = int(headers["Content-Length"]) else: self._expected_content_remaining = None # TODO: headers are supposed to be of type str, but we still have some # cases that let bytes slip through. Remove these native_str calls when those # are fixed. header_lines = (native_str(n) + ": " + native_str(v) for n, v in headers.get_all()) lines.extend(l.encode("latin1") for l in header_lines) for line in lines: if b"\n" in line: raise ValueError("Newline in header: " + repr(line)) future = None if self.stream.closed(): future = self._write_future = Future() future.set_exception(iostream.StreamClosedError()) future.exception() else: future = self._write_future = Future() data = b"\r\n".join(lines) + b"\r\n\r\n" if chunk: data += self._format_chunk(chunk) self._pending_write = self.stream.write(data) future_add_done_callback(self._pending_write, self._on_write_complete) return future def _format_chunk(self, chunk: bytes) -> bytes: if self._expected_content_remaining is not None: self._expected_content_remaining -= len(chunk) if self._expected_content_remaining < 0: # Close the stream now to stop further framing errors. self.stream.close() raise httputil.HTTPOutputError( "Tried to write more data than Content-Length") if self._chunking_output and chunk: # Don't write out empty chunks because that means END-OF-STREAM # with chunked encoding return utf8("%x" % len(chunk)) + b"\r\n" + chunk + b"\r\n" else: return chunk def write(self, chunk: bytes) -> "Future[None]": """Implements `.HTTPConnection.write`. For backwards compatibility it is allowed but deprecated to skip `write_headers` and instead call `write()` with a pre-encoded header block. """ future = None if self.stream.closed(): future = self._write_future = Future() self._write_future.set_exception(iostream.StreamClosedError()) self._write_future.exception() else: future = self._write_future = Future() self._pending_write = self.stream.write(self._format_chunk(chunk)) future_add_done_callback(self._pending_write, self._on_write_complete) return future def finish(self) -> None: """Implements `.HTTPConnection.finish`.""" if (self._expected_content_remaining is not None and self._expected_content_remaining != 0 and not self.stream.closed()): self.stream.close() raise httputil.HTTPOutputError( "Tried to write %d bytes less than Content-Length" % self._expected_content_remaining) if self._chunking_output: if not self.stream.closed(): self._pending_write = self.stream.write(b"0\r\n\r\n") self._pending_write.add_done_callback(self._on_write_complete) self._write_finished = True # If the app finished the request while we're still reading, # divert any remaining data away from the delegate and # close the connection when we're done sending our response. # Closing the connection is the only way to avoid reading the # whole input body. if not self._read_finished: self._disconnect_on_finish = True # No more data is coming, so instruct TCP to send any remaining # data immediately instead of waiting for a full packet or ack. self.stream.set_nodelay(True) if self._pending_write is None: self._finish_request(None) else: future_add_done_callback(self._pending_write, self._finish_request) def _on_write_complete(self, future: "Future[None]") -> None: exc = future.exception() if exc is not None and not isinstance(exc, iostream.StreamClosedError): future.result() if self._write_callback is not None: callback = self._write_callback self._write_callback = None self.stream.io_loop.add_callback(callback) if self._write_future is not None: future = self._write_future self._write_future = None future_set_result_unless_cancelled(future, None) def _can_keep_alive(self, start_line: httputil.RequestStartLine, headers: httputil.HTTPHeaders) -> bool: if self.params.no_keep_alive: return False connection_header = headers.get("Connection") if connection_header is not None: connection_header = connection_header.lower() if start_line.version == "HTTP/1.1": return connection_header != "close" elif ("Content-Length" in headers or headers.get("Transfer-Encoding", "").lower() == "chunked" or getattr(start_line, "method", None) in ("HEAD", "GET")): # start_line may be a request or response start line; only # the former has a method attribute. return connection_header == "keep-alive" return False def _finish_request(self, future: "Optional[Future[None]]") -> None: self._clear_callbacks() if not self.is_client and self._disconnect_on_finish: self.close() return # Turn Nagle's algorithm back on, leaving the stream in its # default state for the next request. self.stream.set_nodelay(False) if not self._finish_future.done(): future_set_result_unless_cancelled(self._finish_future, None) def _parse_headers(self, data: bytes) -> Tuple[str, httputil.HTTPHeaders]: # The lstrip removes newlines that some implementations sometimes # insert between messages of a reused connection. Per RFC 7230, # we SHOULD ignore at least one empty line before the request. # http://tools.ietf.org/html/rfc7230#section-3.5 data_str = native_str(data.decode("latin1")).lstrip("\r\n") # RFC 7230 section allows for both CRLF and bare LF. eol = data_str.find("\n") start_line = data_str[:eol].rstrip("\r") headers = httputil.HTTPHeaders.parse(data_str[eol:]) return start_line, headers def _read_body( self, code: int, headers: httputil.HTTPHeaders, delegate: httputil.HTTPMessageDelegate, ) -> Optional[Awaitable[None]]: if "Content-Length" in headers: if "Transfer-Encoding" in headers: # Response cannot contain both Content-Length and # Transfer-Encoding headers. # http://tools.ietf.org/html/rfc7230#section-3.3.3 raise httputil.HTTPInputError( "Response with both Transfer-Encoding and Content-Length") if "," in headers["Content-Length"]: # Proxies sometimes cause Content-Length headers to get # duplicated. If all the values are identical then we can # use them but if they differ it's an error. pieces = re.split(r",\s*", headers["Content-Length"]) if any(i != pieces[0] for i in pieces): raise httputil.HTTPInputError( "Multiple unequal Content-Lengths: %r" % headers["Content-Length"]) headers["Content-Length"] = pieces[0] try: content_length = int( headers["Content-Length"]) # type: Optional[int] except ValueError: # Handles non-integer Content-Length value. raise httputil.HTTPInputError( "Only integer Content-Length is allowed: %s" % headers["Content-Length"]) if cast(int, content_length) > self._max_body_size: raise httputil.HTTPInputError("Content-Length too long") else: content_length = None if code == 204: # This response code is not allowed to have a non-empty body, # and has an implicit length of zero instead of read-until-close. # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3 if "Transfer-Encoding" in headers or content_length not in (None, 0): raise httputil.HTTPInputError( "Response with code %d should not have body" % code) content_length = 0 if content_length is not None: return self._read_fixed_body(content_length, delegate) if headers.get("Transfer-Encoding", "").lower() == "chunked": return self._read_chunked_body(delegate) if self.is_client: return self._read_body_until_close(delegate) return None async def _read_fixed_body(self, content_length: int, delegate: httputil.HTTPMessageDelegate) -> None: while content_length > 0: body = await self.stream.read_bytes(min(self.params.chunk_size, content_length), partial=True) content_length -= len(body) if not self._write_finished or self.is_client: with _ExceptionLoggingContext(app_log): ret = delegate.data_received(body) if ret is not None: await ret async def _read_chunked_body( self, delegate: httputil.HTTPMessageDelegate) -> None: # TODO: "chunk extensions" http://tools.ietf.org/html/rfc2616#section-3.6.1 total_size = 0 while True: chunk_len_str = await self.stream.read_until(b"\r\n", max_bytes=64) chunk_len = int(chunk_len_str.strip(), 16) if chunk_len == 0: crlf = await self.stream.read_bytes(2) if crlf != b"\r\n": raise httputil.HTTPInputError( "improperly terminated chunked request") return total_size += chunk_len if total_size > self._max_body_size: raise httputil.HTTPInputError("chunked body too large") bytes_to_read = chunk_len while bytes_to_read: chunk = await self.stream.read_bytes(min( bytes_to_read, self.params.chunk_size), partial=True) bytes_to_read -= len(chunk) if not self._write_finished or self.is_client: with _ExceptionLoggingContext(app_log): ret = delegate.data_received(chunk) if ret is not None: await ret # chunk ends with \r\n crlf = await self.stream.read_bytes(2) assert crlf == b"\r\n" async def _read_body_until_close( self, delegate: httputil.HTTPMessageDelegate) -> None: body = await self.stream.read_until_close() if not self._write_finished or self.is_client: with _ExceptionLoggingContext(app_log): ret = delegate.data_received(body) if ret is not None: await ret
class WebSocketClientConnection(simple_httpclient._HTTPConnection): """WebSocket client connection. This class should not be instantiated directly; use the `websocket_connect` function instead. """ def __init__(self, request, on_message_callback=None, compression_options=None, ping_interval=None, ping_timeout=None, max_message_size=None): self.compression_options = compression_options self.connect_future = Future() self.protocol = None self.read_future = None self.read_queue = collections.deque() self.key = base64.b64encode(os.urandom(16)) self._on_message_callback = on_message_callback self.close_code = self.close_reason = None self.ping_interval = ping_interval self.ping_timeout = ping_timeout self.max_message_size = max_message_size scheme, sep, rest = request.url.partition(':') scheme = {'ws': 'http', 'wss': 'https'}[scheme] request.url = scheme + sep + rest request.headers.update({ 'Upgrade': 'websocket', 'Connection': 'Upgrade', 'Sec-WebSocket-Key': self.key, 'Sec-WebSocket-Version': '13', }) if self.compression_options is not None: # Always offer to let the server set our max_wbits (and even though # we don't offer it, we will accept a client_no_context_takeover # from the server). # TODO: set server parameters for deflate extension # if requested in self.compression_options. request.headers['Sec-WebSocket-Extensions'] = ( 'permessage-deflate; client_max_window_bits') self.tcp_client = TCPClient() super(WebSocketClientConnection, self).__init__(None, request, lambda: None, self._on_http_response, 104857600, self.tcp_client, 65536, 104857600) def close(self, code=None, reason=None): """Closes the websocket connection. ``code`` and ``reason`` are documented under `WebSocketHandler.close`. .. versionadded:: 3.2 .. versionchanged:: 4.0 Added the ``code`` and ``reason`` arguments. """ if self.protocol is not None: self.protocol.close(code, reason) self.protocol = None def on_connection_close(self): if not self.connect_future.done(): self.connect_future.set_exception(StreamClosedError()) self.on_message(None) self.tcp_client.close() super(WebSocketClientConnection, self).on_connection_close() def _on_http_response(self, response): if not self.connect_future.done(): if response.error: self.connect_future.set_exception(response.error) else: self.connect_future.set_exception( WebSocketError("Non-websocket response")) def headers_received(self, start_line, headers): if start_line.code != 101: return super(WebSocketClientConnection, self).headers_received(start_line, headers) self.headers = headers self.protocol = self.get_websocket_protocol() self.protocol._process_server_headers(self.key, self.headers) self.protocol.start_pinging() self.protocol._receive_frame() if self._timeout is not None: self.io_loop.remove_timeout(self._timeout) self._timeout = None self.stream = self.connection.detach() self.stream.set_close_callback(self.on_connection_close) # Once we've taken over the connection, clear the final callback # we set on the http request. This deactivates the error handling # in simple_httpclient that would otherwise interfere with our # ability to see exceptions. self.final_callback = None future_set_result_unless_cancelled(self.connect_future, self) def write_message(self, message, binary=False): """Sends a message to the WebSocket server. If the stream is closed, raises `WebSocketClosedError`. Returns a `.Future` which can be used for flow control. .. versionchanged:: 5.0 Exception raised on a closed stream changed from `.StreamClosedError` to `WebSocketClosedError`. """ return self.protocol.write_message(message, binary=binary) def read_message(self, callback=None): """Reads a message from the WebSocket server. If on_message_callback was specified at WebSocket initialization, this function will never return messages Returns a future whose result is the message, or None if the connection is closed. If a callback argument is given it will be called with the future when it is ready. """ assert self.read_future is None future = Future() if self.read_queue: future_set_result_unless_cancelled(future, self.read_queue.popleft()) else: self.read_future = future if callback is not None: self.io_loop.add_future(future, callback) return future def on_message(self, message): if self._on_message_callback: self._on_message_callback(message) elif self.read_future is not None: future_set_result_unless_cancelled(self.read_future, message) self.read_future = None else: self.read_queue.append(message) def on_pong(self, data): pass def on_ping(self, data): pass def get_websocket_protocol(self): return WebSocketProtocol13( self, mask_outgoing=True, compression_options=self.compression_options)
class _Connector(object): """A stateless implementation of the "Happy Eyeballs" algorithm. "Happy Eyeballs" is documented in RFC6555 as the recommended practice for when both IPv4 and IPv6 addresses are available. In this implementation, we partition the addresses by family, and make the first connection attempt to whichever address was returned first by ``getaddrinfo``. If that connection fails or times out, we begin a connection in parallel to the first address of the other family. If there are additional failures we retry with other addresses, keeping one connection attempt per family in flight at a time. http://tools.ietf.org/html/rfc6555 """ def __init__(self, addrinfo, connect): self.io_loop = IOLoop.current() self.connect = connect self.future = Future() self.timeout = None self.connect_timeout = None self.last_error = None self.remaining = len(addrinfo) self.primary_addrs, self.secondary_addrs = self.split(addrinfo) self.streams = set() @staticmethod def split(addrinfo): """Partition the ``addrinfo`` list by address family. Returns two lists. The first list contains the first entry from ``addrinfo`` and all others with the same family, and the second list contains all other addresses (normally one list will be AF_INET and the other AF_INET6, although non-standard resolvers may return additional families). """ primary = [] secondary = [] primary_af = addrinfo[0][0] for af, addr in addrinfo: if af == primary_af: primary.append((af, addr)) else: secondary.append((af, addr)) return primary, secondary def start(self, timeout=_INITIAL_CONNECT_TIMEOUT, connect_timeout=None): self.try_connect(iter(self.primary_addrs)) self.set_timeout(timeout) if connect_timeout is not None: self.set_connect_timeout(connect_timeout) return self.future def try_connect(self, addrs): try: af, addr = next(addrs) except StopIteration: # We've reached the end of our queue, but the other queue # might still be working. Send a final error on the future # only when both queues are finished. if self.remaining == 0 and not self.future.done(): self.future.set_exception(self.last_error or IOError("connection failed")) return stream, future = self.connect(af, addr) self.streams.add(stream) future_add_done_callback( future, functools.partial(self.on_connect_done, addrs, af, addr)) def on_connect_done(self, addrs, af, addr, future): self.remaining -= 1 try: stream = future.result() except Exception as e: if self.future.done(): return # Error: try again (but remember what happened so we have an # error to raise in the end) self.last_error = e self.try_connect(addrs) if self.timeout is not None: # If the first attempt failed, don't wait for the # timeout to try an address from the secondary queue. self.io_loop.remove_timeout(self.timeout) self.on_timeout() return self.clear_timeouts() if self.future.done(): # This is a late arrival; just drop it. stream.close() else: self.streams.discard(stream) self.future.set_result((af, addr, stream)) self.close_streams() def set_timeout(self, timeout): self.timeout = self.io_loop.add_timeout(self.io_loop.time() + timeout, self.on_timeout) def on_timeout(self): self.timeout = None if not self.future.done(): self.try_connect(iter(self.secondary_addrs)) def clear_timeout(self): if self.timeout is not None: self.io_loop.remove_timeout(self.timeout) def set_connect_timeout(self, connect_timeout): self.connect_timeout = self.io_loop.add_timeout( connect_timeout, self.on_connect_timeout) def on_connect_timeout(self): if not self.future.done(): self.future.set_exception(TimeoutError()) self.close_streams() def clear_timeouts(self): if self.timeout is not None: self.io_loop.remove_timeout(self.timeout) if self.connect_timeout is not None: self.io_loop.remove_timeout(self.connect_timeout) def close_streams(self): for stream in self.streams: stream.close()
class HTTP1Connection(httputil.HTTPConnection): def __init__(self, stream, is_client, params=None, context=None): self.is_client = is_client self.stream = stream if params is None: params = HTTP1ConnectionParameters() self.params = params self.context = context self.no_keep_alive = params.no_keep_alive self._max_body_size = (self.params.max_body_size or self.stream.max_buffer_size) self._body_timeout = self.params.body_timeout self._write_finished = False self._read_finished = False self._finish_future = Future() self._disconnect_on_finish = False self._clear_callbacks() self._request_start_line = None self._response_start_line = None self._request_headers = None self._chunking_output = None self._expected_content_remaining = None self._pending_write = None def read_response(self, delegate): return self._read_message(delegate) @gen.coroutine def _read_message(self, delegate): need_delegate_close = False try: header_future = self.stream.read_until_regex( b"\r?\n\r?\n", max_bytes=self.params.max_header_size) if self.params.header_timeout is None: header_data = yield header_future else: try: header_data = yield gen.with_timeout( self.stream.io_loop.time() + self.params.header_timeout, header_future, io_loop=self.stream.io_loop, quiet_exceptions=iostream.StreamClosedError) except gen.TimeoutError: self.close() raise gen.Return(False) start_line, headers = self._parse_headers(header_data) if self.is_client: start_line = httputil.parse_response_start_line(start_line) self._response_start_line = start_line else: start_line = httputil.parse_request_start_line(start_line) self._request_start_line = start_line self._request_headers = headers self._disconnect_on_finish = not self._can_keep_alive( start_line, headers) need_delegate_close = True with _ExceptionLoggingContext(): header_future = delegate.headers_received(start_line, headers) if header_future is not None: yield header_future if self.stream is None: # We've been detached. need_delegate_close = False raise gen.Return(False) skip_body = False if self.is_client: if (self._request_start_line is not None and self._request_start_line.method == 'HEAD'): skip_body = True code = start_line.code if code == 304: skip_body = True if code >= 100 and code < 200: if ('Content-Length' in headers or 'Transfer-Encoding' in headers): raise httputil.HTTPInputError( "Response code %d cannot have body" % code) yield self._read_message(delegate) else: if (headers.get("Expect") == "100-continue" and not self._write_finished): self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n") if not skip_body: body_future = self._read_body( start_line.code if self.is_client else 0, headers, delegate) if body_future is not None: if self._body_timeout is None: yield body_future else: try: yield gen.with_timeout( self.stream.io_loop.time() + self._body_timeout, body_future, self.stream.io_loop, quiet_exceptions=iostream.StreamClosedError) except gen.TimeoutError: print("Timeout reading body from %s" % self.context) self.stream.close() raise gen.Return(False) self._read_finished = True if not self._write_finished or self.is_client: need_delegate_close = False with _ExceptionLoggingContext(): delegate.finish() if (not self._finish_future.done() and self.stream is not None and not self.stream.closed()): self.stream.set_close_callback(self._on_connection_close) yield self._finish_future if self.is_client and self._disconnect_on_finish: self.close() if self.stream is None: raise gen.Return(False) except httputil.HTTPInputError as e: print("Malformed HTTP message from %s: %s", (self.context, e)) self.close() raise gen.Return(False) finally: if need_delegate_close: with _ExceptionLoggingContext(): delegate.on_connection_close() self._clear_callbacks() raise gen.Return(True) def _clear_callbacks(self): self._write_callback = None self._write_future = None self._close_callback = None if self.stream is not None: self.stream.set_close_callback(None) def set_close_callback(self, callback): self._close_callback = stack_context.wrap(callback) def _on_connection_close(self): if self._close_callback is not None: callback = self._close_callback self._close_callback = None callback() if not self._finish_future.done(): self._finish_future.set_result(None) self._clear_callbacks() def close(self): if self.stream is not None: self.stream.close() self._clear_callbacks() if not self._finish_future.done(): self._finish_future.set_result(None) def detach(self): self._clear_callbacks() stream = self.stream self.stream = None if not self._finish_future.done(): self._finish_future.set_result(None) return stream def set_body_timeout(self, timeout): self._body_timeout = timeout def set_max_body_size(self, max_body_size): self._max_body_size = max_body_size def write_headers(self, start_line, headers, chunk=None, callback=None): lines = [] if self.is_client: self._request_start_line = start_line lines.append(utf8('%s %s HTTP/1.1' % (start_line[0], start_line[1]))) self._chunking_output = ( start_line.method in ('POST', 'PUT', 'PATCH') and 'Content-Length' not in headers and 'Transfer-Encoding' not in headers) else: self._response_start_line = start_line lines.append(utf8('HTTP/1.1 %s %s' % (start_line[1], start_line[2]))) self._chunking_output = ( self._request_start_line.version == 'HTTP/1.1' and start_line.code != 304 and 'Content-Length' not in headers and 'Transfer-Encoding' not in headers) if (self._request_start_line.version == 'HTTP/1.0' and (self._request_headers.get('Connection', '').lower() == 'keep-alive')): headers['Connection'] = 'Keep-Alive' if self._chunking_output: headers['Transfer-Encoding'] = 'chunked' if (not self.is_client and (self._request_start_line.method == 'HEAD' or start_line.code == 304)): self._expected_content_remaining = 0 elif 'Content-Length' in headers: self._expected_content_remaining = int(headers['Content-Length']) else: self._expected_content_remaining = None lines.extend([utf8(n) + b": " + utf8(v) for n, v in headers.get_all()]) for line in lines: if b'\n' in line: raise ValueError('Newline in header: ' + repr(line)) future = None if self.stream.closed(): future = self._write_future = Future() future.set_exception(iostream.StreamClosedError()) future.exception() else: if callback is not None: self._write_callback = stack_context.wrap(callback) else: future = self._write_future = Future() data = b"\r\n".join(lines) + b"\r\n\r\n" if chunk: data += self._format_chunk(chunk) self._pending_write = self.stream.write(data) self._pending_write.add_done_callback(self._on_write_complete) return future def _format_chunk(self, chunk): if self._expected_content_remaining is not None: self._expected_content_remaining -= len(chunk) if self._expected_content_remaining < 0: # Close the stream now to stop further framing errors. self.stream.close() raise httputil.HTTPOutputError( "Tried to write more data than Content-Length") if self._chunking_output and chunk: # Don't write out empty chunks because that means END-OF-STREAM # with chunked encoding return utf8("%x" % len(chunk)) + b"\r\n" + chunk + b"\r\n" else: return chunk def write(self, chunk, callback=None): future = None if self.stream.closed(): future = self._write_future = Future() self._write_future.set_exception(iostream.StreamClosedError()) self._write_future.exception() else: if callback is not None: self._write_callback = stack_context.wrap(callback) else: future = self._write_future = Future() self._pending_write = self.stream.write(self._format_chunk(chunk)) self._pending_write.add_done_callback(self._on_write_complete) return future def finish(self): """Implements `.HTTPConnection.finish`.""" if (self._expected_content_remaining is not None and self._expected_content_remaining != 0 and not self.stream.closed()): self.stream.close() raise httputil.HTTPOutputError( "Tried to write %d bytes less than Content-Length" % self._expected_content_remaining) if self._chunking_output: if not self.stream.closed(): self._pending_write = self.stream.write(b"0\r\n\r\n") self._pending_write.add_done_callback(self._on_write_complete) self._write_finished = True if not self._read_finished: self._disconnect_on_finish = True # No more data is coming, so instruct TCP to send any remaining # data immediately instead of waiting for a full packet or ack. self.stream.set_nodelay(True) if self._pending_write is None: self._finish_request(None) else: self._pending_write.add_done_callback(self._finish_request) def _on_write_complete(self, future): exc = future.exception() if exc is not None and not isinstance(exc, iostream.StreamClosedError): future.result() if self._write_callback is not None: callback = self._write_callback self._write_callback = None self.stream.io_loop.add_callback(callback) if self._write_future is not None: future = self._write_future self._write_future = None future.set_result(None) def _can_keep_alive(self, start_line, headers): if self.params.no_keep_alive: return False connection_header = headers.get("Connection") if connection_header is not None: connection_header = connection_header.lower() if start_line.version == "HTTP/1.1": return connection_header != "close" elif ("Content-Length" in headers or headers.get("Transfer-Encoding", "").lower() == "chunked" or start_line.method in ("HEAD", "GET")): return connection_header == "keep-alive" return False def _finish_request(self, future): self._clear_callbacks() if not self.is_client and self._disconnect_on_finish: self.close() return self.stream.set_nodelay(False) if not self._finish_future.done(): self._finish_future.set_result(None) def _parse_headers(self, data): data = native_str(data.decode('latin1')).lstrip("\r\n") # RFC 7230 section allows for both CRLF and bare LF. eol = data.find("\n") start_line = data[:eol].rstrip("\r") try: headers = httputil.HTTPHeaders.parse(data[eol:]) except ValueError: # probably form split() if there was no ':' in the line raise httputil.HTTPInputError("Malformed HTTP headers: %r" % data[eol:100]) return start_line, headers def _read_body(self, code, headers, delegate): if "Content-Length" in headers: if "," in headers["Content-Length"]: pieces = re.split(r',\s*', headers["Content-Length"]) if any(i != pieces[0] for i in pieces): raise httputil.HTTPInputError( "Multiple unequal Content-Lengths: %r" % headers["Content-Length"]) headers["Content-Length"] = pieces[0] content_length = int(headers["Content-Length"]) if content_length > self._max_body_size: raise httputil.HTTPInputError("Content-Length too long") else: content_length = None if code == 204: if ("Transfer-Encoding" in headers or content_length not in (None, 0)): raise httputil.HTTPInputError( "Response with code %d should not have body" % code) content_length = 0 if content_length is not None: return self._read_fixed_body(content_length, delegate) if headers.get("Transfer-Encoding") == "chunked": return self._read_chunked_body(delegate) if self.is_client: return self._read_body_until_close(delegate) return None @gen.coroutine def _read_fixed_body(self, content_length, delegate): while content_length > 0: body = yield self.stream.read_bytes( min(self.params.chunk_size, content_length), partial=True) content_length -= len(body) if not self._write_finished or self.is_client: with _ExceptionLoggingContext(): yield gen.maybe_future(delegate.data_received(body)) @gen.coroutine def _read_chunked_body(self, delegate): # TODO: "chunk extensions" http://tools.ietf.org/html/rfc2616#section-3.6.1 total_size = 0 while True: chunk_len = yield self.stream.read_until(b"\r\n", max_bytes=64) chunk_len = int(chunk_len.strip(), 16) if chunk_len == 0: return total_size += chunk_len if total_size > self._max_body_size: raise httputil.HTTPInputError("chunked body too large") bytes_to_read = chunk_len while bytes_to_read: chunk = yield self.stream.read_bytes( min(bytes_to_read, self.params.chunk_size), partial=True) bytes_to_read -= len(chunk) if not self._write_finished or self.is_client: with _ExceptionLoggingContext(): yield gen.maybe_future(delegate.data_received(chunk)) # chunk ends with \r\n crlf = yield self.stream.read_bytes(2) assert crlf == b"\r\n" @gen.coroutine def _read_body_until_close(self, delegate): body = yield self.stream.read_until_close() if not self._write_finished or self.is_client: with _ExceptionLoggingContext(): delegate.data_received(body)
class Stream(object): def __init__(self, conn, stream_id, delegate, context=None): self.conn = conn self.stream_id = stream_id self.set_delegate(delegate) self.context = context self.finish_future = Future() self.write_lock = Lock() from tornado.util import ObjectDict # TODO: remove self.stream = ObjectDict(io_loop=IOLoop.current(), close=conn.stream.close) self._incoming_content_remaining = None self._outgoing_content_remaining = None self._delegate_started = False self.window = Window(conn.window, stream_id, conn.setting(constants.Setting.INITIAL_WINDOW_SIZE)) self._header_frames = [] self._phase = constants.HTTPPhase.HEADERS def set_delegate(self, delegate): self.orig_delegate = self.delegate = delegate if self.conn.params.decompress: self.delegate = _GzipMessageDelegate(delegate, self.conn.params.chunk_size) def handle_frame(self, frame): if frame.type == constants.FrameType.PRIORITY: self._handle_priority_frame(frame) return elif frame.type == constants.FrameType.RST_STREAM: self._handle_rst_stream_frame(frame) return elif frame.type == constants.FrameType.WINDOW_UPDATE: self._handle_window_update_frame(frame) return elif frame.type in (constants.FrameType.SETTINGS, constants.FrameType.GOAWAY, constants.FrameType.PUSH_PROMISE): raise Exception("invalid frame type %s for stream", frame.type) if self.finish_future.done(): raise StreamError(self.stream_id, constants.ErrorCode.STREAM_CLOSED) if frame.type == constants.FrameType.HEADERS: self._handle_headers_frame(frame) elif frame.type == constants.FrameType.CONTINUATION: self._handle_continuation_frame(frame) elif frame.type == constants.FrameType.DATA: self._handle_data_frame(frame) # Unknown frame types are silently discarded, unless they break # the rule that nothing can come between HEADERS and CONTINUATION. def needs_continuation(self): return bool(self._header_frames) def _handle_headers_frame(self, frame): if self._phase == constants.HTTPPhase.BODY: self._phase = constants.HTTPPhase.TRAILERS frame = frame.without_padding() self._header_frames.append(frame) self._check_header_length() if frame.flags & constants.FrameFlag.END_HEADERS: self._parse_headers() def _handle_continuation_frame(self, frame): if not self._header_frames: raise ConnectionError(constants.ErrorCode.PROTOCOL_ERROR, "CONTINUATION without HEADERS") self._header_frames.append(frame) self._check_header_length() if frame.flags & constants.FrameFlag.END_HEADERS: self._parse_headers() def _check_header_length(self): if (sum(len(f.data) for f in self._header_frames) > self.conn.params.max_header_size): if self.conn.is_client: # TODO: Need tests for client side of headers-too-large. # What's the best way to send an error? self.delegate.on_connection_close() else: # write_headers needs a start line so it can tell # whether this is a HEAD or not. If we're rejecting # the headers we can't know so just make something up. # Note that this means the error response body MUST be # zero bytes so it doesn't matter whether the client # sent a HEAD or a GET. self._request_start_line = RequestStartLine('GET', '/', 'HTTP/2.0') start_line = ResponseStartLine('HTTP/2.0', 431, 'Headers too large') self.write_headers(start_line, HTTPHeaders()) self.finish() return def _parse_headers(self): frame = self._header_frames[0] data = b''.join(f.data for f in self._header_frames) self._header_frames = [] if frame.flags & constants.FrameFlag.PRIORITY: # TODO: support PRIORITY and PADDING. # This is just enough to cover an error case tested in h2spec. stream_dep, weight = struct.unpack('>ib', data[:5]) data = data[5:] # strip off the "exclusive" bit stream_dep = stream_dep & 0x7fffffff if stream_dep == frame.stream_id: raise ConnectionError(constants.ErrorCode.PROTOCOL_ERROR, "stream cannot depend on itself") pseudo_headers = {} headers = HTTPHeaders() try: # Pseudo-headers must come before any regular headers, # and only in the first HEADERS phase. has_regular_header = bool(self._phase == constants.HTTPPhase.TRAILERS) for k, v, idx in self.conn.hpack_decoder.decode(bytearray(data)): if k != k.lower(): # RFC section 8.1.2 raise StreamError(self.stream_id, constants.ErrorCode.PROTOCOL_ERROR) if k.startswith(b':'): if self.conn.is_client: valid_pseudo_headers = (b':status',) else: valid_pseudo_headers = (b':method', b':scheme', b':authority', b':path') if (has_regular_header or k not in valid_pseudo_headers or native_str(k) in pseudo_headers): raise StreamError(self.stream_id, constants.ErrorCode.PROTOCOL_ERROR) pseudo_headers[native_str(k)] = native_str(v) if k == b":authority": headers.add("Host", native_str(v)) else: headers.add(native_str(k), native_str(v)) has_regular_header = True except HpackError: raise ConnectionError(constants.ErrorCode.COMPRESSION_ERROR) if self._phase == constants.HTTPPhase.HEADERS: self._start_request(pseudo_headers, headers) elif self._phase == constants.HTTPPhase.TRAILERS: # TODO: support trailers pass if (not self._maybe_end_stream(frame.flags) and self._phase == constants.HTTPPhase.TRAILERS): # The frame that finishes the trailers must also finish # the stream. raise StreamError(self.stream_id, constants.ErrorCode.PROTOCOL_ERROR) def _start_request(self, pseudo_headers, headers): if "connection" in headers: raise ConnectionError(constants.ErrorCode.PROTOCOL_ERROR, "connection header should not be present") if "te" in headers and headers["te"] != "trailers": raise StreamError(self.stream_id, constants.ErrorCode.PROTOCOL_ERROR) if self.conn.is_client: status = int(pseudo_headers[':status']) start_line = ResponseStartLine('HTTP/2.0', status, responses.get(status, '')) else: for k in (':method', ':scheme', ':path'): if k not in pseudo_headers: raise StreamError(self.stream_id, constants.ErrorCode.PROTOCOL_ERROR) start_line = RequestStartLine(pseudo_headers[':method'], pseudo_headers[':path'], 'HTTP/2.0') self._request_start_line = start_line if (self.conn.is_client and (self._request_start_line.method == 'HEAD' or start_line.code == 304)): self._incoming_content_remaining = 0 elif "content-length" in headers: self._incoming_content_remaining = int(headers["content-length"]) if not self.conn.is_client or status >= 200: self._phase = constants.HTTPPhase.BODY self._delegate_started = True self.delegate.headers_received(start_line, headers) def _handle_data_frame(self, frame): if self._header_frames: raise ConnectionError(constants.ErrorCode.PROTOCOL_ERROR, "DATA without END_HEADERS") if self._phase == constants.HTTPPhase.TRAILERS: raise ConnectionError(constants.ErrorCode.PROTOCOL_ERROR, "DATA after trailers") self._phase = constants.HTTPPhase.BODY frame = frame.without_padding() if self._incoming_content_remaining is not None: self._incoming_content_remaining -= len(frame.data) if self._incoming_content_remaining < 0: raise StreamError(self.stream_id, constants.ErrorCode.PROTOCOL_ERROR) if frame.data and self._delegate_started: future = self.delegate.data_received(frame.data) if future is None: self._send_window_update(len(frame.data)) else: IOLoop.current().add_future( future, lambda f: self._send_window_update(len(frame.data))) self._maybe_end_stream(frame.flags) def _send_window_update(self, amount): encoded = struct.pack('>I', amount) for stream_id in (0, self.stream_id): self.conn._write_frame(Frame( constants.FrameType.WINDOW_UPDATE, 0, stream_id, encoded)) def _maybe_end_stream(self, flags): if flags & constants.FrameFlag.END_STREAM: if (self._incoming_content_remaining is not None and self._incoming_content_remaining != 0): raise StreamError(self.stream_id, constants.ErrorCode.PROTOCOL_ERROR) if self._delegate_started: self._delegate_started = False self.delegate.finish() self.finish_future.set_result(None) return True return False def _handle_priority_frame(self, frame): # TODO: implement priority if len(frame.data) != 5: raise StreamError(self.stream_id, constants.ErrorCode.FRAME_SIZE_ERROR) def _handle_rst_stream_frame(self, frame): if len(frame.data) != 4: raise ConnectionError(constants.ErrorCode.FRAME_SIZE_ERROR) # TODO: expose error code? if self._delegate_started: self.delegate.on_connection_close() def _handle_window_update_frame(self, frame): self.window.apply_window_update(frame) def set_close_callback(self, callback): # TODO: this shouldn't be necessary pass def reset(self): self.conn._write_frame(Frame(constants.FrameType.RST_STREAM, 0, self.stream_id, b'\x00\x00\x00\x00')) @_reset_on_error def write_headers(self, start_line, headers, chunk=None, callback=None): if (not self.conn.is_client and (self._request_start_line.method == 'HEAD' or start_line.code == 304)): self._outgoing_content_remaining = 0 elif 'Content-Length' in headers: self._outgoing_content_remaining = int(headers['Content-Length']) header_list = [] if self.conn.is_client: self._request_start_line = start_line header_list.append((b':method', utf8(start_line.method), constants.HeaderIndexMode.YES)) header_list.append((b':scheme', b'https', constants.HeaderIndexMode.YES)) header_list.append((b':path', utf8(start_line.path), constants.HeaderIndexMode.NO)) else: header_list.append((b':status', utf8(str(start_line.code)), constants.HeaderIndexMode.YES)) for k, v in headers.get_all(): k = utf8(k.lower()) if k == b"connection": # Remove the implicit "connection: close", which is not # allowed in http2. # TODO: move the responsibility for this from httpclient # to http1connection? continue header_list.append((k, utf8(v), constants.HeaderIndexMode.YES)) data = bytes(self.conn.hpack_encoder.encode(header_list)) frame = Frame(constants.FrameType.HEADERS, constants.FrameFlag.END_HEADERS, self.stream_id, data) self.conn._write_frame(frame) return self.write(chunk, callback) @_reset_on_error def write(self, chunk, callback=None): if chunk: if self._outgoing_content_remaining is not None: self._outgoing_content_remaining -= len(chunk) if self._outgoing_content_remaining < 0: raise HTTPOutputError( "Tried to write more data than Content-Length") return self._write_chunk(chunk, callback) @gen.coroutine def _write_chunk(self, chunk, callback=None): try: if chunk: yield self.write_lock.acquire() while chunk: allowance = yield self.window.consume(len(chunk)) yield self.conn._write_frame( Frame(constants.FrameType.DATA, 0, self.stream_id, chunk[:allowance])) chunk = chunk[allowance:] self.write_lock.release() if callback is not None: callback() except Exception: self.reset() raise @_reset_on_error def finish(self): if (self._outgoing_content_remaining is not None and self._outgoing_content_remaining != 0): raise HTTPOutputError( "Tried to write %d bytes less than Content-Length" % self._outgoing_content_remaining) return self._write_end_stream() @gen.coroutine def _write_end_stream(self): # Callers are not required to wait for write() before calling finish, # so we must manually lock. yield self.write_lock.acquire() try: self.conn._write_frame(Frame(constants.FrameType.DATA, constants.FrameFlag.END_STREAM, self.stream_id, b'')) except Exception: self.reset() raise finally: self.write_lock.release() def read_response(self, delegate): assert delegate is self.orig_delegate, 'cannot change delegate' return self.finish_future
class HTTP1Connection(httputil.HTTPConnection): """Implements the HTTP/1.x protocol. This class can be on its own for clients, or via `HTTP1ServerConnection` for servers. """ def __init__(self, stream, is_client, params=None, context=None): """ :arg stream: an `.IOStream` :arg bool is_client: client or server :arg params: a `.HTTP1ConnectionParameters` instance or ``None`` :arg context: an opaque application-defined object that can be accessed as ``connection.context``. """ self.is_client = is_client self.stream = stream if params is None: params = HTTP1ConnectionParameters() self.params = params self.context = context self.no_keep_alive = params.no_keep_alive # The body limits can be altered by the delegate, so save them # here instead of just referencing self.params later. self._max_body_size = (self.params.max_body_size or self.stream.max_buffer_size) self._body_timeout = self.params.body_timeout # _write_finished is set to True when finish() has been called, # i.e. there will be no more data sent. Data may still be in the # stream's write buffer. self._write_finished = False # True when we have read the entire incoming body. self._read_finished = False # _finish_future resolves when all data has been written and flushed # to the IOStream. self._finish_future = Future() # If true, the connection should be closed after this request # (after the response has been written in the server side, # and after it has been read in the client) self._disconnect_on_finish = False self._clear_callbacks() # Save the start lines after we read or write them; they # affect later processing (e.g. 304 responses and HEAD methods # have content-length but no bodies) self._request_start_line = None self._response_start_line = None self._request_headers = None # True if we are writing output with chunked encoding. self._chunking_output = None # While reading a body with a content-length, this is the # amount left to read. self._expected_content_remaining = None # A Future for our outgoing writes, returned by IOStream.write. self._pending_write = None def read_response(self, delegate): """Read a single HTTP response. Typical client-mode usage is to write a request using `write_headers`, `write`, and `finish`, and then call ``read_response``. :arg delegate: a `.HTTPMessageDelegate` Returns a `.Future` that resolves to None after the full response has been read. """ if self.params.decompress: delegate = _GzipMessageDelegate(delegate, self.params.chunk_size) return self._read_message(delegate) @gen.coroutine def _read_message(self, delegate): need_delegate_close = False try: header_future = self.stream.read_until_regex( b"\r?\n\r?\n", max_bytes=self.params.max_header_size) if self.params.header_timeout is None: header_data = yield header_future else: try: header_data = yield gen.with_timeout( self.stream.io_loop.time() + self.params.header_timeout, header_future, io_loop=self.stream.io_loop, quiet_exceptions=iostream.StreamClosedError) except gen.TimeoutError: self.close() raise gen.Return(False) start_line, headers = self._parse_headers(header_data) if self.is_client: start_line = httputil.parse_response_start_line(start_line) self._response_start_line = start_line else: start_line = httputil.parse_request_start_line(start_line) self._request_start_line = start_line self._request_headers = headers self._disconnect_on_finish = not self._can_keep_alive( start_line, headers) need_delegate_close = True with _ExceptionLoggingContext(app_log): header_future = delegate.headers_received(start_line, headers) if header_future is not None: yield header_future if self.stream is None: # We've been detached. need_delegate_close = False raise gen.Return(False) skip_body = False if self.is_client: if (self._request_start_line is not None and self._request_start_line.method == 'HEAD'): skip_body = True code = start_line.code if code == 304: # 304 responses may include the content-length header # but do not actually have a body. # http://tools.ietf.org/html/rfc7230#section-3.3 skip_body = True if code >= 100 and code < 200: # 1xx responses should never indicate the presence of # a body. if ('Content-Length' in headers or 'Transfer-Encoding' in headers): raise httputil.HTTPInputError( "Response code %d cannot have body" % code) # TODO: client delegates will get headers_received twice # in the case of a 100-continue. Document or change? yield self._read_message(delegate) else: if (headers.get("Expect") == "100-continue" and not self._write_finished): self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n") if not skip_body: body_future = self._read_body( start_line.code if self.is_client else 0, headers, delegate) if body_future is not None: if self._body_timeout is None: yield body_future else: try: yield gen.with_timeout( self.stream.io_loop.time() + self._body_timeout, body_future, self.stream.io_loop, quiet_exceptions=iostream.StreamClosedError) except gen.TimeoutError: gen_log.info("Timeout reading body from %s", self.context) self.stream.close() raise gen.Return(False) self._read_finished = True if not self._write_finished or self.is_client: need_delegate_close = False with _ExceptionLoggingContext(app_log): delegate.finish() # If we're waiting for the application to produce an asynchronous # response, and we're not detached, register a close callback # on the stream (we didn't need one while we were reading) if (not self._finish_future.done() and self.stream is not None and not self.stream.closed()): self.stream.set_close_callback(self._on_connection_close) yield self._finish_future if self.is_client and self._disconnect_on_finish: self.close() if self.stream is None: raise gen.Return(False) except httputil.HTTPInputError as e: gen_log.info("Malformed HTTP message from %s: %s", self.context, e) self.close() raise gen.Return(False) finally: if need_delegate_close: with _ExceptionLoggingContext(app_log): delegate.on_connection_close() self._clear_callbacks() raise gen.Return(True) def _clear_callbacks(self): """Clears the callback attributes. This allows the request handler to be garbage collected more quickly in CPython by breaking up reference cycles. """ self._write_callback = None self._write_future = None self._close_callback = None if self.stream is not None: self.stream.set_close_callback(None) def set_close_callback(self, callback): """Sets a callback that will be run when the connection is closed. .. deprecated:: 4.0 Use `.HTTPMessageDelegate.on_connection_close` instead. """ self._close_callback = stack_context.wrap(callback) def _on_connection_close(self): # Note that this callback is only registered on the IOStream # when we have finished reading the request and are waiting for # the application to produce its response. if self._close_callback is not None: callback = self._close_callback self._close_callback = None callback() if not self._finish_future.done(): self._finish_future.set_result(None) self._clear_callbacks() def close(self): if self.stream is not None: self.stream.close() self._clear_callbacks() if not self._finish_future.done(): self._finish_future.set_result(None) def detach(self): """Take control of the underlying stream. Returns the underlying `.IOStream` object and stops all further HTTP processing. May only be called during `.HTTPMessageDelegate.headers_received`. Intended for implementing protocols like websockets that tunnel over an HTTP handshake. """ self._clear_callbacks() stream = self.stream self.stream = None if not self._finish_future.done(): self._finish_future.set_result(None) return stream def set_body_timeout(self, timeout): """Sets the body timeout for a single request. Overrides the value from `.HTTP1ConnectionParameters`. """ self._body_timeout = timeout def set_max_body_size(self, max_body_size): """Sets the body size limit for a single request. Overrides the value from `.HTTP1ConnectionParameters`. """ self._max_body_size = max_body_size def write_headers(self, start_line, headers, chunk=None, callback=None): """Implements `.HTTPConnection.write_headers`.""" lines = [] if self.is_client: self._request_start_line = start_line lines.append( utf8('%s %s HTTP/1.1' % (start_line[0], start_line[1]))) # Client requests with a non-empty body must have either a # Content-Length or a Transfer-Encoding. self._chunking_output = (start_line.method in ('POST', 'PUT', 'PATCH') and 'Content-Length' not in headers and 'Transfer-Encoding' not in headers) else: self._response_start_line = start_line lines.append( utf8('HTTP/1.1 %d %s' % (start_line[1], start_line[2]))) self._chunking_output = ( # TODO: should this use # self._request_start_line.version or # start_line.version? self._request_start_line.version == 'HTTP/1.1' and # 304 responses have no body (not even a zero-length body), and so # should not have either Content-Length or Transfer-Encoding. # headers. start_line.code != 304 and # No need to chunk the output if a Content-Length is specified. 'Content-Length' not in headers and # Applications are discouraged from touching Transfer-Encoding, # but if they do, leave it alone. 'Transfer-Encoding' not in headers) # If a 1.0 client asked for keep-alive, add the header. if (self._request_start_line.version == 'HTTP/1.0' and (self._request_headers.get('Connection', '').lower() == 'keep-alive')): headers['Connection'] = 'Keep-Alive' if self._chunking_output: headers['Transfer-Encoding'] = 'chunked' if (not self.is_client and (self._request_start_line.method == 'HEAD' or start_line.code == 304)): self._expected_content_remaining = 0 elif 'Content-Length' in headers: self._expected_content_remaining = int(headers['Content-Length']) else: self._expected_content_remaining = None lines.extend([utf8(n) + b": " + utf8(v) for n, v in headers.get_all()]) for line in lines: if b'\n' in line: raise ValueError('Newline in header: ' + repr(line)) future = None if self.stream.closed(): future = self._write_future = Future() future.set_exception(iostream.StreamClosedError()) future.exception() else: if callback is not None: self._write_callback = stack_context.wrap(callback) else: future = self._write_future = Future() data = b"\r\n".join(lines) + b"\r\n\r\n" if chunk: data += self._format_chunk(chunk) self._pending_write = self.stream.write(data) self._pending_write.add_done_callback(self._on_write_complete) return future def _format_chunk(self, chunk): if self._expected_content_remaining is not None: self._expected_content_remaining -= len(chunk) if self._expected_content_remaining < 0: # Close the stream now to stop further framing errors. self.stream.close() raise httputil.HTTPOutputError( "Tried to write more data than Content-Length") if self._chunking_output and chunk: # Don't write out empty chunks because that means END-OF-STREAM # with chunked encoding return utf8("%x" % len(chunk)) + b"\r\n" + chunk + b"\r\n" else: return chunk def write(self, chunk, callback=None): """Implements `.HTTPConnection.write`. For backwards compatibility is is allowed but deprecated to skip `write_headers` and instead call `write()` with a pre-encoded header block. """ future = None if self.stream.closed(): future = self._write_future = Future() self._write_future.set_exception(iostream.StreamClosedError()) self._write_future.exception() else: if callback is not None: self._write_callback = stack_context.wrap(callback) else: future = self._write_future = Future() self._pending_write = self.stream.write(self._format_chunk(chunk)) self._pending_write.add_done_callback(self._on_write_complete) return future def finish(self): """Implements `.HTTPConnection.finish`.""" if (self._expected_content_remaining is not None and self._expected_content_remaining != 0 and not self.stream.closed()): self.stream.close() raise httputil.HTTPOutputError( "Tried to write %d bytes less than Content-Length" % self._expected_content_remaining) if self._chunking_output: if not self.stream.closed(): self._pending_write = self.stream.write(b"0\r\n\r\n") self._pending_write.add_done_callback(self._on_write_complete) self._write_finished = True # If the app finished the request while we're still reading, # divert any remaining data away from the delegate and # close the connection when we're done sending our response. # Closing the connection is the only way to avoid reading the # whole input body. if not self._read_finished: self._disconnect_on_finish = True # No more data is coming, so instruct TCP to send any remaining # data immediately instead of waiting for a full packet or ack. self.stream.set_nodelay(True) if self._pending_write is None: self._finish_request(None) else: self._pending_write.add_done_callback(self._finish_request) def _on_write_complete(self, future): exc = future.exception() if exc is not None and not isinstance(exc, iostream.StreamClosedError): future.result() if self._write_callback is not None: callback = self._write_callback self._write_callback = None self.stream.io_loop.add_callback(callback) if self._write_future is not None: future = self._write_future self._write_future = None future.set_result(None) def _can_keep_alive(self, start_line, headers): if self.params.no_keep_alive: return False connection_header = headers.get("Connection") if connection_header is not None: connection_header = connection_header.lower() if start_line.version == "HTTP/1.1": return connection_header != "close" elif ("Content-Length" in headers or headers.get("Transfer-Encoding", "").lower() == "chunked" or start_line.method in ("HEAD", "GET")): return connection_header == "keep-alive" return False def _finish_request(self, future): self._clear_callbacks() if not self.is_client and self._disconnect_on_finish: self.close() return # Turn Nagle's algorithm back on, leaving the stream in its # default state for the next request. self.stream.set_nodelay(False) if not self._finish_future.done(): self._finish_future.set_result(None) def _parse_headers(self, data): # The lstrip removes newlines that some implementations sometimes # insert between messages of a reused connection. Per RFC 7230, # we SHOULD ignore at least one empty line before the request. # http://tools.ietf.org/html/rfc7230#section-3.5 data = native_str(data.decode('latin1')).lstrip("\r\n") # RFC 7230 section allows for both CRLF and bare LF. eol = data.find("\n") start_line = data[:eol].rstrip("\r") try: headers = httputil.HTTPHeaders.parse(data[eol:]) except ValueError: # probably form split() if there was no ':' in the line raise httputil.HTTPInputError("Malformed HTTP headers: %r" % data[eol:100]) return start_line, headers def _read_body(self, code, headers, delegate): if "Content-Length" in headers: if "Transfer-Encoding" in headers: # Response cannot contain both Content-Length and # Transfer-Encoding headers. # http://tools.ietf.org/html/rfc7230#section-3.3.3 raise httputil.HTTPInputError( "Response with both Transfer-Encoding and Content-Length") if "," in headers["Content-Length"]: # Proxies sometimes cause Content-Length headers to get # duplicated. If all the values are identical then we can # use them but if they differ it's an error. pieces = re.split(r',\s*', headers["Content-Length"]) if any(i != pieces[0] for i in pieces): raise httputil.HTTPInputError( "Multiple unequal Content-Lengths: %r" % headers["Content-Length"]) headers["Content-Length"] = pieces[0] content_length = int(headers["Content-Length"]) if content_length > self._max_body_size: raise httputil.HTTPInputError("Content-Length too long") else: content_length = None if code == 204: # This response code is not allowed to have a non-empty body, # and has an implicit length of zero instead of read-until-close. # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3 if ("Transfer-Encoding" in headers or content_length not in (None, 0)): raise httputil.HTTPInputError( "Response with code %d should not have body" % code) content_length = 0 if content_length is not None: return self._read_fixed_body(content_length, delegate) if headers.get("Transfer-Encoding") == "chunked": return self._read_chunked_body(delegate) if self.is_client: return self._read_body_until_close(delegate) return None @gen.coroutine def _read_fixed_body(self, content_length, delegate): while content_length > 0: body = yield self.stream.read_bytes(min(self.params.chunk_size, content_length), partial=True) content_length -= len(body) if not self._write_finished or self.is_client: with _ExceptionLoggingContext(app_log): ret = delegate.data_received(body) if ret is not None: yield ret @gen.coroutine def _read_chunked_body(self, delegate): # TODO: "chunk extensions" http://tools.ietf.org/html/rfc2616#section-3.6.1 total_size = 0 while True: chunk_len = yield self.stream.read_until(b"\r\n", max_bytes=64) chunk_len = int(chunk_len.strip(), 16) if chunk_len == 0: return total_size += chunk_len if total_size > self._max_body_size: raise httputil.HTTPInputError("chunked body too large") bytes_to_read = chunk_len while bytes_to_read: chunk = yield self.stream.read_bytes(min( bytes_to_read, self.params.chunk_size), partial=True) bytes_to_read -= len(chunk) if not self._write_finished or self.is_client: with _ExceptionLoggingContext(app_log): ret = delegate.data_received(chunk) if ret is not None: yield ret # chunk ends with \r\n crlf = yield self.stream.read_bytes(2) assert crlf == b"\r\n" @gen.coroutine def _read_body_until_close(self, delegate): body = yield self.stream.read_until_close() if not self._write_finished or self.is_client: with _ExceptionLoggingContext(app_log): delegate.data_received(body)
class ZMQChannelsHandler(ZMQStreamHandler, WebSocketMixin, KernelAPIHandler): # class-level registry of open sessions # allows checking for conflict on session-id, # which is used as a zmq identity and must be unique. _open_sessions = {} allow_origin = '*' def __repr__(self): return "%s(%s)" % (self.__class__.__name__, getattr(self, 'kernel_id', 'uninitialized')) def initialize(self): """ called by tornado """ super(ZMQChannelsHandler, self).initialize() self.zmq_stream = None self.channels = {} self.kernel_id = None self.kernel_info_channel = None self._kernel_info_future = Future() self._close_future = Future() self.session_key = '' def create_stream(self): km = self.kernel_manager identity = self.session.bsession for channel in ('shell', 'control', 'iopub', 'stdin'): meth = getattr(km, 'connect_' + channel) self.channels[channel] = stream = meth(self.kernel_id, identity=identity) stream.channel = channel def request_kernel_info(self): """send a request for kernel_info""" km = self.kernel_manager kernel = km.get_kernel(self.kernel_id) try: # check for previous request future = kernel._kernel_info_future except AttributeError: self.log.debug("Requesting kernel info from %s", self.kernel_id) # Create a kernel_info channel to query the kernel protocol version. # This channel will be closed after the kernel_info reply is received. if self.kernel_info_channel is None: self.kernel_info_channel = km.connect_shell(self.kernel_id) self.kernel_info_channel.on_recv(self._handle_kernel_info_reply) self.session.send(self.kernel_info_channel, "kernel_info_request") # store the future on the kernel, so only one request is sent kernel._kernel_info_future = self._kernel_info_future else: if not future.done(): self.log.debug("Waiting for pending kernel_info request") future.add_done_callback( lambda f: self._finish_kernel_info(f.result())) return self._kernel_info_future def _handle_kernel_info_reply(self, msg): """process the kernel_info_reply enabling msg spec adaptation, if necessary """ idents, msg = self.session.feed_identities(msg) try: msg = self.session.deserialize(msg) except: self.log.error("Bad kernel_info reply", exc_info=True) self._kernel_info_future.set_result({}) return else: info = msg['content'] self.log.debug("Received kernel info: %s", info) if msg['msg_type'] != 'kernel_info_reply' or 'protocol_version' not in info: self.log.error( "Kernel info request failed, assuming current %s", info) info = {} self._finish_kernel_info(info) # close the kernel_info channel, we don't need it anymore if self.kernel_info_channel: self.kernel_info_channel.close() self.kernel_info_channel = None def _finish_kernel_info(self, info): """Finish handling kernel_info reply Set up protocol adaptation, if needed, and signal that connection can continue. """ protocol_version = info.get('protocol_version', client_protocol_version) if protocol_version != client_protocol_version: self.session.adapt_version = int(protocol_version.split('.')[0]) self.log.info( "Adapting from protocol version {protocol_version} (kernel {kernel_id}) to {client_protocol_version} (client)." .format(protocol_version=protocol_version, kernel_id=self.kernel_id, client_protocol_version=client_protocol_version)) if not self._kernel_info_future.done(): self._kernel_info_future.set_result(info) async def pre_get(self): # authenticate first await super(ZMQChannelsHandler, self).pre_get() # check session collision: await self._register_session() # then request kernel info, waiting up to a certain time before giving up. # We don't want to wait forever, because browsers don't take it well when # servers never respond to websocket connection requests. kernel = self.kernel_manager.get_kernel(self.kernel_id) self.session.key = kernel.session.key future = self.request_kernel_info() def give_up(): """Don't wait forever for the kernel to reply""" if future.done(): return self.log.warning("Timeout waiting for kernel_info reply from %s", self.kernel_id) future.set_result({}) loop = IOLoop.current() loop.add_timeout(KERNEL_INFO_TIMEOUT, give_up) # actually wait for it res = await future return res async def _register_session(self): """Ensure we aren't creating a duplicate session. If a previous identical session is still open, close it to avoid collisions. This is likely due to a client reconnecting from a lost network connection, where the socket on our side has not been cleaned up yet. """ if self.kernel_id is None or self.session.session is None: raise web.HTTPError(500, u'Invalid Kernel ID or Kernel Session') self.session_key = '%s:%s' % (self.kernel_id, self.session.session) stale_handler = self._open_sessions.get(self.session_key) if stale_handler: self.log.warning("Replacing stale connection: %s", self.session_key) await stale_handler.close() self._open_sessions[self.session_key] = self async def get(self, kernel_id): self.kernel_id = cast_unicode(kernel_id, 'ascii') self.session = self.kernel_manager.get_kernel(self.kernel_id).session return await super(ZMQChannelsHandler, self).get(kernel_id=kernel_id) def open(self, kernel_id): super(ZMQChannelsHandler, self).open() km = self.kernel_manager try: self.create_stream() except web.HTTPError as e: self.log.error("Error opening stream: %s", e) # WebSockets don't response to traditional error codes so we # close the connection. for channel, stream in self.channels.items(): if not stream.closed(): stream.close() self.close() return km.add_restart_callback(self.kernel_id, self.on_kernel_restarted) km.add_restart_callback(self.kernel_id, self.on_restart_failed, 'dead') for channel, stream in self.channels.items(): stream.on_recv_stream(self._on_zmq_reply) def close(self): super(ZMQChannelsHandler, self).close() return self._close_future def on_message(self, msg): if not self.channels: # already closed, ignore the message self.log.debug("Received message on closed websocket %r", msg) return if isinstance(msg, bytes): msg = deserialize_binary_message(msg) else: msg = json.loads(msg) channel = msg.pop('channel', None) if channel is None: self.log.warning("No channel specified, assuming shell: %s", msg) channel = 'shell' if channel not in self.channels: self.log.warning("No such channel: %r", channel) return mt = msg['header']['msg_type'] stream = self.channels[channel] self.session.send(stream, msg) def _on_zmq_reply(self, stream, msg_list): idents, fed_msg_list = self.session.feed_identities(msg_list) msg = self.session.deserialize(fed_msg_list) parent = msg['parent_header'] channel = getattr(stream, 'channel', None) msg_type = msg['header']['msg_type'] super(ZMQChannelsHandler, self)._on_zmq_reply(stream, msg) def on_close(self): self.log.debug("Websocket closed %s", self.session_key) # unregister myself as an open session (only if it's really me) if self._open_sessions.get(self.session_key) is self: self._open_sessions.pop(self.session_key) km = self.kernel_manager if self.kernel_id in km: #km.notify_disconnect(self.kernel_id) km.remove_restart_callback( self.kernel_id, self.on_kernel_restarted, ) km.remove_restart_callback( self.kernel_id, self.on_restart_failed, 'dead', ) """ # start buffering instead of closing if this was the last connection if km._kernel_connections[self.kernel_id] == 0: km.start_buffering(self.kernel_id, self.session_key, self.channels) self._close_future.set_result(None) return """ # This method can be called twice, once by self.kernel_died and once # from the WebSocket close event. If the WebSocket connection is # closed before the ZMQ streams are setup, they could be None. for channel, stream in self.channels.items(): if stream is not None and not stream.closed(): stream.on_recv(None) stream.close() self.channels = {} self._close_future.set_result(None) def _send_status_message(self, status): iopub = self.channels.get('iopub', None) if iopub and not iopub.closed(): # flush IOPub before sending a restarting/dead status message # ensures proper ordering on the IOPub channel # that all messages from the stopped kernel have been delivered iopub.flush() msg = self.session.msg("status", {'execution_state': status}) msg['channel'] = 'iopub' self.write_message(json.dumps(msg, default=date_default)) def on_kernel_restarted(self): logging.warn("kernel %s restarted", self.kernel_id) self._send_status_message('restarting') def on_restart_failed(self): logging.error("kernel %s restarted failed!", self.kernel_id) self._send_status_message('dead')
class GatewayWebSocketClient(LoggingConfigurable): """Proxy web socket connection to a kernel/enterprise gateway.""" def __init__(self, **kwargs): super(GatewayWebSocketClient, self).__init__(**kwargs) self.kernel_id = None self.ws = None self.ws_future = Future() self.ws_future_cancelled = False @gen.coroutine def _connect(self, kernel_id): self.kernel_id = kernel_id ws_url = url_path_join(GatewayClient.instance().ws_url, GatewayClient.instance().kernels_endpoint, url_escape(kernel_id), 'channels') self.log.info('Connecting to {}'.format(ws_url)) kwargs = {} kwargs = GatewayClient.instance().load_connection_args(**kwargs) request = HTTPRequest(ws_url, **kwargs) self.ws_future = websocket_connect(request) self.ws_future.add_done_callback(self._connection_done) def _connection_done(self, fut): if not self.ws_future_cancelled: # prevent concurrent.futures._base.CancelledError self.ws = fut.result() self.log.debug("Connection is ready: ws: {}".format(self.ws)) else: self.log.warning( "Websocket connection has been cancelled via client disconnect before its establishment. " "Kernel with ID '{}' may not be terminated on GatewayClient: {}" .format(self.kernel_id, GatewayClient.instance().url)) def _disconnect(self): if self.ws is not None: # Close connection self.ws.close() elif not self.ws_future.done(): # Cancel pending connection. Since future.cancel() is a noop on tornado, we'll track cancellation locally self.ws_future.cancel() self.ws_future_cancelled = True self.log.debug("_disconnect: ws_future_cancelled: {}".format( self.ws_future_cancelled)) @gen.coroutine def _read_messages(self, callback): """Read messages from gateway server.""" while True: message = None if not self.ws_future_cancelled: try: message = yield self.ws.read_message() except Exception as e: self.log.error( "Exception reading message from websocket: {}".format( e)) # , exc_info=True) if message is None: break callback( message ) # pass back to notebook client (see self.on_open and WebSocketChannelsHandler.open) else: # ws cancelled - stop reading break def on_open(self, kernel_id, message_callback, **kwargs): """Web socket connection open against gateway server.""" self._connect(kernel_id) loop = IOLoop.current() loop.add_future(self.ws_future, lambda future: self._read_messages(message_callback)) def on_message(self, message): """Send message to gateway server.""" if self.ws is None: loop = IOLoop.current() loop.add_future(self.ws_future, lambda future: self._write_message(message)) else: self._write_message(message) def _write_message(self, message): """Send message to gateway server.""" try: if not self.ws_future_cancelled: self.ws.write_message(message) except Exception as e: self.log.error("Exception writing message to websocket: {}".format( e)) # , exc_info=True) def on_close(self): """Web socket closed event.""" self._disconnect()
class HTTP1Connection(httputil.HTTPConnection): """Implements the HTTP/1.x protocol. This class can be on its own for clients, or via `HTTP1ServerConnection` for servers. """ def __init__(self, stream, is_client, params=None, context=None): """ :arg stream: an `.IOStream` :arg bool is_client: client or server :arg params: a `.HTTP1ConnectionParameters` instance or ``None`` :arg context: an opaque application-defined object that can be accessed as ``connection.context``. """ self.is_client = is_client self.stream = stream if params is None: params = HTTP1ConnectionParameters() self.params = params self.context = context self.no_keep_alive = params.no_keep_alive # The body limits can be altered by the delegate, so save them # here instead of just referencing self.params later. self._max_body_size = (self.params.max_body_size or self.stream.max_buffer_size) self._body_timeout = self.params.body_timeout # _write_finished is set to True when finish() has been called, # i.e. there will be no more data sent. Data may still be in the # stream's write buffer. self._write_finished = False # True when we have read the entire incoming body. self._read_finished = False # _finish_future resolves when all data has been written and flushed # to the IOStream. self._finish_future = Future() # If true, the connection should be closed after this request # (after the response has been written in the server side, # and after it has been read in the client) self._disconnect_on_finish = False self._clear_callbacks() self.stream.set_close_callback(self._on_connection_close) # Save the start lines after we read or write them; they # affect later processing (e.g. 304 responses and HEAD methods # have content-length but no bodies) self._request_start_line = None self._response_start_line = None self._request_headers = None # True if we are writing output with chunked encoding. self._chunking_output = None # While reading a body with a content-length, this is the # amount left to read. self._expected_content_remaining = None def read_response(self, delegate): """Read a single HTTP response. Typical client-mode usage is to write a request using `write_headers`, `write`, and `finish`, and then call ``read_response``. :arg delegate: a `.HTTPMessageDelegate` Returns a `.Future` that resolves to None after the full response has been read. """ if self.params.use_gzip: delegate = _GzipMessageDelegate(delegate, self.params.chunk_size) return self._read_message(delegate) @gen.coroutine def _read_message(self, delegate): need_delegate_close = False try: header_future = self.stream.read_until_regex( b"\r?\n\r?\n", max_bytes=self.params.max_header_size) if self.params.header_timeout is None: header_data = yield header_future else: try: header_data = yield gen.with_timeout( self.stream.io_loop.time() + self.params.header_timeout, header_future, io_loop=self.stream.io_loop) except gen.TimeoutError: self.close() raise gen.Return(False) start_line, headers = self._parse_headers(header_data) if self.is_client: start_line = httputil.parse_response_start_line(start_line) self._response_start_line = start_line else: start_line = httputil.parse_request_start_line(start_line) self._request_start_line = start_line self._request_headers = headers self._disconnect_on_finish = not self._can_keep_alive( start_line, headers) need_delegate_close = True header_future = delegate.headers_received(start_line, headers) if header_future is not None: yield header_future if self.stream is None: # We've been detached. need_delegate_close = False raise gen.Return(False) skip_body = False if self.is_client: if (self._request_start_line is not None and self._request_start_line.method == 'HEAD'): skip_body = True code = start_line.code if code == 304: skip_body = True if code >= 100 and code < 200: # TODO: client delegates will get headers_received twice # in the case of a 100-continue. Document or change? yield self._read_message(delegate) else: if (headers.get("Expect") == "100-continue" and not self._write_finished): self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n") if not skip_body: body_future = self._read_body(headers, delegate) if body_future is not None: if self._body_timeout is None: yield body_future else: try: yield gen.with_timeout( self.stream.io_loop.time() + self._body_timeout, body_future, self.stream.io_loop) except gen.TimeoutError: gen_log.info("Timeout reading body from %s", self.context) self.stream.close() raise gen.Return(False) self._read_finished = True if not self._write_finished or self.is_client: need_delegate_close = False delegate.finish() yield self._finish_future if self.is_client and self._disconnect_on_finish: self.close() if self.stream is None: raise gen.Return(False) except httputil.HTTPInputException as e: gen_log.info("Malformed HTTP message from %s: %s", self.context, e) self.close() raise gen.Return(False) finally: if need_delegate_close: delegate.on_connection_close() self._clear_callbacks() raise gen.Return(True) def _clear_callbacks(self): """Clears the callback attributes. This allows the request handler to be garbage collected more quickly in CPython by breaking up reference cycles. """ self._write_callback = None self._write_future = None self._close_callback = None def set_close_callback(self, callback): """Sets a callback that will be run when the connection is closed. .. deprecated:: 3.3 Use `.HTTPMessageDelegate.on_connection_close` instead. """ self._close_callback = stack_context.wrap(callback) def _on_connection_close(self): if self._close_callback is not None: callback = self._close_callback self._close_callback = None callback() if not self._finish_future.done(): self._finish_future.set_result(None) self._clear_callbacks() def close(self): self.stream.close() self._clear_callbacks() def detach(self): """Take control of the underlying stream. Returns the underlying `.IOStream` object and stops all further HTTP processing. May only be called during `.HTTPMessageDelegate.headers_received`. Intended for implementing protocols like websockets that tunnel over an HTTP handshake. """ stream = self.stream self.stream = None return stream def set_body_timeout(self, timeout): """Sets the body timeout for a single request. Overrides the value from `.HTTP1ConnectionParameters`. """ self._body_timeout = timeout def set_max_body_size(self, max_body_size): """Sets the body size limit for a single request. Overrides the value from `.HTTP1ConnectionParameters`. """ self._max_body_size = max_body_size def write_headers(self, start_line, headers, chunk=None, callback=None): """Implements `.HTTPConnection.write_headers`.""" if self.is_client: self._request_start_line = start_line # Client requests with a non-empty body must have either a # Content-Length or a Transfer-Encoding. self._chunking_output = ( start_line.method in ('POST', 'PUT', 'PATCH') and 'Content-Length' not in headers and 'Transfer-Encoding' not in headers) else: self._response_start_line = start_line self._chunking_output = ( # TODO: should this use # self._request_start_line.version or # start_line.version? self._request_start_line.version == 'HTTP/1.1' and # 304 responses have no body (not even a zero-length body), and so # should not have either Content-Length or Transfer-Encoding. # headers. start_line.code != 304 and # No need to chunk the output if a Content-Length is specified. 'Content-Length' not in headers and # Applications are discouraged from touching Transfer-Encoding, # but if they do, leave it alone. 'Transfer-Encoding' not in headers) # If a 1.0 client asked for keep-alive, add the header. if (self._request_start_line.version == 'HTTP/1.0' and (self._request_headers.get('Connection', '').lower() == 'keep-alive')): headers['Connection'] = 'Keep-Alive' if self._chunking_output: headers['Transfer-Encoding'] = 'chunked' if (not self.is_client and (self._request_start_line.method == 'HEAD' or start_line.code == 304)): self._expected_content_remaining = 0 elif 'Content-Length' in headers: self._expected_content_remaining = int(headers['Content-Length']) else: self._expected_content_remaining = None lines = [utf8("%s %s %s" % start_line)] lines.extend([utf8(n) + b": " + utf8(v) for n, v in headers.get_all()]) for line in lines: if b'\n' in line: raise ValueError('Newline in header: ' + repr(line)) if self.stream.closed(): self._write_future = Future() self._write_future.set_exception(iostream.StreamClosedError()) else: if callback is not None: self._write_callback = stack_context.wrap(callback) else: self._write_future = Future() data = b"\r\n".join(lines) + b"\r\n\r\n" if chunk: data += self._format_chunk(chunk) self.stream.write(data, self._on_write_complete) return self._write_future def _format_chunk(self, chunk): if self._expected_content_remaining is not None: self._expected_content_remaining -= len(chunk) if self._expected_content_remaining < 0: # Close the stream now to stop further framing errors. self.stream.close() raise httputil.HTTPOutputException( "Tried to write more data than Content-Length") if self._chunking_output and chunk: # Don't write out empty chunks because that means END-OF-STREAM # with chunked encoding return utf8("%x" % len(chunk)) + b"\r\n" + chunk + b"\r\n" else: return chunk def write(self, chunk, callback=None): """Implements `.HTTPConnection.write`. For backwards compatibility is is allowed but deprecated to skip `write_headers` and instead call `write()` with a pre-encoded header block. """ if self.stream.closed(): self._write_future = Future() self._write_future.set_exception(iostream.StreamClosedError()) else: if callback is not None: self._write_callback = stack_context.wrap(callback) else: self._write_future = Future() self.stream.write(self._format_chunk(chunk), self._on_write_complete) return self._write_future def finish(self): """Implements `.HTTPConnection.finish`.""" if (self._expected_content_remaining is not None and self._expected_content_remaining != 0 and not self.stream.closed()): self.stream.close() raise httputil.HTTPOutputException( "Tried to write %d bytes less than Content-Length" % self._expected_content_remaining) if self._chunking_output: if not self.stream.closed(): self.stream.write(b"0\r\n\r\n", self._on_write_complete) self._write_finished = True # If the app finished the request while we're still reading, # divert any remaining data away from the delegate and # close the connection when we're done sending our response. # Closing the connection is the only way to avoid reading the # whole input body. if not self._read_finished: self._disconnect_on_finish = True # No more data is coming, so instruct TCP to send any remaining # data immediately instead of waiting for a full packet or ack. self.stream.set_nodelay(True) if not self.stream.writing(): self._finish_request() def _on_write_complete(self): if self._write_callback is not None: callback = self._write_callback self._write_callback = None callback() if self._write_future is not None: future = self._write_future self._write_future = None future.set_result(None) # _on_write_complete is enqueued on the IOLoop whenever the # IOStream's write buffer becomes empty, but it's possible for # another callback that runs on the IOLoop before it to # simultaneously write more data and finish the request. If # there is still data in the IOStream, a future # _on_write_complete will be responsible for calling # _finish_request. if self._write_finished and not self.stream.writing(): self._finish_request() def _can_keep_alive(self, start_line, headers): if self.params.no_keep_alive: return False connection_header = headers.get("Connection") if connection_header is not None: connection_header = connection_header.lower() if start_line.version == "HTTP/1.1": return connection_header != "close" elif ("Content-Length" in headers or start_line.method in ("HEAD", "GET")): return connection_header == "keep-alive" return False def _finish_request(self): self._clear_callbacks() if not self.is_client and self._disconnect_on_finish: self.close() return # Turn Nagle's algorithm back on, leaving the stream in its # default state for the next request. self.stream.set_nodelay(False) if not self._finish_future.done(): self._finish_future.set_result(None) def _parse_headers(self, data): data = native_str(data.decode('latin1')) eol = data.find("\r\n") start_line = data[:eol] try: headers = httputil.HTTPHeaders.parse(data[eol:]) except ValueError: # probably form split() if there was no ':' in the line raise httputil.HTTPInputException("Malformed HTTP headers: %r" % data[eol:100]) return start_line, headers def _read_body(self, headers, delegate): content_length = headers.get("Content-Length") if content_length: content_length = int(content_length) if content_length > self._max_body_size: raise httputil.HTTPInputException("Content-Length too long") return self._read_fixed_body(content_length, delegate) if headers.get("Transfer-Encoding") == "chunked": return self._read_chunked_body(delegate) if self.is_client: return self._read_body_until_close(delegate) return None @gen.coroutine def _read_fixed_body(self, content_length, delegate): while content_length > 0: body = yield self.stream.read_bytes( min(self.params.chunk_size, content_length), partial=True) content_length -= len(body) if not self._write_finished or self.is_client: yield gen.maybe_future(delegate.data_received(body)) @gen.coroutine def _read_chunked_body(self, delegate): # TODO: "chunk extensions" http://tools.ietf.org/html/rfc2616#section-3.6.1 total_size = 0 while True: chunk_len = yield self.stream.read_until(b"\r\n", max_bytes=64) chunk_len = int(chunk_len.strip(), 16) if chunk_len == 0: return total_size += chunk_len if total_size > self._max_body_size: raise httputil.HTTPInputException("chunked body too large") bytes_to_read = chunk_len while bytes_to_read: chunk = yield self.stream.read_bytes( min(bytes_to_read, self.params.chunk_size), partial=True) bytes_to_read -= len(chunk) if not self._write_finished or self.is_client: yield gen.maybe_future( delegate.data_received(chunk)) # chunk ends with \r\n crlf = yield self.stream.read_bytes(2) assert crlf == b"\r\n" @gen.coroutine def _read_body_until_close(self, delegate): body = yield self.stream.read_until_close() if not self._write_finished or self.is_client: delegate.data_received(body)