def read_message(self):
if self.request is not None:
note('err',derpid(self),self.request)
raise RuntimeError
try:
start_line = yield self.stream.read_until(b'\r\n',max_bytes=self.max_start_header_length)
except iostream.StreamClosedError:
note('done with',self.address,derpid(self))
raise
except iostream.UnsatisfiableReadError:
print("garbage from address",self.address)
self.stream.close()
raise iostream.StreamClosedError
start_line = httputil.parse_request_start_line(start_line.decode('utf-8'))
note('setting request',derpid(self),start_line)
self.request = self.requestFactory(self, self.stream, start_line)
headers = yield maybeTimeout(self.stream, self.header_timeout, self.read_headers())
yield maybeTimeout(self.stream, self.body_timeout, self.read_body(headers))
# Now we've read the request, so start writing it, asynchronously
yield self.startWriting()
note('del self.request (started writing)',derpid(self))
del self.request
assert self.request is None, "boop"
if self.old_client:
self.stream.close()
else:
self.read_next_message('next')
def handle_http_connection(self, conn):
# Read the trailing HTTP request and process it with protocol_switcher.
# We can't rely on ioloop to trigger read because it has been already
# triggered for SSL handshake.
payload = yield conn.stream.read_bytes(1024, partial=True)
logger.debug("Received %r", payload[:128])
# Simulate conn._read_message side effect. This is required by
# HTTP1Connection.write_headers()
conn._request_start_line = parse_request_start_line('GET / HTTP/1.1')
try:
start_line, headers = parse_http_headers(payload)
conn._request_start_line = start_line
request = HTTPRequest(
connection=conn,
headers=headers,
start_line=start_line,
)
request.config = self.request_callback.config
response = protocol_switcher(request)
except Exception as e:
logger.error("Failed to switch to HTTPS: %s", e)
response = HTTPResponse(
request=object(),
code=500,
headers=HTTPHeaders({'Content-Length': '0'}),
effective_url='https://useless_effective_url')
yield conn.write_headers(
start_line=ResponseStartLine(
'HTTP/1.1',
response.code,
response.reason,
),
headers=response.headers,
)
def _handle_request(self, data):
try:
start_line, headers = self.parse_headers(data)
request = httputil.HTTPServerRequest(
headers=headers,
start_line=httputil.parse_request_start_line(start_line))
deli = self._handler.application.find_handler(request)
getattr(self._handler, request.method.lower(),
self._handler.r404)(*deli.path_args, **deli.path_kwargs)
except httputil.HTTPInputError as e:
Log.e(TAG, b"_handle_request error", e)
def parse_http_headers(payload):
# Implements simple HTTP1Connection._read_message but IO-free.
lines = payload.splitlines()
# Drop start line
lines.pop(0)
# Drop contents
if '' in lines:
lines[:] = lines[:lines.index('')]
return (
parse_request_start_line('GET / HTTP/1.1'),
HTTPHeaders.parse('\r\n'.join(lines)),
)
def handle_http_connection(self, conn):
# Read the trailing HTTP request and process it with protocol_switcher.
# We can't rely on ioloop to trigger read because it has been already
# triggered for SSL handshake.
addr, port = conn.stream.socket.getsockname()
try:
# This is not blocking. Just read available bytes.
payload = conn.stream.socket.recv(1024)
except Exception:
# Exception includes EWOULDBLOCK, when no bytes are available. In
# this case just skip.
payload = ""
else:
logger.debug("Received %r", payload[:128])
# Simulate conn._read_message side effect. This is required by
# HTTP1Connection.write_headers()
conn._request_start_line = parse_request_start_line('GET / HTTP/1.1')
conn._request_headers = HTTPHeaders()
try:
start_line, headers = parse_http_headers(payload)
conn._request_start_line = start_line
request = HTTPRequest(
connection=conn,
headers=headers,
start_line=start_line,
)
request.config = self.request_callback.config
response = protocol_switcher(request)
except Exception as e:
logger.error("Failed to switch to HTTPS: %s", e)
response = HTTPResponse(
request=object(),
code=500,
headers=HTTPHeaders({'Content-Length': '0'}),
effective_url='https://useless_effective_url')
yield conn.write_headers(
start_line=ResponseStartLine(
'HTTP/1.1',
response.code,
response.reason,
),
headers=response.headers,
)
def _read_message(self, delegate):
need_delegate_close = False
try:
header_future = self.stream.read_until_regex(
b"\r?\n\r?\n",
max_bytes=self.params.max_header_size)
if self.params.header_timeout is None:
header_data = yield header_future
else:
try:
header_data = yield gen.with_timeout(
self.stream.io_loop.time() + self.params.header_timeout,
header_future,
quiet_exceptions=iostream.StreamClosedError)
except gen.TimeoutError:
self.close()
raise gen.Return(False)
start_line, headers = self._parse_headers(header_data)
if self.is_client:
start_line = httputil.parse_response_start_line(start_line)
self._response_start_line = start_line
else:
start_line = httputil.parse_request_start_line(start_line)
self._request_start_line = start_line
self._request_headers = headers
self._disconnect_on_finish = not self._can_keep_alive(
start_line, headers)
need_delegate_close = True
with _ExceptionLoggingContext(app_log):
header_future = delegate.headers_received(start_line, headers)
if header_future is not None:
yield header_future
if self.stream is None:
# We've been detached.
need_delegate_close = False
raise gen.Return(False)
skip_body = False
if self.is_client:
if (self._request_start_line is not None and
self._request_start_line.method == 'HEAD'):
skip_body = True
code = start_line.code
if code == 304:
# 304 responses may include the content-length header
# but do not actually have a body.
# http://tools.ietf.org/html/rfc7230#section-3.3
skip_body = True
if code >= 100 and code < 200:
# 1xx responses should never indicate the presence of
# a body.
if ('Content-Length' in headers or
'Transfer-Encoding' in headers):
raise httputil.HTTPInputError(
"Response code %d cannot have body" % code)
# TODO: client delegates will get headers_received twice
# in the case of a 100-continue. Document or change?
yield self._read_message(delegate)
else:
if (headers.get("Expect") == "100-continue" and
not self._write_finished):
self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
if not skip_body:
body_future = self._read_body(
start_line.code if self.is_client else 0, headers, delegate)
if body_future is not None:
if self._body_timeout is None:
yield body_future
else:
try:
yield gen.with_timeout(
self.stream.io_loop.time() + self._body_timeout,
body_future,
quiet_exceptions=iostream.StreamClosedError)
except gen.TimeoutError:
gen_log.info("Timeout reading body from %s",
self.context)
self.stream.close()
raise gen.Return(False)
self._read_finished = True
if not self._write_finished or self.is_client:
need_delegate_close = False
with _ExceptionLoggingContext(app_log):
delegate.finish()
# If we're waiting for the application to produce an asynchronous
# response, and we're not detached, register a close callback
# on the stream (we didn't need one while we were reading)
if (not self._finish_future.done() and
self.stream is not None and
not self.stream.closed()):
self.stream.set_close_callback(self._on_connection_close)
yield self._finish_future
if self.is_client and self._disconnect_on_finish:
self.close()
if self.stream is None:
raise gen.Return(False)
except httputil.HTTPInputError as e:
gen_log.info("Malformed HTTP message from %s: %s",
self.context, e)
if not self.is_client:
yield self.stream.write(b'HTTP/1.1 400 Bad Request\r\n\r\n')
self.close()
raise gen.Return(False)
finally:
if need_delegate_close:
with _ExceptionLoggingContext(app_log):
delegate.on_connection_close()
header_future = None
self._clear_callbacks()
raise gen.Return(True)
def test_parse_request_start_line(self):
start_line = " ".join([self.METHOD, self.PATH, self.VERSION])
parsed_start_line = parse_request_start_line(start_line)
self.assertEqual(parsed_start_line.method, self.METHOD)
self.assertEqual(parsed_start_line.path, self.PATH)
self.assertEqual(parsed_start_line.version, self.VERSION)
async def _read_message(self, delegate: httputil.HTTPMessageDelegate) -> bool:
need_delegate_close = False
try:
header_future = self.stream.read_until_regex(
b"\r?\n\r?\n", max_bytes=self.params.max_header_size
)
if self.params.header_timeout is None:
header_data = await header_future
else:
try:
header_data = await gen.with_timeout(
self.stream.io_loop.time() + self.params.header_timeout,
header_future,
quiet_exceptions=iostream.StreamClosedError,
)
except gen.TimeoutError:
self.close()
return False
start_line_str, headers = self._parse_headers(header_data)
if self.is_client:
resp_start_line = httputil.parse_response_start_line(start_line_str)
self._response_start_line = resp_start_line
start_line = (
resp_start_line
) # type: Union[httputil.RequestStartLine, httputil.ResponseStartLine]
# TODO: this will need to change to support client-side keepalive
self._disconnect_on_finish = False
else:
req_start_line = httputil.parse_request_start_line(start_line_str)
self._request_start_line = req_start_line
self._request_headers = headers
start_line = req_start_line
self._disconnect_on_finish = not self._can_keep_alive(
req_start_line, headers
)
need_delegate_close = True
with _ExceptionLoggingContext(app_log):
header_recv_future = delegate.headers_received(start_line, headers)
if header_recv_future is not None:
await header_recv_future
if self.stream is None:
# We've been detached.
need_delegate_close = False
return False
skip_body = False
if self.is_client:
assert isinstance(start_line, httputil.ResponseStartLine)
if (
self._request_start_line is not None
and self._request_start_line.method == "HEAD"
):
skip_body = True
code = start_line.code
if code == 304:
# 304 responses may include the content-length header
# but do not actually have a body.
# http://tools.ietf.org/html/rfc7230#section-3.3
skip_body = True
if 100 <= code < 200:
# 1xx responses should never indicate the presence of
# a body.
if "Content-Length" in headers or "Transfer-Encoding" in headers:
raise httputil.HTTPInputError(
"Response code %d cannot have body" % code
)
# TODO: client delegates will get headers_received twice
# in the case of a 100-continue. Document or change?
await self._read_message(delegate)
else:
if headers.get("Expect") == "100-continue" and not self._write_finished:
self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
if not skip_body:
body_future = self._read_body(
resp_start_line.code if self.is_client else 0, headers, delegate
)
if body_future is not None:
if self._body_timeout is None:
await body_future
else:
try:
await gen.with_timeout(
self.stream.io_loop.time() + self._body_timeout,
body_future,
quiet_exceptions=iostream.StreamClosedError,
)
except gen.TimeoutError:
gen_log.info("Timeout reading body from %s", self.context)
self.stream.close()
return False
self._read_finished = True
if not self._write_finished or self.is_client:
need_delegate_close = False
with _ExceptionLoggingContext(app_log):
delegate.finish()
# If we're waiting for the application to produce an asynchronous
# response, and we're not detached, register a close callback
# on the stream (we didn't need one while we were reading)
if (
not self._finish_future.done()
and self.stream is not None
and not self.stream.closed()
):
self.stream.set_close_callback(self._on_connection_close)
await self._finish_future
if self.is_client and self._disconnect_on_finish:
self.close()
if self.stream is None:
return False
except httputil.HTTPInputError as e:
gen_log.info("Malformed HTTP message from %s: %s", self.context, e)
if not self.is_client:
await self.stream.write(b"HTTP/1.1 400 Bad Request\r\n\r\n")
self.close()
return False
finally:
if need_delegate_close:
with _ExceptionLoggingContext(app_log):
delegate.on_connection_close()
header_future = None # type: ignore
self._clear_callbacks()
return True
def request(self, conn, src, dst):
keep_alive = True
req = None
conn = conn.copy()
req_line = yield
while keep_alive and req_line is not None:
body = ""
headers = MultiOrderedDict()
try:
req = httputil.parse_request_start_line(req_line.strip())
except httputil.HTTPInputError:
if req_line != "":
self.log("HTTP Error: Malformed request start line: '{}'", req_line)
req_line = yield
continue
while True:
header_line = yield
if header_line is None:
break
if not header_line.strip():
break
self.parse_header_line(headers, header_line.strip())
if req.version == "HTTP/1.0":
keep_alive = headers.last("connection", "").lower().strip() == "keep-alive"
else:
keep_alive = headers.last("connection", "").lower().strip() != "close"
if "content-length" in headers:
try:
content_length = int(headers.last("content-length"))
except ValueError:
content_length = None
else:
content_length = None
if req.method != "POST":
content_length = content_length or 0
if header_line is not None:
#body += conn["lbl_buffers"][dst]
#conn["lbl_buffers"][dst] = ""
conn["lbl_disable"](dst)
while len(body) < content_length or content_length is None:
data = yield
if data is None:
break
body += data
if "content-encoding" in headers:
conn["http_decoded"] = False
encoding = headers.last("content-encoding")
self.log("req encoding: {}", encoding)
if encoding in self.DECODERS:
try:
body = self.DECODERS[encoding](body)
conn["http_decoded"] = True
except:
self.log("Unable to decode content '{}' len={}/{}", encoding, len(body), content_length)
else:
conn["http_decoded"] = True
conn["lbl_enable"](dst)
conn["http_headers"] = headers
conn["http_request"] = req
req_line = yield self.bubble(dst, conn, body)
def _read_message(self, delegate):
need_delegate_close = False
try:
header_future = self.stream.read_until_regex(
b"\r?\n\r?\n", max_bytes=self.params.max_header_size)
if self.params.header_timeout is None:
header_data = yield header_future
else:
try:
header_data = yield gen.with_timeout(
self.stream.io_loop.time() +
self.params.header_timeout,
header_future,
io_loop=self.stream.io_loop)
except gen.TimeoutError:
self.close()
raise gen.Return(False)
start_line, headers = self._parse_headers(header_data)
if self.is_client:
start_line = httputil.parse_response_start_line(start_line)
self._response_start_line = start_line
else:
start_line = httputil.parse_request_start_line(start_line)
self._request_start_line = start_line
self._request_headers = headers
self._disconnect_on_finish = not self._can_keep_alive(
start_line, headers)
need_delegate_close = True
header_future = delegate.headers_received(start_line, headers)
if header_future is not None:
yield header_future
if self.stream is None:
# We've been detached.
need_delegate_close = False
raise gen.Return(False)
skip_body = False
if self.is_client:
if (self._request_start_line is not None
and self._request_start_line.method == 'HEAD'):
skip_body = True
code = start_line.code
if code == 304:
skip_body = True
if code >= 100 and code < 200:
# TODO: client delegates will get headers_received twice
# in the case of a 100-continue. Document or change?
yield self._read_message(delegate)
else:
if (headers.get("Expect") == "100-continue"
and not self._write_finished):
self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
if not skip_body:
body_future = self._read_body(headers, delegate)
if body_future is not None:
if self._body_timeout is None:
yield body_future
else:
try:
yield gen.with_timeout(
self.stream.io_loop.time() +
self._body_timeout, body_future,
self.stream.io_loop)
except gen.TimeoutError:
gen_log.info("Timeout reading body from %s",
self.context)
self.stream.close()
raise gen.Return(False)
self._read_finished = True
if not self._write_finished or self.is_client:
need_delegate_close = False
delegate.finish()
# If we're waiting for the application to produce an asynchronous
# response, and we're not detached, register a close callback
# on the stream (we didn't need one while we were reading)
if (not self._finish_future.done() and self.stream is not None
and not self.stream.closed()):
self.stream.set_close_callback(self._on_connection_close)
yield self._finish_future
if self.is_client and self._disconnect_on_finish:
self.close()
if self.stream is None:
raise gen.Return(False)
except httputil.HTTPInputException as e:
gen_log.info("Malformed HTTP message from %s: %s", self.context, e)
self.close()
raise gen.Return(False)
finally:
if need_delegate_close:
delegate.on_connection_close()
self._clear_callbacks()
raise gen.Return(True)
def _read_message(self, delegate):
need_delegate_close = False
try:
header_future = self.stream.read_until_regex(
b"\r?\n\r?\n",
max_bytes=self.params.max_header_size)
if self.params.header_timeout is None:
header_data = yield header_future
else:
try:
header_data = yield gen.with_timeout(
self.stream.io_loop.time() + self.params.header_timeout,
header_future,
io_loop=self.stream.io_loop,
quiet_exceptions=iostream.StreamClosedError)
except gen.TimeoutError:
self.close()
raise gen.Return(False)
start_line, headers = self._parse_headers(header_data)
if self.is_client:
start_line = httputil.parse_response_start_line(start_line)
self._response_start_line = start_line
else:
start_line = httputil.parse_request_start_line(start_line)
self._request_start_line = start_line
self._request_headers = headers
self._disconnect_on_finish = not self._can_keep_alive(
start_line, headers)
need_delegate_close = True
with _ExceptionLoggingContext():
header_future = delegate.headers_received(start_line, headers)
if header_future is not None:
yield header_future
if self.stream is None:
# We've been detached.
need_delegate_close = False
raise gen.Return(False)
skip_body = False
if self.is_client:
if (self._request_start_line is not None and
self._request_start_line.method == 'HEAD'):
skip_body = True
code = start_line.code
if code == 304:
skip_body = True
if code >= 100 and code < 200:
if ('Content-Length' in headers or
'Transfer-Encoding' in headers):
raise httputil.HTTPInputError(
"Response code %d cannot have body" % code)
yield self._read_message(delegate)
else:
if (headers.get("Expect") == "100-continue" and
not self._write_finished):
self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
if not skip_body:
body_future = self._read_body(
start_line.code if self.is_client else 0, headers, delegate)
if body_future is not None:
if self._body_timeout is None:
yield body_future
else:
try:
yield gen.with_timeout(
self.stream.io_loop.time() + self._body_timeout,
body_future, self.stream.io_loop,
quiet_exceptions=iostream.StreamClosedError)
except gen.TimeoutError:
print("Timeout reading body from %s" % self.context)
self.stream.close()
raise gen.Return(False)
self._read_finished = True
if not self._write_finished or self.is_client:
need_delegate_close = False
with _ExceptionLoggingContext():
delegate.finish()
if (not self._finish_future.done() and
self.stream is not None and
not self.stream.closed()):
self.stream.set_close_callback(self._on_connection_close)
yield self._finish_future
if self.is_client and self._disconnect_on_finish:
self.close()
if self.stream is None:
raise gen.Return(False)
except httputil.HTTPInputError as e:
print("Malformed HTTP message from %s: %s", (self.context, e))
self.close()
raise gen.Return(False)
finally:
if need_delegate_close:
with _ExceptionLoggingContext():
delegate.on_connection_close()
self._clear_callbacks()
raise gen.Return(True)
def _read_message(self, delegate):
need_delegate_close = False
try:
# 消息头与消息体之间由一个空行分开
header_future = self.stream.read_until_regex(
b"\r?\n\r?\n",
max_bytes=self.params.max_header_size)
if self.params.header_timeout is None:
header_data = yield header_future
else:
try:
header_data = yield gen.with_timeout(
self.stream.io_loop.time() + self.params.header_timeout,
header_future,
io_loop=self.stream.io_loop)
except gen.TimeoutError:
self.close()
raise gen.Return(False)
# 解析消息头，分离头字段和首行（request-line/status-line）
start_line, headers = self._parse_headers(header_data)
# 作为 client 解析的是 server 的 response，作为 server 解析的是 client 的 request。
# response 与 request 的 start_line(status-line/request-line) 的字段内容不同：
# 1. response's status-line: HTTP-Version SP Status-Code SP Reason-Phrase CRLF
# 2. request's request-line：Method SP Request-URI SP HTTP-Version CRLF
# start_line 的值是一个 namedtuple。
if self.is_client:
start_line = httputil.parse_response_start_line(start_line)
self._response_start_line = start_line
else:
start_line = httputil.parse_request_start_line(start_line)
self._request_start_line = start_line
self._request_headers = headers
# 非 keep-alive 的请求或响应处理完成后要关闭连接。
self._disconnect_on_finish = not self._can_keep_alive(
start_line, headers)
need_delegate_close = True
with _ExceptionLoggingContext(app_log):
header_future = delegate.headers_received(start_line, headers)
if header_future is not None:
# 如果 header_future 是一个 `Future` 实例，则要等到完成才读取 body。
yield header_future
# websocket ？？？
if self.stream is None:
# We've been detached.
need_delegate_close = False
raise gen.Return(False)
skip_body = False
if self.is_client:
# 作为 client 如果发起的是 HEAD 请求，那么 server response 应该无消息体
if (self._request_start_line is not None and
self._request_start_line.method == 'HEAD'):
skip_body = True
code = start_line.code
if code == 304:
# 304 responses may include the content-length header
# but do not actually have a body.
# http://tools.ietf.org/html/rfc7230#section-3.3
skip_body = True
if code >= 100 and code < 200:
# 1xx responses should never indicate the presence of
# a body.
if ('Content-Length' in headers or
'Transfer-Encoding' in headers):
raise httputil.HTTPInputError(
"Response code %d cannot have body" % code)
# TODO: client delegates will get headers_received twice
# in the case of a 100-continue. Document or change?
yield self._read_message(delegate)
else:
# 100-continue 这个状态码是在 HTTP/1.1 中为了提高传输效率而设置的。当
# client 需要 POST 较大数据给 WebServer 时，可以在发送 HTTP 请求时带上
# Expect: 100-continue，WebServer 如果接受这个请求则应答一个
# ``HTTP/1.1 100 (Continue)``，那么 client 就继续传输 request body，
# 否则应答 ``HTTP/1.1 417 Expectation Failed`` client 就放弃传输剩余
# 的数据。（注：Expect 头部域，用于指出客户端要求的特殊服务器行为采用扩展语法
# 定义，以方便扩展。）
if (headers.get("Expect") == "100-continue" and
not self._write_finished):
self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
if not skip_body:
body_future = self._read_body(
start_line.code if self.is_client else 0, headers, delegate)
if body_future is not None:
if self._body_timeout is None:
yield body_future
else:
try:
yield gen.with_timeout(
self.stream.io_loop.time() + self._body_timeout,
body_future, self.stream.io_loop)
except gen.TimeoutError:
gen_log.info("Timeout reading body from %s",
self.context)
self.stream.close()
raise gen.Return(False)
self._read_finished = True
# 对 client mode ，response 解析完成就调用 HTTPMessageDelegate.finish() 方法是合适的；
# 对 server mode ，_write_finished 表示 response 是否发送完成，未完成前调用
# HTTPMessageDelegate.finish() 方法让 delegate 执行请求响应；
if not self._write_finished or self.is_client:
need_delegate_close = False
with _ExceptionLoggingContext(app_log):
delegate.finish()
# If we're waiting for the application to produce an asynchronous
# response, and we're not detached, register a close callback
# on the stream (we didn't need one while we were reading)
#
# NOTE:_finish_future resolves when all data has been written and flushed
# to the IOStream.
#
# hold 住执行流程，直到异步响应完成，所有数据都写入 fd，才继续后续处理，通常调用方执行 `finish` 方法
# 设置 `_finish_future` 完成，详细见 `finish` 和 `_finish_request` 方法实现。
if (not self._finish_future.done() and
self.stream is not None and
not self.stream.closed()):
self.stream.set_close_callback(self._on_connection_close)
yield self._finish_future
# 对于 client mode，处理完响应后如果不是 keep-alive 就断开连接。
# 对于 server mode，需要在 response 完成后才断开连接，详细见 _finish_request/finish 方法实现。
if self.is_client and self._disconnect_on_finish:
self.close()
if self.stream is None:
raise gen.Return(False)
except httputil.HTTPInputError as e:
gen_log.info("Malformed HTTP message from %s: %s",
self.context, e)
self.close()
raise gen.Return(False)
finally:
# 连接 “关闭” 前还没能结束处理请求（call HTTPMessageDelegate.finish()），则
# call HTTPMessageDelegate.on_connection_close()
if need_delegate_close:
with _ExceptionLoggingContext(app_log):
delegate.on_connection_close()
self._clear_callbacks()
raise gen.Return(True)
def _read_message(self, delegate):
need_delegate_close = False
try:
# 这一句会从 iostream 里异步地读取 header
# 的内容，返回的是一个将会填充
# header 内容的值的 future 对象。
# HTTP 协议中，header 和 body 之间使用类似于 \r\n\r\n
# 这样的被称为 CRLF 的字符串来划分，
# 所以作者传入了下面的正则来保证刚好读完 header 的信息
header_future = self.stream.read_until_regex(
b"\r?\n\r?\n", max_bytes=self.params.max_header_size)
if self.params.header_timeout is None:
header_data = yield header_future
else:
# 如果设置了 timeout，则将 Future 对象转化为一个
# gen._Timeout 对象，让 iostream 在
# 限定时间内去填充 future 对象内的 result，否则会报超时的错。
try:
header_data = yield gen.with_timeout(
self.stream.io_loop.time() +
self.params.header_timeout,
header_future,
io_loop=self.stream.io_loop,
quiet_exceptions=iostream.StreamClosedError)
except gen.TimeoutError:
self.close()
raise gen.Return(False)
# 将 header 信息转化为保存有响应信息的对象。因为客户端和服务端中
# http 头信息的形式是不同的，所以要根据 connection 是客户端方还
# 是服务端方来作不同的处理。
# 客户端得到的为 start line 被叫做 status-line，结构类似于：
# HTTP/1.1 200 OK
# 服务端得到的 start line 被叫做
# request-line，结构类似于：GET / HTTP/1.1
start_line, headers = self._parse_headers(header_data)
if self.is_client:
start_line = httputil.parse_response_start_line(start_line)
self._response_start_line = start_line
else:
start_line = httputil.parse_request_start_line(start_line)
self._request_start_line = start_line
self._request_headers = headers
# 根据头信息来判断连接的保持规则。
self._disconnect_on_finish = not self._can_keep_alive(
start_line, headers)
need_delegate_close = True
# 在这个 Context 之下，如果代码抛出异常，将会把 exception 的信息
# 用 app_log 当做日志输出
with _ExceptionLoggingContext(app_log):
# 将解析好的头信息交给 delegate，这时对于服务端来说，会根据
# start line 中的 path 信息来匹配处理请求的那个 Handler。
header_future = delegate.headers_received(start_line, headers)
# 在 stream 的情况下，header_future 则不会是 None。
if header_future is not None:
yield header_future
if self.stream is None:
# We've been detached.
need_delegate_close = False
raise gen.Return(False)
skip_body = False
if self.is_client:
# 对于客户端而言如果使用这个连接的对象是客户端类型的，则进
# 行头信息以及 HTTP 状态码来决定是否继续读取 body 数据。
if (self._request_start_line is not None
and self._request_start_line.method == 'HEAD'):
skip_body = True
code = start_line.code
if code == 304:
# 304 responses may include the content-length header
# but do not actually have a body.
# http://tools.ietf.org/html/rfc7230#section-3.3
skip_body = True
if code >= 100 and code < 200:
# 1xx responses should never indicate the presence of
# a body.
if ('Content-Length' in headers
or 'Transfer-Encoding' in headers):
raise httputil.HTTPInputError(
"Response code %d cannot have body" % code)
# TODO: client delegates will get headers_received twice
# in the case of a 100-continue. Document or change?
yield self._read_message(delegate)
else:
# 对于服务端而言，不存在需要跳过 body 数据的情况。
if (headers.get("Expect") == "100-continue" and
# 处理头中包含『Expect: 100-continue』的客户端请求。
not self._write_finished):
self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
if not skip_body:
body_future = self._read_body(
start_line.code if self.is_client else 0, headers,
delegate)
if body_future is not None:
if self._body_timeout is None:
yield body_future
else:
try:
yield gen.with_timeout(
self.stream.io_loop.time() +
self._body_timeout,
body_future,
self.stream.io_loop,
quiet_exceptions=iostream.StreamClosedError)
except gen.TimeoutError:
gen_log.info("Timeout reading body from %s",
self.context)
self.stream.close()
raise gen.Return(False)
self._read_finished = True
if not self._write_finished or self.is_client:
# 服务端在这里便会调用到匹配到的 handler
# 来向请求方写回返回数据。
# 这时 _RequestDispatcher 对象中已经保有了 header 和 body 的数
# 据，可以稍微处理一下（例如将表单数据 parse 一下）作为参数交给
# Handler 处理了。
need_delegate_close = False
with _ExceptionLoggingContext(app_log):
delegate.finish()
# If we're waiting for the application to produce an asynchronous
# response, and we're not detached, register a close callback
# on the stream (we didn't need one while we were reading)
if (not self._finish_future.done() and self.stream is not None
and not self.stream.closed()):
self.stream.set_close_callback(self._on_connection_close)
yield self._finish_future
if self.is_client and self._disconnect_on_finish:
self.close()
if self.stream is None:
raise gen.Return(False)
except httputil.HTTPInputError as e:
gen_log.info("Malformed HTTP message from %s: %s", self.context, e)
self.close()
raise gen.Return(False)
finally:
if need_delegate_close:
with _ExceptionLoggingContext(app_log):
delegate.on_connection_close()
self._clear_callbacks()
raise gen.Return(True)
def request(self, conn, src, dst):
keep_alive = True
req = None
conn = conn.copy()
req_line = yield
while keep_alive and req_line is not None:
body = ""
headers = MultiOrderedDict()
try:
req = httputil.parse_request_start_line(req_line.strip())
except httputil.HTTPInputError:
if req_line != "":
self.log("HTTP Error: Malformed request start line: '{}'",
req_line)
req_line = yield
continue
while True:
header_line = yield
if header_line is None:
break
if not header_line.strip():
break
self.parse_header_line(headers, header_line.strip())
if req.version == "HTTP/1.0":
keep_alive = headers.last("connection",
"").lower().strip() == "keep-alive"
else:
keep_alive = headers.last("connection",
"").lower().strip() != "close"
if "content-length" in headers:
try:
content_length = int(headers.last("content-length"))
except ValueError:
content_length = None
else:
content_length = None
if req.method != "POST":
content_length = content_length or 0
if header_line is not None:
#body += conn["lbl_buffers"][dst]
#conn["lbl_buffers"][dst] = ""
conn["lbl_disable"](dst)
while len(body) < content_length or content_length is None:
data = yield
if data is None:
break
body += data
if "content-encoding" in headers:
conn["http_decoded"] = False
encoding = headers.last("content-encoding")
self.log("req encoding: {}", encoding)
if encoding in self.DECODERS:
try:
body = self.DECODERS[encoding](body)
conn["http_decoded"] = True
except:
self.log("Unable to decode content '{}' len={}/{}",
encoding, len(body), content_length)
else:
conn["http_decoded"] = True
conn["lbl_enable"](dst)
conn["http_headers"] = headers
conn["http_request"] = req
req_line = yield self.bubble(dst, conn, body)
def test_parse_request_start_line(self):
start_line = " ".join([self.METHOD, self.PATH, self.VERSION])
parsed_start_line = parse_request_start_line(start_line)
self.assertEqual(parsed_start_line.method, self.METHOD)
self.assertEqual(parsed_start_line.path, self.PATH)
self.assertEqual(parsed_start_line.version, self.VERSION)
def _read_message(self, delegate):
need_delegate_close = False
try:
header_future = self.stream.read_until_regex(
b"\r?\n\r?\n",
max_bytes=self.params.max_header_size)
if self.params.header_timeout is None:
header_data = yield header_future
else:
try:
header_data = yield gen.with_timeout(
self.stream.io_loop.time() + self.params.header_timeout,
header_future,
io_loop=self.stream.io_loop)
except gen.TimeoutError:
self.close()
raise gen.Return(False)
start_line, headers = self._parse_headers(header_data)
if self.is_client:
start_line = httputil.parse_response_start_line(start_line)
self._response_start_line = start_line
else:
start_line = httputil.parse_request_start_line(start_line)
self._request_start_line = start_line
self._request_headers = headers
self._disconnect_on_finish = not self._can_keep_alive(
start_line, headers)
need_delegate_close = True
header_future = delegate.headers_received(start_line, headers)
if header_future is not None:
yield header_future
if self.stream is None:
# We've been detached.
need_delegate_close = False
raise gen.Return(False)
skip_body = False
if self.is_client:
if (self._request_start_line is not None and
self._request_start_line.method == 'HEAD'):
skip_body = True
code = start_line.code
if code == 304:
skip_body = True
if code >= 100 and code < 200:
# TODO: client delegates will get headers_received twice
# in the case of a 100-continue. Document or change?
yield self._read_message(delegate)
else:
if (headers.get("Expect") == "100-continue" and
not self._write_finished):
self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
if not skip_body:
body_future = self._read_body(headers, delegate)
if body_future is not None:
if self._body_timeout is None:
yield body_future
else:
try:
yield gen.with_timeout(
self.stream.io_loop.time() + self._body_timeout,
body_future, self.stream.io_loop)
except gen.TimeoutError:
gen_log.info("Timeout reading body from %s",
self.context)
self.stream.close()
raise gen.Return(False)
self._read_finished = True
if not self._write_finished or self.is_client:
need_delegate_close = False
delegate.finish()
yield self._finish_future
if self.is_client and self._disconnect_on_finish:
self.close()
if self.stream is None:
raise gen.Return(False)
except httputil.HTTPInputException as e:
gen_log.info("Malformed HTTP message from %s: %s",
self.context, e)
self.close()
raise gen.Return(False)
finally:
if need_delegate_close:
delegate.on_connection_close()
self._clear_callbacks()
raise gen.Return(True)
def _read_message(self, delegate):
need_delegate_close = False
try:
header_future = self.stream.read_until_regex(
b"\r?\n\r?\n", max_bytes=self.params.max_header_size)
if self.params.header_timeout is None:
header_data = yield header_future
else:
try:
header_data = yield gen.with_timeout(
self.stream.io_loop.time() +
self.params.header_timeout,
header_future,
io_loop=self.stream.io_loop,
quiet_exceptions=iostream.StreamClosedError)
except gen.TimeoutError:
self.close()
raise gen.Return(False)
start_line, headers = self._parse_headers(header_data)
if self.is_client:
start_line = httputil.parse_response_start_line(start_line)
self._response_start_line = start_line
else:
start_line = httputil.parse_request_start_line(start_line)
self._request_start_line = start_line
self._request_headers = headers
self._disconnect_on_finish = not self._can_keep_alive(
start_line, headers)
need_delegate_close = True
with _ExceptionLoggingContext(app_log):
header_future = delegate.headers_received(start_line, headers)
if header_future is not None:
yield header_future
if self.stream is None:
# We've been detached.
need_delegate_close = False
raise gen.Return(False)
skip_body = False
if self.is_client:
if (self._request_start_line is not None
and self._request_start_line.method == 'HEAD'):
skip_body = True
code = start_line.code
if code == 304:
# 304 responses may include the content-length header
# but do not actually have a body.
# http://tools.ietf.org/html/rfc7230#section-3.3
skip_body = True
if code >= 100 and code < 200:
# 1xx responses should never indicate the presence of
# a body.
if ('Content-Length' in headers
or 'Transfer-Encoding' in headers):
raise httputil.HTTPInputError(
"Response code %d cannot have body" % code)
# TODO: client delegates will get headers_received twice
# in the case of a 100-continue. Document or change?
yield self._read_message(delegate)
else:
if (headers.get("Expect") == "100-continue"
and not self._write_finished):
self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
if not skip_body:
body_future = self._read_body(
start_line.code if self.is_client else 0, headers,
delegate)
if body_future is not None:
if self._body_timeout is None:
yield body_future
else:
try:
yield gen.with_timeout(
self.stream.io_loop.time() +
self._body_timeout,
body_future,
self.stream.io_loop,
quiet_exceptions=iostream.StreamClosedError)
except gen.TimeoutError:
gen_log.info("Timeout reading body from %s",
self.context)
self.stream.close()
raise gen.Return(False)
self._read_finished = True
if not self._write_finished or self.is_client:
need_delegate_close = False
with _ExceptionLoggingContext(app_log):
delegate.finish()
# If we're waiting for the application to produce an asynchronous
# response, and we're not detached, register a close callback
# on the stream (we didn't need one while we were reading)
if (not self._finish_future.done() and self.stream is not None
and not self.stream.closed()):
self.stream.set_close_callback(self._on_connection_close)
yield self._finish_future
if self.is_client and self._disconnect_on_finish:
self.close()
if self.stream is None:
raise gen.Return(False)
except httputil.HTTPInputError as e:
gen_log.info("Malformed HTTP message from %s: %s", self.context, e)
self.close()
raise gen.Return(False)
finally:
if need_delegate_close:
with _ExceptionLoggingContext(app_log):
delegate.on_connection_close()
self._clear_callbacks()
raise gen.Return(True)
def _read_message(self, delegate):
need_delegate_close = False
try:
header_future = self.stream.read_until_regex(
b"\r?\n\r?\n", max_bytes=self.params.max_header_size)
if self.params.header_timeout is None:
header_data = yield header_future
else:
try:
header_data = yield gen.with_timeout(
self.stream.io_loop.time() +
self.params.header_timeout,
header_future,
io_loop=self.stream.io_loop,
quiet_exceptions=iostream.StreamClosedError)
except gen.TimeoutError:
self.close()
raise gen.Return(False)
start_line, headers = self._parse_headers(header_data)
if self.is_client:
start_line = httputil.parse_response_start_line(start_line)
self._response_start_line = start_line
else:
start_line = httputil.parse_request_start_line(start_line)
self._request_start_line = start_line
self._request_headers = headers
self._disconnect_on_finish = not self._can_keep_alive(
start_line, headers)
need_delegate_close = True
with _ExceptionLoggingContext():
header_future = delegate.headers_received(start_line, headers)
if header_future is not None:
yield header_future
if self.stream is None:
# We've been detached.
need_delegate_close = False
raise gen.Return(False)
skip_body = False
if self.is_client:
if (self._request_start_line is not None
and self._request_start_line.method == 'HEAD'):
skip_body = True
code = start_line.code
if code == 304:
skip_body = True
if code >= 100 and code < 200:
if ('Content-Length' in headers
or 'Transfer-Encoding' in headers):
raise httputil.HTTPInputError(
"Response code %d cannot have body" % code)
yield self._read_message(delegate)
else:
if (headers.get("Expect") == "100-continue"
and not self._write_finished):
self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
if not skip_body:
body_future = self._read_body(
start_line.code if self.is_client else 0, headers,
delegate)
if body_future is not None:
if self._body_timeout is None:
yield body_future
else:
try:
yield gen.with_timeout(
self.stream.io_loop.time() +
self._body_timeout,
body_future,
self.stream.io_loop,
quiet_exceptions=iostream.StreamClosedError)
except gen.TimeoutError:
print("Timeout reading body from %s" %
self.context)
self.stream.close()
raise gen.Return(False)
self._read_finished = True
if not self._write_finished or self.is_client:
need_delegate_close = False
with _ExceptionLoggingContext():
delegate.finish()
if (not self._finish_future.done() and self.stream is not None
and not self.stream.closed()):
self.stream.set_close_callback(self._on_connection_close)
yield self._finish_future
if self.is_client and self._disconnect_on_finish:
self.close()
if self.stream is None:
raise gen.Return(False)
except httputil.HTTPInputError as e:
print("Malformed HTTP message from %s: %s", (self.context, e))
self.close()
raise gen.Return(False)
finally:
if need_delegate_close:
with _ExceptionLoggingContext():
delegate.on_connection_close()
self._clear_callbacks()
raise gen.Return(True)