def read_message(self):
if self.request is not None:
note('err',derpid(self),self.request)
raise RuntimeError
try:
start_line = yield self.stream.read_until(b'\r\n',max_bytes=self.max_start_header_length)
except iostream.StreamClosedError:
note('done with',self.address,derpid(self))
raise
except iostream.UnsatisfiableReadError:
print("garbage from address",self.address)
self.stream.close()
raise iostream.StreamClosedError
start_line = httputil.parse_request_start_line(start_line.decode('utf-8'))
note('setting request',derpid(self),start_line)
self.request = self.requestFactory(self, self.stream, start_line)
headers = yield maybeTimeout(self.stream, self.header_timeout, self.read_headers())
yield maybeTimeout(self.stream, self.body_timeout, self.read_body(headers))
# Now we've read the request, so start writing it, asynchronously
yield self.startWriting()
note('del self.request (started writing)',derpid(self))
del self.request
assert self.request is None, "boop"
if self.old_client:
self.stream.close()
else:
self.read_next_message('next')
def _read_message(self, delegate):
need_delegate_close = False
try:
header_future = self.stream.read_until_regex(
b"\r?\n\r?\n",
max_bytes=self.params.max_header_size)
if self.params.header_timeout is None:
header_data = yield header_future
else:
try:
header_data = yield gen.with_timeout(
self.stream.io_loop.time() + self.params.header_timeout,
header_future,
quiet_exceptions=iostream.StreamClosedError)
except gen.TimeoutError:
self.close()
raise gen.Return(False)
start_line, headers = self._parse_headers(header_data)
if self.is_client:
start_line = httputil.parse_response_start_line(start_line)
self._response_start_line = start_line
else:
start_line = httputil.parse_request_start_line(start_line)
self._request_start_line = start_line
self._request_headers = headers
self._disconnect_on_finish = not self._can_keep_alive(
start_line, headers)
need_delegate_close = True
with _ExceptionLoggingContext(app_log):
header_future = delegate.headers_received(start_line, headers)
if header_future is not None:
yield header_future
if self.stream is None:
# We've been detached.
need_delegate_close = False
raise gen.Return(False)
skip_body = False
if self.is_client:
if (self._request_start_line is not None and
self._request_start_line.method == 'HEAD'):
skip_body = True
code = start_line.code
if code == 304:
# 304 responses may include the content-length header
# but do not actually have a body.
# http://tools.ietf.org/html/rfc7230#section-3.3
skip_body = True
if code >= 100 and code < 200:
# 1xx responses should never indicate the presence of
# a body.
if ('Content-Length' in headers or
'Transfer-Encoding' in headers):
raise httputil.HTTPInputError(
"Response code %d cannot have body" % code)
# TODO: client delegates will get headers_received twice
# in the case of a 100-continue. Document or change?
yield self._read_message(delegate)
else:
if (headers.get("Expect") == "100-continue" and
not self._write_finished):
self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
if not skip_body:
body_future = self._read_body(
start_line.code if self.is_client else 0, headers, delegate)
if body_future is not None:
if self._body_timeout is None:
yield body_future
else:
try:
yield gen.with_timeout(
self.stream.io_loop.time() + self._body_timeout,
body_future,
quiet_exceptions=iostream.StreamClosedError)
except gen.TimeoutError:
gen_log.info("Timeout reading body from %s",
self.context)
self.stream.close()
raise gen.Return(False)
self._read_finished = True
if not self._write_finished or self.is_client:
need_delegate_close = False
with _ExceptionLoggingContext(app_log):
delegate.finish()
# If we're waiting for the application to produce an asynchronous
# response, and we're not detached, register a close callback
# on the stream (we didn't need one while we were reading)
if (not self._finish_future.done() and
self.stream is not None and
not self.stream.closed()):
self.stream.set_close_callback(self._on_connection_close)
yield self._finish_future
if self.is_client and self._disconnect_on_finish:
self.close()
if self.stream is None:
raise gen.Return(False)
except httputil.HTTPInputError as e:
gen_log.info("Malformed HTTP message from %s: %s",
self.context, e)
if not self.is_client:
yield self.stream.write(b'HTTP/1.1 400 Bad Request\r\n\r\n')
self.close()
raise gen.Return(False)
finally:
if need_delegate_close:
with _ExceptionLoggingContext(app_log):
delegate.on_connection_close()
header_future = None
self._clear_callbacks()
raise gen.Return(True)
def test_parse_request_start_line(self):
start_line = " ".join([self.METHOD, self.PATH, self.VERSION])
parsed_start_line = parse_request_start_line(start_line)
self.assertEqual(parsed_start_line.method, self.METHOD)
self.assertEqual(parsed_start_line.path, self.PATH)
self.assertEqual(parsed_start_line.version, self.VERSION)
def _read_message(self, delegate):
need_delegate_close = False
try:
header_future = self.stream.read_until_regex(
b"\r?\n\r?\n",
max_bytes=self.params.max_header_size)
if self.params.header_timeout is None:
header_data = yield header_future
else:
try:
header_data = yield gen.with_timeout(
self.stream.io_loop.time() + self.params.header_timeout,
header_future,
io_loop=self.stream.io_loop)
except gen.TimeoutError:
self.close()
raise gen.Return(False)
start_line, headers = self._parse_headers(header_data)
if self.is_client:
start_line = httputil.parse_response_start_line(start_line)
self._response_start_line = start_line
else:
start_line = httputil.parse_request_start_line(start_line)
self._request_start_line = start_line
self._request_headers = headers
self._disconnect_on_finish = not self._can_keep_alive(
start_line, headers)
need_delegate_close = True
header_future = delegate.headers_received(start_line, headers)
if header_future is not None:
yield header_future
if self.stream is None:
# We've been detached.
need_delegate_close = False
raise gen.Return(False)
skip_body = False
if self.is_client:
if (self._request_start_line is not None and
self._request_start_line.method == 'HEAD'):
skip_body = True
code = start_line.code
if code == 304:
skip_body = True
if code >= 100 and code < 200:
# TODO: client delegates will get headers_received twice
# in the case of a 100-continue. Document or change?
yield self._read_message(delegate)
else:
if (headers.get("Expect") == "100-continue" and
not self._write_finished):
self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
if not skip_body:
body_future = self._read_body(headers, delegate)
if body_future is not None:
if self._body_timeout is None:
yield body_future
else:
try:
yield gen.with_timeout(
self.stream.io_loop.time() + self._body_timeout,
body_future, self.stream.io_loop)
except gen.TimeoutError:
gen_log.info("Timeout reading body from %s",
self.context)
self.stream.close()
raise gen.Return(False)
self._read_finished = True
if not self._write_finished or self.is_client:
need_delegate_close = False
delegate.finish()
yield self._finish_future
if self.is_client and self._disconnect_on_finish:
self.close()
if self.stream is None:
raise gen.Return(False)
except httputil.HTTPInputException as e:
gen_log.info("Malformed HTTP message from %s: %s",
self.context, e)
self.close()
raise gen.Return(False)
finally:
if need_delegate_close:
delegate.on_connection_close()
self._clear_callbacks()
raise gen.Return(True)
def _read_message(self, delegate):
need_delegate_close = False
try:
header_future = self.stream.read_until_regex(
b"\r?\n\r?\n",
max_bytes=self.params.max_header_size)
if self.params.header_timeout is None:
header_data = yield header_future
else:
try:
header_data = yield gen.with_timeout(
self.stream.io_loop.time() + self.params.header_timeout,
header_future,
io_loop=self.stream.io_loop,
quiet_exceptions=iostream.StreamClosedError)
except gen.TimeoutError:
self.close()
raise gen.Return(False)
start_line, headers = self._parse_headers(header_data)
if self.is_client:
start_line = httputil.parse_response_start_line(start_line)
self._response_start_line = start_line
else:
start_line = httputil.parse_request_start_line(start_line)
self._request_start_line = start_line
self._request_headers = headers
self._disconnect_on_finish = not self._can_keep_alive(
start_line, headers)
need_delegate_close = True
with _ExceptionLoggingContext():
header_future = delegate.headers_received(start_line, headers)
if header_future is not None:
yield header_future
if self.stream is None:
# We've been detached.
need_delegate_close = False
raise gen.Return(False)
skip_body = False
if self.is_client:
if (self._request_start_line is not None and
self._request_start_line.method == 'HEAD'):
skip_body = True
code = start_line.code
if code == 304:
skip_body = True
if code >= 100 and code < 200:
if ('Content-Length' in headers or
'Transfer-Encoding' in headers):
raise httputil.HTTPInputError(
"Response code %d cannot have body" % code)
yield self._read_message(delegate)
else:
if (headers.get("Expect") == "100-continue" and
not self._write_finished):
self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
if not skip_body:
body_future = self._read_body(
start_line.code if self.is_client else 0, headers, delegate)
if body_future is not None:
if self._body_timeout is None:
yield body_future
else:
try:
yield gen.with_timeout(
self.stream.io_loop.time() + self._body_timeout,
body_future, self.stream.io_loop,
quiet_exceptions=iostream.StreamClosedError)
except gen.TimeoutError:
print("Timeout reading body from %s" % self.context)
self.stream.close()
raise gen.Return(False)
self._read_finished = True
if not self._write_finished or self.is_client:
need_delegate_close = False
with _ExceptionLoggingContext():
delegate.finish()
if (not self._finish_future.done() and
self.stream is not None and
not self.stream.closed()):
self.stream.set_close_callback(self._on_connection_close)
yield self._finish_future
if self.is_client and self._disconnect_on_finish:
self.close()
if self.stream is None:
raise gen.Return(False)
except httputil.HTTPInputError as e:
print("Malformed HTTP message from %s: %s", (self.context, e))
self.close()
raise gen.Return(False)
finally:
if need_delegate_close:
with _ExceptionLoggingContext():
delegate.on_connection_close()
self._clear_callbacks()
raise gen.Return(True)
def _read_message(self, delegate):
need_delegate_close = False
try:
# 消息头与消息体之间由一个空行分开
header_future = self.stream.read_until_regex(
b"\r?\n\r?\n",
max_bytes=self.params.max_header_size)
if self.params.header_timeout is None:
header_data = yield header_future
else:
try:
header_data = yield gen.with_timeout(
self.stream.io_loop.time() + self.params.header_timeout,
header_future,
io_loop=self.stream.io_loop)
except gen.TimeoutError:
self.close()
raise gen.Return(False)
# 解析消息头，分离头字段和首行（request-line/status-line）
start_line, headers = self._parse_headers(header_data)
# 作为 client 解析的是 server 的 response，作为 server 解析的是 client 的 request。
# response 与 request 的 start_line(status-line/request-line) 的字段内容不同：
# 1. response's status-line: HTTP-Version SP Status-Code SP Reason-Phrase CRLF
# 2. request's request-line：Method SP Request-URI SP HTTP-Version CRLF
# start_line 的值是一个 namedtuple。
if self.is_client:
start_line = httputil.parse_response_start_line(start_line)
self._response_start_line = start_line
else:
start_line = httputil.parse_request_start_line(start_line)
self._request_start_line = start_line
self._request_headers = headers
# 非 keep-alive 的请求或响应处理完成后要关闭连接。
self._disconnect_on_finish = not self._can_keep_alive(
start_line, headers)
need_delegate_close = True
with _ExceptionLoggingContext(app_log):
header_future = delegate.headers_received(start_line, headers)
if header_future is not None:
# 如果 header_future 是一个 `Future` 实例，则要等到完成才读取 body。
yield header_future
# websocket ？？？
if self.stream is None:
# We've been detached.
need_delegate_close = False
raise gen.Return(False)
skip_body = False
if self.is_client:
# 作为 client 如果发起的是 HEAD 请求，那么 server response 应该无消息体
if (self._request_start_line is not None and
self._request_start_line.method == 'HEAD'):
skip_body = True
code = start_line.code
if code == 304:
# 304 responses may include the content-length header
# but do not actually have a body.
# http://tools.ietf.org/html/rfc7230#section-3.3
skip_body = True
if code >= 100 and code < 200:
# 1xx responses should never indicate the presence of
# a body.
if ('Content-Length' in headers or
'Transfer-Encoding' in headers):
raise httputil.HTTPInputError(
"Response code %d cannot have body" % code)
# TODO: client delegates will get headers_received twice
# in the case of a 100-continue. Document or change?
yield self._read_message(delegate)
else:
# 100-continue 这个状态码是在 HTTP/1.1 中为了提高传输效率而设置的。当
# client 需要 POST 较大数据给 WebServer 时，可以在发送 HTTP 请求时带上
# Expect: 100-continue，WebServer 如果接受这个请求则应答一个
# ``HTTP/1.1 100 (Continue)``，那么 client 就继续传输 request body，
# 否则应答 ``HTTP/1.1 417 Expectation Failed`` client 就放弃传输剩余
# 的数据。（注：Expect 头部域，用于指出客户端要求的特殊服务器行为采用扩展语法
# 定义，以方便扩展。）
if (headers.get("Expect") == "100-continue" and
not self._write_finished):
self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
if not skip_body:
body_future = self._read_body(
start_line.code if self.is_client else 0, headers, delegate)
if body_future is not None:
if self._body_timeout is None:
yield body_future
else:
try:
yield gen.with_timeout(
self.stream.io_loop.time() + self._body_timeout,
body_future, self.stream.io_loop)
except gen.TimeoutError:
gen_log.info("Timeout reading body from %s",
self.context)
self.stream.close()
raise gen.Return(False)
self._read_finished = True
# 对 client mode ，response 解析完成就调用 HTTPMessageDelegate.finish() 方法是合适的；
# 对 server mode ，_write_finished 表示 response 是否发送完成，未完成前调用
# HTTPMessageDelegate.finish() 方法让 delegate 执行请求响应；
if not self._write_finished or self.is_client:
need_delegate_close = False
with _ExceptionLoggingContext(app_log):
delegate.finish()
# If we're waiting for the application to produce an asynchronous
# response, and we're not detached, register a close callback
# on the stream (we didn't need one while we were reading)
#
# NOTE:_finish_future resolves when all data has been written and flushed
# to the IOStream.
#
# hold 住执行流程，直到异步响应完成，所有数据都写入 fd，才继续后续处理，通常调用方执行 `finish` 方法
# 设置 `_finish_future` 完成，详细见 `finish` 和 `_finish_request` 方法实现。
if (not self._finish_future.done() and
self.stream is not None and
not self.stream.closed()):
self.stream.set_close_callback(self._on_connection_close)
yield self._finish_future
# 对于 client mode，处理完响应后如果不是 keep-alive 就断开连接。
# 对于 server mode，需要在 response 完成后才断开连接，详细见 _finish_request/finish 方法实现。
if self.is_client and self._disconnect_on_finish:
self.close()
if self.stream is None:
raise gen.Return(False)
except httputil.HTTPInputError as e:
gen_log.info("Malformed HTTP message from %s: %s",
self.context, e)
self.close()
raise gen.Return(False)
finally:
# 连接 “关闭” 前还没能结束处理请求（call HTTPMessageDelegate.finish()），则
# call HTTPMessageDelegate.on_connection_close()
if need_delegate_close:
with _ExceptionLoggingContext(app_log):
delegate.on_connection_close()
self._clear_callbacks()
raise gen.Return(True)
def request(self, conn, src, dst):
keep_alive = True
req = None
conn = conn.copy()
req_line = yield
while keep_alive and req_line is not None:
body = ""
headers = MultiOrderedDict()
try:
req = httputil.parse_request_start_line(req_line.strip())
except httputil.HTTPInputError:
if req_line != "":
self.log("HTTP Error: Malformed request start line: '{}'", req_line)
req_line = yield
continue
while True:
header_line = yield
if header_line is None:
break
if not header_line.strip():
break
self.parse_header_line(headers, header_line.strip())
if req.version == "HTTP/1.0":
keep_alive = headers.last("connection", "").lower().strip() == "keep-alive"
else:
keep_alive = headers.last("connection", "").lower().strip() != "close"
if "content-length" in headers:
try:
content_length = int(headers.last("content-length"))
except ValueError:
content_length = None
else:
content_length = None
if req.method != "POST":
content_length = content_length or 0
if header_line is not None:
#body += conn["lbl_buffers"][dst]
#conn["lbl_buffers"][dst] = ""
conn["lbl_disable"](dst)
while len(body) < content_length or content_length is None:
data = yield
if data is None:
break
body += data
if "content-encoding" in headers:
conn["http_decoded"] = False
encoding = headers.last("content-encoding")
self.log("req encoding: {}", encoding)
if encoding in self.DECODERS:
try:
body = self.DECODERS[encoding](body)
conn["http_decoded"] = True
except:
self.log("Unable to decode content '{}' len={}/{}", encoding, len(body), content_length)
else:
conn["http_decoded"] = True
conn["lbl_enable"](dst)
conn["http_headers"] = headers
conn["http_request"] = req
req_line = yield self.bubble(dst, conn, body)
