def _prepare_request(self, messages):
# Determine the URL for the messages
url = self.url
if self._append_message_type and len(messages) == 1 and messages[0].channel.is_meta():
message_type = '/'.join(messages[0].channel.parts()[1:])
if not url.endswith('/'):
url += '/'
url += message_type
# Get the headers for the request
headers = HTTPHeaders()
for header, values in self.get_headers().iteritems():
for value in values:
headers.add(header, value)
for header, value in headers.get_all():
self.log.debug('Request header %s: %s' % (header, value))
# Get the body for the request
body = Message.to_json(messages, encoding='utf8')
self.log.debug('Request body (length: %d): %s' % (len(body), body))
# Get the timeout (in seconds)
timeout = self.get_timeout(messages) / 1000.0
self.log.debug('Request timeout: %ss' % timeout)
# Build and return the request
return HTTPRequest(
url,
method='POST',
headers=headers,
body=body,
connect_timeout=timeout,
request_timeout=timeout
)
def compose_response(self):
headers = HTTPHeaders()
headers = self.process_headers(headers)
lines = []
lines.append("HTTP/1.1 %d %s" % (
self.response.code,
responses[self.response.code]
))
for k, v in headers.get_all():
lines.append(k + ": " + v)
head = "\r\n".join(lines) + "\r\n\r\n"
head = head.encode("ascii")
body = self.process_body(self.response.body)
if body is not None:
return head + self.response.body
else:
return head
def write_error(self, status_code, **kwargs):
exc_info = kwargs.pop('exc_info')
kwargs['exception'] = exc_info[1]
if debug:
message = "<h4>Error Code: " + str(status_code) + "</h4>"
message += "<h4>Error Type: " + str(exc_info[0]) + "</h4>"
message += "<h4>Error Detail: " + str(exc_info[1]) + "</h4>"
message += "<h4>Header:</h4>"
message += "<br />".join(
'%s: "%s"' % (elem[0], elem[1]) for elem in HTTPHeaders.get_all(self.request.headers))
message += "<h4>Content:</h4>"
message += "<br />".join(
['%s: "%s"' % (key, ', '.join(value)) for key, value in self.request.arguments.items()])
if "exc_info" in kwargs:
message += "<h4>Traceback:</h4>"
message += "<br />".join(traceback.format_exception(*kwargs["exc_info"]))
message = message.replace("<", "").replace(">", "")
if status_code == 404:
sendEmail(u"404 页面找不到", message.decode('utf-8'))
self.render('404.html')
elif status_code == 500:
sendEmail(u"500 页面找不到", message.decode('utf-8'))
# self.render('500.html')
else:
sendEmail(u"*** 未知异常", message.decode('utf-8'))
tornado.web.RequestHandler.write_error(self, status_code, **kwargs)
else:
tornado.web.RequestHandler.write_error(self, status_code, **kwargs)
def _prepare_request(self, messages):
# Determine the URL for the messages
url = self.url
if self._append_message_type and len(messages) == 1 and messages[0].channel.is_meta():
message_type = '/'.join(messages[0].channel.parts()[1:])
if not url.endswith('/'):
url += '/'
url += message_type
# Get the headers for the request
headers = HTTPHeaders()
for header, values in self.get_headers().items():
for value in values:
headers.add(header, value)
for header, value in headers.get_all():
self.log.debug('Request header %s: %s' % (header, value))
# Get the body for the request
body = Message.to_json(messages, encoding='utf8')
self.log.debug('Request body (length: %d): %s' % (len(body), body))
# Get the timeout (in seconds)
timeout = self.get_timeout(messages) / 1000.0
self.log.debug('Request timeout: %ss' % timeout)
# Build and return the request
return HTTPRequest(
url,
method='POST',
headers=headers,
body=body,
connect_timeout=timeout,
request_timeout=timeout
)
def assert_headers_contains(self, headers: HTTPHeaders, contained: dict):
self.assertTrue(
all(item in headers.get_all() for item in contained.items()),
"Headers does not contain expected headers"
"\n Expected headers:"
f"\n {pformat(contained, indent=6)}"
"\n All headers:"
f"\n {pformat(dict(headers.get_all()), indent=6)}")
def test_pickle_roundtrip(self):
headers = HTTPHeaders()
headers.add('Set-Cookie', 'a=b')
headers.add('Set-Cookie', 'c=d')
headers.add('Content-Type', 'text/html')
pickled = pickle.dumps(headers)
unpickled = pickle.loads(pickled)
self.assertEqual(sorted(headers.get_all()), sorted(unpickled.get_all()))
self.assertEqual(sorted(headers.items()), sorted(unpickled.items()))
def assert_headers_contains(self, headers: HTTPHeaders, contained: dict):
self.assertTrue(
all(item in headers.get_all() for item in contained.items()),
"Headers does not contain expected headers"
"\n Expected headers:"
f"\n {pformat(contained, indent=6)}"
"\n All headers:"
f"\n {pformat(dict(headers.get_all()), indent=6)}"
)
def test_pickle_roundtrip(self):
headers = HTTPHeaders()
headers.add('Set-Cookie', 'a=b')
headers.add('Set-Cookie', 'c=d')
headers.add('Content-Type', 'text/html')
pickled = pickle.dumps(headers)
unpickled = pickle.loads(pickled)
self.assertEqual(sorted(headers.get_all()), sorted(unpickled.get_all()))
self.assertEqual(sorted(headers.items()), sorted(unpickled.items()))
def test_pickle_roundtrip(self):
headers = HTTPHeaders()
headers.add("Set-Cookie", "a=b")
headers.add("Set-Cookie", "c=d")
headers.add("Content-Type", "text/html")
pickled = pickle.dumps(headers)
unpickled = pickle.loads(pickled)
self.assertEqual(sorted(headers.get_all()), sorted(unpickled.get_all()))
self.assertEqual(sorted(headers.items()), sorted(unpickled.items()))
def test_setdefault(self):
headers = HTTPHeaders()
headers["foo"] = "bar"
# If a value is present, setdefault returns it without changes.
self.assertEqual(headers.setdefault("foo", "baz"), "bar")
self.assertEqual(headers["foo"], "bar")
# If a value is not present, setdefault sets it for future use.
self.assertEqual(headers.setdefault("quux", "xyzzy"), "xyzzy")
self.assertEqual(headers["quux"], "xyzzy")
self.assertEqual(sorted(headers.get_all()), [("Foo", "bar"), ("Quux", "xyzzy")])
def test_pickle_roundtrip(self):
headers = HTTPHeaders()
headers.add("Set-Cookie", "a=b")
headers.add("Set-Cookie", "c=d")
headers.add("Content-Type", "text/html")
pickled = pickle.dumps(headers)
unpickled = pickle.loads(pickled)
self.assertEqual(sorted(headers.get_all()),
sorted(unpickled.get_all()))
self.assertEqual(sorted(headers.items()), sorted(unpickled.items()))
def test_setdefault(self):
headers = HTTPHeaders()
headers['foo'] = 'bar'
# If a value is present, setdefault returns it without changes.
self.assertEqual(headers.setdefault('foo', 'baz'), 'bar')
self.assertEqual(headers['foo'], 'bar')
# If a value is not present, setdefault sets it for future use.
self.assertEqual(headers.setdefault('quux', 'xyzzy'), 'xyzzy')
self.assertEqual(headers['quux'], 'xyzzy')
self.assertEqual(sorted(headers.get_all()), [('Foo', 'bar'), ('Quux', 'xyzzy')])
def test_setdefault(self):
headers = HTTPHeaders()
headers['foo'] = 'bar'
# If a value is present, setdefault returns it without changes.
self.assertEqual(headers.setdefault('foo', 'baz'), 'bar')
self.assertEqual(headers['foo'], 'bar')
# If a value is not present, setdefault sets it for future use.
self.assertEqual(headers.setdefault('quux', 'xyzzy'), 'xyzzy')
self.assertEqual(headers['quux'], 'xyzzy')
self.assertEqual(sorted(headers.get_all()), [('Foo', 'bar'), ('Quux', 'xyzzy')])
def test_setdefault(self):
headers = HTTPHeaders()
headers["foo"] = "bar"
# If a value is present, setdefault returns it without changes.
self.assertEqual(headers.setdefault("foo", "baz"), "bar")
self.assertEqual(headers["foo"], "bar")
# If a value is not present, setdefault sets it for future use.
self.assertEqual(headers.setdefault("quux", "xyzzy"), "xyzzy")
self.assertEqual(headers["quux"], "xyzzy")
self.assertEqual(sorted(headers.get_all()), [("Foo", "bar"),
("Quux", "xyzzy")])
def test_serialize_response(self):
req = HTTPRequest("http://foo.com")
dct = {'foo': 'bar'}
headers = HTTPHeaders()
headers.add("Foo", "bar")
headers.add("Foo", "bar2")
headers.add("Foo2", "bar3")
buf = BytesIO(b"foo")
response = HTTPResponse(req, 200, headers=headers, buffer=buf)
msg = serialize_http_response(response, dict_to_inject=dct)
(status_code, body, body_link, headers, extra_dict) = \
unserialize_response_message(msg)
self.assertEquals(status_code, 200)
self.assertEquals(body_link, None)
self.assertEquals(len(extra_dict), 1)
self.assertEquals(extra_dict['foo'], 'bar')
self.assertEquals(body, b"foo")
self.assertEquals(len(list(headers.get_all())), 3)
self.assertEquals(headers['Foo2'], "bar3")
self.assertEquals(headers['Foo'], "bar,bar2")
def write_error(self, status_code, **kwargs):
exc_info = kwargs.pop('exc_info')
kwargs['exception'] = exc_info[1]
if debug:
message = "<h4>Error Code: " + str(status_code) + "</h4>"
message += "<h4>Error Type: " + str(exc_info[0]) + "</h4>"
message += "<h4>Error Detail: " + str(exc_info[1]) + "</h4>"
message += "<h4>Header:</h4>"
message += "<br />".join(
'%s: "%s"' % (elem[0], elem[1])
for elem in HTTPHeaders.get_all(self.request.headers))
message += "<h4>Content:</h4>"
message += "<br />".join([
'%s: "%s"' % (key, ', '.join(value))
for key, value in self.request.arguments.items()
])
if "exc_info" in kwargs:
message += "<h4>Traceback:</h4>"
message += "<br />".join(
traceback.format_exception(*kwargs["exc_info"]))
message = message.replace("<", "").replace(">", "")
if status_code == 404:
sendEmail(u"404 页面找不到", message.decode('utf-8'))
self.render('404.html')
elif status_code == 500:
sendEmail(u"500 页面找不到", message.decode('utf-8'))
# self.render('500.html')
else:
sendEmail(u"*** 未知异常", message.decode('utf-8'))
tornado.web.RequestHandler.write_error(self, status_code,
**kwargs)
else:
tornado.web.RequestHandler.write_error(self, status_code, **kwargs)
def test_header_reuse(self):
# Apps may reuse a headers object if they are only passing in constant
# headers like user-agent. The header object should not be modified.
headers = HTTPHeaders({'User-Agent': 'Foo'})
self.fetch("/hello", headers=headers)
self.assertEqual(list(headers.get_all()), [('User-Agent', 'Foo')])
def write_headers(
self,
start_line: Union[httputil.RequestStartLine,
httputil.ResponseStartLine],
headers: httputil.HTTPHeaders,
chunk: bytes = None,
) -> "Future[None]":
"""Implements `.HTTPConnection.write_headers`."""
lines = []
if self.is_client:
assert isinstance(start_line, httputil.RequestStartLine)
self._request_start_line = start_line
lines.append(
utf8("%s %s HTTP/1.1" % (start_line[0], start_line[1])))
# Client requests with a non-empty body must have either a
# Content-Length or a Transfer-Encoding.
self._chunking_output = (
start_line.method in ("POST", "PUT", "PATCH")
and "Content-Length" not in headers
and ("Transfer-Encoding" not in headers
or headers["Transfer-Encoding"] == "chunked"))
else:
assert isinstance(start_line, httputil.ResponseStartLine)
assert self._request_start_line is not None
assert self._request_headers is not None
self._response_start_line = start_line
lines.append(
utf8("HTTP/1.1 %d %s" % (start_line[1], start_line[2])))
self._chunking_output = (
# TODO: should this use
# self._request_start_line.version or
# start_line.version?
self._request_start_line.version == "HTTP/1.1"
# 1xx, 204 and 304 responses have no body (not even a zero-length
# body), and so should not have either Content-Length or
# Transfer-Encoding headers.
and start_line.code not in (204, 304) and
(start_line.code < 100 or start_line.code >= 200)
# No need to chunk the output if a Content-Length is specified.
and "Content-Length" not in headers
# Applications are discouraged from touching Transfer-Encoding,
# but if they do, leave it alone.
and "Transfer-Encoding" not in headers)
# If connection to a 1.1 client will be closed, inform client
if (self._request_start_line.version == "HTTP/1.1"
and self._disconnect_on_finish):
headers["Connection"] = "close"
# If a 1.0 client asked for keep-alive, add the header.
if (self._request_start_line.version == "HTTP/1.0"
and self._request_headers.get("Connection",
"").lower() == "keep-alive"):
headers["Connection"] = "Keep-Alive"
if self._chunking_output:
headers["Transfer-Encoding"] = "chunked"
if not self.is_client and (self._request_start_line.method == "HEAD"
or cast(httputil.ResponseStartLine,
start_line).code == 304):
self._expected_content_remaining = 0
elif "Content-Length" in headers:
self._expected_content_remaining = int(headers["Content-Length"])
else:
self._expected_content_remaining = None
# TODO: headers are supposed to be of type str, but we still have some
# cases that let bytes slip through. Remove these native_str calls when those
# are fixed.
header_lines = (native_str(n) + ": " + native_str(v)
for n, v in headers.get_all())
lines.extend(l.encode("latin1") for l in header_lines)
for line in lines:
if b"\n" in line:
raise ValueError("Newline in header: " + repr(line))
future = None
if self.stream.closed():
future = self._write_future = Future()
future.set_exception(iostream.StreamClosedError())
future.exception()
else:
future = self._write_future = Future()
data = b"\r\n".join(lines) + b"\r\n\r\n"
if chunk:
data += self._format_chunk(chunk)
self._pending_write = self.stream.write(data)
future_add_done_callback(self._pending_write,
self._on_write_complete)
return future
def write_headers(
self,
start_line: Union[httputil.RequestStartLine, httputil.ResponseStartLine],
headers: httputil.HTTPHeaders,
chunk: bytes = None,
) -> "Future[None]":
"""Implements `.HTTPConnection.write_headers`."""
lines = []
if self.is_client:
assert isinstance(start_line, httputil.RequestStartLine)
self._request_start_line = start_line
lines.append(utf8("%s %s HTTP/1.1" % (start_line[0], start_line[1])))
# Client requests with a non-empty body must have either a
# Content-Length or a Transfer-Encoding.
self._chunking_output = (
start_line.method in ("POST", "PUT", "PATCH")
and "Content-Length" not in headers
and "Transfer-Encoding" not in headers
)
else:
assert isinstance(start_line, httputil.ResponseStartLine)
assert self._request_start_line is not None
assert self._request_headers is not None
self._response_start_line = start_line
lines.append(utf8("HTTP/1.1 %d %s" % (start_line[1], start_line[2])))
self._chunking_output = (
# TODO: should this use
# self._request_start_line.version or
# start_line.version?
self._request_start_line.version == "HTTP/1.1"
# 1xx, 204 and 304 responses have no body (not even a zero-length
# body), and so should not have either Content-Length or
# Transfer-Encoding headers.
and start_line.code not in (204, 304)
and (start_line.code < 100 or start_line.code >= 200)
# No need to chunk the output if a Content-Length is specified.
and "Content-Length" not in headers
# Applications are discouraged from touching Transfer-Encoding,
# but if they do, leave it alone.
and "Transfer-Encoding" not in headers
)
# If connection to a 1.1 client will be closed, inform client
if (
self._request_start_line.version == "HTTP/1.1"
and self._disconnect_on_finish
):
headers["Connection"] = "close"
# If a 1.0 client asked for keep-alive, add the header.
if (
self._request_start_line.version == "HTTP/1.0"
and self._request_headers.get("Connection", "").lower() == "keep-alive"
):
headers["Connection"] = "Keep-Alive"
if self._chunking_output:
headers["Transfer-Encoding"] = "chunked"
if not self.is_client and (
self._request_start_line.method == "HEAD"
or cast(httputil.ResponseStartLine, start_line).code == 304
):
self._expected_content_remaining = 0
elif "Content-Length" in headers:
self._expected_content_remaining = int(headers["Content-Length"])
else:
self._expected_content_remaining = None
# TODO: headers are supposed to be of type str, but we still have some
# cases that let bytes slip through. Remove these native_str calls when those
# are fixed.
header_lines = (
native_str(n) + ": " + native_str(v) for n, v in headers.get_all()
)
lines.extend(l.encode("latin1") for l in header_lines)
for line in lines:
if b"\n" in line:
raise ValueError("Newline in header: " + repr(line))
future = None
if self.stream.closed():
future = self._write_future = Future()
future.set_exception(iostream.StreamClosedError())
future.exception()
else:
future = self._write_future = Future()
data = b"\r\n".join(lines) + b"\r\n\r\n"
if chunk:
data += self._format_chunk(chunk)
self._pending_write = self.stream.write(data)
future_add_done_callback(self._pending_write, self._on_write_complete)
return future
def dump_headers(headers: HTTPHeaders) -> str:
return json.dumps(list(headers.get_all()))
def test_header_reuse(self):
# Apps may reuse a headers object if they are only passing in constant
# headers like user-agent. The header object should not be modified.
headers = HTTPHeaders({'User-Agent': 'Foo'})
self.fetch("/hello", headers=headers)
self.assertEqual(list(headers.get_all()), [('User-Agent', 'Foo')])
def test_header_reuse(self: typing.Any):
# Apps may reuse a headers object if they are only passing in constant
# headers like user-agent. The header object should not be modified.
headers = HTTPHeaders({"User-Agent": "Foo"})
self.fetch("/hello", headers=headers)
self.assertEqual(list(headers.get_all()), [("User-Agent", "Foo")])
class ResponseHandler(object):
timeout = 10
chunked = False
length = None
finished_headers = False
length_sent = False
code = message = path = None
def __init__(self, conn, stream, start_line):
self.conn = conn
self.stream = stream
self.start_time = time.time()
self.method, self.path, self.version = start_line
self.version = self.version.rstrip()
if not self.conn.old_client and self.version == 'HTTP/1.0':
self.conn.old_client = True
self.conn.old_client = True
else:
assert self.version == 'HTTP/1.1'
self.headers = HTTPHeaders()
self.headers.add("Server", "MYOB/1.0")
self.pending = [] # delay body writes until headers sent
def date_time_string(self,timestamp=None):
"""Return the current date and time formatted for a message header."""
if timestamp is None:
timestamp = time.time()
if isinstance(timestamp,(int,float)):
year, month, day, hh, mm, ss, wd, y, z = time.gmtime(timestamp)
else:
year, month, day, hh, mm, ss, wd, y, z = timestamp
s = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % (
self.weekdayname[wd],
day, self.monthname[month], year,
hh, mm, ss)
return s
weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
monthname = [None,
'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
def set_chunked(self):
assert False, "uuu"
assert self.length is None, "You can't both be chunked and have a length!"
self.headers["Transport-Encoding"] = "chunked"
self.chunked = True
return self.actually_send_header("Transport-Encoding")
def set_length(self,length):
if self.conn.old_client: return # connection terminates at end of data anyway
assert self.chunked is not True, "You can't specify a length when chunking"
assert self.code not in {301,302,303,304,204},"No length for these codes"
self.headers['Content-Length'] = denumber(length)
self.length = length
note('set the length te',length)
return self.actually_send_header('Content-Length')
def check_header(self,name,value):
if not self.chunked and name == 'Transport-Encoding' and 'chunked' in self.headers[name]:
assert False, 'wonk'
self.chunked = True
elif name == 'Content-Length':
if self.conn.old_client: return True # connection terminates at end of data anyway
if not self.length:
assert self.code not in {204},"No length for these codes"
self.length = value
status_sent = False
def send_status(self,code,message):
print('status',code,message)
self.code = code
self.message = message
self.status_sent = True
return self.stream.write(b'HTTP/1.1 '+
utf8(denumber(code))+
b' '+utf8(message)+b'\r\n')
def send_header(self,name,value=None):
if value is not None:
self.headers.add(name,decodeHeader(name,value))
if self.check_header(name,value):
del self.headers[name]
else:
return self.actually_send_header(name)
needDate = True
@gen.coroutine
def actually_send_header(self,name):
if self.status_sent is not True:
if self.code:
yield self.send_status(self.code,self.message)
else:
print("need to send status first!")
raise RuntimeError('please send status')
yield send_header(self.stream, name, self.headers[name])
if name == 'Date':
self.needDate = False
del self.headers[name]
@gen.coroutine
def end_headers(self):
if self.finished_headers:
raise RuntimeError('finished headers already!')
if not self.conn.old_client:
self.headers.add("Connection","keep-alive")
if self.needDate:
yield self.send_header('Date',datetime.now())
for name,normalized_value in self.headers.get_all():
self.check_header(name,normalized_value)
yield send_header(self.stream, name, normalized_value)
if not self.chunked and self.length is None:
if self.code in {304,204}: #...?
assert not self.pending,"No data for these codes allowed (or length header)"
else:
if not self.conn.old_client:
length = 0
for chunk in self.pending:
# no reason to chunk, since we got all the body already
length += len(chunk)
self.headers.add("Content-Length",denumber(length))
yield self.actually_send_header("Content-Length")
self.length = length
yield self.stream.write(b'\r\n')
self.finished_headers = True
yield self.flush_pending()
@gen.coroutine
def flush_pending(self):
pending = self.pending
self.pending = None
for chunk in pending:
yield self.write(chunk)
written = 0
def write(self,chunk):
if self.pending is not None:
self.pending.append(chunk)
return success
if self.chunked:
chunk = self.conn._format_chunk(chunk)
elif self.length:
if isinstance(chunk,str):
chunk = utf8(chunk)
self.length -= len(chunk)
elif self.conn.old_client:
if isinstance(chunk,str):
chunk = utf8(chunk)
elif self.length == 0:
raise RuntimeError("Either tried to send 2 chunks while setting a length, or body was supposed to be empty.")
else:
raise RuntimeError("Can't add to the body and automatically calculate content length. Either set chunked, or set the length, or write the whole body before ending headers.")
self.written += len(chunk)
return self.stream.write(chunk)
@gen.coroutine
def respond(self):
try:
response = yield self.do()
note('got response',derpid(self))
if not self.finished_headers:
yield self.end_headers()
except Redirect as e:
yield self.send_status(e.code,e.message)
yield self.send_header('Location',e.location)
yield self.end_headers()
finally:
self.recordAccess()
def redirect(self,location,code=302,message='boink'):
raise Redirect(self,location,code,message)
ip = None
def recordAccess(self):
print(json.dumps((self.ip or self.conn.address[0],self.method,self.code,self.path,self.written,time.time())))
def received_headers(self): pass
def received_header(self,name,value):
"received a header just now, can setup, or raise an error if this is not a good header"
if name == 'Content-Length':
note('setting length')
self.length = int(value)
elif name == 'Transport-Encoding':
if 'chunked' in value:
assert False, 'uhhh'
self.chunked = True
def OK(self):
"Check headers/IP if this request's body is OK to push."
return True
def do(self):
"return a Future for when writing the response is finished."
"override this to wrap all requests in context"
return getattr(self, self.method.lower())()
def abort(self,stage):
"called when a request was in the process of being received, or waiting to start writing back and the connection dies."
