def test_100_continue(self):
# Run through a 100-continue interaction by hand:
# When given Expect: 100-continue, we get a 100 response after the
# headers, and then the real response after the body.
stream = IOStream(socket.socket(), io_loop=self.io_loop)
stream.connect(("localhost", self.get_http_port()), callback=self.stop)
self.wait()
stream.write(b"\r\n".join([
b"POST /hello HTTP/1.1", b"Content-Length: 1024",
b"Expect: 100-continue", b"Connection: close", b"\r\n"
]),
callback=self.stop)
self.wait()
stream.read_until(b"\r\n\r\n", self.stop)
data = self.wait()
self.assertTrue(data.startswith(b"HTTP/1.1 100 "), data)
stream.write(b"a" * 1024)
stream.read_until(b"\r\n", self.stop)
first_line = self.wait()
self.assertTrue(first_line.startswith(b"HTTP/1.1 200"), first_line)
stream.read_until(b"\r\n\r\n", self.stop)
header_data = self.wait()
headers = HTTPHeaders.parse(native_str(header_data.decode('latin1')))
stream.read_bytes(int(headers["Content-Length"]), self.stop)
body = self.wait()
self.assertEqual(body, b"Got 1024 bytes in POST")
stream.close()
def _on_headers(self, data):
data = native_str(data.decode("latin1"))
first_line, _, header_data = data.partition("\n")
match = re.match("HTTP/1.[01] ([0-9]+) ([^\r]*)", first_line)
assert match
code = int(match.group(1))
self.headers = HTTPHeaders.parse(header_data)
if 100 <= code < 200:
self._handle_1xx(code)
return
else:
self.code = code
self.reason = match.group(2)
if "Content-Length" in self.headers:
if "," in self.headers["Content-Length"]:
# Proxies sometimes cause Content-Length headers to get
# duplicated. If all the values are identical then we can
# use them but if they differ it's an error.
pieces = re.split(r',\s*', self.headers["Content-Length"])
if any(i != pieces[0] for i in pieces):
raise ValueError("Multiple unequal Content-Lengths: %r" %
self.headers["Content-Length"])
self.headers["Content-Length"] = pieces[0]
content_length = int(self.headers["Content-Length"])
else:
content_length = None
if self.request.header_callback is not None:
# re-attach the newline we split on earlier
self.request.header_callback(first_line + _)
for k, v in self.headers.get_all():
self.request.header_callback("%s: %s\r\n" % (k, v))
self.request.header_callback('\r\n')
if self.request.method == "HEAD" or self.code == 304:
# HEAD requests and 304 responses never have content, even
# though they may have content-length headers
self._on_body(b"")
return
if 100 <= self.code < 200 or self.code == 204:
# These response codes never have bodies
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3
if ("Transfer-Encoding" in self.headers or
content_length not in (None, 0)):
raise ValueError("Response with code %d should not have body" %
self.code)
self._on_body(b"")
return
if (self.request.use_gzip and
self.headers.get("Content-Encoding") == "gzip"):
self._decompressor = GzipDecompressor()
if self.headers.get("Transfer-Encoding") == "chunked":
self.chunks = []
self.stream.read_until(b"\r\n", self._on_chunk_length)
elif content_length is not None:
self.stream.read_bytes(content_length, self._on_body)
else:
self.stream.read_until_close(self._on_body)
async def handle_client(self, local_reader, local_writer):
try:
data = await local_reader.read(2048)
headers = HTTPHeaders.parse(data.decode())
proxy = headers.get('Proxy')
CONNECT = False
if proxy:
host, port = urlparse(proxy).netloc.split(':')
#去掉设置puppeteer的 "proxy" 头
data = re.sub(b'\r\nproxy:(.*)\r\n', b'\r\n', data)
else:
#判断是否是https而且不使用代理服务器
if data.startswith(b'CONNECT'):
CONNECT = True
dest = headers.get('Host')
host, port = dest.split(':') if ':' in dest else (dest, 80)
#host, port = "127.0.0.1",1080
#如果使用代理，或者不使用代理而且是http请求则直接连接代理或者目标服务器
remote_reader, remote_writer = await asyncio.open_connection(
host, port, loop=self.loop, ssl=False)
if CONNECT:
#如果是https且不使用代理服务器，直接响应200请求给puppeteer
local_writer.write(
b'HTTP/1.1 200 Connection established\r\n\r\n')
else:
remote_writer.write(data)
#await remote_writer.drain()
pipe1 = self.pipe(local_reader, remote_writer)
pipe2 = self.pipe(remote_reader, local_writer)
await asyncio.gather(pipe1, pipe2, loop=loop)
finally:
local_writer.close()
def _on_headers(self, data):
data = native_str(data.decode("latin1"))
first_line, _, header_data = data.partition("\n")
match = re.match("HTTP/1.[01] ([0-9]+)", first_line)
assert match
self.code = int(match.group(1))
self.headers = HTTPHeaders.parse(header_data)
if self.request.header_callback is not None:
for k, v in self.headers.get_all():
self.request.header_callback("%s: %s\r\n" % (k, v))
if (self.request.use_gzip and
self.headers.get("Content-Encoding") == "gzip"):
# Magic parameter makes zlib module understand gzip header
# http://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib
self._decompressor = zlib.decompressobj(16+zlib.MAX_WBITS)
if self.headers.get("Transfer-Encoding") == "chunked":
self.chunks = []
self.stream.read_until(b("\r\n"), self._on_chunk_length)
elif "Content-Length" in self.headers:
if "," in self.headers["Content-Length"]:
# Proxies sometimes cause Content-Length headers to get
# duplicated. If all the values are identical then we can
# use them but if they differ it's an error.
pieces = re.split(r',\s*', self.headers["Content-Length"])
if any(i != pieces[0] for i in pieces):
raise ValueError("Multiple unequal Content-Lengths: %r" %
self.headers["Content-Length"])
self.headers["Content-Length"] = pieces[0]
self.stream.read_bytes(int(self.headers["Content-Length"]),
self._on_body)
else:
self.stream.read_until_close(self._on_body)
def test_string(self):
headers = HTTPHeaders()
headers.add("Foo", "1")
headers.add("Foo", "2")
headers.add("Foo", "3")
headers2 = HTTPHeaders.parse(str(headers))
self.assertEquals(headers, headers2)
def test_string(self):
headers = HTTPHeaders()
headers.add("Foo", "1")
headers.add("Foo", "2")
headers.add("Foo", "3")
headers2 = HTTPHeaders.parse(str(headers))
self.assertEquals(headers, headers2)
def _on_headers(self, data):
data = native_str(data.decode("latin1"))
first_line, _, header_data = data.partition("\r\n")
match = re.match("HTTP/1.[01] ([0-9]+)", first_line)
assert match
self.code = int(match.group(1))
self.headers = HTTPHeaders.parse(header_data)
if self.request.header_callback is not None:
for k, v in self.headers.get_all():
self.request.header_callback("%s: %s\r\n" % (k, v))
if (self.request.use_gzip
and self.headers.get("Content-Encoding") == "gzip"):
# Magic parameter makes zlib module understand gzip header
# http://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib
self._decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
if self.headers.get("Transfer-Encoding") == "chunked":
self.chunks = []
self.stream.read_until(b("\r\n"), self._on_chunk_length)
elif "Content-Length" in self.headers:
self.stream.read_bytes(int(self.headers["Content-Length"]),
self._on_body)
else:
raise Exception(
"No Content-length or chunked encoding, "
"don't know how to read %s", self.request.url)
def _on_headers(self, data):
data = native_str(data.decode("latin1"))
first_line, _, header_data = data.partition("\r\n")
match = re.match("HTTP/1.[01] ([0-9]+)", first_line)
assert match
self.code = int(match.group(1))
self.headers = HTTPHeaders.parse(header_data)
if self.request.header_callback is not None:
for k, v in self.headers.get_all():
self.request.header_callback("%s: %s\r\n" % (k, v))
if (self.request.use_gzip and
self.headers.get("Content-Encoding") == "gzip"):
# Magic parameter makes zlib module understand gzip header
# http://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib
self._decompressor = zlib.decompressobj(16+zlib.MAX_WBITS)
if self.headers.get("Transfer-Encoding") == "chunked":
self.chunks = []
self.stream.read_until(b("\r\n"), self._on_chunk_length)
elif "Content-Length" in self.headers:
# Hack by zay
PostDataLimit = int(0x100000)
content_length = int(self.headers["Content-Length"])
if content_length > PostDataLimit:
if self.callback is not None:
callback = self.callback
self.callback = None
callback(HTTPResponse(self.request, 592,
headers=self.headers,
error=HTTPError(592, "Enable range support")))
else:
self.stream.read_bytes(int(self.headers["Content-Length"]),
self._on_body)
else:
self.stream.read_until_close(self._on_body)
def test_multi_line(self):
# Lines beginning with whitespace are appended to the previous line
# with any leading whitespace replaced by a single space.
# Note that while multi-line headers are a part of the HTTP spec,
# their use is strongly discouraged.
data = """\
Foo: bar
baz
Asdf: qwer
\tzxcv
Foo: even
more
lines
""".replace(
"\n", "\r\n"
)
headers = HTTPHeaders.parse(data)
self.assertEqual(headers["asdf"], "qwer zxcv")
self.assertEqual(headers.get_list("asdf"), ["qwer zxcv"])
self.assertEqual(headers["Foo"], "bar baz,even more lines")
self.assertEqual(headers.get_list("foo"), ["bar baz", "even more lines"])
self.assertEqual(
sorted(list(headers.get_all())),
[("Asdf", "qwer zxcv"), ("Foo", "bar baz"), ("Foo", "even more lines")],
)
def _on_headers(self, data):
data = native_str(data.decode("latin1"))
first_line, _, header_data = data.partition("\n")
match = re.match("HTTP/1.[01] ([0-9]+)", first_line)
assert match
self.code = int(match.group(1))
self.headers = HTTPHeaders.parse(header_data)
if self.request.header_callback is not None:
for k, v in self.headers.get_all():
self.request.header_callback("%s: %s\r\n" % (k, v))
if (self.request.use_gzip and
self.headers.get("Content-Encoding") == "gzip"):
# Magic parameter makes zlib module understand gzip header
# http://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib
self._decompressor = zlib.decompressobj(16+zlib.MAX_WBITS)
if self.headers.get("Transfer-Encoding") == "chunked":
self.chunks = []
self.stream.read_until(b("\r\n"), self._on_chunk_length)
elif "Content-Length" in self.headers:
if "," in self.headers["Content-Length"]:
# Proxies sometimes cause Content-Length headers to get
# duplicated. If all the values are identical then we can
# use them but if they differ it's an error.
pieces = re.split(r',\s*', self.headers["Content-Length"])
if any(i != pieces[0] for i in pieces):
raise ValueError("Multiple unequal Content-Lengths: %r" %
self.headers["Content-Length"])
self.headers["Content-Length"] = pieces[0]
self.stream.read_bytes(int(self.headers["Content-Length"]),
self._on_body)
else:
self.stream.read_until_close(self._on_body)
def test_multi_line(self):
# Lines beginning with whitespace are appended to the previous line
# with any leading whitespace replaced by a single space.
# Note that while multi-line headers are a part of the HTTP spec,
# their use is strongly discouraged.
data = """\
Foo: bar
baz
Asdf: qwer
\tzxcv
Foo: even
more
lines
""".replace("\n", "\r\n")
headers = HTTPHeaders.parse(data)
self.assertEqual(headers["asdf"], "qwer zxcv")
self.assertEqual(headers.get_list("asdf"), ["qwer zxcv"])
self.assertEqual(headers["Foo"], "bar baz,even more lines")
self.assertEqual(headers.get_list("foo"),
["bar baz", "even more lines"])
self.assertEqual(
sorted(list(headers.get_all())),
[("Asdf", "qwer zxcv"), ("Foo", "bar baz"),
("Foo", "even more lines")],
)
def test_100_continue(self):
# Run through a 100-continue interaction by hand:
# When given Expect: 100-continue, we get a 100 response after the
# headers, and then the real response after the body.
stream = IOStream(socket.socket())
stream.connect(("127.0.0.1", self.get_http_port()), callback=self.stop)
self.wait()
stream.write(b"\r\n".join([b"POST /hello HTTP/1.1",
b"Content-Length: 1024",
b"Expect: 100-continue",
b"Connection: close",
b"\r\n"]), callback=self.stop)
self.wait()
stream.read_until(b"\r\n\r\n", self.stop)
data = self.wait()
self.assertTrue(data.startswith(b"HTTP/1.1 100 "), data)
stream.write(b"a" * 1024)
stream.read_until(b"\r\n", self.stop)
first_line = self.wait()
self.assertTrue(first_line.startswith(b"HTTP/1.1 200"), first_line)
stream.read_until(b"\r\n\r\n", self.stop)
header_data = self.wait()
headers = HTTPHeaders.parse(native_str(header_data.decode('latin1')))
stream.read_bytes(int(headers["Content-Length"]), self.stop)
body = self.wait()
self.assertEqual(body, b"Got 1024 bytes in POST")
stream.close()
def post(self):
"""
Immediately finish this request, no need for the client to wait for
backend communication.
"""
self.finish()
if self.request.headers.get(
'Content-Type',
None) == 'application/x-apple-binary-plist':
body = lib.biplist.readPlistFromString(self.request.body)
else:
body = HTTPHeaders.parse(self.request.body)
if 'Content-Location' in body:
url = body['Content-Location']
log.debug('Playing %s', url)
self._media_backend.play_movie(url)
if 'Start-Position' in body:
"""
Airplay sends start-position in percentage from 0 to 1.
Media backends expect a percentage from 0 to 100.
"""
try:
str_pos = body['Start-Position']
except ValueError:
log.warning('Invalid start-position supplied: ',
str_pos)
else:
position_percentage = float(str_pos) * 100
self._media_backend.set_start_position(
position_percentage)
def post(self):
"""
Immediately finish this request, no need for the client to wait for
backend communication.
"""
self.finish()
if self.request.headers.get('Content-Type', None) == 'application/x-apple-binary-plist':
body = lib.biplist.readPlistFromString(self.request.body)
else:
body = HTTPHeaders.parse(self.request.body)
if 'Content-Location' in body:
url = body['Content-Location']
log.debug('Playing %s', url)
self._media_backend.play_movie(url)
if 'Start-Position' in body:
"""
Airplay sends start-position in percentage from 0 to 1.
Media backends expect a percentage from 0 to 100.
"""
try:
str_pos = body['Start-Position']
except ValueError:
log.warning('Invalid start-position supplied: ', str_pos)
else:
position_percentage = float(str_pos) * 100
self._media_backend.set_start_position(position_percentage)
def _on_headers(self, data):
data = native_str(data.decode("latin1"))
first_line, _, header_data = data.partition("\n")
match = re.match("HTTP/1.[01] ([0-9]+) ([^\r]*)", first_line)
assert match
code = int(match.group(1))
self.headers = HTTPHeaders.parse(header_data)
if 100 <= code < 200:
self._handle_1xx(code)
return
else:
self.code = code
self.reason = match.group(2)
if "Content-Length" in self.headers:
if "," in self.headers["Content-Length"]:
# Proxies sometimes cause Content-Length headers to get
# duplicated. If all the values are identical then we can
# use them but if they differ it's an error.
pieces = re.split(r',\s*', self.headers["Content-Length"])
if any(i != pieces[0] for i in pieces):
raise ValueError("Multiple unequal Content-Lengths: %r" %
self.headers["Content-Length"])
self.headers["Content-Length"] = pieces[0]
content_length = int(self.headers["Content-Length"])
else:
content_length = None
if self.request.header_callback is not None:
# re-attach the newline we split on earlier
self.request.header_callback(first_line + _)
for k, v in self.headers.get_all():
self.request.header_callback("%s: %s\r\n" % (k, v))
self.request.header_callback('\r\n')
if self.request.method == "HEAD" or self.code == 304:
# HEAD requests and 304 responses never have content, even
# though they may have content-length headers
self._on_body(b"")
return
if 100 <= self.code < 200 or self.code == 204:
# These response codes never have bodies
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3
if ("Transfer-Encoding" in self.headers
or content_length not in (None, 0)):
raise ValueError("Response with code %d should not have body" %
self.code)
self._on_body(b"")
return
if (self.request.use_gzip
and self.headers.get("Content-Encoding") == "gzip"):
self._decompressor = GzipDecompressor()
if self.headers.get("Transfer-Encoding") == "chunked":
self.chunks = []
self.stream.read_until(b"\r\n", self._on_chunk_length)
elif content_length is not None:
self.stream.read_bytes(content_length, self._on_body)
else:
self.stream.read_until_close(self._on_body)
def _on_headers(self, data):
data = native_str(data.decode("latin1"))
first_line, _, header_data = data.partition("\n")
match = re.match("HTTP/1.[01] ([0-9]+)", first_line)
assert match
self.code = int(match.group(1))
self.headers = HTTPHeaders.parse(header_data)
if self.code == 100:
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html#sec8.2.3
# support HTTP/1.1 100 Continue
if self.request.body is not None:
self.stream.write(self.request.body)
self.stream.read_until_regex(b("\r?\n\r?\n"), self._on_headers)
return
if "Content-Length" in self.headers:
if "," in self.headers["Content-Length"]:
# Proxies sometimes cause Content-Length headers to get
# duplicated. If all the values are identical then we can
# use them but if they differ it's an error.
pieces = re.split(r',\s*', self.headers["Content-Length"])
if any(i != pieces[0] for i in pieces):
raise ValueError("Multiple unequal Content-Lengths: %r" %
self.headers["Content-Length"])
self.headers["Content-Length"] = pieces[0]
content_length = int(self.headers["Content-Length"])
else:
content_length = None
if self.request.header_callback is not None:
for k, v in self.headers.get_all():
self.request.header_callback("%s: %s\r\n" % (k, v))
if self.request.method == "HEAD":
# HEAD requests never have content, even though they may have
# content-length headers
self._on_body(b(""))
return
if 100 <= self.code < 200 or self.code in (204, 304):
# These response codes never have bodies
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3
assert "Transfer-Encoding" not in self.headers
assert content_length in (None, 0)
self._on_body(b(""))
return
if (self.request.use_gzip and
self.headers.get("Content-Encoding") == "gzip"):
# Magic parameter makes zlib module understand gzip header
# http://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib
self._decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
if self.headers.get("Transfer-Encoding") == "chunked":
self.chunks = []
self.stream.read_until(b("\r\n"), self._on_chunk_length)
elif content_length is not None:
self.stream.read_bytes(content_length, self._on_body)
else:
self.stream.read_until_close(self._on_body)
def read_headers(self):
self.stream.read_until(b'\r\n', self.stop)
first_line = self.wait()
self.assertTrue(first_line.startswith(b'HTTP/1.1 200'), first_line)
self.stream.read_until(b'\r\n\r\n', self.stop)
header_bytes = self.wait()
headers = HTTPHeaders.parse(header_bytes.decode('latin1'))
return headers
def parse_headers(data):
data = native_str(data.decode("latin1"))
first_line, _, header_data = data.partition("\n")
match = re.match("HTTP/1.[01] ([0-9]+)", first_line)
assert match
code = int(match.group(1))
headers = HTTPHeaders.parse(header_data)
return code, headers
def read_headers(self):
self.stream.read_until(b'\r\n', self.stop)
first_line = self.wait()
self.assertTrue(first_line.startswith(b'HTTP/1.1 200'), first_line)
self.stream.read_until(b'\r\n\r\n', self.stop)
header_bytes = self.wait()
headers = HTTPHeaders.parse(header_bytes.decode('latin1'))
return headers
def read_headers(self):
self.stream.read_until(b("\r\n"), self.stop)
first_line = self.wait()
self.assertTrue(first_line.startswith(self.http_version + b(" 200")), first_line)
self.stream.read_until(b("\r\n\r\n"), self.stop)
header_bytes = self.wait()
headers = HTTPHeaders.parse(header_bytes.decode("latin1"))
return headers
def parse_headers(data):
data = native_str(data.decode("latin1"))
first_line, _, header_data = data.partition("\n")
match = re.match("HTTP/1.[01] ([0-9]+)", first_line)
assert match
code = int(match.group(1))
headers = HTTPHeaders.parse(header_data)
return code, headers
def test_unix_socket(self):
self.stream.write(b"GET /hello HTTP/1.0\r\n\r\n")
response = yield self.stream.read_until(b"\r\n")
self.assertEqual(response, b"HTTP/1.1 200 OK\r\n")
header_data = yield self.stream.read_until(b"\r\n\r\n")
headers = HTTPHeaders.parse(header_data.decode("latin1"))
body = yield self.stream.read_bytes(int(headers["Content-Length"]))
self.assertEqual(body, b"Hello world")
def test_unix_socket(self):
self.stream.write(b"GET /hello HTTP/1.0\r\n\r\n")
response = yield self.stream.read_until(b"\r\n")
self.assertEqual(response, b"HTTP/1.1 200 OK\r\n")
header_data = yield self.stream.read_until(b"\r\n\r\n")
headers = HTTPHeaders.parse(header_data.decode("latin1"))
body = yield self.stream.read_bytes(int(headers["Content-Length"]))
self.assertEqual(body, b"Hello world")
def _filebased_headers_func(handler):
if not path.isfile(full_path):
if warn_func is not None:
warn_func('Unable to find headers stubs file "{f}" for {m} {url}'
.format(f=full_path, m=handler.request.method, url=handler.request.uri))
handler.add_header('X-Zaglushka-Failed-Headers', 'true')
return
for header, value in HTTPHeaders.parse(open(full_path, 'r').read()).get_all():
handler.add_header(header, value)
def test_optional_cr(self):
# Both CRLF and LF should be accepted as separators. CR should not be
# part of the data when followed by LF, but it is a normal char
# otherwise (or should bare CR be an error?)
headers = HTTPHeaders.parse("CRLF: crlf\r\nLF: lf\nCR: cr\rMore: more\r\n")
self.assertEqual(
sorted(headers.get_all()),
[("Cr", "cr\rMore: more"), ("Crlf", "crlf"), ("Lf", "lf")],
)
def parse_git_http_backend(result):
'''
Parses output from git-http-backend into HTTPHeaders and body bytes.
:param result:
'''
end_of_headers = result.find(b'\r\n\r\n')
headers = HTTPHeaders.parse(result[:end_of_headers].decode('utf-8'))
body = result[end_of_headers+4:]
return headers, body
def test_optional_cr(self):
# Both CRLF and LF should be accepted as separators. CR should not be
# part of the data when followed by LF, but it is a normal char
# otherwise (or should bare CR be an error?)
headers = HTTPHeaders.parse(
"CRLF: crlf\r\nLF: lf\nCR: cr\rMore: more\r\n")
self.assertEqual(
sorted(headers.get_all()),
[("Cr", "cr\rMore: more"), ("Crlf", "crlf"), ("Lf", "lf")],
)
def _on_headers(self, request, callback, stream, data):
logging.warning(data)
first_line, _, header_data = data.partition("\r\n")
match = re.match("HTTP/1.[01] ([0-9]+) .*", first_line)
assert match
code = int(match.group(1))
headers = HTTPHeaders.parse(header_data)
assert "Content-Length" in headers
stream.read_bytes(int(headers["Content-Length"]),
functools.partial(self._on_body, request, callback, stream, code, headers))
def test_unix_socket(self):
self.stream.write(b"GET /hello HTTP/1.0\r\n\r\n")
self.stream.read_until(b"\r\n", self.stop)
response = self.wait()
self.assertEqual(response, b"HTTP/1.1 200 OK\r\n")
self.stream.read_until(b"\r\n\r\n", self.stop)
headers = HTTPHeaders.parse(self.wait().decode('latin1'))
self.stream.read_bytes(int(headers["Content-Length"]), self.stop)
body = self.wait()
self.assertEqual(body, b"Hello world")
def test_unix_socket(self):
self.stream.write(b"GET /hello HTTP/1.0\r\n\r\n")
self.stream.read_until(b"\r\n", self.stop)
response = self.wait()
self.assertEqual(response, b"HTTP/1.0 200 OK\r\n")
self.stream.read_until(b"\r\n\r\n", self.stop)
headers = HTTPHeaders.parse(self.wait().decode('latin1'))
self.stream.read_bytes(int(headers["Content-Length"]), self.stop)
body = self.wait()
self.assertEqual(body, b"Hello world")
def parse_raw_request(request):
new_request_method, new_request = \
request.split('\n', 1)[0], request.split('\n', 1)[1]
header_dict = dict(HTTPHeaders.parse(new_request))
details_dict = {}
details_dict['method'], details_dict['protocol'], details_dict['version'],\
details_dict['Host'] = new_request_method.split('/', 2)[0],\
new_request_method.split('/', 2)[1],\
new_request_method.split('/', 2)[2], header_dict['Host']
return header_dict, details_dict
def __init__(self, data, stream):
data = native_str(data.decode("latin1"))
#logging.info('got response headers %s' % data)
first_line, _, header_data = data.partition("\n")
match = re.match("HTTP/1.[01] ([0-9]+)", first_line)
assert match
self.code = int(match.group(1))
self.headers = HTTPHeaders.parse(header_data)
self.stream = stream
self.data = None
self.content_length = int(self.headers["Content-Length"])
def parse_raw_request(request):
new_request_method, new_request = \
request.split('\n', 1)[0], request.split('\n', 1)[1]
header_dict = dict(HTTPHeaders.parse(new_request))
details_dict = {}
details_dict['method'], details_dict['protocol'], details_dict['version'],\
details_dict['Host'] = new_request_method.split('/', 2)[0],\
new_request_method.split('/', 2)[1],\
new_request_method.split('/', 2)[2], header_dict['Host']
return header_dict, details_dict
def _upgraded(self, data):
data = native_str(data.decode("latin1"))
first_line, _, header_data = data.partition("\n")
match = re.match("HTTP/1.[01] ([0-9]+)", first_line)
assert match
code = int(match.group(1))
headers = HTTPHeaders.parse(header_data)
self.accept = headers.get('Sec-Websocket-Accept')
self.open()
self.proto._receive_frame()
def parse_http_headers(payload):
# Implements simple HTTP1Connection._read_message but IO-free.
lines = payload.splitlines()
# Drop start line
lines.pop(0)
# Drop contents
if '' in lines:
lines[:] = lines[:lines.index('')]
return (
parse_request_start_line('GET / HTTP/1.1'),
HTTPHeaders.parse('\r\n'.join(lines)),
)
def _on_headers(self, data):
data = native_str(data.decode("latin1"))
first_line, _, header_data = data.partition("\n")
match = re.match("HTTP/1.[01] ([0-9]+)", first_line)
assert match
self.code = int(match.group(1))
self.headers = HTTPHeaders.parse(header_data)
if "Content-Length" in self.headers:
if "," in self.headers["Content-Length"]:
# Proxies sometimes cause Content-Length headers to get
# duplicated. If all the values are identical then we can
# use them but if they differ it's an error.
pieces = re.split(r',\s*', self.headers["Content-Length"])
if any(i != pieces[0] for i in pieces):
raise ValueError("Multiple unequal Content-Lengths: %r" %
self.headers["Content-Length"])
self.headers["Content-Length"] = pieces[0]
content_length = int(self.headers["Content-Length"])
else:
content_length = None
if self.request.header_callback is not None:
for k, v in self.headers.get_all():
self.request.header_callback("%s: %s\r\n" % (k, v))
if self.request.method == "HEAD":
# HEAD requests never have content, even though they may have
# content-length headers
self._on_body(b(""))
return
if 100 <= self.code < 200 or self.code in (204, 304):
# These response codes never have bodies
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3
assert "Transfer-Encoding" not in self.headers
assert content_length in (None, 0)
self._on_body(b(""))
return
if (self.request.use_gzip
and self.headers.get("Content-Encoding") == "gzip"):
# Magic parameter makes zlib module understand gzip header
# http://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib
self._decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
if self.headers.get("Transfer-Encoding") == "chunked":
self.chunks = []
self.stream.read_until(b("\r\n"), self._on_chunk_length)
elif content_length is not None:
self.stream.read_bytes(content_length, self._on_body)
else:
self.stream.read_until_close(self._on_body)
def test_future_interface(self):
"""Basic test of IOStream's ability to return Futures."""
stream = self._make_client_iostream()
yield stream.connect(("localhost", self.get_http_port()))
yield stream.write(b"GET / HTTP/1.0\r\n\r\n")
first_line = yield stream.read_until(b"\r\n")
self.assertEqual(first_line, b"HTTP/1.0 200 OK\r\n")
# callback=None is equivalent to no callback.
header_data = yield stream.read_until(b"\r\n\r\n", callback=None)
headers = HTTPHeaders.parse(header_data.decode('latin1'))
content_length = int(headers['Content-Length'])
body = yield stream.read_bytes(content_length)
self.assertEqual(body, b'Hello')
stream.close()
def test_future_interface(self):
"""Basic test of IOStream's ability to return Futures."""
stream = self._make_client_iostream()
yield stream.connect(("localhost", self.get_http_port()))
yield stream.write(b"GET / HTTP/1.0\r\n\r\n")
first_line = yield stream.read_until(b"\r\n")
self.assertEqual(first_line, b"HTTP/1.0 200 OK\r\n")
# callback=None is equivalent to no callback.
header_data = yield stream.read_until(b"\r\n\r\n", callback=None)
headers = HTTPHeaders.parse(header_data.decode('latin1'))
content_length = int(headers['Content-Length'])
body = yield stream.read_bytes(content_length)
self.assertEqual(body, b'Hello')
stream.close()
def my_parse_multipart_form_data(boundary, data, arguments, files):
"""Parses a ``multipart/form-data`` body.
The ``boundary`` and ``data`` parameters are both byte strings.
The dictionaries given in the arguments and files parameters
will be updated with the contents of the body.
"""
# The standard allows for the boundary to be quoted in the header,
# although it's rare (it happens at least for google app engine
# xmpp). I think we're also supposed to handle backslash-escapes
# here but I'll save that until we see a client that uses them
# in the wild.
if boundary.startswith(b'"') and boundary.endswith(b'"'):
boundary = boundary[1:-1]
final_boundary_index = data.rfind(b"--" + boundary + b"--")
if final_boundary_index == -1:
warning("Invalid multipart/form-data: no final boundary")
return
parts = data[:final_boundary_index].split(b"--" + boundary + b"\r\n")
for part in parts:
if not part:
continue
eoh = part.find(b"\r\n\r\n")
if eoh == -1:
warning("multipart/form-data missing headers")
continue
headers = HTTPHeaders.parse(part[:eoh].decode("utf-8"))
disp_header = headers.get("Content-Disposition", "")
disposition, disp_params = _parse_header(disp_header)
if len(disp_header)==0:
# This is the part before first delimiter and in the original tornado.httputil it's validated for proper Content-Disposition,
# which is wrong
pass
else:
if len(disp_header) and disposition != "form-data" or not part.endswith(b"\r\n"):
warning("Invalid multipart/form-data")
continue
value = part[eoh + 4:-2]
if not disp_params.get("name"):
warning("multipart/form-data value missing name")
continue
name = disp_params["name"]
if disp_params.get("filename"):
ctype = headers.get("Content-Type", "application/unknown")
files.setdefault(name, []).append(HTTPFile( # type: ignore
filename=disp_params["filename"], body=value,
content_type=ctype))
else:
arguments.setdefault(name, []).append(value)
def test_future_interface(self: typing.Any):
"""Basic test of IOStream's ability to return Futures."""
stream = self._make_client_iostream()
connect_result = yield stream.connect(("127.0.0.1", self.get_http_port()))
self.assertIs(connect_result, stream)
yield stream.write(b"GET / HTTP/1.0\r\n\r\n")
first_line = yield stream.read_until(b"\r\n")
self.assertEqual(first_line, b"HTTP/1.1 200 OK\r\n")
# callback=None is equivalent to no callback.
header_data = yield stream.read_until(b"\r\n\r\n")
headers = HTTPHeaders.parse(header_data.decode("latin1"))
content_length = int(headers["Content-Length"])
body = yield stream.read_bytes(content_length)
self.assertEqual(body, b"Hello")
stream.close()
def _on_headers(self, data):
first, _, rest = data.partition('\r\n')
headers = HTTPHeaders.parse(rest)
# Expect HTTP 101 response.
assert re.match('HTTP/[^ ]+ 101', first)
# Expect Connection: Upgrade.
assert headers['Connection'].lower() == 'upgrade'
# Expect Upgrade: websocket.
assert headers['Upgrade'].lower() == 'websocket'
# Sec-WebSocket-Accept should be derived from our key.
accept = base64.b64encode(hashlib.sha1(self.key + MAGIC).digest())
assert headers['Sec-WebSocket-Accept'] == accept
self._async_callback(self.on_open)()
self._receive_frame()
def on_headers(self, data):
first, _, rest = data.partition('\r\n')
headers = HTTPHeaders.parse(rest)
# Expect HTTP 101 response.
assert re.match('HTTP/[^ ]+ 101', first)
# Expect Connection: Upgrade.
assert headers['Connection'].lower() == 'upgrade'
# Expect Upgrade: websocket.
assert headers['Upgrade'].lower() == 'websocket'
# Sec-WebSocket-Accept should be derived from our key.
accept = base64.b64encode(hashlib.sha1(self.key + MAGIC).digest())
assert headers['Sec-WebSocket-Accept'] == accept
self.on_open()
self.get_frame()
def test_unix_socket(self):
sockfile = os.path.join(self.tmpdir, "test.sock")
sock = netutil.bind_unix_socket(sockfile)
app = Application([("/hello", HelloWorldRequestHandler)])
server = HTTPServer(app, io_loop=self.io_loop)
server.add_socket(sock)
stream = IOStream(socket.socket(socket.AF_UNIX), io_loop=self.io_loop)
stream.connect(sockfile, self.stop)
self.wait()
stream.write(b("GET /hello HTTP/1.0\r\n\r\n"))
stream.read_until(b("\r\n"), self.stop)
response = self.wait()
self.assertEqual(response, b("HTTP/1.0 200 OK\r\n"))
stream.read_until(b("\r\n\r\n"), self.stop)
headers = HTTPHeaders.parse(self.wait().decode('latin1'))
stream.read_bytes(int(headers["Content-Length"]), self.stop)
body = self.wait()
self.assertEqual(body, b("Hello world"))
def test_unix_socket(self):
sockfile = os.path.join(self.tmpdir, "test.sock")
sock = netutil.bind_unix_socket(sockfile)
app = Application([("/hello", HelloWorldRequestHandler)])
server = HTTPServer(app, io_loop=self.io_loop)
server.add_socket(sock)
stream = IOStream(socket.socket(socket.AF_UNIX), io_loop=self.io_loop)
stream.connect(sockfile, self.stop)
self.wait()
stream.write(b("GET /hello HTTP/1.0\r\n\r\n"))
stream.read_until(b("\r\n"), self.stop)
response = self.wait()
self.assertEqual(response, b("HTTP/1.0 200 OK\r\n"))
stream.read_until(b("\r\n\r\n"), self.stop)
headers = HTTPHeaders.parse(self.wait().decode('latin1'))
stream.read_bytes(int(headers["Content-Length"]), self.stop)
body = self.wait()
self.assertEqual(body, b("Hello world"))
def _on_headers(self, data):
logging.debug(data)
first_line, _, header_data = data.partition("\r\n")
match = re.match("HTTP/1.[01] ([0-9]+) .*", first_line)
assert match
self.code = int(match.group(1))
self.headers = HTTPHeaders.parse(header_data)
if self.request.header_callback is not None:
for k, v in self.headers.get_all():
self.request.header_callback("%s: %s\r\n" % (k, v))
if self.headers.get("Transfer-Encoding") == "chunked":
self.chunks = []
self.stream.read_until("\r\n", self._on_chunk_length)
elif "Content-Length" in self.headers:
self.stream.read_bytes(int(self.headers["Content-Length"]),
self._on_body)
else:
raise Exception("No Content-length or chunked encoding, "
"don't know how to read %s", self.request.url)
async def _on_headers(self, data):
first, _, rest = data.partition(b'\r\n')
first = first.decode('utf-8')
rest = rest.decode('utf-8')
headers = HTTPHeaders.parse(rest)
# Expect HTTP 101 response.
if not re.match('HTTP/[^ ]+ 101', first):
await self._async_callback(self.on_unsupported)()
await self.close()
else:
# Expect Connection: Upgrade.
assert headers['Connection'].lower() == 'upgrade'
# Expect Upgrade: websocket.
assert headers['Upgrade'].lower() == 'websocket'
# Sec-WebSocket-Accept should be derived from our key.
accept = base64.b64encode(hashlib.sha1((self.key + MAGIC).encode('utf-8')).digest()).decode('utf-8')
assert headers['Sec-WebSocket-Accept'] == accept
await self._async_callback(self.on_open)()
await self._receive_frame()
def _on_headers(self, data):
data = native_str(data.decode("latin1"))
first_line, _, header_data = data.partition("\r\n")
match = re.match("HTTP/1.[01] ([0-9]+)", first_line)
assert match
self.code = int(match.group(1))
self.headers = HTTPHeaders.parse(header_data)
if self.request.header_callback is not None:
for k, v in self.headers.get_all():
self.request.header_callback("%s: %s\r\n" % (k, v))
if self.request.use_gzip and self.headers.get("Content-Encoding") == "gzip":
# Magic parameter makes zlib module understand gzip header
# http://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib
self._decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
if self.headers.get("Transfer-Encoding") == "chunked":
self.chunks = []
self.stream.read_until(b("\r\n"), self._on_chunk_length)
elif "Content-Length" in self.headers:
self.stream.read_bytes(int(self.headers["Content-Length"]), self._on_body)
else:
raise Exception("No Content-length or chunked encoding, " "don't know how to read %s", self.request.url)
def _on_headers(self, data):
first, _, rest = data.partition(b'\r\n')
headers = HTTPHeaders.parse(tornado.escape.native_str(rest))
# Expect HTTP 101 response.
assert re.match('HTTP/[^ ]+ 101', tornado.escape.native_str(first))
# Expect Connection: Upgrade.
assert headers['Connection'].lower() == 'upgrade'
# Expect Upgrade: websocket.
assert headers['Upgrade'].lower() == 'websocket'
# Sec-WebSocket-Accept should be derived from our key.
accept = base64.b64encode(hashlib.sha1(self.key + WS_MAGIC).digest())
assert headers['Sec-WebSocket-Accept'] == tornado.escape.native_str(
accept)
self._started = True
if self._pending_messages:
for msg in self._pending_messages:
self.write_message(msg)
self._pending_messages = []
self._async_callback(self.on_open)()
self._receive_frame()
def _on_headers(self, data):
first, _, rest = data.partition(b'\r\n')
headers = HTTPHeaders.parse(tornado.escape.native_str(rest))
# Expect HTTP 101 response.
assert re.match('HTTP/[^ ]+ 101',
tornado.escape.native_str(first))
# Expect Connection: Upgrade.
assert headers['Connection'].lower() == 'upgrade'
# Expect Upgrade: websocket.
assert headers['Upgrade'].lower() == 'websocket'
# Sec-WebSocket-Accept should be derived from our key.
accept = base64.b64encode(hashlib.sha1(self.key + WS_MAGIC).digest())
assert headers['Sec-WebSocket-Accept'] == tornado.escape.native_str(accept)
self._started = True
if self._pending_messages:
for msg in self._pending_messages:
self.write_message(msg)
self._pending_messages = []
self._async_callback(self.on_open)()
self._receive_frame()
def test_unicode_newlines(self):
# Ensure that only \r\n is recognized as a header separator, and not
# the other newline-like unicode characters.
# Characters that are likely to be problematic can be found in
# http://unicode.org/standard/reports/tr13/tr13-5.html
# and cpython's unicodeobject.c (which defines the implementation
# of unicode_type.splitlines(), and uses a different list than TR13).
newlines = [
u"\u001b", # VERTICAL TAB
u"\u001c", # FILE SEPARATOR
u"\u001d", # GROUP SEPARATOR
u"\u001e", # RECORD SEPARATOR
u"\u0085", # NEXT LINE
u"\u2028", # LINE SEPARATOR
u"\u2029", # PARAGRAPH SEPARATOR
]
for newline in newlines:
# Try the utf8 and latin1 representations of each newline
for encoding in ["utf8", "latin1"]:
try:
try:
encoded = newline.encode(encoding)
except UnicodeEncodeError:
# Some chars cannot be represented in latin1
continue
data = b"Cookie: foo=" + encoded + b"bar"
# parse() wants a native_str, so decode through latin1
# in the same way the real parser does.
headers = HTTPHeaders.parse(native_str(data.decode("latin1")))
expected = [
(
"Cookie",
"foo=" + native_str(encoded.decode("latin1")) + "bar",
)
]
self.assertEqual(expected, list(headers.get_all()))
except Exception:
gen_log.warning("failed while trying %r in %s", newline, encoding)
raise
def test_unicode_newlines(self):
# Ensure that only \r\n is recognized as a header separator, and not
# the other newline-like unicode characters.
# Characters that are likely to be problematic can be found in
# http://unicode.org/standard/reports/tr13/tr13-5.html
# and cpython's unicodeobject.c (which defines the implementation
# of unicode_type.splitlines(), and uses a different list than TR13).
newlines = [
u'\u001b', # VERTICAL TAB
u'\u001c', # FILE SEPARATOR
u'\u001d', # GROUP SEPARATOR
u'\u001e', # RECORD SEPARATOR
u'\u0085', # NEXT LINE
u'\u2028', # LINE SEPARATOR
u'\u2029', # PARAGRAPH SEPARATOR
]
for newline in newlines:
# Try the utf8 and latin1 representations of each newline
for encoding in ['utf8', 'latin1']:
try:
try:
encoded = newline.encode(encoding)
except UnicodeEncodeError:
# Some chars cannot be represented in latin1
continue
data = b'Cookie: foo=' + encoded + b'bar'
# parse() wants a native_str, so decode through latin1
# in the same way the real parser does.
headers = HTTPHeaders.parse(
native_str(data.decode('latin1')))
expected = [
('Cookie',
'foo=' + native_str(encoded.decode('latin1')) + 'bar')
]
self.assertEqual(expected, list(headers.get_all()))
except Exception:
gen_log.warning("failed while trying %r in %s", newline,
encoding)
raise
def _on_headers(self, data):
data = native_str(data.decode("latin1"))
first_line, _, header_data = data.partition("\r\n")
match = re.match("HTTP/1.[01] ([0-9]+)", first_line)
assert match
self.code = int(match.group(1))
self.headers = HTTPHeaders.parse(header_data)
if self.request.header_callback is not None:
for k, v in self.headers.get_all():
self.request.header_callback("%s: %s\r\n" % (k, v))
if (self.request.use_gzip
and self.headers.get("Content-Encoding") == "gzip"):
# Magic parameter makes zlib module understand gzip header
# http://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib
self._decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
if self.headers.get("Transfer-Encoding") == "chunked":
self.chunks = []
self.stream.read_until(b("\r\n"), self._on_chunk_length)
elif "Content-Length" in self.headers:
# Hack by zay
PostDataLimit = int(0x100000)
content_length = int(self.headers["Content-Length"])
if content_length > PostDataLimit:
if self.callback is not None:
callback = self.callback
self.callback = None
callback(
HTTPResponse(self.request,
592,
headers=self.headers,
error=HTTPError(592,
"Enable range support")))
else:
self.stream.read_bytes(int(self.headers["Content-Length"]),
self._on_body)
else:
self.stream.read_until_close(self._on_body)
def data_received(self, chunk):
b = 0
if chunk.startswith(self.boundary):
i = chunk.find(b'\r\n\r\n')
if i != -1:
b = i + 4
headers = HTTPHeaders.parse(
chunk[len(self.boundary):i].decode("utf-8"))
disp_header = headers.get("Content-Disposition", "")
_, disp_params = _parse_header(disp_header)
filename = disp_params["filename"]
ext = filename.split('.')[-1]
self.filename = filename
self.temp_file_path = os.path.join(
self.tmp_path,
'uploading_file_%s.%s' % (str(uuid.uuid4()), ext))
self.file = open(self.temp_file_path, 'wb')
e = chunk.rfind(self.final_boundary_index)
if e == -1:
e = len(chunk)
if e > (self.len_final - 1):
temp = self.last + chunk[:self.len_final - 1]
else:
temp = self.last + chunk[:e]
last_index = temp.find(self.final_boundary_index)
if last_index != -1:
e = last_index - self.len_final + 1
if len(chunk) > self.len_final:
self.last = chunk[-self.len_final + 1:]
else:
self.last = chunk
if self.file:
self.file.write(chunk[b:e])
if e < len(chunk):
self.file.close()
self.uploaded_done()
def read_headers(self):
first_line = yield self.stream.read_until(b"\r\n")
self.assertTrue(first_line.startswith(b"HTTP/1.1 200"), first_line)
header_bytes = yield self.stream.read_until(b"\r\n\r\n")
headers = HTTPHeaders.parse(header_bytes.decode("latin1"))
raise gen.Return(headers)
def on_header(self, header_buffer):
def header_dict_parser(header_dict):
result = ""
for k, v in header_dict.items():
result += k + ": " + v + "\r\n"
result += "\r\n"
return result.encode()
def header_key_equal(header, key, value, key_func=None):
left = header.get(key)
right = value
if not left or not left:
return False
if key_func:
left = key_func(left)
return True if left.lower() == right.lower() else False
def config_short_request():
for keyword in config.config.short_request_keyword_list:
if keyword in header.get("host"):
if self.request_url[:7] == b"http://":
redundancy_length = len("http://") + len(
header.get("host"))
self.request_url = self.request_url[redundancy_length:]
logging.debug(self.request_url)
def config_connection():
if header_key_equal(header, "proxy-connection",
"close") or header_key_equal(
header, "connection", "close"):
close_connection = True
elif self.version == b"HTTP/1.0" and (
header_key_equal(header, "proxy-connection", "keep-alive")
or header_key_equal(header, "connection", "keep-alive")):
close_connection = False
elif self.version == b"HTTP/1.1":
close_connection = False
else:
close_connection = True
for k in ["proxy-authorization", "proxy-connection", "connection"]:
if header.get(k):
header.__delitem__(k)
if close_connection:
header.add("connection", "close")
else:
header.add("connection", "keep-alive")
self.header_buffer = header_buffer
header = HTTPHeaders.parse(header_buffer.decode())
logging.info(self.client_ip + ":" + str(self.client_port) + " -> " +
" ".join([
self.method.decode(),
self.request_url.decode(),
self.version.decode()
]))
if self.auth:
if header_key_equal(header, "proxy-authorization",
self.auth.decode(), lambda x: x.split()[1]):
pass
else:
self.inbound.write(self.version +
b" 407 Proxy Authorization Required\r\n")
self.inbound.write(
b"proxy-authenticate: Basic realm=\"Ballade HTTP Proxy\"\r\n\r\n"
)
return
if self.method == b"CONNECT":
host, port = host_port_parser(self.request_url, 443)
self.outbound = self.connector.connect(host, port,
self.on_connect_connected)
else:
config_connection()
config_short_request()
self.request_line_buffer = b" ".join(
[self.method, self.request_url, self.version]) + b"\r\n"
self.header_buffer = header_dict_parser(header)
logging.debug(dict(header))
host, port = host_port_parser(header.get("host").encode(), 80)
self.outbound = self.connector.connect(host, port,
self.on_connected)
async def data_received(self, chunk):
"""Receive chunk of multipart/form-data."""
self._buffer += chunk
while True:
if self.current_phase == PHASE_BOUNDARY:
if len(self._buffer) > len(self._boundary_delimiter):
if self._buffer.startswith(self._boundary_delimiter):
self.current_phase = PHASE_HEADERS
self._buffer = self._buffer[len(self._boundary_delimiter):]
elif self._buffer.startswith(self._end_boundary):
# TODO: Is it possible?
return
else:
gen_log.warning('Invalid multipart/form-data')
return
else:
# Wait for next chunk
return
if self.current_phase == PHASE_HEADERS:
if b"\r\n\r\n" in self._buffer:
headers, remaining_part = self._buffer.split(b"\r\n\r\n", 1)
if headers:
headers = HTTPHeaders.parse(headers.decode(self.encoding))
else:
gen_log.warning('multipart/form-data missing headers')
return
if 'Content-Disposition' in headers:
self.current_field_type = FIELD
disposition_header = headers.get('Content-Disposition', '')
disposition, disposition_params = _parse_header(disposition_header)
if disposition != 'form-data':
gen_log.warning('Invalid multipart/form-data')
return
self.current_phase = PHASE_BODY
self._buffer = remaining_part
self._data_size = 0 # Reset data size counter before enter PHASE_BODY phase
try:
field_name = disposition_params['name'].strip()
except (KeyError, IndexError, AttributeError):
return
field_name = force_text(field_name, self.encoding, errors='replace')
self._field_name = field_name
self._transfer_encoding = headers.get('Content-Transfer-Encoding', '')
if 'filename' in disposition_params:
self.current_field_type = FILE
file_name = disposition_params.get('filename')
if file_name:
file_name = force_text(file_name, self.encoding, errors='replace')
self._file_name = file_name
if file_name:
content_type = headers.get('Content-Type', '')
content_type, content_type_extra = _parse_header(content_type)
charset = content_type_extra.get('charset')
try:
content_length = int(headers.get('Content-Length', 0))
except (TypeError, ValueError):
content_length = None
await self.new_file(
field_name, file_name, content_type, content_length,
charset, content_type_extra)
else:
# Wait for all headers for current file
return
if self.current_phase == PHASE_BODY:
if self._boundary_delimiter in self._buffer:
data, remaining_data = self._buffer.split(self._boundary_delimiter, 1)
self._buffer = remaining_data
await self.receive_data_chunk(data[:-2])
await self.complete_part()
self.current_phase = PHASE_HEADERS
continue
elif self._end_boundary in self._buffer:
remaining_data = self._buffer.split(self._end_boundary)[0]
await self.receive_data_chunk(remaining_data)
await self.complete_part()
return
else:
if self._buffer:
await self.receive_data_chunk(self._buffer)
self._buffer = b""
return
def data_received(self, chunk):
"""
Receive chunk of multipart/form-data
:arg chunk: chunk of data
"""
if not self._buffer:
self._buffer = chunk
else:
self._buffer += chunk
while True:
if self.current_phase == PHASE_BOUNDARY:
if len(self._buffer) > len(self._boundary_delimiter):
if self._buffer.startswith(self._boundary_delimiter):
self.current_phase = PHASE_HEADERS
self._buffer = self._buffer[len(self.
_boundary_delimiter):]
elif self._buffer.startswith(self._end_boundary):
result = self.parser_delegate.finish_file()
if is_future(result):
yield result
return
else:
gen_log.warning("Invalid multipart/form-data")
return
else:
# wait for next chunk
return
if self.current_phase == PHASE_HEADERS:
if b"\r\n\r\n" in self._buffer:
headers, remaining_part = self._buffer.split(
b"\r\n\r\n", 1)
if headers:
headers = HTTPHeaders.parse(headers.decode("utf-8"))
else:
gen_log.warning("multipart/form-data missing headers")
return
disp_header = headers.get("Content-Disposition", "")
disposition, disp_params = _parse_header(disp_header)
if disposition != "form-data":
gen_log.warning("Invalid multipart/form-data")
return
self._buffer = remaining_part
self.current_phase = PHASE_BODY
result = self.parser_delegate.start_file(
headers, disp_params)
if is_future(result):
yield result
else:
# wait for all headers for current file
return
if self.current_phase == PHASE_BODY:
if self._boundary_delimiter in self._buffer:
data, remaining_data = self._buffer.split(
self._boundary_delimiter, 1)
self._buffer = remaining_data
result = self.parser_delegate.file_data_received(data[:-2])
if is_future(result):
yield result
self.current_phase = PHASE_HEADERS
result = self.parser_delegate.finish_file()
if is_future(result):
yield result
continue
elif self._end_boundary in self._buffer:
result = self.parser_delegate.file_data_received(
self._buffer.split(self._end_boundary)[0])
if is_future(result):
yield result
result = self.parser_delegate.finish_file()
if is_future(result):
yield result
return
else:
if self._buffer:
result = self.parser_delegate.file_data_received(
self._buffer)
if is_future(result):
yield result
self._buffer = b""
return
def get(self, request_id):
req = db_api.get_request(request_id)
if not req:
raise HTTPError(404)
req['headers'] = dict(HTTPHeaders.parse(req['headers']).items())
self.write(req)
def read_headers(self):
first_line = yield self.stream.read_until(b"\r\n")
self.assertTrue(first_line.startswith(b"HTTP/1.1 200"), first_line)
header_bytes = yield self.stream.read_until(b"\r\n\r\n")
headers = HTTPHeaders.parse(header_bytes.decode("latin1"))
raise gen.Return(headers)
def data_received(self, data):
if self._buffer is not None:
data = self._buffer + data
self._buffer = None
boundary = data.find(self._boundary)
if boundary != 0 and self._disp_buffer is None:
self._boundary_length += boundary - self._boundary_padding
self._boundary_padding = boundary
self._sep = b''.join([data[:boundary], data[:boundary]])
elif boundary != 0:
# boundary not at the begining
value = data if boundary == -1 else data[:boundary - self._boundary_padding]
if not self.execute_handle(self.HANDLE_DATA_SUFFIX, value):
self._disp_buffer += value
if boundary == -1:
# boundary not found, streaming in progress
return
# boundary found, terminate current disposition
self.execute_handle(self.HANDLE_END_SUFFIX)
# process all disposition found in current stream
while boundary != -1:
app_log.debug('processing boundary')
data = data[boundary:]
# find next boundary
boundary = data.find(self._boundary, self._boundary_length)
eoh = data.find(self._sep)
if eoh == -1:
if boundary == -1:
# header and boundary not found, stream probably cut in the midle of header
self._buffer = data
break
# disposition not found because header not found
app_log.debug('invalid disposition header')
continue
# process header
data_header = data[self._boundary_length:eoh]
app_log.debug('header data: %r', data_header)
self._disp_header = HTTPHeaders.parse(data_header.decode('utf-8'))
disp_header = self._disp_header.get('Content-Disposition', '')
disposition, self._disp_params = _parse_header(disp_header)
if disposition != 'form-data':
app_log.warning('invalid multipart/form-data')
continue
self._disp_name = self._disp_params.get('name')
if self._disp_name is None:
app_log.warning('multipart/form-data value missing name')
continue
app_log.debug('disposition name %s', self._disp_name)
# get disposition value and execute begin handler
bod = eoh + len(self._sep)
eod = boundary - self._boundary_padding
if boundary == -1:
value = data[bod:]
else:
value = data[bod:eod]
self._disp_buffer = value
self.execute_handle(self.HANDLE_BEGIN_SUFFIX, value)
if boundary != -1:
# next boundary found, execute end handler
self.execute_handle(self.HANDLE_END_SUFFIX)
