def _on_request_body(self, data):
self.reset_connection_timeout()
self._request.body = data
content_type = self._request.headers.get("Content-Type", "")
if self._request.method in ("POST", "PUT"):
if content_type.startswith("application/x-www-form-urlencoded"):
arguments = parse_qs_bytes(native_str(self._request.body))
for name, values in arguments.iteritems():
values = [v for v in values if v]
if values:
self._request.arguments.setdefault(name, []).extend(
values)
elif content_type.startswith("multipart/form-data"):
fields = content_type.split(";")
for field in fields:
k, sep, v = field.strip().partition("=")
if k == "boundary" and v:
httputil.parse_multipart_form_data(
utf8(v), data,
self._request.arguments,
self._request.files)
break
else:
logging.warning("Invalid multipart/form-data")
self.request_callback(self._request)
def test_special_filenames(self):
filenames = [
"a;b.txt",
'a"b.txt',
'a";b.txt',
'a;"b.txt',
'a";";.txt',
'a\\"b.txt',
"a\\b.txt",
]
for filename in filenames:
logging.debug("trying filename %r", filename)
str_data = """\
--1234
Content-Disposition: form-data; name="files"; filename="%s"
Foo
--1234--""" % filename.replace(
"\\", "\\\\"
).replace(
'"', '\\"'
)
data = utf8(str_data.replace("\n", "\r\n"))
args, files = form_data_args()
parse_multipart_form_data(b"1234", data, args, files)
file = files["files"][0]
self.assertEqual(file["filename"], filename)
self.assertEqual(file["body"], b"Foo")
def _on_request_body(self, data):
try:
self._request.body = data
content_type = self._request.headers.get("Content-Type", "")
if self._request.method in ("POST", "PUT"):
if content_type.startswith("application/x-www-form-urlencoded"):
arguments = parse_qs_bytes(native_str(self._request.body), keep_blank_values = True)
for name, values in arguments.iteritems():
values = [v for v in values if v is not None]
if values:
self._request.arguments.setdefault(name, []).extend(
values)
elif content_type.startswith("multipart/form-data"):
fields = content_type.split(";")
for field in fields:
k, sep, v = field.strip().partition("=")
if k == "boundary" and v:
httputil.parse_multipart_form_data(
utf8(v), data,
self._request.arguments,
self._request.files)
break
else:
logging.warning("Invalid multipart/form-data")
self.request_callback(self._request)
except BadRequestException, e:
logging.info("Malformed HTTP request from %s: %s",
self.address[0], e)
logging.info('Request:\n%s', data)
self.no_keep_alive = True
if not self.stream.closed():
self.stream.write("HTTP/1.1 400 Bad request\r\n\r\n", self._finish_request)
return
def test_special_filenames(self):
filenames = [
'a;b.txt',
'a"b.txt',
'a";b.txt',
'a;"b.txt',
'a";";.txt',
'a\\"b.txt',
'a\\b.txt',
]
for filename in filenames:
logging.debug("trying filename %r", filename)
data = """\
--1234
Content-Disposition: form-data; name="files"; filename="%s"
Foo
--1234--""" % filename.replace('\\', '\\\\').replace('"', '\\"')
data = utf8(data.replace("\n", "\r\n"))
args = {}
files = {}
parse_multipart_form_data(b"1234", data, args, files)
file = files["files"][0]
self.assertEqual(file["filename"], filename)
self.assertEqual(file["body"], b"Foo")
def post(self):
boundary = self.request.body.split(b"\r\n")[0][2:]
parse_multipart_form_data(
boundary=utf8(boundary),
data=self.request.body,
arguments=self.request.arguments,
files=self.request.files,
)
if "file" in self.request.files:
file = self.request.files["file"][0]
uploaded_dataset = create_dataset(file=file["body"],
filename=file["filename"])
local_file = create_dataset_locally(
file=file["body"],
filename=uploaded_dataset["filename"],
name=uploaded_dataset["name"])
if local_file:
self.set_status(200)
self.set_header("Content-Type", "application/json")
data = json.dumps(local_file, default=date_default)
self.finish(data)
else:
self.set_status(500)
self.finish(u"Unable to create a dataset from file")
else:
self.set_status(400)
self.finish(u"No file in request")
def _parse_distutils_message(self):
# http://groups.google.com/group/python-tornado/browse_thread/thread/d0531e331c189c56?pli=1
httputil.parse_multipart_form_data(
self._boundary(),
self.request.body.replace("\n", "\r\n"),
self.request.arguments,
self.request.files
)
def __init__(self, environ):
"""Parses the given WSGI environ to construct the request."""
self.method = environ["REQUEST_METHOD"]
self.path = urllib.quote(environ.get("SCRIPT_NAME", ""))
self.path += urllib.quote(environ.get("PATH_INFO", ""))
self.uri = self.path
self.arguments = {}
self.query = environ.get("QUERY_STRING", "")
if self.query:
self.uri += "?" + self.query
arguments = parse_qs_bytes(native_str(self.query))
for name, values in arguments.iteritems():
values = [v for v in values if v]
if values:
self.arguments[name] = values
self.version = "HTTP/1.1"
self.headers = httputil.HTTPHeaders()
if environ.get("CONTENT_TYPE"):
self.headers["Content-Type"] = environ["CONTENT_TYPE"]
if environ.get("CONTENT_LENGTH"):
self.headers["Content-Length"] = environ["CONTENT_LENGTH"]
for key in environ:
if key.startswith("HTTP_"):
self.headers[key[5:].replace("_", "-")] = environ[key]
if self.headers.get("Content-Length"):
self.body = environ["wsgi.input"].read(
int(self.headers["Content-Length"]))
else:
self.body = ""
self.protocol = environ["wsgi.url_scheme"]
self.remote_ip = environ.get("REMOTE_ADDR", "")
if environ.get("HTTP_HOST"):
self.host = environ["HTTP_HOST"]
else:
self.host = environ["SERVER_NAME"]
# Parse request body
self.files = {}
content_type = self.headers.get("Content-Type", "")
if content_type.startswith("application/x-www-form-urlencoded"):
for name, values in parse_qs_bytes(native_str(
self.body)).iteritems():
self.arguments.setdefault(name, []).extend(values)
elif content_type.startswith("multipart/form-data"):
if 'boundary=' in content_type:
boundary = content_type.split('boundary=', 1)[1]
if boundary:
httputil.parse_multipart_form_data(utf8(boundary),
self.body,
self.arguments,
self.files)
else:
logging.warning("Invalid multipart/form-data")
self._start_time = monotime()
self._finish_time = None
def test_missing_headers(self):
data = b'''\
--1234
Foo
--1234--'''.replace(b"\n", b"\r\n")
args, files = form_data_args()
with ExpectLog(gen_log, "multipart/form-data missing headers"):
parse_multipart_form_data(b"1234", data, args, files)
self.assertEqual(files, {})
def test_line_does_not_end_with_correct_line_break(self):
data = b"""\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"
Foo--1234--""".replace(b"\n", b"\r\n")
args, files = form_data_args()
with ExpectLog(gen_log, "Invalid multipart/form-data"):
parse_multipart_form_data(b"1234", data, args, files)
self.assertEqual(files, {})
def test_missing_headers(self):
data = b"""\
--1234
Foo
--1234--""".replace(b"\n", b"\r\n")
args, files = form_data_args()
with ExpectLog(gen_log, "multipart/form-data missing headers"):
parse_multipart_form_data(b"1234", data, args, files)
self.assertEqual(files, {})
def test_missing_headers(self):
data = b('''\
--1234
Foo
--1234--''').replace(b("\n"), b("\r\n"))
args = {}
files = {}
parse_multipart_form_data(b("1234"), data, args, files)
self.assertEqual(files, {})
def test_line_does_not_end_with_correct_line_break(self):
data = b('''\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"
Foo--1234--''').replace(b("\n"), b("\r\n"))
args = {}
files = {}
parse_multipart_form_data(b("1234"), data, args, files)
self.assertEqual(files, {})
def test_missing_headers(self):
data = b('''\
--1234
Foo
--1234--''').replace(b("\n"), b("\r\n"))
args = {}
files = {}
parse_multipart_form_data(b("1234"), data, args, files)
self.assertEqual(files, {})
def test_line_does_not_end_with_correct_line_break(self):
data = b('''\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"
Foo--1234--''').replace(b("\n"), b("\r\n"))
args = {}
files = {}
parse_multipart_form_data(b("1234"), data, args, files)
self.assertEqual(files, {})
def test_line_does_not_end_with_correct_line_break(self):
data = b'''\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"
Foo--1234--'''.replace(b"\n", b"\r\n")
args, files = form_data_args()
with ExpectLog(gen_log, "Invalid multipart/form-data"):
parse_multipart_form_data(b"1234", data, args, files)
self.assertEqual(files, {})
def test_header_has_content_disposition_but_nothing_more(self):
data = b("""\
--1234
Content-Disposition: form-data;
Foo
--1234--""").replace(b("\n"), b("\r\n"))
args = {}
files = {}
parse_multipart_form_data(b("1234"), data, args, files)
self.assertEqual(files, {})
def test_missing_headers(self):
data = b('''\
--1234
Foo
--1234--''').replace(b("\n"), b("\r\n"))
args = {}
files = {}
with ExpectLog(gen_log, "multipart/form-data missing headers"):
parse_multipart_form_data(b("1234"), data, args, files)
self.assertEqual(files, {})
def test_content_disposition_header_without_name_parameter(self):
data = b"""\
--1234
Content-Disposition: form-data; filename="ab.txt"
Foo
--1234--""".replace(b"\n", b"\r\n")
args, files = form_data_args()
with ExpectLog(gen_log, "multipart/form-data value missing name"):
parse_multipart_form_data(b"1234", data, args, files)
self.assertEqual(files, {})
def test_content_disposition_header_without_name_parameter(self):
data = b("""\
--1234
Content-Disposition: form-data; filename="ab.txt"
Foo
--1234--""").replace(b("\n"), b("\r\n"))
args = {}
files = {}
parse_multipart_form_data(b("1234"), data, args, files)
self.assertEqual(files, {})
def test_invalid_content_disposition(self):
data = b('''\
--1234
Content-Disposition: invalid; name="files"; filename="ab.txt"
Foo
--1234--''').replace(b("\n"), b("\r\n"))
args = {}
files = {}
parse_multipart_form_data(b("1234"), data, args, files)
self.assertEqual(files, {})
def test_invalid_content_disposition(self):
data = b'''\
--1234
Content-Disposition: invalid; name="files"; filename="ab.txt"
Foo
--1234--'''.replace(b"\n", b"\r\n")
args, files = form_data_args()
with ExpectLog(gen_log, "Invalid multipart/form-data"):
parse_multipart_form_data(b"1234", data, args, files)
self.assertEqual(files, {})
def test_missing_headers(self):
data = b('''\
--1234
Foo
--1234--''').replace(b("\n"), b("\r\n"))
args = {}
files = {}
with ExpectLog(gen_log, "multipart/form-data missing headers"):
parse_multipart_form_data(b("1234"), data, args, files)
self.assertEqual(files, {})
def test_invalid_content_disposition(self):
data = b('''\
--1234
Content-Disposition: invalid; name="files"; filename="ab.txt"
Foo
--1234--''').replace(b("\n"), b("\r\n"))
args = {}
files = {}
parse_multipart_form_data(b("1234"), data, args, files)
self.assertEqual(files, {})
def test_content_disposition_header_without_name_parameter(self):
data = b"""\
--1234
Content-Disposition: form-data; filename="ab.txt"
Foo
--1234--""".replace(b"\n", b"\r\n")
args, files = form_data_args()
with ExpectLog(gen_log, "multipart/form-data value missing name"):
parse_multipart_form_data(b"1234", data, args, files)
self.assertEqual(files, {})
def test_content_disposition_header_without_name_parameter(self):
data = b("""\
--1234
Content-Disposition: form-data; filename="ab.txt"
Foo
--1234--""").replace(b("\n"), b("\r\n"))
args = {}
files = {}
parse_multipart_form_data(b("1234"), data, args, files)
self.assertEqual(files, {})
def test_invalid_content_disposition(self):
data = b"""\
--1234
Content-Disposition: invalid; name="files"; filename="ab.txt"
Foo
--1234--""".replace(b"\n", b"\r\n")
args, files = form_data_args()
with ExpectLog(gen_log, "Invalid multipart/form-data"):
parse_multipart_form_data(b"1234", data, args, files)
self.assertEqual(files, {})
def test_file_upload(self):
data = b"""\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"
Foo
--1234--""".replace(b"\n", b"\r\n")
args, files = form_data_args()
parse_multipart_form_data(b"1234", data, args, files)
file = files["files"][0]
self.assertEqual(file["filename"], "ab.txt")
self.assertEqual(file["body"], b"Foo")
def __init__(self, environ):
"""Parses the given WSGI environ to construct the request."""
self.method = environ["REQUEST_METHOD"]
self.path = urllib.quote(from_wsgi_str(environ.get("SCRIPT_NAME", "")))
self.path += urllib.quote(from_wsgi_str(environ.get("PATH_INFO", "")))
self.uri = self.path
self.arguments = {}
self.query = environ.get("QUERY_STRING", "")
if self.query:
self.uri += "?" + self.query
arguments = parse_qs_bytes(native_str(self.query))
for name, values in arguments.iteritems():
values = [v for v in values if v]
if values:
self.arguments[name] = values
self.version = "HTTP/1.1"
self.headers = httputil.HTTPHeaders()
if environ.get("CONTENT_TYPE"):
self.headers["Content-Type"] = environ["CONTENT_TYPE"]
if environ.get("CONTENT_LENGTH"):
self.headers["Content-Length"] = environ["CONTENT_LENGTH"]
for key in environ:
if key.startswith("HTTP_"):
self.headers[key[5:].replace("_", "-")] = environ[key]
if self.headers.get("Content-Length"):
self.body = environ["wsgi.input"].read(
int(self.headers["Content-Length"]))
else:
self.body = ""
self.protocol = environ["wsgi.url_scheme"]
self.remote_ip = environ.get("REMOTE_ADDR", "")
if environ.get("HTTP_HOST"):
self.host = environ["HTTP_HOST"]
else:
self.host = environ["SERVER_NAME"]
# Parse request body
self.files = {}
content_type = self.headers.get("Content-Type", "")
if content_type.startswith("application/x-www-form-urlencoded"):
for name, values in parse_qs_bytes(native_str(self.body)).iteritems():
self.arguments.setdefault(name, []).extend(values)
elif content_type.startswith("multipart/form-data"):
if 'boundary=' in content_type:
boundary = content_type.split('boundary=', 1)[1]
if boundary:
httputil.parse_multipart_form_data(
utf8(boundary), self.body, self.arguments, self.files)
else:
logging.warning("Invalid multipart/form-data")
self._start_time = time.time()
self._finish_time = None
def test_boundary_starts_and_ends_with_quotes(self):
data = b'''\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"
Foo
--1234--'''.replace(b"\n", b"\r\n")
args, files = form_data_args()
parse_multipart_form_data(b'"1234"', data, args, files)
file = files["files"][0]
self.assertEqual(file["filename"], "ab.txt")
self.assertEqual(file["body"], b"Foo")
def test_non_ascii_filename(self):
data = b"""\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"; filename*=UTF-8''%C3%A1b.txt
Foo
--1234--""".replace(b"\n", b"\r\n")
args, files = form_data_args()
parse_multipart_form_data(b"1234", data, args, files)
file = files["files"][0]
self.assertEqual(file["filename"], u"áb.txt")
self.assertEqual(file["body"], b"Foo")
def test_boundary_starts_and_ends_with_quotes(self):
data = b"""\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"
Foo
--1234--""".replace(b"\n", b"\r\n")
args, files = form_data_args()
parse_multipart_form_data(b'"1234"', data, args, files)
file = files["files"][0]
self.assertEqual(file["filename"], "ab.txt")
self.assertEqual(file["body"], b"Foo")
def test_unquoted_names(self):
# quotes are optional unless special characters are present
data = b"""\
--1234
Content-Disposition: form-data; name=files; filename=ab.txt
Foo
--1234--""".replace(b"\n", b"\r\n")
args, files = form_data_args()
parse_multipart_form_data(b"1234", data, args, files)
file = files["files"][0]
self.assertEqual(file["filename"], "ab.txt")
self.assertEqual(file["body"], b"Foo")
def test_file_upload(self):
data = b"""\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"
Foo
--1234--""".replace(b"\n", b"\r\n")
args = {}
files = {}
parse_multipart_form_data(b"1234", data, args, files)
file = files["files"][0]
self.assertEqual(file["filename"], "ab.txt")
self.assertEqual(file["body"], b"Foo")
def test_unquoted_names(self):
# quotes are optional unless special characters are present
data = b"""\
--1234
Content-Disposition: form-data; name=files; filename=ab.txt
Foo
--1234--""".replace(b"\n", b"\r\n")
args, files = form_data_args()
parse_multipart_form_data(b"1234", data, args, files)
file = files["files"][0]
self.assertEqual(file["filename"], "ab.txt")
self.assertEqual(file["body"], b"Foo")
def test_boundary_starts_and_ends_with_quotes(self):
data = b('''\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"
Foo
--1234--''').replace(b("\n"), b("\r\n"))
args = {}
files = {}
parse_multipart_form_data(b('"1234"'), data, args, files)
file = files["files"][0]
self.assertEqual(file["filename"], "ab.txt")
self.assertEqual(file["body"], b("Foo"))
def test_non_ascii_filename(self):
data = b"""\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"; filename*=UTF-8''%C3%A1b.txt
Foo
--1234--""".replace(b"\n", b"\r\n")
args = {}
files = {}
parse_multipart_form_data(b"1234", data, args, files)
file = files["files"][0]
self.assertEqual(file["filename"], u"áb.txt")
self.assertEqual(file["body"], b"Foo")
def test_invalid_content_disposition(self):
data = b(
"""\
--1234
Content-Disposition: invalid; name="files"; filename="ab.txt"
Foo
--1234--"""
).replace(b("\n"), b("\r\n"))
args = {}
files = {}
with ExpectLog(gen_log, "Invalid multipart/form-data"):
parse_multipart_form_data(b("1234"), data, args, files)
self.assertEqual(files, {})
def test_boundary_starts_and_ends_with_quotes(self):
data = b(
"""\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"
Foo
--1234--"""
).replace(b("\n"), b("\r\n"))
args = {}
files = {}
parse_multipart_form_data(b('"1234"'), data, args, files)
file = files["files"][0]
self.assertEqual(file["filename"], "ab.txt")
self.assertEqual(file["body"], b("Foo"))
def test_data_after_final_boundary(self):
# The spec requires that data after the final boundary be ignored.
# http://www.w3.org/Protocols/rfc1341/7_2_Multipart.html
# In practice, some libraries include an extra CRLF after the boundary.
data = b"""\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"
Foo
--1234--
""".replace(b"\n", b"\r\n")
args, files = form_data_args()
parse_multipart_form_data(b"1234", data, args, files)
file = files["files"][0]
self.assertEqual(file["filename"], "ab.txt")
self.assertEqual(file["body"], b"Foo")
def test_data_after_final_boundary(self):
# The spec requires that data after the final boundary be ignored.
# http://www.w3.org/Protocols/rfc1341/7_2_Multipart.html
# In practice, some libraries include an extra CRLF after the boundary.
data = b"""\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"
Foo
--1234--
""".replace(b"\n", b"\r\n")
args, files = form_data_args()
parse_multipart_form_data(b"1234", data, args, files)
file = files["files"][0]
self.assertEqual(file["filename"], "ab.txt")
self.assertEqual(file["body"], b"Foo")
def post(self):
boundary = self.request.headers['Content-Type'].split('boundary=',
1)[1]
args = {}
files = {}
data = parse_multipart_form_data(boundary, self.tmp_buffer, args,
files)
f = open('uploads/%s' % files['to_upload'][0]['filename'], 'wb+')
f.write(files['to_upload'][0]['body'])
f.close()
# curpos = 0
# chunk = 1024
# file = self.request.files['to_upload'][0]['body']
# length = len(file)
# REG['length'] = length
# f = open(self.request.files['to_upload'][0]['filename'], 'wb+')
# while curpos < length:
# part = file[curpos:min(curpos+chunk, length)]
# f.write(part)
# curpos += chunk
# REG['writed'] = curpos
#
# # DOWNLOAD['writed'] = 0
self.write('Done!')
self.finish()
def post(self, *args, **kwargs):
# 1. close file
file_size = util.human_data_size(self.writed_size)
logger.info(f"Uploaded file finished at {self.temporary_file_path}, file size {file_size} .")
if self.temporary_file is not None:
self.temporary_file.flush()
self.temporary_file.close()
content_type = self.request.headers['Content-Type']
with open(self.temporary_file_path, 'rb') as f:
fields = content_type.split(";")
for field in fields:
k, sep, v = field.strip().partition("=")
if k == "boundary" and v:
from tornado.escape import utf8
files = {}
httputil.parse_multipart_form_data(utf8(v), f.read(), {}, files)
handle_tornado_upload_file(self, files, self.start_time)
return
raise Exception("Handle upload failed.")
def _on_request_body(self, data):
self._request.body = data
content_type = self._request.headers.get("Content-Type", "")
if self._request.method in ("POST", "PATCH", "PUT"):
if content_type.startswith("application/x-www-form-urlencoded"):
arguments = parse_qs_bytes(native_str(self._request.body))
for name, values in arguments.iteritems():
values = [v for v in values if v]
if values:
self._request.arguments.setdefault(name,
[]).extend(values)
elif content_type.startswith("multipart/form-data"):
fields = content_type.split(";")
for field in fields:
k, sep, v = field.strip().partition("=")
if k == "boundary" and v:
httputil.parse_multipart_form_data(
utf8(v), data, self._request.arguments,
self._request.files)
break
else:
logging.warning("Invalid multipart/form-data")
self.request_callback(self._request)
def test_special_filenames(self):
filenames = ['a;b.txt',
'a"b.txt',
'a";b.txt',
'a;"b.txt',
'a";";.txt',
'a\\"b.txt',
'a\\b.txt',
]
for filename in filenames:
logging.info("trying filename %r", filename)
data = """\
--1234
Content-Disposition: form-data; name="files"; filename="%s"
Foo
--1234--""" % filename.replace('\\', '\\\\').replace('"', '\\"')
data = utf8(data.replace("\n", "\r\n"))
args = {}
files = {}
parse_multipart_form_data(b("1234"), data, args, files)
file = files["files"][0]
self.assertEqual(file["filename"], filename)
self.assertEqual(file["body"], b("Foo"))
def post(self):
boundary = self.request.headers["Content-Type"].split("boundary=", 1)[1]
args = {}
files = {}
data = parse_multipart_form_data(boundary, self.tmp_buffer, args, files)
f = open("uploads/%s" % files["to_upload"][0]["filename"], "wb+")
f.write(files["to_upload"][0]["body"])
f.close()
# curpos = 0
# chunk = 1024
# file = self.request.files['to_upload'][0]['body']
# length = len(file)
# REG['length'] = length
# f = open(self.request.files['to_upload'][0]['filename'], 'wb+')
# while curpos < length:
# part = file[curpos:min(curpos+chunk, length)]
# f.write(part)
# curpos += chunk
# REG['writed'] = curpos
#
# # DOWNLOAD['writed'] = 0
self.write("Done!")
self.finish()
def _parse_distutils_message(self):
# http://groups.google.com/group/python-tornado/browse_thread/thread/d0531e331c189c56?pli=1
httputil.parse_multipart_form_data(
self._boundary(), self.request.body.replace("\n", "\r\n"),
self.request.arguments, self.request.files)
