def request(self, method, uri, *args, **kwargs):
params = kwargs.pop("params")
if params:
uri = uri + "?" + urllib.urlencode(params)
connection = kwargs.pop('connection')
if not connection:
connection = HTTPConnection()
connection.xheaders = False
kwargs['connection'] = connection
connection.factory = self.app
request = HTTPRequest(method, uri, *args, **kwargs)
connection.connectionMade()
connection._request = request
connection.transport = proto_helpers.StringTransport()
request.remote_ip = connection.transport.getHost().host
handler = self.app(request)
def setup_response():
response_body = connection.transport.io.getvalue()
handler.content = response_body.split("\r\n\r\n", 1)[1]
handler.headers = handler._headers
if handler._finished:
setup_response()
returnValue(handler)
yield connection.notifyFinish()
setup_response()
returnValue(handler)
def request(self, method, path, body=None):
app = Application()
connection = MockConnection()
req = HTTPRequest(method='GET',
path='/ping',
remote_ip=None,
protocol="http",
connection=connection)
req.body = body
req.headers = headers or {}
application(req)
yield connection.notifyFinish()
(body, status) = self.parse_response(connection.buffer)
defer.returnValue((body, status))
def test_init_with_invalid_protocol_xheaders(self):
connection = Mock()
connection.xheaders = True
protocol = "ftp"
req = HTTPRequest(
"GET", "/something", connection=connection, protocol=protocol)
self.assertEqual(req.protocol, "http")
def setUp(self):
self.secure_application = SecureApplication([])
self.request = HTTPRequest('GET', 'http://localhost/foobar.txt')
super(SecureApplicationTest, self).setUp()
valid_caller = Caller(username='******', token='groot?')
DBSession.SESSIONMAKER().query(Caller).filter_by(
username='******').first.return_value = valid_caller
def test_init_with_https(self):
connection = Mock()
connection.xheaders = False
connection.transport = StringTransport()
interfaces.ISSLTransport.providedBy = lambda x: True
req = HTTPRequest("GET", "/something", connection=connection)
self.assertEqual(req.protocol, "https")
def request(self, method, uri, *args, **kwargs):
params = kwargs.pop("params", {}) or {}
if method in ["GET", "HEAD", "OPTIONS"] and params:
uri = uri + "?" + urllib.urlencode(params)
elif method in ["POST", "PATCH", "PUT"]\
and params and not kwargs['body']:
kwargs['body'] = urllib.urlencode(params)
connection = kwargs.pop('connection')
if not connection:
connection = HTTPConnection()
connection.xheaders = False
kwargs['connection'] = connection
connection.factory = self.app
cookie_value = self.cookies.output(header="")
if cookie_value.strip():
if kwargs['headers'] is None:
kwargs['headers'] = {}
kwargs['headers']['Cookie'] = cookie_value.strip()
request = HTTPRequest(method, uri, *args, **kwargs)
for k, p in params.items():
request.arguments.setdefault(k, []).append(p)
connection.connectionMade()
connection._request = request
connection.transport = proto_helpers.StringTransport()
request.remote_ip = connection.transport.getHost().host
handler = self.app(request)
def setup_response():
headers = HTTPHeaders()
for line in handler._generate_headers().split("\r\n"):
if line.startswith("HTTP") or not line.strip():
continue
headers.parse_line(line)
for cookie in headers.get_list("Set-Cookie"):
self.cookies.load(cookie)
response_body = connection.transport.io.getvalue()
handler.content = response_body.split("\r\n\r\n", 1)[1]
handler.headers = headers
if handler._finished:
setup_response()
returnValue(handler)
yield connection.notifyFinish()
setup_response()
returnValue(handler)
def request(self, method, uri, *args, **kwargs):
params = kwargs.pop("params", {}) or {}
if method in ["GET", "HEAD", "OPTIONS"] and params:
uri = uri + "?" + urllib.urlencode(params)
elif method in ["POST", "PATCH", "PUT"]\
and params and not kwargs['body']:
kwargs['body'] = urllib.urlencode(params)
connection = kwargs.pop('connection')
if not connection:
connection = HTTPConnection()
connection.xheaders = False
kwargs['connection'] = connection
connection.factory = self.app
cookie_value = self.cookies.output(header="")
if cookie_value.strip():
if kwargs['headers'] is None:
kwargs['headers'] = {}
kwargs['headers']['Cookie'] = cookie_value.strip()
request = HTTPRequest(method, uri, *args, **kwargs)
for k, p in params.items():
request.arguments.setdefault(k, []).append(p)
connection.connectionMade()
connection._request = request
connection.transport = proto_helpers.StringTransport()
request.remote_ip = connection.transport.getHost().host
handler = self.app(request)
def setup_response():
headers = HTTPHeaders()
for line in handler._generate_headers().split("\r\n"):
if line.startswith("HTTP") or not line.strip():
continue
headers.parse_line(line)
for cookie in headers.get_list("Set-Cookie"):
self.cookies.load(cookie)
response_body = connection.transport.io.getvalue()
handler.content = response_body.split("\r\n\r\n", 1)[1]
handler.headers = headers
if handler._finished:
setup_response()
returnValue(handler)
yield connection.notifyFinish()
setup_response()
returnValue(handler)
def test_init_with_invalid_connection_xheaders(self):
connection = Mock()
connection.xheaders = True
headers = {"X-Real-Ip": "256.0.0.1"}
req = HTTPRequest("GET",
"/something",
headers=headers,
connection=connection)
self.assertEqual(req.remote_ip, None)
self.assertEqual(req.protocol, "http")
def setUp(self):
self.app = Application()
self.request = HTTPRequest("GET", "/", connection=Mock())
self.request.headers["Accept"] = "application/json; v=v1"
self.resource_class = Mock()
self.handler = RESTHandler(
self.app, self.request,
resource_set=ResourceSet("test", versions={"v1": self.resource_class}))
self.handler._transforms = []
self.handler._request_summary = lambda: None
self.app.log_request = lambda x: None
def _on_headers(self, data):
try:
data = native_str(data.decode("latin1"))
eol = data.find("\r\n")
start_line = data[:eol]
try:
method, uri, version = start_line.split(" ")
except ValueError:
raise _BadRequestException("Malformed HTTP request line")
if not version.startswith("HTTP/"):
raise _BadRequestException(
"Malformed HTTP version in HTTP Request-Line")
try:
headers = httputil.HTTPHeaders.parse(data[eol:])
content_length = int(headers.get("Content-Length", 0))
except ValueError:
raise _BadRequestException("Malformed Content-Length header")
self._request = HTTPRequest(connection=self,
method=method,
uri=uri,
version=version,
headers=headers,
remote_ip=self._remote_ip)
if content_length:
megabytes = int(content_length) / (1024 * 1024)
if megabytes > GLSettings.memory_copy.maximum_filesize:
raise _BadRequestException(
"Request exceeded size limit %d" %
GLSettings.memory_copy.maximum_filesize)
if headers.get("Expect") == "100-continue":
self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")
if content_length < 100000:
self._contentbuffer = StringIO()
else:
self._contentbuffer = GLSecureTemporaryFile(
GLSettings.tmp_upload_path)
self.content_length = content_length
self.setRawMode()
return
self.request_callback(self._request)
except _BadRequestException as e:
log.msg("Exception while handling HTTP request from %s: %s" %
(self._remote_ip, e))
self.transport.loseConnection()
def test_init(self):
req = HTTPRequest("GET", "/something")
self.assertTrue(hasattr(req, "method"))
self.assertTrue(hasattr(req, "uri"))
self.assertTrue(hasattr(req, "version"))
self.assertTrue(hasattr(req, "headers"))
self.assertTrue(hasattr(req, "body"))
self.assertTrue(hasattr(req, "host"))
self.assertTrue(hasattr(req, "files"))
self.assertTrue(hasattr(req, "connection"))
self.assertTrue(hasattr(req, "path"))
self.assertTrue(hasattr(req, "arguments"))
self.assertTrue(hasattr(req, "remote_ip"))
def _make_dummy_request(self, method="GET", uri="/", **kwargs):
from cyclone.httpserver import HTTPRequest
req = HTTPRequest(method, uri, **kwargs)
req.connection = Mock()
return req
class GLHTTPConnection(HTTPConnection):
file_upload = False
def __init__(self):
self.uploaded_file = {}
def rawDataReceived(self, data):
if self.content_length is not None:
data, rest = data[:self.content_length], data[self.content_length:]
self.content_length -= len(data)
else:
rest = ''
self._contentbuffer.write(data)
if self.content_length == 0 and self._contentbuffer is not None:
tmpbuf = self._contentbuffer
self.content_length = self._contentbuffer = None
self.setLineMode(rest)
tmpbuf.seek(0, 0)
if self.file_upload:
self._on_request_body(self.uploaded_file)
self.file_upload = False
self.uploaded_file = {}
else:
self._on_request_body(tmpbuf.read())
def _on_headers(self, data):
try:
data = native_str(data.decode("latin1"))
eol = data.find("\r\n")
start_line = data[:eol]
try:
method, uri, version = start_line.split(" ")
except ValueError:
raise _BadRequestException("Malformed HTTP request line")
if not version.startswith("HTTP/"):
raise _BadRequestException(
"Malformed HTTP version in HTTP Request-Line")
headers = httputil.HTTPHeaders.parse(data[eol:])
self._request = HTTPRequest(
connection=self, method=method, uri=uri, version=version,
headers=headers, remote_ip=self._remote_ip)
try:
self.content_length = int(headers.get("Content-Length", 0))
except ValueError:
raise _BadRequestException("Malformed Content-Length header")
# we always use secure temporary files in case of large json or file uploads
if self.content_length < 100000 and self._request.headers.get("Content-Disposition") is None:
self._contentbuffer = StringIO('')
else:
self._contentbuffer = GLSecureTemporaryFile(GLSetting.tmp_upload_path)
if headers.get("Expect") == "100-continue":
self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")
c_d_header = self._request.headers.get("Content-Disposition")
if c_d_header is not None:
key, pdict = parse_header(c_d_header)
if key != 'attachment' or 'filename' not in pdict:
raise _BadRequestException("Malformed Content-Disposition header")
self.file_upload = True
self.uploaded_file['filename'] = pdict['filename']
self.uploaded_file['content_type'] = self._request.headers.get("Content-Type",
'application/octet-stream')
self.uploaded_file['body'] = self._contentbuffer
self.uploaded_file['body_len'] = int(self.content_length)
self.uploaded_file['body_filepath'] = self._contentbuffer.filepath
megabytes = int(self.content_length) / (1024 * 1024)
if self.file_upload:
limit_type = "upload"
limit = GLSetting.memory_copy.maximum_filesize
else:
limit_type = "json"
limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize
# is 1MB probably too high. probably this variable must be in kB
# less than 1 megabytes is always accepted
if megabytes > limit:
log.err("Tried %s request larger than expected (%dMb > %dMb)" %
(limit_type,
megabytes,
limit))
# In HTTP Protocol errors need to be managed differently than handlers
raise errors.HTTPRawLimitReach
if self.content_length > 0:
self.setRawMode()
return
elif self.file_upload:
self._on_request_body(self.uploaded_file)
self.file_upload = False
self.uploaded_file = {}
return
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
def _on_request_body(self, data):
try:
self._request.body = data
content_type = self._request.headers.get("Content-Type", "")
if self._request.method in ("POST", "PATCH", "PUT"):
if content_type.startswith("application/x-www-form-urlencoded") and self.content_length < GLSetting.www_form_urlencoded_maximum_size:
arguments = parse_qs_bytes(native_str(self._request.body))
for name, values in arguments.iteritems():
values = [v for v in values if v]
if values:
self._request.arguments.setdefault(name,
[]).extend(values)
elif content_type.startswith("application/x-www-form-urlencoded"):
raise errors.InvalidInputFormat("content type application/x-www-form-urlencoded not supported")
elif content_type.startswith("multipart/form-data"):
raise errors.InvalidInputFormat("content type multipart/form-data not supported")
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
def test_supports_http_1_1(self):
req = HTTPRequest("GET", "/something", version="HTTP/1.0")
self.assertFalse(req.supports_http_1_1())
req = HTTPRequest("GET", "/something", version="HTTP/1.1")
self.assertTrue(req.supports_http_1_1())
def _on_headers(self, data):
try:
data = native_str(data.decode("latin1"))
eol = data.find("\r\n")
start_line = data[:eol]
try:
method, uri, version = start_line.split(" ")
except ValueError:
raise _BadRequestException("Malformed HTTP request line")
if not version.startswith("HTTP/"):
raise _BadRequestException(
"Malformed HTTP version in HTTP Request-Line")
headers = httputil.HTTPHeaders.parse(data[eol:])
self._request = HTTPRequest(
connection=self, method=method, uri=uri, version=version,
headers=headers, remote_ip=self._remote_ip)
try:
self.content_length = int(headers.get("Content-Length", 0))
except ValueError:
raise _BadRequestException("Malformed Content-Length header")
# we always use secure temporary files in case of large json or file uploads
if self.content_length < 100000 and self._request.headers.get("Content-Disposition") is None:
self._contentbuffer = StringIO('')
else:
self._contentbuffer = GLSecureTemporaryFile(GLSetting.tmp_upload_path)
if headers.get("Expect") == "100-continue":
self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")
c_d_header = self._request.headers.get("Content-Disposition")
if c_d_header is not None:
key, pdict = parse_header(c_d_header)
if key != 'attachment' or 'filename' not in pdict:
raise _BadRequestException("Malformed Content-Disposition header")
self.file_upload = True
self.uploaded_file['filename'] = pdict['filename']
self.uploaded_file['content_type'] = self._request.headers.get("Content-Type",
'application/octet-stream')
self.uploaded_file['body'] = self._contentbuffer
self.uploaded_file['body_len'] = int(self.content_length)
self.uploaded_file['body_filepath'] = self._contentbuffer.filepath
megabytes = int(self.content_length) / (1024 * 1024)
if self.file_upload:
limit_type = "upload"
limit = GLSetting.memory_copy.maximum_filesize
else:
limit_type = "json"
limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize
# is 1MB probably too high. probably this variable must be in kB
# less than 1 megabytes is always accepted
if megabytes > limit:
log.err("Tried %s request larger than expected (%dMb > %dMb)" %
(limit_type,
megabytes,
limit))
# In HTTP Protocol errors need to be managed differently than handlers
raise errors.HTTPRawLimitReach
if self.content_length > 0:
self.setRawMode()
return
elif self.file_upload:
self._on_request_body(self.uploaded_file)
self.file_upload = False
self.uploaded_file = {}
return
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
class HTTPRequestTest(unittest.TestCase):
def setUp(self):
self.req = HTTPRequest("GET", "/something")
def test_init(self):
req = HTTPRequest("GET", "/something")
self.assertTrue(hasattr(req, "method"))
self.assertTrue(hasattr(req, "uri"))
self.assertTrue(hasattr(req, "version"))
self.assertTrue(hasattr(req, "headers"))
self.assertTrue(hasattr(req, "body"))
self.assertTrue(hasattr(req, "host"))
self.assertTrue(hasattr(req, "files"))
self.assertTrue(hasattr(req, "connection"))
self.assertTrue(hasattr(req, "path"))
self.assertTrue(hasattr(req, "arguments"))
self.assertTrue(hasattr(req, "remote_ip"))
def test_init_with_connection_xheaders(self):
connection = Mock()
connection.xheaders = True
headers = {
"X-Real-Ip": "127.0.0.1"
}
req = HTTPRequest(
"GET", "/something", headers=headers, connection=connection)
self.assertEqual(req.remote_ip, "127.0.0.1")
self.assertEqual(req.protocol, "http")
def test_init_with_invalid_connection_xheaders(self):
connection = Mock()
connection.xheaders = True
headers = {
"X-Real-Ip": "256.0.0.1"
}
req = HTTPRequest(
"GET", "/something", headers=headers, connection=connection)
self.assertEqual(req.remote_ip, None)
self.assertEqual(req.protocol, "http")
def test_init_with_invalid_protocol_xheaders(self):
connection = Mock()
connection.xheaders = True
protocol = "ftp"
req = HTTPRequest(
"GET", "/something", connection=connection, protocol=protocol)
self.assertEqual(req.protocol, "http")
def test_init_with_https(self):
connection = Mock()
connection.xheaders = False
connection.transport = StringTransport()
interfaces.ISSLTransport.providedBy = lambda x: True
req = HTTPRequest(
"GET", "/something", connection=connection)
self.assertEqual(req.protocol, "https")
def test_supports_http_1_1(self):
req = HTTPRequest("GET", "/something", version="HTTP/1.0")
self.assertFalse(req.supports_http_1_1())
req = HTTPRequest("GET", "/something", version="HTTP/1.1")
self.assertTrue(req.supports_http_1_1())
def test_cookies_create(self):
cookies = self.req.cookies
self.assertFalse(cookies)
def test_cookies_load(self):
self.req.headers = {
"Cookie": "a=b"
}
cookies = self.req.cookies
self.assertEqual(cookies['a'].value, 'b')
def test_cookies_invalid(self):
self.req.headers = {
"Cookie": "a"
}
def throw_exc(ignore):
raise Exception()
old_cookie = Cookie.SimpleCookie
Cookie.SimpleCookie = Mock()
Cookie.SimpleCookie.return_value.load = throw_exc
self.req.cookies
cookies = self.req.cookies
self.assertEqual(cookies, {})
Cookie.SimpleCookie = old_cookie
def test_full_url(self):
expected = "http://127.0.0.1/something"
self.assertEqual(self.req.full_url(), expected)
def test_request_time_empty_finish(self):
self.req._finish_time = None
self.assertTrue(self.req.request_time() < 0.01)
def test_request_time(self):
self.assertTrue(self.req.request_time() < 0.01)
def test_repr(self):
"""
Purely for coverage.
"""
repr(self.req)
def setUp(self):
self.req = HTTPRequest("GET", "/something")
class HTTPRequestTest(unittest.TestCase):
def setUp(self):
self.req = HTTPRequest("GET", "/something")
def test_init(self):
req = HTTPRequest("GET", "/something")
self.assertTrue(hasattr(req, "method"))
self.assertTrue(hasattr(req, "uri"))
self.assertTrue(hasattr(req, "version"))
self.assertTrue(hasattr(req, "headers"))
self.assertTrue(hasattr(req, "body"))
self.assertTrue(hasattr(req, "host"))
self.assertTrue(hasattr(req, "files"))
self.assertTrue(hasattr(req, "connection"))
self.assertTrue(hasattr(req, "path"))
self.assertTrue(hasattr(req, "arguments"))
self.assertTrue(hasattr(req, "remote_ip"))
def test_init_with_connection_xheaders(self):
connection = Mock()
connection.xheaders = True
headers = {"X-Real-Ip": "127.0.0.1"}
req = HTTPRequest("GET",
"/something",
headers=headers,
connection=connection)
self.assertEqual(req.remote_ip, "127.0.0.1")
self.assertEqual(req.protocol, "http")
def test_init_with_invalid_connection_xheaders(self):
connection = Mock()
connection.xheaders = True
headers = {"X-Real-Ip": "256.0.0.1"}
req = HTTPRequest("GET",
"/something",
headers=headers,
connection=connection)
self.assertEqual(req.remote_ip, None)
self.assertEqual(req.protocol, "http")
def test_init_with_invalid_protocol_xheaders(self):
connection = Mock()
connection.xheaders = True
protocol = "ftp"
req = HTTPRequest("GET",
"/something",
connection=connection,
protocol=protocol)
self.assertEqual(req.protocol, "http")
def test_init_with_https(self):
connection = Mock()
connection.xheaders = False
connection.transport = StringTransport()
interfaces.ISSLTransport.providedBy = lambda x: True
req = HTTPRequest("GET", "/something", connection=connection)
self.assertEqual(req.protocol, "https")
def test_supports_http_1_1(self):
req = HTTPRequest("GET", "/something", version="HTTP/1.0")
self.assertFalse(req.supports_http_1_1())
req = HTTPRequest("GET", "/something", version="HTTP/1.1")
self.assertTrue(req.supports_http_1_1())
def test_cookies_create(self):
cookies = self.req.cookies
self.assertFalse(cookies)
def test_cookies_load(self):
self.req.headers = {"Cookie": "a=b"}
cookies = self.req.cookies
self.assertEqual(cookies['a'].value, 'b')
def test_cookies_invalid(self):
self.req.headers = {"Cookie": "a"}
def throw_exc(ignore):
raise Exception()
old_cookie = Cookie.SimpleCookie
Cookie.SimpleCookie = Mock()
Cookie.SimpleCookie.return_value.load = throw_exc
self.req.cookies
cookies = self.req.cookies
self.assertEqual(cookies, {})
Cookie.SimpleCookie = old_cookie
def test_full_url(self):
expected = "http://127.0.0.1/something"
self.assertEqual(self.req.full_url(), expected)
def test_request_time_empty_finish(self):
self.req._finish_time = None
self.assertTrue(self.req.request_time() < 0.01)
def test_request_time(self):
self.assertTrue(self.req.request_time() < 0.01)
def test_repr(self):
"""
Purely for coverage.
"""
repr(self.req)
def _on_headers(self, data):
try:
data = native_str(data.decode("latin1"))
eol = data.find("\r\n")
start_line = data[:eol]
try:
method, uri, version = start_line.split(" ")
except ValueError:
raise _BadRequestException("Malformed HTTP request line")
if not version.startswith("HTTP/"):
raise _BadRequestException(
"Malformed HTTP version in HTTP Request-Line")
headers = httputil.HTTPHeaders.parse(data[eol:])
self._request = HTTPRequest(
connection=self, method=method, uri=uri, version=version,
headers=headers, remote_ip=self._remote_ip)
try:
self.content_length = int(headers.get("Content-Length", 0))
except ValueError:
raise _BadRequestException("Malformed Content-Length header")
# we always use secure temporary files in case of large json or file uploads
if self.content_length < 100000 and self._request.headers.get("Content-Disposition") is None:
self._contentbuffer = StringIO('')
else:
self._contentbuffer = GLSecureTemporaryFile(GLSetting.tmp_upload_path)
if headers.get("Expect") == "100-continue":
self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")
c_d_header = self._request.headers.get("Content-Disposition")
if c_d_header is not None:
key, pdict = parse_header(c_d_header)
if key != 'attachment' or 'filename' not in pdict:
raise _BadRequestException("Malformed Content-Disposition header")
self.file_upload = True
self.uploaded_file['filename'] = pdict['filename']
self.uploaded_file['content_type'] = self._request.headers.get("Content-Type",
'application/octet-stream')
self.uploaded_file['body'] = self._contentbuffer
self.uploaded_file['body_len'] = int(self.content_length)
self.uploaded_file['body_filepath'] = self._contentbuffer.filepath
megabytes = int(self.content_length) / (1024 * 1024)
if self.file_upload:
limit_type = "upload"
limit = GLSetting.memory_copy.maximum_filesize
else:
limit_type = "json"
limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize
# is 1MB probably too high. probably this variable must be in kB
# less than 1 megabytes is always accepted
if megabytes > limit:
log.err("Tried %s request larger than expected (%dMb > %dMb)" %
(limit_type,
megabytes,
limit))
# In HTTP Protocol errors need to be managed differently than handlers
raise errors.HTTPRawLimitReach
if self.content_length > 0:
self.setRawMode()
return
elif self.file_upload:
self._on_request_body(self.uploaded_file)
self.file_upload = False
self.uploaded_file = {}
return
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
class GLHTTPServer(HTTPConnection):
file_upload = False
def __init__(self):
self.uploaded_file = {}
def rawDataReceived(self, data):
if self.content_length is not None:
data, rest = data[:self.content_length], data[self.content_length:]
self.content_length -= len(data)
else:
rest = ''
self._contentbuffer.write(data)
if self.content_length == 0 and self._contentbuffer is not None:
tmpbuf = self._contentbuffer
self.content_length = self._contentbuffer = None
self.setLineMode(rest)
tmpbuf.seek(0, 0)
if self.file_upload:
self._on_request_body(self.uploaded_file)
self.file_upload = False
self.uploaded_file = {}
else:
self._on_request_body(tmpbuf.read())
def _on_headers(self, data):
try:
data = native_str(data.decode("latin1"))
eol = data.find("\r\n")
start_line = data[:eol]
try:
method, uri, version = start_line.split(" ")
except ValueError:
raise _BadRequestException("Malformed HTTP request line")
if not version.startswith("HTTP/"):
raise _BadRequestException(
"Malformed HTTP version in HTTP Request-Line")
headers = httputil.HTTPHeaders.parse(data[eol:])
self._request = HTTPRequest(
connection=self, method=method, uri=uri, version=version,
headers=headers, remote_ip=self._remote_ip)
try:
self.content_length = int(headers.get("Content-Length", 0))
except ValueError:
raise _BadRequestException("Malformed Content-Length header")
# we always use secure temporary files in case of large json or file uploads
if self.content_length < 100000 and self._request.headers.get("Content-Disposition") is None:
self._contentbuffer = StringIO('')
else:
self._contentbuffer = GLSecureTemporaryFile(GLSetting.tmp_upload_path)
if headers.get("Expect") == "100-continue":
self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")
c_d_header = self._request.headers.get("Content-Disposition")
if c_d_header is not None:
key, pdict = parse_header(c_d_header)
if key != 'attachment' or 'filename' not in pdict:
raise _BadRequestException("Malformed Content-Disposition header")
self.file_upload = True
self.uploaded_file['filename'] = pdict['filename']
self.uploaded_file['content_type'] = self._request.headers.get("Content-Type",
'application/octet-stream')
self.uploaded_file['body'] = self._contentbuffer
self.uploaded_file['body_len'] = int(self.content_length)
self.uploaded_file['body_filepath'] = self._contentbuffer.filepath
megabytes = int(self.content_length) / (1024 * 1024)
if self.file_upload:
limit_type = "upload"
limit = GLSetting.memory_copy.maximum_filesize
else:
limit_type = "json"
limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize
# is 1MB probably too high. probably this variable must be in kB
# less than 1 megabytes is always accepted
if megabytes > limit:
log.err("Tried %s request larger than expected (%dMb > %dMb)" %
(limit_type,
megabytes,
limit))
# In HTTP Protocol errors need to be managed differently than handlers
raise errors.HTTPRawLimitReach
if self.content_length > 0:
self.setRawMode()
return
elif self.file_upload:
self._on_request_body(self.uploaded_file)
self.file_upload = False
self.uploaded_file = {}
return
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
def _on_request_body(self, data):
try:
self._request.body = data
content_type = self._request.headers.get("Content-Type", "")
if self._request.method in ("POST", "PATCH", "PUT"):
if content_type.startswith("application/x-www-form-urlencoded") and self.content_length < GLSetting.www_form_urlencoded_maximum_size:
arguments = parse_qs_bytes(native_str(self._request.body))
for name, values in arguments.iteritems():
values = [v for v in values if v]
if values:
self._request.arguments.setdefault(name,
[]).extend(values)
elif content_type.startswith("application/x-www-form-urlencoded"):
raise errors.InvalidInputFormat("content type application/x-www-form-urlencoded not supported")
elif content_type.startswith("multipart/form-data"):
raise errors.InvalidInputFormat("content type multipart/form-data not supported")
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
def _on_headers(self, data):
try:
data = native_str(data.decode("latin1"))
eol = data.find("\r\n")
start_line = data[:eol]
try:
method, uri, version = start_line.split(" ")
except ValueError:
raise _BadRequestException("Malformed HTTP request line")
if not version.startswith("HTTP/"):
raise _BadRequestException("Malformed HTTP version in HTTP Request-Line")
headers = httputil.HTTPHeaders.parse(data[eol:])
self._request = HTTPRequest(
connection=self, method=method, uri=uri, version=version, headers=headers, remote_ip=self._remote_ip
)
content_length = int(headers.get("Content-Length", 0))
self.content_length = content_length
if content_length:
if headers.get("Expect") == "100-continue":
self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")
if content_length < 100000:
self._contentbuffer = StringIO("")
else:
self._contentbuffer = TemporaryFile()
c_d_header = self._request.headers.get("Content-Disposition")
if c_d_header is not None:
c_d_header = c_d_header.lower()
m = content_disposition_re.match(c_d_header)
if m is None:
raise Exception
self.file_upload = True
self.uploaded_file["filename"] = m.group(1)
self.uploaded_file["content_type"] = self._request.headers.get(
"Content-Type", "application/octet-stream"
)
self.uploaded_file["body"] = self._contentbuffer
self.uploaded_file["body_len"] = int(content_length)
megabytes = int(content_length) / (1024 * 1024)
if self.file_upload:
limit_type = "upload"
limit = GLSetting.memory_copy.maximum_filesize
else:
limit_type = "json"
limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize
# is 1MB probably too high. probably this variable must be
# in kB
# less than 1 megabytes is always accepted
if megabytes > limit:
log.err("Tried %s request larger than expected (%dMb > %dMb)" % (limit_type, megabytes, limit))
# In HTTP Protocol errors need to be managed differently than handlers
raise errors.HTTPRawLimitReach
self.setRawMode()
return
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
def setUp(self):
self.req = HTTPRequest("GET", "/something")
def test_supports_http_1_1(self):
req = HTTPRequest("GET", "/something", version="HTTP/1.0")
self.assertFalse(req.supports_http_1_1())
req = HTTPRequest("GET", "/something", version="HTTP/1.1")
self.assertTrue(req.supports_http_1_1())
