def process_request(self, request):
encoding = request.META.get('HTTP_CONTENT_ENCODING', None)
if encoding == 'gzip':
if is_ratelimited(request,
group='gunzip_request_middleware',
key='ip',
rate='300/1m',
increment=True):
return http.HttpResponse(
'Rate limit exceeded: too many gzipped request bodies',
status=429)
data = request._stream.read()
if len(data) > settings.GUNZIP_MAX_COMPRESSED_SIZE:
logger.warning('Compressed request body is too large: %s',
request.path,
extra={
'status_code': 400,
'request': request,
})
return http.HttpResponseBadRequest(
'Compressed request body is too large')
try:
zfile = GzipFile(mode='rb', fileobj=StringIO(data))
uncompressed = zfile.read()
request._stream = LimitedStream(StringIO(uncompressed),
len(uncompressed))
del request.META['HTTP_CONTENT_ENCODING']
except IOError:
return http.HttpResponseBadRequest(
'Invalid content-encoding, could not gunzip')
return None
def __init__(self, stdin, stdout, stderr, environ, **kwargs):
"""
Use a LimitedStream so that unread request data will be ignored at
the end of the request. WSGIRequest uses a LimitedStream but it
shouldn't discard the data since the upstream servers usually do this.
This fix applies only for testserver/runserver.
"""
try:
content_length = int(environ.get('CONTENT_LENGTH'))
except (ValueError, TypeError):
content_length = 0
super().__init__(LimitedStream(stdin, content_length), stdout, stderr, environ, **kwargs)
def __init__(self, stdin, stdout, stderr, environ, **kwargs):
"""
Setup a limited stream, so we can discard unread request data
at the end of the request. Django already uses `LimitedStream`
in `WSGIRequest` but it shouldn't discard the data since the
upstream servers usually do this. Hence we fix this only for
our testserver/runserver.
"""
try:
content_length = int(environ.get('CONTENT_LENGTH'))
except (ValueError, TypeError):
content_length = 0
super().__init__(LimitedStream(stdin, content_length), stdout, stderr,
environ, **kwargs)
def api_handler(request, service):
service_handler = Settings.get_service_handler(service)
if service_handler:
logger.debug("Hitting service %s." % service)
log = LogEntry(
url="%s %s" % (request.method, request.get_full_path()),
application=service)
request_body = request.META["wsgi.input"].read(
request._stream.remaining)
request.META["wsgi.input"] = StringIO(request_body)
request._stream = LimitedStream(
request.META["wsgi.input"], request._stream.remaining)
service_handler.event_manager.add_listener(
"wsgi_close", partial(handle_wsgi_close, log=log))
service_handler.app.event_manager.add_listener(
"method_call_exception", partial(handle_exception, log=log))
try:
response = csrf_exempt(service_handler)(request)
except Exception:
log.traceback = unicode(traceback.format_exc(), errors="ignore")
raise
else:
if response.content:
log.response = response.content
finally:
if request_body:
log.request = request_body
log.save()
service_handler.event_manager = EventManager(service_handler)
service_handler.app.event_manager = EventManager(
service_handler.app)
return response
else:
msg = "Service %s not found" % service
logger.info(msg)
raise Http404(msg)
def test_limited_stream(self):
# Read all of a limited stream
stream = LimitedStream(BytesIO(b'test'), 2)
self.assertEqual(stream.read(), b'te')
# Reading again returns nothing.
self.assertEqual(stream.read(), b'')
# Read a number of characters greater than the stream has to offer
stream = LimitedStream(BytesIO(b'test'), 2)
self.assertEqual(stream.read(5), b'te')
# Reading again returns nothing.
self.assertEqual(stream.readline(5), b'')
# Read sequentially from a stream
stream = LimitedStream(BytesIO(b'12345678'), 8)
self.assertEqual(stream.read(5), b'12345')
self.assertEqual(stream.read(5), b'678')
# Reading again returns nothing.
self.assertEqual(stream.readline(5), b'')
# Read lines from a stream
stream = LimitedStream(BytesIO(b'1234\n5678\nabcd\nefgh\nijkl'), 24)
# Read a full line, unconditionally
self.assertEqual(stream.readline(), b'1234\n')
# Read a number of characters less than a line
self.assertEqual(stream.readline(2), b'56')
# Read the rest of the partial line
self.assertEqual(stream.readline(), b'78\n')
# Read a full line, with a character limit greater than the line length
self.assertEqual(stream.readline(6), b'abcd\n')
# Read the next line, deliberately terminated at the line end
self.assertEqual(stream.readline(4), b'efgh')
# Read the next line... just the line end
self.assertEqual(stream.readline(), b'\n')
# Read everything else.
self.assertEqual(stream.readline(), b'ijkl')
# Regression for #15018
# If a stream contains a newline, but the provided length
# is less than the number of provided characters, the newline
# doesn't reset the available character count
stream = LimitedStream(BytesIO(b'1234\nabcdef'), 9)
self.assertEqual(stream.readline(10), b'1234\n')
self.assertEqual(stream.readline(3), b'abc')
# Now expire the available characters
self.assertEqual(stream.readline(3), b'd')
# Reading again returns nothing.
self.assertEqual(stream.readline(2), b'')
# Same test, but with read, not readline.
stream = LimitedStream(BytesIO(b'1234\nabcdef'), 9)
self.assertEqual(stream.read(6), b'1234\na')
self.assertEqual(stream.read(2), b'bc')
self.assertEqual(stream.read(2), b'd')
self.assertEqual(stream.read(2), b'')
self.assertEqual(stream.read(), b'')
def test_limited_stream(self):
# Read all of a limited stream
stream = LimitedStream(StringIO('test'), 2)
self.assertEqual(stream.read(), 'te')
# Reading again returns nothing.
self.assertEqual(stream.read(), '')
# Read a number of characters greater than the stream has to offer
stream = LimitedStream(StringIO('test'), 2)
self.assertEqual(stream.read(5), 'te')
# Reading again returns nothing.
self.assertEqual(stream.readline(5), '')
# Read sequentially from a stream
stream = LimitedStream(StringIO('12345678'), 8)
self.assertEqual(stream.read(5), '12345')
self.assertEqual(stream.read(5), '678')
# Reading again returns nothing.
self.assertEqual(stream.readline(5), '')
# Read lines from a stream
stream = LimitedStream(StringIO('1234\n5678\nabcd\nefgh\nijkl'), 24)
# Read a full line, unconditionally
self.assertEqual(stream.readline(), '1234\n')
# Read a number of characters less than a line
self.assertEqual(stream.readline(2), '56')
# Read the rest of the partial line
self.assertEqual(stream.readline(), '78\n')
# Read a full line, with a character limit greater than the line length
self.assertEqual(stream.readline(6), 'abcd\n')
# Read the next line, deliberately terminated at the line end
self.assertEqual(stream.readline(4), 'efgh')
# Read the next line... just the line end
self.assertEqual(stream.readline(), '\n')
# Read everything else.
self.assertEqual(stream.readline(), 'ijkl')
# Regression for #15018
# If a stream contains a newline, but the provided length
# is less than the number of provided characters, the newline
# doesn't reset the available character count
stream = LimitedStream(StringIO('1234\nabcdef'), 9)
self.assertEqual(stream.readline(10), '1234\n')
self.assertEqual(stream.readline(3), 'abc')
# Now expire the available characters
self.assertEqual(stream.readline(3), 'd')
# Reading again returns nothing.
self.assertEqual(stream.readline(2), '')
# Same test, but with read, not readline.
stream = LimitedStream(StringIO('1234\nabcdef'), 9)
self.assertEqual(stream.read(6), '1234\na')
self.assertEqual(stream.read(2), 'bc')
self.assertEqual(stream.read(2), 'd')
self.assertEqual(stream.read(2), '')
self.assertEqual(stream.read(), '')
def test_limited_stream(self):
# Read all of a limited stream
stream = LimitedStream(StringIO('test'), 2)
self.assertEqual(stream.read(), 'te')
# Read a number of characters greater than the stream has to offer
stream = LimitedStream(StringIO('test'), 2)
self.assertEqual(stream.read(5), 'te')
# Read sequentially from a stream
stream = LimitedStream(StringIO('12345678'), 8)
self.assertEqual(stream.read(5), '12345')
self.assertEqual(stream.read(5), '678')
# Read lines from a stream
stream = LimitedStream(StringIO('1234\n5678\nabcd\nefgh\nijkl'), 24)
# Read a full line, unconditionally
self.assertEqual(stream.readline(), '1234\n')
# Read a number of characters less than a line
self.assertEqual(stream.readline(2), '56')
# Read the rest of the partial line
self.assertEqual(stream.readline(), '78\n')
# Read a full line, with a character limit greater than the line length
self.assertEqual(stream.readline(6), 'abcd\n')
# Read the next line, deliberately terminated at the line end
self.assertEqual(stream.readline(4), 'efgh')
# Read the next line... just the line end
self.assertEqual(stream.readline(), '\n')
# Read everything else.
self.assertEqual(stream.readline(), 'ijkl')
def test_limited_stream(self):
# Read all of a limited stream
stream = LimitedStream(BytesIO(b"test"), 2)
self.assertEqual(stream.read(), b"te")
# Reading again returns nothing.
self.assertEqual(stream.read(), b"")
# Read a number of characters greater than the stream has to offer
stream = LimitedStream(BytesIO(b"test"), 2)
self.assertEqual(stream.read(5), b"te")
# Reading again returns nothing.
self.assertEqual(stream.readline(5), b"")
# Read sequentially from a stream
stream = LimitedStream(BytesIO(b"12345678"), 8)
self.assertEqual(stream.read(5), b"12345")
self.assertEqual(stream.read(5), b"678")
# Reading again returns nothing.
self.assertEqual(stream.readline(5), b"")
# Read lines from a stream
stream = LimitedStream(BytesIO(b"1234\n5678\nabcd\nefgh\nijkl"), 24)
# Read a full line, unconditionally
self.assertEqual(stream.readline(), b"1234\n")
# Read a number of characters less than a line
self.assertEqual(stream.readline(2), b"56")
# Read the rest of the partial line
self.assertEqual(stream.readline(), b"78\n")
# Read a full line, with a character limit greater than the line length
self.assertEqual(stream.readline(6), b"abcd\n")
# Read the next line, deliberately terminated at the line end
self.assertEqual(stream.readline(4), b"efgh")
# Read the next line... just the line end
self.assertEqual(stream.readline(), b"\n")
# Read everything else.
self.assertEqual(stream.readline(), b"ijkl")
# Regression for #15018
# If a stream contains a newline, but the provided length
# is less than the number of provided characters, the newline
# doesn't reset the available character count
stream = LimitedStream(BytesIO(b"1234\nabcdef"), 9)
self.assertEqual(stream.readline(10), b"1234\n")
self.assertEqual(stream.readline(3), b"abc")
# Now expire the available characters
self.assertEqual(stream.readline(3), b"d")
# Reading again returns nothing.
self.assertEqual(stream.readline(2), b"")
# Same test, but with read, not readline.
stream = LimitedStream(BytesIO(b"1234\nabcdef"), 9)
self.assertEqual(stream.read(6), b"1234\na")
self.assertEqual(stream.read(2), b"bc")
self.assertEqual(stream.read(2), b"d")
self.assertEqual(stream.read(2), b"")
self.assertEqual(stream.read(), b"")
def test_limited_stream(self):
# Read all of a limited stream
stream = LimitedStream(StringIO('test'), 2)
self.assertEqual(stream.read(), 'te')
# Read a number of characters greater than the stream has to offer
stream = LimitedStream(StringIO('test'), 2)
self.assertEqual(stream.read(5), 'te')
# Read sequentially from a stream
stream = LimitedStream(StringIO('12345678'), 8)
self.assertEqual(stream.read(5), '12345')
self.assertEqual(stream.read(5), '678')
# Read lines from a stream
stream = LimitedStream(StringIO('1234\n5678\nabcd\nefgh\nijkl'), 24)
# Read a full line, unconditionally
self.assertEqual(stream.readline(), '1234\n')
# Read a number of characters less than a line
self.assertEqual(stream.readline(2), '56')
# Read the rest of the partial line
self.assertEqual(stream.readline(), '78\n')
# Read a full line, with a character limit greater than the line length
self.assertEqual(stream.readline(6), 'abcd\n')
# Read the next line, deliberately terminated at the line end
self.assertEqual(stream.readline(4), 'efgh')
# Read the next line... just the line end
self.assertEqual(stream.readline(), '\n')
# Read everything else.
self.assertEqual(stream.readline(), 'ijkl')
