def process_request(self, request):
        encoding = request.META.get('HTTP_CONTENT_ENCODING', None)

        if encoding == 'gzip':
            if is_ratelimited(request,
                              group='gunzip_request_middleware',
                              key='ip',
                              rate='300/1m',
                              increment=True):
                return http.HttpResponse(
                    'Rate limit exceeded: too many gzipped request bodies',
                    status=429)

            data = request._stream.read()
            if len(data) > settings.GUNZIP_MAX_COMPRESSED_SIZE:
                logger.warning('Compressed request body is too large: %s',
                               request.path,
                               extra={
                                   'status_code': 400,
                                   'request': request,
                               })
                return http.HttpResponseBadRequest(
                    'Compressed request body is too large')

            try:
                zfile = GzipFile(mode='rb', fileobj=StringIO(data))
                uncompressed = zfile.read()
                request._stream = LimitedStream(StringIO(uncompressed),
                                                len(uncompressed))
                del request.META['HTTP_CONTENT_ENCODING']
            except IOError:
                return http.HttpResponseBadRequest(
                    'Invalid content-encoding, could not gunzip')

        return None
Exemple #2
0
 def __init__(self, stdin, stdout, stderr, environ, **kwargs):
     """
     Use a LimitedStream so that unread request data will be ignored at
     the end of the request. WSGIRequest uses a LimitedStream but it
     shouldn't discard the data since the upstream servers usually do this.
     This fix applies only for testserver/runserver.
     """
     try:
         content_length = int(environ.get('CONTENT_LENGTH'))
     except (ValueError, TypeError):
         content_length = 0
     super().__init__(LimitedStream(stdin, content_length), stdout, stderr, environ, **kwargs)
Exemple #3
0
 def __init__(self, stdin, stdout, stderr, environ, **kwargs):
     """
     Setup a limited stream, so we can discard unread request data
     at the end of the request. Django already uses `LimitedStream`
     in `WSGIRequest` but it shouldn't discard the data since the
     upstream servers usually do this. Hence we fix this only for
     our testserver/runserver.
     """
     try:
         content_length = int(environ.get('CONTENT_LENGTH'))
     except (ValueError, TypeError):
         content_length = 0
     super().__init__(LimitedStream(stdin, content_length), stdout, stderr,
                      environ, **kwargs)
Exemple #4
0
def api_handler(request, service):
    service_handler = Settings.get_service_handler(service)
    if service_handler:
        logger.debug("Hitting service %s." % service)
        log = LogEntry(
            url="%s %s" % (request.method, request.get_full_path()),
            application=service)
        request_body = request.META["wsgi.input"].read(
            request._stream.remaining)
        request.META["wsgi.input"] = StringIO(request_body)
        request._stream = LimitedStream(
            request.META["wsgi.input"], request._stream.remaining)
        service_handler.event_manager.add_listener(
            "wsgi_close", partial(handle_wsgi_close, log=log))
        service_handler.app.event_manager.add_listener(
            "method_call_exception", partial(handle_exception, log=log))

        try:
            response = csrf_exempt(service_handler)(request)
        except Exception:
            log.traceback = unicode(traceback.format_exc(), errors="ignore")
            raise
        else:
            if response.content:
                log.response = response.content
        finally:
            if request_body:
                log.request = request_body
            log.save()
            service_handler.event_manager = EventManager(service_handler)
            service_handler.app.event_manager = EventManager(
                service_handler.app)

        return response
    else:
        msg = "Service %s not found" % service
        logger.info(msg)
        raise Http404(msg)
Exemple #5
0
    def test_limited_stream(self):
        # Read all of a limited stream
        stream = LimitedStream(BytesIO(b'test'), 2)
        self.assertEqual(stream.read(), b'te')
        # Reading again returns nothing.
        self.assertEqual(stream.read(), b'')

        # Read a number of characters greater than the stream has to offer
        stream = LimitedStream(BytesIO(b'test'), 2)
        self.assertEqual(stream.read(5), b'te')
        # Reading again returns nothing.
        self.assertEqual(stream.readline(5), b'')

        # Read sequentially from a stream
        stream = LimitedStream(BytesIO(b'12345678'), 8)
        self.assertEqual(stream.read(5), b'12345')
        self.assertEqual(stream.read(5), b'678')
        # Reading again returns nothing.
        self.assertEqual(stream.readline(5), b'')

        # Read lines from a stream
        stream = LimitedStream(BytesIO(b'1234\n5678\nabcd\nefgh\nijkl'), 24)
        # Read a full line, unconditionally
        self.assertEqual(stream.readline(), b'1234\n')
        # Read a number of characters less than a line
        self.assertEqual(stream.readline(2), b'56')
        # Read the rest of the partial line
        self.assertEqual(stream.readline(), b'78\n')
        # Read a full line, with a character limit greater than the line length
        self.assertEqual(stream.readline(6), b'abcd\n')
        # Read the next line, deliberately terminated at the line end
        self.assertEqual(stream.readline(4), b'efgh')
        # Read the next line... just the line end
        self.assertEqual(stream.readline(), b'\n')
        # Read everything else.
        self.assertEqual(stream.readline(), b'ijkl')

        # Regression for #15018
        # If a stream contains a newline, but the provided length
        # is less than the number of provided characters, the newline
        # doesn't reset the available character count
        stream = LimitedStream(BytesIO(b'1234\nabcdef'), 9)
        self.assertEqual(stream.readline(10), b'1234\n')
        self.assertEqual(stream.readline(3), b'abc')
        # Now expire the available characters
        self.assertEqual(stream.readline(3), b'd')
        # Reading again returns nothing.
        self.assertEqual(stream.readline(2), b'')

        # Same test, but with read, not readline.
        stream = LimitedStream(BytesIO(b'1234\nabcdef'), 9)
        self.assertEqual(stream.read(6), b'1234\na')
        self.assertEqual(stream.read(2), b'bc')
        self.assertEqual(stream.read(2), b'd')
        self.assertEqual(stream.read(2), b'')
        self.assertEqual(stream.read(), b'')
Exemple #6
0
    def test_limited_stream(self):
        # Read all of a limited stream
        stream = LimitedStream(StringIO('test'), 2)
        self.assertEqual(stream.read(), 'te')
        # Reading again returns nothing.
        self.assertEqual(stream.read(), '')

        # Read a number of characters greater than the stream has to offer
        stream = LimitedStream(StringIO('test'), 2)
        self.assertEqual(stream.read(5), 'te')
        # Reading again returns nothing.
        self.assertEqual(stream.readline(5), '')

        # Read sequentially from a stream
        stream = LimitedStream(StringIO('12345678'), 8)
        self.assertEqual(stream.read(5), '12345')
        self.assertEqual(stream.read(5), '678')
        # Reading again returns nothing.
        self.assertEqual(stream.readline(5), '')

        # Read lines from a stream
        stream = LimitedStream(StringIO('1234\n5678\nabcd\nefgh\nijkl'), 24)
        # Read a full line, unconditionally
        self.assertEqual(stream.readline(), '1234\n')
        # Read a number of characters less than a line
        self.assertEqual(stream.readline(2), '56')
        # Read the rest of the partial line
        self.assertEqual(stream.readline(), '78\n')
        # Read a full line, with a character limit greater than the line length
        self.assertEqual(stream.readline(6), 'abcd\n')
        # Read the next line, deliberately terminated at the line end
        self.assertEqual(stream.readline(4), 'efgh')
        # Read the next line... just the line end
        self.assertEqual(stream.readline(), '\n')
        # Read everything else.
        self.assertEqual(stream.readline(), 'ijkl')

        # Regression for #15018
        # If a stream contains a newline, but the provided length
        # is less than the number of provided characters, the newline
        # doesn't reset the available character count
        stream = LimitedStream(StringIO('1234\nabcdef'), 9)
        self.assertEqual(stream.readline(10), '1234\n')
        self.assertEqual(stream.readline(3), 'abc')
        # Now expire the available characters
        self.assertEqual(stream.readline(3), 'd')
        # Reading again returns nothing.
        self.assertEqual(stream.readline(2), '')

        # Same test, but with read, not readline.
        stream = LimitedStream(StringIO('1234\nabcdef'), 9)
        self.assertEqual(stream.read(6), '1234\na')
        self.assertEqual(stream.read(2), 'bc')
        self.assertEqual(stream.read(2), 'd')
        self.assertEqual(stream.read(2), '')
        self.assertEqual(stream.read(), '')
Exemple #7
0
    def test_limited_stream(self):
        # Read all of a limited stream
        stream = LimitedStream(StringIO('test'), 2)
        self.assertEqual(stream.read(), 'te')

        # Read a number of characters greater than the stream has to offer
        stream = LimitedStream(StringIO('test'), 2)
        self.assertEqual(stream.read(5), 'te')

        # Read sequentially from a stream
        stream = LimitedStream(StringIO('12345678'), 8)
        self.assertEqual(stream.read(5), '12345')
        self.assertEqual(stream.read(5), '678')

        # Read lines from a stream
        stream = LimitedStream(StringIO('1234\n5678\nabcd\nefgh\nijkl'), 24)
        # Read a full line, unconditionally
        self.assertEqual(stream.readline(), '1234\n')
        # Read a number of characters less than a line
        self.assertEqual(stream.readline(2), '56')
        # Read the rest of the partial line
        self.assertEqual(stream.readline(), '78\n')
        # Read a full line, with a character limit greater than the line length
        self.assertEqual(stream.readline(6), 'abcd\n')
        # Read the next line, deliberately terminated at the line end
        self.assertEqual(stream.readline(4), 'efgh')
        # Read the next line... just the line end
        self.assertEqual(stream.readline(), '\n')
        # Read everything else.
        self.assertEqual(stream.readline(), 'ijkl')
Exemple #8
0
    def test_limited_stream(self):
        # Read all of a limited stream
        stream = LimitedStream(BytesIO(b"test"), 2)
        self.assertEqual(stream.read(), b"te")
        # Reading again returns nothing.
        self.assertEqual(stream.read(), b"")

        # Read a number of characters greater than the stream has to offer
        stream = LimitedStream(BytesIO(b"test"), 2)
        self.assertEqual(stream.read(5), b"te")
        # Reading again returns nothing.
        self.assertEqual(stream.readline(5), b"")

        # Read sequentially from a stream
        stream = LimitedStream(BytesIO(b"12345678"), 8)
        self.assertEqual(stream.read(5), b"12345")
        self.assertEqual(stream.read(5), b"678")
        # Reading again returns nothing.
        self.assertEqual(stream.readline(5), b"")

        # Read lines from a stream
        stream = LimitedStream(BytesIO(b"1234\n5678\nabcd\nefgh\nijkl"), 24)
        # Read a full line, unconditionally
        self.assertEqual(stream.readline(), b"1234\n")
        # Read a number of characters less than a line
        self.assertEqual(stream.readline(2), b"56")
        # Read the rest of the partial line
        self.assertEqual(stream.readline(), b"78\n")
        # Read a full line, with a character limit greater than the line length
        self.assertEqual(stream.readline(6), b"abcd\n")
        # Read the next line, deliberately terminated at the line end
        self.assertEqual(stream.readline(4), b"efgh")
        # Read the next line... just the line end
        self.assertEqual(stream.readline(), b"\n")
        # Read everything else.
        self.assertEqual(stream.readline(), b"ijkl")

        # Regression for #15018
        # If a stream contains a newline, but the provided length
        # is less than the number of provided characters, the newline
        # doesn't reset the available character count
        stream = LimitedStream(BytesIO(b"1234\nabcdef"), 9)
        self.assertEqual(stream.readline(10), b"1234\n")
        self.assertEqual(stream.readline(3), b"abc")
        # Now expire the available characters
        self.assertEqual(stream.readline(3), b"d")
        # Reading again returns nothing.
        self.assertEqual(stream.readline(2), b"")

        # Same test, but with read, not readline.
        stream = LimitedStream(BytesIO(b"1234\nabcdef"), 9)
        self.assertEqual(stream.read(6), b"1234\na")
        self.assertEqual(stream.read(2), b"bc")
        self.assertEqual(stream.read(2), b"d")
        self.assertEqual(stream.read(2), b"")
        self.assertEqual(stream.read(), b"")
Exemple #9
0
    def test_limited_stream(self):
        # Read all of a limited stream
        stream = LimitedStream(StringIO('test'), 2)
        self.assertEqual(stream.read(), 'te')

        # Read a number of characters greater than the stream has to offer
        stream = LimitedStream(StringIO('test'), 2)
        self.assertEqual(stream.read(5), 'te')

        # Read sequentially from a stream
        stream = LimitedStream(StringIO('12345678'), 8)
        self.assertEqual(stream.read(5), '12345')
        self.assertEqual(stream.read(5), '678')

        # Read lines from a stream
        stream = LimitedStream(StringIO('1234\n5678\nabcd\nefgh\nijkl'), 24)
        # Read a full line, unconditionally
        self.assertEqual(stream.readline(), '1234\n')
        # Read a number of characters less than a line
        self.assertEqual(stream.readline(2), '56')
        # Read the rest of the partial line
        self.assertEqual(stream.readline(), '78\n')
        # Read a full line, with a character limit greater than the line length
        self.assertEqual(stream.readline(6), 'abcd\n')
        # Read the next line, deliberately terminated at the line end
        self.assertEqual(stream.readline(4), 'efgh')
        # Read the next line... just the line end
        self.assertEqual(stream.readline(), '\n')
        # Read everything else.
        self.assertEqual(stream.readline(), 'ijkl')