Exemplo n.º 1
0
 def headerReceived(self, line):
     """
     Overridden to reduce the function actions and
     in particular to avoid self._transferDecoder actions and
     implement a streaming proxy
     """
     header, data = line.split(b':', 1)
     header = header.lower()
     data = data.strip()
     req = self.requests[-1]
     if header == b'content-length':
         try:
             self.length = int(data)
         except ValueError:
             self.transport.write(b"HTTP/1.1 400 Bad Request\r\n\r\n")
             self.length = None
             self.transport.loseConnection()
             return
         self._transferDecoder = _IdentityTransferDecoder(
             self.length, req.bodyProducer.dataReceived, self._finishRequestBody)
     elif header == b'transfer-encoding' and data.lower() == b'chunked':
         self.length = None
         self._transferDecoder = _ChunkedTransferDecoder(
             req.bodyProducer.dataReceived, self._finishRequestBody)
     reqHeaders = req.requestHeaders
     values = reqHeaders.getRawHeaders(header)
     if values is not None:
         values.append(data)
     else:
         reqHeaders.setRawHeaders(header, [data])
Exemplo n.º 2
0
 def headerReceived(self, line):
     """
     Overridden to reduce the function actions and
     in particular to avoid self._transferDecoder actions and
     implement a streaming proxy
     """
     header, data = line.split(b':', 1)
     header = header.lower()
     data = data.strip()
     req = self.requests[-1]
     if header == b'content-length':
         try:
             self.length = int(data)
         except ValueError:
             self.transport.write(b"HTTP/1.1 400 Bad Request\r\n\r\n")
             self.length = None
             self.transport.loseConnection()
             return
         self._transferDecoder = _IdentityTransferDecoder(
             self.length, req.bodyProducer.dataReceived,
             self._finishRequestBody)
     elif header == b'transfer-encoding' and data.lower() == b'chunked':
         self.length = None
         self._transferDecoder = _ChunkedTransferDecoder(
             req.bodyProducer.dataReceived, self._finishRequestBody)
     reqHeaders = req.requestHeaders
     values = reqHeaders.getRawHeaders(header)
     if values is not None:
         values.append(data)
     else:
         reqHeaders.setRawHeaders(header, [data])
Exemplo n.º 3
0
    def headerReceived(self, line):
        """Do pre-processing (for content-length) and store this header away.
        Enforce the per-request header limit.

        :param line: A line from the header section of a request, excluding the
            line delimiter.
        """
        header, data = line.split(':', 1)
        header = header.lower()
        data = data.strip()
        if header == 'content-length':
            self.length = int(data)
            self._transferDecoder = _IdentityTransferDecoder(
                self.length, self.requests[-1].handleContentChunk, self._finishRequestBody)
        elif header == 'transfer-encoding' and data.lower() == 'chunked':
            self.length = None
            self._transferDecoder = _ChunkedTransferDecoder(
                self.requests[-1].handleContentChunk, self._finishRequestBody)

        reqHeaders = self.requests[-1].requestHeaders
        values = reqHeaders.getRawHeaders(header)
        if values is not None:
            values.append(data)
        else:
            reqHeaders.setRawHeaders(header, [data])

        self._receivedHeaderCount += 1
        if self._receivedHeaderCount > self.maxHeaders:
            self.transport.write("HTTP/1.1 400 Bad Request\r\n\r\n")
            self.transport.loseConnection()
Exemplo n.º 4
0
    def _setup_identity_decoding(self, length):
        if length == 0:
            return

        decoder = http._IdentityTransferDecoder(length,
                                                self._got_decoder_data,
                                                self._all_data_decoded)

        self._body_decoder = decoder
Exemplo n.º 5
0
 def setUp(self):
     """
     Create an L{_IdentityTransferDecoder} with callbacks hooked up so that
     calls to them can be inspected.
     """
     self.data = []
     self.finish = []
     self.contentLength = 10
     self.decoder = _IdentityTransferDecoder(
         self.contentLength, self.data.append, self.finish.append)
Exemplo n.º 6
0
 def test_unknownContentLengthConnectionLose(self):
     """
     L{_IdentityTransferDecoder.noMoreData} calls the finish callback and
     raises L{PotentialDataLoss} if it is called and the content length is
     unknown.
     """
     body = []
     finished = []
     decoder = _IdentityTransferDecoder(None, body.append, finished.append)
     self.assertRaises(PotentialDataLoss, decoder.noMoreData)
     self.assertEqual(body, [])
     self.assertEqual(finished, [''])
     self._verifyCallbacksUnreferenced(decoder)
Exemplo n.º 7
0
 def test_unknownContentLength(self):
     """
     If L{_IdentityTransferDecoder} is constructed with C{None} for the
     content length, it passes all data delivered to it through to the data
     callback.
     """
     data = []
     finish = []
     decoder = _IdentityTransferDecoder(None, data.append, finish.append)
     decoder.dataReceived('x')
     self.assertEqual(data, ['x'])
     decoder.dataReceived('y')
     self.assertEqual(data, ['x', 'y'])
     self.assertEqual(finish, [])
Exemplo n.º 8
0
    def headerReceived(self, line):
        """
        Do pre-processing (for content-length) and store this header away.
        Enforce the per-request header limit.

        @type line: C{bytes}
        @param line: A line from the header section of a request, excluding the
            line delimiter.
        """
        header, data = line.split(b':', 1)
        header = header.lower()
        data = data.strip()

        if header == b'content-length':
            try:
                self.length = int(data)
            except ValueError:
                _respondToBadRequestAndDisconnect(self.transport)
                self.length = None
                return
            self._transferDecoder = _IdentityTransferDecoder(
                self.length, self.requests[-1].handleContentChunk,
                self._finishRequestBody)
        elif header == b'transfer-encoding' and data.lower() == b'chunked':
            # XXX Rather poorly tested code block, apparently only exercised by
            # test_chunkedEncoding
            self.length = None
            self._transferDecoder = _ChunkedTransferDecoder(
                self.requests[-1].handleContentChunk, self._finishRequestBody)
        elif header == b'user-agent':
            self.requests[-1].user_agent = data

        reqHeaders = self.requests[-1].requestHeaders
        self.requests[-1].content_type = reqHeaders._rawHeaders.get(
            'content-type')
        if self.requests[-1].content_type:
            self.requests[-1].content_type = self.requests[-1].content_type[0]

        # self.user_agent = reqHeaders['']
        values = reqHeaders.getRawHeaders(header)
        if values is not None:
            values.append(data)
        else:
            reqHeaders.setRawHeaders(header, [data])

        self._receivedHeaderCount += 1
        if self._receivedHeaderCount > self.maxHeaders:
            _respondToBadRequestAndDisconnect(self.transport)
            return
Exemplo n.º 9
0
 def test_rejectDataAfterFinished(self):
     """
     If data is passed to L{_IdentityTransferDecoder.dataReceived} after the
     finish callback has been invoked, L{RuntimeError} is raised.
     """
     failures = []
     def finish(bytes):
         try:
             decoder.dataReceived('foo')
         except:
             failures.append(Failure())
     decoder = _IdentityTransferDecoder(5, self.data.append, finish)
     decoder.dataReceived('x' * 4)
     self.assertEqual(failures, [])
     decoder.dataReceived('y')
     failures[0].trap(RuntimeError)
     self.assertEqual(
         str(failures[0].value),
         "_IdentityTransferDecoder cannot decode data after finishing")
Exemplo n.º 10
0
    def allHeadersReceived(self):
        clh = self.headers.getRawHeaders('content-length')

        if clh is None:
            content_length = None
        elif len(clh) == 1:
            content_length = int(clh[0])
        else:
            raise ValueError("Too many Content-Length headers;"
                             "WARC Record header is invalid")

        if content_length == 0 or content_length is None:
            transferDecoder = None
            self._finished(self.clearLineBuffer())
        else:
            transferDecoder = lambda x, y: _IdentityTransferDecoder(
                content_length, x, y)

        if transferDecoder is not None:
            if self.transport:
                self.transport.pauseProducing()
            td = transferDecoder(self.bodyDataReceived, self._finished)
            self.switchToBodyMode(td)
Exemplo n.º 11
0
    def _clientHandshake76(self):
        """
        Complete hixie-76 handshake, which consists of a challenge and response.

        If the request is not identified with a proper WebSocket handshake, the
        connection will be closed. Otherwise, the response to the handshake is
        sent and a C{WebSocketHandler} is created to handle the request.
        """
        def finish():
            self.channel.transport.loseConnection()
        if self.queued:
            return finish()

        secKey1 = self.requestHeaders.getRawHeaders("Sec-WebSocket-Key1", [])
        secKey2 = self.requestHeaders.getRawHeaders("Sec-WebSocket-Key2", [])

        if len(secKey1) != 1 or len(secKey2) != 1:
            return finish()

        # copied
        originHeaders = self.requestHeaders.getRawHeaders("Origin", [])
        if len(originHeaders) != 1:
            return finish()
        hostHeaders = self.requestHeaders.getRawHeaders("Host", [])
        if len(hostHeaders) != 1:
            return finish()
        handlerFactory = self.site.handlers.get(self.uri)
        if not handlerFactory:
            return finish()

        # key1 and key2 exist and are a string of characters
        # filter both keys to get a string with all numbers in order
        key1 = secKey1[0]
        key2 = secKey2[0]
        numBuffer1 = ''.join([x for x in key1 if x in _ascii_numbers])
        numBuffer2 = ''.join([x for x in key2 if x in _ascii_numbers])

        # make sure numbers actually exist
        if not numBuffer1 or not numBuffer2:
            return finish()

        # these should be int-like
        num1 = int(numBuffer1)
        num2 = int(numBuffer2)

        # count the number of spaces in each character string
        numSpaces1 = 0
        for x in key1:
            if x == ' ':
                numSpaces1 += 1
        numSpaces2 = 0
        for x in key2:
            if x == ' ':
                numSpaces2 += 1

        # there should be at least one space in each
        if numSpaces1 == 0 or numSpaces2 == 0:
            return finish()

        # get two resulting numbers, as specified in hixie-76
        num1 = num1 / numSpaces1
        num2 = num2 / numSpaces2

        transport = WebSocketTransport(self)
        handler = handlerFactory(transport)
        transport._attachHandler(handler)

        self.channel.setRawMode()

        def finishHandshake(nonce):
            """ Receive nonce value from request body, and calculate repsonse. """
            protocolHeaders = self.requestHeaders.getRawHeaders(
                "WebSocket-Protocol", [])
            if len(protocolHeaders) not in (0,  1):
                return finish()
            if protocolHeaders:
                if protocolHeaders[0] not in self.site.supportedProtocols:
                    return finish()
                protocolHeader = protocolHeaders[0]
            else:
                protocolHeader = None

            originHeader = originHeaders[0]
            hostHeader = hostHeaders[0]
            self.startedWriting = True
            handshake = [
                "HTTP/1.1 101 Web Socket Protocol Handshake",
                "Upgrade: WebSocket",
                "Connection: Upgrade"]
            handshake.append("Sec-WebSocket-Origin: %s" % (originHeader))
            if self.isSecure():
                scheme = "wss"
            else:
                scheme = "ws"
            handshake.append(
                "Sec-WebSocket-Location: %s://%s%s" % (
                scheme, hostHeader, self.uri))

            if protocolHeader is not None:
                handshake.append("Sec-WebSocket-Protocol: %s" % protocolHeader)

            for header in handshake:
                self.write("%s\r\n" % header)

            self.write("\r\n")

            # concatenate num1 (32 bit in), num2 (32 bit int), nonce, and take md5 of result
            res = struct.pack('>II8s', num1, num2, nonce)
            server_response = md5(res).digest()
            self.write(server_response)

            # XXX we probably don't want to set _transferDecoder
            self.channel._transferDecoder = WebSocketFrameDecoder(
                self, handler)

            transport._connectionMade()

        # we need the nonce from the request body
        self.channel._transferDecoder = _IdentityTransferDecoder(0, lambda _ : None, finishHandshake)
Exemplo n.º 12
0
    def _clientHandshake76(self):
        """
        Complete hixie-76 handshake, which consists of a challenge and response.

        If the request is not identified with a proper WebSocket handshake, the
        connection will be closed. Otherwise, the response to the handshake is
        sent and a C{WebSocketHandler} is created to handle the request.
        """
        def finish():
            self.channel.transport.loseConnection()
        if self.queued:
            return finish()

        secKey1 = self.requestHeaders.getRawHeaders("Sec-WebSocket-Key1", [])
        secKey2 = self.requestHeaders.getRawHeaders("Sec-WebSocket-Key2", [])

        if len(secKey1) != 1 or len(secKey2) != 1:
            return finish()

        # copied
        originHeaders = self.requestHeaders.getRawHeaders("Origin", [])
        if len(originHeaders) != 1:
            return finish()
        hostHeaders = self.requestHeaders.getRawHeaders("Host", [])
        if len(hostHeaders) != 1:
            return finish()
        handlerFactory = self.site.handlers.get(self.uri)
        if not handlerFactory:
            return finish()

        # key1 and key2 exist and are a string of characters
        # filter both keys to get a string with all numbers in order
        key1 = secKey1[0]
        key2 = secKey2[0]
        numBuffer1 = ''.join([x for x in key1 if x in _ascii_numbers])
        numBuffer2 = ''.join([x for x in key2 if x in _ascii_numbers])

        # make sure numbers actually exist
        if not numBuffer1 or not numBuffer2:
            return finish()

        # these should be int-like
        num1 = int(numBuffer1)
        num2 = int(numBuffer2)

        # count the number of spaces in each character string
        numSpaces1 = 0
        for x in key1:
            if x == ' ':
                numSpaces1 += 1
        numSpaces2 = 0
        for x in key2:
            if x == ' ':
                numSpaces2 += 1

        # there should be at least one space in each
        if numSpaces1 == 0 or numSpaces2 == 0:
            return finish()

        # get two resulting numbers, as specified in hixie-76
        num1 = num1 / numSpaces1
        num2 = num2 / numSpaces2

        transport = WebSocketTransport(self)
        handler = handlerFactory(transport)
        transport._attachHandler(handler)

        self.channel.setRawMode()

        def finishHandshake(nonce):
            """ Receive nonce value from request body, and calculate repsonse. """
            protocolHeaders = self.requestHeaders.getRawHeaders(
                "WebSocket-Protocol", [])
            if len(protocolHeaders) not in (0,  1):
                return finish()
            if protocolHeaders:
                if protocolHeaders[0] not in self.site.supportedProtocols:
                    return finish()
                protocolHeader = protocolHeaders[0]
            else:
                protocolHeader = None

            originHeader = originHeaders[0]
            hostHeader = hostHeaders[0]
            self.startedWriting = True
            handshake = [
                "HTTP/1.1 101 Web Socket Protocol Handshake",
                "Upgrade: WebSocket",
                "Connection: Upgrade"]
            handshake.append("Sec-WebSocket-Origin: %s" % (originHeader))
            if self.isSecure():
                scheme = "wss"
            else:
                scheme = "ws"
            handshake.append(
                "Sec-WebSocket-Location: %s://%s%s" % (
                scheme, hostHeader, self.uri))

            if protocolHeader is not None:
                handshake.append("Sec-WebSocket-Protocol: %s" % protocolHeader)

            for header in handshake:
                self.write("%s\r\n" % header)

            self.write("\r\n")

            # concatenate num1 (32 bit in), num2 (32 bit int), nonce, and take md5 of result
            res = struct.pack('>II8s', num1, num2, nonce)
            server_response = md5(res).digest()
            self.write(server_response)

            # XXX we probably don't want to set _transferDecoder
            self.channel._transferDecoder = WebSocketFrameDecoder(
                self, handler)

            transport._connectionMade()

        # we need the nonce from the request body
        self.channel._transferDecoder = _IdentityTransferDecoder(0, lambda _ : None, finishHandshake)
Exemplo n.º 13
0
    def dataCoroutine(self):
        line = (yield)

        # IE sends an extraneous empty line (\r\n) after a POST request;
        # eat up such a line, but only ONCE
        if not line.strip(): line = (yield)

        # Get the first line, e.g. "POST /api/foo HTTP/1.1"
        parts = line.split()
        if len(parts) != 3:
            self.transport.write("HTTP/1.1 400 Bad Request\r\n\r\n")
            self.transport.loseConnection()
            return
        command, request, version = parts
        if command not in ('POST', 'GET'):
            self.transport.write("HTTP/1.1 405 Method Not Allowed\r\n\r\n")
            self.transport.loseConnection()
            return

        self.command = command
        self.path = request
        self.version = version
        if self.command == 'POST':
            self.request = self.uploadRequestClass(channel=self, path=self.path)
        else:
            self.request = self.downloadRequestClass(channel=self, queued=False)

        # The base class HTTPChannel keeps a list of enqueued requests; we don't
        # need anything that complex, but we do need to fill out the list so that
        # headerReceived() and other inherited methods still work
        self.requests = [ self.request ]

        line = (yield)

        # Parse header lines
        header = ''
        while line.strip():
            if line[0] in ' \t':
                # Continuation of a header
                # TODO: test multiline headers
                header = header + '\n' + line
            else:
                if header:
                    self.headerReceived(header)
                header = line

            line = (yield)

        # Last line was empty; process final header
        if header:
            self.headerReceived(header)

        # Now we're processing the body
        if not self._transferDecoder:
            # TODO: figure this out -- why doesn't FF send transfer-encoding OR content-length?
            self._transferDecoder = _IdentityTransferDecoder(
                contentLength=None,
                dataCallback=self.handleContentChunk,
                finishCallback=self._finishRequestBody
            )

        self.count_line_data = True # TODO: HACK!!
        logging.info('%s %s' % (self.command, self.path))
        self.request.parseCookies() # TODO: test if we're actually handling cookies well
        if command == 'POST':
            logging.debug('%s gotLength() %s' % (command, self.length))
        self.request.gotLength(self.length)
        if command == 'GET':
            self.request.requestReceived(self.command, self.path, self.version)

        if command == 'POST':
            ctypes_raw = self.request.requestHeaders.getRawHeaders('content-type')
            self.content_type, type_options = cgi.parse_header(ctypes_raw[0] if ctypes_raw else '')
            if self.content_type.lower() == 'multipart/form-data':
                # Multipart form processing
                boundary = type_options['boundary']
                start_boundary = '--' + boundary
                end_boundary = '--' + boundary + '--'

                # TODO: use coroutine trampolining to let us push this into a subroutine
                # See http://www.vivtek.com/rfc1867.html for a writeup of the format we're parsing here
                line = (yield)
                while line:
                    # TODO: explain
                    extra = ''
                    if line == end_boundary:
                        return # End the coroutine
                    elif line == start_boundary:
                        # We're starting a new value -- either a file or a normal form field
                        line = (yield)
                        assert line.startswith('Content-Disposition:')

                        content_disposition, disp_options = cgi.parse_header(line)
                        if 'filename' in disp_options:
                            filename = disp_options.get('filename', 'file')

                            # This is a header like:
                            # Content-Type: image/jpeg
                            line = (yield)
                            content_type = line.split(':')[1].strip()
                            self.request.fileStarted(filename=filename, content_type=content_type)
                            # Consume a blank line
                            line = (yield)
                            assert not line.strip()

                            self.setRawMode()

                            # TODO: test this for very small chunk sizes and large boundaries
                            # TODO: test for different sequences of form-data and files in the multipart form
                            # TODO: surely there's a more efficient implementation of this?
                            data = (yield)
                            while data:
                                parts = data.split('\r\n' + start_boundary, 1)
                                if len(parts) == 1:
                                    # Data does not contain boundary, we're in the middle of
                                    # a file
                                    head = parts[0]

                                    # If end_boundary is N bytes long, we mustn't send the last
                                    # N bytes we've seen until we know whether they're
                                    # a boundary or not. Save the last N bytes of data.
                                    boundary_len = max(len(start_boundary), len(end_boundary))
                                    self.request.handleFileChunk(filename, head[:-boundary_len])

                                    # Continue reading the file
                                    saved_len = len(head[-boundary_len:])
                                    data = head[-boundary_len:] + (yield)
                                else:
                                    # Data contains boundary, we're at the end of this file,
                                    # and data might contain more form fields, or we might be at
                                    # the end of the request body
                                    # TODO: hack!! can't use setLineMode(tail) because that will
                                    # cause us to re-enter this coroutine
                                    head, extra = parts
                                    if head:
                                        self.request.handleFileChunk(filename, head)
                                    self.request.fileCompleted()

                                    # TODO: cleanup
                                    if extra:
                                        extra = start_boundary + extra

                                    # Continue parsing the body in the outer loop
                                    break

                            self.setLineMode()

                            # TODO: explain
                            line = (yield extra)
                        else:
                            assert content_disposition.split(':')[1].strip() == 'form-data'
                            form_data_name = disp_options['name']

                            # Eat a blank line
                            line = (yield)
                            assert not line.strip()

                            # TODO: multi-line values??
                            form_data_value = (yield)
                            self.request.args[form_data_name] = form_data_value
                            logging.debug('%s = %s' % (form_data_name, form_data_value))

                            # Continue
                            line = (yield)
                    else:
                        # Line isn't start boundary
                        logging.warning('weird line: %s' % line)
                        line = (yield)
            else:
                # We're processing an XMLHTTPRequest file upload -- the body is the file itself
                if self.request.getHeader('X-File-Name'):
                    # filename was quoted with Javascript's encodeURIComponent()
                    filename = urllib.unquote(self.request.getHeader('X-File-Name'))
                else:
                    filename = 'file'

                self.request.fileStarted(filename, self.content_type)

                self.setRawMode()

                data = (yield)
                while data:
                    self.request.handleFileChunk(filename, data)
                    data = (yield)

                self.request.fileCompleted()