def headerReceived(self, line): """ Overridden to reduce the function actions and in particular to avoid self._transferDecoder actions and implement a streaming proxy """ header, data = line.split(b':', 1) header = header.lower() data = data.strip() req = self.requests[-1] if header == b'content-length': try: self.length = int(data) except ValueError: self.transport.write(b"HTTP/1.1 400 Bad Request\r\n\r\n") self.length = None self.transport.loseConnection() return self._transferDecoder = _IdentityTransferDecoder( self.length, req.bodyProducer.dataReceived, self._finishRequestBody) elif header == b'transfer-encoding' and data.lower() == b'chunked': self.length = None self._transferDecoder = _ChunkedTransferDecoder( req.bodyProducer.dataReceived, self._finishRequestBody) reqHeaders = req.requestHeaders values = reqHeaders.getRawHeaders(header) if values is not None: values.append(data) else: reqHeaders.setRawHeaders(header, [data])
def handleHeader(self, key, value): keyLower = key.lower() valueLower = value.lower() if keyLower == 'location': self.location = t2w.fix_link(self.obj, valueLower) return elif keyLower == 'transfer-encoding' and valueLower == 'chunked': self.decoderChunked = http._ChunkedTransferDecoder( self.handleResponsePart, self.handleResponseEnd) return elif keyLower == 'content-encoding' and valueLower == 'gzip': self.obj.server_response_is_gzip = True return elif keyLower == 'content-type' and re.search('text/html', valueLower): self.obj.contentNeedFix = True self.html = True elif keyLower == 'content-length': return elif keyLower == 'cache-control': return elif keyLower == 'connection' and valueLower == 'keep-alive': self.obj.server_supports_keepalive = True return proxy.ProxyClient.handleHeader(self, key, value)
def handleHeader(self, key, value): keyLower = key.lower() valueLower = value.lower() if keyLower == 'location': self.location = t2w.fix_link(self.obj, valueLower) return elif keyLower == 'transfer-encoding' and valueLower == 'chunked': self.decoderChunked = http._ChunkedTransferDecoder(self.handleResponsePart, self.handleResponseEnd) return elif keyLower == 'content-encoding' and valueLower == 'gzip': self.obj.server_response_is_gzip = True return elif keyLower == 'content-type' and re.search('text/html', valueLower): self.obj.contentNeedFix = True self.html = True elif keyLower == 'content-length': return elif keyLower == 'cache-control': return elif keyLower == 'connection' and valueLower == 'keep-alive': self.obj.server_supports_keepalive = True return proxy.ProxyClient.handleHeader(self, key, value)
def test_finishedConnectionLose(self): """ L{_ChunkedTransferDecoder.noMoreData} does not raise any exception if it is called after the terminal zero length chunk is received. """ parser = http._ChunkedTransferDecoder(None, lambda bytes: None) parser.dataReceived('0\r\n\r\n') parser.noMoreData()
def test_afterFinished(self): """ L{_ChunkedTransferDecoder.dataReceived} raises L{RuntimeError} if it is called after it has seen the last chunk. """ p = http._ChunkedTransferDecoder(None, lambda bytes: None) p.dataReceived('0\r\n\r\n') self.assertRaises(RuntimeError, p.dataReceived, 'hello')
def test_extra(self): """ L{_ChunkedTransferDecoder.dataReceived} passes any bytes which come after the terminating zero-length chunk to the completion callback. """ finished = [] p = http._ChunkedTransferDecoder(None, finished.append) p.dataReceived('0\r\n\r\nhello') self.assertEqual(finished, ['hello'])
def test_extensions(self): """ L{_ChunkedTransferDecoder.dataReceived} disregards chunk-extension fields. """ L = [] p = http._ChunkedTransferDecoder(L.append, None) p.dataReceived('3; x-foo=bar\r\nabc\r\n') self.assertEqual(L, ['abc'])
def test_newlines(self): """ L{_ChunkedTransferDecoder.dataReceived} doesn't treat CR LF pairs embedded in chunk bodies specially. """ L = [] p = http._ChunkedTransferDecoder(L.append, None) p.dataReceived('2\r\n\r\n\r\n') self.assertEqual(L, ['\r\n'])
def handleHeader(self, key, val): """ Handle header comes from tunnel/proxy """ self.request.responseHeaders.addRawHeader(key, val) if key.lower() == b'transfer-encoding' and val.lower() == b'chunked': logger.debug('%s: CHUNKED -> On' % self) self._responseDecoder = http._ChunkedTransferDecoder(self.handleChunk, self.chunkResponseEnded) self._chunkProcessing = True
def test_finish(self): """ L{_ChunkedTransferDecoder.dataReceived} interprets a zero-length chunk as the end of the chunked data stream and calls the completion callback. """ finished = [] p = http._ChunkedTransferDecoder(None, finished.append) p.dataReceived('0\r\n\r\n') self.assertEqual(finished, [''])
def test_decoding(self): """ L{_ChunkedTransferDecoder.dataReceived} decodes chunked-encoded data and passes the result to the specified callback. """ L = [] p = http._ChunkedTransferDecoder(L.append, None) p.dataReceived('3\r\nabc\r\n5\r\n12345\r\n') p.dataReceived('a\r\n0123456789\r\n') self.assertEqual(L, ['abc', '12345', '0123456789'])
def test_earlyConnectionLose(self): """ L{_ChunkedTransferDecoder.noMoreData} raises L{_DataLoss} if it is called and the end of the last trailer has not yet been received. """ parser = http._ChunkedTransferDecoder(None, lambda bytes: None) parser.dataReceived('0\r\n\r') exc = self.assertRaises(_DataLoss, parser.noMoreData) self.assertEqual( str(exc), "Chunked decoder in 'trailer' state, still expecting more data " "to get to finished state.")
def test_short(self): """ L{_ChunkedTransferDecoder.dataReceived} decodes chunks broken up and delivered in multiple calls. """ L = [] finished = [] p = http._ChunkedTransferDecoder(L.append, finished.append) for s in '3\r\nabc\r\n5\r\n12345\r\n0\r\n\r\n': p.dataReceived(s) self.assertEqual(L, ['a', 'b', 'c', '1', '2', '3', '4', '5']) self.assertEqual(finished, [''])
def test_reentrantFinishedNoMoreData(self): """ L{_ChunkedTransferDecoder.noMoreData} can be called from the finished callback without raising an exception. """ errors = [] successes = [] def finished(extra): try: parser.noMoreData() except: errors.append(Failure()) else: successes.append(True) parser = http._ChunkedTransferDecoder(None, finished) parser.dataReceived('0\r\n\r\n') self.assertEqual(errors, []) self.assertEqual(successes, [True])
def _setup_chunked_decoding(self): decoder = http._ChunkedTransferDecoder(self._got_decoder_data, self._all_data_decoded) self._body_decoder = decoder