def headerReceived(self, line):
"""
Overridden to reduce the function actions and
in particular to avoid self._transferDecoder actions and
implement a streaming proxy
"""
header, data = line.split(b':', 1)
header = header.lower()
data = data.strip()
req = self.requests[-1]
if header == b'content-length':
try:
self.length = int(data)
except ValueError:
self.transport.write(b"HTTP/1.1 400 Bad Request\r\n\r\n")
self.length = None
self.transport.loseConnection()
return
self._transferDecoder = _IdentityTransferDecoder(
self.length, req.bodyProducer.dataReceived,
self._finishRequestBody)
elif header == b'transfer-encoding' and data.lower() == b'chunked':
self.length = None
self._transferDecoder = _ChunkedTransferDecoder(
req.bodyProducer.dataReceived, self._finishRequestBody)
reqHeaders = req.requestHeaders
values = reqHeaders.getRawHeaders(header)
if values is not None:
values.append(data)
else:
reqHeaders.setRawHeaders(header, [data])
def headerReceived(self, line):
"""
Overridden to reduce the function actions and
in particular to avoid self._transferDecoder actions and
implement a streaming proxy
"""
header, data = line.split(b':', 1)
header = header.lower()
data = data.strip()
req = self.requests[-1]
if header == b'content-length':
try:
self.length = int(data)
except ValueError:
self.transport.write(b"HTTP/1.1 400 Bad Request\r\n\r\n")
self.length = None
self.transport.loseConnection()
return
self._transferDecoder = _IdentityTransferDecoder(
self.length, req.bodyProducer.dataReceived, self._finishRequestBody)
elif header == b'transfer-encoding' and data.lower() == b'chunked':
self.length = None
self._transferDecoder = _ChunkedTransferDecoder(
req.bodyProducer.dataReceived, self._finishRequestBody)
reqHeaders = req.requestHeaders
values = reqHeaders.getRawHeaders(header)
if values is not None:
values.append(data)
else:
reqHeaders.setRawHeaders(header, [data])
def handleHeader(self, key, value):
keyLower = key.lower()
valueLower = value.lower()
if keyLower == 'location':
self.location = t2w.fix_link(self.obj, valueLower)
return
elif keyLower == 'transfer-encoding' and valueLower == 'chunked':
self.decoderChunked = http._ChunkedTransferDecoder(
self.handleResponsePart, self.handleResponseEnd)
return
elif keyLower == 'content-encoding' and valueLower == 'gzip':
self.obj.server_response_is_gzip = True
return
elif keyLower == 'content-type' and re.search('text/html', valueLower):
self.obj.contentNeedFix = True
self.html = True
elif keyLower == 'content-length':
return
elif keyLower == 'cache-control':
return
elif keyLower == 'connection' and valueLower == 'keep-alive':
self.obj.server_supports_keepalive = True
return
proxy.ProxyClient.handleHeader(self, key, value)
def handleHeader(self, key, value):
keyLower = key.lower()
valueLower = value.lower()
if keyLower == 'location':
self.location = t2w.fix_link(self.obj, valueLower)
return
elif keyLower == 'transfer-encoding' and valueLower == 'chunked':
self.decoderChunked = http._ChunkedTransferDecoder(self.handleResponsePart, self.handleResponseEnd)
return
elif keyLower == 'content-encoding' and valueLower == 'gzip':
self.obj.server_response_is_gzip = True
return
elif keyLower == 'content-type' and re.search('text/html', valueLower):
self.obj.contentNeedFix = True
self.html = True
elif keyLower == 'content-length':
return
elif keyLower == 'cache-control':
return
elif keyLower == 'connection' and valueLower == 'keep-alive':
self.obj.server_supports_keepalive = True
return
proxy.ProxyClient.handleHeader(self, key, value)
def test_finishedConnectionLose(self):
"""
L{_ChunkedTransferDecoder.noMoreData} does not raise any exception if
it is called after the terminal zero length chunk is received.
"""
parser = http._ChunkedTransferDecoder(None, lambda bytes: None)
parser.dataReceived('0\r\n\r\n')
parser.noMoreData()
def test_afterFinished(self):
"""
L{_ChunkedTransferDecoder.dataReceived} raises L{RuntimeError} if it
is called after it has seen the last chunk.
"""
p = http._ChunkedTransferDecoder(None, lambda bytes: None)
p.dataReceived('0\r\n\r\n')
self.assertRaises(RuntimeError, p.dataReceived, 'hello')
def test_extra(self):
"""
L{_ChunkedTransferDecoder.dataReceived} passes any bytes which come
after the terminating zero-length chunk to the completion callback.
"""
finished = []
p = http._ChunkedTransferDecoder(None, finished.append)
p.dataReceived('0\r\n\r\nhello')
self.assertEqual(finished, ['hello'])
def test_extensions(self):
"""
L{_ChunkedTransferDecoder.dataReceived} disregards chunk-extension
fields.
"""
L = []
p = http._ChunkedTransferDecoder(L.append, None)
p.dataReceived('3; x-foo=bar\r\nabc\r\n')
self.assertEqual(L, ['abc'])
def test_newlines(self):
"""
L{_ChunkedTransferDecoder.dataReceived} doesn't treat CR LF pairs
embedded in chunk bodies specially.
"""
L = []
p = http._ChunkedTransferDecoder(L.append, None)
p.dataReceived('2\r\n\r\n\r\n')
self.assertEqual(L, ['\r\n'])
def handleHeader(self, key, val):
"""
Handle header comes from tunnel/proxy
"""
self.request.responseHeaders.addRawHeader(key, val)
if key.lower() == b'transfer-encoding' and val.lower() == b'chunked':
logger.debug('%s: CHUNKED -> On' % self)
self._responseDecoder = http._ChunkedTransferDecoder(self.handleChunk, self.chunkResponseEnded)
self._chunkProcessing = True
def test_finish(self):
"""
L{_ChunkedTransferDecoder.dataReceived} interprets a zero-length
chunk as the end of the chunked data stream and calls the completion
callback.
"""
finished = []
p = http._ChunkedTransferDecoder(None, finished.append)
p.dataReceived('0\r\n\r\n')
self.assertEqual(finished, [''])
def test_decoding(self):
"""
L{_ChunkedTransferDecoder.dataReceived} decodes chunked-encoded data
and passes the result to the specified callback.
"""
L = []
p = http._ChunkedTransferDecoder(L.append, None)
p.dataReceived('3\r\nabc\r\n5\r\n12345\r\n')
p.dataReceived('a\r\n0123456789\r\n')
self.assertEqual(L, ['abc', '12345', '0123456789'])
def test_earlyConnectionLose(self):
"""
L{_ChunkedTransferDecoder.noMoreData} raises L{_DataLoss} if it is
called and the end of the last trailer has not yet been received.
"""
parser = http._ChunkedTransferDecoder(None, lambda bytes: None)
parser.dataReceived('0\r\n\r')
exc = self.assertRaises(_DataLoss, parser.noMoreData)
self.assertEqual(
str(exc),
"Chunked decoder in 'trailer' state, still expecting more data "
"to get to finished state.")
def test_short(self):
"""
L{_ChunkedTransferDecoder.dataReceived} decodes chunks broken up and
delivered in multiple calls.
"""
L = []
finished = []
p = http._ChunkedTransferDecoder(L.append, finished.append)
for s in '3\r\nabc\r\n5\r\n12345\r\n0\r\n\r\n':
p.dataReceived(s)
self.assertEqual(L, ['a', 'b', 'c', '1', '2', '3', '4', '5'])
self.assertEqual(finished, [''])
def test_reentrantFinishedNoMoreData(self):
"""
L{_ChunkedTransferDecoder.noMoreData} can be called from the finished
callback without raising an exception.
"""
errors = []
successes = []
def finished(extra):
try:
parser.noMoreData()
except:
errors.append(Failure())
else:
successes.append(True)
parser = http._ChunkedTransferDecoder(None, finished)
parser.dataReceived('0\r\n\r\n')
self.assertEqual(errors, [])
self.assertEqual(successes, [True])
def _setup_chunked_decoding(self):
decoder = http._ChunkedTransferDecoder(self._got_decoder_data,
self._all_data_decoded)
self._body_decoder = decoder