def test_mock_httpresponse_stream(self):
# Mock out a HTTP Request that does enough to make it through urllib3's
# read() and close() calls, and also exhausts and underlying file
# object.
class MockHTTPRequest(object):
self.fp = None
def read(self, amt):
data = self.fp.read(amt)
if not data:
self.fp = None
return data
def close(self):
self.fp = None
bio = BytesIO(b'foo')
fp = MockHTTPRequest()
fp.fp = bio
resp = HTTPResponse(fp, preload_content=False)
stream = resp.stream(2)
self.assertEqual(next(stream), b'fo')
self.assertEqual(next(stream), b'o')
self.assertRaises(StopIteration, next, stream)
def test_empty_stream(self):
fp = BytesIO(b'')
resp = HTTPResponse(fp, preload_content=False)
stream = resp.stream(2, decode_content=False)
with pytest.raises(StopIteration):
next(stream)
def test_io(self):
import socket
try:
from http.client import HTTPResponse as OldHTTPResponse
except:
from httplib import HTTPResponse as OldHTTPResponse
fp = BytesIO(b'foo')
resp = HTTPResponse(fp, preload_content=False)
self.assertEqual(resp.closed, False)
self.assertEqual(resp.readable(), True)
self.assertEqual(resp.writable(), False)
self.assertRaises(IOError, resp.fileno)
resp.close()
self.assertEqual(resp.closed, True)
# Try closing with an `httplib.HTTPResponse`, because it has an
# `isclosed` method.
hlr = OldHTTPResponse(socket.socket())
resp2 = HTTPResponse(hlr, preload_content=False)
self.assertEqual(resp2.closed, False)
resp2.close()
self.assertEqual(resp2.closed, True)
#also try when only data is present.
resp3 = HTTPResponse('foodata')
self.assertRaises(IOError, resp3.fileno)
resp3._fp = 2
# A corner case where _fp is present but doesn't have `closed`,
# `isclosed`, or `fileno`. Unlikely, but possible.
self.assertEqual(resp3.closed, True)
self.assertRaises(IOError, resp3.fileno)
def test_reference_read(self):
fp = BytesIO(b'foo')
r = HTTPResponse(fp, preload_content=False)
assert r.read(1) == b'f'
assert r.read(2) == b'oo'
assert r.read() == b''
assert r.read() == b''
def test_streaming(self):
fp = BytesIO(b'foo')
resp = HTTPResponse(fp, preload_content=False)
stream = resp.stream(2, decode_content=False)
self.assertEqual(next(stream), b'fo')
self.assertEqual(next(stream), b'o')
self.assertRaises(StopIteration, next, stream)
def test_reference_read(self):
fp = BytesIO(b'foo')
r = HTTPResponse(fp, preload_content=False)
self.assertEqual(r.read(1), b'f')
self.assertEqual(r.read(2), b'oo')
self.assertEqual(r.read(), b'')
self.assertEqual(r.read(), b'')
def test_reference_read(self):
fp = BytesIO(b"foo")
r = HTTPResponse(fp, preload_content=False)
self.assertEqual(r.read(1), b"f")
self.assertEqual(r.read(2), b"oo")
self.assertEqual(r.read(), b"")
self.assertEqual(r.read(), b"")
def test_chunked_response_with_extensions(self):
stream = [b"foo", b"bar"]
fp = MockChunkedEncodingWithExtensions(stream)
r = httplib.HTTPResponse(MockSock)
r.fp = fp
r.chunked = True
r.chunk_left = None
resp = HTTPResponse(r, preload_content=False, headers={'transfer-encoding': 'chunked'})
assert stream == list(resp.stream())
def test_mock_transfer_encoding_chunked(self):
stream = [b"fo", b"o", b"bar"]
fp = MockChunkedEncodingResponse(stream)
r = httplib.HTTPResponse(MockSock)
r.fp = fp
resp = HTTPResponse(r, preload_content=False, headers={'transfer-encoding': 'chunked'})
for i, c in enumerate(resp.stream()):
assert c == stream[i]
def test_invalid_chunks(self):
stream = [b"foooo", b"bbbbaaaaar"]
fp = MockChunkedInvalidEncoding(stream)
r = httplib.HTTPResponse(MockSock)
r.fp = fp
r.chunked = True
r.chunk_left = None
resp = HTTPResponse(r, preload_content=False, headers={'transfer-encoding': 'chunked'})
self.assertRaises(ProtocolError, next, resp.read_chunked())
def test_mock_transfer_encoding_chunked_unlmtd_read(self):
stream = [b"foooo", b"bbbbaaaaar"]
fp = MockChunkedEncodingResponse(stream)
r = httplib.HTTPResponse(MockSock)
r.fp = fp
r.chunked = True
r.chunk_left = None
resp = HTTPResponse(r, preload_content=False, headers={'transfer-encoding': 'chunked'})
assert stream == list(resp.read_chunked())
def test_streaming(self):
fp = BytesIO(b'foo')
resp = HTTPResponse(fp, preload_content=False)
stream = resp.stream(2, decode_content=False)
assert next(stream) == b'fo'
assert next(stream) == b'o'
with pytest.raises(StopIteration):
next(stream)
def test_geturl_retries(self):
fp = BytesIO(b'')
resp = HTTPResponse(fp, request_url='http://example.com')
request_histories = [
RequestHistory(method='GET', url='http://example.com', error=None,
status=301, redirect_location='https://example.com/'),
RequestHistory(method='GET', url='https://example.com/', error=None,
status=301, redirect_location='https://www.example.com')]
retry = Retry(history=request_histories)
resp = HTTPResponse(fp, retries=retry)
assert resp.geturl() == 'https://www.example.com'
def test_chunked_decoding_deflate(self):
import zlib
data = zlib.compress(b"foo")
fp = BytesIO(data)
r = HTTPResponse(fp, headers={"content-encoding": "deflate"}, preload_content=False)
self.assertEqual(r.read(3), b"")
self.assertEqual(r.read(1), b"f")
self.assertEqual(r.read(2), b"oo")
def test_chunked_decoding_deflate(self):
import zlib
data = zlib.compress(b'foo')
fp = BytesIO(data)
r = HTTPResponse(fp, headers={'content-encoding': 'deflate'},
preload_content=False)
self.assertEqual(r.read(3), b'')
self.assertEqual(r.read(1), b'f')
self.assertEqual(r.read(2), b'oo')
def test_mock_transfer_encoding_chunked_custom_read(self):
stream = [b"foooo", b"bbbbaaaaar"]
fp = MockChunkedEncodingResponse(stream)
r = httplib.HTTPResponse(MockSock)
r.fp = fp
r.chunked = True
r.chunk_left = None
resp = HTTPResponse(r, preload_content=False, headers={'transfer-encoding': 'chunked'})
expected_response = [b'fo', b'oo', b'o', b'bb', b'bb', b'aa', b'aa', b'ar']
response = list(resp.read_chunked(2))
assert expected_response == response
def test_mock_transfer_encoding_chunked(self):
stream = [b"fo", b"o", b"bar"]
fp = MockChunkedEncodingResponse(stream)
r = httplib.HTTPResponse(MockSock)
r.fp = fp
resp = HTTPResponse(r, preload_content=False, headers={'transfer-encoding': 'chunked'})
i = 0
for c in resp.stream():
self.assertEqual(c, stream[i])
i += 1
def test_deflate_streaming(self):
import zlib
data = zlib.compress(b'foo')
fp = BytesIO(data)
resp = HTTPResponse(fp, headers={'content-encoding': 'deflate'},
preload_content=False)
stream = resp.stream(2)
self.assertEqual(next(stream), b'f')
self.assertEqual(next(stream), b'oo')
self.assertRaises(StopIteration, next, stream)
def test_deflate_streaming(self):
data = zlib.compress(b'foo')
fp = BytesIO(data)
resp = HTTPResponse(fp, headers={'content-encoding': 'deflate'},
preload_content=False)
stream = resp.stream(2)
assert next(stream) == b'f'
assert next(stream) == b'oo'
with pytest.raises(StopIteration):
next(stream)
def test_deflate_streaming_tell_intermediate_point(self):
# Ensure that ``tell()`` returns the correct number of bytes when
# part-way through streaming compressed content.
import zlib
NUMBER_OF_READS = 10
class MockCompressedDataReading(BytesIO):
"""
A ByteIO-like reader returning ``payload`` in ``NUMBER_OF_READS``
calls to ``read``.
"""
def __init__(self, payload, payload_part_size):
self.payloads = [
payload[i*payload_part_size:(i+1)*payload_part_size]
for i in range(NUMBER_OF_READS+1)]
assert b"".join(self.payloads) == payload
def read(self, _):
# Amount is unused.
if len(self.payloads) > 0:
return self.payloads.pop(0)
return b""
uncompressed_data = zlib.decompress(ZLIB_PAYLOAD)
payload_part_size = len(ZLIB_PAYLOAD) // NUMBER_OF_READS
fp = MockCompressedDataReading(ZLIB_PAYLOAD, payload_part_size)
resp = HTTPResponse(fp, headers={'content-encoding': 'deflate'},
preload_content=False)
stream = resp.stream()
parts_positions = [(part, resp.tell()) for part in stream]
end_of_stream = resp.tell()
with pytest.raises(StopIteration):
next(stream)
parts, positions = zip(*parts_positions)
# Check that the payload is equal to the uncompressed data
payload = b"".join(parts)
assert uncompressed_data == payload
# Check that the positions in the stream are correct
expected = [(i+1)*payload_part_size for i in range(NUMBER_OF_READS)]
assert expected == list(positions)
# Check that the end of the stream is in the correct place
assert len(ZLIB_PAYLOAD) == end_of_stream
def test_chunked_decoding_deflate2(self):
import zlib
compress = zlib.compressobj(6, zlib.DEFLATED, -zlib.MAX_WBITS)
data = compress.compress(b'foo')
data += compress.flush()
fp = BytesIO(data)
r = HTTPResponse(fp, headers={'content-encoding': 'deflate'},
preload_content=False)
self.assertEqual(r.read(1), b'')
self.assertEqual(r.read(1), b'f')
self.assertEqual(r.read(2), b'oo')
def test_chunked_decoding_gzip(self):
import zlib
compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
data = compress.compress(b"foo")
data += compress.flush()
fp = BytesIO(data)
r = HTTPResponse(fp, headers={"content-encoding": "gzip"}, preload_content=False)
self.assertEqual(r.read(11), b"")
self.assertEqual(r.read(1), b"f")
self.assertEqual(r.read(2), b"oo")
def test_chunked_decoding_brotli(self):
data = brotli.compress(b'foobarbaz')
fp = BytesIO(data)
r = HTTPResponse(
fp, headers={'content-encoding': 'br'}, preload_content=False)
ret = b''
for _ in range(100):
ret += r.read(1)
if r.closed:
break
assert ret == b'foobarbaz'
def test_deflate2_streaming(self):
compress = zlib.compressobj(6, zlib.DEFLATED, -zlib.MAX_WBITS)
data = compress.compress(b'foo')
data += compress.flush()
fp = BytesIO(data)
resp = HTTPResponse(fp, headers={'content-encoding': 'deflate'},
preload_content=False)
stream = resp.stream(2)
assert next(stream) == b'f'
assert next(stream) == b'oo'
with pytest.raises(StopIteration):
next(stream)
def test_mock_transfer_encoding_chunked_unlmtd_read(self):
stream = [b"foooo", b"bbbbaaaaar"]
fp = MockChunkedEncodingResponse(stream)
r = httplib.HTTPResponse(MockSock)
r.fp = fp
r.chunked = True
r.chunk_left = None
resp = HTTPResponse(r, preload_content=False, headers={"transfer-encoding": "chunked"})
if getattr(self, "assertListEqual", False):
self.assertListEqual(stream, list(resp.read_chunked()))
else:
for index, item in enumerate(resp.read_chunked()):
v = stream[index]
self.assertEqual(item, v)
def test_gzipped_streaming(self):
import zlib
compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
data = compress.compress(b"foo")
data += compress.flush()
fp = BytesIO(data)
resp = HTTPResponse(fp, headers={"content-encoding": "gzip"}, preload_content=False)
stream = resp.stream(2)
self.assertEqual(next(stream), b"f")
self.assertEqual(next(stream), b"oo")
self.assertRaises(StopIteration, next, stream)
def test_chunked_response_with_extensions(self):
stream = [b"foo", b"bar"]
fp = MockChunkedEncodingWithExtensions(stream)
r = httplib.HTTPResponse(MockSock)
r.fp = fp
r.chunked = True
r.chunk_left = None
resp = HTTPResponse(r, preload_content=False, headers={'transfer-encoding': 'chunked'})
if getattr(self, "assertListEqual", False):
self.assertListEqual(stream, list(resp.stream()))
else:
for index, item in enumerate(resp.stream()):
v = stream[index]
self.assertEqual(item, v)
def test_chunked_decoding_gzip(self):
compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
data = compress.compress(b'foo')
data += compress.flush()
fp = BytesIO(data)
r = HTTPResponse(fp, headers={'content-encoding': 'gzip'},
preload_content=False)
assert r.read(11) == b''
assert r.read(1) == b'f'
assert r.read(2) == b'oo'
assert r.read() == b''
assert r.read() == b''
def test_deflate2_streaming(self):
import zlib
compress = zlib.compressobj(6, zlib.DEFLATED, -zlib.MAX_WBITS)
data = compress.compress(b'foo')
data += compress.flush()
fp = BytesIO(data)
resp = HTTPResponse(fp, headers={'content-encoding': 'deflate'},
preload_content=False)
stream = resp.stream(2)
self.assertEqual(next(stream), b'f')
self.assertEqual(next(stream), b'oo')
self.assertRaises(StopIteration, next, stream)
def test_io_closed_consistently(self):
hlr = httplib.HTTPResponse(socket.socket())
hlr.fp = BytesIO(b'foo')
hlr.chunked = 0
hlr.length = 3
resp = HTTPResponse(hlr, preload_content=False)
self.assertEqual(resp.closed, False)
self.assertEqual(resp._fp.isclosed(), False)
self.assertEqual(is_fp_closed(resp._fp), False)
resp.read()
self.assertEqual(resp.closed, True)
self.assertEqual(resp._fp.isclosed(), True)
self.assertEqual(is_fp_closed(resp._fp), True)
def test_none(self):
r = HTTPResponse(None)
self.assertEqual(r.data, None)
def test_read_not_chunked_response_as_chunks(self):
fp = BytesIO(b'foo')
resp = HTTPResponse(fp, preload_content=False)
r = resp.read_chunked()
self.assertRaises(ResponseNotChunked, next, r)
def test_cache_content(self):
r = HTTPResponse("foo")
assert r.data == "foo"
assert r._body == "foo"
def test_body_blob(self):
resp = HTTPResponse(b'foo')
self.assertEqual(resp.data, b'foo')
self.assertTrue(resp.closed)
def test_getheader(self):
headers = {'host': 'example.com'}
r = HTTPResponse(headers=headers)
self.assertEqual(r.getheader('host'), 'example.com')
def test_empty_stream(self):
fp = BytesIO(b'')
resp = HTTPResponse(fp, preload_content=False)
stream = resp.stream(2, decode_content=False)
self.assertRaises(StopIteration, next, stream)
def test_decode_brotli_error(self):
fp = BytesIO(b"foo")
with pytest.raises(DecodeError):
HTTPResponse(fp, headers={"content-encoding": "br"})
def test_cache_content(self):
r = HTTPResponse('foo')
self.assertEqual(r.data, 'foo')
self.assertEqual(r._body, 'foo')
def test_decode_bad_data(self):
fp = BytesIO(b"\x00" * 10)
with pytest.raises(DecodeError):
HTTPResponse(fp, headers={"content-encoding": "deflate"})
def test__iter__(self, payload, expected_stream):
actual_stream = []
for chunk in HTTPResponse(BytesIO(payload), preload_content=False):
actual_stream.append(chunk)
assert actual_stream == expected_stream
def test_geturl(self):
fp = BytesIO(b"")
request_url = "https://example.com"
resp = HTTPResponse(fp, request_url=request_url)
assert resp.geturl() == request_url
def test_get_case_insensitive_headers(self):
headers = {"host": "example.com"}
r = HTTPResponse(headers=headers)
assert r.headers.get("host") == "example.com"
assert r.headers.get("Host") == "example.com"
def test_none(self):
r = HTTPResponse(None)
assert r.data is None
def test_default(self):
r = HTTPResponse()
self.assertEqual(r.data, None)
def test_length_after_read(self):
headers = {"content-length": "5"}
# Test no defined length
fp = BytesIO(b'12345')
resp = HTTPResponse(fp, preload_content=False)
resp.read()
self.assertEqual(resp.length_remaining, None)
# Test our update from content-length
fp = BytesIO(b'12345')
resp = HTTPResponse(fp, headers=headers, preload_content=False)
resp.read()
self.assertEqual(resp.length_remaining, 0)
# Test partial read
fp = BytesIO(b'12345')
resp = HTTPResponse(fp, headers=headers, preload_content=False)
data = resp.stream(2)
next(data)
self.assertEqual(resp.length_remaining, 3)
def test_default(self):
r = HTTPResponse()
assert r.data is None
def test_length_w_valid_header(self):
headers = {"content-length": "5"}
fp = BytesIO(b'12345')
resp = HTTPResponse(fp, headers=headers, preload_content=False)
self.assertEqual(resp.length_remaining, 5)
def test_length_no_header(self):
fp = BytesIO(b"12345")
resp = HTTPResponse(fp, preload_content=False)
assert resp.length_remaining is None
def test_length_no_header(self):
fp = BytesIO(b'12345')
resp = HTTPResponse(fp, preload_content=False)
self.assertEqual(resp.length_remaining, None)
def test_decode_brotli(self):
data = brotli.compress(b"foo")
fp = BytesIO(data)
r = HTTPResponse(fp, headers={"content-encoding": "br"})
assert r.data == b"foo"
def test_RemoteReader_fetch_multi(self, http_request):
test_finders = RemoteFinder.factory()
finder = test_finders[0]
startTime = 1496262000
endTime = 1496262060
# no path or bulk_query
reader = RemoteReader(finder, {})
self.assertEqual(reader.bulk_query, [])
result = reader.fetch_multi(startTime, endTime)
self.assertEqual(result, [])
self.assertEqual(http_request.call_count, 0)
# path
reader = RemoteReader(finder, {'intervals': [], 'path': 'a.b.c.d'})
data = [{
'start': startTime,
'step': 60,
'end': endTime,
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
'name': 'a.b.c.d'
}]
responseObject = HTTPResponse(body=StringIO(pickle.dumps(data)),
status=200)
http_request.return_value = responseObject
result = reader.fetch_multi(startTime, endTime)
expected_response = [{
'pathExpression': 'a.b.c.d',
'name': 'a.b.c.d',
'time_info': (1496262000, 1496262060, 60),
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
}]
self.assertEqual(result, expected_response)
self.assertEqual(http_request.call_args[0], (
'GET',
'http://127.0.0.1/render/',
))
self.assertEqual(
http_request.call_args[1], {
'fields': [
('format', 'pickle'),
('local', '1'),
('noCache', '1'),
('from', startTime),
('until', endTime),
('target', 'a.b.c.d'),
],
'headers':
None,
'timeout':
10,
})
# bulk_query & now
reader = RemoteReader(finder, {
'intervals': [],
'path': 'a.b.c.d'
},
bulk_query=['a.b.c.d'])
result = reader.fetch_multi(
startTime,
endTime,
now=endTime,
requestContext={'forwardHeaders': {
'Authorization': 'Basic xxxx'
}})
expected_response = [{
'pathExpression': 'a.b.c.d',
'name': 'a.b.c.d',
'time_info': (1496262000, 1496262060, 60),
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
}]
self.assertEqual(result, expected_response)
self.assertEqual(http_request.call_args[0], (
'GET',
'http://127.0.0.1/render/',
))
self.assertEqual(
http_request.call_args[1], {
'fields': [
('format', 'pickle'),
('local', '1'),
('noCache', '1'),
('from', startTime),
('until', endTime),
('target', 'a.b.c.d'),
('now', endTime),
],
'headers': {
'Authorization': 'Basic xxxx'
},
'timeout':
10,
})
# non-pickle response
responseObject = HTTPResponse(body='error', status=200)
http_request.return_value = responseObject
with self.assertRaisesRegexp(
Exception,
'Error decoding render response from http://[^ ]+: .+'):
reader.fetch(startTime, endTime)
# non-200 response
responseObject = HTTPResponse(body='error', status=500)
http_request.return_value = responseObject
with self.assertRaisesRegexp(Exception,
'Error response 500 from http://[^ ]+'):
reader.fetch(startTime, endTime)
# exception raised by request()
http_request.side_effect = Exception('error')
with self.assertRaisesRegexp(Exception,
'Error requesting http://[^ ]+: error'):
reader.fetch(startTime, endTime)
def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None):
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
try:
conn = self.get_connection(request.url, proxies)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies)
chunked = not (request.body is None or 'Content-Length' in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
timeout = TimeoutSauce(connect=connect, read=read)
except ValueError as e:
# this may raise a string formatting error.
err = ("Invalid timeout {}. Pass a (connect, read) "
"timeout tuple, or a single float to set "
"both timeouts to the same value".format(timeout))
raise ValueError(err)
elif isinstance(timeout, TimeoutSauce):
pass
else:
timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
if not chunked:
resp = conn.urlopen(
method=request.method,
url=url,
body=request.body,
headers=request.headers,
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=timeout
)
# Send the request.
else:
if hasattr(conn, 'proxy_pool'):
conn = conn.proxy_pool
low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT)
try:
low_conn.putrequest(request.method,
url,
skip_accept_encoding=True)
for header, value in request.headers.items():
low_conn.putheader(header, value)
low_conn.endheaders()
for i in request.body:
low_conn.send(hex(len(i))[2:].encode('utf-8'))
low_conn.send(b'\r\n')
low_conn.send(i)
low_conn.send(b'\r\n')
low_conn.send(b'0\r\n\r\n')
# Receive the response from the server
try:
# For Python 2.7, use buffering of HTTP responses
r = low_conn.getresponse(buffering=True)
except TypeError:
# For compatibility with Python 3.3+
r = low_conn.getresponse()
resp = HTTPResponse.from_httplib(
r,
pool=conn,
connection=low_conn,
preload_content=False,
decode_content=False
)
except:
# If we hit any problems here, clean up the connection.
# Then, reraise so that we can handle the actual exception.
low_conn.close()
raise
except (ProtocolError, socket.error) as err:
raise ConnectionError(err, request=request)
except MaxRetryError as e:
if isinstance(e.reason, ConnectTimeoutError):
# TODO: Remove this in 3.0.0: see #2811
if not isinstance(e.reason, NewConnectionError):
raise ConnectTimeout(e, request=request)
if isinstance(e.reason, ResponseError):
raise RetryError(e, request=request)
if isinstance(e.reason, _ProxyError):
raise ProxyError(e, request=request)
if isinstance(e.reason, _SSLError):
# This branch is for urllib3 v1.22 and later.
raise SSLError(e, request=request)
raise ConnectionError(e, request=request)
except ClosedPoolError as e:
raise ConnectionError(e, request=request)
except _ProxyError as e:
raise ProxyError(e)
except (_SSLError, _HTTPError) as e:
if isinstance(e, _SSLError):
# This branch is for urllib3 versions earlier than v1.22
raise SSLError(e, request=request)
elif isinstance(e, ReadTimeoutError):
raise ReadTimeout(e, request=request)
else:
raise
return self.build_response(request, resp)
def test_io(self):
import socket
fp = BytesIO(b'foo')
resp = HTTPResponse(fp, preload_content=False)
self.assertEqual(resp.closed, False)
self.assertEqual(resp.readable(), True)
self.assertEqual(resp.writable(), False)
self.assertRaises(IOError, resp.fileno)
resp.close()
self.assertEqual(resp.closed, True)
# Try closing with an `httplib.HTTPResponse`, because it has an
# `isclosed` method.
hlr = httplib.HTTPResponse(socket.socket())
resp2 = HTTPResponse(hlr, preload_content=False)
self.assertEqual(resp2.closed, False)
resp2.close()
self.assertEqual(resp2.closed, True)
#also try when only data is present.
resp3 = HTTPResponse('foodata')
self.assertRaises(IOError, resp3.fileno)
resp3._fp = 2
# A corner case where _fp is present but doesn't have `closed`,
# `isclosed`, or `fileno`. Unlikely, but possible.
self.assertEqual(resp3.closed, True)
self.assertRaises(IOError, resp3.fileno)
def test_getheader(self):
headers = {"host": "example.com"}
r = HTTPResponse(headers=headers)
assert r.getheader("host") == "example.com"
def test_io(self, sock):
fp = BytesIO(b"foo")
resp = HTTPResponse(fp, preload_content=False)
assert not resp.closed
assert resp.readable()
assert not resp.writable()
with pytest.raises(IOError):
resp.fileno()
resp.close()
assert resp.closed
# Try closing with an `httplib.HTTPResponse`, because it has an
# `isclosed` method.
try:
hlr = httplib.HTTPResponse(sock)
resp2 = HTTPResponse(hlr, preload_content=False)
assert not resp2.closed
resp2.close()
assert resp2.closed
finally:
hlr.close()
# also try when only data is present.
resp3 = HTTPResponse("foodata")
with pytest.raises(IOError):
resp3.fileno()
resp3._fp = 2
# A corner case where _fp is present but doesn't have `closed`,
# `isclosed`, or `fileno`. Unlikely, but possible.
assert resp3.closed
with pytest.raises(IOError):
resp3.fileno()
def test_length_after_read(self):
headers = {"content-length": "5"}
# Test no defined length
fp = BytesIO(b"12345")
resp = HTTPResponse(fp, preload_content=False)
resp.read()
assert resp.length_remaining is None
# Test our update from content-length
fp = BytesIO(b"12345")
resp = HTTPResponse(fp, headers=headers, preload_content=False)
resp.read()
assert resp.length_remaining == 0
# Test partial read
fp = BytesIO(b"12345")
resp = HTTPResponse(fp, headers=headers, preload_content=False)
data = resp.stream(2)
next(data)
assert resp.length_remaining == 3
def test_get_case_insensitive_headers(self):
headers = {'host': 'example.com'}
r = HTTPResponse(headers=headers)
self.assertEqual(r.headers.get('host'), 'example.com')
self.assertEqual(r.headers.get('Host'), 'example.com')
def test_length_w_valid_header(self):
headers = {"content-length": "5"}
fp = BytesIO(b"12345")
resp = HTTPResponse(fp, headers=headers, preload_content=False)
assert resp.length_remaining == 5
def test_read_not_chunked_response_as_chunks(self):
fp = BytesIO(b"foo")
resp = HTTPResponse(fp, preload_content=False)
r = resp.read_chunked()
with pytest.raises(ResponseNotChunked):
next(r)
def test_body_blob(self):
resp = HTTPResponse(b"foo")
assert resp.data == b"foo"
assert resp.closed