def test_redirect_with_failed_tell(self):
"""Abort request if failed to get a position from tell()"""
class BadTellObject(io.BytesIO):
def tell(self):
raise IOError
body = BadTellObject(b"the data")
url = "/redirect?target=/successful_retry"
# httplib uses fileno if Content-Length isn't supplied,
# which is unsupported by BytesIO.
headers = {"Content-Length": "8"}
with HTTPConnectionPool(self.host, self.port, timeout=0.1) as pool:
try:
pool.urlopen("PUT", url, headers=headers, body=body)
self.fail("PUT successful despite failed rewind.")
except UnrewindableBodyError as e:
assert "Unable to record file position for" in str(e)
def test_defaults_are_applied(self):
"""Test that modifying the default socket options works."""
# This test needs to be here in order to be run. socket.create_connection actually tries to
# connect to the host provided so we need a dummyserver to be running.
pool = HTTPConnectionPool(self.host, self.port)
# Get the HTTPConnection instance
conn = pool._new_conn()
# Update the default socket options
conn.default_socket_options += [(socket.SOL_SOCKET,
socket.SO_KEEPALIVE, 1)]
s = conn._new_conn()
nagle_disabled = s.getsockopt(socket.IPPROTO_TCP,
socket.TCP_NODELAY) > 0
using_keepalive = s.getsockopt(socket.SOL_SOCKET,
socket.SO_KEEPALIVE) > 0
self.assertTrue(nagle_disabled)
self.assertTrue(using_keepalive)
def test_stream_keepalive(self):
x = 2
with HTTPConnectionPool(self.host, self.port) as pool:
for _ in range(x):
response = pool.request(
"GET",
"/chunked",
headers={"Connection": "keep-alive"},
preload_content=False,
retries=False,
)
for chunk in response.stream():
assert chunk == b"123"
assert pool.num_connections == 1
assert pool.num_requests == x
def test_socket_options(self):
"""Test that connections accept socket options."""
# This test needs to be here in order to be run. socket.create_connection actually tries to
# connect to the host provided so we need a dummyserver to be running.
with HTTPConnectionPool(
self.host,
self.port,
socket_options=[(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)],
) as pool:
s = pool._new_conn()._new_conn() # Get the socket
try:
using_keepalive = (
s.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) > 0
)
assert using_keepalive
finally:
s.close()
def test_multi_setcookie(self):
def multicookie_response_handler(listener):
sock = listener.accept()[0]
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf += sock.recv(65536)
sock.send(b'HTTP/1.1 200 OK\r\n'
b'Set-Cookie: foo=1\r\n'
b'Set-Cookie: bar=1\r\n'
b'\r\n')
self._start_server(multicookie_response_handler)
pool = HTTPConnectionPool(self.host, self.port)
r = pool.request('GET', '/', retries=0)
self.assertEquals(r.headers, {'set-cookie': 'foo=1, bar=1'})
def _test_body(self, data):
self.start_chunked_handler()
pool = HTTPConnectionPool(self.host, self.port, retries=False)
r = pool.urlopen('GET', '/', data, chunked=True)
header, body = self.buffer.split(b'\r\n\r\n', 1)
self.assertTrue(b'Transfer-Encoding: chunked' in header.split(b'\r\n'))
if data:
bdata = data if isinstance(
data, six.binary_type) else data.encode('utf-8')
self.assertTrue(b'\r\n' + bdata + b'\r\n' in body)
self.assertTrue(body.endswith(b'\r\n0\r\n\r\n'))
len_str = body.split(b'\r\n', 1)[0]
stated_len = int(len_str, 16)
self.assertEqual(stated_len, len(bdata))
else:
self.assertEqual(body, b'0\r\n\r\n')
def test_chunks(self):
self.start_chunked_handler()
chunks = ['foo', 'bar', '', 'bazzzzzzzzzzzzzzzzzzzzzz']
pool = HTTPConnectionPool(self.host, self.port, retries=False)
r = pool.urlopen('GET',
'/',
chunks,
headers=dict(DNT='1'),
chunked=True)
self.assertTrue(b'Transfer-Encoding' in self.buffer)
body = self.buffer.split(b'\r\n\r\n', 1)[1]
lines = body.split(b'\r\n')
# Empty chunks should have been skipped, as this could not be distinguished
# from terminating the transmission
for i, chunk in enumerate([c for c in chunks if c]):
self.assertEqual(lines[i * 2], hex(len(chunk))[2:].encode('utf-8'))
self.assertEqual(lines[i * 2 + 1], chunk.encode('utf-8'))
def _test_body(self, data):
self.start_chunked_handler()
with HTTPConnectionPool(self.host, self.port, retries=False) as pool:
pool.urlopen("GET", "/", data, chunked=True)
header, body = self.buffer.split(b"\r\n\r\n", 1)
assert b"Transfer-Encoding: chunked" in header.split(b"\r\n")
if data:
bdata = data if isinstance(data,
bytes) else data.encode("utf-8")
assert b"\r\n" + bdata + b"\r\n" in body
assert body.endswith(b"\r\n0\r\n\r\n")
len_str = body.split(b"\r\n", 1)[0]
stated_len = int(len_str, 16)
assert stated_len == len(bdata)
else:
assert body == b"0\r\n\r\n"
def test_chunks(self):
self.start_chunked_handler()
chunks = ["foo", "bar", "", "bazzzzzzzzzzzzzzzzzzzzzz"]
with HTTPConnectionPool(self.host, self.port, retries=False) as pool:
pool.urlopen("GET",
"/",
chunks,
headers=dict(DNT="1"),
chunked=True)
assert b"Transfer-Encoding" in self.buffer
body = self.buffer.split(b"\r\n\r\n", 1)[1]
lines = body.split(b"\r\n")
# Empty chunks should have been skipped, as this could not be distinguished
# from terminating the transmission
for i, chunk in enumerate([c for c in chunks if c]):
assert lines[i * 2] == hex(len(chunk))[2:].encode("utf-8")
assert lines[i * 2 + 1] == chunk.encode("utf-8")
def _make_pool(self, profile):
if profile.secure:
from ssl import CERT_NONE, CERT_REQUIRED
from certifi import where as cert_where
self.http_pool = HTTPSConnectionPool(
host=profile.host,
port=profile.port_number,
maxsize=1,
block=True,
cert_reqs=CERT_REQUIRED if profile.verify else CERT_NONE,
ca_certs=cert_where())
else:
self.http_pool = HTTPConnectionPool(
host=profile.host,
port=profile.port_number,
maxsize=1,
block=True,
)
def __http_pool(self):
"""
Create HTTP connection pool
:raise HttpRequestError
:return: urllib3.HTTPConnectionPool
"""
try:
pool = HTTPConnectionPool(self.__cfg.host,
port=self.__cfg.port,
maxsize=self.__cfg.threads,
timeout=self.__cfg.timeout,
block=True)
if self._HTTP_DBG_LEVEL <= self.__debug.level:
self.__debug.debug_connection_pool('http_pool_start', pool)
return pool
except Exception as e:
raise HttpRequestError(e)
def test_api_convert_sample_01(self):
"""
Multiple API convert calls to isolated HTTP Server process
"""
pool = HTTPConnectionPool('localhost', port=self.port, maxsize=1)
def proc():
response = pool.request(
'POST',
'/api/convert',
headers={'Content-Type': 'application/json'},
body=json.dumps({'text': data}))
self.assertEqual(response.status, 200)
with open(self.resources_path + 'oracle.txt', 'r',
encoding='utf-8') as f:
data = f.read()
print(timeit.timeit(proc, number=100))
def test_connection_timeout(self):
timed_out = Event()
def socket_handler(listener):
timed_out.wait()
sock = listener.accept()[0]
sock.close()
self._start_server(socket_handler)
pool = HTTPConnectionPool(self.host, self.port, timeout=0.001)
self.assertRaises(ReadTimeoutError,
pool.request,
'GET',
'/',
retries=0)
timed_out.set()
def test_preserve_user_agent_header(self) -> None:
self.start_chunked_handler()
chunks = [b"foo", b"bar", b"", b"bazzzzzzzzzzzzzzzzzzzzzz"]
with HTTPConnectionPool(self.host, self.port, retries=False) as pool:
pool.urlopen(
"GET",
"/",
body=chunks,
headers={"user-Agent": "test-agent"},
chunked=True,
)
ua_headers = self._get_header_lines(b"user-agent")
# Validate that there is only one User-Agent header.
assert len(ua_headers) == 1
# Validate that the existing User-Agent header is the one that was
# provided.
assert ua_headers[0] == b"user-agent: test-agent"
def _test_broken_header_parsing(self, headers):
self.start_response_handler((
b'HTTP/1.1 200 OK\r\n'
b'Content-Length: 0\r\n'
b'Content-type: text/plain\r\n'
) + b'\r\n'.join(headers) + b'\r\n'
)
pool = HTTPConnectionPool(self.host, self.port, retries=False)
with LogRecorder() as logs:
pool.request('GET', '/')
for record in logs:
if 'Failed to parse headers' in record.msg and \
pool._absolute_url('/') == record.args[0]:
return
self.fail('Missing log about unparsed headers')
def test_conn_closed(self):
block_event = Event()
self.start_basic_handler(block_send=block_event, num=1)
with HTTPConnectionPool(
self.host, self.port, timeout=SHORT_TIMEOUT, retries=False
) as pool:
conn = pool._get_conn()
pool._put_conn(conn)
try:
with pytest.raises(ReadTimeoutError):
pool.urlopen("GET", "/")
if conn.sock:
with pytest.raises(socket.error):
conn.sock.recv(1024)
finally:
pool._put_conn(conn)
block_event.set()
def test_preserve_transfer_encoding_header(self) -> None:
self.start_chunked_handler()
chunks = [b"foo", b"bar", b"", b"bazzzzzzzzzzzzzzzzzzzzzz"]
with HTTPConnectionPool(self.host, self.port, retries=False) as pool:
pool.urlopen(
"GET",
"/",
body=chunks,
headers={"transfer-Encoding": "test-transfer-encoding"},
chunked=True,
)
te_headers = self._get_header_lines(b"transfer-encoding")
# Validate that there is only one Transfer-Encoding header.
assert len(te_headers) == 1
# Validate that the existing Transfer-Encoding header is the one that
# was provided.
assert te_headers[
0] == b"transfer-encoding: test-transfer-encoding"
def test_read_timeout_dont_retry_method_not_in_whitelist(self):
timed_out = Event()
def socket_handler(listener):
sock = listener.accept()[0]
sock.recv(65536)
timed_out.wait()
sock.close()
self._start_server(socket_handler)
pool = HTTPConnectionPool(self.host,
self.port,
timeout=0.001,
retries=True)
try:
self.assertRaises(ReadTimeoutError, pool.request, 'POST', '/')
finally:
timed_out.set()
def test_disabled_retry(self):
""" Disabled retries should disable redirect handling. """
r = self.pool.request("GET",
"/redirect",
fields={"target": "/"},
retries=False)
assert r.status == 303
r = self.pool.request("GET",
"/redirect",
fields={"target": "/"},
retries=Retry(redirect=False))
assert r.status == 303
pool = HTTPConnectionPool("thishostdoesnotexist.invalid",
self.port,
timeout=0.001)
with pytest.raises(NewConnectionError):
pool.request("GET", "/test", retries=False)
def test_connection_read_timeout(self):
timed_out = Event()
def socket_handler(listener):
sock = listener.accept()[0]
while not sock.recv(65536).endswith(b'\r\n\r\n'):
pass
timed_out.wait()
sock.close()
self._start_server(socket_handler)
http = HTTPConnectionPool(self.host, self.port, timeout=0.001, retries=False, maxsize=3, block=True)
try:
self.assertRaises(ReadTimeoutError, http.request, 'GET', '/', release_conn=False)
finally:
timed_out.set()
self.assertEqual(http.pool.qsize(), http.pool.maxsize)
def getting_citation_by_arxiv_id(arxiv_id):
query = arxiv_id.split('v')[0]
url = 'https://scholar.google.com/scholar?hl=en&as_sdt=0%2C47&q=' + query + '&btnG='
pool = HTTPConnectionPool(url)
http = urllib3.PoolManager()
r = http.request('GET', url)
html = BeautifulSoup(r.data)
if 'Why did this happen' in html.getText():
raise Exception('Forbidden')
entries = html.find_all('div', class_="gs_ri")
for entry in entries:
scraped_title = entry.h3.getText()
text = entry.getText()
if 'Cited' in text:
text = text.split()
indx = text.index('Cited')
return int(text[indx + 2])
return 0
def test_lazy_load_twice(self):
# This test is sad and confusing. Need to figure out what's
# going on with partial reads and socket reuse.
pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1, timeout=2)
payload_size = 1024 * 2
first_chunk = 512
boundary = 'foo'
req_data = {'count': 'a' * payload_size}
resp_data = encode_multipart_formdata(req_data, boundary=boundary)[0]
req2_data = {'count': 'b' * payload_size}
resp2_data = encode_multipart_formdata(req2_data, boundary=boundary)[0]
r1 = pool.request('POST', '/echo',
fields=req_data,
multipart_boundary=boundary,
preload_content=False)
self.assertEqual(r1.read(first_chunk), resp_data[:first_chunk])
try:
r2 = pool.request('POST', '/echo', fields=req2_data, multipart_boundary=boundary,
preload_content=False, pool_timeout=0.001)
# This branch should generally bail here, but maybe someday it will
# work? Perhaps by some sort of magic. Consider it a TODO.
self.assertEqual(r2.read(first_chunk), resp2_data[:first_chunk])
self.assertEqual(r1.read(), resp_data[first_chunk:])
self.assertEqual(r2.read(), resp2_data[first_chunk:])
self.assertEqual(pool.num_requests, 2)
except EmptyPoolError:
self.assertEqual(r1.read(), resp_data[first_chunk:])
self.assertEqual(pool.num_requests, 1)
self.assertEqual(pool.num_connections, 1)
def test_retry_weird_http_version(self):
""" Retry class should handle httplib.BadStatusLine errors properly """
def socket_handler(listener):
sock = listener.accept()[0]
# First request.
# Pause before responding so the first request times out.
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf += sock.recv(65536)
# send unknown http protocol
body = "bad http 0.5 response"
sock.send(('HTTP/0.5 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Content-Length: %d\r\n'
'\r\n'
'%s' % (len(body), body)).encode('utf-8'))
sock.close()
# Second request.
sock = listener.accept()[0]
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf += sock.recv(65536)
# Now respond immediately.
sock.send(('HTTP/1.1 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Content-Length: %d\r\n'
'\r\n'
'foo' % (len('foo'))).encode('utf-8'))
sock.close() # Close the socket.
self._start_server(socket_handler)
pool = HTTPConnectionPool(self.host, self.port)
self.addCleanup(pool.close)
retry = Retry(read=1)
response = pool.request('GET', '/', retries=retry)
self.assertEqual(response.status, 200)
self.assertEqual(response.data, b'foo')
