def test_pools_keyed_with_from_host(self):
"""Assert pools are still keyed correctly with connection_from_host."""
ssl_kw = {
'key_file': '/root/totally_legit.key',
'cert_file': '/root/totally_legit.crt',
'cert_reqs': 'CERT_REQUIRED',
'ca_certs': '/root/path_to_pem',
'ssl_version': 'SSLv23_METHOD',
}
p = PoolManager(5, **ssl_kw)
conns = []
conns.append(
p.connection_from_host('example.com', 443, scheme='https')
)
for k in ssl_kw:
p.connection_pool_kw[k] = 'newval'
conns.append(
p.connection_from_host('example.com', 443, scheme='https')
)
assert all(
x is not y
for i, x in enumerate(conns)
for j, y in enumerate(conns)
if i != j
)
def test_redirect_cross_host_set_removed_headers(self):
http = PoolManager()
self.addCleanup(http.clear)
r = http.request('GET', '%s/redirect' % self.base_url,
fields={'target': '%s/headers' % self.base_url_alt},
headers={'X-API-Secret': 'foo',
'Authorization': 'bar'},
retries=Retry(remove_headers_on_redirect=['X-API-Secret']))
self.assertEqual(r.status, 200)
data = json.loads(r.data.decode('utf-8'))
self.assertNotIn('X-API-Secret', data)
self.assertEqual(data['Authorization'], 'bar')
r = http.request('GET', '%s/redirect' % self.base_url,
fields={'target': '%s/headers' % self.base_url_alt},
headers={'x-api-secret': 'foo',
'authorization': 'bar'},
retries=Retry(remove_headers_on_redirect=['X-API-Secret']))
self.assertEqual(r.status, 200)
data = json.loads(r.data.decode('utf-8'))
self.assertNotIn('x-api-secret', data)
self.assertNotIn('X-API-Secret', data)
self.assertEqual(data['Authorization'], 'bar')
def test_merge_pool_kwargs_none(self):
"""Assert false-y values to _merge_pool_kwargs result in defaults"""
p = PoolManager(strict=True)
merged = p._merge_pool_kwargs({})
assert p.connection_pool_kw == merged
merged = p._merge_pool_kwargs(None)
assert p.connection_pool_kw == merged
def test_request_survives_missing_port_number(self):
# Can a URL that lacks an explicit port like ':80' succeed, or
# will all such URLs fail with an error?
def server(listener):
sock = listener.accept()[0]
read_request(sock)
sock.send('HTTP/1.1 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Content-Length: 8\r\n'
'\r\n'
'Inspire.')
sock.close()
# By globally adjusting `port_by_scheme` we pretend for a moment
# that HTTP's default port is not 80, but is the port at which
# our test server happens to be listening.
p = PoolManager()
host, port = start_server(server)
port_by_scheme['http'] = port
try:
response = p.request('GET', 'http://%s/' % host, retries=0)
finally:
port_by_scheme['http'] = 80
self.assertEqual(response.status, 200)
self.assertEqual(response.data, 'Inspire.')
def test_raise_on_status(self):
http = PoolManager()
self.addCleanup(http.clear)
try:
# the default is to raise
r = http.request('GET', '%s/status' % self.base_url,
fields={'status': '500 Internal Server Error'},
retries=Retry(total=1, status_forcelist=range(500, 600)))
self.fail("Failed to raise MaxRetryError exception, returned %r" % r.status)
except MaxRetryError:
pass
try:
# raise explicitly
r = http.request('GET', '%s/status' % self.base_url,
fields={'status': '500 Internal Server Error'},
retries=Retry(total=1,
status_forcelist=range(500, 600),
raise_on_status=True))
self.fail("Failed to raise MaxRetryError exception, returned %r" % r.status)
except MaxRetryError:
pass
# don't raise
r = http.request('GET', '%s/status' % self.base_url,
fields={'status': '500 Internal Server Error'},
retries=Retry(total=1,
status_forcelist=range(500, 600),
raise_on_status=False))
self.assertEqual(r.status, 500)
def test_assert_hostname_and_fingerprint_flag(self):
"""Assert that pool manager can accept hostname and fingerprint flags."""
fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:BF:93:CF:F9:71:CC:07:7D:0A'
p = PoolManager(assert_hostname=True, assert_fingerprint=fingerprint)
pool = p.connection_from_url('https://example.com/')
assert 1 == len(p.pools)
assert pool.assert_hostname
assert fingerprint == pool.assert_fingerprint
def test_raise_on_redirect(self):
http = PoolManager()
r = http.request('GET', '%s/redirect' % self.base_url,
fields={'target': '%s/redirect?target=%s/' % (self.base_url, self.base_url)},
retries=Retry(total=None, redirect=1, raise_on_redirect=False))
self.assertEqual(r.status, 303)
def test_http_connection_from_url_case_insensitive(self):
"""Assert scheme case is ignored when pooling HTTP connections."""
p = PoolManager()
pool = p.connection_from_url('http://example.com/')
other_pool = p.connection_from_url('HTTP://EXAMPLE.COM/')
assert 1 == len(p.pools)
assert pool is other_pool
assert all(isinstance(key, PoolKey) for key in p.pools.keys())
def test_http_connection_from_host_case_insensitive(self):
"""Assert scheme case is ignored when getting the https key class."""
p = PoolManager()
pool = p.connection_from_host('example.com', scheme='http')
other_pool = p.connection_from_host('EXAMPLE.COM', scheme='HTTP')
assert 1 == len(p.pools)
assert pool is other_pool
assert all(isinstance(key, PoolKey) for key in p.pools.keys())
def test_assert_hostname_and_fingerprint_flag(self):
"""Assert that pool manager can accept hostname and fingerprint flags."""
fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:BF:93:CF:F9:71:CC:07:7D:0A'
p = PoolManager(assert_hostname=True, assert_fingerprint=fingerprint)
self.addCleanup(p.clear)
pool = p.connection_from_url('https://example.com/')
self.assertEqual(1, len(p.pools))
self.assertTrue(pool.assert_hostname)
self.assertEqual(fingerprint, pool.assert_fingerprint)
def test_https_connection_from_url_case_insensitive(self):
"""Assert scheme case is ignored when pooling HTTPS connections."""
p = PoolManager()
self.addCleanup(p.clear)
pool = p.connection_from_url('https://example.com/')
other_pool = p.connection_from_url('HTTPS://EXAMPLE.COM/')
self.assertEqual(1, len(p.pools))
self.assertTrue(pool is other_pool)
self.assertTrue(all(isinstance(key, PoolKey) for key in p.pools.keys()))
def test_http_connection_from_host_case_insensitive(self):
"""Assert scheme case is ignored when getting the https key class."""
p = PoolManager()
self.addCleanup(p.clear)
pool = p.connection_from_host('example.com', scheme='http')
other_pool = p.connection_from_host('EXAMPLE.COM', scheme='HTTP')
self.assertEqual(1, len(p.pools))
self.assertTrue(pool is other_pool)
self.assertTrue(all(isinstance(key, PoolKey) for key in p.pools.keys()))
def test_redirect(self):
http = PoolManager()
r = http.request("GET", "%s/redirect" % self.base_url, fields={"target": "%s/" % self.base_url}, redirect=False)
self.assertEqual(r.status, 303)
r = http.request("GET", "%s/redirect" % self.base_url, fields={"target": "%s/" % self.base_url})
self.assertEqual(r.status, 200)
self.assertEqual(r.data, b"Dummy server!")
def test_http_connection_from_context_case_insensitive(self):
"""Assert scheme case is ignored when getting the https key class."""
p = PoolManager()
context = {'scheme': 'http', 'host': 'example.com', 'port': '8080'}
other_context = {'scheme': 'HTTP', 'host': 'EXAMPLE.COM', 'port': '8080'}
pool = p.connection_from_context(context)
other_pool = p.connection_from_context(other_context)
assert 1 == len(p.pools)
assert pool is other_pool
assert all(isinstance(key, PoolKey) for key in p.pools.keys())
def test_raise_on_redirect(self):
http = PoolManager()
r = http.request(
"GET",
"%s/redirect" % self.base_url,
fields={"target": "%s/redirect?target=%s/" % (self.base_url, self.base_url)},
retries=Retry(total=None, redirect=1, raise_on_redirect=False),
)
self.assertEqual(r.status, 303)
def test_https_connection_from_context_case_insensitive(self):
"""Assert scheme case is ignored when getting the https key class."""
p = PoolManager()
self.addCleanup(p.clear)
context = {'scheme': 'https', 'host': 'example.com', 'port': '443'}
other_context = {'scheme': 'HTTPS', 'host': 'EXAMPLE.COM', 'port': '443'}
pool = p.connection_from_context(context)
other_pool = p.connection_from_context(other_context)
self.assertEqual(1, len(p.pools))
self.assertTrue(pool is other_pool)
self.assertTrue(all(isinstance(key, PoolKey) for key in p.pools.keys()))
def test_redirect_cross_host_remove_headers(self):
http = PoolManager()
self.addCleanup(http.clear)
r = http.request('GET', '%s/redirect' % self.base_url,
fields={'target': '%s/headers' % self.base_url_alt},
headers={'Authorization': 'foo'})
self.assertEqual(r.status, 200)
data = json.loads(r.data.decode('utf-8'))
self.assertNotIn('Authorization', data)
def test_same_url(self):
# Convince ourselves that normally we don't get the same object
conn1 = connection_from_url('http://localhost:8081/foo')
conn2 = connection_from_url('http://localhost:8081/bar')
self.assertNotEqual(conn1, conn2)
# Now try again using the PoolManager
p = PoolManager(1)
conn1 = p.connection_from_url('http://localhost:8081/foo')
conn2 = p.connection_from_url('http://localhost:8081/bar')
self.assertEqual(conn1, conn2)
def test_redirect_to_relative_url(self):
http = PoolManager()
r = http.request('GET', '%s/redirect' % self.base_url,
fields = {'target': '/redirect'},
redirect = False)
self.assertEqual(r.status, 303)
r = http.request('GET', '%s/redirect' % self.base_url,
fields = {'target': '/redirect'})
self.assertEqual(r.status, 200)
self.assertEqual(r.data, b'Dummy server!')
def test_redirect_twice(self):
http = PoolManager()
r = http.request('GET', '%s/redirect' % self.base_url,
fields={'target': '%s/redirect' % self.base_url},
redirect=False)
self.assertEqual(r.status, 303)
r = http.request('GET', '%s/redirect' % self.base_url,
fields={'target': '%s/redirect?target=%s/' % (self.base_url, self.base_url)})
self.assertEqual(r.status, 200)
self.assertEqual(r.data, b'Dummy server!')
def test_redirect(self):
http = PoolManager()
self.addCleanup(http.clear)
r = http.request('GET', '%s/redirect' % self.base_url,
fields={'target': '%s/' % self.base_url},
redirect=False)
self.assertEqual(r.status, 303)
r = http.request('GET', '%s/redirect' % self.base_url,
fields={'target': '%s/' % self.base_url})
self.assertEqual(r.status, 200)
self.assertEqual(r.data, b'Dummy server!')
def test_post_request(self):
pool_manager = PoolManager()
url = "http://www.snee.com/xml/crud/posttest.cgi"
params = { "fname": "test firstname", "lname": "test lastname" }
response = pool_manager.request(method="POST", url=url,
fields=params)
self.assertEqual(response.status, 200)
self.assertTrue(response.data.find('''<p>First name: "test firstname"</p><p>Last
name: "test lastname"</p>'''))
def test_custom_pool_key(self):
"""Assert it is possible to define a custom key function."""
p = PoolManager(10)
p.key_fn_by_scheme['http'] = lambda x: tuple(x['key'])
pool1 = p.connection_from_url(
'http://example.com', pool_kwargs={'key': 'value'})
pool2 = p.connection_from_url(
'http://example.com', pool_kwargs={'key': 'other'})
pool3 = p.connection_from_url(
'http://example.com', pool_kwargs={'key': 'value', 'x': 'y'})
assert 2 == len(p.pools)
assert pool1 is pool3
assert pool1 is not pool2
def test_custom_pool_key(self):
"""Assert it is possible to define a custom key function."""
p = PoolManager(10)
self.addCleanup(p.clear)
p.key_fn_by_scheme['http'] = lambda x: tuple(x['key'])
pool1 = p.connection_from_url(
'http://example.com', pool_kwargs={'key': 'value'})
pool2 = p.connection_from_url(
'http://example.com', pool_kwargs={'key': 'other'})
pool3 = p.connection_from_url(
'http://example.com', pool_kwargs={'key': 'value', 'x': 'y'})
self.assertEqual(2, len(p.pools))
self.assertTrue(pool1 is pool3)
self.assertFalse(pool1 is pool2)
def test_override_pool_kwargs_host(self):
"""Assert overriding pool kwargs works with connection_from_host"""
p = PoolManager(strict=True)
pool_kwargs = {'strict': False, 'retries': 100, 'block': True}
default_pool = p.connection_from_host('example.com', scheme='http')
override_pool = p.connection_from_host('example.com', scheme='http',
pool_kwargs=pool_kwargs)
assert default_pool.strict
assert retry.Retry.DEFAULT == default_pool.retries
assert not default_pool.block
assert not override_pool.strict
assert 100 == override_pool.retries
assert override_pool.block
def test_pool_kwargs_socket_options(self):
"""Assert passing socket options works with connection_from_host"""
p = PoolManager(socket_options=[])
override_opts = [
(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1),
(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
]
pool_kwargs = {'socket_options': override_opts}
default_pool = p.connection_from_host('example.com', scheme='http')
override_pool = p.connection_from_host(
'example.com', scheme='http', pool_kwargs=pool_kwargs
)
assert default_pool.conn_kw['socket_options'] == []
assert override_pool.conn_kw['socket_options'] == override_opts
def test_override_pool_kwargs_host(self):
"""Assert overriding pool kwargs works with connection_from_host"""
p = PoolManager(strict=True)
pool_kwargs = {'strict': False, 'retries': 100, 'block': True}
default_pool = p.connection_from_host('example.com', scheme='http')
override_pool = p.connection_from_host('example.com', scheme='http',
pool_kwargs=pool_kwargs)
self.assertTrue(default_pool.strict)
self.assertEqual(retry.Retry.DEFAULT, default_pool.retries)
self.assertFalse(default_pool.block)
self.assertFalse(override_pool.strict)
self.assertEqual(100, override_pool.retries)
self.assertTrue(override_pool.block)
def init_poolmanager(
self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs
):
"""Initializes a urllib3 PoolManager.
This method should not be called from user code, and is only
exposed for use when subclassing the
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
:param connections: The number of urllib3 connection pools to cache.
:param maxsize: The maximum number of connections to save in the pool.
:param block: Block when no free connections are available.
:param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager.
"""
# save these values for pickling
self._pool_connections = connections
self._pool_maxsize = maxsize
self._pool_block = block
self.poolmanager = PoolManager(
num_pools=connections,
maxsize=maxsize,
block=block,
strict=True,
**pool_kwargs
)
def test_missing_port(self):
# Can a URL that lacks an explicit port like ':80' succeed, or
# will all such URLs fail with an error?
http = PoolManager()
# By globally adjusting `port_by_scheme` we pretend for a moment
# that HTTP's default port is not 80, but is the port at which
# our test server happens to be listening.
port_by_scheme['http'] = self.port
try:
r = http.request('GET', 'http://%s/' % self.host, retries=0)
finally:
port_by_scheme['http'] = 80
self.assertEqual(r.status, 200)
self.assertEqual(r.data, b'Dummy server!')
def test_headers(self):
http = PoolManager(headers={'Foo': 'bar'})
r = http.request('GET', '%s/headers' % self.base_url)
returned_headers = json.loads(r.data.decode())
self.assertEqual(returned_headers.get('Foo'), 'bar')
r = http.request('POST', '%s/headers' % self.base_url)
returned_headers = json.loads(r.data.decode())
self.assertEqual(returned_headers.get('Foo'), 'bar')
r = http.request_encode_url('GET', '%s/headers' % self.base_url)
returned_headers = json.loads(r.data.decode())
self.assertEqual(returned_headers.get('Foo'), 'bar')
r = http.request_encode_body('POST', '%s/headers' % self.base_url)
returned_headers = json.loads(r.data.decode())
self.assertEqual(returned_headers.get('Foo'), 'bar')
r = http.request_encode_url('GET', '%s/headers' % self.base_url, headers={'Baz': 'quux'})
returned_headers = json.loads(r.data.decode())
self.assertEqual(returned_headers.get('Foo'), None)
self.assertEqual(returned_headers.get('Baz'), 'quux')
r = http.request_encode_body('GET', '%s/headers' % self.base_url, headers={'Baz': 'quux'})
returned_headers = json.loads(r.data.decode())
self.assertEqual(returned_headers.get('Foo'), None)
self.assertEqual(returned_headers.get('Baz'), 'quux')
def test_pools_keyed_with_from_host(self):
"""Assert pools are still keyed correctly with connection_from_host."""
ssl_kw = [
('key_file', DEFAULT_CERTS['keyfile']),
('cert_file', DEFAULT_CERTS['certfile']),
('cert_reqs', 'CERT_REQUIRED'),
('ca_certs', DEFAULT_CA),
('ca_cert_dir', DEFAULT_CA_DIR),
('ssl_version', 'SSLv23'),
('ssl_context', ssl_.create_urllib3_context()),
]
p = PoolManager()
conns = [p.connection_from_host('example.com', 443, scheme='https')]
for k, v in ssl_kw:
p.connection_pool_kw[k] = v
conns.append(
p.connection_from_host('example.com', 443, scheme='https'))
assert all(x is not y for i, x in enumerate(conns)
for j, y in enumerate(conns) if i != j)
def test_many_urls(self) -> None:
urls = [
"http://localhost:8081/foo",
"http://www.google.com/mail",
"http://localhost:8081/bar",
"https://www.google.com/",
"https://www.google.com/mail",
"http://yahoo.com",
"http://bing.com",
"http://yahoo.com/",
]
connections = set()
p = PoolManager(10)
for url in urls:
conn = p.connection_from_url(url)
connections.add(conn)
assert len(connections) == 5
def test_pools_keyed_with_from_host(self):
"""Assert pools are still keyed correctly with connection_from_host."""
ssl_kw = {
'key_file': '/root/totally_legit.key',
'cert_file': '/root/totally_legit.crt',
'cert_reqs': 'CERT_REQUIRED',
'ca_certs': '/root/path_to_pem',
'ssl_version': 'SSLv23_METHOD',
}
p = PoolManager(5, **ssl_kw)
conns = []
conns.append(p.connection_from_host('example.com', 443,
scheme='https'))
for k in ssl_kw:
p.connection_pool_kw[k] = 'newval'
conns.append(
p.connection_from_host('example.com', 443, scheme='https'))
assert all(x is not y for i, x in enumerate(conns)
for j, y in enumerate(conns) if i != j)
def test_retry_reuse_safe(self):
""" It should be possible to reuse a Retry object across requests """
headers = {'test-name': 'test_retry_safe'}
retry = Retry(total=1, status_forcelist=[418])
with PoolManager() as http:
resp = http.request('GET', '%s/successful_retry' % self.base_url,
headers=headers, retries=retry)
self.assertEqual(resp.status, 200)
resp = http.request('GET', '%s/successful_retry' % self.base_url,
headers=headers, retries=retry)
self.assertEqual(resp.status, 200)
def test_too_many_redirects(self):
http = PoolManager()
self.addCleanup(http.clear)
try:
r = http.request('GET', '%s/redirect' % self.base_url,
fields={'target': '%s/redirect?target=%s/' % (self.base_url,
self.base_url)},
retries=1)
self.fail("Failed to raise MaxRetryError exception, returned %r" % r.status)
except MaxRetryError:
pass
try:
r = http.request('GET', '%s/redirect' % self.base_url,
fields={'target': '%s/redirect?target=%s/' % (self.base_url,
self.base_url)},
retries=Retry(total=None, redirect=1))
self.fail("Failed to raise MaxRetryError exception, returned %r" % r.status)
except MaxRetryError:
pass
def test_redirect_cross_host_set_removed_headers(self):
http = PoolManager()
self.addCleanup(http.clear)
r = http.request(
"GET",
"%s/redirect" % self.base_url,
fields={"target": "%s/headers" % self.base_url_alt},
headers={
"X-API-Secret": "foo",
"Authorization": "bar"
},
retries=Retry(remove_headers_on_redirect=["X-API-Secret"]),
)
self.assertEqual(r.status, 200)
data = json.loads(r.data.decode("utf-8"))
self.assertNotIn("X-API-Secret", data)
self.assertEqual(data["Authorization"], "bar")
r = http.request(
"GET",
"%s/redirect" % self.base_url,
fields={"target": "%s/headers" % self.base_url_alt},
headers={
"x-api-secret": "foo",
"authorization": "bar"
},
retries=Retry(remove_headers_on_redirect=["X-API-Secret"]),
)
self.assertEqual(r.status, 200)
data = json.loads(r.data.decode("utf-8"))
self.assertNotIn("x-api-secret", data)
self.assertNotIn("X-API-Secret", data)
self.assertEqual(data["Authorization"], "bar")
def test_read_total_retries(self):
""" HTTP response w/ status code in the whitelist should be retried """
headers = {"test-name": "test_read_total_retries"}
retry = Retry(total=1, status_forcelist=[418])
with PoolManager() as http:
resp = http.request(
"GET",
"%s/successful_retry" % self.base_url,
headers=headers,
retries=retry,
)
assert resp.status == 200
def test_raise_on_redirect(self) -> None:
with PoolManager() as http:
r = http.request(
"GET",
f"{self.base_url}/redirect",
fields={
"target":
f"{self.base_url}/redirect?target={self.base_url}/"
},
retries=Retry(total=None, redirect=1, raise_on_redirect=False),
)
assert r.status == 303
def test_retries_wrong_method_list(self):
"""Method not in our whitelist should not be retried, even if code matches"""
headers = {"test-name": "test_wrong_method_whitelist"}
retry = Retry(total=1, status_forcelist=[418], method_whitelist=["POST"])
with PoolManager() as http:
resp = http.request(
"GET",
"%s/successful_retry" % self.base_url,
headers=headers,
retries=retry,
)
assert resp.status == 418
def test_pools_keyed_with_from_host(self):
"""Assert pools are still keyed correctly with connection_from_host."""
ssl_kw = {
"key_file": "/root/totally_legit.key",
"cert_file": "/root/totally_legit.crt",
"cert_reqs": "CERT_REQUIRED",
"ca_certs": "/root/path_to_pem",
"ssl_version": "SSLv23_METHOD",
}
p = PoolManager(5, **ssl_kw)
conns = [p.connection_from_host("example.com", 443, scheme="https")]
for k in ssl_kw:
p.connection_pool_kw[k] = "newval"
conns.append(p.connection_from_host("example.com", 443, scheme="https"))
assert all(
x is not y
for i, x in enumerate(conns)
for j, y in enumerate(conns)
if i != j
)
def test_retry_return_in_response(self):
headers = {'test-name': 'test_retry_return_in_response'}
retry = Retry(total=2, status_forcelist=[418])
with PoolManager() as http:
resp = http.request('GET',
'%s/successful_retry' % self.base_url,
headers=headers,
retries=retry)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.retries.total, 1)
self.assertEqual(resp.retries.history, (RequestHistory(
'GET', '/successful_retry', None, 418, None), ))
def test_retries_wrong_method_list(self):
"""Method not in our whitelist should not be retried, even if code matches"""
headers = {'test-name': 'test_wrong_method_whitelist'}
retry = Retry(total=1,
status_forcelist=[418],
method_whitelist=['POST'])
with PoolManager() as http:
resp = http.request('GET',
'%s/successful_retry' % self.base_url,
headers=headers,
retries=retry)
self.assertEqual(resp.status, 418)
def test_raise_on_redirect(self):
with PoolManager() as http:
r = http.request(
"GET",
"%s/redirect" % self.base_url,
fields={
"target":
"%s/redirect?target=%s/" % (self.base_url, self.base_url)
},
retries=Retry(total=None, redirect=1, raise_on_redirect=False),
)
assert r.status == 303
def test_many_urls(self):
urls = [
"http://localhost:8081/foo",
"http://www.google.com/mail",
"http://localhost:8081/bar",
"https://www.google.com/",
"https://www.google.com/mail",
"http://yahoo.com",
"http://bing.com",
"http://yahoo.com/",
]
connections = set()
p = PoolManager(10)
self.addCleanup(p.clear)
for url in urls:
conn = p.connection_from_url(url)
connections.add(conn)
self.assertEqual(len(connections), 5)
def test_raise_on_status(self):
http = PoolManager()
self.addCleanup(http.clear)
try:
# the default is to raise
r = http.request('GET',
'%s/status' % self.base_url,
fields={'status': '500 Internal Server Error'},
retries=Retry(total=1,
status_forcelist=range(500, 600)))
self.fail("Failed to raise MaxRetryError exception, returned %r" %
r.status)
except MaxRetryError:
pass
try:
# raise explicitly
r = http.request('GET',
'%s/status' % self.base_url,
fields={'status': '500 Internal Server Error'},
retries=Retry(total=1,
status_forcelist=range(500, 600),
raise_on_status=True))
self.fail("Failed to raise MaxRetryError exception, returned %r" %
r.status)
except MaxRetryError:
pass
# don't raise
r = http.request('GET',
'%s/status' % self.base_url,
fields={'status': '500 Internal Server Error'},
retries=Retry(total=1,
status_forcelist=range(500, 600),
raise_on_status=False))
self.assertEqual(r.status, 500)
def test_redirect_cross_host_set_removed_headers(self):
http = PoolManager()
self.addCleanup(http.clear)
r = http.request(
'GET',
'%s/redirect' % self.base_url,
fields={'target': '%s/headers' % self.base_url_alt},
headers={
'X-API-Secret': 'foo',
'Authorization': 'bar'
},
retries=Retry(remove_headers_on_redirect=['X-API-Secret']))
self.assertEqual(r.status, 200)
data = json.loads(r.data.decode('utf-8'))
self.assertNotIn('X-API-Secret', data)
self.assertEqual(data['Authorization'], 'bar')
r = http.request(
'GET',
'%s/redirect' % self.base_url,
fields={'target': '%s/headers' % self.base_url_alt},
headers={
'x-api-secret': 'foo',
'authorization': 'bar'
},
retries=Retry(remove_headers_on_redirect=['X-API-Secret']))
self.assertEqual(r.status, 200)
data = json.loads(r.data.decode('utf-8'))
self.assertNotIn('x-api-secret', data)
self.assertNotIn('X-API-Secret', data)
self.assertEqual(data['Authorization'], 'bar')
def __init__(self, config):
super().__init__()
try:
access_key = config['storage_access_key']
except Exception:
access_key = 'minio'
try:
secret_key = config['storage_secret_key']
except Exception:
secret_key = 'minio123'
try:
self.bucket = config['storage_bucket']
except Exception:
self.bucket = 'fedn-models'
try:
self.context_bucket = config['context_bucket']
except Exception:
self.bucket = 'fedn-context'
try:
self.secure_mode = bool(config['storage_secure_mode'])
except Exception:
self.secure_mode = False
if not self.secure_mode:
print(
"\n\n\nWARNING : S3/MINIO RUNNING IN **INSECURE** MODE! THIS IS NOT FOR PRODUCTION!\n\n\n"
)
if self.secure_mode:
from urllib3.poolmanager import PoolManager
manager = PoolManager(num_pools=100,
cert_reqs='CERT_NONE',
assert_hostname=False)
self.client = Minio("{0}:{1}".format(config['storage_hostname'],
config['storage_port']),
access_key=access_key,
secret_key=secret_key,
secure=self.secure_mode,
http_client=manager)
else:
self.client = Minio("{0}:{1}".format(config['storage_hostname'],
config['storage_port']),
access_key=access_key,
secret_key=secret_key,
secure=self.secure_mode)
# TODO: generalize
self.context_bucket = 'fedn-context'
self.create_bucket(self.context_bucket)
self.create_bucket(self.bucket)
def init_poolmanager(
self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs
):
self._pool_connections = connections
self._pool_maxsize = maxsize
self._pool_block = block
self.poolmanager = PoolManager(
num_pools=connections,
maxsize=maxsize,
block=block,
strict=True,
cert_reqs='CERT_NONE',
**pool_kwargs
)
def test_max_retry(self):
with PoolManager() as http:
try:
r = http.request(
"GET",
"%s/redirect" % self.base_url,
fields={"target": "/"},
retries=0,
)
self.fail(
"Failed to raise MaxRetryError exception, returned %r" % r.status
)
except MaxRetryError:
pass
def init_poolmanager(self,
connections,
maxsize,
block=False,
*args,
**kwargs):
context = create_urllib3_context(ciphers=DESAdapter.CIPHERS)
kwargs["ssl_context"] = context
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
ssl_version=ssl.PROTOCOL_SSLv23,
*args,
**kwargs)
def test_redirect_cross_host_no_remove_headers(self):
with PoolManager() as http:
r = http.request(
"GET",
"%s/redirect" % self.base_url,
fields={"target": "%s/headers" % self.base_url_alt},
headers={"Authorization": "foo"},
retries=Retry(remove_headers_on_redirect=[]),
)
assert r.status == 200
data = json.loads(r.data.decode("utf-8"))
assert data["Authorization"] == "foo"
def test_cross_host_redirect(self):
http = PoolManager()
cross_host_location = '%s/echo?a=b' % self.base_url_alt
try:
http.request('GET',
'%s/redirect' % self.base_url,
fields={'target': cross_host_location},
timeout=0.01,
retries=0)
self.fail(
"Request succeeded instead of raising an exception like it should."
)
except MaxRetryError:
pass
r = http.request('GET',
'%s/redirect' % self.base_url,
fields={'target': '%s/echo?a=b' % self.base_url_alt},
timeout=0.01,
retries=1)
self.assertEqual(r._pool.host, self.host_alt)
def test_redirect_twice(self):
http = PoolManager()
self.addCleanup(http.clear)
r = http.request(
"GET",
"%s/redirect" % self.base_url,
fields={"target": "%s/redirect" % self.base_url},
redirect=False,
)
self.assertEqual(r.status, 303)
r = http.request(
"GET",
"%s/redirect" % self.base_url,
fields={
"target":
"%s/redirect?target=%s/" % (self.base_url, self.base_url)
},
)
self.assertEqual(r.status, 200)
self.assertEqual(r.data, b"Dummy server!")
def test_connection_from_context_strict_param(
self, connection_from_pool_key: mock.MagicMock) -> None:
p = PoolManager()
context = {
"scheme": "http",
"host": "example.com",
"port": 8080,
"strict": True,
}
with pytest.warns(DeprecationWarning) as records:
p.connection_from_context(context)
msg = ("The 'strict' parameter is no longer needed on Python 3+. "
"This will raise an error in urllib3 v3.0.0.")
record = records[0]
assert isinstance(record.message, Warning)
assert record.message.args[0] == msg
_, kwargs = connection_from_pool_key.call_args
assert kwargs["request_context"] == {
"scheme": "http",
"host": "example.com",
"port": 8080,
}
def test_pools_keyed_with_from_host(self):
"""Assert pools are still keyed correctly with connection_from_host."""
ssl_kw = [
("key_file", DEFAULT_CERTS["keyfile"]),
("cert_file", DEFAULT_CERTS["certfile"]),
("cert_reqs", "CERT_REQUIRED"),
("ca_certs", DEFAULT_CA),
("ca_cert_dir", DEFAULT_CA_DIR),
("ssl_version", "SSLv23"),
("ssl_context", ssl_.create_urllib3_context()),
]
p = PoolManager()
conns = [p.connection_from_host("example.com", 443, scheme="https")]
for k, v in ssl_kw:
p.connection_pool_kw[k] = v
conns.append(p.connection_from_host("example.com", 443, scheme="https"))
assert all(
x is not y
for i, x in enumerate(conns)
for j, y in enumerate(conns)
if i != j
)
def test_retry_return_in_response(self):
headers = {"test-name": "test_retry_return_in_response"}
retry = Retry(total=2, status_forcelist=[418])
with PoolManager() as http:
resp = http.request(
"GET",
"%s/successful_retry" % self.base_url,
headers=headers,
retries=retry,
)
assert resp.status == 200
assert resp.retries.total == 1
assert resp.retries.history == (RequestHistory(
"GET", "/successful_retry", None, 418, None), )
def test_redirect_cross_host_no_remove_headers(self) -> None:
with PoolManager() as http:
r = http.request(
"GET",
f"{self.base_url}/redirect",
fields={"target": f"{self.base_url_alt}/headers"},
headers={"Authorization": "foo"},
retries=Retry(remove_headers_on_redirect=[]),
)
assert r.status == 200
data = r.json()
assert data["Authorization"] == "foo"
def test_contextmanager(self):
with PoolManager(1) as p:
conn_pool = p.connection_from_url('http://google.com')
self.assertEqual(len(p.pools), 1)
conn = conn_pool._get_conn()
self.assertEqual(len(p.pools), 0)
self.assertRaises(ClosedPoolError, conn_pool._get_conn)
conn_pool._put_conn(conn)
self.assertRaises(ClosedPoolError, conn_pool._get_conn)
self.assertEqual(len(p.pools), 0)
class ConnectionPool:
NUM_POOLS = 5 # one pool per host, usually just 1 pool is needed
# when redirect happens, one additional pool will be created
def __init__(self, host, path, timeout=0, maxsize=50):
self.host = host
self.path = path
self.pool = PoolManager(
self.NUM_POOLS,
headers=None,
cert_reqs='CERT_REQUIRED', # Force certificate check
ca_certs=certifi.where(), # Path to the Certifi bundle
strict=True, # TODO more comments to explain these parameters
timeout=timeout,
maxsize=maxsize,
block=True,
)
def send_receive(self, url, request_headers, request_body):
global _network_io_time
if _NETWORK_IO_TIME_COUNT_FLAG:
begin = time.time()
# if isinstance(request_body, bytes):
# request_body = str(request_body, encoding='utf-8')
response = self.pool.urlopen(
'POST',
self.host + self.path + url,
body=request_body,
headers=request_headers,
redirect=False,
assert_same_host=False,
)
if _NETWORK_IO_TIME_COUNT_FLAG:
end = time.time()
_network_io_time += end - begin
# TODO error handling
response_headers = dict(response.getheaders())
response_body = response.data # TODO figure out why response.read() don't work
return response.status, response.reason, response_headers, response_body
def test_redirect_with_failed_tell(self):
"""Abort request if failed to get a position from tell()"""
class BadTellObject(io.BytesIO):
def tell(self):
raise IOError
body = BadTellObject(b"the data")
url = "%s/redirect?target=/successful_retry" % self.base_url
# httplib uses fileno if Content-Length isn't supplied,
# which is unsupported by BytesIO.
headers = {"Content-Length": "8"}
with PoolManager() as http:
with pytest.raises(UnrewindableBodyError) as e:
http.urlopen("PUT", url, headers=headers, body=body)
assert "Unable to record file position for" in str(e.value)
