def test_pools_keyed_with_from_host(self): """Assert pools are still keyed correctly with connection_from_host.""" ssl_kw = { 'key_file': '/root/totally_legit.key', 'cert_file': '/root/totally_legit.crt', 'cert_reqs': 'CERT_REQUIRED', 'ca_certs': '/root/path_to_pem', 'ssl_version': 'SSLv23_METHOD', } p = PoolManager(5, **ssl_kw) conns = [] conns.append( p.connection_from_host('example.com', 443, scheme='https') ) for k in ssl_kw: p.connection_pool_kw[k] = 'newval' conns.append( p.connection_from_host('example.com', 443, scheme='https') ) assert all( x is not y for i, x in enumerate(conns) for j, y in enumerate(conns) if i != j )
def test_redirect_cross_host_set_removed_headers(self): http = PoolManager() self.addCleanup(http.clear) r = http.request('GET', '%s/redirect' % self.base_url, fields={'target': '%s/headers' % self.base_url_alt}, headers={'X-API-Secret': 'foo', 'Authorization': 'bar'}, retries=Retry(remove_headers_on_redirect=['X-API-Secret'])) self.assertEqual(r.status, 200) data = json.loads(r.data.decode('utf-8')) self.assertNotIn('X-API-Secret', data) self.assertEqual(data['Authorization'], 'bar') r = http.request('GET', '%s/redirect' % self.base_url, fields={'target': '%s/headers' % self.base_url_alt}, headers={'x-api-secret': 'foo', 'authorization': 'bar'}, retries=Retry(remove_headers_on_redirect=['X-API-Secret'])) self.assertEqual(r.status, 200) data = json.loads(r.data.decode('utf-8')) self.assertNotIn('x-api-secret', data) self.assertNotIn('X-API-Secret', data) self.assertEqual(data['Authorization'], 'bar')
def test_merge_pool_kwargs_none(self): """Assert false-y values to _merge_pool_kwargs result in defaults""" p = PoolManager(strict=True) merged = p._merge_pool_kwargs({}) assert p.connection_pool_kw == merged merged = p._merge_pool_kwargs(None) assert p.connection_pool_kw == merged
def test_request_survives_missing_port_number(self): # Can a URL that lacks an explicit port like ':80' succeed, or # will all such URLs fail with an error? def server(listener): sock = listener.accept()[0] read_request(sock) sock.send('HTTP/1.1 200 OK\r\n' 'Content-Type: text/plain\r\n' 'Content-Length: 8\r\n' '\r\n' 'Inspire.') sock.close() # By globally adjusting `port_by_scheme` we pretend for a moment # that HTTP's default port is not 80, but is the port at which # our test server happens to be listening. p = PoolManager() host, port = start_server(server) port_by_scheme['http'] = port try: response = p.request('GET', 'http://%s/' % host, retries=0) finally: port_by_scheme['http'] = 80 self.assertEqual(response.status, 200) self.assertEqual(response.data, 'Inspire.')
def test_raise_on_status(self): http = PoolManager() self.addCleanup(http.clear) try: # the default is to raise r = http.request('GET', '%s/status' % self.base_url, fields={'status': '500 Internal Server Error'}, retries=Retry(total=1, status_forcelist=range(500, 600))) self.fail("Failed to raise MaxRetryError exception, returned %r" % r.status) except MaxRetryError: pass try: # raise explicitly r = http.request('GET', '%s/status' % self.base_url, fields={'status': '500 Internal Server Error'}, retries=Retry(total=1, status_forcelist=range(500, 600), raise_on_status=True)) self.fail("Failed to raise MaxRetryError exception, returned %r" % r.status) except MaxRetryError: pass # don't raise r = http.request('GET', '%s/status' % self.base_url, fields={'status': '500 Internal Server Error'}, retries=Retry(total=1, status_forcelist=range(500, 600), raise_on_status=False)) self.assertEqual(r.status, 500)
def test_assert_hostname_and_fingerprint_flag(self): """Assert that pool manager can accept hostname and fingerprint flags.""" fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:BF:93:CF:F9:71:CC:07:7D:0A' p = PoolManager(assert_hostname=True, assert_fingerprint=fingerprint) pool = p.connection_from_url('https://example.com/') assert 1 == len(p.pools) assert pool.assert_hostname assert fingerprint == pool.assert_fingerprint
def test_raise_on_redirect(self): http = PoolManager() r = http.request('GET', '%s/redirect' % self.base_url, fields={'target': '%s/redirect?target=%s/' % (self.base_url, self.base_url)}, retries=Retry(total=None, redirect=1, raise_on_redirect=False)) self.assertEqual(r.status, 303)
def test_http_connection_from_url_case_insensitive(self): """Assert scheme case is ignored when pooling HTTP connections.""" p = PoolManager() pool = p.connection_from_url('http://example.com/') other_pool = p.connection_from_url('HTTP://EXAMPLE.COM/') assert 1 == len(p.pools) assert pool is other_pool assert all(isinstance(key, PoolKey) for key in p.pools.keys())
def test_http_connection_from_host_case_insensitive(self): """Assert scheme case is ignored when getting the https key class.""" p = PoolManager() pool = p.connection_from_host('example.com', scheme='http') other_pool = p.connection_from_host('EXAMPLE.COM', scheme='HTTP') assert 1 == len(p.pools) assert pool is other_pool assert all(isinstance(key, PoolKey) for key in p.pools.keys())
def test_assert_hostname_and_fingerprint_flag(self): """Assert that pool manager can accept hostname and fingerprint flags.""" fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:BF:93:CF:F9:71:CC:07:7D:0A' p = PoolManager(assert_hostname=True, assert_fingerprint=fingerprint) self.addCleanup(p.clear) pool = p.connection_from_url('https://example.com/') self.assertEqual(1, len(p.pools)) self.assertTrue(pool.assert_hostname) self.assertEqual(fingerprint, pool.assert_fingerprint)
def test_https_connection_from_url_case_insensitive(self): """Assert scheme case is ignored when pooling HTTPS connections.""" p = PoolManager() self.addCleanup(p.clear) pool = p.connection_from_url('https://example.com/') other_pool = p.connection_from_url('HTTPS://EXAMPLE.COM/') self.assertEqual(1, len(p.pools)) self.assertTrue(pool is other_pool) self.assertTrue(all(isinstance(key, PoolKey) for key in p.pools.keys()))
def test_http_connection_from_host_case_insensitive(self): """Assert scheme case is ignored when getting the https key class.""" p = PoolManager() self.addCleanup(p.clear) pool = p.connection_from_host('example.com', scheme='http') other_pool = p.connection_from_host('EXAMPLE.COM', scheme='HTTP') self.assertEqual(1, len(p.pools)) self.assertTrue(pool is other_pool) self.assertTrue(all(isinstance(key, PoolKey) for key in p.pools.keys()))
def test_redirect(self): http = PoolManager() r = http.request("GET", "%s/redirect" % self.base_url, fields={"target": "%s/" % self.base_url}, redirect=False) self.assertEqual(r.status, 303) r = http.request("GET", "%s/redirect" % self.base_url, fields={"target": "%s/" % self.base_url}) self.assertEqual(r.status, 200) self.assertEqual(r.data, b"Dummy server!")
def test_http_connection_from_context_case_insensitive(self): """Assert scheme case is ignored when getting the https key class.""" p = PoolManager() context = {'scheme': 'http', 'host': 'example.com', 'port': '8080'} other_context = {'scheme': 'HTTP', 'host': 'EXAMPLE.COM', 'port': '8080'} pool = p.connection_from_context(context) other_pool = p.connection_from_context(other_context) assert 1 == len(p.pools) assert pool is other_pool assert all(isinstance(key, PoolKey) for key in p.pools.keys())
def test_raise_on_redirect(self): http = PoolManager() r = http.request( "GET", "%s/redirect" % self.base_url, fields={"target": "%s/redirect?target=%s/" % (self.base_url, self.base_url)}, retries=Retry(total=None, redirect=1, raise_on_redirect=False), ) self.assertEqual(r.status, 303)
def test_https_connection_from_context_case_insensitive(self): """Assert scheme case is ignored when getting the https key class.""" p = PoolManager() self.addCleanup(p.clear) context = {'scheme': 'https', 'host': 'example.com', 'port': '443'} other_context = {'scheme': 'HTTPS', 'host': 'EXAMPLE.COM', 'port': '443'} pool = p.connection_from_context(context) other_pool = p.connection_from_context(other_context) self.assertEqual(1, len(p.pools)) self.assertTrue(pool is other_pool) self.assertTrue(all(isinstance(key, PoolKey) for key in p.pools.keys()))
def test_redirect_cross_host_remove_headers(self): http = PoolManager() self.addCleanup(http.clear) r = http.request('GET', '%s/redirect' % self.base_url, fields={'target': '%s/headers' % self.base_url_alt}, headers={'Authorization': 'foo'}) self.assertEqual(r.status, 200) data = json.loads(r.data.decode('utf-8')) self.assertNotIn('Authorization', data)
def test_same_url(self): # Convince ourselves that normally we don't get the same object conn1 = connection_from_url('http://localhost:8081/foo') conn2 = connection_from_url('http://localhost:8081/bar') self.assertNotEqual(conn1, conn2) # Now try again using the PoolManager p = PoolManager(1) conn1 = p.connection_from_url('http://localhost:8081/foo') conn2 = p.connection_from_url('http://localhost:8081/bar') self.assertEqual(conn1, conn2)
def test_redirect_to_relative_url(self): http = PoolManager() r = http.request('GET', '%s/redirect' % self.base_url, fields = {'target': '/redirect'}, redirect = False) self.assertEqual(r.status, 303) r = http.request('GET', '%s/redirect' % self.base_url, fields = {'target': '/redirect'}) self.assertEqual(r.status, 200) self.assertEqual(r.data, b'Dummy server!')
def test_redirect_twice(self): http = PoolManager() r = http.request('GET', '%s/redirect' % self.base_url, fields={'target': '%s/redirect' % self.base_url}, redirect=False) self.assertEqual(r.status, 303) r = http.request('GET', '%s/redirect' % self.base_url, fields={'target': '%s/redirect?target=%s/' % (self.base_url, self.base_url)}) self.assertEqual(r.status, 200) self.assertEqual(r.data, b'Dummy server!')
def test_redirect(self): http = PoolManager() self.addCleanup(http.clear) r = http.request('GET', '%s/redirect' % self.base_url, fields={'target': '%s/' % self.base_url}, redirect=False) self.assertEqual(r.status, 303) r = http.request('GET', '%s/redirect' % self.base_url, fields={'target': '%s/' % self.base_url}) self.assertEqual(r.status, 200) self.assertEqual(r.data, b'Dummy server!')
def test_post_request(self): pool_manager = PoolManager() url = "http://www.snee.com/xml/crud/posttest.cgi" params = { "fname": "test firstname", "lname": "test lastname" } response = pool_manager.request(method="POST", url=url, fields=params) self.assertEqual(response.status, 200) self.assertTrue(response.data.find('''<p>First name: "test firstname"</p><p>Last name: "test lastname"</p>'''))
def test_custom_pool_key(self): """Assert it is possible to define a custom key function.""" p = PoolManager(10) p.key_fn_by_scheme['http'] = lambda x: tuple(x['key']) pool1 = p.connection_from_url( 'http://example.com', pool_kwargs={'key': 'value'}) pool2 = p.connection_from_url( 'http://example.com', pool_kwargs={'key': 'other'}) pool3 = p.connection_from_url( 'http://example.com', pool_kwargs={'key': 'value', 'x': 'y'}) assert 2 == len(p.pools) assert pool1 is pool3 assert pool1 is not pool2
def test_custom_pool_key(self): """Assert it is possible to define a custom key function.""" p = PoolManager(10) self.addCleanup(p.clear) p.key_fn_by_scheme['http'] = lambda x: tuple(x['key']) pool1 = p.connection_from_url( 'http://example.com', pool_kwargs={'key': 'value'}) pool2 = p.connection_from_url( 'http://example.com', pool_kwargs={'key': 'other'}) pool3 = p.connection_from_url( 'http://example.com', pool_kwargs={'key': 'value', 'x': 'y'}) self.assertEqual(2, len(p.pools)) self.assertTrue(pool1 is pool3) self.assertFalse(pool1 is pool2)
def test_override_pool_kwargs_host(self): """Assert overriding pool kwargs works with connection_from_host""" p = PoolManager(strict=True) pool_kwargs = {'strict': False, 'retries': 100, 'block': True} default_pool = p.connection_from_host('example.com', scheme='http') override_pool = p.connection_from_host('example.com', scheme='http', pool_kwargs=pool_kwargs) assert default_pool.strict assert retry.Retry.DEFAULT == default_pool.retries assert not default_pool.block assert not override_pool.strict assert 100 == override_pool.retries assert override_pool.block
def test_pool_kwargs_socket_options(self): """Assert passing socket options works with connection_from_host""" p = PoolManager(socket_options=[]) override_opts = [ (socket.SOL_SOCKET, socket.SO_REUSEADDR, 1), (socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) ] pool_kwargs = {'socket_options': override_opts} default_pool = p.connection_from_host('example.com', scheme='http') override_pool = p.connection_from_host( 'example.com', scheme='http', pool_kwargs=pool_kwargs ) assert default_pool.conn_kw['socket_options'] == [] assert override_pool.conn_kw['socket_options'] == override_opts
def test_override_pool_kwargs_host(self): """Assert overriding pool kwargs works with connection_from_host""" p = PoolManager(strict=True) pool_kwargs = {'strict': False, 'retries': 100, 'block': True} default_pool = p.connection_from_host('example.com', scheme='http') override_pool = p.connection_from_host('example.com', scheme='http', pool_kwargs=pool_kwargs) self.assertTrue(default_pool.strict) self.assertEqual(retry.Retry.DEFAULT, default_pool.retries) self.assertFalse(default_pool.block) self.assertFalse(override_pool.strict) self.assertEqual(100, override_pool.retries) self.assertTrue(override_pool.block)
def init_poolmanager( self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs ): """Initializes a urllib3 PoolManager. This method should not be called from user code, and is only exposed for use when subclassing the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`. :param connections: The number of urllib3 connection pools to cache. :param maxsize: The maximum number of connections to save in the pool. :param block: Block when no free connections are available. :param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager. """ # save these values for pickling self._pool_connections = connections self._pool_maxsize = maxsize self._pool_block = block self.poolmanager = PoolManager( num_pools=connections, maxsize=maxsize, block=block, strict=True, **pool_kwargs )
def test_missing_port(self): # Can a URL that lacks an explicit port like ':80' succeed, or # will all such URLs fail with an error? http = PoolManager() # By globally adjusting `port_by_scheme` we pretend for a moment # that HTTP's default port is not 80, but is the port at which # our test server happens to be listening. port_by_scheme['http'] = self.port try: r = http.request('GET', 'http://%s/' % self.host, retries=0) finally: port_by_scheme['http'] = 80 self.assertEqual(r.status, 200) self.assertEqual(r.data, b'Dummy server!')
def test_headers(self): http = PoolManager(headers={'Foo': 'bar'}) r = http.request('GET', '%s/headers' % self.base_url) returned_headers = json.loads(r.data.decode()) self.assertEqual(returned_headers.get('Foo'), 'bar') r = http.request('POST', '%s/headers' % self.base_url) returned_headers = json.loads(r.data.decode()) self.assertEqual(returned_headers.get('Foo'), 'bar') r = http.request_encode_url('GET', '%s/headers' % self.base_url) returned_headers = json.loads(r.data.decode()) self.assertEqual(returned_headers.get('Foo'), 'bar') r = http.request_encode_body('POST', '%s/headers' % self.base_url) returned_headers = json.loads(r.data.decode()) self.assertEqual(returned_headers.get('Foo'), 'bar') r = http.request_encode_url('GET', '%s/headers' % self.base_url, headers={'Baz': 'quux'}) returned_headers = json.loads(r.data.decode()) self.assertEqual(returned_headers.get('Foo'), None) self.assertEqual(returned_headers.get('Baz'), 'quux') r = http.request_encode_body('GET', '%s/headers' % self.base_url, headers={'Baz': 'quux'}) returned_headers = json.loads(r.data.decode()) self.assertEqual(returned_headers.get('Foo'), None) self.assertEqual(returned_headers.get('Baz'), 'quux')
def test_pools_keyed_with_from_host(self): """Assert pools are still keyed correctly with connection_from_host.""" ssl_kw = [ ('key_file', DEFAULT_CERTS['keyfile']), ('cert_file', DEFAULT_CERTS['certfile']), ('cert_reqs', 'CERT_REQUIRED'), ('ca_certs', DEFAULT_CA), ('ca_cert_dir', DEFAULT_CA_DIR), ('ssl_version', 'SSLv23'), ('ssl_context', ssl_.create_urllib3_context()), ] p = PoolManager() conns = [p.connection_from_host('example.com', 443, scheme='https')] for k, v in ssl_kw: p.connection_pool_kw[k] = v conns.append( p.connection_from_host('example.com', 443, scheme='https')) assert all(x is not y for i, x in enumerate(conns) for j, y in enumerate(conns) if i != j)
def test_many_urls(self) -> None: urls = [ "http://localhost:8081/foo", "http://www.google.com/mail", "http://localhost:8081/bar", "https://www.google.com/", "https://www.google.com/mail", "http://yahoo.com", "http://bing.com", "http://yahoo.com/", ] connections = set() p = PoolManager(10) for url in urls: conn = p.connection_from_url(url) connections.add(conn) assert len(connections) == 5
def test_pools_keyed_with_from_host(self): """Assert pools are still keyed correctly with connection_from_host.""" ssl_kw = { 'key_file': '/root/totally_legit.key', 'cert_file': '/root/totally_legit.crt', 'cert_reqs': 'CERT_REQUIRED', 'ca_certs': '/root/path_to_pem', 'ssl_version': 'SSLv23_METHOD', } p = PoolManager(5, **ssl_kw) conns = [] conns.append(p.connection_from_host('example.com', 443, scheme='https')) for k in ssl_kw: p.connection_pool_kw[k] = 'newval' conns.append( p.connection_from_host('example.com', 443, scheme='https')) assert all(x is not y for i, x in enumerate(conns) for j, y in enumerate(conns) if i != j)
def test_retry_reuse_safe(self): """ It should be possible to reuse a Retry object across requests """ headers = {'test-name': 'test_retry_safe'} retry = Retry(total=1, status_forcelist=[418]) with PoolManager() as http: resp = http.request('GET', '%s/successful_retry' % self.base_url, headers=headers, retries=retry) self.assertEqual(resp.status, 200) resp = http.request('GET', '%s/successful_retry' % self.base_url, headers=headers, retries=retry) self.assertEqual(resp.status, 200)
def test_too_many_redirects(self): http = PoolManager() self.addCleanup(http.clear) try: r = http.request('GET', '%s/redirect' % self.base_url, fields={'target': '%s/redirect?target=%s/' % (self.base_url, self.base_url)}, retries=1) self.fail("Failed to raise MaxRetryError exception, returned %r" % r.status) except MaxRetryError: pass try: r = http.request('GET', '%s/redirect' % self.base_url, fields={'target': '%s/redirect?target=%s/' % (self.base_url, self.base_url)}, retries=Retry(total=None, redirect=1)) self.fail("Failed to raise MaxRetryError exception, returned %r" % r.status) except MaxRetryError: pass
def test_redirect_cross_host_set_removed_headers(self): http = PoolManager() self.addCleanup(http.clear) r = http.request( "GET", "%s/redirect" % self.base_url, fields={"target": "%s/headers" % self.base_url_alt}, headers={ "X-API-Secret": "foo", "Authorization": "bar" }, retries=Retry(remove_headers_on_redirect=["X-API-Secret"]), ) self.assertEqual(r.status, 200) data = json.loads(r.data.decode("utf-8")) self.assertNotIn("X-API-Secret", data) self.assertEqual(data["Authorization"], "bar") r = http.request( "GET", "%s/redirect" % self.base_url, fields={"target": "%s/headers" % self.base_url_alt}, headers={ "x-api-secret": "foo", "authorization": "bar" }, retries=Retry(remove_headers_on_redirect=["X-API-Secret"]), ) self.assertEqual(r.status, 200) data = json.loads(r.data.decode("utf-8")) self.assertNotIn("x-api-secret", data) self.assertNotIn("X-API-Secret", data) self.assertEqual(data["Authorization"], "bar")
def test_read_total_retries(self): """ HTTP response w/ status code in the whitelist should be retried """ headers = {"test-name": "test_read_total_retries"} retry = Retry(total=1, status_forcelist=[418]) with PoolManager() as http: resp = http.request( "GET", "%s/successful_retry" % self.base_url, headers=headers, retries=retry, ) assert resp.status == 200
def test_raise_on_redirect(self) -> None: with PoolManager() as http: r = http.request( "GET", f"{self.base_url}/redirect", fields={ "target": f"{self.base_url}/redirect?target={self.base_url}/" }, retries=Retry(total=None, redirect=1, raise_on_redirect=False), ) assert r.status == 303
def test_retries_wrong_method_list(self): """Method not in our whitelist should not be retried, even if code matches""" headers = {"test-name": "test_wrong_method_whitelist"} retry = Retry(total=1, status_forcelist=[418], method_whitelist=["POST"]) with PoolManager() as http: resp = http.request( "GET", "%s/successful_retry" % self.base_url, headers=headers, retries=retry, ) assert resp.status == 418
def test_pools_keyed_with_from_host(self): """Assert pools are still keyed correctly with connection_from_host.""" ssl_kw = { "key_file": "/root/totally_legit.key", "cert_file": "/root/totally_legit.crt", "cert_reqs": "CERT_REQUIRED", "ca_certs": "/root/path_to_pem", "ssl_version": "SSLv23_METHOD", } p = PoolManager(5, **ssl_kw) conns = [p.connection_from_host("example.com", 443, scheme="https")] for k in ssl_kw: p.connection_pool_kw[k] = "newval" conns.append(p.connection_from_host("example.com", 443, scheme="https")) assert all( x is not y for i, x in enumerate(conns) for j, y in enumerate(conns) if i != j )
def test_retry_return_in_response(self): headers = {'test-name': 'test_retry_return_in_response'} retry = Retry(total=2, status_forcelist=[418]) with PoolManager() as http: resp = http.request('GET', '%s/successful_retry' % self.base_url, headers=headers, retries=retry) self.assertEqual(resp.status, 200) self.assertEqual(resp.retries.total, 1) self.assertEqual(resp.retries.history, (RequestHistory( 'GET', '/successful_retry', None, 418, None), ))
def test_retries_wrong_method_list(self): """Method not in our whitelist should not be retried, even if code matches""" headers = {'test-name': 'test_wrong_method_whitelist'} retry = Retry(total=1, status_forcelist=[418], method_whitelist=['POST']) with PoolManager() as http: resp = http.request('GET', '%s/successful_retry' % self.base_url, headers=headers, retries=retry) self.assertEqual(resp.status, 418)
def test_raise_on_redirect(self): with PoolManager() as http: r = http.request( "GET", "%s/redirect" % self.base_url, fields={ "target": "%s/redirect?target=%s/" % (self.base_url, self.base_url) }, retries=Retry(total=None, redirect=1, raise_on_redirect=False), ) assert r.status == 303
def test_many_urls(self): urls = [ "http://localhost:8081/foo", "http://www.google.com/mail", "http://localhost:8081/bar", "https://www.google.com/", "https://www.google.com/mail", "http://yahoo.com", "http://bing.com", "http://yahoo.com/", ] connections = set() p = PoolManager(10) self.addCleanup(p.clear) for url in urls: conn = p.connection_from_url(url) connections.add(conn) self.assertEqual(len(connections), 5)
def test_raise_on_status(self): http = PoolManager() self.addCleanup(http.clear) try: # the default is to raise r = http.request('GET', '%s/status' % self.base_url, fields={'status': '500 Internal Server Error'}, retries=Retry(total=1, status_forcelist=range(500, 600))) self.fail("Failed to raise MaxRetryError exception, returned %r" % r.status) except MaxRetryError: pass try: # raise explicitly r = http.request('GET', '%s/status' % self.base_url, fields={'status': '500 Internal Server Error'}, retries=Retry(total=1, status_forcelist=range(500, 600), raise_on_status=True)) self.fail("Failed to raise MaxRetryError exception, returned %r" % r.status) except MaxRetryError: pass # don't raise r = http.request('GET', '%s/status' % self.base_url, fields={'status': '500 Internal Server Error'}, retries=Retry(total=1, status_forcelist=range(500, 600), raise_on_status=False)) self.assertEqual(r.status, 500)
def test_redirect_cross_host_set_removed_headers(self): http = PoolManager() self.addCleanup(http.clear) r = http.request( 'GET', '%s/redirect' % self.base_url, fields={'target': '%s/headers' % self.base_url_alt}, headers={ 'X-API-Secret': 'foo', 'Authorization': 'bar' }, retries=Retry(remove_headers_on_redirect=['X-API-Secret'])) self.assertEqual(r.status, 200) data = json.loads(r.data.decode('utf-8')) self.assertNotIn('X-API-Secret', data) self.assertEqual(data['Authorization'], 'bar') r = http.request( 'GET', '%s/redirect' % self.base_url, fields={'target': '%s/headers' % self.base_url_alt}, headers={ 'x-api-secret': 'foo', 'authorization': 'bar' }, retries=Retry(remove_headers_on_redirect=['X-API-Secret'])) self.assertEqual(r.status, 200) data = json.loads(r.data.decode('utf-8')) self.assertNotIn('x-api-secret', data) self.assertNotIn('X-API-Secret', data) self.assertEqual(data['Authorization'], 'bar')
def __init__(self, config): super().__init__() try: access_key = config['storage_access_key'] except Exception: access_key = 'minio' try: secret_key = config['storage_secret_key'] except Exception: secret_key = 'minio123' try: self.bucket = config['storage_bucket'] except Exception: self.bucket = 'fedn-models' try: self.context_bucket = config['context_bucket'] except Exception: self.bucket = 'fedn-context' try: self.secure_mode = bool(config['storage_secure_mode']) except Exception: self.secure_mode = False if not self.secure_mode: print( "\n\n\nWARNING : S3/MINIO RUNNING IN **INSECURE** MODE! THIS IS NOT FOR PRODUCTION!\n\n\n" ) if self.secure_mode: from urllib3.poolmanager import PoolManager manager = PoolManager(num_pools=100, cert_reqs='CERT_NONE', assert_hostname=False) self.client = Minio("{0}:{1}".format(config['storage_hostname'], config['storage_port']), access_key=access_key, secret_key=secret_key, secure=self.secure_mode, http_client=manager) else: self.client = Minio("{0}:{1}".format(config['storage_hostname'], config['storage_port']), access_key=access_key, secret_key=secret_key, secure=self.secure_mode) # TODO: generalize self.context_bucket = 'fedn-context' self.create_bucket(self.context_bucket) self.create_bucket(self.bucket)
def init_poolmanager( self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs ): self._pool_connections = connections self._pool_maxsize = maxsize self._pool_block = block self.poolmanager = PoolManager( num_pools=connections, maxsize=maxsize, block=block, strict=True, cert_reqs='CERT_NONE', **pool_kwargs )
def test_max_retry(self): with PoolManager() as http: try: r = http.request( "GET", "%s/redirect" % self.base_url, fields={"target": "/"}, retries=0, ) self.fail( "Failed to raise MaxRetryError exception, returned %r" % r.status ) except MaxRetryError: pass
def init_poolmanager(self, connections, maxsize, block=False, *args, **kwargs): context = create_urllib3_context(ciphers=DESAdapter.CIPHERS) kwargs["ssl_context"] = context self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize, block=block, ssl_version=ssl.PROTOCOL_SSLv23, *args, **kwargs)
def test_redirect_cross_host_no_remove_headers(self): with PoolManager() as http: r = http.request( "GET", "%s/redirect" % self.base_url, fields={"target": "%s/headers" % self.base_url_alt}, headers={"Authorization": "foo"}, retries=Retry(remove_headers_on_redirect=[]), ) assert r.status == 200 data = json.loads(r.data.decode("utf-8")) assert data["Authorization"] == "foo"
def test_cross_host_redirect(self): http = PoolManager() cross_host_location = '%s/echo?a=b' % self.base_url_alt try: http.request('GET', '%s/redirect' % self.base_url, fields={'target': cross_host_location}, timeout=0.01, retries=0) self.fail( "Request succeeded instead of raising an exception like it should." ) except MaxRetryError: pass r = http.request('GET', '%s/redirect' % self.base_url, fields={'target': '%s/echo?a=b' % self.base_url_alt}, timeout=0.01, retries=1) self.assertEqual(r._pool.host, self.host_alt)
def test_redirect_twice(self): http = PoolManager() self.addCleanup(http.clear) r = http.request( "GET", "%s/redirect" % self.base_url, fields={"target": "%s/redirect" % self.base_url}, redirect=False, ) self.assertEqual(r.status, 303) r = http.request( "GET", "%s/redirect" % self.base_url, fields={ "target": "%s/redirect?target=%s/" % (self.base_url, self.base_url) }, ) self.assertEqual(r.status, 200) self.assertEqual(r.data, b"Dummy server!")
def test_connection_from_context_strict_param( self, connection_from_pool_key: mock.MagicMock) -> None: p = PoolManager() context = { "scheme": "http", "host": "example.com", "port": 8080, "strict": True, } with pytest.warns(DeprecationWarning) as records: p.connection_from_context(context) msg = ("The 'strict' parameter is no longer needed on Python 3+. " "This will raise an error in urllib3 v3.0.0.") record = records[0] assert isinstance(record.message, Warning) assert record.message.args[0] == msg _, kwargs = connection_from_pool_key.call_args assert kwargs["request_context"] == { "scheme": "http", "host": "example.com", "port": 8080, }
def test_pools_keyed_with_from_host(self): """Assert pools are still keyed correctly with connection_from_host.""" ssl_kw = [ ("key_file", DEFAULT_CERTS["keyfile"]), ("cert_file", DEFAULT_CERTS["certfile"]), ("cert_reqs", "CERT_REQUIRED"), ("ca_certs", DEFAULT_CA), ("ca_cert_dir", DEFAULT_CA_DIR), ("ssl_version", "SSLv23"), ("ssl_context", ssl_.create_urllib3_context()), ] p = PoolManager() conns = [p.connection_from_host("example.com", 443, scheme="https")] for k, v in ssl_kw: p.connection_pool_kw[k] = v conns.append(p.connection_from_host("example.com", 443, scheme="https")) assert all( x is not y for i, x in enumerate(conns) for j, y in enumerate(conns) if i != j )
def test_retry_return_in_response(self): headers = {"test-name": "test_retry_return_in_response"} retry = Retry(total=2, status_forcelist=[418]) with PoolManager() as http: resp = http.request( "GET", "%s/successful_retry" % self.base_url, headers=headers, retries=retry, ) assert resp.status == 200 assert resp.retries.total == 1 assert resp.retries.history == (RequestHistory( "GET", "/successful_retry", None, 418, None), )
def test_redirect_cross_host_no_remove_headers(self) -> None: with PoolManager() as http: r = http.request( "GET", f"{self.base_url}/redirect", fields={"target": f"{self.base_url_alt}/headers"}, headers={"Authorization": "foo"}, retries=Retry(remove_headers_on_redirect=[]), ) assert r.status == 200 data = r.json() assert data["Authorization"] == "foo"
def test_contextmanager(self): with PoolManager(1) as p: conn_pool = p.connection_from_url('http://google.com') self.assertEqual(len(p.pools), 1) conn = conn_pool._get_conn() self.assertEqual(len(p.pools), 0) self.assertRaises(ClosedPoolError, conn_pool._get_conn) conn_pool._put_conn(conn) self.assertRaises(ClosedPoolError, conn_pool._get_conn) self.assertEqual(len(p.pools), 0)
class ConnectionPool: NUM_POOLS = 5 # one pool per host, usually just 1 pool is needed # when redirect happens, one additional pool will be created def __init__(self, host, path, timeout=0, maxsize=50): self.host = host self.path = path self.pool = PoolManager( self.NUM_POOLS, headers=None, cert_reqs='CERT_REQUIRED', # Force certificate check ca_certs=certifi.where(), # Path to the Certifi bundle strict=True, # TODO more comments to explain these parameters timeout=timeout, maxsize=maxsize, block=True, ) def send_receive(self, url, request_headers, request_body): global _network_io_time if _NETWORK_IO_TIME_COUNT_FLAG: begin = time.time() # if isinstance(request_body, bytes): # request_body = str(request_body, encoding='utf-8') response = self.pool.urlopen( 'POST', self.host + self.path + url, body=request_body, headers=request_headers, redirect=False, assert_same_host=False, ) if _NETWORK_IO_TIME_COUNT_FLAG: end = time.time() _network_io_time += end - begin # TODO error handling response_headers = dict(response.getheaders()) response_body = response.data # TODO figure out why response.read() don't work return response.status, response.reason, response_headers, response_body
def test_redirect_with_failed_tell(self): """Abort request if failed to get a position from tell()""" class BadTellObject(io.BytesIO): def tell(self): raise IOError body = BadTellObject(b"the data") url = "%s/redirect?target=/successful_retry" % self.base_url # httplib uses fileno if Content-Length isn't supplied, # which is unsupported by BytesIO. headers = {"Content-Length": "8"} with PoolManager() as http: with pytest.raises(UnrewindableBodyError) as e: http.urlopen("PUT", url, headers=headers, body=body) assert "Unable to record file position for" in str(e.value)