def test_same_host(self): same_host = [ ('http://google.com/', '/'), ('http://google.com/', 'http://google.com/'), ('http://google.com/', 'http://google.com'), ('http://google.com/', 'http://google.com/abra/cadabra'), ('http://google.com:42/', 'http://google.com:42/abracadabra'), # Test comparison using default ports ('http://google.com:80/', 'http://google.com/abracadabra'), ('http://google.com/', 'http://google.com:80/abracadabra'), ('https://google.com:443/', 'https://google.com/abracadabra'), ('https://google.com/', 'https://google.com:443/abracadabra'), ('http://[2607:f8b0:4005:805::200e%25eth0]/', 'http://[2607:f8b0:4005:805::200e%eth0]/'), ('https://[2607:f8b0:4005:805::200e%25eth0]:443/', 'https://[2607:f8b0:4005:805::200e%eth0]:443/'), ('http://[::1]/', 'http://[::1]'), ('http://[2001:558:fc00:200:f816:3eff:fef9:b954%lo]/', 'http://[2001:558:fc00:200:f816:3eff:fef9:b954%25lo]') ] for a, b in same_host: c = connection_from_url(a) self.addCleanup(c.close) self.assertTrue(c.is_same_host(b), "%s =? %s" % (a, b)) not_same_host = [ ('https://google.com/', 'http://google.com/'), ('http://google.com/', 'https://google.com/'), ('http://yahoo.com/', 'http://google.com/'), ('http://google.com:42', 'https://google.com/abracadabra'), ('http://google.com', 'https://google.net/'), # Test comparison with default ports ('http://google.com:42', 'http://google.com'), ('https://google.com:42', 'https://google.com'), ('http://google.com:443', 'http://google.com'), ('https://google.com:80', 'https://google.com'), ('http://google.com:443', 'https://google.com'), ('https://google.com:80', 'http://google.com'), ('https://google.com:443', 'http://google.com'), ('http://google.com:80', 'https://google.com'), # Zone identifiers are unique connection end points and should # never be equivalent. ('http://[dead::beef]', 'https://[dead::beef%en5]/'), ] for a, b in not_same_host: c = connection_from_url(a) self.addCleanup(c.close) self.assertFalse(c.is_same_host(b), "%s =? %s" % (a, b)) c = connection_from_url(b) self.addCleanup(c.close) self.assertFalse(c.is_same_host(a), "%s =? %s" % (b, a))
def test_cleanup_on_extreme_connection_error(self): """ This test validates that we clean up properly even on exceptions that we'd not otherwise catch, i.e. those that inherit from BaseException like KeyboardInterrupt or gevent.Timeout. See #805 for more details. """ class RealBad(BaseException): pass def kaboom(*args, **kwargs): raise RealBad() c = connection_from_url('http://localhost:80') c._make_request = kaboom initial_pool_size = c.pool.qsize() try: # We need to release_conn this way or we'd put it away regardless. c.urlopen('GET', '/', release_conn=False) except RealBad: pass new_pool_size = c.pool.qsize() self.assertEqual(initial_pool_size, new_pool_size)
def test_cleanup_on_extreme_connection_error(self) -> None: """ This test validates that we clean up properly even on exceptions that we'd not otherwise catch, i.e. those that inherit from BaseException like KeyboardInterrupt or gevent.Timeout. See #805 for more details. """ class RealBad(BaseException): pass def kaboom(*args: Any, **kwargs: Any) -> None: raise RealBad() with connection_from_url("http://localhost:80") as c: with patch.object(c, "_make_request", kaboom): assert c.pool is not None initial_pool_size = c.pool.qsize() try: # We need to release_conn this way or we'd put it away # regardless. c.urlopen("GET", "/", release_conn=False) except RealBad: pass new_pool_size = c.pool.qsize() assert initial_pool_size == new_pool_size
def test_pool_close(self): pool = connection_from_url('http://google.com:80') # Populate with some connections conn1 = pool._get_conn() conn2 = pool._get_conn() conn3 = pool._get_conn() pool._put_conn(conn1) pool._put_conn(conn2) old_pool_queue = pool.pool pool.close() self.assertEqual(pool.pool, None) with self.assertRaises(ClosedPoolError): pool._get_conn() pool._put_conn(conn3) with self.assertRaises(ClosedPoolError): pool._get_conn() with self.assertRaises(Empty): old_pool_queue.get(block=False)
def test_pool_close(self): pool = connection_from_url("http://google.com:80") # Populate with some connections conn1 = pool._get_conn() conn2 = pool._get_conn() conn3 = pool._get_conn() pool._put_conn(conn1) pool._put_conn(conn2) old_pool_queue = pool.pool pool.close() assert pool.pool is None with pytest.raises(ClosedPoolError): pool._get_conn() pool._put_conn(conn3) with pytest.raises(ClosedPoolError): pool._get_conn() with pytest.raises(Empty): old_pool_queue.get(block=False)
def test_oldapi(self): http = ProxyManager(connection_from_url(self.proxy_url)) r = http.request('GET', '%s/' % self.http_url) self.assertEqual(r.status, 200) r = http.request('GET', '%s/' % self.https_url) self.assertEqual(r.status, 200)
def test_oldapi(self): with ProxyManager(connection_from_url(self.proxy_url), ca_certs=DEFAULT_CA) as http: r = http.request("GET", f"{self.http_url}/") assert r.status == 200 r = http.request("GET", f"{self.https_url}/") assert r.status == 200
def test_assert_same_host(self): c = connection_from_url('http://google.com:80') self.assertRaises(HostChangedError, c.request, 'GET', 'http://yahoo.com:80', assert_same_host=True)
def test_oldapi(self): http = ProxyManager(connection_from_url(self.proxy_url), ca_certs=DEFAULT_CA) self.addCleanup(http.clear) r = http.request('GET', '%s/' % self.http_url) self.assertEqual(r.status, 200) r = http.request('GET', '%s/' % self.https_url) self.assertEqual(r.status, 200)
def connect(self, node=None): node_url = node or next(self._nodes) self.disconnect() self._connection = connection_from_url(node_url, **self.connection_options) logger.info("Current node changed to `{}`".format( self._connection.host)) return node_url
def test_oldapi(self): http = ProxyManager(connection_from_url(self.proxy_url), ca_certs=DEFAULT_CA) self.addCleanup(http.clear) r = http.request("GET", "%s/" % self.http_url) assert r.status == 200 r = http.request("GET", "%s/" % self.https_url) assert r.status == 200
def __init__(self, base_url): self.base_url = base_url # Get a ConnectionPool object, same as what you're doing in your question self.http = connection_from_url(self.base_url) # Override the connection class to force the unverified HTTPSConnection class self.http.ConnectionCls = UnverifiedHTTPSConnection disable_warnings(InsecureRequestWarning)
def test_same_host(self): same_host = [ ('http://google.com/', '/'), ('http://google.com/', 'http://google.com/'), ('http://google.com/', 'http://google.com'), ('http://google.com/', 'http://google.com/abra/cadabra'), ('http://google.com:42/', 'http://google.com:42/abracadabra'), # Test comparison using default ports ('http://google.com:80/', 'http://google.com/abracadabra'), ('http://google.com/', 'http://google.com:80/abracadabra'), ('https://google.com:443/', 'https://google.com/abracadabra'), ('https://google.com/', 'https://google.com:443/abracadabra'), ] for a, b in same_host: c = connection_from_url(a) self.assertTrue(c.is_same_host(b), "%s =? %s" % (a, b)) not_same_host = [ ('https://google.com/', 'http://google.com/'), ('http://google.com/', 'https://google.com/'), ('http://yahoo.com/', 'http://google.com/'), ('http://google.com:42', 'https://google.com/abracadabra'), ('http://google.com', 'https://google.net/'), # Test comparison with default ports ('http://google.com:42', 'http://google.com'), ('https://google.com:42', 'https://google.com'), ('http://google.com:443', 'http://google.com'), ('https://google.com:80', 'https://google.com'), ('http://google.com:443', 'https://google.com'), ('https://google.com:80', 'http://google.com'), ('https://google.com:443', 'http://google.com'), ('http://google.com:80', 'https://google.com'), ] for a, b in not_same_host: c = connection_from_url(a) self.assertFalse(c.is_same_host(b), "%s =? %s" % (a, b)) c = connection_from_url(b) self.assertFalse(c.is_same_host(a), "%s =? %s" % (b, a))
def test_same_host(self): same_host = [ ('http://google.com/', '/'), ('http://google.com/', 'http://google.com/'), ('http://google.com/', 'http://google.com'), ('http://google.com/', 'http://google.com/abra/cadabra'), ('http://google.com:42/', 'http://google.com:42/abracadabra'), ] for a, b in same_host: c = connection_from_url(a) self.assertTrue(c.is_same_host(b), "%s =? %s" % (a, b)) not_same_host = [ ('http://yahoo.com/', 'http://google.com/'), ('http://google.com:42', 'https://google.com/abracadabra'), ('http://google.com', 'https://google.net/'), ] for a, b in not_same_host: c = connection_from_url(a) self.assertFalse(c.is_same_host(b), "%s =? %s" % (a, b))
def test_pool_close_twice(self): pool = connection_from_url("http://google.com:80") # Populate with some connections conn1 = pool._get_conn() conn2 = pool._get_conn() pool._put_conn(conn1) pool._put_conn(conn2) pool.close() assert pool.pool is None try: pool.close() except AttributeError: pytest.fail("Pool of the ConnectionPool is None and has no attribute get.")
def test_pool_close_twice(self): pool = connection_from_url('http://google.com:80') # Populate with some connections conn1 = pool._get_conn() conn2 = pool._get_conn() pool._put_conn(conn1) pool._put_conn(conn2) pool.close() assert pool.pool is None try: pool.close() except AttributeError: pytest.fail("Pool of the ConnectionPool is None and has no attribute get.")
def test_contextmanager(self): with connection_from_url('http://google.com:80') as pool: # Populate with some connections conn1 = pool._get_conn() conn2 = pool._get_conn() conn3 = pool._get_conn() pool._put_conn(conn1) pool._put_conn(conn2) old_pool_queue = pool.pool self.assertEqual(pool.pool, None) self.assertRaises(ClosedPoolError, pool._get_conn) pool._put_conn(conn3) self.assertRaises(ClosedPoolError, pool._get_conn) self.assertRaises(Empty, old_pool_queue.get, block=False)
def session(self): if self._session is not None: return self._session session = connection_from_url(self.base_url) if session.scheme == "https": import certifi session.ca_certs = certifi.where() session.headers.update( { "Accept": "application/json", "User-Agent": "freight-cli", "Authorization": f"Key {self.api_key}", } ) self._session = session return session
def https_detector(url, parse_file): global ip_list global ip_index ip_index = 0 ip_list = parse_ip_file(parse_file) for i in range(0, len(ip_list)): try: connection.create_connection = patched_create_connection conn = connectionpool.connection_from_url(url) resp = conn.request('GET', url) if resp.data: print("{} {}".format(ip_list[ip_index] - 1), url) input('press any key to exit') sys.exit(0) except: pass
def status_view(request): """Status endpoint for monitoring app availability.""" dependent_services_ok = True secondary_services_ok = True # Test database connection for db_connection_key in DATABASES.keys(): try: connections[db_connection_key].cursor() except Exception as e: dependent_services_ok = False logger.error('Database "{}" connection error: {}'.format(db_connection_key, e)) # Test redis connection try: redis.StrictRedis(host=REDIS_SERVICE_HOSTNAME, socket_connect_timeout=3).ping() except Exception as e: secondary_services_ok = False logger.error('Redis connection error: {}'.format(str(e))) # Test elasticsearch connection try: if not get_es_client(timeout=3, max_retries=0).ping(): raise ValueError('No response from elasticsearch ping') except Exception as e: dependent_services_ok = False logger.error('Elasticsearch connection error: {}'.format(str(e))) # Test kibana connection try: resp = connection_from_url('http://{}'.format(KIBANA_SERVER)).urlopen('HEAD', '/status', timeout=3, retries=3) if resp.status >= 400: raise ValueError('Error {}: {}'.format(resp.status, resp.reason)) except Exception as e: secondary_services_ok = False logger.error('Kibana connection error: {}'.format(str(e))) return create_json_response( {'version': SEQR_VERSION, 'dependent_services_ok': dependent_services_ok, 'secondary_services_ok': secondary_services_ok}, status= 200 if dependent_services_ok else 400 )
def test_contextmanager(self): with connection_from_url('http://google.com:80') as pool: # Populate with some connections conn1 = pool._get_conn() conn2 = pool._get_conn() conn3 = pool._get_conn() pool._put_conn(conn1) pool._put_conn(conn2) old_pool_queue = pool.pool assert pool.pool is None with pytest.raises(ClosedPoolError): pool._get_conn() pool._put_conn(conn3) with pytest.raises(ClosedPoolError): pool._get_conn() with pytest.raises(Empty): old_pool_queue.get(block=False)
def http(url, directory, gpg_key=None): "Download a url via http and save to location" try: conn = connection_from_url( get_url_base(url), maxsize=20, retries=5, cert_reqs='CERT_REQUIRED', ca_certs=certifi.where(), ) filename = get_url_filename(url) req = conn.request('GET', url) data = req.data if len(data) and req.status == 200: filename = os.path.join(directory, filename) with open(filename, 'w') as handle: handle.write(data) return check_file(filename, gpg_key) return False else:
def test_absolute_url(self): c = connection_from_url('http://google.com:80') self.assertEqual('http://google.com:80/path?query=foo', c._absolute_url('path?query=foo'))
return self._follow_redirect(resp, request) except Exception, e: raise e return resp def _make_request(self, request): try: request = self._handlers.dispatch("request_prepare", request) except Exception, e: response = Response(status=400, message="Bad request", request=request) return response url = request.url conn = connectionpool.connection_from_url(str(url)) headers = self._merge_headers(request.headers) try: dispatch_response = self._handlers.dispatch("request_send", request) if isinstance(dispatch_response, Response): return dispatch_response except Exception, e: raise e # XXX fix in Url path = str(request.url.path) or "/" r = conn.urlopen( method=request.method, url=path, headers=headers, timeout=self.timeout, body=request.content, redirect=False )
def test_connection_refused(self): # Assume that port 31456 is not bound to by any server. pool = connection_from_url('http://localhost:31456') with self.assertRaises(MaxRetryError): pool.request('GET', '/')
def test_assert_same_host(self): with connection_from_url('http://google.com:80') as c: with pytest.raises(HostChangedError): c.request('GET', 'http://yahoo.com:80', assert_same_host=True)
def test_same_host(self, a, b): with connection_from_url(a) as c: assert c.is_same_host(b)
#!/usr/bin/env python # coding:utf-8 import threading, logging from urllib3 import connectionpool connectionpool.log.setLevel(logging.DEBUG) connectionpool.log.addHandler(logging.StreamHandler()) #pool = connectionpool.connection_from_url('http://5uproxy.net/http_fast.html', maxsize=10) # #def get(): # for i in range(3): # pool.get_url('http://5uproxy.net/http_fast.html') # #for i in range(10): # threading.Thread(target=get).start() pool = connectionpool.connection_from_url('www.python.org', maxsize=1) pool.urlopen('GET', '/') pool.urlopen('HEAD', '/') pool.urlopen('HEAD', '/') pool.urlopen('HEAD', '/') pool.urlopen('HEAD', '/') pool.urlopen('HEAD', '/')
def test_assert_same_host(self): with connection_from_url("http://google.com:80") as c: with pytest.raises(HostChangedError): c.request("GET", "http://yahoo.com:80", assert_same_host=True)
def test_ca_certs_default_cert_required(self): with connection_from_url('https://google.com:80', ca_certs=DEFAULT_CA) as pool: conn = pool._get_conn() self.assertEqual(conn.cert_reqs, 'CERT_REQUIRED')
def test_absolute_url(self): with connection_from_url('http://google.com:80') as c: assert 'http://google.com:80/path?query=foo' == c._absolute_url('path?query=foo')
def test_ca_certs_default_cert_required(self): with connection_from_url('https://google.com:80', ca_certs=DEFAULT_CA) as pool: conn = pool._get_conn() assert conn.cert_reqs == 'CERT_REQUIRED'
def test_not_same_host(self, a, b): with connection_from_url(a) as c: assert not c.is_same_host(b) with connection_from_url(b) as c: assert not c.is_same_host(a)
raise e return resp def _make_request(self, request): try: request = self._handlers.dispatch('request_prepare', request) except Exception, e: response = Response(status=400, message='Bad request', request=request) return response url = request.url conn = connectionpool.connection_from_url(str(url)) headers = self._merge_headers(request.headers) try: dispatch_response = self._handlers.dispatch( 'request_send', request) if (isinstance(dispatch_response, Response)): return dispatch_response except Exception, e: raise e # XXX fix in Url path = str(request.url.path) or '/' r = conn.urlopen( method=request.method,
def test_absolute_url(self): with connection_from_url('http://google.com:80') as c: assert 'http://google.com:80/path?query=foo' == c._absolute_url( 'path?query=foo')
def test_absolute_url(self): with connection_from_url("http://google.com:80") as c: assert "http://google.com:80/path?query=foo" == c._absolute_url( "path?query=foo")
def test_ca_certs_default_cert_required(self): with connection_from_url("https://google.com:80", ca_certs=DEFAULT_CA) as pool: conn = pool._get_conn() assert conn.cert_reqs == ssl.CERT_REQUIRED
def test_ca_certs_default_cert_required(self): with connection_from_url('https://google.com:80', ca_certs='/etc/ssl/certs/custom.pem') as pool: conn = pool._get_conn() self.assertEqual(conn.cert_reqs, 'CERT_REQUIRED')
def test_absolute_url(self): c = connection_from_url('http://google.com:80') self.assertEqual( 'http://google.com:80/path?query=foo', c._absolute_url('path?query=foo'))
def __init__(self, url): self._http = connection_from_url(url) self.url = url if not self.url.endswith('/'): self.url += '/'