class TestHTTPS_TLSv1(HTTPSDummyServerTestCase):
certs = DEFAULT_CERTS.copy()
certs["ssl_version"] = ssl.PROTOCOL_TLSv1
def setUp(self):
self._pool = HTTPSConnectionPool(self.host, self.port)
def test_set_ssl_version_to_sslv3(self):
self._pool.ssl_version = ssl.PROTOCOL_SSLv3
self.assertRaises(SSLError, self._pool.request, "GET", "/")
def test_ssl_version_as_string(self):
self._pool.ssl_version = "PROTOCOL_SSLv3"
self.assertRaises(SSLError, self._pool.request, "GET", "/")
def test_ssl_version_as_short_string(self):
self._pool.ssl_version = "SSLv3"
self.assertRaises(SSLError, self._pool.request, "GET", "/")
def test_discards_connection_on_sslerror(self):
self._pool.cert_reqs = "CERT_REQUIRED"
self.assertRaises(SSLError, self._pool.request, "GET", "/")
self._pool.ca_certs = DEFAULT_CA
self._pool.request("GET", "/")
def test_set_cert_default_cert_required(self):
conn = VerifiedHTTPSConnection(self.host, self.port)
conn.set_cert(ca_certs="/etc/ssl/certs/custom.pem")
self.assertEqual(conn.cert_reqs, "CERT_REQUIRED")
def test_invalid_common_name(self):
https_pool = HTTPSConnectionPool("127.0.0.1", self.port, cert_reqs="CERT_REQUIRED", ca_certs=DEFAULT_CA)
try:
https_pool.request("GET", "/")
self.fail("Didn't raise SSL invalid common name")
except SSLError as e:
self.assertTrue("doesn't match" in str(e))
def test_assert_fingerprint_sha256(self):
https_pool = HTTPSConnectionPool("localhost", self.port, cert_reqs="CERT_REQUIRED", ca_certs=DEFAULT_CA)
https_pool.assert_fingerprint = (
"9A:29:9D:4F:47:85:1C:51:23:F5:9A:A3:" "0F:5A:EF:96:F9:2E:3C:22:2E:FC:E8:BC:" "0E:73:90:37:ED:3B:AA:AB"
)
https_pool.request("GET", "/")
def request():
try:
pool = HTTPSConnectionPool(self.host, self.port, assert_fingerprint=fingerprint)
response = pool.urlopen("GET", "/", preload_content=False, timeout=Timeout(connect=1, read=0.001))
response.read()
finally:
pool.close()
def test_can_validate_ip_san(self):
"""Ensure that urllib3 can validate SANs with IP addresses in them."""
https_pool = HTTPSConnectionPool('127.0.0.1', self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
r = https_pool.request('GET', '/')
self.assertEqual(r.status, 200)
def test_verified(self):
https_pool = HTTPSConnectionPool(self.host, self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
self.addCleanup(https_pool.close)
conn = https_pool._new_conn()
self.assertEqual(conn.__class__, VerifiedHTTPSConnection)
with mock.patch('warnings.warn') as warn:
r = https_pool.request('GET', '/')
self.assertEqual(r.status, 200)
# Modern versions of Python, or systems using PyOpenSSL, don't
# emit warnings.
if sys.version_info >= (2, 7, 9) or util.IS_PYOPENSSL \
or util.IS_SECURETRANSPORT:
self.assertFalse(warn.called, warn.call_args_list)
else:
self.assertTrue(warn.called)
if util.HAS_SNI:
call = warn.call_args_list[0]
else:
call = warn.call_args_list[1]
error = call[0][1]
self.assertEqual(error, InsecurePlatformWarning)
class TestHTTPS_TLSv1(HTTPSDummyServerTestCase):
certs = DEFAULT_CERTS.copy()
certs['ssl_version'] = ssl.PROTOCOL_TLSv1
def setUp(self):
self._pool = HTTPSConnectionPool(self.host, self.port)
def test_set_ssl_version_to_sslv3(self):
self._pool.ssl_version = ssl.PROTOCOL_SSLv3
self.assertRaises(SSLError, self._pool.request, 'GET', '/')
def test_ssl_version_as_string(self):
self._pool.ssl_version = 'PROTOCOL_SSLv3'
self.assertRaises(SSLError, self._pool.request, 'GET', '/')
def test_ssl_version_as_short_string(self):
self._pool.ssl_version = 'SSLv3'
self.assertRaises(SSLError, self._pool.request, 'GET', '/')
def test_discards_connection_on_sslerror(self):
self._pool.cert_reqs = 'CERT_REQUIRED'
self.assertRaises(SSLError, self._pool.request, 'GET', '/')
self._pool.ca_certs = DEFAULT_CA
self._pool.request('GET', '/')
def test_set_cert_default_cert_required(self):
conn = VerifiedHTTPSConnection(self.host, self.port)
conn.set_cert(ca_certs=DEFAULT_CA)
self.assertEqual(conn.cert_reqs, 'CERT_REQUIRED')
def test_assert_hostname_false(self):
https_pool = HTTPSConnectionPool('127.0.0.1', self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
https_pool.assert_hostname = False
https_pool.request('GET', '/')
def test_ssl_verified_with_platform_ca_certs(self):
"""
We should rely on the platform CA file to validate authenticity of SSL
certificates. Since this file is used by many components of the OS,
such as curl, apt-get, etc., we decided to not touch it, in order to
not compromise the security of the OS running the test suite (typically
urllib3 developer's OS).
This test assumes that httpbin.org uses a certificate signed by a well
known Certificate Authority.
"""
try:
import urllib3.contrib.pyopenssl
except ImportError:
raise SkipTest('Test requires PyOpenSSL')
if (urllib3.connection.ssl_wrap_socket is
urllib3.contrib.pyopenssl.orig_connection_ssl_wrap_socket):
# Not patched
raise SkipTest('Test should only be run after PyOpenSSL '
'monkey patching')
https_pool = HTTPSConnectionPool('httpbin.org', 443,
cert_reqs=ssl.CERT_REQUIRED)
https_pool.request('HEAD', '/')
def test_invalid_ca_certs(self):
https_pool = HTTPSConnectionPool(self.host, self.port, cert_reqs="CERT_REQUIRED")
# Empty string won't throw on py2
https_pool.ca_certs = "/no_valid_path_to_ca_certs"
self.assertRaises(SSLError, https_pool.request, "GET", "/")
def test_verified_with_context(self):
ctx = util.ssl_.create_urllib3_context(cert_reqs=ssl.CERT_REQUIRED)
ctx.load_verify_locations(cafile=DEFAULT_CA)
https_pool = HTTPSConnectionPool(self.host, self.port,
ssl_context=ctx)
self.addCleanup(https_pool.close)
conn = https_pool._new_conn()
self.assertEqual(conn.__class__, VerifiedHTTPSConnection)
with mock.patch('warnings.warn') as warn:
r = https_pool.request('GET', '/')
self.assertEqual(r.status, 200)
# Modern versions of Python, or systems using PyOpenSSL, don't
# emit warnings.
if sys.version_info >= (2, 7, 9) or util.IS_PYOPENSSL:
self.assertFalse(warn.called, warn.call_args_list)
else:
self.assertTrue(warn.called)
if util.HAS_SNI:
call = warn.call_args_list[0]
else:
call = warn.call_args_list[1]
error = call[0][1]
self.assertEqual(error, InsecurePlatformWarning)
def test_assert_specific_hostname(self):
https_pool = HTTPSConnectionPool('127.0.0.1', self.port,
cert_reqs='CERT_REQUIRED')
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_hostname = 'localhost'
https_pool.request('GET', '/')
def test_strip_square_brackets_before_validating(self):
"""Test that the fix for #760 works."""
if not HAS_IPV6:
raise SkipTest("Only runs on IPv6 systems")
https_pool = HTTPSConnectionPool("[::1]", self.port, cert_reqs="CERT_REQUIRED", ca_certs=IPV6_ADDR_CA)
r = https_pool.request("GET", "/")
self.assertEqual(r.status, 200)
def test_cert_reqs_as_constant(self):
https_pool = HTTPSConnectionPool(self.host, self.port,
cert_reqs=ssl.CERT_REQUIRED)
https_pool.ca_certs = DEFAULT_CA_BAD
# if we pass in an invalid value it defaults to CERT_NONE
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
def test_assert_fingerprint_md5(self):
https_pool = HTTPSConnectionPool('127.0.0.1', self.port,
cert_reqs='CERT_REQUIRED')
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_fingerprint = 'CA:84:E1:AD0E5a:ef:2f:C3:09' \
':E7:30:F8:CD:C8:5B'
https_pool.request('GET', '/')
def test_source_address(self):
for addr in VALID_SOURCE_ADDRESSES:
https_pool = HTTPSConnectionPool(
self.host, self.port, cert_reqs='CERT_REQUIRED',
source_address=addr)
https_pool.ca_certs = DEFAULT_CA
r = https_pool.request('GET', '/source_address')
assert r.data == b(addr[0])
def test_source_address_error(self):
for addr in INVALID_SOURCE_ADDRESSES:
https_pool = HTTPSConnectionPool(
self.host, self.port, cert_reqs='CERT_REQUIRED',
source_address=addr)
https_pool.ca_certs = DEFAULT_CA
self.assertRaises(
MaxRetryError, https_pool.request, 'GET', '/source_address')
def test_assert_specific_hostname(self):
https_pool = HTTPSConnectionPool('localhost', self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
self.addCleanup(https_pool.close)
https_pool.assert_hostname = 'localhost'
https_pool.request('GET', '/')
def test_warning_for_certs_without_a_san(self):
"""Ensure that a warning is raised when the cert from the server has
no Subject Alternative Name."""
with mock.patch("warnings.warn") as warn:
https_pool = HTTPSConnectionPool(self.host, self.port, cert_reqs="CERT_REQUIRED", ca_certs=NO_SAN_CA)
r = https_pool.request("GET", "/")
self.assertEqual(r.status, 200)
self.assertTrue(warn.called)
def test_verify_none_and_good_fingerprint(self):
https_pool = HTTPSConnectionPool('127.0.0.1', self.port,
cert_reqs='CERT_NONE',
ca_certs=DEFAULT_CA_BAD)
https_pool.assert_fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:' \
'BF:93:CF:F9:71:CC:07:7D:0A'
https_pool.request('GET', '/')
def test_assert_fingerprint_sha1(self):
https_pool = HTTPSConnectionPool('localhost', self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
https_pool.assert_fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:' \
'BF:93:CF:F9:71:CC:07:7D:0A'
https_pool.request('GET', '/')
def test_no_ssl(self):
pool = HTTPSConnectionPool(self.host, self.port)
pool.ConnectionCls = None
self.addCleanup(pool.close)
self.assertRaises(SSLError, pool._new_conn)
with self.assertRaises(MaxRetryError) as cm:
pool.request('GET', '/', retries=0)
self.assertIsInstance(cm.exception.reason, SSLError)
def test_good_fingerprint_and_hostname_mismatch(self):
https_pool = HTTPSConnectionPool('127.0.0.1', self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
https_pool.assert_fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:' \
'BF:93:CF:F9:71:CC:07:7D:0A'
https_pool.request('GET', '/')
def test_strip_square_brackets_before_validating(self):
"""Test that the fix for #760 works."""
https_pool = HTTPSConnectionPool('[::1]', self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=IPV6_ADDR_CA)
self.addCleanup(https_pool.close)
r = https_pool.request('GET', '/')
self.assertEqual(r.status, 200)
def test_verify_none_and_bad_fingerprint(self):
https_pool = HTTPSConnectionPool('127.0.0.1', self.port,
cert_reqs='CERT_NONE',
ca_certs=DEFAULT_CA_BAD)
https_pool.assert_fingerprint = 'AA:AA:AA:AA:AA:AAAA:AA:AAAA:AA:' \
'AA:AA:AA:AA:AA:AA:AA:AA:AA'
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
def test_verify_none_and_good_fingerprint(self):
https_pool = HTTPSConnectionPool('127.0.0.1', self.port,
cert_reqs='CERT_NONE',
ca_certs=DEFAULT_CA_BAD)
https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \
'7A:F2:8A:D7:1E:07:33:67:DE'
https_pool.request('GET', '/')
def test_assert_fingerprint_sha1(self):
https_pool = HTTPSConnectionPool('127.0.0.1', self.port,
cert_reqs='CERT_REQUIRED')
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \
'7A:F2:8A:D7:1E:07:33:67:DE'
https_pool.request('GET', '/')
def test_enhanced_ssl_connection(self):
conn = VerifiedHTTPSConnection(self.host, self.port)
https_pool = HTTPSConnectionPool(
self.host, self.port, timeout=Timeout(total=None, connect=5), cert_reqs="CERT_REQUIRED"
)
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_fingerprint = "CC:45:6A:90:82:F7FF:C0:8218:8e:" "7A:F2:8A:D7:1E:07:33:67:DE"
https_pool._make_request(conn, "GET", "/")
def test_enhanced_ssl_connection(self):
fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:7A:F2:8A:D7:1E:07:33:67:DE'
conn = VerifiedHTTPSConnection(self.host, self.port)
https_pool = HTTPSConnectionPool(self.host, self.port,
cert_reqs='CERT_REQUIRED', ca_certs=DEFAULT_CA,
assert_fingerprint=fingerprint)
https_pool._make_request(conn, 'GET', '/')
def test_enhanced_ssl_connection(self):
fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:BF:93:CF:F9:71:CC:07:7D:0A'
conn = VerifiedHTTPSConnection(self.host, self.port)
https_pool = HTTPSConnectionPool(self.host, self.port,
cert_reqs='CERT_REQUIRED', ca_certs=DEFAULT_CA,
assert_fingerprint=fingerprint)
https_pool._make_request(conn, 'GET', '/')
def __init__(self): self.handle = HTTPSConnectionPool(RapleafApi.host, timeout = RapleafApi.timeout)
class TestHTTPS(HTTPSDummyServerTestCase):
def setUp(self):
self._pool = HTTPSConnectionPool(self.host, self.port)
def test_simple(self):
r = self._pool.request('GET', '/')
self.assertEqual(r.status, 200, r.data)
def test_set_ssl_version_to_tlsv1(self):
self._pool.ssl_version = ssl.PROTOCOL_TLSv1
r = self._pool.request('GET', '/')
self.assertEqual(r.status, 200, r.data)
def test_verified(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_REQUIRED')
conn = https_pool._new_conn()
self.assertEqual(conn.__class__, VerifiedHTTPSConnection)
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
https_pool.ca_certs = DEFAULT_CA_BAD
try:
https_pool.request('GET', '/')
self.fail("Didn't raise SSL error with wrong CA")
except SSLError as e:
self.assertTrue(
'certificate verify failed' in str(e),
"Expected 'certificate verify failed',"
"instead got: %r" % e)
https_pool.ca_certs = DEFAULT_CA
https_pool.request('GET', '/') # Should succeed without exceptions.
https_fail_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_REQUIRED')
https_fail_pool.ca_certs = DEFAULT_CA
try:
https_fail_pool.request('GET', '/')
self.fail("Didn't raise SSL invalid common name")
except SSLError as e:
self.assertTrue("doesn't match" in str(e))
def test_no_ssl(self):
OriginalConnectionCls = self._pool.ConnectionCls
try:
self._pool.ConnectionCls = None
self.assertRaises(SSLError, self._pool._new_conn)
self.assertRaises(SSLError, self._pool.request, 'GET', '/')
finally:
self._pool.ConnectionCls = OriginalConnectionCls
def test_unverified_ssl(self):
""" Test that bare HTTPSConnection can connect, make requests """
try:
OriginalConnectionCls = self._pool.ConnectionCls
self._pool.ConnectionCls = UnverifiedHTTPSConnection
self._pool.request('GET', '/')
finally:
self._pool.ConnectionCls = OriginalConnectionCls
def test_cert_reqs_as_constant(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs=ssl.CERT_REQUIRED)
https_pool.ca_certs = DEFAULT_CA_BAD
# if we pass in an invalid value it defaults to CERT_NONE
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
def test_cert_reqs_as_short_string(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='REQUIRED')
https_pool.ca_certs = DEFAULT_CA_BAD
# if we pass in an invalid value it defaults to CERT_NONE
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
def test_ssl_unverified_with_ca_certs(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_NONE')
https_pool.ca_certs = DEFAULT_CA_BAD
https_pool.request('GET', '/')
@requires_network
def test_ssl_verified_with_platform_ca_certs(self):
"""
This test check that whe rely on platform CA file to validate
authenticity of SSL certificate. Since this file is used by many
components of the OS, such as curl, apt-get, etc., we decided to not
touch it, in order to not compromise the security of the OS
running the test suite (typically urllib3 developer's OS).
This test assume that httpbin.org use a certificate signed
by a well known Certificate Authority.
"""
try:
import urllib3.contrib.pyopenssl
except ImportError:
raise SkipTest('This test needs pyopenssl support')
if (urllib3.connection.ssl_wrap_socket is
urllib3.contrib.pyopenssl.orig_connection_ssl_wrap_socket):
# Not patched
raise SkipTest('This test needs pyopenssl support')
https_pool = HTTPSConnectionPool('httpbin.org',
443,
cert_reqs=ssl.CERT_REQUIRED)
https_pool.request('HEAD', '/')
def test_verified_without_ca_certs(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_REQUIRED')
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
def test_invalid_ca_certs(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_REQUIRED')
# Empty string won't throw on py2
https_pool.ca_certs = '/no_valid_path_to_ca_certs'
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
def test_assert_hostname_false(self):
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_REQUIRED')
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_hostname = False
https_pool.request('GET', '/')
def test_assert_specific_hostname(self):
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_REQUIRED')
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_hostname = 'localhost'
https_pool.request('GET', '/')
def test_assert_fingerprint_md5(self):
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_REQUIRED')
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_fingerprint = 'CA:84:E1:AD0E5a:ef:2f:C3:09' \
':E7:30:F8:CD:C8:5B'
https_pool.request('GET', '/')
def test_assert_fingerprint_sha1(self):
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_REQUIRED')
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \
'7A:F2:8A:D7:1E:07:33:67:DE'
https_pool.request('GET', '/')
def test_assert_invalid_fingerprint(self):
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_REQUIRED')
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_fingerprint = 'AA:AA:AA:AA:AA:AAAA:AA:AAAA:AA:' \
'AA:AA:AA:AA:AA:AA:AA:AA:AA'
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
# invalid length
https_pool.assert_fingerprint = 'AA'
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
# uneven length
https_pool.assert_fingerprint = 'AA:A'
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
@requires_network
def test_https_timeout(self):
timeout = Timeout(connect=0.001)
https_pool = HTTPSConnectionPool(TARPIT_HOST,
self.port,
timeout=timeout,
cert_reqs='CERT_REQUIRED')
timeout = Timeout(total=None, connect=0.001)
https_pool = HTTPSConnectionPool(TARPIT_HOST,
self.port,
timeout=timeout,
cert_reqs='CERT_REQUIRED')
self.assertRaises(ConnectTimeoutError, https_pool.request, 'GET', '/')
timeout = Timeout(read=0.001)
https_pool = HTTPSConnectionPool(self.host,
self.port,
timeout=timeout,
cert_reqs='CERT_REQUIRED')
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \
'7A:F2:8A:D7:1E:07:33:67:DE'
url = '/sleep?seconds=0.005'
self.assertRaises(ReadTimeoutError, https_pool.request, 'GET', url)
timeout = Timeout(total=None)
https_pool = HTTPSConnectionPool(self.host,
self.port,
timeout=timeout,
cert_reqs='CERT_NONE')
https_pool.request('GET', '/')
def test_tunnel(self):
""" test the _tunnel behavior """
timeout = Timeout(total=None)
https_pool = HTTPSConnectionPool(self.host,
self.port,
timeout=timeout,
cert_reqs='CERT_NONE')
conn = https_pool._new_conn()
try:
conn.set_tunnel(self.host, self.port)
except AttributeError: # python 2.6
conn._set_tunnel(self.host, self.port)
conn._tunnel = mock.Mock()
https_pool._make_request(conn, 'GET', '/')
conn._tunnel.assert_called_once_with()
def test_tunnel_old_python(self):
"""HTTPSConnection can still make connections if _tunnel_host isn't set
The _tunnel_host attribute was added in 2.6.3 - because our test runners
generally use the latest Python 2.6, we simulate the old version by
deleting the attribute from the HTTPSConnection.
"""
conn = self._pool._new_conn()
del conn._tunnel_host
self._pool._make_request(conn, 'GET', '/')
@requires_network
def test_enhanced_timeout(self):
def new_pool(timeout, cert_reqs='CERT_REQUIRED'):
https_pool = HTTPSConnectionPool(TARPIT_HOST,
self.port,
timeout=timeout,
cert_reqs=cert_reqs)
return https_pool
https_pool = new_pool(Timeout(connect=0.001))
conn = https_pool._new_conn()
self.assertRaises(ConnectTimeoutError, https_pool.request, 'GET', '/')
self.assertRaises(ConnectTimeoutError, https_pool._make_request, conn,
'GET', '/')
https_pool = new_pool(Timeout(connect=5))
self.assertRaises(ConnectTimeoutError,
https_pool.request,
'GET',
'/',
timeout=Timeout(connect=0.001))
t = Timeout(total=None)
https_pool = new_pool(t)
conn = https_pool._new_conn()
self.assertRaises(ConnectTimeoutError,
https_pool.request,
'GET',
'/',
timeout=Timeout(total=None, connect=0.001))
def test_enhanced_ssl_connection(self):
conn = VerifiedHTTPSConnection(self.host, self.port)
https_pool = HTTPSConnectionPool(self.host,
self.port,
timeout=Timeout(total=None,
connect=5),
cert_reqs='CERT_REQUIRED')
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \
'7A:F2:8A:D7:1E:07:33:67:DE'
https_pool._make_request(conn, 'GET', '/')
def test_simple(self):
with HTTPSConnectionPool(self.host, self.port,
ca_certs=DEFAULT_CA) as https_pool:
r = https_pool.request("GET", "/")
assert r.status == 200, r.data
def setup_method(self, method):
self._pool = HTTPSConnectionPool(self.host,
self.port,
ca_certs=DEFAULT_CA)
def __init__(self, api_key):
self.handle = HTTPSConnectionPool(RapleafApi.HOST, timeout=RapleafApi.TIMEOUT)
self.base_path = RapleafApi.BASE_PATH + '?api_key=%s' % (api_key)
class RapleafApi:
HEADERS = {'User-Agent' : 'RapleafApi/Python/1.1'}
BASE_PATH = '/v4/dr'
HOST = 'personalize.rapleaf.com'
TIMEOUT = 2.0
def __init__(self, api_key):
self.handle = HTTPSConnectionPool(RapleafApi.HOST, timeout=RapleafApi.TIMEOUT)
self.base_path = RapleafApi.BASE_PATH + '?api_key=%s' % (api_key)
def query_by_email(self, email, hash_email=False, show_available=False):
"""
Takes an e-mail and returns a hash which maps attribute fields onto attributes
If the hash_email option is set, then the email will be hashed before it's sent to Rapleaf.
If the show_available option is set, then the string "Data Available" will be returned for
those fields which the API account is not subscribed but for which RapLeaf has data.
"""
if hash_email:
s = hashlib.sha1()
s.update(email.lower())
return self.query_by_sha1(s.hexdigest(), show_available)
url = '%s&email=%s' % (self.base_path, quote(email))
return self.__get_json_response(url, show_available)
def query_by_md5(self, md5_email, show_available=False):
"""
Takes an e-mail that has already been hashed by md5
and returns a hash which maps attribute fields onto attributes.
If the show_available option is set, then the string "Data Available" will be returned for
those fields which the API account is not subscribed but for which RapLeaf has data.
"""
url = '%s&md5_email=%s' % (self.base_path, quote(md5_email))
return self.__get_json_response(url, show_available)
def query_by_sha1(self, sha1_email, show_available=False):
"""
Takes an e-mail that has already been hashed by sha1
and returns a hash which maps attribute fields onto attributes.
If the show_available option is set, then the string "Data Available" will be returned for
those fields which the API account is not subscribed but for which RapLeaf has data.
"""
url = '%s&sha1_email=%s' % (self.base_path, quote(sha1_email))
return self.__get_json_response(url, show_available)
def query_by_nap(self, first, last, street, city, state, email=None, show_available=False):
"""
Takes first name, last name, and postal (street, city, and state acronym),
and returns a hash which maps attribute fields onto attributes
Though not necessary, adding an e-mail increases hit rate.
If the show_available option is set, then the string "Data Available" will be returned for
those fields which the API account is not subscribed but for which RapLeaf has data.
"""
url = '%s&first=%s&last=%s&street=%s&city=%s&state=%s' % (
self.base_path, quote(first), quote(last),
quote(street), quote(city), quote(state))
if email:
url = '%s&email=%s' % (url, quote(email))
return self.__get_json_response(url, show_available)
def query_by_naz(self, first, last, zip4, email=None, show_available=False):
"""
Takes first name, last name, and zip4 code (5-digit zip
and 4-digit extension separated by a dash as a string),
and returns a hash which maps attribute fields onto attributes
Though not necessary, adding an e-mail increases hit rate.
If the show_available option is set, then the string "Data Available" will be returned for
those fields which the API account is not subscribed but for which RapLeaf has data.
"""
url = '%s&first=%s&last=%s&zip4=%s' % (
self.base_path, quote(first), quote(last), zip4)
if email:
url = '%s&email=%s' % (url, quote(email))
return self.__get_json_response(url, show_available)
def __get_json_response(self, path, show_available=False):
"""
Pre: Path is an extension to personalize.rapleaf.com
Note that an exception is raised if an HTTP response code
other than 200 is sent back. In this case, both the error code
the error code and error body are accessible from the exception raised
"""
if show_available:
path += '&show_available'
json_response = self.handle.request('GET', path, headers=RapleafApi.HEADERS)
if 200 <= json_response.status < 300:
if json_response.data:
return json.JSONDecoder().decode(json_response.data)
else:
return {}
else:
raise Exception(json_response.status, json_response.data.decode("utf-8"))
class HTTP(object):
""" Wrapper for HTTP method calls.
"""
@staticmethod
def authorization(user, password):
return 'Basic ' + b64encode(
(user + ":" + password).encode("utf-8")).decode("ascii")
def __init__(self, uri, headers, verified):
self.uri = uri
self.verified = verified
parts = urlsplit(uri)
scheme = parts.scheme
host = parts.hostname
port = parts.port
if scheme == "http":
from urllib3 import HTTPConnectionPool
self._http = HTTPConnectionPool("%s:%d" % (host, port))
elif scheme == "https":
from urllib3 import HTTPSConnectionPool
if verified:
from certifi import where
self._http = HTTPSConnectionPool("%s:%d" % (host, port),
cert_reqs="CERT_REQUIRED",
ca_certs=where())
else:
self._http = HTTPSConnectionPool("%s:%d" % (host, port))
else:
raise ValueError("Unsupported scheme %r" % scheme)
self.path = parts.path
if "auth" in headers:
user, password = headers.pop("auth")
headers["Authorization"] = 'Basic ' + b64encode(
(ustr(user) + u":" +
ustr(password)).encode("utf-8")).decode("ascii")
self.headers = headers
def __del__(self):
self.close()
def __eq__(self, other):
try:
return self.uri == other.uri
except AttributeError:
return False
def __ne__(self, other):
return not self.__eq__(other)
def request(self, method, url, fields=None, headers=None, **urlopen_kw):
from neo4j.v1 import ServiceUnavailable
from urllib3.exceptions import MaxRetryError
try:
if self.verified:
return self._http.request(method, url, fields, headers,
**urlopen_kw)
else:
with catch_warnings():
simplefilter("ignore")
return self._http.request(method, url, fields, headers,
**urlopen_kw)
except MaxRetryError:
raise ServiceUnavailable("Cannot send %s request to <%s>" %
(method, url))
def get_json(self, ref):
""" Perform an HTTP GET to this resource and return JSON.
"""
rs = self.request("GET", self.path + ref, headers=self.headers)
try:
if rs.status == 200:
return json_loads(rs.data.decode('utf-8'))
else:
self.raise_error(rs.status, rs.data)
finally:
rs.close()
def post(self, ref, json, expected):
""" Perform an HTTP POST to this resource.
"""
headers = dict(self.headers)
if json is not None:
headers["Content-Type"] = "application/json"
json = json_dumps(json).encode('utf-8')
rs = self.request("POST",
self.path + ref,
headers=self.headers,
body=json)
if rs.status not in expected:
self.raise_error(rs.status, rs.data)
return rs
def delete(self, ref, expected):
""" Perform an HTTP DELETE to this resource.
"""
rs = self.request("DELETE", self.path + ref, headers=self.headers)
if rs.status not in expected:
self.raise_error(rs.status, rs.data)
return rs
def close(self):
if self._http and self._http.pool:
self._http.close()
def raise_error(self, status_code, data):
if status_code == UNAUTHORIZED:
raise AuthError(self.uri)
if status_code == FORBIDDEN:
raise Forbidden(self.uri)
if data:
content = json_loads(data.decode('utf-8'))
else:
content = {}
message = content.pop(
"message",
"HTTP request to <%s> returned unexpected status code %s" %
(self.uri, status_code))
error = GraphError(message, **content)
error.http_status_code = status_code
raise error
class TestHTTPS(HTTPSDummyServerTestCase):
tls_protocol_name = None
def setup_method(self, method):
self._pool = HTTPSConnectionPool(self.host,
self.port,
ca_certs=DEFAULT_CA)
def teardown_method(self, method):
self._pool.close()
def test_simple(self):
r = self._pool.request("GET", "/")
assert r.status == 200, r.data
@fails_on_travis_gce
def test_dotted_fqdn(self):
with HTTPSConnectionPool(self.host + ".",
self.port,
ca_certs=DEFAULT_CA) as pool:
r = pool.request("GET", "/")
assert r.status == 200, r.data
def test_client_intermediate(self):
client_cert, client_key = (
DEFAULT_CLIENT_CERTS["certfile"],
DEFAULT_CLIENT_CERTS["keyfile"],
)
with HTTPSConnectionPool(
self.host,
self.port,
key_file=client_key,
cert_file=client_cert,
ca_certs=DEFAULT_CA,
) as https_pool:
r = https_pool.request("GET", "/certificate")
subject = json.loads(r.data.decode("utf-8"))
assert subject["organizationalUnitName"].startswith(
"Testing server cert")
def test_client_no_intermediate(self):
client_cert, client_key = (
DEFAULT_CLIENT_NO_INTERMEDIATE_CERTS["certfile"],
DEFAULT_CLIENT_NO_INTERMEDIATE_CERTS["keyfile"],
)
with HTTPSConnectionPool(
self.host,
self.port,
cert_file=client_cert,
key_file=client_key,
ca_certs=DEFAULT_CA,
) as https_pool:
try:
https_pool.request("GET", "/certificate", retries=False)
except SSLError as e:
if not ("alert unknown ca" in str(e)
or "invalid certificate chain" in str(e)
or "unknown Cert Authority" in str(e) or
# https://github.com/urllib3/urllib3/issues/1422
"connection closed via error" in str(e)
or "WSAECONNRESET" in str(e)):
raise
except ProtocolError as e:
if not ("An existing connection was forcibly closed by the remote host"
in str(e)
# Python 3.7.4+
or "WSAECONNRESET" in str(e) # Windows
or "EPIPE" in str(e) # macOS
):
raise
@requires_ssl_context_keyfile_password
def test_client_key_password(self):
client_cert, client_key = (
DEFAULT_CLIENT_CERTS["certfile"],
PASSWORD_CLIENT_KEYFILE,
)
with HTTPSConnectionPool(
self.host,
self.port,
ca_certs=DEFAULT_CA,
key_file=client_key,
cert_file=client_cert,
key_password="******",
) as https_pool:
r = https_pool.request("GET", "/certificate")
subject = json.loads(r.data.decode("utf-8"))
assert subject["organizationalUnitName"].startswith(
"Testing server cert")
@requires_ssl_context_keyfile_password
def test_client_encrypted_key_requires_password(self):
client_cert, client_key = (
DEFAULT_CLIENT_CERTS["certfile"],
PASSWORD_CLIENT_KEYFILE,
)
with pytest.raises(SSLError) as e:
HTTPSConnectionPool(
self.host,
self.port,
key_file=client_key,
cert_file=client_cert,
key_password=None,
)
assert "password is required" in str(e.value)
def test_verified(self):
with HTTPSConnectionPool(self.host,
self.port,
cert_reqs="CERT_REQUIRED",
ca_certs=DEFAULT_CA) as https_pool:
with mock.patch("warnings.warn") as warn:
r = https_pool.request("GET", "/")
assert r.status == 200
# Modern versions of Python, or systems using PyOpenSSL, don't
# emit warnings.
if (sys.version_info >= (2, 7, 9) or util.IS_PYOPENSSL
or util.IS_SECURETRANSPORT):
assert not warn.called, warn.call_args_list
else:
assert warn.called
if util.HAS_SNI:
call = warn.call_args_list[0]
else:
call = warn.call_args_list[1]
error = call[0][1]
assert error == InsecurePlatformWarning
def test_verified_with_context(self):
ctx = util.ssl_.create_urllib3_context(cert_reqs=ssl.CERT_REQUIRED)
ctx.load_verify_locations(cafile=DEFAULT_CA)
with HTTPSConnectionPool(self.host, self.port,
ssl_context=ctx) as https_pool:
with mock.patch("warnings.warn") as warn:
r = https_pool.request("GET", "/")
assert r.status == 200
# Modern versions of Python, or systems using PyOpenSSL, don't
# emit warnings.
if (sys.version_info >= (2, 7, 9) or util.IS_PYOPENSSL
or util.IS_SECURETRANSPORT):
assert not warn.called, warn.call_args_list
else:
assert warn.called
if util.HAS_SNI:
call = warn.call_args_list[0]
else:
call = warn.call_args_list[1]
error = call[0][1]
assert error == InsecurePlatformWarning
def test_context_combines_with_ca_certs(self):
ctx = util.ssl_.create_urllib3_context(cert_reqs=ssl.CERT_REQUIRED)
with HTTPSConnectionPool(self.host,
self.port,
ca_certs=DEFAULT_CA,
ssl_context=ctx) as https_pool:
with mock.patch("warnings.warn") as warn:
r = https_pool.request("GET", "/")
assert r.status == 200
# Modern versions of Python, or systems using PyOpenSSL, don't
# emit warnings.
if (sys.version_info >= (2, 7, 9) or util.IS_PYOPENSSL
or util.IS_SECURETRANSPORT):
assert not warn.called, warn.call_args_list
else:
assert warn.called
if util.HAS_SNI:
call = warn.call_args_list[0]
else:
call = warn.call_args_list[1]
error = call[0][1]
assert error == InsecurePlatformWarning
@onlyPy279OrNewer
@notSecureTransport # SecureTransport does not support cert directories
@notOpenSSL098 # OpenSSL 0.9.8 does not support cert directories
def test_ca_dir_verified(self):
with HTTPSConnectionPool(self.host,
self.port,
cert_reqs="CERT_REQUIRED",
ca_cert_dir=DEFAULT_CA_DIR) as https_pool:
with mock.patch("warnings.warn") as warn:
r = https_pool.request("GET", "/")
assert r.status == 200
assert not warn.called, warn.call_args_list
def test_invalid_common_name(self):
with HTTPSConnectionPool("127.0.0.1",
self.port,
cert_reqs="CERT_REQUIRED",
ca_certs=DEFAULT_CA) as https_pool:
with pytest.raises(MaxRetryError) as e:
https_pool.request("GET", "/")
assert isinstance(e.value.reason, SSLError)
assert "doesn't match" in str(
e.value.reason) or "certificate verify failed" in str(
e.value.reason)
def test_verified_with_bad_ca_certs(self):
with HTTPSConnectionPool(self.host,
self.port,
cert_reqs="CERT_REQUIRED",
ca_certs=DEFAULT_CA_BAD) as https_pool:
with pytest.raises(MaxRetryError) as e:
https_pool.request("GET", "/")
assert isinstance(e.value.reason, SSLError)
assert "certificate verify failed" in str(e.value.reason), (
"Expected 'certificate verify failed', instead got: %r" %
e.value.reason)
def test_verified_without_ca_certs(self):
# default is cert_reqs=None which is ssl.CERT_NONE
with HTTPSConnectionPool(self.host,
self.port,
cert_reqs="CERT_REQUIRED") as https_pool:
with pytest.raises(MaxRetryError) as e:
https_pool.request("GET", "/")
assert isinstance(e.value.reason, SSLError)
# there is a different error message depending on whether or
# not pyopenssl is injected
assert ("No root certificates specified" in str(e.value.reason)
# PyPy sometimes uses all-caps here
or
"certificate verify failed" in str(e.value.reason).lower()
or "invalid certificate chain" in str(e.value.reason)), (
"Expected 'No root certificates specified', "
"'certificate verify failed', or "
"'invalid certificate chain', "
"instead got: %r" % e.value.reason)
def test_unverified_ssl(self):
""" Test that bare HTTPSConnection can connect, make requests """
with HTTPSConnectionPool(self.host, self.port,
cert_reqs=ssl.CERT_NONE) as pool:
with mock.patch("warnings.warn") as warn:
r = pool.request("GET", "/")
assert r.status == 200
assert warn.called
# Modern versions of Python, or systems using PyOpenSSL, only emit
# the unverified warning. Older systems may also emit other
# warnings, which we want to ignore here.
calls = warn.call_args_list
assert InsecureRequestWarning in [x[0][1] for x in calls]
def test_ssl_unverified_with_ca_certs(self):
with HTTPSConnectionPool(self.host,
self.port,
cert_reqs="CERT_NONE",
ca_certs=DEFAULT_CA_BAD) as pool:
with mock.patch("warnings.warn") as warn:
r = pool.request("GET", "/")
assert r.status == 200
assert warn.called
# Modern versions of Python, or systems using PyOpenSSL, only emit
# the unverified warning. Older systems may also emit other
# warnings, which we want to ignore here.
calls = warn.call_args_list
if (sys.version_info >= (2, 7, 9) or util.IS_PYOPENSSL
or util.IS_SECURETRANSPORT):
category = calls[0][0][1]
elif util.HAS_SNI:
category = calls[1][0][1]
else:
category = calls[2][0][1]
assert category == InsecureRequestWarning
def test_assert_hostname_false(self):
with HTTPSConnectionPool("localhost",
self.port,
cert_reqs="CERT_REQUIRED",
ca_certs=DEFAULT_CA) as https_pool:
https_pool.assert_hostname = False
https_pool.request("GET", "/")
def test_assert_specific_hostname(self):
with HTTPSConnectionPool("localhost",
self.port,
cert_reqs="CERT_REQUIRED",
ca_certs=DEFAULT_CA) as https_pool:
https_pool.assert_hostname = "localhost"
https_pool.request("GET", "/")
def test_server_hostname(self):
with HTTPSConnectionPool(
"127.0.0.1",
self.port,
cert_reqs="CERT_REQUIRED",
ca_certs=DEFAULT_CA,
server_hostname="localhost",
) as https_pool:
conn = https_pool._new_conn()
https_pool._start_conn(conn, connect_timeout=None)
# Assert the wrapping socket is using the passed-through SNI name.
# pyopenssl doesn't let you pull the server_hostname back off the
# socket, so only add this assertion if the attribute is there (i.e.
# the python ssl module).
# XXX This is highly-specific to SyncBackend
# See https://github.com/python-trio/urllib3/pull/54#discussion_r241683895
# for potential solutions
sock = conn._sock._sock
if hasattr(sock, "server_hostname"):
assert sock.server_hostname == "localhost"
def test_assert_fingerprint_md5(self):
with HTTPSConnectionPool("localhost",
self.port,
cert_reqs="CERT_REQUIRED",
ca_certs=DEFAULT_CA) as https_pool:
https_pool.assert_fingerprint = (
"F2:06:5A:42:10:3F:45:1C:17:FE:E6:07:1E:8A:86:E5")
https_pool.request("GET", "/")
def test_assert_fingerprint_sha1(self):
with HTTPSConnectionPool("localhost",
self.port,
cert_reqs="CERT_REQUIRED",
ca_certs=DEFAULT_CA) as https_pool:
https_pool.assert_fingerprint = (
"92:81:FE:85:F7:0C:26:60:EC:D6:B3:BF:93:CF:F9:71:CC:07:7D:0A")
https_pool.request("GET", "/")
def test_assert_fingerprint_sha256(self):
with HTTPSConnectionPool("localhost",
self.port,
cert_reqs="CERT_REQUIRED",
ca_certs=DEFAULT_CA) as https_pool:
https_pool.assert_fingerprint = (
"C5:4D:0B:83:84:89:2E:AE:B4:58:BB:12:"
"F7:A6:C4:76:05:03:88:D8:57:65:51:F3:"
"1E:60:B0:8B:70:18:64:E6")
https_pool.request("GET", "/")
def test_assert_invalid_fingerprint(self):
with HTTPSConnectionPool("127.0.0.1",
self.port,
cert_reqs="CERT_REQUIRED",
ca_certs=DEFAULT_CA) as https_pool:
https_pool.assert_fingerprint = (
"AA:AA:AA:AA:AA:AAAA:AA:AAAA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA")
def _test_request(pool):
with pytest.raises(MaxRetryError) as cm:
pool.request("GET", "/", retries=0)
assert isinstance(cm.value.reason, SSLError)
_test_request(https_pool)
https_pool._get_conn()
# Uneven length
https_pool.assert_fingerprint = "AA:A"
_test_request(https_pool)
https_pool._get_conn()
# Invalid length
https_pool.assert_fingerprint = "AA"
_test_request(https_pool)
def test_verify_none_and_bad_fingerprint(self):
with HTTPSConnectionPool("127.0.0.1",
self.port,
cert_reqs="CERT_NONE",
ca_certs=DEFAULT_CA_BAD) as https_pool:
https_pool.assert_fingerprint = (
"AA:AA:AA:AA:AA:AAAA:AA:AAAA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA")
with pytest.raises(MaxRetryError) as cm:
https_pool.request("GET", "/", retries=0)
assert isinstance(cm.value.reason, SSLError)
def test_verify_none_and_good_fingerprint(self):
with HTTPSConnectionPool("127.0.0.1",
self.port,
cert_reqs="CERT_NONE",
ca_certs=DEFAULT_CA_BAD) as https_pool:
https_pool.assert_fingerprint = (
"92:81:FE:85:F7:0C:26:60:EC:D6:B3:BF:93:CF:F9:71:CC:07:7D:0A")
https_pool.request("GET", "/")
@notSecureTransport
def test_good_fingerprint_and_hostname_mismatch(self):
# This test doesn't run with SecureTransport because we don't turn off
# hostname validation without turning off all validation, which this
# test doesn't do (deliberately). We should revisit this if we make
# new decisions.
with HTTPSConnectionPool("127.0.0.1",
self.port,
cert_reqs="CERT_REQUIRED",
ca_certs=DEFAULT_CA) as https_pool:
https_pool.assert_fingerprint = (
"92:81:FE:85:F7:0C:26:60:EC:D6:B3:BF:93:CF:F9:71:CC:07:7D:0A")
https_pool.request("GET", "/")
@requires_network
def test_https_timeout(self):
timeout = Timeout(total=None, connect=SHORT_TIMEOUT)
with HTTPSConnectionPool(
TARPIT_HOST,
self.port,
timeout=timeout,
retries=False,
cert_reqs="CERT_REQUIRED",
) as https_pool:
with pytest.raises(ConnectTimeoutError):
https_pool.request("GET", "/")
timeout = Timeout(read=0.01)
with HTTPSConnectionPool(
self.host,
self.port,
timeout=timeout,
retries=False,
cert_reqs="CERT_REQUIRED",
) as https_pool:
https_pool.assert_fingerprint = (
"92:81:FE:85:F7:0C:26:60:EC:D6:B3:BF:93:CF:F9:71:CC:07:7D:0A")
timeout = Timeout(total=None)
with HTTPSConnectionPool(self.host,
self.port,
timeout=timeout,
cert_reqs="CERT_NONE") as https_pool:
https_pool.request("GET", "/")
@requires_network
def test_enhanced_timeout(self):
with HTTPSConnectionPool(
TARPIT_HOST,
self.port,
timeout=Timeout(connect=SHORT_TIMEOUT),
retries=False,
cert_reqs="CERT_REQUIRED",
) as https_pool:
conn = https_pool._new_conn()
with pytest.raises(ConnectTimeoutError):
https_pool.request("GET", "/")
with pytest.raises(ConnectTimeoutError):
https_pool._make_request(conn, "GET", "/")
with HTTPSConnectionPool(
TARPIT_HOST,
self.port,
timeout=Timeout(connect=LONG_TIMEOUT),
retries=False,
cert_reqs="CERT_REQUIRED",
) as https_pool:
with pytest.raises(ConnectTimeoutError):
https_pool.request("GET",
"/",
timeout=Timeout(connect=SHORT_TIMEOUT))
with HTTPSConnectionPool(
TARPIT_HOST,
self.port,
timeout=Timeout(total=None),
retries=False,
cert_reqs="CERT_REQUIRED",
) as https_pool:
conn = https_pool._new_conn()
try:
with pytest.raises(ConnectTimeoutError):
https_pool.request("GET",
"/",
timeout=Timeout(total=None,
connect=SHORT_TIMEOUT))
finally:
conn.close()
def test_enhanced_ssl_connection(self):
fingerprint = "92:81:FE:85:F7:0C:26:60:EC:D6:B3:BF:93:CF:F9:71:CC:07:7D:0A"
with HTTPSConnectionPool(
self.host,
self.port,
cert_reqs="CERT_REQUIRED",
ca_certs=DEFAULT_CA,
assert_fingerprint=fingerprint,
) as https_pool:
r = https_pool.urlopen("GET", "/")
assert r.status == 200
@onlyPy279OrNewer
def test_ssl_correct_system_time(self):
w = self._request_without_resource_warnings("GET", "/")
assert [] == w
@onlyPy279OrNewer
def test_ssl_wrong_system_time(self):
with mock.patch("urllib3._sync.connection.datetime") as mock_date:
mock_date.date.today.return_value = datetime.date(1970, 1, 1)
w = self._request_without_resource_warnings("GET", "/")
assert len(w) == 1
warning = w[0]
assert SystemTimeWarning == warning.category
assert str(RECENT_DATE) in warning.message.args[0]
def _request_without_resource_warnings(self, method, url):
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
with HTTPSConnectionPool(self.host,
self.port,
cert_reqs="CERT_REQUIRED",
ca_certs=DEFAULT_CA) as pool:
pool.request(method, url)
return [x for x in w if not isinstance(x.message, ResourceWarning)]
def test_set_ssl_version_to_tls_version(self):
if self.tls_protocol_name is None:
pytest.skip("Skipping base test class")
self._pool.ssl_version = self.certs["ssl_version"]
r = self._pool.request("GET", "/")
assert r.status == 200, r.data
def test_set_cert_default_cert_required(self):
pool = HTTPSConnectionPool(self.host, self.port, ca_certs=DEFAULT_CA)
assert pool.ssl_context.verify_mode == ssl.CERT_REQUIRED
def test_tls_protocol_name_of_socket(self):
if self.tls_protocol_name is None:
pytest.skip("Skipping base test class")
conn = self._pool._get_conn()
conn.connect(self._pool.ssl_context)
if not hasattr(conn._sock, "_version"):
pytest.skip("_version() not available in backend")
assert conn._sock._version() == self.tls_protocol_name
def test_https_timeout(self):
timeout = Timeout(connect=0.001)
https_pool = HTTPSConnectionPool(TARPIT_HOST,
self.port,
timeout=timeout,
retries=False,
cert_reqs='CERT_REQUIRED')
timeout = Timeout(total=None, connect=0.001)
https_pool = HTTPSConnectionPool(TARPIT_HOST,
self.port,
timeout=timeout,
retries=False,
cert_reqs='CERT_REQUIRED')
self.assertRaises(ConnectTimeoutError, https_pool.request, 'GET', '/')
timeout = Timeout(read=0.001)
https_pool = HTTPSConnectionPool(self.host,
self.port,
timeout=timeout,
retries=False,
cert_reqs='CERT_REQUIRED')
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \
'7A:F2:8A:D7:1E:07:33:67:DE'
url = '/sleep?seconds=0.005'
self.assertRaises(ReadTimeoutError, https_pool.request, 'GET', url)
timeout = Timeout(total=None)
https_pool = HTTPSConnectionPool(self.host,
self.port,
timeout=timeout,
cert_reqs='CERT_NONE')
https_pool.request('GET', '/')
class TestHTTPS(HTTPSDummyServerTestCase):
def setUp(self):
self._pool = HTTPSConnectionPool(self.host, self.port)
def test_simple(self):
r = self._pool.request('GET', '/')
self.assertEqual(r.status, 200, r.data)
def test_set_ssl_version_to_tlsv1(self):
self._pool.ssl_version = ssl.PROTOCOL_TLSv1
r = self._pool.request('GET', '/')
self.assertEqual(r.status, 200, r.data)
def test_verified(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
conn = https_pool._new_conn()
self.assertEqual(conn.__class__, VerifiedHTTPSConnection)
with mock.patch('warnings.warn') as warn:
r = https_pool.request('GET', '/')
self.assertEqual(r.status, 200)
if sys.version_info >= (2, 7, 9):
self.assertFalse(warn.called, warn.call_args_list)
else:
self.assertTrue(warn.called)
call, = warn.call_args_list
error = call[0][1]
self.assertEqual(error, InsecurePlatformWarning)
def test_invalid_common_name(self):
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
try:
https_pool.request('GET', '/')
self.fail("Didn't raise SSL invalid common name")
except SSLError as e:
self.assertTrue("doesn't match" in str(e))
def test_verified_with_bad_ca_certs(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA_BAD)
try:
https_pool.request('GET', '/')
self.fail("Didn't raise SSL error with bad CA certs")
except SSLError as e:
self.assertTrue(
'certificate verify failed' in str(e),
"Expected 'certificate verify failed',"
"instead got: %r" % e)
def test_verified_without_ca_certs(self):
# default is cert_reqs=None which is ssl.CERT_NONE
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_REQUIRED')
try:
https_pool.request('GET', '/')
self.fail("Didn't raise SSL error with no CA certs when"
"CERT_REQUIRED is set")
except SSLError as e:
# there is a different error message depending on whether or
# not pyopenssl is injected
self.assertTrue(
'No root certificates specified' in str(e)
or 'certificate verify failed' in str(e),
"Expected 'No root certificates specified' or "
"'certificate verify failed', "
"instead got: %r" % e)
def test_no_ssl(self):
pool = HTTPSConnectionPool(self.host, self.port)
pool.ConnectionCls = None
self.assertRaises(SSLError, pool._new_conn)
self.assertRaises(SSLError, pool.request, 'GET', '/')
def test_unverified_ssl(self):
""" Test that bare HTTPSConnection can connect, make requests """
pool = HTTPSConnectionPool(self.host, self.port)
pool.ConnectionCls = UnverifiedHTTPSConnection
with mock.patch('warnings.warn') as warn:
r = pool.request('GET', '/')
self.assertEqual(r.status, 200)
self.assertTrue(warn.called)
call, = warn.call_args_list
category = call[0][1]
self.assertEqual(category, InsecureRequestWarning)
def test_ssl_unverified_with_ca_certs(self):
pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_NONE',
ca_certs=DEFAULT_CA_BAD)
with mock.patch('warnings.warn') as warn:
r = pool.request('GET', '/')
self.assertEqual(r.status, 200)
self.assertTrue(warn.called)
calls = warn.call_args_list
if sys.version_info >= (2, 7, 9):
category = calls[0][0][1]
else:
category = calls[1][0][1]
self.assertEqual(category, InsecureRequestWarning)
@requires_network
def test_ssl_verified_with_platform_ca_certs(self):
"""
We should rely on the platform CA file to validate authenticity of SSL
certificates. Since this file is used by many components of the OS,
such as curl, apt-get, etc., we decided to not touch it, in order to
not compromise the security of the OS running the test suite (typically
urllib3 developer's OS).
This test assumes that httpbin.org uses a certificate signed by a well
known Certificate Authority.
"""
try:
import urllib3.contrib.pyopenssl
except ImportError:
raise SkipTest('Test requires PyOpenSSL')
if (urllib3.connection.ssl_wrap_socket is
urllib3.contrib.pyopenssl.orig_connection_ssl_wrap_socket):
# Not patched
raise SkipTest('Test should only be run after PyOpenSSL '
'monkey patching')
https_pool = HTTPSConnectionPool('httpbin.org',
443,
cert_reqs=ssl.CERT_REQUIRED)
https_pool.request('HEAD', '/')
def test_assert_hostname_false(self):
https_pool = HTTPSConnectionPool('localhost',
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
https_pool.assert_hostname = False
https_pool.request('GET', '/')
def test_assert_specific_hostname(self):
https_pool = HTTPSConnectionPool('localhost',
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
https_pool.assert_hostname = 'localhost'
https_pool.request('GET', '/')
def test_assert_fingerprint_md5(self):
https_pool = HTTPSConnectionPool('localhost',
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
https_pool.assert_fingerprint = 'CA:84:E1:AD0E5a:ef:2f:C3:09' \
':E7:30:F8:CD:C8:5B'
https_pool.request('GET', '/')
def test_assert_fingerprint_sha1(self):
https_pool = HTTPSConnectionPool('localhost',
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \
'7A:F2:8A:D7:1E:07:33:67:DE'
https_pool.request('GET', '/')
def test_assert_fingerprint_sha256(self):
https_pool = HTTPSConnectionPool('localhost',
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
https_pool.assert_fingerprint = ('9A:29:9D:4F:47:85:1C:51:23:F5:9A:A3:'
'0F:5A:EF:96:F9:2E:3C:22:2E:FC:E8:BC:'
'0E:73:90:37:ED:3B:AA:AB')
https_pool.request('GET', '/')
def test_assert_invalid_fingerprint(self):
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
https_pool.assert_fingerprint = 'AA:AA:AA:AA:AA:AAAA:AA:AAAA:AA:' \
'AA:AA:AA:AA:AA:AA:AA:AA:AA'
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
https_pool._get_conn()
# Uneven length
https_pool.assert_fingerprint = 'AA:A'
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
https_pool._get_conn()
# Invalid length
https_pool.assert_fingerprint = 'AA'
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
def test_verify_none_and_bad_fingerprint(self):
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_NONE',
ca_certs=DEFAULT_CA_BAD)
https_pool.assert_fingerprint = 'AA:AA:AA:AA:AA:AAAA:AA:AAAA:AA:' \
'AA:AA:AA:AA:AA:AA:AA:AA:AA'
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
def test_verify_none_and_good_fingerprint(self):
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_NONE',
ca_certs=DEFAULT_CA_BAD)
https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \
'7A:F2:8A:D7:1E:07:33:67:DE'
https_pool.request('GET', '/')
def test_good_fingerprint_and_hostname_mismatch(self):
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \
'7A:F2:8A:D7:1E:07:33:67:DE'
https_pool.request('GET', '/')
@requires_network
def test_https_timeout(self):
timeout = Timeout(connect=0.001)
https_pool = HTTPSConnectionPool(TARPIT_HOST,
self.port,
timeout=timeout,
retries=False,
cert_reqs='CERT_REQUIRED')
timeout = Timeout(total=None, connect=0.001)
https_pool = HTTPSConnectionPool(TARPIT_HOST,
self.port,
timeout=timeout,
retries=False,
cert_reqs='CERT_REQUIRED')
self.assertRaises(ConnectTimeoutError, https_pool.request, 'GET', '/')
timeout = Timeout(read=0.001)
https_pool = HTTPSConnectionPool(self.host,
self.port,
timeout=timeout,
retries=False,
cert_reqs='CERT_REQUIRED')
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \
'7A:F2:8A:D7:1E:07:33:67:DE'
url = '/sleep?seconds=0.005'
self.assertRaises(ReadTimeoutError, https_pool.request, 'GET', url)
timeout = Timeout(total=None)
https_pool = HTTPSConnectionPool(self.host,
self.port,
timeout=timeout,
cert_reqs='CERT_NONE')
https_pool.request('GET', '/')
def test_tunnel(self):
""" test the _tunnel behavior """
timeout = Timeout(total=None)
https_pool = HTTPSConnectionPool(self.host,
self.port,
timeout=timeout,
cert_reqs='CERT_NONE')
conn = https_pool._new_conn()
try:
conn.set_tunnel(self.host, self.port)
except AttributeError: # python 2.6
conn._set_tunnel(self.host, self.port)
conn._tunnel = mock.Mock()
https_pool._make_request(conn, 'GET', '/')
conn._tunnel.assert_called_once_with()
@onlyPy26OrOlder
def test_tunnel_old_python(self):
"""HTTPSConnection can still make connections if _tunnel_host isn't set
The _tunnel_host attribute was added in 2.6.3 - because our test runners
generally use the latest Python 2.6, we simulate the old version by
deleting the attribute from the HTTPSConnection.
"""
conn = self._pool._new_conn()
del conn._tunnel_host
self._pool._make_request(conn, 'GET', '/')
@requires_network
def test_enhanced_timeout(self):
def new_pool(timeout, cert_reqs='CERT_REQUIRED'):
https_pool = HTTPSConnectionPool(TARPIT_HOST,
self.port,
timeout=timeout,
retries=False,
cert_reqs=cert_reqs)
return https_pool
https_pool = new_pool(Timeout(connect=0.001))
conn = https_pool._new_conn()
self.assertRaises(ConnectTimeoutError, https_pool.request, 'GET', '/')
self.assertRaises(ConnectTimeoutError, https_pool._make_request, conn,
'GET', '/')
https_pool = new_pool(Timeout(connect=5))
self.assertRaises(ConnectTimeoutError,
https_pool.request,
'GET',
'/',
timeout=Timeout(connect=0.001))
t = Timeout(total=None)
https_pool = new_pool(t)
conn = https_pool._new_conn()
self.assertRaises(ConnectTimeoutError,
https_pool.request,
'GET',
'/',
timeout=Timeout(total=None, connect=0.001))
def test_enhanced_ssl_connection(self):
fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:7A:F2:8A:D7:1E:07:33:67:DE'
conn = VerifiedHTTPSConnection(self.host, self.port)
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA,
assert_fingerprint=fingerprint)
https_pool._make_request(conn, 'GET', '/')
def test_ssl_correct_system_time(self):
self._pool.cert_reqs = 'CERT_REQUIRED'
self._pool.ca_certs = DEFAULT_CA
w = self._request_without_resource_warnings('GET', '/')
self.assertEqual([], w)
def test_ssl_wrong_system_time(self):
self._pool.cert_reqs = 'CERT_REQUIRED'
self._pool.ca_certs = DEFAULT_CA
with mock.patch('urllib3.connection.datetime') as mock_date:
mock_date.date.today.return_value = datetime.date(1970, 1, 1)
w = self._request_without_resource_warnings('GET', '/')
self.assertEqual(len(w), 1)
warning = w[0]
self.assertEqual(SystemTimeWarning, warning.category)
self.assertTrue(str(RECENT_DATE) in warning.message.args[0])
def _request_without_resource_warnings(self, method, url):
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
self._pool.request(method, url)
return [x for x in w if not isinstance(x.message, ResourceWarning)]
class TestHTTPS(HTTPSDummyServerTestCase):
def setUp(self):
self._pool = HTTPSConnectionPool(self.host, self.port)
def test_simple(self):
r = self._pool.request('GET', '/')
self.assertEqual(r.status, 200, r.data)
def test_set_ssl_version_to_tlsv1(self):
self._pool.ssl_version = ssl.PROTOCOL_TLSv1
r = self._pool.request('GET', '/')
self.assertEqual(r.status, 200, r.data)
def test_verified(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_REQUIRED')
conn = https_pool._new_conn()
self.assertEqual(conn.__class__, VerifiedHTTPSConnection)
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
https_pool.ca_certs = DEFAULT_CA_BAD
try:
https_pool.request('GET', '/')
self.fail("Didn't raise SSL error with wrong CA")
except SSLError as e:
self.assertTrue(
'certificate verify failed' in str(e),
"Expected 'certificate verify failed',"
"instead got: %r" % e)
https_pool.ca_certs = DEFAULT_CA
https_pool.request('GET', '/') # Should succeed without exceptions.
https_fail_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_REQUIRED')
https_fail_pool.ca_certs = DEFAULT_CA
try:
https_fail_pool.request('GET', '/')
self.fail("Didn't raise SSL invalid common name")
except SSLError as e:
self.assertTrue("doesn't match" in str(e))
def test_no_ssl(self):
import urllib3.connectionpool
OriginalHTTPSConnection = urllib3.connectionpool.HTTPSConnection
OriginalSSL = urllib3.connectionpool.ssl
urllib3.connectionpool.HTTPSConnection = None
urllib3.connectionpool.ssl = None
self.assertRaises(SSLError, self._pool._new_conn)
self.assertRaises(SSLError, self._pool.request, 'GET', '/')
# Undo
urllib3.HTTPSConnection = OriginalHTTPSConnection
urllib3.connectionpool.ssl = OriginalSSL
def test_cert_reqs_as_constant(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs=ssl.CERT_REQUIRED)
https_pool.ca_certs = DEFAULT_CA_BAD
# if we pass in an invalid value it defaults to CERT_NONE
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
def test_cert_reqs_as_short_string(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='REQUIRED')
https_pool.ca_certs = DEFAULT_CA_BAD
# if we pass in an invalid value it defaults to CERT_NONE
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
def test_ssl_unverified_with_ca_certs(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_NONE')
https_pool.ca_certs = DEFAULT_CA_BAD
https_pool.request('GET', '/')
def test_verified_without_ca_certs(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_REQUIRED')
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
def test_invalid_ca_certs(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_REQUIRED')
# Empty string won't throw on py2
https_pool.ca_certs = '/no_valid_path_to_ca_certs'
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
def test_assert_specific_hostname(self):
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_REQUIRED')
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_hostname = 'localhost'
https_pool.request('GET', '/')
def test_assert_fingerprint_md5(self):
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_REQUIRED')
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_fingerprint = 'CA:84:E1:AD0E5a:ef:2f:C3:09' \
':E7:30:F8:CD:C8:5B'
https_pool.request('GET', '/')
def test_assert_fingerprint_sha1(self):
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_REQUIRED')
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \
'7A:F2:8A:D7:1E:07:33:67:DE'
https_pool.request('GET', '/')
def test_assert_invalid_fingerprint(self):
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_REQUIRED')
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_fingerprint = 'AA:AA:AA:AA:AA:AAAA:AA:AAAA:AA:' \
'AA:AA:AA:AA:AA:AA:AA:AA:AA'
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
# invalid length
https_pool.assert_fingerprint = 'AA'
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
# uneven length
https_pool.assert_fingerprint = 'AA:A'
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
def setUp(self):
self._pool = HTTPSConnectionPool(self.host, self.port)
self.addCleanup(self._pool.close)
class RapleafApi:
apiKey = 'SET_ME' # Set your API key here
headers = {'User-Agent' : 'RapleafApi/Python/1.1'}
basePath = '/v4/dr?api_key=%s' %(apiKey)
host = 'personalize.rapleaf.com'
timeout = 2.0
def __init__(self):
self.handle = HTTPSConnectionPool(RapleafApi.host, timeout = RapleafApi.timeout)
def query_by_email(self, email, hash_email = False):
"""
Takes an e-mail and returns a hash which maps attribute fields onto attributes
If the hash_email option is set, then the email will be hashed before it's sent to Rapleaf
"""
if hash_email:
s = hashlib.sha1()
s.update(email)
return self.query_by_sha1(s.digest())
url = '%s&email=%s' % (RapleafApi.basePath, urllib.quote(email))
return self.__get_json_response(url)
def query_by_md5(self, md5_email):
"""
Takes an e-mail that has already been hashed by md5
and returns a hash which maps attribute fields onto attributes
"""
url = '%s&md5_email=%s' % (RapleafApi.basePath, urllib.quote(md5_email))
return self.__get_json_response(url)
def query_by_sha1(self, sha1_email):
"""
Takes an e-mail that has already been hashed by sha1
and returns a hash which maps attribute fields onto attributes
"""
url = '%s&sha1_email=%s' % (RapleafApi.basePath, urllib.quote(sha1_email))
return self.__get_json_response(url)
def query_by_nap(self, first, last, street, city, state, email = None):
"""
Takes first name, last name, and postal (street, city, and state acronym),
and returns a hash which maps attribute fields onto attributes
Though not necessary, adding an e-mail increases hit rate
"""
if email:
url = '%s&email=%s&first=%s&last=%s&street=%s&city=%s&state=%s' % (RapleafApi.basePath,
urllib.quote(email), urllib.quote(first), urllib.quote(last),
urllib.quote(street), urllib.quote(city), urllib.quote(state))
else:
url = '%s&first=%s&last=%s&street=%s&city=%s&state=%s' % (RapleafApi.basePath,
urllib.quote(first), urllib.quote(last),
urllib.quote(street), urllib.quote(city), urllib.quote(state))
return self.__get_json_response(url)
def query_by_naz(self, first, last, zip4, email = None):
"""
Takes first name, last name, and zip4 code (5-digit zip
and 4-digit extension separated by a dash as a string),
and returns a hash which maps attribute fields onto attributes
Though not necessary, adding an e-mail increases hit rate
"""
if email:
url = '%s&email=%s&first=%s&last=%s&zip4=%s' % (RapleafApi.basePath,
urllib.quote(email), urllib.quote(first), urllib.quote(last), zip4)
else:
url = '%s&first=%s&last=%s&zip4=%s' % (RapleafApi.basePath,
urllib.quote(first), urllib.quote(last), zip4)
return self.__get_json_response(url)
def __get_json_response(self, path):
"""
Pre: Path is an extension to personalize.rapleaf.com
Note that an exception is raised if an HTTP response code
other than 200 is sent back. In this case, both the error code
the error code and error body are accessible from the exception raised
"""
json_response = self.handle.get_url(path, headers = RapleafApi.headers)
if 200 <= json_response.status < 300:
if json_response.data:
return json.JSONDecoder().decode(json_response.data)
else:
return {}
else:
raise Exception(json_response.status, json_response.data)
def test_assert_specific_hostname(self):
with HTTPSConnectionPool(
"localhost", self.port, cert_reqs="CERT_REQUIRED", ca_certs=DEFAULT_CA
) as https_pool:
https_pool.assert_hostname = "localhost"
https_pool.request("GET", "/")
def test_dotted_fqdn(self):
pool = HTTPSConnectionPool(self.host + '.', self.port)
r = pool.request('GET', '/')
self.assertEqual(r.status, 200, r.data)
def setUp(self):
self._pool = HTTPSConnectionPool(self.host, self.port, ca_certs=DEFAULT_CA)
self.addCleanup(self._pool.close)
class HandleInfrastructure(DOInfrastructure):
"""
Specialization of the general Digital Object Infrastructure based on the Handle System.
Connects to the Handle System via a RESTful interface.
"""
def __init__(self, host, port, user, user_index, password, path, prefix = None, additional_identifier_element = None, unsafe_ssl=False):
'''
Constructor.
:param host: Host name of the Handle server.
:param port: Port for the Handle server, usually 443 or 8443.
:param user: Full user Handle name to authenticate for administrative permissions for modifying Handles. The full user name will usually be of the form 'prefix/suffix'.
:param user_index: The index to use within the user Handle.
:param password: The user password.
:param path: The server path for the API endpoint, e.g. 'api/handles'. Does not include the host name.
:param prefix: The Handle prefix to use (without trailing slash). If not given, all operations will work
nonetheless, except for random handle creation. Note that setting a prefix does not mean that identifier
strings can omit it - all identifiers must ALWAYS include the prefix, no matter what.
:param additional_identifier_element: A string that is inserted inbetween Handle prefix and suffix, e.g. if set
to "test-", 10876/identifier becomes 10876/test-identifier.
:unsafe_ssl: If set to True, SSL certificate warnings will be ignored. Do not activate this in productive environments!
'''
super(HandleInfrastructure, self).__init__()
self._host = host
self._port = port
self._path = path
self._prefix = prefix
if unsafe_ssl:
disable_warnings()
self.__connpool = HTTPSConnectionPool(host, port=port, assert_hostname=False, cert_reqs="CERT_NONE")
else:
self.__connpool = HTTPSConnectionPool(host, port=port)
self.__user_handle = prefix+"/"+user
self.__user_index = user_index
self.__authstring = b64encode(user_index+"%3A"+user+":"+password)
self.__http_headers = {"Content-Type": "application/json", "Authorization": "Basic %s" % self.__authstring}
if not self._path.endswith("/"):
self._path = self._path + "/"
self._additional_identifier_element = additional_identifier_element
def _generate_random_identifier(self):
if not self._prefix:
raise ValueError("Cannot generate random Handles if no _prefix is provided!")
rid = super(HandleInfrastructure, self)._generate_random_identifier()
return self._prefix+"/"+rid
def _prepare_identifier(self, identifier):
# check identifier string for validity
if " " in identifier:
raise ValueError("Illegal Handle identifier string character; spaces are not supported! (identifier: %s)" % identifier)
identifier = identifier.strip()
if (self._additional_identifier_element):
# split identifier into _prefix and suffix, insert additional element inbetween
parts = identifier.split("/", 1)
if len(parts) != 2:
raise ValueError("Invalid identifier - no separating slash between _prefix and suffix: %s" % identifier)
if (parts[1].startswith(self._additional_identifier_element)):
return self._path+identifier, identifier
newident = parts[0]+"/"+self._additional_identifier_element+parts[1]
return self._path+newident, newident
else:
return self._path+identifier, identifier
def __generate_admin_value(self):
return {"index":100,"type":"HS_ADMIN","data":{"format":"admin","value":{"handle":self.__user_handle,"index":self.__user_index,"permissions":"011111110011"}}}
def _acquire_pid(self, identifier):
path, identifier_prep = self._prepare_identifier(identifier)
# Try to create Handle, but do not ovewrite existing
values = {"values": [self.__generate_admin_value()]}
resp = self.__connpool.urlopen("PUT", path+"?overwrite=false", str(values), self.__http_headers)
# status check; 409 = Conflict on existing Handle
if (resp.status == 409):
raise PIDAlreadyExistsError("Handle already exists: %s" % identifier_prep)
if not(200 <= resp.status <= 299):
raise IOError("Could not create Handle %s: %s" % (identifier_prep, resp.reason))
return identifier_prep
def _do_from_json(self, piddata, identifier, aliases):
"""
Construct a DO instance from given JSON data.
:param piddata: JSON loaded data.
:param identifier: Identifier of the DO.
:param aliases: A list of aliases that were used to get to this identifier (may be empty). The list must be
ordered in the order of alias resolution, i.e. aliases[0] pointed to aliases[1] etc. The last entry pointed
to the actual identifier.
:returns: A fully fledged DigitalObject instance
"""
# piddata is an array of dicts, where each dict has keys: index, type, data
references = {}
res_type = None
if not "values" in piddata:
raise IOError("Illegal format of JSON response from Handle server: 'values' not found in JSON record!")
for ele in piddata["values"]:
idx = int(ele["index"])
if idx == 2:
res_type = ele["data"]["value"]
continue
if ele["type"] == "HS_ADMIN":
# ignore HS_ADMIN values; these are taken care of by the REST service server-side
continue
# no special circumstances --> assign to annotations or references
if REFERENCE_INDEX_END >= idx >= REFERENCE_INDEX_START:
# reference; first, parse element data using json to a list
list_data = json.loads(ele["data"]["value"])
if not isinstance(list_data, list):
raise IOError("Illegal format of JSON response from Handle server: Cannot load reference list! Input: %s" % ele["data"])
if ele["type"] not in references:
references[ele["type"]] = list_data
else:
references[ele["type"]].extend(list_data)
continue
# create special instances for special resource types
if res_type == DigitalObjectSet.RESOURCE_TYPE:
return DigitalObjectSet(self, identifier, references=references, alias_identifiers=aliases)
if res_type == DigitalObjectArray.RESOURCE_TYPE:
return DigitalObjectArray(self, identifier, references=references, alias_identifiers=aliases)
if res_type == DigitalObjectLinkedList.RESOURCE_TYPE:
return DigitalObjectLinkedList(self, identifier, references=references, alias_identifiers=aliases)
return DigitalObject(self, identifier, references, alias_identifiers=aliases)
def lookup_pid(self, identifier):
aliases = []
while True:
path, identifier = self._prepare_identifier(identifier)
resp = self.__connpool.request("GET", path, None, self.__http_headers)
if resp.status == 404:
# Handle not found
if len(aliases) > 0:
raise PIDAliasBrokenError("Alias %s does not exist. Already resolved aliases: %s" % (identifier, aliases))
return None
elif not(200 <= resp.status <= 299):
raise IOError("Failed to look up Handle %s due to the following reason (HTTP Code %s): %s" % (identifier, resp.status, resp.reason))
else:
# check for HS_ALIAS redirect
piddata = json.loads(resp.data)
isa, alias_id = self._check_json_for_alias(piddata)
if isa:
# write down alias identifier and redo lookup with target identifier
aliases.append(identifier)
identifier = alias_id
continue
dobj = self._do_from_json(piddata, identifier, aliases)
return dobj
def _determine_index(self, identifier, handledata, key, index_start, index_end=None):
"""
Finds an index in the Handle key-metadata record to store a value for the given key. If the key is already
present, its index will be reused. If it is not present, a free index will be determined.
:param identifier: The current Handle.
:param key: The key that will be assigned.
:param index_start: At which index the search should start.
:param index_end: Where should the search end? Use None to search all indices greater than the start index.
:raises: :exc:`IndexError` if all possible indices are already taken by other keys.
:returns: an index value.
"""
matching_values = []
free_index = index_start
taken_indices = []
if not "values" in handledata:
raise IOError("Illegal format of JSON response from Handle server: 'values' not found in JSON record!")
for ele in handledata["values"]:
idx = int(ele["index"])
if (index_end and (index_start <= idx <= index_end))\
or (not index_end and (index_start <= idx)):
taken_indices.append(idx)
if ele["type"] == key:
matching_values.append(ele)
if len(matching_values) > 1:
raise IllegalHandleStructureError("Handle %s contains more than one entry of type %s!" % (identifier, key))
elif len(matching_values) == 1:
return int(matching_values[0]["index"])
else:
# key not present in Handle; must assign a new index
# check for free index within bounds
if taken_indices == []:
return index_start
m = min(taken_indices)
if m == index_end:
raise IllegalHandleStructureError("Handle %s does not have any more available index slots between %s and %s!" % (index_start, index_end))
return m
def _write_pid_value(self, identifier, index, valuetype, value):
"""
Writes a single (index, type, value) to the Handle with given identifier.
:param identifier: The Handle identifier.
:param index: Index (positive 32 bit int).
:param valuetype: Type (arbitrary)
:param value: Value (arbitrary)
"""
path, identifier = self._prepare_identifier(identifier)
if type(index) is not int:
raise ValueError("Index must be an integer! (was: type %s, value %s)" % (type(index), index))
# write the raw (index, type, value) triple
data = json.dumps([{"index": index, "type": valuetype, "data": {"format": "string", "value": value}}])
resp = self.__connpool.urlopen("PUT", path+"?index=various", data, self.__http_headers)
if not(200 <= resp.status <= 299):
raise IOError("Could not write raw value to Handle %s: %s" % (identifier, resp.reason))
def _read_pid_value(self, identifier, index):
"""
Reads a single indexed type and value from the Handle with given identifier.
:returns: A tuple (type, value) or None if the given index is unassigned.
:raises: :exc:`IOError` if no Handle with given identifier exists.
"""
path, identifier = self._prepare_identifier(str(identifier))
if type(index) is not int:
raise ValueError("Index must be an integer! (was: type %s, value %s)" % (type(index), index))
# read only the given index
resp = self.__connpool.request("GET", path+"?index=%s" % index, "", self.__http_headers)
if resp.status == 404:
# value not found; the Handle may exist, but the index is unused
return None
if not(200 <= resp.status <= 299):
raise IOError("Could not read raw value from Handle %s: %s" % (identifier, resp.reason))
respdata = json.loads(resp.data)
if not "values" in respdata:
raise IOError("Illegal format of JSON response from Handle server: 'values' not found in JSON record!")
for ele in respdata["values"]:
if int(ele["index"]) == index:
return (ele["type"], ele["data"]["value"])
return None
def _remove_pid_value(self, identifier, index):
"""
Removes a single Handle value at Handle of given identifier at given index.
:raises: :exc:`IOError` if no Handle with given identifier exists.
"""
path, identifier = self._prepare_identifier(str(identifier))
if type(index) is not int:
raise ValueError("Index must be an integer! (was: type %s, value %s)" % (type(index), index))
# read only the given index
resp = self.__connpool.urlopen("DELETE", path+"?index=%s" % index, "", self.__http_headers)
if not(200 <= resp.status <= 299):
raise IOError("Could not remove raw value from Handle %s: %s" % (identifier, resp.reason))
def _read_all_pid_values(self, identifier):
"""
Reads the full Handle record of given identifier.
:return: a dict with indexes as keys and (type, value) tuples as values.
"""
path, identifier = self._prepare_identifier(identifier)
# read full record
resp = self.__connpool.request("GET", path, "", self.__http_headers)
if not(200 <= resp.status <= 299):
raise IOError("Could not read raw values from Handle %s: %s" % (identifier, resp.reason))
respdata = json.loads(resp.data)
res = {}
if not "values" in respdata:
raise IOError("Illegal format of JSON response from Handle server: 'values' not found in JSON record!")
for ele in respdata["values"]:
res[int(ele["index"])] = (ele["type"], ele["data"]["value"])
return res
def _write_resource_information(self, identifier, resource_location, resource_type=None):
path, identifier = self._prepare_identifier(identifier)
handle_values = []
if resource_location:
handle_values = [{"index": INDEX_RESOURCE_LOCATION, "type": "URL", "data": {"format": "string", "value": resource_location}}]
if resource_type:
handle_values.append({"index": INDEX_RESOURCE_TYPE, "type": "", "data": {"format": "string", "value": resource_type}})
data = json.dumps(handle_values)
resp = self.__connpool.urlopen("PUT", path, data, self.__http_headers)
if not(200 <= resp.status <= 299):
raise IOError("Could not write resource location to Handle %s: %s" % (identifier, resp.reason))
def delete_do(self, identifier):
path, identifier = self._prepare_identifier(identifier)
resp = self.__connpool.urlopen("DELETE", path, headers=self.__http_headers)
if resp.status == 404:
raise KeyError("Handle not found: %s" % identifier)
if not(200 <= resp.status <= 299):
raise IOError("Could not delete Handle %s: %s" % (identifier, resp.reason))
def _write_reference(self, identifier, key, reference):
path, identifier = self._prepare_identifier(identifier)
# first, we need to determine the index to use by looking at the key
resp = self.__connpool.request("GET", path, headers=self.__http_headers)
if not(200 <= resp.status <= 299):
raise IOError("Unknown Handle: %s" % identifier)
dodata = json.loads(resp.data)
index = self._determine_index(identifier, dodata, key, REFERENCE_INDEX_START, REFERENCE_INDEX_END)
# now we can write the reference; note that reference may be a list. But this is okay, we
# convert it to a string and take care of reconversion in the JSON-to-DO method
reference_s = json.dumps(reference)
data = json.dumps({"values": [{"index": index, "type": key, "data": {"format": "string", "value": reference_s}}]})
resp = self.__connpool.urlopen("PUT", path+"?index=various", data, self.__http_headers)
if not(200 <= resp.status <= 299):
raise IOError("Could not write references to Handle %s: %s" % (identifier, resp.reason))
def create_alias(self, original, alias_identifier):
if isinstance(original, DigitalObject):
original_identifier = original.identifier
else:
original_identifier = str(original)
path, identifier = self._prepare_identifier(alias_identifier)
# check for existing Handle
resp = self.__connpool.request("GET", path, None, self.__http_headers)
if (resp.status == 200):
# Handle already exists
raise PIDAlreadyExistsError("Handle already exists, cannot use it as an alias: %s" % identifier)
if (resp.status != 404):
raise IOError("Failed to check for existing Handle %s (HTTP Code %s): %s" % (identifier, resp.status, resp.reason))
# okay, alias is available. Now create it.
values = {"values": [self.__generate_admin_value(), {"index": 1, "type": "HS_ALIAS", "data": {"format": "string", "value": str(original_identifier)}}]}
resp = self.__connpool.urlopen("PUT", path, str(values), self.__http_headers)
if not(200 <= resp.status <= 299):
raise IOError("Could not create Alias Handle %s: %s" % (identifier, resp.reason))
return identifier
def delete_alias(self, alias_identifier):
# resolve to check if this is really an alias
isa = self.is_alias(alias_identifier)
if not isa:
return False
self.delete_do(alias_identifier)
return True
def is_alias(self, alias_identifier):
path, identifier = self._prepare_identifier(alias_identifier)
resp = self.__connpool.request("GET", path, None, self.__http_headers)
if resp.status == 404:
raise KeyError("Handle not found: %s" % identifier)
if not(200 <= resp.status <= 299):
raise IOError("Failed to lookup Handle %s for alias check: %s" % (identifier, resp.reason))
# parse JSON, but do not create a Digital Object instance, as this might cause inefficient subsequent calls
isa, a_id = self._check_json_for_alias(json.loads(resp.data))
return isa
def _check_json_for_alias(self, piddata):
"""
Checks the given JSON data structure for presence of an HS_ALIAS marker.
:returns: a tuple (b, id) where b is True or False and if b is True, id is the Handle string of the target
Handle.
"""
res = (False, None)
for ele in piddata["values"]:
if ele["type"] == "HS_ALIAS":
res = (True, ele["data"]["value"])
break
return res
def prefix_pid(self, suffix):
"""
Prepends a given (incomplete) identifier with the current Handle _prefix.
"""
return self._prefix + "/" + suffix
def manufacture_hashmap(self, identifier, characteristic_segment_number):
"""
Factory method. Constructs Handle-based Hashmap implementation objects.
:identifier: The PID of the record that should hold the hash map.
:param: characteristic_segment_number: Since there can be multiple hash maps in a single record, this number
is used to separate them from each other.
"""
return HandleHashmapImpl(self, identifier, characteristic_segment_number)
def test_no_ssl(self):
pool = HTTPSConnectionPool(self.host, self.port)
pool.ConnectionCls = None
self.assertRaises(SSLError, pool._new_conn)
self.assertRaises(SSLError, pool.request, 'GET', '/')
def test_verified(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_REQUIRED')
conn = https_pool._new_conn()
self.assertEqual(conn.__class__, VerifiedHTTPSConnection)
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
https_pool.ca_certs = DEFAULT_CA_BAD
try:
https_pool.request('GET', '/')
self.fail("Didn't raise SSL error with wrong CA")
except SSLError as e:
self.assertTrue(
'certificate verify failed' in str(e),
"Expected 'certificate verify failed',"
"instead got: %r" % e)
https_pool.ca_certs = DEFAULT_CA
https_pool.request('GET', '/') # Should succeed without exceptions.
https_fail_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_REQUIRED')
https_fail_pool.ca_certs = DEFAULT_CA
try:
https_fail_pool.request('GET', '/')
self.fail("Didn't raise SSL invalid common name")
except SSLError as e:
self.assertTrue("doesn't match" in str(e))
def test_dotted_fqdn(self):
with HTTPSConnectionPool(self.host + ".",
self.port,
ca_certs=DEFAULT_CA) as pool:
r = pool.request("GET", "/")
assert r.status == 200, r.data
def test_verified_without_ca_certs(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_REQUIRED')
self.assertRaises(SSLError, https_pool.request, 'GET', '/')
def setUp(self):
self._pool = HTTPSConnectionPool(self.host, self.port)
def _test_request(pool: HTTPSConnectionPool) -> SSLError:
with pytest.raises(MaxRetryError) as cm:
pool.request("GET", "/", retries=0)
assert isinstance(cm.value.reason, SSLError)
return cm.value.reason
class Client:
'''
A fast, lightweight Python interface to the AlphaVantage API
Usage:
client = Client()
client.get_quote("MSFT")
'''
def __init__(self):
self.session = HTTPSConnectionPool(API_HOST)
self.ThreadedRequest = ThreadedRequest
def get_quote(self, symbol):
'''
fetch realtime quote for a stock
:param symbol: a stock symbol
:returns: realtime quote of the given symbol in json format
'''
payload = {
"function": "GLOBAL_QUOTE",
"symbol": symbol,
"apikey": API_KEY
}
r = self.session.request('GET', '/query', fields=payload)
return json.loads(r.data)
def get_quotes(self, symbols):
'''
threaded request for fetching multiple realtime stock quotes
:param symbols: a list of stock symbols
:returns: list of stock quotes in json format
'''
results = ThreadedRequest(symbols, self.get_quote)
return results
def get_historical_quote(self, symbol, interval, output_size="full"):
'''
fetch historical data for a stock
:param symbol: a stock symbol
:param interval: aggragated time period for historical data (1min, 5min, 15min, 30min, 60min)
:param output_size: ammount of data returned (full, compact)
:returns: a list of historical stock quotes in json format
'''
assert interval in INTERVALS
assert output_size in OUTPUT_SIZE
payload = {
"function": "TIME_SERIES_INTRADAY",
"symbol": symbol,
"interval": interval,
"outputsize": output_size,
"apikey": API_KEY
}
r = self.session.request('GET', '/query', fields=payload)
return json.loads(r.data)
def get_historical_quotes(self, symbols, interval, output_size="full"):
'''
threaded request for fetching multiple historical stock quotes
:param symbols: a list of stock symbols
:param interval: aggragated time period for historical data (1min, 5min, 15min, 30min, 60min)
:param output_size: ammount of data returned (full, compact)
:returns: a list of multiple historical stock quotes in json format
'''
results = ThreadedRequest(
symbols, lambda symbol: self.get_historical_quote(
symbol, interval, output_size))
return results
def get_sector_preformances(self):
'''
fetch realtime sector preformance
:returns: list of sector preformances in json format
'''
payload = {"function": "SECTOR", "apikey": API_KEY}
r = self.session.request('GET', '/query', fields=payload)
return json.loads(r.data)
def test_set_cert_default_cert_required(self):
pool = HTTPSConnectionPool(self.host, self.port, ca_certs=DEFAULT_CA)
assert pool.ssl_context.verify_mode == ssl.CERT_REQUIRED
def __init__(self):
self.session = HTTPSConnectionPool(API_HOST)
self.ThreadedRequest = ThreadedRequest
def test_https_timeout(self):
timeout = Timeout(connect=0.001)
https_pool = HTTPSConnectionPool(TARPIT_HOST,
self.port,
timeout=timeout,
retries=False,
cert_reqs='CERT_REQUIRED')
self.addCleanup(https_pool.close)
timeout = Timeout(total=None, connect=0.001)
https_pool = HTTPSConnectionPool(TARPIT_HOST,
self.port,
timeout=timeout,
retries=False,
cert_reqs='CERT_REQUIRED')
self.addCleanup(https_pool.close)
self.assertRaises(ConnectTimeoutError, https_pool.request, 'GET', '/')
timeout = Timeout(read=0.001)
https_pool = HTTPSConnectionPool(self.host,
self.port,
timeout=timeout,
retries=False,
cert_reqs='CERT_REQUIRED')
self.addCleanup(https_pool.close)
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:' \
'BF:93:CF:F9:71:CC:07:7D:0A'
timeout = Timeout(total=None)
https_pool = HTTPSConnectionPool(self.host,
self.port,
timeout=timeout,
cert_reqs='CERT_NONE')
self.addCleanup(https_pool.close)
https_pool.request('GET', '/')
class TowerDataApi:
HEADERS = {'User-Agent' : 'TowerDataApi/Python/1.2'}
POST_HEADERS = HEADERS.copy()
POST_HEADERS.update({
'Content-Type': 'application/json'
})
BASE_PATH = '/v5/td'
EV_BASE_PATH = '/v5/ev'
EPPEND_BASE_PATH = '/v5/eppend'
BULK_BASE_PATH = '/v5/ei/bulk'
HOST = 'api.towerdata.com'
TIMEOUT = 30.0
def __init__(self, api_key):
self.handle = HTTPSConnectionPool(TowerDataApi.HOST, timeout=TowerDataApi.TIMEOUT)
self.base_path = TowerDataApi.BASE_PATH + '?api_key=%s' % (api_key)
self.base_ev_path = TowerDataApi.EV_BASE_PATH + '?api_key=%s' % (api_key)
self.base_eppend_path = TowerDataApi.EPPEND_BASE_PATH + '?api_key=%s' % (api_key)
self.base_bulk_path = (TowerDataApi.BULK_BASE_PATH +
'?api_key=%s' % (api_key))
def validate_email(self, email, timeout=None):
"""
Takes an e-mail and returns a map of validation attributes.
The timeout option is the server-side timeout value in seconds; floating-point numbers (e.g. 4.9, 3.55) are permitted; max is 30 (seconds).
If timeout is set, it overrides the default server-side timeout.
"""
if timeout:
url = '%s&email=%s&timeout=%.1f' % (self.base_ev_path, quote(email), timeout)
else:
url = '%s&email=%s' % (self.base_ev_path, quote(email))
return self.__do_get_request(url)
def query_by_email(self, email, hash_email=False, fields=None):
"""
Takes an e-mail and returns a hash which maps attribute fields onto attributes
If the hash_email option is set, then the email will be hashed before it's sent to TowerData.
If the fields parameter is set, this comma-separated string specifies the attributes you want returned,
otherwise, all the attributes configured in your API key are returned.
You will only be charged for the data you receive.
"""
if hash_email:
s = hashlib.sha1()
s.update(email.lower())
return self.query_by_sha1(s.hexdigest(), fields)
if fields:
url = '%s&email=%s&fields=%s' % (self.base_path, quote(email), quote(fields))
else:
url = '%s&email=%s' % (self.base_path, quote(email))
return self.__do_get_request(url)
def do_bulk_query(self, data, fields=None):
"""
Takes a list of e-mails and returns a hash with emails as keys and api information for each email
The data should be following form: [{'email': '*****@*****.**'}, {'email': '*****@*****.**'}]
If the fields parameter is set, this comma-separated string specifies the attributes you want returned,
otherwise, all the attributes configured in your API key are returned.
You will only be charged for the data you receive.
For more information refer to http://docs.towerdata.com/#bulk-email-intelligence-introduction
"""
if fields:
url = '%s&fields=%s' % (self.base_path, quote(fields))
else:
url = self.base_path
return self.__do_post_request(data)
def query_by_md5(self, md5_email, fields=None):
"""
Takes an e-mail that has already been hashed by md5
and returns a hash which maps attribute fields onto attributes.
If the fields parameter is set, this comma-separated string specifies the attributes you want returned,
otherwise, all the attributes configured in your API key are returned.
You will only be charged for the data you receive.
"""
if fields:
url = '%s&md5_email=%s&fields=%s' % (self.base_path, quote(md5_email), quote(fields))
else:
url = '%s&md5_email=%s' % (self.base_path, quote(md5_email))
return self.__do_get_request(url)
def query_by_sha1(self, sha1_email, fields=None):
"""
Takes an e-mail that has already been hashed by sha1
and returns a hash which maps attribute fields onto attributes.
If the fields parameter is set, this comma-separated string specifies the attributes you want returned,
otherwise, all the attributes configured in your API key are returned.
You will only be charged for the data you receive.
"""
if fields:
url = '%s&sha1_email=%s&fields=%s' % (self.base_path, quote(sha1_email), quote(fields))
else:
url = '%s&sha1_email=%s' % (self.base_path, quote(sha1_email))
return self.__do_get_request(url)
def query_by_nap(self, first, last, street, city, state, email=None):
"""
Takes first name, last name, and postal (street, city, and state acronym),
and returns a hash which maps attribute fields onto attributes
Though not necessary, adding an e-mail increases hit rate.
"""
url = '%s&first=%s&last=%s&street=%s&city=%s&state=%s' % (
self.base_path, quote(first), quote(last),
quote(street), quote(city), quote(state))
if email:
url = '%s&email=%s' % (url, quote(email))
return self.__do_get_request(url)
def query_by_naz(self, first, last, zip4, email=None):
"""
Takes first name, last name, and zip4 code (5-digit zip
and 4-digit extension separated by a dash as a string),
and returns a hash which maps attribute fields onto attributes
Though not necessary, adding an e-mail increases hit rate.
"""
url = '%s&first=%s&last=%s&zip4=%s' % (
self.base_path, quote(first), quote(last), zip4)
if email:
url = '%s&email=%s' % (url, quote(email))
return self.__do_get_request(url)
def append_email(self, first, last, street, city, state, zip):
"""
Takes first name, last name, and postal (street, city, state acronym, and zip),
and returns a hash which maps attribute fields onto attributes
"""
url = '%s&first=%s&last=%s&street=%s&city=%s&state=%s&zip=%s' % (
self.base_eppend_path, quote(first), quote(last),
quote(street), quote(city), quote(state), quote(zip))
return self.__do_get_request(url)
def append_postal(self, email):
"""
Takes email and returns a hash with first name, last name, and postal (street, city, state acronym, and zip) attributes.
"""
url = '%s&email=%s' % (self.base_eppend_path, quote(email))
return self.__do_get_request(url)
def __do_get_request(self, path):
"""
Pre: Path is an extension to personalize.rapleaf.com
Note that an exception is raised if an HTTP response code
other than 200 is sent back. In this case, both the error code
the error code and error body are accessible from the exception raised
"""
json_response = self.handle.request('GET', path, headers=TowerDataApi.HEADERS)
if 200 <= json_response.status < 300:
if json_response.data:
return http_response_to_json(json_response.data)
else:
return {}
else:
raise Exception(json_response.status, json_response.data.decode("utf-8"))
def __do_post_request(self, data):
"""
Note that an exception is raised if an HTTP response code
other than 200 is sent back. In this case, both the error code
the error code and error body are accessible from the exception raised
"""
json_response = self.handle.request(
'POST', self.base_bulk_path, headers=TowerDataApi.POST_HEADERS,
body=json.JSONEncoder().encode(data)
)
if 200 <= json_response.status < 300:
if json_response.data:
resp = http_response_to_json(json_response.data)
ret = {}
for i, tmp in enumerate(data):
ret[tmp['email']] = resp[i]
return ret
else:
return {}
else:
raise Exception(
json_response.status, json_response.data.decode("utf-8")
)
class TestHTTPS(HTTPSDummyServerTestCase):
def setUp(self):
self._pool = HTTPSConnectionPool(self.host, self.port)
self.addCleanup(self._pool.close)
def test_simple(self):
r = self._pool.request('GET', '/')
self.assertEqual(r.status, 200, r.data)
def test_dotted_fqdn(self):
pool = HTTPSConnectionPool(self.host + '.', self.port)
r = pool.request('GET', '/')
self.assertEqual(r.status, 200, r.data)
def test_set_ssl_version_to_tlsv1(self):
self._pool.ssl_version = ssl.PROTOCOL_TLSv1
r = self._pool.request('GET', '/')
self.assertEqual(r.status, 200, r.data)
def test_client_intermediate(self):
client_cert, client_key, client_subject = (
DEFAULT_CLIENT_CERTS['certfile'], DEFAULT_CLIENT_CERTS['keyfile'],
DEFAULT_CLIENT_CERTS['subject'])
https_pool = HTTPSConnectionPool(self.host,
self.port,
key_file=client_key,
cert_file=client_cert)
r = https_pool.request('GET', '/certificate')
self.assertDictEqual(json.loads(r.data.decode('utf-8')),
client_subject, r.data)
def test_client_no_intermediate(self):
client_cert, client_key = (
DEFAULT_CLIENT_NO_INTERMEDIATE_CERTS['certfile'],
DEFAULT_CLIENT_NO_INTERMEDIATE_CERTS['keyfile'])
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_file=client_cert,
key_file=client_key)
try:
https_pool.request('GET', '/certificate', retries=False)
except SSLError as e:
self.assertTrue('alert unknown ca' in str(e)
or 'invalid certificate chain' in str(e)
or 'unknown Cert Authority' in str(e))
def test_verified(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
self.addCleanup(https_pool.close)
conn = https_pool._new_conn()
self.assertEqual(conn.__class__, VerifiedHTTPSConnection)
with mock.patch('warnings.warn') as warn:
r = https_pool.request('GET', '/')
self.assertEqual(r.status, 200)
# Modern versions of Python, or systems using PyOpenSSL, don't
# emit warnings.
if sys.version_info >= (2, 7, 9) or util.IS_PYOPENSSL:
self.assertFalse(warn.called, warn.call_args_list)
else:
self.assertTrue(warn.called)
if util.HAS_SNI:
call = warn.call_args_list[0]
else:
call = warn.call_args_list[1]
error = call[0][1]
self.assertEqual(error, InsecurePlatformWarning)
def test_verified_with_context(self):
ctx = util.ssl_.create_urllib3_context(cert_reqs=ssl.CERT_REQUIRED)
ctx.load_verify_locations(cafile=DEFAULT_CA)
https_pool = HTTPSConnectionPool(self.host, self.port, ssl_context=ctx)
self.addCleanup(https_pool.close)
conn = https_pool._new_conn()
self.assertEqual(conn.__class__, VerifiedHTTPSConnection)
with mock.patch('warnings.warn') as warn:
r = https_pool.request('GET', '/')
self.assertEqual(r.status, 200)
# Modern versions of Python, or systems using PyOpenSSL, don't
# emit warnings.
if sys.version_info >= (2, 7, 9) or util.IS_PYOPENSSL:
self.assertFalse(warn.called, warn.call_args_list)
else:
self.assertTrue(warn.called)
if util.HAS_SNI:
call = warn.call_args_list[0]
else:
call = warn.call_args_list[1]
error = call[0][1]
self.assertEqual(error, InsecurePlatformWarning)
def test_context_combines_with_ca_certs(self):
ctx = util.ssl_.create_urllib3_context(cert_reqs=ssl.CERT_REQUIRED)
https_pool = HTTPSConnectionPool(self.host,
self.port,
ca_certs=DEFAULT_CA,
ssl_context=ctx)
self.addCleanup(https_pool.close)
conn = https_pool._new_conn()
self.assertEqual(conn.__class__, VerifiedHTTPSConnection)
with mock.patch('warnings.warn') as warn:
r = https_pool.request('GET', '/')
self.assertEqual(r.status, 200)
# Modern versions of Python, or systems using PyOpenSSL, don't
# emit warnings.
if sys.version_info >= (2, 7, 9) or util.IS_PYOPENSSL:
self.assertFalse(warn.called, warn.call_args_list)
else:
self.assertTrue(warn.called)
if util.HAS_SNI:
call = warn.call_args_list[0]
else:
call = warn.call_args_list[1]
error = call[0][1]
self.assertEqual(error, InsecurePlatformWarning)
@onlyPy279OrNewer
@notSecureTransport
def test_ca_dir_verified(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_REQUIRED',
ca_cert_dir=DEFAULT_CA_DIR)
self.addCleanup(https_pool.close)
conn = https_pool._new_conn()
self.assertEqual(conn.__class__, VerifiedHTTPSConnection)
with mock.patch('warnings.warn') as warn:
r = https_pool.request('GET', '/')
self.assertEqual(r.status, 200)
self.assertFalse(warn.called, warn.call_args_list)
def test_invalid_common_name(self):
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
self.addCleanup(https_pool.close)
try:
https_pool.request('GET', '/')
self.fail("Didn't raise SSL invalid common name")
except MaxRetryError as e:
self.assertIsInstance(e.reason, SSLError)
self.assertTrue("doesn't match" in str(e.reason)
or "certificate verify failed" in str(e.reason))
def test_verified_with_bad_ca_certs(self):
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA_BAD)
self.addCleanup(https_pool.close)
try:
https_pool.request('GET', '/')
self.fail("Didn't raise SSL error with bad CA certs")
except MaxRetryError as e:
self.assertIsInstance(e.reason, SSLError)
self.assertTrue(
'certificate verify failed' in str(e.reason),
"Expected 'certificate verify failed',"
"instead got: %r" % e.reason)
def test_verified_without_ca_certs(self):
# default is cert_reqs=None which is ssl.CERT_NONE
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_REQUIRED')
self.addCleanup(https_pool.close)
try:
https_pool.request('GET', '/')
self.fail("Didn't raise SSL error with no CA certs when"
"CERT_REQUIRED is set")
except MaxRetryError as e:
self.assertIsInstance(e.reason, SSLError)
# there is a different error message depending on whether or
# not pyopenssl is injected
self.assertTrue(
'No root certificates specified' in str(e.reason)
or 'certificate verify failed' in str(e.reason)
or 'invalid certificate chain' in str(e.reason),
"Expected 'No root certificates specified', "
"'certificate verify failed', or "
"'invalid certificate chain', "
"instead got: %r" % e.reason)
def test_no_ssl(self):
pool = HTTPSConnectionPool(self.host, self.port)
pool.ConnectionCls = None
self.addCleanup(pool.close)
self.assertRaises(SSLError, pool._new_conn)
with self.assertRaises(MaxRetryError) as cm:
pool.request('GET', '/', retries=0)
self.assertIsInstance(cm.exception.reason, SSLError)
def test_unverified_ssl(self):
""" Test that bare HTTPSConnection can connect, make requests """
pool = HTTPSConnectionPool(self.host, self.port)
pool.ConnectionCls = UnverifiedHTTPSConnection
self.addCleanup(pool.close)
with mock.patch('warnings.warn') as warn:
r = pool.request('GET', '/')
self.assertEqual(r.status, 200)
self.assertTrue(warn.called)
# Modern versions of Python, or systems using PyOpenSSL, only emit
# the unverified warning. Older systems may also emit other
# warnings, which we want to ignore here.
calls = warn.call_args_list
if sys.version_info >= (2, 7, 9) or util.IS_PYOPENSSL:
category = calls[0][0][1]
elif util.HAS_SNI:
category = calls[1][0][1]
else:
category = calls[2][0][1]
self.assertEqual(category, InsecureRequestWarning)
def test_ssl_unverified_with_ca_certs(self):
pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_NONE',
ca_certs=DEFAULT_CA_BAD)
self.addCleanup(pool.close)
with mock.patch('warnings.warn') as warn:
r = pool.request('GET', '/')
self.assertEqual(r.status, 200)
self.assertTrue(warn.called)
# Modern versions of Python, or systems using PyOpenSSL, only emit
# the unverified warning. Older systems may also emit other
# warnings, which we want to ignore here.
calls = warn.call_args_list
if sys.version_info >= (2, 7, 9) or util.IS_PYOPENSSL:
category = calls[0][0][1]
elif util.HAS_SNI:
category = calls[1][0][1]
else:
category = calls[2][0][1]
self.assertEqual(category, InsecureRequestWarning)
def test_assert_hostname_false(self):
https_pool = HTTPSConnectionPool('localhost',
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
self.addCleanup(https_pool.close)
https_pool.assert_hostname = False
https_pool.request('GET', '/')
def test_assert_specific_hostname(self):
https_pool = HTTPSConnectionPool('localhost',
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
self.addCleanup(https_pool.close)
https_pool.assert_hostname = 'localhost'
https_pool.request('GET', '/')
def test_assert_fingerprint_md5(self):
https_pool = HTTPSConnectionPool('localhost',
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
self.addCleanup(https_pool.close)
https_pool.assert_fingerprint = 'F2:06:5A:42:10:3F:45:1C:17:FE:E6:' \
'07:1E:8A:86:E5'
https_pool.request('GET', '/')
def test_assert_fingerprint_sha1(self):
https_pool = HTTPSConnectionPool('localhost',
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
self.addCleanup(https_pool.close)
https_pool.assert_fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:' \
'BF:93:CF:F9:71:CC:07:7D:0A'
https_pool.request('GET', '/')
def test_assert_fingerprint_sha256(self):
https_pool = HTTPSConnectionPool('localhost',
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
self.addCleanup(https_pool.close)
https_pool.assert_fingerprint = ('C5:4D:0B:83:84:89:2E:AE:B4:58:BB:12:'
'F7:A6:C4:76:05:03:88:D8:57:65:51:F3:'
'1E:60:B0:8B:70:18:64:E6')
https_pool.request('GET', '/')
def test_assert_invalid_fingerprint(self):
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
self.addCleanup(https_pool.close)
https_pool.assert_fingerprint = 'AA:AA:AA:AA:AA:AAAA:AA:AAAA:AA:' \
'AA:AA:AA:AA:AA:AA:AA:AA:AA'
def _test_request(pool):
with self.assertRaises(MaxRetryError) as cm:
pool.request('GET', '/', retries=0)
self.assertIsInstance(cm.exception.reason, SSLError)
_test_request(https_pool)
https_pool._get_conn()
# Uneven length
https_pool.assert_fingerprint = 'AA:A'
_test_request(https_pool)
https_pool._get_conn()
# Invalid length
https_pool.assert_fingerprint = 'AA'
_test_request(https_pool)
def test_verify_none_and_bad_fingerprint(self):
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_NONE',
ca_certs=DEFAULT_CA_BAD)
self.addCleanup(https_pool.close)
https_pool.assert_fingerprint = 'AA:AA:AA:AA:AA:AAAA:AA:AAAA:AA:' \
'AA:AA:AA:AA:AA:AA:AA:AA:AA'
with self.assertRaises(MaxRetryError) as cm:
https_pool.request('GET', '/', retries=0)
self.assertIsInstance(cm.exception.reason, SSLError)
def test_verify_none_and_good_fingerprint(self):
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_NONE',
ca_certs=DEFAULT_CA_BAD)
self.addCleanup(https_pool.close)
https_pool.assert_fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:' \
'BF:93:CF:F9:71:CC:07:7D:0A'
https_pool.request('GET', '/')
@notSecureTransport
def test_good_fingerprint_and_hostname_mismatch(self):
# This test doesn't run with SecureTransport because we don't turn off
# hostname validation without turning off all validation, which this
# test doesn't do (deliberately). We should revisit this if we make
# new decisions.
https_pool = HTTPSConnectionPool('127.0.0.1',
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA)
self.addCleanup(https_pool.close)
https_pool.assert_fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:' \
'BF:93:CF:F9:71:CC:07:7D:0A'
https_pool.request('GET', '/')
@requires_network
def test_https_timeout(self):
timeout = Timeout(connect=0.001)
https_pool = HTTPSConnectionPool(TARPIT_HOST,
self.port,
timeout=timeout,
retries=False,
cert_reqs='CERT_REQUIRED')
self.addCleanup(https_pool.close)
timeout = Timeout(total=None, connect=0.001)
https_pool = HTTPSConnectionPool(TARPIT_HOST,
self.port,
timeout=timeout,
retries=False,
cert_reqs='CERT_REQUIRED')
self.addCleanup(https_pool.close)
self.assertRaises(ConnectTimeoutError, https_pool.request, 'GET', '/')
timeout = Timeout(read=0.001)
https_pool = HTTPSConnectionPool(self.host,
self.port,
timeout=timeout,
retries=False,
cert_reqs='CERT_REQUIRED')
self.addCleanup(https_pool.close)
https_pool.ca_certs = DEFAULT_CA
https_pool.assert_fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:' \
'BF:93:CF:F9:71:CC:07:7D:0A'
timeout = Timeout(total=None)
https_pool = HTTPSConnectionPool(self.host,
self.port,
timeout=timeout,
cert_reqs='CERT_NONE')
self.addCleanup(https_pool.close)
https_pool.request('GET', '/')
def test_tunnel(self):
""" test the _tunnel behavior """
timeout = Timeout(total=None)
https_pool = HTTPSConnectionPool(self.host,
self.port,
timeout=timeout,
cert_reqs='CERT_NONE')
self.addCleanup(https_pool.close)
conn = https_pool._new_conn()
self.addCleanup(conn.close)
try:
conn.set_tunnel(self.host, self.port)
except AttributeError: # python 2.6
conn._set_tunnel(self.host, self.port)
conn._tunnel = mock.Mock()
https_pool._make_request(conn, 'GET', '/')
conn._tunnel.assert_called_once_with()
@onlyPy26OrOlder
def test_tunnel_old_python(self):
"""HTTPSConnection can still make connections if _tunnel_host isn't set
The _tunnel_host attribute was added in 2.6.3 - because our test runners
generally use the latest Python 2.6, we simulate the old version by
deleting the attribute from the HTTPSConnection.
"""
conn = self._pool._new_conn()
del conn._tunnel_host
self._pool._make_request(conn, 'GET', '/')
@requires_network
def test_enhanced_timeout(self):
def new_pool(timeout, cert_reqs='CERT_REQUIRED'):
https_pool = HTTPSConnectionPool(TARPIT_HOST,
self.port,
timeout=timeout,
retries=False,
cert_reqs=cert_reqs)
self.addCleanup(https_pool.close)
return https_pool
https_pool = new_pool(Timeout(connect=0.001))
conn = https_pool._new_conn()
self.assertRaises(ConnectTimeoutError, https_pool.request, 'GET', '/')
self.assertRaises(ConnectTimeoutError, https_pool._make_request, conn,
'GET', '/')
https_pool = new_pool(Timeout(connect=5))
self.assertRaises(ConnectTimeoutError,
https_pool.request,
'GET',
'/',
timeout=Timeout(connect=0.001))
t = Timeout(total=None)
https_pool = new_pool(t)
conn = https_pool._new_conn()
self.assertRaises(ConnectTimeoutError,
https_pool.request,
'GET',
'/',
timeout=Timeout(total=None, connect=0.001))
def test_enhanced_ssl_connection(self):
fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:BF:93:CF:F9:71:CC:07:7D:0A'
conn = VerifiedHTTPSConnection(self.host, self.port)
self.addCleanup(conn.close)
https_pool = HTTPSConnectionPool(self.host,
self.port,
cert_reqs='CERT_REQUIRED',
ca_certs=DEFAULT_CA,
assert_fingerprint=fingerprint)
self.addCleanup(https_pool.close)
https_pool._make_request(conn, 'GET', '/')
def test_ssl_correct_system_time(self):
self._pool.cert_reqs = 'CERT_REQUIRED'
self._pool.ca_certs = DEFAULT_CA
w = self._request_without_resource_warnings('GET', '/')
self.assertEqual([], w)
def test_ssl_wrong_system_time(self):
self._pool.cert_reqs = 'CERT_REQUIRED'
self._pool.ca_certs = DEFAULT_CA
with mock.patch('urllib3.connection.datetime') as mock_date:
mock_date.date.today.return_value = datetime.date(1970, 1, 1)
w = self._request_without_resource_warnings('GET', '/')
self.assertEqual(len(w), 1)
warning = w[0]
self.assertEqual(SystemTimeWarning, warning.category)
self.assertTrue(str(RECENT_DATE) in warning.message.args[0])
def _request_without_resource_warnings(self, method, url):
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
self._pool.request(method, url)
return [x for x in w if not isinstance(x.message, ResourceWarning)]
def new_pool(timeout, cert_reqs='CERT_REQUIRED'):
https_pool = HTTPSConnectionPool(TARPIT_HOST,
self.port,
timeout=timeout,
cert_reqs=cert_reqs)
return https_pool
