def __init__(self, cipher_suite=None, **kwargs):
self.cipher_suite = cipher_suite
super(CipherSuiteAdapter, self).__init__(**kwargs)
if hasattr(ssl, 'PROTOCOL_TLS'):
self.ssl_context = create_urllib3_context(ssl_version=getattr(ssl, 'PROTOCOL_TLSv1_3', ssl.PROTOCOL_TLSv1_2), ciphers=self.cipher_suite)
else:
self.ssl_context = create_urllib3_context(ssl_version=ssl.PROTOCOL_TLSv1)
def __init__(self, url):
try:
self.url = url
create_urllib3_context()
self.parser = UrlParser(self.url)
self.response_arr = [
grequests.get(url) for url in self.parser.urls
]
self.res = []
except Exception as E:
raise E
def __init__(self, cipherSuite=None, **kwargs):
self.cipherSuite = cipherSuite
if hasattr(ssl, 'PROTOCOL_TLS'):
self.ssl_context = create_urllib3_context(
ssl_version=getattr(ssl, 'PROTOCOL_TLSv1_3', ssl.PROTOCOL_TLSv1_2),
ciphers=self.cipherSuite
)
else:
self.ssl_context = create_urllib3_context(ssl_version=ssl.PROTOCOL_TLSv1)
super(CipherSuiteAdapter, self).__init__(**kwargs)
def init_poolmanager(self, *args, **kwargs):
"""Overrides the parent method to include call to load_default_certs()"""
context = create_urllib3_context()
kwargs['ssl_context'] = context
context.load_default_certs(
) # this loads the OS default trusted CA certs
return super(SSLContextAdapter, self).init_poolmanager(*args, **kwargs)
def init_poolmanager(self, *args, **kwargs):
context = create_urllib3_context(
ciphers=
'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:!DH'
)
kwargs['ssl_context'] = context
return super(NODHAdapter, self).init_poolmanager(*args, **kwargs)
def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs):
"""
Initialize the connection pool manager.
Args:
connections (int): Initial number of connections to be used.
maxsize (int): Maximum size of the connection pool.
block (object): Blocking policy.
**pool_kwargs: Additional arguments for the connection pool.
Returns:
None
"""
if self._cbc_sdk_force_tls_1_2 and REQUESTS_HAS_URLLIB_SSL_CONTEXT:
# Force the use of TLS v1.2 when talking to this Cb Response server.
context = create_urllib3_context(ciphers=('TLSv1.2:!aNULL:!eNULL:!MD5'))
context.options |= ssl.OP_NO_SSLv2
context.options |= ssl.OP_NO_SSLv3
context.options |= ssl.OP_NO_TLSv1
context.options |= ssl.OP_NO_TLSv1_1
pool_kwargs['ssl_context'] = context
if not self._cbc_sdk_verify_hostname:
# Provide the ability to validate a Carbon Black server's SSL certificate without validating the hostname
# (by default Carbon Black certificates are "issued" as CN=Self-signed Carbon Black Enterprise Server
# HTTPS Certificate)
pool_kwargs["assert_hostname"] = False
return super(CBCSDKSessionAdapter, self).init_poolmanager(connections, maxsize, block, **pool_kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
context = create_urllib3_context(ciphers=CIPHERS)
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
ssl_version=ssl.PROTOCOL_TLSv1,
ssl_context=context)
def proxy_manager_for(self, proxy, **proxy_kwargs):
"""Ensure cipher and Tlsv1"""
context = create_urllib3_context(ciphers=self.CIPHERS,
ssl_version=ssl.PROTOCOL_TLSv1)
proxy_kwargs['ssl_context'] = context
return super(TLSv1Adapter,
self).proxy_manager_for(proxy, **proxy_kwargs)
def __init__(self, cipher_suite=None, **kwargs):
self.cipher_suite = cipher_suite
self.ssl_context = create_urllib3_context(ssl_version=ssl.PROTOCOL_TLS,
ciphers=self.cipher_suite)
super(CipherSuiteAdapter, self).__init__(**kwargs)
def proxy_manager_for(self, *args, **kwargs):
context = create_urllib3_context()
if self.ctx_options is not None:
context._ctx.use_certificate(self.ctx_options.get('cert'))
context._ctx.use_privatekey(self.ctx_options.get('key'))
context._ctx.load_verify_locations(self.ctx_options.get('cafile'))
kwargs['ssl_context'] = context
return super(CustomHTTPSAdapter, self).proxy_manager_for(*args, **kwargs)
def init_poolmanager(self, *args, **kwargs):
context = create_urllib3_context()
if self.ctx_options is not None:
# Probably there is a better (pythonic) way to setting this up
context._ctx.use_certificate(self.ctx_options.get('cert'))
context._ctx.use_privatekey(self.ctx_options.get('key'))
context._ctx.load_verify_locations(self.ctx_options.get('cafile'))
kwargs['ssl_context'] = context
return super(CustomHTTPSAdapter, self).init_poolmanager(*args, **kwargs)
def init_poolmanager(self, *pool_args, **pool_kwargs):
ctx = ssl_.create_urllib3_context(ssl.PROTOCOL_TLS)
# extend the default context options, which is to disable SSL2, SSL3
# and SSL compression, see:
# https://github.com/shazow/urllib3/blob/6a6cfe9/urllib3/util/ssl_.py#L241
ctx.options |= self.ssl_options
self.poolmanager = PoolManager(*pool_args,
ssl_context=ctx,
**pool_kwargs)
def init_poolmanager(self, connections, maxsize,
block=requests.adapters.DEFAULT_POOLBLOCK,
**pool_kwargs):
"""Initialize poolmanager with cipher and Tlsv1"""
context = create_urllib3_context(ciphers=self.CIPHERS,
ssl_version=ssl.PROTOCOL_TLSv1)
pool_kwargs['ssl_context'] = context
return super(TLSv1Adapter, self).init_poolmanager(connections, maxsize,
block, **pool_kwargs)
def get_connection(self, *args, **kwargs):
from requests.packages.urllib3.util.ssl_ import create_urllib3_context, DEFAULT_CIPHERS####
DEFAULT_CIPHERS += ":!ECDHE+SHA:!AES128-SHA"######
conn = super(CloudflareAdapter, self).get_connection(*args, **kwargs)
if conn.conn_kw.get("ssl_context"):
conn.conn_kw["ssl_context"].set_ciphers(DEFAULT_CIPHERS)
else:
context = create_urllib3_context(ciphers=DEFAULT_CIPHERS)
conn.conn_kw["ssl_context"] = context
return conn
def new_connect(self, **httplib_request_kw):
# Add certificate verification
conn = self._new_conn()
import datetime
hostname = self.host
if getattr(self, '_tunnel_host', None):
# _tunnel_host was added in Python 2.6.3
# (See: http://hg.python.org/cpython/rev/0f57b30a152f)
self.sock = conn
# Calls self._set_hostport(), so self.host is
# self._tunnel_host below.
self._tunnel()
# Mark this connection as not reusable
self.auto_open = 0
# Override the host with the one we're requesting data from.
hostname = self._tunnel_host
if 'Host' in httplib_request_kw['headers']:
hostname = httplib_request_kw['headers']['Host']
# Wrap socket using verification with the root certs in
# trusted_root_certs
if self.ssl_context is None:
self.ssl_context = create_urllib3_context(
ssl_version=resolve_ssl_version(self.ssl_version),
cert_reqs=resolve_cert_reqs(self.cert_reqs),
)
context = self.ssl_context
context.verify_mode = resolve_cert_reqs(self.cert_reqs)
self.sock = ssl_wrap_socket(sock=conn,
keyfile=self.key_file,
certfile=self.cert_file,
ca_certs=self.ca_certs,
ca_cert_dir=self.ca_cert_dir,
server_hostname=hostname,
ssl_context=context)
if self.assert_fingerprint:
assert_fingerprint(self.sock.getpeercert(binary_form=True),
self.assert_fingerprint)
elif context.verify_mode != ssl.CERT_NONE \
and self.assert_hostname is not False:
cert = self.sock.getpeercert()
if not cert.get('subjectAltName', ()):
warnings.warn((
'Certificate for {0} has no `subjectAltName`, falling back to check for a '
'`commonName` for now. This feature is being removed by major browsers and '
'deprecated by RFC 2818. (See https://github.com/shazow/urllib3/issues/497 '
'for details.)'.format(hostname)), SubjectAltNameWarning)
_match_hostname(cert, self.assert_hostname or hostname)
self.is_verified = (context.verify_mode == ssl.CERT_REQUIRED
or self.assert_fingerprint is not None)
def loadCipherSuite(self):
if self.cipherSuite:
return self.cipherSuite
if hasattr(ssl, 'PROTOCOL_TLS') or hasattr(ssl, 'PROTOCOL_TLSv1_2'):
for cipher in self.user_agent.cipherSuite[:]:
try:
create_urllib3_context(
getattr(ssl, 'PROTOCOL_TLS', ssl.PROTOCOL_TLSv1_2),
ciphers=cipher
)
except (OpenSSL.SSL.Error, ssl.SSLError):
self.user_agent.cipherSuite.remove(cipher)
pass
if self.user_agent.cipherSuite:
self.cipherSuite = ':'.join(self.user_agent.cipherSuite)
return self.cipherSuite
raise RuntimeError("Your SSL compiled in python does not meet the minimum cipher suite requirements.")
def init_poolmanager(self, *pool_args, **pool_kwargs):
ctx = ssl_.create_urllib3_context(
ciphers=':'.join(self.CIPHERS),
cert_reqs=ssl.CERT_REQUIRED,
options=self.ssl_options
)
self.poolmanager = PoolManager(
*pool_args,
ssl_context=ctx,
**pool_kwargs
)
def __init__(self, cipherSuite=None, **kwargs):
self.cipherSuite = cipherSuite
self.ssl_context = create_urllib3_context(ssl_version=getattr(
ssl, 'PROTOCOL_TLS', ssl.PROTOCOL_TLSv1_2),
ciphers=self.cipherSuite)
self.ssl_context.options |= ssl.OP_NO_SSLv2
self.ssl_context.options |= ssl.OP_NO_SSLv3
self.ssl_context.options |= ssl.OP_NO_TLSv1
self.ssl_context.options |= ssl.OP_NO_TLSv1_1
super(CipherSuiteAdapter, self).__init__(**kwargs)
def __init__(self, cert, key):
import certifi
from OpenSSL import crypto
import urllib3.contrib.pyopenssl # type: ignore
urllib3.contrib.pyopenssl.inject_into_urllib3()
pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, key)
x509 = crypto.load_certificate(crypto.FILETYPE_PEM, cert)
ctx_poolmanager = create_urllib3_context()
ctx_poolmanager.load_verify_locations(cafile=certifi.where())
ctx_poolmanager._ctx.use_certificate(x509)
ctx_poolmanager._ctx.use_privatekey(pkey)
self._ctx_poolmanager = ctx_poolmanager
ctx_proxymanager = create_urllib3_context()
ctx_proxymanager.load_verify_locations(cafile=certifi.where())
ctx_proxymanager._ctx.use_certificate(x509)
ctx_proxymanager._ctx.use_privatekey(pkey)
self._ctx_proxymanager = ctx_proxymanager
super(_MutualTlsAdapter, self).__init__()
def init_poolmanager(self,
connections,
maxsize,
block=False,
**pool_kwargs):
self._pool_connections = connections
self._pool_maxsize = maxsize
self._pool_block = block
ssl_context = create_urllib3_context(ciphers=CIPHERS)
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
strict=True,
ssl_context=ssl_context,
**pool_kwargs)
def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs):
if self._cbapi_force_tls_1_2:
# Force the use of TLS v1.2 when talking to this Cb Response server.
context = create_urllib3_context(ciphers=('TLSv1.2:!aNULL:!eNULL:!MD5'))
context.options |= ssl.OP_NO_SSLv2
context.options |= ssl.OP_NO_SSLv3
context.options |= ssl.OP_NO_TLSv1
context.options |= ssl.OP_NO_TLSv1_1
pool_kwargs['ssl_context'] = context
if not self._cbapi_verify_hostname:
# Provide the ability to validate a Carbon Black server's SSL certificate without validating the hostname
# (by default Carbon Black certificates are "issued" as CN=Self-signed Carbon Black Enterprise Server
# HTTPS Certificate)
pool_kwargs["assert_hostname"] = False
return super(CbAPISessionAdapter, self).init_poolmanager(connections, maxsize, block, **pool_kwargs)
def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs):
if self._cbapi_force_tls_1_2 and REQUESTS_HAS_URLLIB_SSL_CONTEXT:
# Force the use of TLS v1.2 when talking to this Cb Response server.
context = create_urllib3_context(ciphers=('TLSv1.2:!aNULL:!eNULL:!MD5'))
context.options |= ssl.OP_NO_SSLv2
context.options |= ssl.OP_NO_SSLv3
context.options |= ssl.OP_NO_TLSv1
context.options |= ssl.OP_NO_TLSv1_1
pool_kwargs['ssl_context'] = context
if not self._cbapi_verify_hostname:
# Provide the ability to validate a Carbon Black server's SSL certificate without validating the hostname
# (by default Carbon Black certificates are "issued" as CN=Self-signed Carbon Black Enterprise Server
# HTTPS Certificate)
pool_kwargs["assert_hostname"] = False
return super(CbAPISessionAdapter, self).init_poolmanager(connections, maxsize, block, **pool_kwargs)
def init_poolmanager(self, connections, maxsize, block, *args, **kwargs):
context = create_urllib3_context(ciphers=CIPHERS)
kwargs['ssl_context'] = context
return super(SSLAdapter, self).init_poolmanager(
connections, maxsize, block, *args, **kwargs
)
def init_poolmanager(self, *args, **kwargs):
context = create_urllib3_context()
kwargs['ssl_context'] = context
context.load_default_certs() # this loads the OS defaults on Windows
return super(SSLContextAdapter, self).init_poolmanager(*args, **kwargs)
def proxy_manager_for(self, *args, **kwargs):
kwargs['ssl_context'] = create_urllib3_context(
ciphers=self.cipher_suite)
return super(CipherSuiteAdapter,
self).proxy_manager_for(*args, **kwargs)
import gevent
from gevent import monkey
import requests
from pyquery import PyQuery as pq
import threading
from requests.packages.urllib3.util.ssl_ import create_urllib3_context
create_urllib3_context()
monkey.patch_all() # 对所有io操作打上补丁，固定加这一句
def get_title(i):
print(threading.current_thread().name) # 打印出当前线程名称
url = 'https://movie.douban.com/top250?start={}&filter='.format(i * 25)
headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36'
}
res = requests.get(url=url, headers=headers)
doc = pq(res.text)
ids = doc('.pic').items()
for id in ids:
print({id.text(): id.find('img').attr('alt')})
gevent.joinall([gevent.spawn(get_title, i) for i in range(10)])
def proxy_manager_for(self, *args, **kwargs):
context = create_urllib3_context(ciphers=CIPHERS)
kwargs['ssl_context'] = context
return super(DESAdapter, self).proxy_manager_for(*args, **kwargs)
def __init__(self, default_timeout):
self.default_timeout = default_timeout
self.ssl_context = create_urllib3_context(ciphers="ECDHE+AESGCM:!ECDSA") # avoid DH_KEY_TOO_SMALL error
super().__init__(
max_retries=Retry(total=4, backoff_factor=2, status_forcelist=[500, 502, 503, 504])
)
def init_poolmanager(self, *args, **kwargs):
context = create_urllib3_context(ciphers=CIPHERS)
kwargs['ssl_context'] = context
return super(DESAdapter, self).init_poolmanager(*args, **kwargs)
def init_poolmanager(self, *args, **kwargs):
# Override poolmanager setup to stash a copy of the context
# for later access.
self.context = create_urllib3_context()
kwargs['ssl_context'] = self.context
return super().init_poolmanager(*args, **kwargs)
def init_poolmanager(self, *args, **kwargs):
kwargs['ssl_context'] = create_urllib3_context(
ciphers=self.cipherSuite)
return super(CipherSuiteAdapter,
self).init_poolmanager(*args, **kwargs)
def proxy_manager_for(self, *args, **kwargs):
context = create_urllib3_context()
context.load_verify_locations(cadata=keychain.get_truststore_data())
kwargs['ssl_context'] = context
return super().proxy_manager_for(*args, **kwargs)
import requests
# https://github.com/requests/requests/issues/3833
# https://github.com/requests/requests/issues/4046
CIPHERS = (
'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:'
'ECDH+HIGH:DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:'
'RSA+3DES:ECDH+RC4:DH+RC4:RSA+RC4:!aNULL:!eNULL:!MD5'
)
#CIPHERS=('ALL:!aNULL:!SSLv2:!LOW:!EXP:!MD5')
#CIPHERS = (
# 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:'
# 'ECDH+HIGH:DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:'
# 'RSA+3DES:ECDH+RC4:DH+RC4:RSA+RC4:!aNULL:!eNULL:!EXP:-MD5:RSA+RC4+MD5'
# )
context = create_urllib3_context(ciphers=CIPHERS,ssl_version=ssl.PROTOCOL_TLSv1)
class TLSAdapter(HTTPAdapter):
def init_poolmanager(self, *args, **kwargs):
kwargs['ssl_context'] = context
return super(TLSAdapter, self).init_poolmanager(*args, **kwargs)
def proxy_manager_for(self, *args, **kwargs):
kwargs['ssl_context'] = self.create_ssl_context()
return super(TLSAdapter, self).proxy_manager_for(*args, **kwargs)
# session = requests.Session()
# session.mount('https://', TLSAdapter())
# session.get('https://www.jpxgyw.com')