def on_success(cls, metadata):
libraries = metadata['libraries']
if not libraries['standard']['contexts']:
log.debug('Standard SSL library doesn\'t support "SSLContext"')
elif not libraries['standard']['sslwrap']:
log.debug('Standard SSL library doesn\'t support "sslwrap"')
elif libraries['bundled']['version'] > libraries['standard']['version']:
log.debug('Standard SSL library is out of date')
# Initialize ssl library
if metadata['type'] == 'bundled':
# Inject pyOpenSSL into requests
log.debug('Using bundled SSL library (pyOpenSSL)')
try:
from requests.packages.urllib3.contrib.pyopenssl import inject_into_urllib3
inject_into_urllib3()
except Exception as ex:
log.warn('Unable to inject pyOpenSSL into urllib3 - %s',
ex,
exc_info=True)
return
else:
log.debug('Using standard SSL library (ssl)')
def on_success(cls, metadata):
libraries = metadata["libraries"]
if not libraries["standard"]["contexts"]:
log.debug('Standard SSL library doesn\'t support "SSLContext"')
# Initialize ssl library
if metadata["type"] == "bundled":
if libraries["bundled"]["version"] > libraries["standard"]["version"]:
log.debug("Standard SSL library is out of date")
# Inject pyOpenSSL into requests
log.debug("Using bundled SSL library (pyOpenSSL)")
try:
from requests.packages.urllib3.contrib.pyopenssl import inject_into_urllib3
inject_into_urllib3()
except Exception as ex:
log.warn("Unable to inject pyOpenSSL into urllib3 - %s", ex, exc_info=True)
return
else:
log.debug("Using standard SSL library (ssl)")
# Enable secure error reporting
from plugin.core.logger.handlers.error_reporter import RAVEN
RAVEN.set_protocol("threaded+requests+https")
def on_success(cls, metadata):
# Inject pyopenssl into requests
from requests.packages.urllib3.contrib.pyopenssl import inject_into_urllib3
inject_into_urllib3()
# Enable secure error reporting
from plugin.core.logger.handlers.error_reporter import RAVEN
RAVEN.set_protocol('threaded+requests+https')
def test_import(cls):
standard_version = cls._standard_version()
standard_contexts = cls._standard_has_contexts()
bundled_version = cls._bundled_version()
libraries = {
"standard": {"version": standard_version, "contexts": standard_contexts},
"bundled": {"version": bundled_version},
}
# Check if we should use the standard ssl library
if standard_contexts and (bundled_version is None or (standard_version and standard_version > bundled_version)):
return {"type": "standard", "libraries": libraries, "versions": {"openssl": standard_version}}
# Test pyOpenSSL availability
import OpenSSL.SSL
# Try construct SSL context
ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
# Ensure library has SNI support
cnx = OpenSSL.SSL.Connection(ctx)
if not hasattr(cnx, "set_tlsext_host_name"):
raise Exception("Missing SNI extension")
# Ensure binding can be imported
from cryptography.hazmat.bindings.openssl.binding import Binding
assert Binding
# Ensure secure connections work with requests
from requests.packages.urllib3.contrib.pyopenssl import inject_into_urllib3
import requests
inject_into_urllib3()
try:
requests.head("https://api-v2launch.trakt.tv", timeout=3)
except requests.RequestException as ex:
# Ignore failed requests (server error, network problem, etc..)
log.warn("Request failed: %s", ex, exc_info=True)
return {
"type": "bundled",
"libraries": libraries,
"versions": {"openssl": bundled_version, "pyopenssl": getattr(OpenSSL, "__version__", None)},
}
def test_httpretty_fails_when_pyopenssl_is_not_replaced():
('HTTPretty should fail if PyOpenSSL is installed and we do not remove the monkey patch')
# When we don't replace the PyOpenSSL monkeypatch
inject_into_urllib3()
# And we use HTTPretty on as ssl site
HTTPretty.register_uri(HTTPretty.GET, "https://yipit.com/",
body="Find the best daily deals")
# Then we get an SSL error
requests.get.when.called_with('https://yipit.com').should.throw(requests.exceptions.SSLError)
# Undo injection after test
extract_from_urllib3()
def test_import():
import OpenSSL
# Inject pyopenssl into requests
from requests.packages.urllib3.contrib.pyopenssl import inject_into_urllib3
inject_into_urllib3()
# Enable secure error reporting
RAVEN.set_protocol('threaded+requests+https')
return {
'versions': {
'pyopenssl': getattr(OpenSSL, '__version__', None)
}
}
def test_httpretty_fails_when_pyopenssl_is_not_replaced():
('HTTPretty should fail if PyOpenSSL is installed and we do not remove the monkey patch')
# When we don't replace the PyOpenSSL monkeypatch
inject_into_urllib3()
# And we use HTTPretty on as ssl site
HTTPretty.register_uri(HTTPretty.GET, "https://yipit.com/",
body="Find the best daily deals")
# Then we get an SSL error
requests.get.when.called_with('https://yipit.com').should.throw(requests.exceptions.SSLError)
# Undo injection after test
extract_from_urllib3()
def disable(cls):
cls._is_enabled = False
socket.socket = old_socket
if not BAD_SOCKET_SHADOW:
socket.SocketType = old_socket
socket._socketobject = old_socket
socket.create_connection = old_create_connection
socket.gethostname = old_gethostname
socket.gethostbyname = old_gethostbyname
socket.getaddrinfo = old_getaddrinfo
socket.__dict__["socket"] = old_socket
socket.__dict__["_socketobject"] = old_socket
if not BAD_SOCKET_SHADOW:
socket.__dict__["SocketType"] = old_socket
socket.__dict__["create_connection"] = old_create_connection
socket.__dict__["gethostname"] = old_gethostname
socket.__dict__["gethostbyname"] = old_gethostbyname
socket.__dict__["getaddrinfo"] = old_getaddrinfo
if socks:
socks.socksocket = old_socksocket
socks.__dict__["socksocket"] = old_socksocket
if ssl:
ssl.wrap_socket = old_ssl_wrap_socket
ssl.SSLSocket = old_sslsocket
try:
ssl.SSLContext.wrap_socket = old_sslcontext_wrap_socket
except AttributeError:
pass
ssl.__dict__["wrap_socket"] = old_ssl_wrap_socket
ssl.__dict__["SSLSocket"] = old_sslsocket
if not PY3:
ssl.sslwrap_simple = old_sslwrap_simple
ssl.__dict__["sslwrap_simple"] = old_sslwrap_simple
if pyopenssl_override:
inject_into_urllib3()
def disable(cls):
cls._is_enabled = False
socket.socket = old_socket
if not BAD_SOCKET_SHADOW:
socket.SocketType = old_socket
socket._socketobject = old_socket
socket.create_connection = old_create_connection
socket.gethostname = old_gethostname
socket.gethostbyname = old_gethostbyname
socket.getaddrinfo = old_getaddrinfo
socket.__dict__['socket'] = old_socket
socket.__dict__['_socketobject'] = old_socket
if not BAD_SOCKET_SHADOW:
socket.__dict__['SocketType'] = old_socket
socket.__dict__['create_connection'] = old_create_connection
socket.__dict__['gethostname'] = old_gethostname
socket.__dict__['gethostbyname'] = old_gethostbyname
socket.__dict__['getaddrinfo'] = old_getaddrinfo
if socks:
socks.socksocket = old_socksocket
socks.__dict__['socksocket'] = old_socksocket
if ssl:
ssl.wrap_socket = old_ssl_wrap_socket
ssl.SSLSocket = old_sslsocket
try:
ssl.SSLContext.wrap_socket = old_sslcontext_wrap_socket
except AttributeError:
pass
ssl.__dict__['wrap_socket'] = old_ssl_wrap_socket
ssl.__dict__['SSLSocket'] = old_sslsocket
if not PY3:
ssl.sslwrap_simple = old_sslwrap_simple
ssl.__dict__['sslwrap_simple'] = old_sslwrap_simple
if pyopenssl_override:
inject_into_urllib3()
def check_requests_ssl():
global _CHECKED
if _CHECKED or os.environ.get(ENV_VAR_IGNORE):
return
try:
from requests.packages.urllib3.contrib import pyopenssl
pyopenssl.inject_into_urllib3()
except ImportError:
print(
'WARNING: You do not have proper SSL libraries installed for python. '
'You may be at risk for man in the middle attacks. Installing '
'pyOpenSSL (see https://urllib3.readthedocs.io/en/latest/user-guide.'
'html#ssl-py2 for more details), or using the recipe engine '
'`--use-bootstrap` flag will ensure proper python SSL libraries.')
from requests.packages import urllib3
# TODO(martiniss): Make this mandatory by default, once chrome-infra is
# using proper python SSL libraries.
urllib3.disable_warnings()
_CHECKED = True
def disable(cls):
cls._is_enabled = False
socket.socket = old_socket
if not BAD_SOCKET_SHADOW:
socket.SocketType = old_socket
socket._socketobject = old_socket
socket.create_connection = old_create_connection
socket.gethostname = old_gethostname
socket.gethostbyname = old_gethostbyname
socket.getaddrinfo = old_getaddrinfo
socket.__dict__['socket'] = old_socket
socket.__dict__['_socketobject'] = old_socket
if not BAD_SOCKET_SHADOW:
socket.__dict__['SocketType'] = old_socket
socket.__dict__['create_connection'] = old_create_connection
socket.__dict__['gethostname'] = old_gethostname
socket.__dict__['gethostbyname'] = old_gethostbyname
socket.__dict__['getaddrinfo'] = old_getaddrinfo
if socks:
socks.socksocket = old_socksocket
socks.__dict__['socksocket'] = old_socksocket
if ssl:
ssl.wrap_socket = old_ssl_wrap_socket
ssl.SSLSocket = old_sslsocket
ssl.__dict__['wrap_socket'] = old_ssl_wrap_socket
ssl.__dict__['SSLSocket'] = old_sslsocket
if not PY3:
ssl.sslwrap_simple = old_sslwrap_simple
ssl.__dict__['sslwrap_simple'] = old_sslwrap_simple
if pyopenssl_override:
inject_into_urllib3()
def on_success(cls, metadata):
libraries = metadata['libraries']
if not libraries['standard']['contexts']:
log.debug('Standard SSL library doesn\'t support "SSLContext"')
elif not libraries['standard']['sslwrap']:
log.debug('Standard SSL library doesn\'t support "sslwrap"')
elif libraries['bundled']['version'] > libraries['standard']['version']:
log.debug('Standard SSL library is out of date')
# Initialize ssl library
if metadata['type'] == 'bundled':
# Inject pyOpenSSL into requests
log.debug('Using bundled SSL library (pyOpenSSL)')
try:
from requests.packages.urllib3.contrib.pyopenssl import inject_into_urllib3
inject_into_urllib3()
except Exception as ex:
log.warn('Unable to inject pyOpenSSL into urllib3 - %s', ex, exc_info=True)
return
else:
log.debug('Using standard SSL library (ssl)')
def disable(cls):
cls._is_enabled = False
socket.socket = old_socket
socket.SocketType = old_SocketType
socket._socketobject = old_socket
socket.create_connection = old_create_connection
socket.gethostname = old_gethostname
socket.gethostbyname = old_gethostbyname
socket.getaddrinfo = old_getaddrinfo
socket.__dict__['socket'] = old_socket
socket.__dict__['_socketobject'] = old_socket
socket.__dict__['SocketType'] = old_SocketType
socket.__dict__['create_connection'] = old_create_connection
socket.__dict__['gethostname'] = old_gethostname
socket.__dict__['gethostbyname'] = old_gethostbyname
socket.__dict__['getaddrinfo'] = old_getaddrinfo
if socks:
socks.socksocket = old_socksocket
socks.__dict__['socksocket'] = old_socksocket
if ssl:
ssl.wrap_socket = old_ssl_wrap_socket
ssl.SSLSocket = old_sslsocket
ssl.__dict__['wrap_socket'] = old_ssl_wrap_socket
ssl.__dict__['SSLSocket'] = old_sslsocket
if not PY3:
ssl.sslwrap_simple = old_sslwrap_simple
ssl.__dict__['sslwrap_simple'] = old_sslwrap_simple
if pyopenssl_override:
# Replace PyOpenSSL Monkeypatching
inject_into_urllib3()
import pkg_resources
import cliquet
from pyramid.config import Configurator
from syncto.heartbeat import ping_sync_cluster
# Module version, as defined in PEP-0396.
__version__ = pkg_resources.get_distribution(__package__).version
try:
# Verify that we are using the Py2 urllib3 version with OpenSSL installed
from requests.packages.urllib3.contrib import pyopenssl
except ImportError: # Pragma: no cover
pass
else:
pyopenssl.inject_into_urllib3() # Pragma: no cover
AUTHORIZATION_HEADER = 'Authorization'
CLIENT_STATE_HEADER = 'X-Client-State'
CLIENT_STATE_LENGTH = 32
DEFAULT_SETTINGS = {
'project_name': 'syncto',
'project_docs': 'https://syncto.readthedocs.io/',
'cache_hmac_secret': None,
'cache_credentials_ttl_seconds': 300,
'token_server_url': 'https://token.services.mozilla.com/',
'token_server_heartbeat_timeout_seconds': 5,
'certificate_ca_bundle': None,
}
def test_import(cls):
standard_version = cls._standard_version()
standard_contexts = cls._standard_has_contexts()
standard_sslwrap = cls._standard_has_sslwrap()
bundled_version = cls._bundled_version()
libraries = {
'standard': {
'version': standard_version,
'contexts': standard_contexts,
'sslwrap': standard_sslwrap
},
'bundled': {
'version': bundled_version
}
}
# Check if we should use the standard ssl library
if cls._use_standard(libraries):
return {
'type': 'standard',
'libraries': libraries,
'versions': {
'openssl': standard_version
}
}
# Test pyOpenSSL availability
import OpenSSL.SSL
# Try construct SSL context
ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
# Ensure library has SNI support
cnx = OpenSSL.SSL.Connection(ctx)
if not hasattr(cnx, 'set_tlsext_host_name'):
raise Exception('Missing SNI extension')
# Ensure binding can be imported
from cryptography.hazmat.bindings.openssl.binding import Binding
assert Binding
# Ensure secure connections work with requests
from requests.packages.urllib3.contrib.pyopenssl import inject_into_urllib3
import requests
inject_into_urllib3()
try:
requests.head('https://api-v2launch.trakt.tv', timeout=3)
except requests.RequestException as ex:
# Ignore failed requests (server error, network problem, etc..)
log.warn('Request failed: %s', ex, exc_info=True)
return {
'type': 'bundled',
'libraries': libraries,
'versions': {
'openssl': bundled_version,
'pyopenssl': getattr(OpenSSL, '__version__', None)
}
}
def disable(cls):
OriginalHTTPretty.disable()
if pyopenssl_override:
# Put the pyopenssl version back in place
inject_into_urllib3()
def test_import(cls):
standard_version = cls._standard_version()
standard_contexts = cls._standard_has_contexts()
standard_sslwrap = cls._standard_has_sslwrap()
bundled_version = cls._bundled_version()
libraries = {
'standard': {
'version': standard_version,
'contexts': standard_contexts,
'sslwrap': standard_sslwrap
},
'bundled': {
'version': bundled_version
}
}
# Check if we should use the standard ssl library
if cls._use_standard(libraries):
return {
'type': 'standard',
'libraries': libraries,
'versions': {
'openssl': standard_version
}
}
# Test pyOpenSSL availability
import OpenSSL.SSL
# Try construct SSL context
ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
# Ensure library has SNI support
cnx = OpenSSL.SSL.Connection(ctx)
if not hasattr(cnx, 'set_tlsext_host_name'):
raise Exception('Missing SNI extension')
# Ensure binding can be imported
from cryptography.hazmat.bindings.openssl.binding import Binding
assert Binding
# Ensure secure connections work with requests
from requests.packages.urllib3.contrib.pyopenssl import inject_into_urllib3
import requests
inject_into_urllib3()
try:
requests.head('https://api-v2launch.trakt.tv', timeout=3)
except requests.RequestException as ex:
# Ignore failed requests (server error, network problem, etc..)
log.warn('Request failed: %s', ex, exc_info=True)
return {
'type': 'bundled',
'libraries': libraries,
'versions': {
'openssl': bundled_version,
'pyopenssl': getattr(OpenSSL, '__version__', None)
}
}
try:
from requests.packages.urllib3 import disable_warnings
from requests.packages.urllib3.contrib.pyopenssl import inject_into_urllib3
except ImportError:
from urllib3 import disable_warnings
from urllib3.contrib.pyopenssl import inject_into_urllib3
from .check_impl import (
add_check_prefix,
make_check,
sequential_check,
)
# Ensure we always use pyOpenSSL instead of the ssl builtin
inject_into_urllib3()
CA_CERTS = []
def load_tls_certs(path):
cert_map = {}
for filepath in glob.glob("{}/*.pem".format(os.path.abspath(path))):
# There might be some dead symlinks in there,
# so let's make sure it's real.
if os.path.isfile(filepath):
data = open(filepath).read()
x509 = load_certificate(FILETYPE_PEM, data)
# Now, de-duplicate in case the same cert has multiple names.
cert_map[x509.digest('sha1')] = x509
import time
import os
VERIFY_SSL = False
VERIFY_MSGS = []
import requests
from requests.exceptions import SSLError
# py3.2
if sys.hexversion >= 0x30200f0:
VERIFY_SSL = True
else:
try: # import and enable SNI support for py2
from requests.packages.urllib3.contrib import pyopenssl
pyopenssl.inject_into_urllib3()
VERIFY_SSL = True
VERIFY_MSGS = ["Successfully enabled ssl certificate verification."]
except ImportError as e:
VERIFY_MSGS = [
"Failed to import and inject pyopenssl/SNI support into urllib3",
"Disabling certificate verification",
"Error was: {0}".format(e)
]
VERIFY_SSL = False
def fileopen(path, mode='r', enc="UTF-8"):
"""py2, py3 compatibility function"""
try:
f = open(path, mode, encoding=enc)
def disable(cls):
OriginalHTTPretty.disable()
if pyopenssl_override:
# Put the pyopenssl version back in place
inject_into_urllib3()
import pkg_resources
import cliquet
from pyramid.config import Configurator
from syncto.heartbeat import ping_sync_cluster
# Module version, as defined in PEP-0396.
__version__ = pkg_resources.get_distribution(__package__).version
try:
# Verify that we are using the Py2 urllib3 version with OpenSSL installed
from requests.packages.urllib3.contrib import pyopenssl
except ImportError: # Pragma: no cover
pass
else:
pyopenssl.inject_into_urllib3() # Pragma: no cover
AUTHORIZATION_HEADER = 'Authorization'
CLIENT_STATE_HEADER = 'X-Client-State'
DEFAULT_SETTINGS = {
'project_name': 'syncto',
'project_docs': 'https://syncto.readthedocs.org/',
'cache_hmac_secret': None,
'cache_credentials_ttl_seconds': 300,
'token_server_url': 'https://token.services.mozilla.com/',
'token_server_heartbeat_timeout_seconds': 5,
}
def main(global_config, **settings):
from io import StringIO
from multiprocessing import Pool
import requests
from .config import (
NgdConfig,
)
from .jobs import DownloadJob
from . import metadata
from .summary import SummaryReader
# Python < 2.7.9 hack: fix ssl support
if sys.version_info < (2, 7, 9): # pragma: no cover
from requests.packages.urllib3.contrib import pyopenssl
pyopenssl.inject_into_urllib3()
# Get the user's cache dir in a system-independent manner
CACHE_DIR = user_cache_dir(appname="ncbi-genome-download", appauthor="kblin")
def argument_parser(version=None):
"""Create the argument parser for ncbi-genome-download."""
parser = argparse.ArgumentParser()
parser.add_argument('group',
default=NgdConfig.get_default('group'),
help='The NCBI taxonomic group to download (default: %(default)s). '
'A comma-separated list of taxonomic groups is also possible. For example: "bacteria,viral"'
'Choose from: {choices}'.format(choices=NgdConfig.get_choices('group')))
parser.add_argument('-s', '--section', dest='section',
try:
from requests.packages.urllib3 import disable_warnings
from requests.packages.urllib3.contrib.pyopenssl import inject_into_urllib3
except ImportError:
from urllib3 import disable_warnings
from urllib3.contrib.pyopenssl import inject_into_urllib3
from .check_impl import (
add_check_prefix,
make_check,
sequential_check,
)
# Ensure we always use pyOpenSSL instead of the ssl builtin
inject_into_urllib3()
CA_CERTS = []
def load_tls_certs(path):
cert_map = {}
for filepath in glob.glob("{}/*.pem".format(os.path.abspath(path))):
# There might be some dead symlinks in there,
# so let's make sure it's real.
if os.path.isfile(filepath):
data = open(filepath).read()
x509 = load_certificate(FILETYPE_PEM, data)
# Now, de-duplicate in case the same cert has multiple names.
cert_map[x509.digest('sha1')] = x509
