def _create_opener():
"""
Creates an opener for the internet.
It also attaches the :class:`CachingRedirectHandler` to the opener and
sets its User-agent to ``Mozilla/5.0``.
If the Network Proxy settings are set and recognized, it creates the
opener and attaches the proxy_handler to it. The opener is tested and
returned if the test passes.
If the test fails an opener without the proxy settings is created instead
and is returned instead.
"""
use_proxy = False
proxy_handler = None
if NETWORK_PROXY_TYPE == 'http':
use_proxy = True
proxyurl = _get_http_proxy_url()
proxy_handler = ProxyHandler({'http': proxyurl, 'https': proxyurl})
if use_proxy:
openr = build_opener(HTTPHandler(), HTTPSHandler(), proxy_handler,
CachingRedirectHandler)
else:
openr = build_opener(HTTPSHandler(), HTTPSHandler(),
CachingRedirectHandler)
openr.addheaders = [('User-agent', 'Mozilla/5.0')]
global _internet_connected
_internet_connected = _test_opener(openr)
return openr
def create_urllib_https_handler():
context = create_https_context()
try:
return HTTPSHandler(context=context)
except TypeError:
# Older Python versions doesn't have context parameter.
# (Prior to Python 2.7.9/3.4.3
return HTTPSHandler()
def __init__(self,
url,
username=None,
password=None,
cookie=None,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
insecure=False):
super(SFJenkins, self).__init__(urljoin(url, 'jenkins/'), username,
password, timeout)
if not cookie and (not username or not password):
raise ValueError("Authentication needed")
if not cookie:
self.cookie = sfauth.get_cookie(url,
username,
password,
use_ssl=True,
verify=not insecure)
else:
self.cookie = cookie
self.insecure = insecure
self.opener = build_opener()
if self.insecure:
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
self.opener.add_handler(HTTPSHandler(context=ctx))
def _send_request(self, url):
'''
Performs a GET against the given url.
'''
LOGGER.debug('Making an api call to {0}'.format(url))
data = None
try:
ctx = ssl.create_default_context()
ctx.check_hostname = False
if self.ssl_certs['ca_cert']:
ctx.load_verify_locations(cafile=self.ssl_certs.ca_cert)
ctx.load_cert_chain(certfile=self.ssl_certs.client_cert,
keyfile=self.ssl_certs.client_key)
opener = urllib.request.build_opener(HTTPSHandler(context=ctx))
opener.addheaders = []
for k, v in self.headers.items():
opener.addheaders.append((k, v))
response = opener.open(url)
data = response.read()
data = json.loads(data)
except HTTPError as e:
LOGGER.error('HTTPError - status code: {0}, '
'received from {1}'.format(e.code, url))
except URLError as e:
LOGGER.error('URLError - {0} {1}'.format(url, e.reason))
except ValueError as e:
LOGGER.error('Error parsing JSON for url {0}. {1}'.format(url, e))
return data
def debug():
"""
Activate debugging on urllib2.
"""
handler = HTTPSHandler(debuglevel=1)
opener = build_opener(handler)
install_opener(opener)
def debug():
"""Activate debugging on urllib2."""
if six.PY2:
handler = HTTPSHandler(debuglevel=1)
opener = build_opener(handler)
install_opener(opener)
else:
http_client.HTTPConnection.debuglevel = 1
def https_request(self, req):
# Make sure that if we're using an iterable object as the request
# body, that we've also specified Content-Length
data = req.data
if data:
if hasattr(data, 'read') or hasattr(data, '__next__'):
if not req.has_header('Content-length'):
raise ValueError(
"No Content-Length specified for iterable body")
return HTTPSHandler.do_request_(self, req)
def check_nltk():
try:
from nltk.tokenize import word_tokenize
word_tokenize('It\'s.')
except Exception:
import nltk
if not sslVerify:
from ssl import _create_unverified_context
from six.moves.urllib.request import install_opener, HTTPSHandler, build_opener
# TODO: This needs still proxy support !
ctx = _create_unverified_context()
opener = build_opener(HTTPSHandler(context=ctx))
install_opener(opener)
if 'HTTP_PROXY' in os.environ:
nltk.set_proxy(os.environ.get('HTTP_PROXY'))
nltk.download('punkt')
return
def _request(self, method, url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=None,
cert=None,
json=None):
if not headers:
headers = {}
url = quote(url, safe="%/:=&?~#+!$,;'@()*[]")
handlers = []
import sys
# starting with python 2.7.9 urllib verifies every https request
if False == verify and sys.version_info >= (2, 7, 9):
import ssl
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers.append(HTTPSHandler(context=ssl_context))
# handlers.append(HTTPCookieProcessor())
# handlers.append(ErrorHandler)
if not allow_redirects:
handlers.append(NoRedirectHandler)
opener = build_opener(*handlers)
if params:
url = '{0}?{1}'.format(url, urlencode(params))
request = _request(url)
if headers:
for key in headers:
request.add_header(key, headers[key])
if data or json:
if self.plugin.get_dict_value(headers, 'content-type').startswith('application/x-www-form-urlencoded') and data:
# transform a string into a map of values
if isinstance(data, six.string_types):
_data = data.split('&')
data = {}
for item in _data:
name, value = item.split('=')
data[name] = value
request.data = urlencode(data)
elif self.plugin.get_dict_value(headers, 'content-type').startswith('application/json') and data:
request.data = dumps(data).encode('utf-8')
elif json:
request.data = dumps(json).encode('utf-8')
else:
if not isinstance(data, six.string_types):
data = str(data)
if isinstance(data, str):
data = data.encode('utf-8')
request.data = data
elif method.upper() in ['POST', 'PUT']:
request.data = 'null'.encode('utf-8')
request.get_method = lambda: method
result = Response()
response = None
try:
response = opener.open(request, timeout=30)
except HTTPError as e:
# HTTPError implements addinfourl, so we can use the exception to construct a response
if isinstance(e, addinfourl):
response = e
except Exception as e:
result.text = e
return result
# process response
result.headers.update(response.headers)
result.status_code = response.getcode()
if method.upper() == 'HEAD':
return result
elif response.headers.get('Content-Encoding', '').startswith('gzip'):
buf = StringIO(response.read())
f = GzipFile(fileobj=buf)
result.text = f.read()
else:
result.text = response.read()
return result
def __init__(self, context):
HTTPSHandler.__init__(self)
self.context = context
def __init__(self,
url,
cookie_file=None,
username=None,
password=None,
api_token=None,
agent=None,
session=None,
disable_proxy=False,
auth_callback=None,
otp_token_callback=None,
verify_ssl=True,
save_cookies=True,
ext_auth_cookies=None):
if not url.endswith('/'):
url += '/'
self.url = url + 'api/'
self.save_cookies = save_cookies
self.ext_auth_cookies = ext_auth_cookies
if self.save_cookies:
self.cookie_jar, self.cookie_file = create_cookie_jar(
cookie_file=cookie_file)
try:
self.cookie_jar.load(ignore_expires=True)
except IOError:
pass
else:
self.cookie_jar = CookieJar()
self.cookie_file = None
if self.ext_auth_cookies:
try:
self.cookie_jar.load(ext_auth_cookies, ignore_expires=True)
except IOError as e:
logging.critical(
'There was an error while loading a '
'cookie file: %s', e)
pass
# Get the cookie domain from the url. If the domain
# does not contain a '.' (e.g. 'localhost'), we assume
# it is a local domain and suffix it (See RFC 2109).
parsed_url = urlparse(url)
self.domain = parsed_url[1].partition(':')[0] # Remove Port.
if self.domain.count('.') < 1:
self.domain = '%s.local' % self.domain
if session:
cookie = Cookie(version=0,
name=RB_COOKIE_NAME,
value=session,
port=None,
port_specified=False,
domain=self.domain,
domain_specified=True,
domain_initial_dot=True,
path=parsed_url[2],
path_specified=True,
secure=False,
expires=None,
discard=False,
comment=None,
comment_url=None,
rest={'HttpOnly': None})
self.cookie_jar.set_cookie(cookie)
if self.save_cookies:
self.cookie_jar.save()
if username:
# If the username parameter is given, we have to clear the session
# cookie manually or it will override the username:password
# combination retrieved from the authentication callback.
try:
self.cookie_jar.clear(self.domain, parsed_url[2],
RB_COOKIE_NAME)
except KeyError:
pass
# Set up the HTTP libraries to support all of the features we need.
password_mgr = ReviewBoardHTTPPasswordMgr(self.url, username, password,
api_token, auth_callback,
otp_token_callback)
self.preset_auth_handler = PresetHTTPAuthHandler(
self.url, password_mgr)
handlers = []
if not verify_ssl:
context = ssl._create_unverified_context()
handlers.append(HTTPSHandler(context=context))
if disable_proxy:
handlers.append(ProxyHandler({}))
handlers += [
HTTPCookieProcessor(self.cookie_jar),
ReviewBoardHTTPBasicAuthHandler(password_mgr),
HTTPDigestAuthHandler(password_mgr),
self.preset_auth_handler,
ReviewBoardHTTPErrorProcessor(),
]
if agent:
self.agent = agent
else:
self.agent = ('RBTools/' + get_package_version()).encode('utf-8')
opener = build_opener(*handlers)
opener.addheaders = [
(str('User-agent'), str(self.agent)),
]
install_opener(opener)
self._cache = None
self._urlopen = urlopen
def fetchUrl(self, url, ca_certs=None):
"""
:param url: a string identifying the url to fetch
:param ca_certs: a string identifying a file with the CA cert to be
used to trust the remote cert. Pass None to proceed in insecure mode.
If a CA cert filename is passed, it will try to verify the remote cert
signature and it will verify that it matches the remote host hostname.
It handles the differences between python < / >= 2.7.9.
Returns http exit code, http info and the URL content.
Raise RuntimeError if something goes wrong.
"""
self.logger.debug(
'Downloading from {url} - ca_certs={ca_certs}'.format(
url=url,
ca_certs=ca_certs,
))
from six.moves.urllib.request import HTTPSHandler
from six.moves.urllib.request import build_opener
if getattr(ssl, 'create_default_context', None):
# new in python 3.4/2.7.9 (backported by Redhat to 2.7.5)
# TODO: calling context = ssl.create_default_context()
# will load by default also the system defined CA certs which
# in general is a good idea but oVirt python SDK will instead
# ignore them so, until rhbz#1326386 will get fixed, it's better
# to ignore them also here for coherency reasons.
if ca_certs:
context = ssl.create_default_context(cafile=ca_certs)
context.verify_mode = ssl.CERT_REQUIRED
context.check_hostname = ssl.match_hostname
else:
context = ssl.create_default_context()
context.check_hostname = None
context.verify_mode = ssl.CERT_NONE
https_context = contextlib.closing(
build_opener(HTTPSHandler(context=context)).open(url))
else:
# for compatibility with older python releases
import socket
if sys.version_info[0] < 3:
from httplib import HTTPSConnection
else:
from http.client import HTTPSConnection
class MyHTTPSConnection(HTTPSConnection):
def __init__(self, host, **kwargs):
self._ca_certs = kwargs.pop('ca_certs', None)
HTTPSConnection.__init__(self, host, **kwargs)
def connect(self):
self.sock = ssl.wrap_socket(
socket.create_connection((self.host, self.port)),
cert_reqs=(ssl.CERT_REQUIRED
if self._ca_certs else ssl.CERT_NONE),
ca_certs=self._ca_certs,
)
if self._ca_certs:
cert = self.sock.getpeercert()
for field in cert.get('subject', []):
if field[0][0] == 'commonName':
expected = field[0][1]
break
else:
raise RuntimeError(_('No CN in peer certificate'))
if expected != self.host:
raise RuntimeError(
_("Invalid host '{host}' "
"expected '{expected}'").format(
expected=expected,
host=self.host,
))
class MyHTTPSHandler(HTTPSHandler):
def __init__(self, ca_certs=None):
HTTPSHandler.__init__(self)
self._ca_certs = ca_certs
def https_open(self, req):
return self.do_open(self._get_connection, req)
def _get_connection(self, host, timeout):
return MyHTTPSConnection(
host=host,
timeout=timeout,
ca_certs=self._ca_certs,
)
https_context = contextlib.closing(
build_opener(MyHTTPSHandler(ca_certs=ca_certs)).open(url))
with https_context as cont:
info = {}
for key, value in cont.info().items():
info[key] = value
code = cont.getcode()
content = cont.read().decode('utf-8', 'replace')
self.logger.debug('code: %s', code)
self.logger.debug('info: %s', info)
self.logger.debug('content: %s', content)
return code, info, content
def __init__(self, ca_certs=None):
HTTPSHandler.__init__(self)
self._ca_certs = ca_certs
def __init__(self):
HTTPSHandler.__init__(self)
def __init__(self):
HTTPSHandler.__init__(self)