def _create_opener(): """ Creates an opener for the internet. It also attaches the :class:`CachingRedirectHandler` to the opener and sets its User-agent to ``Mozilla/5.0``. If the Network Proxy settings are set and recognized, it creates the opener and attaches the proxy_handler to it. The opener is tested and returned if the test passes. If the test fails an opener without the proxy settings is created instead and is returned instead. """ use_proxy = False proxy_handler = None if NETWORK_PROXY_TYPE == 'http': use_proxy = True proxyurl = _get_http_proxy_url() proxy_handler = ProxyHandler({'http': proxyurl, 'https': proxyurl}) if use_proxy: openr = build_opener(HTTPHandler(), HTTPSHandler(), proxy_handler, CachingRedirectHandler) else: openr = build_opener(HTTPSHandler(), HTTPSHandler(), CachingRedirectHandler) openr.addheaders = [('User-agent', 'Mozilla/5.0')] global _internet_connected _internet_connected = _test_opener(openr) return openr
def create_urllib_https_handler(): context = create_https_context() try: return HTTPSHandler(context=context) except TypeError: # Older Python versions doesn't have context parameter. # (Prior to Python 2.7.9/3.4.3 return HTTPSHandler()
def __init__(self, url, username=None, password=None, cookie=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, insecure=False): super(SFJenkins, self).__init__(urljoin(url, 'jenkins/'), username, password, timeout) if not cookie and (not username or not password): raise ValueError("Authentication needed") if not cookie: self.cookie = sfauth.get_cookie(url, username, password, use_ssl=True, verify=not insecure) else: self.cookie = cookie self.insecure = insecure self.opener = build_opener() if self.insecure: ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE self.opener.add_handler(HTTPSHandler(context=ctx))
def _send_request(self, url): ''' Performs a GET against the given url. ''' LOGGER.debug('Making an api call to {0}'.format(url)) data = None try: ctx = ssl.create_default_context() ctx.check_hostname = False if self.ssl_certs['ca_cert']: ctx.load_verify_locations(cafile=self.ssl_certs.ca_cert) ctx.load_cert_chain(certfile=self.ssl_certs.client_cert, keyfile=self.ssl_certs.client_key) opener = urllib.request.build_opener(HTTPSHandler(context=ctx)) opener.addheaders = [] for k, v in self.headers.items(): opener.addheaders.append((k, v)) response = opener.open(url) data = response.read() data = json.loads(data) except HTTPError as e: LOGGER.error('HTTPError - status code: {0}, ' 'received from {1}'.format(e.code, url)) except URLError as e: LOGGER.error('URLError - {0} {1}'.format(url, e.reason)) except ValueError as e: LOGGER.error('Error parsing JSON for url {0}. {1}'.format(url, e)) return data
def debug(): """ Activate debugging on urllib2. """ handler = HTTPSHandler(debuglevel=1) opener = build_opener(handler) install_opener(opener)
def debug(): """Activate debugging on urllib2.""" if six.PY2: handler = HTTPSHandler(debuglevel=1) opener = build_opener(handler) install_opener(opener) else: http_client.HTTPConnection.debuglevel = 1
def https_request(self, req): # Make sure that if we're using an iterable object as the request # body, that we've also specified Content-Length data = req.data if data: if hasattr(data, 'read') or hasattr(data, '__next__'): if not req.has_header('Content-length'): raise ValueError( "No Content-Length specified for iterable body") return HTTPSHandler.do_request_(self, req)
def check_nltk(): try: from nltk.tokenize import word_tokenize word_tokenize('It\'s.') except Exception: import nltk if not sslVerify: from ssl import _create_unverified_context from six.moves.urllib.request import install_opener, HTTPSHandler, build_opener # TODO: This needs still proxy support ! ctx = _create_unverified_context() opener = build_opener(HTTPSHandler(context=ctx)) install_opener(opener) if 'HTTP_PROXY' in os.environ: nltk.set_proxy(os.environ.get('HTTP_PROXY')) nltk.download('punkt') return
def _request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=None, cert=None, json=None): if not headers: headers = {} url = quote(url, safe="%/:=&?~#+!$,;'@()*[]") handlers = [] import sys # starting with python 2.7.9 urllib verifies every https request if False == verify and sys.version_info >= (2, 7, 9): import ssl ssl_context = ssl.create_default_context() ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE handlers.append(HTTPSHandler(context=ssl_context)) # handlers.append(HTTPCookieProcessor()) # handlers.append(ErrorHandler) if not allow_redirects: handlers.append(NoRedirectHandler) opener = build_opener(*handlers) if params: url = '{0}?{1}'.format(url, urlencode(params)) request = _request(url) if headers: for key in headers: request.add_header(key, headers[key]) if data or json: if self.plugin.get_dict_value(headers, 'content-type').startswith('application/x-www-form-urlencoded') and data: # transform a string into a map of values if isinstance(data, six.string_types): _data = data.split('&') data = {} for item in _data: name, value = item.split('=') data[name] = value request.data = urlencode(data) elif self.plugin.get_dict_value(headers, 'content-type').startswith('application/json') and data: request.data = dumps(data).encode('utf-8') elif json: request.data = dumps(json).encode('utf-8') else: if not isinstance(data, six.string_types): data = str(data) if isinstance(data, str): data = data.encode('utf-8') request.data = data elif method.upper() in ['POST', 'PUT']: request.data = 'null'.encode('utf-8') request.get_method = lambda: method result = Response() response = None try: response = opener.open(request, timeout=30) except HTTPError as e: # HTTPError implements addinfourl, so we can use the exception to construct a response if isinstance(e, addinfourl): response = e except Exception as e: result.text = e return result # process response result.headers.update(response.headers) result.status_code = response.getcode() if method.upper() == 'HEAD': return result elif response.headers.get('Content-Encoding', '').startswith('gzip'): buf = StringIO(response.read()) f = GzipFile(fileobj=buf) result.text = f.read() else: result.text = response.read() return result
def __init__(self, context): HTTPSHandler.__init__(self) self.context = context
def __init__(self, url, cookie_file=None, username=None, password=None, api_token=None, agent=None, session=None, disable_proxy=False, auth_callback=None, otp_token_callback=None, verify_ssl=True, save_cookies=True, ext_auth_cookies=None): if not url.endswith('/'): url += '/' self.url = url + 'api/' self.save_cookies = save_cookies self.ext_auth_cookies = ext_auth_cookies if self.save_cookies: self.cookie_jar, self.cookie_file = create_cookie_jar( cookie_file=cookie_file) try: self.cookie_jar.load(ignore_expires=True) except IOError: pass else: self.cookie_jar = CookieJar() self.cookie_file = None if self.ext_auth_cookies: try: self.cookie_jar.load(ext_auth_cookies, ignore_expires=True) except IOError as e: logging.critical( 'There was an error while loading a ' 'cookie file: %s', e) pass # Get the cookie domain from the url. If the domain # does not contain a '.' (e.g. 'localhost'), we assume # it is a local domain and suffix it (See RFC 2109). parsed_url = urlparse(url) self.domain = parsed_url[1].partition(':')[0] # Remove Port. if self.domain.count('.') < 1: self.domain = '%s.local' % self.domain if session: cookie = Cookie(version=0, name=RB_COOKIE_NAME, value=session, port=None, port_specified=False, domain=self.domain, domain_specified=True, domain_initial_dot=True, path=parsed_url[2], path_specified=True, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest={'HttpOnly': None}) self.cookie_jar.set_cookie(cookie) if self.save_cookies: self.cookie_jar.save() if username: # If the username parameter is given, we have to clear the session # cookie manually or it will override the username:password # combination retrieved from the authentication callback. try: self.cookie_jar.clear(self.domain, parsed_url[2], RB_COOKIE_NAME) except KeyError: pass # Set up the HTTP libraries to support all of the features we need. password_mgr = ReviewBoardHTTPPasswordMgr(self.url, username, password, api_token, auth_callback, otp_token_callback) self.preset_auth_handler = PresetHTTPAuthHandler( self.url, password_mgr) handlers = [] if not verify_ssl: context = ssl._create_unverified_context() handlers.append(HTTPSHandler(context=context)) if disable_proxy: handlers.append(ProxyHandler({})) handlers += [ HTTPCookieProcessor(self.cookie_jar), ReviewBoardHTTPBasicAuthHandler(password_mgr), HTTPDigestAuthHandler(password_mgr), self.preset_auth_handler, ReviewBoardHTTPErrorProcessor(), ] if agent: self.agent = agent else: self.agent = ('RBTools/' + get_package_version()).encode('utf-8') opener = build_opener(*handlers) opener.addheaders = [ (str('User-agent'), str(self.agent)), ] install_opener(opener) self._cache = None self._urlopen = urlopen
def fetchUrl(self, url, ca_certs=None): """ :param url: a string identifying the url to fetch :param ca_certs: a string identifying a file with the CA cert to be used to trust the remote cert. Pass None to proceed in insecure mode. If a CA cert filename is passed, it will try to verify the remote cert signature and it will verify that it matches the remote host hostname. It handles the differences between python < / >= 2.7.9. Returns http exit code, http info and the URL content. Raise RuntimeError if something goes wrong. """ self.logger.debug( 'Downloading from {url} - ca_certs={ca_certs}'.format( url=url, ca_certs=ca_certs, )) from six.moves.urllib.request import HTTPSHandler from six.moves.urllib.request import build_opener if getattr(ssl, 'create_default_context', None): # new in python 3.4/2.7.9 (backported by Redhat to 2.7.5) # TODO: calling context = ssl.create_default_context() # will load by default also the system defined CA certs which # in general is a good idea but oVirt python SDK will instead # ignore them so, until rhbz#1326386 will get fixed, it's better # to ignore them also here for coherency reasons. if ca_certs: context = ssl.create_default_context(cafile=ca_certs) context.verify_mode = ssl.CERT_REQUIRED context.check_hostname = ssl.match_hostname else: context = ssl.create_default_context() context.check_hostname = None context.verify_mode = ssl.CERT_NONE https_context = contextlib.closing( build_opener(HTTPSHandler(context=context)).open(url)) else: # for compatibility with older python releases import socket if sys.version_info[0] < 3: from httplib import HTTPSConnection else: from http.client import HTTPSConnection class MyHTTPSConnection(HTTPSConnection): def __init__(self, host, **kwargs): self._ca_certs = kwargs.pop('ca_certs', None) HTTPSConnection.__init__(self, host, **kwargs) def connect(self): self.sock = ssl.wrap_socket( socket.create_connection((self.host, self.port)), cert_reqs=(ssl.CERT_REQUIRED if self._ca_certs else ssl.CERT_NONE), ca_certs=self._ca_certs, ) if self._ca_certs: cert = self.sock.getpeercert() for field in cert.get('subject', []): if field[0][0] == 'commonName': expected = field[0][1] break else: raise RuntimeError(_('No CN in peer certificate')) if expected != self.host: raise RuntimeError( _("Invalid host '{host}' " "expected '{expected}'").format( expected=expected, host=self.host, )) class MyHTTPSHandler(HTTPSHandler): def __init__(self, ca_certs=None): HTTPSHandler.__init__(self) self._ca_certs = ca_certs def https_open(self, req): return self.do_open(self._get_connection, req) def _get_connection(self, host, timeout): return MyHTTPSConnection( host=host, timeout=timeout, ca_certs=self._ca_certs, ) https_context = contextlib.closing( build_opener(MyHTTPSHandler(ca_certs=ca_certs)).open(url)) with https_context as cont: info = {} for key, value in cont.info().items(): info[key] = value code = cont.getcode() content = cont.read().decode('utf-8', 'replace') self.logger.debug('code: %s', code) self.logger.debug('info: %s', info) self.logger.debug('content: %s', content) return code, info, content
def __init__(self, ca_certs=None): HTTPSHandler.__init__(self) self._ca_certs = ca_certs
def __init__(self): HTTPSHandler.__init__(self)
def __init__(self): HTTPSHandler.__init__(self)