def _get_response(request): # type: (Any) -> Any logger.debug("_get_response()") if logger.isEnabledFor(logging.DEBUG): opener = build_opener(HTTPSHandler(debuglevel=1)) else: opener = build_opener(HTTPSHandler()) try: response = opener.open(request) except HTTPError as error_response: response = error_response content_encoding = response.headers.get("content-encoding", "") content_type = response.headers.get("content-type") encoding = content_type.split("charset=")[-1] r = response.read() if content_encoding == "gzip": s = gzip.decompress(r) else: s = r.decode(encoding) if content_type.startswith("application/json"): return json.loads(s) return s
def mock_urlopen(cls, url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, cafile=None, capath=None, cadefault=False, context=None): # Check for passed-through headers if not isinstance(url, urllib2.Request) or \ cls._headers.items()[0] not in url.header_items(): raise urllib2.URLError('Headers not passed through') global _opener if cafile or capath or cadefault: if context is not None: raise ValueError( "You can't pass both context and any of cafile, capath, " "and cadefault" ) if not _have_ssl: raise ValueError('SSL support not available') context = ssl.create_default_context( purpose=ssl.Purpose.SERVER_AUTH, cafile=cafile, capath=capath) https_handler = HTTPSHandler(context=context) opener = build_opener(https_handler) elif context: https_handler = HTTPSHandler(context=context) opener = build_opener(https_handler) elif _opener is None: _opener = opener = build_opener() else: opener = _opener return opener.open(url, data, timeout)
def __init__(self, timeout=None, debuglevel=0): """Initialize the object. Args: timeout: the socket connect timeout value. debuglevel: the debuglevel level. """ _HS.__init__(self, debuglevel) TimeoutHTTPSConnection._timeout = timeout
def __init__(self, *args, **kwargs): try: kwargs['context'] = ssl._create_unverified_context() except AttributeError: # Python prior to 2.7.9 doesn't have default-enabled certificate # verification pass HTTPSHandler.__init__(self, *args, **kwargs)
def __init__(self, key, cert): ctx = ssl._create_unverified_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE HTTPSHandler.__init__(self, context=ctx) self.context = ctx self.key = key self.cert = cert
def u2handlers(self): handlers = [] handlers.append(ProxyHandler(self.proxy)) handlers.append(HTTPBasicAuthHandler(self.pm)) # python ssl Context support - PEP 0466 if hasattr(ssl, '_create_unverified_context'): ssl_context = ssl._create_unverified_context() handlers.append(HTTPSHandler(context=ssl_context)) else: handlers.append(HTTPSHandler()) return handlers
def __init__(self): """Build an HTTPS opener.""" # Based on pip 1.4.1's URLOpener # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): self._opener = build_opener( HTTPSHandler(context=create_CERT_NONE_context())) else: self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: for handler in self._opener.handlers: if isinstance(handler, HTTPHandler): self._opener.handlers.remove(handler)
def build_opener(self, debug=False): """Create handlers with the appropriate debug level. We intentionally create new ones because the OpenerDirector class in urllib2 is smart enough to replace its internal versions with ours if we pass them into the urllib2.build_opener method. This is much easier than trying to introspect into the OpenerDirector to find the existing handlers. Based on http://code.activestate.com/recipes/440574/#c1 TODO: Implement workaround for http://bugs.python.org/issue7152 """ http_handler = HTTPHandler(debuglevel=debug) https_handler = HTTPSHandler(debuglevel=debug) proxy_handler = ProxyHandler(debuglevel=debug) unknown_handler = UnknownHandler(debuglevel=debug) http_default_error_handler = HTTPDefaultErrorHandler(debuglevel=debug) http_redirect_handler = HTTPRedirectHandler(debuglevel=debug) http_error_processor = HTTPErrorProcessor(debuglevel=debug) handlers = [http_handler, https_handler, proxy_handler, \ unknown_handler, http_default_error_handler, \ http_redirect_handler, http_error_processor] opener = build_opener(handlers) return opener
def opener(): opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: for handler in opener.handlers: if isinstance(handler, HTTPHandler): opener.handlers.remove(handler) return opener
def get_cert(cls, url): context = ssl.create_default_context() context.load_cert_chain(cls.ssl_file('client-certificate.pem'), cls.ssl_file('client-private-key.pem'), 'client-password') context.load_verify_locations(cls.ssl_file('ca-certificate.pem')) opener = build_opener(HTTPSHandler(context=context)) return opener.open(url).read().decode('utf-8')
def get_new_cookie(self): # Start by prompting user to input their credentials # Another Python2/3 workaround try: new_username = raw_input("Username: "******"Username: "******"Password (will not be displayed): ") # Build URS4 Cookie request auth_cookie_url = self.asf_urs4['url'] + '?client_id=' + self.asf_urs4['client'] + '&redirect_uri=' + \ self.asf_urs4['redir'] + '&response_type=code&state=' try: # python2 user_pass = base64.b64encode(bytes(new_username + ":" + new_password)) except TypeError: # python3 user_pass = base64.b64encode(bytes(new_username + ":" + new_password, "utf-8")) user_pass = user_pass.decode("utf-8") # Authenticate against URS, grab all the cookies self.cookie_jar = MozillaCookieJar() opener = build_opener(HTTPCookieProcessor(self.cookie_jar), HTTPHandler(), HTTPSHandler(**self.context)) request = Request(auth_cookie_url, headers={"Authorization": "Basic {0}".format(user_pass)}) # Watch out cookie rejection! try: response = opener.open(request) except HTTPError as e: if e.code == 401: print(" > Username and Password combo was not successful. Please try again.") return False else: # If an error happens here, the user most likely has not confirmed EULA. print("\nIMPORTANT: There was an error obtaining a download cookie!") print("Your user appears to lack permission to download data from the ASF Datapool.") print( "\n\nNew users: you must first log into Vertex and accept the EULA. In addition, your Study Area must be set at Earthdata https://urs.earthdata.nasa.gov") exit(-1) except URLError as e: print("\nIMPORTANT: There was a problem communicating with URS, unable to obtain cookie. ") print("Try cookie generation later.") exit(-1) # Did we get a cookie? if self.check_cookie_is_logged_in(self.cookie_jar): # COOKIE SUCCESS! self.cookie_jar.save(self.cookie_jar_path) return True # if we aren't successful generating the cookie, nothing will work. Stop here! print("WARNING: Could not generate new cookie! Cannot proceed. Please try Username and Password again.") print("Response was {0}.".format(response.getcode())) print( "\n\nNew users: you must first log into Vertex and accept the EULA. In addition, your Study Area must be set at Earthdata https://urs.earthdata.nasa.gov") exit(-1)
def u2handlers(self): """Get a collection of urllib handlers. """ handlers = suds.transport.http.HttpTransport.u2handlers(self) if self.ssl_context: try: handlers.append(HTTPSHandler(context=self.ssl_context, check_hostname=self.verify)) except TypeError: # Python 2.7.9 HTTPSHandler does not accept the # check_hostname keyword argument. # # Note that even older Python versions would also # croak on the context keyword argument. But these # old versions do not have SSLContext either, so we # will not end up here in the first place. handlers.append(HTTPSHandler(context=self.ssl_context)) return handlers
def __init__(self): """Build an HTTPS opener.""" # Based on pip 1.4.1's URLOpener # This verifies certs on only Python >=2.7.9. self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: for handler in self._opener.handlers: if isinstance(handler, HTTPHandler): self._opener.handlers.remove(handler)
def u2handlers(self): handlers = HttpAuthenticated.u2handlers(self) context = ssl._create_unverified_context() # add an HTTPS handler, using the custom context handlers.append(HTTPSHandler(context=context)) return handlers
def __init__(self, host, auth_header, debug=0): self._host = host self._headers['Authorization'] = auth_header self._opener = build_opener(HTTPHandler(debuglevel=debug), HTTPSHandler(debuglevel=debug), HTTPCookieProcessor(CookieJar()), LoggingHandler(), HTTPDefaultErrorHandler())
def check_cookie(self): if self.cookie_jar is None: print(" > Cookiejar is bunk: {0}".format(self.cookie_jar)) return False # File we know is valid, used to validate cookie file_check = 'https://urs.earthdata.nasa.gov/profile' # Apply custom Redirect Hanlder opener = build_opener(HTTPCookieProcessor( self.cookie_jar), HTTPHandler(), HTTPSHandler(**self.context)) install_opener(opener) # Attempt a HEAD request request = Request(file_check) request.get_method = lambda: 'HEAD' try: print(" > attempting to download {0}".format(file_check)) response = urlopen(request, timeout=30) resp_code = response.getcode() # Make sure we're logged in if not self.check_cookie_is_logged_in(self.cookie_jar): return False # Save cookiejar self.cookie_jar.save(self.cookie_jar_path) except HTTPError: # If we ge this error, again, it likely means the user has not agreed to current EULA print("\nIMPORTANT: ") print( "Your user appears to lack permissions to download data from the ASF Datapool.") print("\n\nNew users: you must first log into Vertex and accept the EULA. In addition, your Study Area must be set at Earthdata https://urs.earthdata.nasa.gov") exit(-1) # This return codes indicate the USER has not been approved to download the data if resp_code in (300, 301, 302, 303): try: redir_url = response.info().getheader('Location') except AttributeError: redir_url = response.getheader('Location') # Funky Test env: if ("vertex-retired.daac.asf.alaska.edu" in redir_url and "test" in self.asf_urs4['redir']): print("Cough, cough. It's dusty in this test env!") return True print( "Redirect ({0}) occured, invalid cookie value!".format(resp_code)) return False # These are successes! if resp_code in (200, 307): return True return False
def u2handlers(self): # use handlers from superclass handlers = HttpAuthenticated.u2handlers(self) # create custom ssl context, e.g. ctx = ssl.create_default_context(cafile=CERTIFICATE_LOCATION) # configure context as needed... ctx.check_hostname = False # add a https handler using the custom context handlers.append(HTTPSHandler(context=ctx)) return handlers
def __init__(self, config): self.opener = build_opener(HTTPSHandler()) self._appid = str(config['app_id']) self._app_notify_url = config['notify_url'] self._mch_id = config['mch_id'] self._mch_key = config['mch_key'] self._appsecret = config['app_secret'] # 证书pem格式 self._api_cert_path = config.setdefault('api_cert_path', '') # 证书密钥pem格式 self._api_key_path = config.setdefault('api_key_path', '')
def handle(): request = Request(url, data=data) request.get_method = lambda: method for k, v in headers: request.add_header(k, v) resp = build_opener( HTTPSHandler(context=tlscontext)).open( request, timeout=timeout) if resp.code >= 400: raise IOError("error talking to pushgateway: {0} {1}".format( resp.code, resp.msg))
def _add_basic_auth(self): auth_handler = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm()) auth_handler.add_password( realm=None, uri=self.host, user=self.user, passwd=self.password, ) install_opener( build_opener( auth_handler, HTTPHandler(debuglevel=self.request_debug_level), HTTPSHandler(debuglevel=self.request_debug_level), ))
def sendRequest(text, lang='en-ru'): ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE proxy = ProxyHandler({'https': '199.201.121.139:3128'}) try: opener = build_opener(proxy) opener.add_handler(HTTPSHandler(context=ctx)) req = Request(url_request.format(text=text.encode('utf-8'), lang=lang)) req.get_method = lambda: 'POST' response = json.loads(opener.open(req).read().decode('utf-8')) except: response = '' return response
def __init__(self, url, sessionId=None, sslContext=None, proxy=None): """Create an IDSClient. """ self.url = url if not self.url.endswith("/"): self.url += "/" self.sessionId = sessionId if sslContext: verify = (sslContext.verify_mode != ssl.CERT_NONE) try: httpsHandler = HTTPSHandler(context=sslContext, check_hostname=verify) chunkedHTTPSHandler = ChunkedHTTPSHandler(context=sslContext, check_hostname=verify) except TypeError: # Python 2.7.9 HTTPSHandler does not accept the # check_hostname keyword argument. httpsHandler = HTTPSHandler(context=sslContext) chunkedHTTPSHandler = ChunkedHTTPSHandler(context=sslContext) else: httpsHandler = HTTPSHandler() chunkedHTTPSHandler = ChunkedHTTPSHandler() if proxy: proxyhandler = ProxyHandler(proxy) self.default = build_opener(proxyhandler, httpsHandler, IDSHTTPErrorHandler) self.chunked = build_opener(proxyhandler, ChunkedHTTPHandler, chunkedHTTPSHandler, IDSHTTPErrorHandler) else: self.default = build_opener(httpsHandler, IDSHTTPErrorHandler) self.chunked = build_opener(ChunkedHTTPHandler, chunkedHTTPSHandler, IDSHTTPErrorHandler) apiversion = self.getApiVersion() # Translate a version having a trailing '-SNAPSHOT' into # something that StrictVersion would accept. apiversion = re.sub(r'-SNAPSHOT$', 'a1', apiversion) self.apiversion = Version(apiversion)
def __init__(self, username, password, host, debug=0): logger.debug('Creating new connection with username=%s host=%s', username, host) self._host = host base64string = base64.encodestring( ('%s:%s' % (username, password)).encode('utf-8')).replace( b'\n', b'') self._headers['Authorization'] = b'Basic ' + base64string self._opener = build_opener(HTTPHandler(debuglevel=debug), HTTPSHandler(debuglevel=debug), HTTPCookieProcessor(CookieJar()), LoggingHandler(), HTTPDefaultErrorHandler())
def urlopen_with_ssl(request, timeout): result = None if (sys.version_info[0] == 3 and sys.version_info >= (3, 5)) or \ (sys.version_info[0] == 2 and sys.version_info >= (2, 7)): result = urlopen(request, context=get_ssl_no_verify_context(), timeout=timeout) else: if sys.version_info[0] == 2: from urllib2 import HTTPSHandler, build_opener, install_opener # pylint: disable=import-error else: from urllib.request import HTTPSHandler, build_opener, install_opener # pylint: disable=import-error,no-name-in-module install_opener(build_opener(HTTPSHandler())) result = urlopen(request, timeout=timeout) return result
def _setup_http(self): self._api_uri = '{http}://{host}:{port}{uri}'.format( http=self._proto, host=self._hostname, port=self._port, uri=DEFAULT_API_URI) auth_mgr = HTTPPasswordMgrWithDefaultRealm() auth_mgr.add_password(None, self._api_uri, self._user, self._password) auth_hndlr = HTTPBasicAuthHandler(auth_mgr) http_hndlr = HTTPHandler(debuglevel=0) https_hndlr = HTTPSHandler(debuglevel=0) self._http_api = build_opener(auth_hndlr, https_hndlr, http_hndlr) self._auth_base64 = base64.encodestring( '%s:%s' % (self._user, self._password))[:-1]
def _setup_opener(self): if HAVE_SSLCONTEXT and self._server_cert_validation == 'ignore': sslcontext = create_default_context() sslcontext.check_hostname = False sslcontext.verify_mode = CERT_NONE else: sslcontext = None https_transport_handler = HTTPSHandler( context=sslcontext) if sslcontext else None if not self._cert_pem: super(HttpSSL, self)._setup_opener(root_handler=https_transport_handler) else: handler = HTTPSClientAuthHandler(self._cert_pem, self._cert_key_pem, sslcontext) self.opener = build_opener(handler)
def __init__ (self, args) : self.modulus = None self.exponent = None self.args = args self.jar = j = LWPCookieJar () self.has_cookies = False context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE if self.args.cookiefile : self.has_cookies = True try : j.load (self.args.cookiefile, ignore_discard = True) except IOError : self.has_cookies = False handlers = [HTTPCookieProcessor (j)] if self.args.ssl_noverify: handlers.append(HTTPSHandler(context=context)) self.opener = build_opener (*handlers) self.nextfile = args.file
def get(self, uri, params={}, headers={}, with_status_code=False, timeout=10, user=None, password=None): data = None # always none in GET if params: uri = "%s?%s" % (uri, urlencode(params)) # SSL, user/password and basic # NOTE: currently don't manage ssl & user/password if uri.startswith('https://'): handler = HTTPSHandler(context=self.ssl_context) elif user and password: passwordMgr = HTTPPasswordMgrWithDefaultRealm() passwordMgr.add_password(None, uri, user, password) handler = HTTPBasicAuthHandler(passwordMgr) else: handler = HTTPHandler url_opener = build_opener(handler) req = Request(uri, data) req.get_method = lambda: 'GET' for (k, v) in headers.items(): req.add_header(k, v) request = url_opener.open(req, timeout=timeout) response = request.read() status_code = request.code request.close() if not with_status_code: return response else: return (status_code, response)
def query(self, query, ts_start, ts_end): # target = 'summarize({},"{}","avg")'.format( # query, '99year') @TODO remove if not needed # build graphite url args = { '__auth_token': self.token, 'target': query, 'format': 'json', 'from': ts_start, 'until': ts_end, } url = '{}/render?'.format(self.url) for k, v in args.iteritems(): print k print v url += '{}={}&'.format(quote(k), quote(v)) logger.debug('Query URL is {}'.format(url)) # Basic auth header password_mgr = HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password( None, self.graphite_url, self.username, self.password, ) auth_handler = HTTPBasicAuthHandler(password_mgr) # Ignore ssl cert check ctx = create_default_context() ctx.check_hostname = False ctx.verify_mode = CERT_NONE ssl_handler = HTTPSHandler(context=ctx) opener = build_opener(ssl_handler, auth_handler) install_opener(opener) result = json.loads(urlopen(url).read()) return result
def https_requests(server, insecure): from urllib2 import HTTPSHandler, HTTPCookieProcessor, Request, build_opener import cookielib import ssl context = ssl.create_default_context() if insecure: context.check_hostname = False context.verify_mode = ssl.CERT_NONE cookies = cookielib.LWPCookieJar() opener = build_opener(HTTPSHandler(context=context), HTTPCookieProcessor(cookies)) # Get CSRF req = Request(server) req.get_method = lambda: "HEAD" response = opener.open(req) csrf = response.info().getheader("X-CSRF-Token") return HTTPSRequests(opener, cookies, server, csrf)
def __init__(self, ca_certs=None): HTTPSHandler.__init__(self) self._ca_certs = ca_certs
def __init__(self, cert, key, sslcontext=None): HTTPSHandler.__init__(self) self.cert = cert self.key = key self._context = sslcontext
def __init__(self, cert, key): HTTPSHandler.__init__(self) self.key = key self.cert = cert