def _get_response(request): # type: (Any) -> Any
logger.debug("_get_response()")
if logger.isEnabledFor(logging.DEBUG):
opener = build_opener(HTTPSHandler(debuglevel=1))
else:
opener = build_opener(HTTPSHandler())
try:
response = opener.open(request)
except HTTPError as error_response:
response = error_response
content_encoding = response.headers.get("content-encoding", "")
content_type = response.headers.get("content-type")
encoding = content_type.split("charset=")[-1]
r = response.read()
if content_encoding == "gzip":
s = gzip.decompress(r)
else:
s = r.decode(encoding)
if content_type.startswith("application/json"):
return json.loads(s)
return s
def mock_urlopen(cls, url, data=None,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
cafile=None, capath=None, cadefault=False, context=None):
# Check for passed-through headers
if not isinstance(url, urllib2.Request) or \
cls._headers.items()[0] not in url.header_items():
raise urllib2.URLError('Headers not passed through')
global _opener
if cafile or capath or cadefault:
if context is not None:
raise ValueError(
"You can't pass both context and any of cafile, capath, "
"and cadefault"
)
if not _have_ssl:
raise ValueError('SSL support not available')
context = ssl.create_default_context(
purpose=ssl.Purpose.SERVER_AUTH,
cafile=cafile,
capath=capath)
https_handler = HTTPSHandler(context=context)
opener = build_opener(https_handler)
elif context:
https_handler = HTTPSHandler(context=context)
opener = build_opener(https_handler)
elif _opener is None:
_opener = opener = build_opener()
else:
opener = _opener
return opener.open(url, data, timeout)
def __init__(self, timeout=None, debuglevel=0):
"""Initialize the object.
Args:
timeout: the socket connect timeout value.
debuglevel: the debuglevel level.
"""
_HS.__init__(self, debuglevel)
TimeoutHTTPSConnection._timeout = timeout
def __init__(self, *args, **kwargs):
try:
kwargs['context'] = ssl._create_unverified_context()
except AttributeError:
# Python prior to 2.7.9 doesn't have default-enabled certificate
# verification
pass
HTTPSHandler.__init__(self, *args, **kwargs)
def __init__(self, timeout=None, debuglevel=0):
"""Initialize the object.
Args:
timeout: the socket connect timeout value.
debuglevel: the debuglevel level.
"""
_HS.__init__(self, debuglevel)
TimeoutHTTPSConnection._timeout = timeout
def __init__(self, key, cert):
ctx = ssl._create_unverified_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
HTTPSHandler.__init__(self, context=ctx)
self.context = ctx
self.key = key
self.cert = cert
def u2handlers(self):
handlers = []
handlers.append(ProxyHandler(self.proxy))
handlers.append(HTTPBasicAuthHandler(self.pm))
# python ssl Context support - PEP 0466
if hasattr(ssl, '_create_unverified_context'):
ssl_context = ssl._create_unverified_context()
handlers.append(HTTPSHandler(context=ssl_context))
else:
handlers.append(HTTPSHandler())
return handlers
def __init__(self):
"""Build an HTTPS opener."""
# Based on pip 1.4.1's URLOpener
# This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set.
if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'):
self._opener = build_opener(
HTTPSHandler(context=create_CERT_NONE_context()))
else:
self._opener = build_opener(HTTPSHandler())
# Strip out HTTPHandler to prevent MITM spoof:
for handler in self._opener.handlers:
if isinstance(handler, HTTPHandler):
self._opener.handlers.remove(handler)
def build_opener(self, debug=False):
"""Create handlers with the appropriate debug level.
We intentionally create new ones because the OpenerDirector
class in urllib2 is smart enough to replace its internal
versions with ours if we pass them into the
urllib2.build_opener method. This is much easier than
trying to introspect into the OpenerDirector to find the
existing handlers.
Based on http://code.activestate.com/recipes/440574/#c1
TODO: Implement workaround for http://bugs.python.org/issue7152
"""
http_handler = HTTPHandler(debuglevel=debug)
https_handler = HTTPSHandler(debuglevel=debug)
proxy_handler = ProxyHandler(debuglevel=debug)
unknown_handler = UnknownHandler(debuglevel=debug)
http_default_error_handler = HTTPDefaultErrorHandler(debuglevel=debug)
http_redirect_handler = HTTPRedirectHandler(debuglevel=debug)
http_error_processor = HTTPErrorProcessor(debuglevel=debug)
handlers = [http_handler, https_handler, proxy_handler, \
unknown_handler, http_default_error_handler, \
http_redirect_handler, http_error_processor]
opener = build_opener(handlers)
return opener
def opener():
opener = build_opener(HTTPSHandler())
# Strip out HTTPHandler to prevent MITM spoof:
for handler in opener.handlers:
if isinstance(handler, HTTPHandler):
opener.handlers.remove(handler)
return opener
def get_cert(cls, url):
context = ssl.create_default_context()
context.load_cert_chain(cls.ssl_file('client-certificate.pem'),
cls.ssl_file('client-private-key.pem'),
'client-password')
context.load_verify_locations(cls.ssl_file('ca-certificate.pem'))
opener = build_opener(HTTPSHandler(context=context))
return opener.open(url).read().decode('utf-8')
def get_new_cookie(self):
# Start by prompting user to input their credentials
# Another Python2/3 workaround
try:
new_username = raw_input("Username: "******"Username: "******"Password (will not be displayed): ")
# Build URS4 Cookie request
auth_cookie_url = self.asf_urs4['url'] + '?client_id=' + self.asf_urs4['client'] + '&redirect_uri=' + \
self.asf_urs4['redir'] + '&response_type=code&state='
try:
# python2
user_pass = base64.b64encode(bytes(new_username + ":" + new_password))
except TypeError:
# python3
user_pass = base64.b64encode(bytes(new_username + ":" + new_password, "utf-8"))
user_pass = user_pass.decode("utf-8")
# Authenticate against URS, grab all the cookies
self.cookie_jar = MozillaCookieJar()
opener = build_opener(HTTPCookieProcessor(self.cookie_jar), HTTPHandler(), HTTPSHandler(**self.context))
request = Request(auth_cookie_url, headers={"Authorization": "Basic {0}".format(user_pass)})
# Watch out cookie rejection!
try:
response = opener.open(request)
except HTTPError as e:
if e.code == 401:
print(" > Username and Password combo was not successful. Please try again.")
return False
else:
# If an error happens here, the user most likely has not confirmed EULA.
print("\nIMPORTANT: There was an error obtaining a download cookie!")
print("Your user appears to lack permission to download data from the ASF Datapool.")
print(
"\n\nNew users: you must first log into Vertex and accept the EULA. In addition, your Study Area must be set at Earthdata https://urs.earthdata.nasa.gov")
exit(-1)
except URLError as e:
print("\nIMPORTANT: There was a problem communicating with URS, unable to obtain cookie. ")
print("Try cookie generation later.")
exit(-1)
# Did we get a cookie?
if self.check_cookie_is_logged_in(self.cookie_jar):
# COOKIE SUCCESS!
self.cookie_jar.save(self.cookie_jar_path)
return True
# if we aren't successful generating the cookie, nothing will work. Stop here!
print("WARNING: Could not generate new cookie! Cannot proceed. Please try Username and Password again.")
print("Response was {0}.".format(response.getcode()))
print(
"\n\nNew users: you must first log into Vertex and accept the EULA. In addition, your Study Area must be set at Earthdata https://urs.earthdata.nasa.gov")
exit(-1)
def u2handlers(self):
"""Get a collection of urllib handlers.
"""
handlers = suds.transport.http.HttpTransport.u2handlers(self)
if self.ssl_context:
try:
handlers.append(HTTPSHandler(context=self.ssl_context,
check_hostname=self.verify))
except TypeError:
# Python 2.7.9 HTTPSHandler does not accept the
# check_hostname keyword argument.
#
# Note that even older Python versions would also
# croak on the context keyword argument. But these
# old versions do not have SSLContext either, so we
# will not end up here in the first place.
handlers.append(HTTPSHandler(context=self.ssl_context))
return handlers
def __init__(self):
"""Build an HTTPS opener."""
# Based on pip 1.4.1's URLOpener
# This verifies certs on only Python >=2.7.9.
self._opener = build_opener(HTTPSHandler())
# Strip out HTTPHandler to prevent MITM spoof:
for handler in self._opener.handlers:
if isinstance(handler, HTTPHandler):
self._opener.handlers.remove(handler)
def u2handlers(self):
handlers = HttpAuthenticated.u2handlers(self)
context = ssl._create_unverified_context()
# add an HTTPS handler, using the custom context
handlers.append(HTTPSHandler(context=context))
return handlers
def __init__(self, host, auth_header, debug=0):
self._host = host
self._headers['Authorization'] = auth_header
self._opener = build_opener(HTTPHandler(debuglevel=debug),
HTTPSHandler(debuglevel=debug),
HTTPCookieProcessor(CookieJar()),
LoggingHandler(),
HTTPDefaultErrorHandler())
def check_cookie(self):
if self.cookie_jar is None:
print(" > Cookiejar is bunk: {0}".format(self.cookie_jar))
return False
# File we know is valid, used to validate cookie
file_check = 'https://urs.earthdata.nasa.gov/profile'
# Apply custom Redirect Hanlder
opener = build_opener(HTTPCookieProcessor(
self.cookie_jar), HTTPHandler(), HTTPSHandler(**self.context))
install_opener(opener)
# Attempt a HEAD request
request = Request(file_check)
request.get_method = lambda: 'HEAD'
try:
print(" > attempting to download {0}".format(file_check))
response = urlopen(request, timeout=30)
resp_code = response.getcode()
# Make sure we're logged in
if not self.check_cookie_is_logged_in(self.cookie_jar):
return False
# Save cookiejar
self.cookie_jar.save(self.cookie_jar_path)
except HTTPError:
# If we ge this error, again, it likely means the user has not agreed to current EULA
print("\nIMPORTANT: ")
print(
"Your user appears to lack permissions to download data from the ASF Datapool.")
print("\n\nNew users: you must first log into Vertex and accept the EULA. In addition, your Study Area must be set at Earthdata https://urs.earthdata.nasa.gov")
exit(-1)
# This return codes indicate the USER has not been approved to download the data
if resp_code in (300, 301, 302, 303):
try:
redir_url = response.info().getheader('Location')
except AttributeError:
redir_url = response.getheader('Location')
# Funky Test env:
if ("vertex-retired.daac.asf.alaska.edu" in redir_url and "test" in self.asf_urs4['redir']):
print("Cough, cough. It's dusty in this test env!")
return True
print(
"Redirect ({0}) occured, invalid cookie value!".format(resp_code))
return False
# These are successes!
if resp_code in (200, 307):
return True
return False
def u2handlers(self):
# use handlers from superclass
handlers = HttpAuthenticated.u2handlers(self)
# create custom ssl context, e.g.
ctx = ssl.create_default_context(cafile=CERTIFICATE_LOCATION)
# configure context as needed...
ctx.check_hostname = False
# add a https handler using the custom context
handlers.append(HTTPSHandler(context=ctx))
return handlers
def __init__(self, config):
self.opener = build_opener(HTTPSHandler())
self._appid = str(config['app_id'])
self._app_notify_url = config['notify_url']
self._mch_id = config['mch_id']
self._mch_key = config['mch_key']
self._appsecret = config['app_secret']
# 证书pem格式
self._api_cert_path = config.setdefault('api_cert_path', '')
# 证书密钥pem格式
self._api_key_path = config.setdefault('api_key_path', '')
def handle():
request = Request(url, data=data)
request.get_method = lambda: method
for k, v in headers:
request.add_header(k, v)
resp = build_opener(
HTTPSHandler(context=tlscontext)).open(
request, timeout=timeout)
if resp.code >= 400:
raise IOError("error talking to pushgateway: {0} {1}".format(
resp.code, resp.msg))
def _add_basic_auth(self):
auth_handler = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm())
auth_handler.add_password(
realm=None,
uri=self.host,
user=self.user,
passwd=self.password,
)
install_opener(
build_opener(
auth_handler,
HTTPHandler(debuglevel=self.request_debug_level),
HTTPSHandler(debuglevel=self.request_debug_level),
))
def sendRequest(text, lang='en-ru'):
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
proxy = ProxyHandler({'https': '199.201.121.139:3128'})
try:
opener = build_opener(proxy)
opener.add_handler(HTTPSHandler(context=ctx))
req = Request(url_request.format(text=text.encode('utf-8'), lang=lang))
req.get_method = lambda: 'POST'
response = json.loads(opener.open(req).read().decode('utf-8'))
except:
response = ''
return response
def __init__(self, url, sessionId=None, sslContext=None, proxy=None):
"""Create an IDSClient.
"""
self.url = url
if not self.url.endswith("/"): self.url += "/"
self.sessionId = sessionId
if sslContext:
verify = (sslContext.verify_mode != ssl.CERT_NONE)
try:
httpsHandler = HTTPSHandler(context=sslContext,
check_hostname=verify)
chunkedHTTPSHandler = ChunkedHTTPSHandler(context=sslContext,
check_hostname=verify)
except TypeError:
# Python 2.7.9 HTTPSHandler does not accept the
# check_hostname keyword argument.
httpsHandler = HTTPSHandler(context=sslContext)
chunkedHTTPSHandler = ChunkedHTTPSHandler(context=sslContext)
else:
httpsHandler = HTTPSHandler()
chunkedHTTPSHandler = ChunkedHTTPSHandler()
if proxy:
proxyhandler = ProxyHandler(proxy)
self.default = build_opener(proxyhandler, httpsHandler,
IDSHTTPErrorHandler)
self.chunked = build_opener(proxyhandler,
ChunkedHTTPHandler, chunkedHTTPSHandler,
IDSHTTPErrorHandler)
else:
self.default = build_opener(httpsHandler, IDSHTTPErrorHandler)
self.chunked = build_opener(ChunkedHTTPHandler, chunkedHTTPSHandler,
IDSHTTPErrorHandler)
apiversion = self.getApiVersion()
# Translate a version having a trailing '-SNAPSHOT' into
# something that StrictVersion would accept.
apiversion = re.sub(r'-SNAPSHOT$', 'a1', apiversion)
self.apiversion = Version(apiversion)
def __init__(self, username, password, host, debug=0):
logger.debug('Creating new connection with username=%s host=%s',
username, host)
self._host = host
base64string = base64.encodestring(
('%s:%s' % (username, password)).encode('utf-8')).replace(
b'\n', b'')
self._headers['Authorization'] = b'Basic ' + base64string
self._opener = build_opener(HTTPHandler(debuglevel=debug),
HTTPSHandler(debuglevel=debug),
HTTPCookieProcessor(CookieJar()),
LoggingHandler(),
HTTPDefaultErrorHandler())
def urlopen_with_ssl(request, timeout):
result = None
if (sys.version_info[0] == 3 and sys.version_info >= (3, 5)) or \
(sys.version_info[0] == 2 and sys.version_info >= (2, 7)):
result = urlopen(request,
context=get_ssl_no_verify_context(),
timeout=timeout)
else:
if sys.version_info[0] == 2:
from urllib2 import HTTPSHandler, build_opener, install_opener # pylint: disable=import-error
else:
from urllib.request import HTTPSHandler, build_opener, install_opener # pylint: disable=import-error,no-name-in-module
install_opener(build_opener(HTTPSHandler()))
result = urlopen(request, timeout=timeout)
return result
def _setup_http(self):
self._api_uri = '{http}://{host}:{port}{uri}'.format(
http=self._proto,
host=self._hostname,
port=self._port,
uri=DEFAULT_API_URI)
auth_mgr = HTTPPasswordMgrWithDefaultRealm()
auth_mgr.add_password(None, self._api_uri, self._user, self._password)
auth_hndlr = HTTPBasicAuthHandler(auth_mgr)
http_hndlr = HTTPHandler(debuglevel=0)
https_hndlr = HTTPSHandler(debuglevel=0)
self._http_api = build_opener(auth_hndlr, https_hndlr, http_hndlr)
self._auth_base64 = base64.encodestring(
'%s:%s' % (self._user, self._password))[:-1]
def _setup_opener(self):
if HAVE_SSLCONTEXT and self._server_cert_validation == 'ignore':
sslcontext = create_default_context()
sslcontext.check_hostname = False
sslcontext.verify_mode = CERT_NONE
else:
sslcontext = None
https_transport_handler = HTTPSHandler(
context=sslcontext) if sslcontext else None
if not self._cert_pem:
super(HttpSSL,
self)._setup_opener(root_handler=https_transport_handler)
else:
handler = HTTPSClientAuthHandler(self._cert_pem,
self._cert_key_pem, sslcontext)
self.opener = build_opener(handler)
def __init__ (self, args) :
self.modulus = None
self.exponent = None
self.args = args
self.jar = j = LWPCookieJar ()
self.has_cookies = False
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
if self.args.cookiefile :
self.has_cookies = True
try :
j.load (self.args.cookiefile, ignore_discard = True)
except IOError :
self.has_cookies = False
handlers = [HTTPCookieProcessor (j)]
if self.args.ssl_noverify:
handlers.append(HTTPSHandler(context=context))
self.opener = build_opener (*handlers)
self.nextfile = args.file
def get(self,
uri,
params={},
headers={},
with_status_code=False,
timeout=10,
user=None,
password=None):
data = None # always none in GET
if params:
uri = "%s?%s" % (uri, urlencode(params))
# SSL, user/password and basic
# NOTE: currently don't manage ssl & user/password
if uri.startswith('https://'):
handler = HTTPSHandler(context=self.ssl_context)
elif user and password:
passwordMgr = HTTPPasswordMgrWithDefaultRealm()
passwordMgr.add_password(None, uri, user, password)
handler = HTTPBasicAuthHandler(passwordMgr)
else:
handler = HTTPHandler
url_opener = build_opener(handler)
req = Request(uri, data)
req.get_method = lambda: 'GET'
for (k, v) in headers.items():
req.add_header(k, v)
request = url_opener.open(req, timeout=timeout)
response = request.read()
status_code = request.code
request.close()
if not with_status_code:
return response
else:
return (status_code, response)
def query(self, query, ts_start, ts_end):
# target = 'summarize({},"{}","avg")'.format(
# query, '99year') @TODO remove if not needed
# build graphite url
args = {
'__auth_token': self.token,
'target': query,
'format': 'json',
'from': ts_start,
'until': ts_end,
}
url = '{}/render?'.format(self.url)
for k, v in args.iteritems():
print k
print v
url += '{}={}&'.format(quote(k), quote(v))
logger.debug('Query URL is {}'.format(url))
# Basic auth header
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(
None,
self.graphite_url,
self.username,
self.password,
)
auth_handler = HTTPBasicAuthHandler(password_mgr)
# Ignore ssl cert check
ctx = create_default_context()
ctx.check_hostname = False
ctx.verify_mode = CERT_NONE
ssl_handler = HTTPSHandler(context=ctx)
opener = build_opener(ssl_handler, auth_handler)
install_opener(opener)
result = json.loads(urlopen(url).read())
return result
def https_requests(server, insecure):
from urllib2 import HTTPSHandler, HTTPCookieProcessor, Request, build_opener
import cookielib
import ssl
context = ssl.create_default_context()
if insecure:
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
cookies = cookielib.LWPCookieJar()
opener = build_opener(HTTPSHandler(context=context),
HTTPCookieProcessor(cookies))
# Get CSRF
req = Request(server)
req.get_method = lambda: "HEAD"
response = opener.open(req)
csrf = response.info().getheader("X-CSRF-Token")
return HTTPSRequests(opener, cookies, server, csrf)
def __init__(self, ca_certs=None):
HTTPSHandler.__init__(self)
self._ca_certs = ca_certs
def __init__(self, cert, key, sslcontext=None):
HTTPSHandler.__init__(self)
self.cert = cert
self.key = key
self._context = sslcontext
def __init__(self, cert, key):
HTTPSHandler.__init__(self)
self.key = key
self.cert = cert
