def _create_opener():
"""
Creates an opener for the internet.
It also attaches the :class:`CachingRedirectHandler` to the opener and
sets its User-agent to ``Mozilla/5.0``.
If the Network Proxy settings are set and recognized, it creates the
opener and attaches the proxy_handler to it. The opener is tested and
returned if the test passes.
If the test fails an opener without the proxy settings is created instead
and is returned instead.
"""
use_proxy = False
proxy_handler = None
if NETWORK_PROXY_TYPE == 'http':
use_proxy = True
proxyurl = _get_http_proxy_url()
proxy_handler = ProxyHandler({'http': proxyurl,
'https': proxyurl})
if use_proxy:
openr = build_opener(HTTPHandler(), HTTPSHandler(),
proxy_handler, CachingRedirectHandler)
else:
openr = build_opener(HTTPSHandler(), HTTPSHandler(),
CachingRedirectHandler)
openr.addheaders = [('User-agent', 'Mozilla/5.0')]
global _internet_connected
_internet_connected = _test_opener(openr)
return openr
def putDataUrllib2(url,payload,timeout=900,logger=None):
"""
Wrapper method for urllib2 that supports PUTs to a url.
Parameters
----------
url : `string`
Ex: 'https://dqsegdb5.phy.syr.edu/L1/DMT-SCIENCE/1'
payload : `string`
JSON formatted string
"""
socket.setdefaulttimeout(timeout)
#BEFORE HTTPS: opener = urllib2.build_opener(urllib2.HTTPHandler)
if urlparse(url).scheme == 'https':
opener=urllib_request.build_opener(HTTPSClientAuthHandler)
else:
opener = urllib_request.build_opener(urllib_request.HTTPHandler)
request = urllib_request.Request(url, data=payload)
request.add_header('Content-Type', 'JSON')
request.get_method = lambda: 'PUT'
if logger:
logger.debug("Beginning url call: %s" % url)
try:
urlreturned = opener.open(request)
except urllib_error.HTTPError as e:
handleHTTPError("PUT",url,e)
##print(e.read())
#if int(e.code)==404:
# print("Flag does not exist in database yet for url: %s" % url)
#else:
# print("Warning: Issue accessing url: %s" % url)
# print("Code: ")
# print(e.code)
# print("Message: ")
# print(e.msg)
# #print(e.reason)
# #print(url)
# print("May be handled cleanly by calling instance: otherwise will result in an error.")
##print(e.reason)
##print(urlreturned)
raise
except urllib_error.URLError as e:
#print(e.read())
warnmsg="Warning: Issue accessing url: %s" % url
warnmsg+="; "
warnmsg+=str(e.reason)
warnmsg+="; "
warnmsg+="May be handled cleanly by calling instance: otherwise will result in an error."
warn(warnmsg)
raise
if logger:
logger.debug("Completed url call: %s" % url)
return url
def open_no_proxy(*args, **kwargs):
# NOTE(jamespage):
# Deal with more secure certification chain verficiation
# introduced in python 2.7.9 under PEP-0476
# https://github.com/python/peps/blob/master/pep-0476.txt
if hasattr(ssl, "_create_unverified_context"):
opener = urlrequest.build_opener(
urlrequest.ProxyHandler({}), urlrequest.HTTPSHandler(context=ssl._create_unverified_context())
)
else:
opener = urlrequest.build_opener(urlrequest.ProxyHandler({}))
return opener.open(*args, **kwargs)
def _get_opener(self):
if not self.opener:
if (CONF.dashboard.disable_ssl_certificate_validation and
self._ssl_default_context_supported()):
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
self.opener = request.build_opener(
request.HTTPSHandler(context=ctx),
request.HTTPCookieProcessor())
else:
self.opener = request.build_opener(
request.HTTPCookieProcessor())
return self.opener
def url_get(base_url, password_mgr=None, pathspec=None, params=None):
"""Make contact with the uri provided and return any contents."""
# Uses system proxy settings if they exist.
proxy = urlrequest.ProxyHandler()
if password_mgr is not None:
auth = urlrequest.HTTPDigestAuthHandler(password_mgr)
urlopener = urlrequest.build_opener(proxy, auth)
else:
urlopener = urlrequest.build_opener(proxy)
urlrequest.install_opener(urlopener)
full_url = build_url(base_url, pathspec=pathspec, params=params)
response = urlopener.open(full_url)
content = response.read()
response.close()
return content
def set_proxy(proxy, user=None, password=''):
"""
Set the HTTP proxy for Python to download through.
If ``proxy`` is None then tries to set proxy from environment or system
settings.
:param proxy: The HTTP proxy server to use. For example:
'http://proxy.example.com:3128/'
:param user: The username to authenticate with. Use None to disable
authentication.
:param password: The password to authenticate with.
"""
from nltk import compat
if proxy is None:
# Try and find the system proxy settings
try:
proxy = getproxies()['http']
except KeyError:
raise ValueError('Could not detect default proxy settings')
# Set up the proxy handler
proxy_handler = ProxyHandler({'https': proxy, 'http': proxy})
opener = build_opener(proxy_handler)
if user is not None:
# Set up basic proxy authentication if provided
password_manager = HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(realm=None, uri=proxy, user=user, passwd=password)
opener.add_handler(ProxyBasicAuthHandler(password_manager))
opener.add_handler(ProxyDigestAuthHandler(password_manager))
# Overide the existing url opener
install_opener(opener)
def __init__(self, url, username=None, password=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
super().__init__(url, username, password, timeout)
# Setting ssl key verificationas false
context = ssl._create_stdlib_context(check_hostname=False)
unverified_handler = HTTPSHandler(context=context, check_hostname=False)
install_opener(build_opener(unverified_handler))
def _read_from_url(url):
"""Reads data from *url* with an HTTP *GET*.
This function supports fetching from resources which use basic HTTP auth as
laid out by RFC1738 § 3.1. See § 5 for grammar definitions for URLs.
.. seealso:
https://www.ietf.org/rfc/rfc1738.txt
:param url: URL of an HTTP resource
:type url: ``str``
:return: data read from resource described by *url*
:rtype: ``file``-like object
"""
url, username, password = _strip_basic_auth(url)
if username is not None and password is not None:
# case: url contains basic auth creds
password_mgr = request.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, url, username, password)
handler = request.HTTPBasicAuthHandler(password_mgr)
opener = request.build_opener(default_handlers + [handler])
else:
opener = default_opener
return opener.open(url)
def __init__(self, settings, address, handler_class, backend_port, username):
# type: (CtlSettings, Tuple[str, int], Type[BaseHTTPRequestHandler], int, str) -> None
self.backend_port = backend_port
self.user_auth_header = settings.user_auth_header
self.username = username
self.opener = build_opener(NoRedirectHandler)
super(ProxyServer, self).__init__(address, handler_class)
def open(self, proxy=None, cache=True):
if self.local:
self._last_update_time = os.stat(self.url).st_mtime
return open(self.url)
elif cache and os.path.exists(self.cache):
self._last_update_time = os.stat(self.cache).st_mtime
return open(self.cache)
else:
r = Request(self.url)
if proxy or self.proxy:
proxy = proxy if proxy else self.proxy
opener = build_opener(SocksiPyHandler(
socks.PROXY_TYPES[proxy['type'].upper()],
proxy['ip'],
proxy['port'],
))
data_io = opener.open(r)
else:
data_io = urlopen(r)
data = data_io.read()
if self.is_base64(data):
logger.debug('BASE64 decode...')
data = base64.b64decode(data)
if not isinstance(data, str):
data = data.decode('utf-8')
if self.cache:
with open(self.cache, 'w') as f:
f.write(data)
self._last_update_time = os.stat(self.cache).st_mtime
return open(self.cache)
self._last_update_time = time.time()
return StringIO(data)
def user_login(self, username, password):
self.opener = request.build_opener(request.HTTPCookieProcessor())
response = self.opener.open(CONF.dashboard.dashboard_url).read()
# Grab the CSRF token and default region
parser = HorizonHTMLParser()
parser.feed(response)
# construct login url for dashboard, discovery accommodates non-/ web
# root for dashboard
login_url = parse.urljoin(CONF.dashboard.dashboard_url, parser.login)
# Prepare login form request
req = request.Request(login_url)
req.add_header("Content-type", "application/x-www-form-urlencoded")
req.add_header("Referer", CONF.dashboard.dashboard_url)
# Pass the default domain name regardless of the auth version in order
# to test the scenario of when horizon is running with keystone v3
params = {
"username": username,
"password": password,
"region": parser.region,
"domain": CONF.auth.default_credentials_domain_name,
"csrfmiddlewaretoken": parser.csrf_token,
}
self.opener.open(req, parse.urlencode(params))
def srtm_login_or_skip(monkeypatch):
import os
try:
srtm_username = os.environ['SRTM_USERNAME']
except KeyError:
pytest.skip('SRTM_USERNAME environment variable is unset.')
try:
srtm_password = os.environ['SRTM_PASSWORD']
except KeyError:
pytest.skip('SRTM_PASSWORD environment variable is unset.')
from six.moves.urllib.request import (HTTPBasicAuthHandler,
HTTPCookieProcessor,
HTTPPasswordMgrWithDefaultRealm,
build_opener)
from six.moves.http_cookiejar import CookieJar
password_manager = HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(
None,
"https://urs.earthdata.nasa.gov",
srtm_username,
srtm_password)
cookie_jar = CookieJar()
opener = build_opener(HTTPBasicAuthHandler(password_manager),
HTTPCookieProcessor(cookie_jar))
monkeypatch.setattr(cartopy.io, 'urlopen', opener.open)
def _set_cookies(self, src):
'''
function that returns a urllib2 opener for retrieving data from *src*
input:
*src* : 'asos' or 'wunderground' or 'wunder_nonairport'
'''
jar = http_cookiejar.CookieJar()
handler = request.HTTPCookieProcessor(jar)
opener = request.build_opener(handler)
try:
if src.lower() == 'wunderground':
url1 = 'http://www.wunderground.com/history/airport/%s/2011/12/4/DailyHistory.html?' % self.sta_id
url2 = 'http://www.wunderground.com/cgi-bin/findweather/getForecast?setpref=SHOWMETAR&value=1'
url3 = 'http://www.wunderground.com/history/airport/%s/2011/12/4/DailyHistory.html?&&theprefset=SHOWMETAR&theprefvalue=1&format=1' % self.sta_id
opener.open(url1)
opener.open(url2)
opener.open(url3)
elif src.lower() == 'asos':
url = 'ftp://ftp.ncdc.noaa.gov/pub/data/asos-fivemin/'
opener.open(url)
elif src.lower() == 'wunder_nonairport':
url = 'http://www.wunderground.com/weatherstation/WXDailyHistory.asp?ID=MEGKO3&day=1&year=2013&month=1&graphspan=day&format=1'
opener.open(url)
except error.URLError:
print(('connection to %s not available. working locally' % src))
return opener
def func(args):
if args.outputdocument:
fdout = open(args.outputdocument, 'w')
else:
fdout = sys.stdout
if args.useragent:
useragent = args.useragent
else:
useragent = 'Cmdix/' + cmdix.__version__
opener = build_opener()
opener.addheaders = [('User-agent', useragent)]
for url in args.url:
try:
fdin = opener.open(url)
except HTTPError as e:
exception.StdErrException(
"HTTP error opening {0}: {1}".format(url, e))
length = int(fdin.headers['content-length'])
print("Getting {0} bytes from {1}...".format(length, url))
shutil.copyfileobj(fdin, fdout)
print("Done")
def __init__(self, url, keystone_url, credentials, **kwargs):
logger.info('Initiate HTTPClient with url %s', url)
self.url = url
self.keystone_url = keystone_url
self.creds = dict(credentials, **kwargs)
self.keystone = None
self.opener = request.build_opener(request.HTTPHandler)
def pytest_unconfigure(config):
"""
Called after all tests are completed.
"""
global enabled, submit_url, commit, branch, environment, username, password
if not enabled:
return
executable = "{}-{}-{}".format(platform.python_implementation(),
platform.python_version(),
platform.system())
data = [x.to_codespeed_dict(commit=commit,
branch=branch,
environment=environment,
executable=executable,
project=project)
for x in function_profile_list]
try:
json_submit_url = submit_url + 'result/add/json/'
if username:
password_mgr = urllib_request.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, json_submit_url, username, password)
handler = urllib_request.HTTPBasicAuthHandler(password_mgr)
opener = urllib_request.build_opener(handler)
else:
opener = urllib_request.build_opener()
# use the opener to fetch a URL
f = opener.open(json_submit_url, urlencode({'json': json.dumps(data)}).encode('UTF-8'))
response = f.read()
except HTTPError as e:
print('Error while connecting to Codespeed:')
print('Exception: {}'.format(str(e)))
fd, name = tempfile.mkstemp(suffix='.html')
os.close(fd)
with open(name, 'wb') as f:
f.write(e.read())
print('HTTP Response written to {}'.format(name))
raise e
if not response.startswith('All result data saved successfully'.encode('UTF-8')):
print("Unexpected response while connecting to Codespeed:")
raise ValueError('Unexpected response from Codespeed server: {}'.format(response))
else:
print("{} test benchmarks sumbitted.".format(len(function_profile_list)))
def reload(self, store): """ Force a reload of this response """ opener = request.build_opener() cr = self.from_response(opener.open(self.url)) self.__dict__.update(vars(cr)) store.set(self.url, self.save())
def __insert_binary_file(self, url):
"""
Insert a binary file. First download the file and then insert.
"""
opener = build_opener()
page = opener.open(url)
image = binascii.b2a_hex(page.read())
return image.decode()
def _make_opener(self, realm, base_url, username, password):
"""uTorrent API need HTTP Basic Auth and cookie support for token verify."""
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm=realm,
uri=base_url,
user=username,
passwd=password)
opener = build_opener(auth_handler)
install_opener(opener)
cookie_jar = CookieJar()
cookie_handler = HTTPCookieProcessor(cookie_jar)
handlers = [auth_handler, cookie_handler]
opener = build_opener(*handlers)
return opener
def __init__(self, url):
warn(
'HTTPClientZabbix is deprecated and not used now. '
'It will be dropped in short term period.',
DeprecationWarning
)
self.url = url
self.opener = request.build_opener(request.HTTPHandler)
def get_cookie_opener( gs_username, gs_token, gs_toolname=None ):
""" Create a GenomeSpace cookie opener """
cj = http_cookiejar.CookieJar()
for cookie_name, cookie_value in [ ( 'gs-token', gs_token ), ( 'gs-username', gs_username ) ]:
# create a super-cookie, valid for all domains
cookie = http_cookiejar.Cookie(version=0, name=cookie_name, value=cookie_value, port=None, port_specified=False, domain='', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False )
cj.set_cookie( cookie )
cookie_opener = build_opener( HTTPCookieProcessor( cj ) )
cookie_opener.addheaders.append( ( 'gs-toolname', gs_toolname or DEFAULT_GENOMESPACE_TOOLNAME ) )
return cookie_opener
def __init__(self, server_url, opener=None):
self.logger = logging.getLogger(__name__ + '.Client')
self.server_url = server_url
if opener is None:
opener = build_opener(create_urllib_https_handler())
elif not isinstance(opener, OpenerDirector):
raise TypeError('opener must be {0.__module__}.{0.__name__}, not '
'{1!r}'.format(OpenerDirector, opener))
self.opener = opener
self.public_keys = PublicKeyDict(self)
def debug():
"""
Activate debugging on urllib2.
"""
if six.PY2:
handler = HTTPSHandler(debuglevel=1)
opener = build_opener(handler)
install_opener(opener)
else:
http_client.HTTPConnection.debuglevel = 1
def authfetch(username,password,path="/",realm=realm):
server.accept(2)
import socket
socket.setdefaulttimeout(5)
uri = ("http://%s:%s" % server.server_address) + path
auth = HTTPDigestAuthHandler()
auth.add_password(realm,uri,username,password)
opener = build_opener(auth)
result = opener.open(uri)
return result.read()
def request():
endpoint = config.endpoint
if "://" not in endpoint:
endpoint = config.get_endpoint()
req = Request(endpoint, payload.encode("utf-8", "replace"), {"Content-Type": "application/json"})
if config.proxy_host:
proxies = ProxyHandler({"https": config.proxy_host, "http": config.proxy_host})
opener = build_opener(proxies)
else:
opener = build_opener()
resp = opener.open(req)
status = resp.getcode()
if status != 200:
bugsnag.logger.warning("Notification to %s failed, status %d" % (config.endpoint, status))
def build_request_with_data(url, data, api_key, method):
"""Build a request with the received method."""
http_redirect_with_data_handler = HTTPRedirectWithDataHandler(method=method)
opener = build_opener(http_redirect_with_data_handler)
install_opener(opener)
url = make_url(url, api_key=api_key, args=None)
request = Request(url, headers={'Content-Type': 'application/json'}, data=json.dumps(data))
request_method = request.get_method()
if request_method != method:
request.get_method = lambda: method
return opener, request
def clear_all_queries(self, cluster_name=DEFAULT_CLUSTER):
"""
Clear all the primed queries from a particular cluster
:param cluster_name: cluster to clear queries from
"""
opener = build_opener(HTTPHandler)
request = Request("http://{0}/{1}/{2}".format(
self.admin_addr, "prime", cluster_name))
request.get_method = lambda: 'DELETE'
connection = opener.open(request)
return connection.read()
def submit_request(self, query):
opener = build_opener(HTTPHandler)
data = json.dumps(query.fetch_json()).encode('utf8')
request = Request("http://{}/{}{}".format(
self.admin_addr, query.path, query.fetch_url_params()), data=data)
request.get_method = lambda: 'POST'
request.add_header("Content-Type", 'application/json')
request.add_header("Content-Length", len(data))
connection = opener.open(request)
return connection.read().decode('utf-8')
def __init__(self, configuration, debug=True):
super(XMLAPIConnector, self).__init__()
self.storage_ip = configuration.emc_nas_server
self.user_name = configuration.emc_nas_login
self.pass_word = configuration.emc_nas_password
self.debug = debug
self.auth_url = "https://" + self.storage_ip + "/Login"
self._url = "https://" + self.storage_ip + "/servlets/CelerraManagementServices"
https_handler = url_request.HTTPSHandler()
cookie_jar = http_cookiejar.CookieJar()
cookie_handler = url_request.HTTPCookieProcessor(cookie_jar)
self.url_opener = url_request.build_opener(https_handler, cookie_handler)
self._do_setup()
def open_with_basic_auth(url, auth):
"""
opens an url protected with basic http authentication
:param url: string - the url to open
:param auth:
:return:
"""
user, passwd = auth
p = six.moves.urllib.request.HTTPPasswordMgrWithDefaultRealm()
p.add_password(None, url, user, passwd)
auth_handler = six.moves.urllib.request.HTTPBasicAuthHandler(p)
opener = request.build_opener(auth_handler)
request.install_opener(opener)
return opener.open(url)
def main():
opts, args = parse_opts()
exitcode = 0
if not inside_project():
_log("Error: no Scrapy project found in this location")
sys.exit(1)
install_opener(build_opener(HTTPRedirectHandler))
if opts.list_targets:
for name, target in _get_targets().items():
print("%-20s %s" % (name, target['url']))
return
if opts.list_projects:
target = _get_target(opts.list_projects)
req = Request(_url(target, 'listprojects.json'))
_add_auth_header(req, target)
f = urlopen(req)
projects = json.loads(f.read())['projects']
print(os.linesep.join(projects))
return
tmpdir = None
if opts.build_egg: # build egg only
egg, tmpdir = _build_egg()
_log("Writing egg to %s" % opts.build_egg)
shutil.copyfile(egg, opts.build_egg)
elif opts.deploy_all_targets:
version = None
for name, target in _get_targets().items():
if version is None:
version = _get_version(target, opts)
_build_egg_and_deploy_target(target, version, opts)
else: # buld egg and deploy
target_name = _get_target_name(args)
target = _get_target(target_name)
version = _get_version(target, opts)
exitcode, tmpdir = _build_egg_and_deploy_target(target, version, opts)
if tmpdir:
if opts.debug:
_log("Output dir not removed: %s" % tmpdir)
else:
shutil.rmtree(tmpdir)
sys.exit(exitcode)
def DetectGce():
"""Determine whether or not we're running on GCE.
This is based on:
https://cloud.google.com/compute/docs/metadata#runninggce
Returns:
True iff we're running on a GCE instance.
"""
try:
o = urllib_request.build_opener(urllib_request.ProxyHandler({})).open(
urllib_request.Request('http://metadata.google.internal'))
except urllib_error.URLError:
return False
return (o.getcode() == http_client.OK
and o.headers.get('metadata-flavor') == 'Google')
def opener(self):
"""
Create an opener object.
By calling :meth:`add_authentication` before calling this property for
the first time, authentication credentials can be set.
EXAMPLES::
sage: from sage.dev.digest_transport import DigestTransport
sage: DigestTransport().opener
<urllib2.OpenerDirector instance at 0x...>
"""
if self._opener is None:
self._opener = build_opener(HTTPDigestAuthHandler())
return self._opener
def setup_urllib_proxies():
global _urllib_proxies_installed, SYSTEM_PROXIES
if _urllib_proxies_installed:
return
_urllib_proxies_installed = True
if not SYSTEM_PROXIES:
return
proxies = dict(
(k, "%s://%s:%s" % (k, SYSTEM_PROXIES[k][0], SYSTEM_PROXIES[k][1])) for k in SYSTEM_PROXIES
)
from six.moves.urllib.request import ProxyHandler, build_opener, install_opener
proxy_handler = ProxyHandler(proxies)
opener = build_opener(proxy_handler)
install_opener(opener)
def run(self, args, opts):
try:
import setuptools
except ImportError:
raise UsageError("setuptools not installed")
request.install_opener(request.build_opener(HTTPRedirectHandler))
if opts.list_targets:
for name, target in _get_targets().items():
print("%-20s %s" % (name, target['url']))
return
if opts.list_projects:
target = _get_target(opts.list_projects)
req = request.Request(_url(target, 'listprojects.json'))
_add_auth_header(req, target)
f = request.urlopen(req)
projects = json.loads(f.read())['projects']
print(os.linesep.join(projects))
return
tmpdir = None
if opts.build_egg: # build egg only
egg, tmpdir = _build_egg()
_log("Writing egg to %s" % opts.build_egg)
shutil.copyfile(egg, opts.build_egg)
else: # buld egg and deploy
target_name = _get_target_name(args)
target = _get_target(target_name)
project = _get_project(target, opts)
version = _get_version(target, opts)
if opts.egg:
_log("Using egg: %s" % opts.egg)
egg = opts.egg
else:
_log("Packing version %s" % version)
egg, tmpdir = _build_egg()
if not _upload_egg(target, egg, project, version):
self.exitcode = 1
if tmpdir:
if opts.debug:
_log("Output dir not removed: %s" % tmpdir)
else:
shutil.rmtree(tmpdir)
def _follow_redirects(uri_list, http_timeout):
""" Follow HTTP redirects from servers. Needed so that we can create
RewriteRules for all repository URLs that pkg clients may encounter.
We return a sorted list of URIs that were found having followed all
redirects in 'uri_list'. We also return a boolean, True if we timed out
when following any of the URIs.
"""
ret_uris = set(uri_list)
timed_out = False
class SysrepoRedirectHandler(HTTPRedirectHandler):
""" A HTTPRedirectHandler that saves URIs we've been
redirected to along the path to our eventual destination."""
def __init__(self):
self.redirects = set()
def redirect_request(self, req, fp, code, msg, hdrs, newurl):
self.redirects.add(newurl)
return HTTPRedirectHandler.redirect_request(
self, req, fp, code, msg, hdrs, newurl)
for uri in uri_list:
handler = SysrepoRedirectHandler()
opener = build_opener(handler)
if not uri.startswith("http:"):
ret_uris.update([uri])
continue
# otherwise, open a known url to check for redirects
try:
opener.open("{0}/versions/0".format(uri), None, http_timeout)
ret_uris.update(
set([
item.replace("/versions/0", "").rstrip("/")
for item in handler.redirects
]))
except URLError as err:
# We need to log this, and carry on - the url
# could become available at a later date.
msg(
_("WARNING: unable to access {uri} when checking "
"for redirects: {err}").format(**locals()))
timed_out = True
return sorted(list(ret_uris)), timed_out
def try_del(httpd, querystr):
"""Try DEL calls to the server."""
num_requests["del_handler"] = 0
opener = build_opener(HTTPHandler)
request = Request(httpd_url(httpd, "/api/resource/1", querystr))
request.get_method = lambda: "DEL"
f = opener.open(request)
assert f.getcode() == 200
assert json.loads(f.read()) == {
"called": 1,
"id": "1",
"query": querystr
}
assert num_requests["del_handler"] == 1
def __init__(self):
self.server = server
self.handler = handler
self.waited = -1
self.tool = 'echo_' + handler
self._hda_id = None
self._hda_state = None
self._history_id = None
if not self.server.startswith('http'):
self.server = 'http://' + self.server
self.cookie_jar = os.path.join(var_dir, "cookie_jar")
dprint("cookie jar path: %s" % self.cookie_jar)
if not os.access(self.cookie_jar, os.R_OK):
dprint("no cookie jar at above path, creating")
tc.save_cookies(self.cookie_jar)
tc.load_cookies(self.cookie_jar)
self.opener = build_opener(HTTPCookieProcessor(tc.get_browser().cj))
def upload(self, filename):
if not self.cdash_upload_url:
return
# Compute md5 checksum for the contents of this file.
md5sum = checksum(hashlib.md5, filename, block_size=8192)
opener = build_opener(HTTPHandler)
with open(filename, 'rb') as f:
url = "{0}&MD5={1}".format(self.cdash_upload_url, md5sum)
request = Request(url, data=f)
request.add_header('Content-Type', 'text/xml')
request.add_header('Content-Length', os.path.getsize(filename))
# By default, urllib2 only support GET and POST.
# CDash needs expects this file to be uploaded via PUT.
request.get_method = lambda: 'PUT'
url = opener.open(request)
def add_authentication(self, realm, url, username, password):
"""
Set authentication credentials for the opener returned by
:meth:`opener`.
EXAMPLES::
sage: from sage.dev.digest_transport import DigestTransport
sage: dt = DigestTransport()
sage: dt.add_authentication("realm", "url", "username", "password")
sage: dt.opener
<urllib2.OpenerDirector instance at 0x...>
"""
assert self._opener is None
authhandler = HTTPDigestAuthHandler()
authhandler.add_password(realm, url, username, password)
self._opener = build_opener(authhandler)
def populate_buildgroup(job_names, group_name, project, site, credentials,
cdash_url):
url = "{0}/api/v1/buildgroup.php".format(cdash_url)
headers = {
'Authorization': 'Bearer {0}'.format(credentials),
'Content-Type': 'application/json',
}
opener = build_opener(HTTPHandler)
parent_group_id = _create_buildgroup(opener, headers, url, project,
group_name, 'Daily')
group_id = _create_buildgroup(opener, headers, url, project,
'Latest {0}'.format(group_name), 'Latest')
if not parent_group_id or not group_id:
msg = 'Failed to create or retrieve buildgroups for {0}'.format(
group_name)
raise SpackError(msg)
data = {
'project':
project,
'buildgroupid':
group_id,
'dynamiclist': [{
'match': name,
'parentgroupid': parent_group_id,
'site': site
} for name in job_names]
}
enc_data = json.dumps(data).encode('utf-8')
request = Request(url, data=enc_data, headers=headers)
request.get_method = lambda: 'PUT'
response = opener.open(request)
response_code = response.getcode()
if response_code != 200:
msg = 'Error response code ({0}) in populate_buildgroup'.format(
response_code)
raise SpackError(msg)
def http_call(method, url, data=None):
"""Utility method for making HTTP requests."""
LOG.debug("http_call(): Calling %s %s" % (method, url))
opener = build_opener(HTTPHandler)
if data:
data = simplejson.dumps(data)
LOG.debug("http_call(): With body: %s" % data)
request = Request(url, data)
request.add_header('Accept', 'application/json')
if data:
request.add_header('Content-Type', 'application/json')
request.get_method = lambda: method
resp = opener.open(request)
if resp.getcode() >= 400:
raise exceptions.RomanaException("Error in %s %s with payload %s: %s", method, url, data, resp)
body = resp.read()
data = simplejson.loads(body)
return data
def read(self, size=-1):
if self.fileobj:
self.fileobj.close()
opener = build_opener(HTTPRangeHandler)
install_opener(opener)
if size < 0:
rangeheader = {'Range': 'bytes=%s-' % (self.pos)}
else:
rangeheader = {'Range': 'bytes=%s-%s' % (self.pos, self.pos + size - 1)}
req = Request(self.url, headers=rangeheader)
res = urlopen(req)
self.pos += size
data = res.read()
return data
def http_request(self, url, method, data="", headers=None, timeout=None):
if url[0:7].lower() != "http://":
url = "http://%s" % url
if hasattr(self, 'logger') and self.logger is not None:
self.logger.debug("Sending http request. Url: %s, Data: %s, Headers: %s" % (url, str(data), str(headers)))
req = Request(url, data, headers)
req.get_method = lambda: method
# The timeout parameter in urllib2.urlopen has strange behavior, and
# seems to raise errors when set to a number. Using an opener works however.
opener = build_opener()
if timeout is None:
response = opener.open(req)
else:
response = opener.open(req, timeout=timeout)
return response
def do_call(self, url, data=None, method=None,
calltimeout=constants.SOCKET_TIMEOUT):
"""Send requests to server.
Send HTTPS call, get response in JSON.
Convert response into Python Object and return it.
"""
if self.url:
url = self.url + url
if "xx/sessions" not in url:
LOG.debug('Request URL: %(url)s\n'
'Call Method: %(method)s\n'
'Request Data: %(data)s\n',
{'url': url,
'method': method,
'data': data})
opener = urlreq.build_opener(urlreq.HTTPCookieProcessor(self.cookie))
urlreq.install_opener(opener)
result = None
try:
req = urlreq.Request(url, data, self.headers)
if method:
req.get_method = lambda: method
res_temp = urlreq.urlopen(req, timeout=calltimeout)
res = res_temp.read().decode("utf-8")
LOG.debug('Response Data: %(res)s.', {'res': res})
except Exception as err:
LOG.error(_LE('\nBad response from server: %(url)s.'
' Error: %(err)s'), {'url': url, 'err': err})
res = '{"error":{"code":%s,' \
'"description":"Connect server error"}}' \
% constants.ERROR_CONNECT_TO_SERVER
try:
result = jsonutils.loads(res)
except Exception as err:
err_msg = (_('JSON transfer error: %s.') % err)
LOG.error(err_msg)
raise exception.InvalidInput(reason=err_msg)
return result
def __init__(self, configuration, debug=True):
super(XMLAPIConnector, self).__init__()
self.storage_ip = configuration.emc_nas_server
self.username = configuration.emc_nas_login
self.password = configuration.emc_nas_password
self.debug = debug
self.auth_url = 'https://' + self.storage_ip + '/Login'
self._url = 'https://{}/servlets/CelerraManagementServices'.format(
self.storage_ip)
context = enas_utils.create_ssl_context(configuration)
if context:
https_handler = url_request.HTTPSHandler(context=context)
else:
https_handler = url_request.HTTPSHandler()
cookie_handler = url_request.HTTPCookieProcessor(
http_cookiejar.CookieJar())
self.url_opener = url_request.build_opener(https_handler,
cookie_handler)
self._do_setup()
def prepare(self):
"""
Read options for uploading, check that they're sane
"""
super(BlazeMeterUploader, self).prepare()
self.client.address = self.settings.get("address", self.client.address)
self.client.data_address = self.settings.get("data-address", self.client.data_address)
self.client.timeout = dehumanize_time(self.settings.get("timeout", self.client.timeout))
self.send_interval = dehumanize_time(self.settings.get("send-interval", self.send_interval))
self.browser_open = self.settings.get("browser-open", self.browser_open)
token = self.settings.get("token", "")
proxy_settings = self.engine.config.get("settings").get("proxy")
if proxy_settings:
if proxy_settings.get("address"):
proxy_url = urlsplit(proxy_settings.get("address"))
username = proxy_settings.get("username")
pwd = proxy_settings.get("password")
if username and pwd:
proxy_uri = "%s://%s:%s@%s" % (proxy_url.scheme, username, pwd, proxy_url.netloc)
else:
proxy_uri = "%s://%s" % (proxy_url.scheme, proxy_url.netloc)
proxy_handler = ProxyHandler({"https": proxy_uri, "http": proxy_uri})
opener = build_opener(proxy_handler)
install_opener(opener)
if not token:
self.log.warning("No BlazeMeter API key provided, will upload anonymously")
self.client.token = token
self.client.active_session_id = self.parameters.get("session-id", None)
self.client.test_id = self.parameters.get("test-id", None)
self.client.user_id = self.parameters.get("user-id", None)
self.client.data_signature = self.parameters.get("signature", None)
if not self.client.test_id:
test_name = self.parameters.get("test", "Taurus Test") # TODO: provide a way to put datetime into test name
try:
self.client.ping() # to check connectivity and auth
if token:
self.test_id = self.client.test_by_name(test_name, {"type": "external"})
except HTTPError:
self.log.error("Cannot reach online results storage, maybe the address/token is wrong")
raise
def fixture_httpd_with_proxy_handler(docroot):
"""Yields a started MozHttpd server for the proxy test."""
httpd = mozhttpd.MozHttpd(port=0, docroot=str(docroot))
httpd.start(block=False)
port = httpd.httpd.server_port
proxy_support = ProxyHandler({
"http":
"http://127.0.0.1:{port:d}".format(port=port),
})
install_opener(build_opener(proxy_support))
yield httpd
httpd.stop()
# Reset proxy opener in case it changed
install_opener(None)
def user_login(self, username, password):
self.opener = request.build_opener(request.HTTPCookieProcessor())
response = self.opener.open(CONF.dashboard.dashboard_url).read()
# Grab the CSRF token and default region
parser = HorizonHTMLParser()
parser.feed(response)
# Prepare login form request
req = request.Request(CONF.dashboard.login_url)
req.add_header('Content-type', 'application/x-www-form-urlencoded')
req.add_header('Referer', CONF.dashboard.dashboard_url)
params = {
'username': username,
'password': password,
'region': parser.region,
'csrfmiddlewaretoken': parser.csrf_token
}
self.opener.open(req, parse.urlencode(params))
def try_del(self, server_port, querystr):
self.resource_del_called = 0
opener = build_opener(HTTPHandler)
request = Request(
self.get_url('/api/resource/1', server_port, querystr))
request.get_method = lambda: 'DEL'
f = opener.open(request)
try:
self.assertEqual(f.getcode(), 200)
except AttributeError:
pass # python 2.4
self.assertEqual(json.loads(f.read()), {
'called': 1,
'id': str(1),
'query': querystr
})
self.assertEqual(self.resource_del_called, 1)
def relate_cdash_builds(spec_map, cdash_base_url, job_build_id, cdash_project,
cdashids_mirror_url):
if not job_build_id:
return
dep_map = spec_map['deps']
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json',
}
cdash_api_url = '{0}/api/v1/relateBuilds.php'.format(cdash_base_url)
for dep_pkg_name in dep_map:
tty.debug('Fetching cdashid file for {0}'.format(dep_pkg_name))
dep_spec = dep_map[dep_pkg_name]
dep_build_id = read_cdashid_from_mirror(dep_spec, cdashids_mirror_url)
payload = {
"project": cdash_project,
"buildid": job_build_id,
"relatedid": dep_build_id,
"relationship": "depends on"
}
enc_data = json.dumps(payload).encode('utf-8')
opener = build_opener(HTTPHandler)
request = Request(cdash_api_url, data=enc_data, headers=headers)
response = opener.open(request)
response_code = response.getcode()
if response_code != 200 and response_code != 201:
msg = 'Relate builds ({0} -> {1}) failed (resp code = {2})'.format(
job_build_id, dep_build_id, response_code)
raise SpackError(msg)
response_text = response.read()
tty.debug('Relate builds response: {0}'.format(response_text))
def wait_for_spark_workers(num_of_expected_workers, timeout):
"""
This queries the spark master and checks for the expected number of workers
"""
start_time = time.time()
while True:
opener = build_opener(HTTPHandler)
request = Request("http://{0}:7080".format(CASSANDRA_IP))
request.get_method = lambda: 'GET'
connection = opener.open(request)
match = re.search('Alive Workers:.*(\d+)</li>', connection.read().decode('utf-8'))
num_workers = int(match.group(1))
if num_workers == num_of_expected_workers:
match = True
break
elif time.time() - start_time > timeout:
match = True
break
time.sleep(1)
return match
def get_file(url):
opener = build_opener()
opener.addheaders = [('User-agent', 'pgxnclient/%s' % __version__)]
logger.debug('opening url: %s', url)
try:
return closing(opener.open(url))
except HTTPError as e:
if e.code == 404:
raise ResourceNotFound(_("resource not found: '%s'") % e.url)
elif e.code == 400:
raise BadRequestError(_("bad request on '%s'") % e.url)
elif e.code == 500:
raise NetworkError(_("server error"))
elif e.code == 503:
raise NetworkError(_("service unavailable"))
else:
raise NetworkError(
_("unexpected response %d for '%s'") % (e.code, e.url))
except URLError as e:
raise NetworkError(_("network error: %s") % e.reason)
def __init__(self, base_url, endpoint, username, password):
self.base_url = base_url.strip()
# remove left '/' if any
self.endpoint = endpoint.strip().lstrip('/')
self.username = username.strip()
self.password = password.strip()
# set other things
self.services_link = "%s/%s" % (base_url, endpoint)
self.services_session_token = None
self.http_user_agent = 'DrupalComputingAgent'
self.http_content_type = 'application/json'
# set cookie handler
# first, create an opener than has cookie support.
opener = urllib_request.build_opener(
urllib_request.HTTPCookieProcessor())
# then install the opener to request instead of using the default BaseHandler.
urllib_request.install_opener(opener)
def make_image_object_from_url(image_url):
# parse url
parsed_url = urlparse(image_url)
# handle absolute and relative urls, Assuming http for now.
if not parsed_url.scheme:
image_url = '%s%s' % (get_setting('site', 'global', 'siteurl'), image_url)
request = Request(image_url)
request.add_header('User-Agent', settings.TENDENCI_USER_AGENT)
opener = build_opener()
# make image object
try:
socket.setdefaulttimeout(1.5)
data = opener.open(request).read() # get data
im = Image.open(BytesIO(data))
except:
im = None
return im
def test_nonexistent_resources(httpd_no_urlhandlers):
# GET: Return 404 for non-existent endpoint
with pytest.raises(HTTPError) as excinfo:
urlopen(httpd_url(httpd_no_urlhandlers, "/api/resource/"))
assert excinfo.value.code == 404
# POST: POST should also return 404
with pytest.raises(HTTPError) as excinfo:
urlopen(httpd_url(httpd_no_urlhandlers, "/api/resource/"),
data=json.dumps({}))
assert excinfo.value.code == 404
# DEL: DEL should also return 404
opener = build_opener(HTTPHandler)
request = Request(httpd_url(httpd_no_urlhandlers, "/api/resource/"))
request.get_method = lambda: "DEL"
with pytest.raises(HTTPError) as excinfo:
opener.open(request)
assert excinfo.value.code == 404
def http_emitter(message, log, url):
"""Send payload
"""
log.debug('http_emitter: attempting postback to ' + url)
# Post back the data
partial_payload = []
for measurement in message:
partial_payload.append(measurement)
payload = json.dumps(partial_payload)
if PY3:
payload = payload.encode('utf-8')
url = "%s/intake" % url
headers = post_headers(payload)
try:
# Make sure no proxy is autodetected for this localhost connection
proxy_handler = ProxyHandler({})
# Should this be installed as the default opener and reused?
opener = build_opener(proxy_handler)
request = Request(url, payload, headers)
response = None
try:
response = opener.open(request)
log.debug('http_emitter: postback response: ' +
str(response.read()))
except Exception as exc:
log.error("""Forwarder at {0} is down or not responding...
Error is {1}
Please restart the monasca-agent.""".format(
url, repr(exc)))
finally:
if response:
response.close()
except HTTPError as e:
if e.code == 202:
log.debug("http payload accepted")
else:
raise
def check_nltk():
try:
from nltk.tokenize import word_tokenize
word_tokenize('It\'s.')
except Exception:
import nltk
if not sslVerify:
from ssl import _create_unverified_context
from six.moves.urllib.request import install_opener, HTTPSHandler, build_opener
# TODO: This needs still proxy support !
ctx = _create_unverified_context()
opener = build_opener(HTTPSHandler(context=ctx))
install_opener(opener)
if 'HTTP_PROXY' in os.environ:
nltk.set_proxy(os.environ.get('HTTP_PROXY'))
nltk.download('punkt')
return
def test_nonexistent_resources(self):
# Create a server with a placeholder handler so we don't fall back
# to serving local files
httpd = mozhttpd.MozHttpd(port=0)
httpd.start(block=False)
server_port = httpd.httpd.server_port
# GET: Return 404 for non-existent endpoint
exception_thrown = False
try:
urlopen(self.get_url('/api/resource/', server_port, None))
except HTTPError as e:
self.assertEqual(e.code, 404)
exception_thrown = True
self.assertTrue(exception_thrown)
# POST: POST should also return 404
exception_thrown = False
try:
urlopen(
self.get_url('/api/resource/', server_port, None),
data=json.dumps({}),
)
except HTTPError as e:
self.assertEqual(e.code, 404)
exception_thrown = True
self.assertTrue(exception_thrown)
# DEL: DEL should also return 404
exception_thrown = False
try:
opener = build_opener(HTTPHandler)
request = Request(self.get_url('/api/resource/', server_port,
None))
request.get_method = lambda: 'DEL'
opener.open(request)
except HTTPError:
self.assertEqual(e.code, 404)
exception_thrown = True
self.assertTrue(exception_thrown)
def main():
import tempfile, sys
validatorURL = "http://validator.w3.org/check"
opener = build_opener(MultipartPostHandler)
def validateFile(url):
temp = tempfile.mkstemp(suffix=".html")
write(temp[0], opener.open(url).read())
params = {
"ss": "0", # show source
"doctype": "Inline",
"uploaded_file": open(temp[1], "rb")
}
print(opener.open(validatorURL, params).read())
remove(temp[1])
if len(sys.argv[1:]) > 0:
for arg in sys.argv[1:]:
validateFile(arg)
else:
validateFile("http://www.google.com")
