def test_generic_urlparse():
url = 'https://ansible.com/blog'
parts = urlparse(url)
generic_parts = generic_urlparse(parts)
assert generic_parts.as_list() == list(parts)
assert urlunparse(generic_parts.as_list()) == url
def test_generic_urlparse():
url = 'https://ansible.com/blog'
parts = urlparse(url)
generic_parts = generic_urlparse(parts)
assert generic_parts.as_list() == list(parts)
assert urlunparse(generic_parts.as_list()) == url
def update_url_query(url, **kwargs):
parse_result = urlparse(url)
query = parse_qs(parse_result.query)
query.update({k: v for k, v in kwargs.items() if v is not None})
query_string = urlencode(query, doseq=True)
parse_result = parse_result._replace(query=query_string)
return urlunparse(parse_result)
def check(module, name, state, service_id, integration_key, api_key, incident_key=None, http_call=fetch_url):
url = 'https://api.pagerduty.com/incidents'
headers = {
"Content-type": "application/json",
"Authorization": "Token token=%s" % api_key,
'Accept': 'application/vnd.pagerduty+json;version=2'
}
params = {
'service_ids[]': service_id,
'sort_by': 'incident_number:desc',
'time_zone': 'UTC'
}
if incident_key:
params['incident_key'] = incident_key
url_parts = list(urlparse(url))
url_parts[4] = urlencode(params, True)
url = urlunparse(url_parts)
response, info = http_call(module, url, method='get', headers=headers)
if info['status'] != 200:
module.fail_json(msg="failed to check current incident status."
"Reason: %s" % info['msg'])
json_out = json.loads(response.read())["incidents"][0]
if state != json_out["status"]:
return json_out, True
return json_out, False
def do_notify_bearychat(module, url, payload):
response, info = fetch_url(module, url, data=payload)
if info['status'] != 200:
url_info = urlparse(url)
obscured_incoming_webhook = urlunparse(
(url_info.scheme, url_info.netloc, '[obscured]', '', '', ''))
module.fail_json(msg=" failed to send %s to %s: %s" %
(payload, obscured_incoming_webhook, info['msg']))
def test_generic_urlparse_netloc():
url = 'https://ansible.com:443/blog'
parts = urlparse(url)
generic_parts = generic_urlparse(parts)
assert generic_parts.hostname == parts.hostname
assert generic_parts.hostname == 'ansible.com'
assert generic_parts.port == 443
assert urlunparse(generic_parts.as_list()) == url
def test_generic_urlparse_netloc():
url = 'https://ansible.com:443/blog'
parts = urlparse(url)
generic_parts = generic_urlparse(parts)
assert generic_parts.hostname == parts.hostname
assert generic_parts.hostname == 'ansible.com'
assert generic_parts.port == 443
assert urlunparse(generic_parts.as_list()) == url
def do_notify_bearychat(module, url, payload):
response, info = fetch_url(module, url, data=payload)
if info['status'] != 200:
url_info = urlparse(url)
obscured_incoming_webhook = urlunparse(
(url_info.scheme, url_info.netloc, '[obscured]', '', '', ''))
module.fail_json(
msg=" failed to send %s to %s: %s" % (
payload, obscured_incoming_webhook, info['msg']))
def test_generic_urlparse_no_netloc():
url = 'https://*****:*****@ansible.com:443/blog'
parts = list(urlparse(url))
generic_parts = generic_urlparse(parts)
assert generic_parts.hostname == 'ansible.com'
assert generic_parts.port == 443
assert generic_parts.username == 'user'
assert generic_parts.password == 'passwd'
assert urlunparse(generic_parts.as_list()) == url
def test_generic_urlparse_no_netloc():
url = 'https://*****:*****@ansible.com:443/blog'
parts = list(urlparse(url))
generic_parts = generic_urlparse(parts)
assert generic_parts.hostname == 'ansible.com'
assert generic_parts.port == 443
assert generic_parts.username == 'user'
assert generic_parts.password == 'passwd'
assert urlunparse(generic_parts.as_list()) == url
def update_qsl(url, params):
''' Add or update a URL query string '''
if HAS_URLPARSE:
url_parts = list(urlparse(url))
query = dict(parse_qsl(url_parts[4]))
query.update(params)
url_parts[4] = urlencode(query)
return urlunparse(url_parts)
elif '?' in url:
return url + '&' + '&'.join(['%s=%s' % (k, v) for k, v in params.items()])
else:
return url + '?' + '&'.join(['%s=%s' % (k, v) for k, v in params.items()])
def update_qsl(url, params):
''' Add or update a URL query string '''
if HAS_URLPARSE:
url_parts = list(urlparse(url))
query = dict(parse_qsl(url_parts[4]))
query.update(params)
url_parts[4] = urlencode(query)
return urlunparse(url_parts)
elif '?' in url:
return url + '&' + '&'.join(['%s=%s' % (k, v) for k, v in params.items()])
else:
return url + '?' + '&'.join(['%s=%s' % (k, v) for k, v in params.items()])
def install_package(module, pkg, pkg_file, upgrade):
check_build_environment(module)
git_source = module.params['git_source']
gpg_key = module.params['gpg_key']
if not upgrade and query_package(module, pkg):
module.exit_json(changed=False, msg="package already installed", package=pkg)
if gpg_key:
if should_add_gpg_key(module, gpg_key):
add_gpg_key(module, gpg_key)
if git_source:
pkg_path = make_package(module, pkg, None)
elif not pkg_file:
# this is an aur pkg
rpc_params = urllib.urlencode({"type": "info", "arg": pkg})
rpc_req = urlunparse((AUR_SCHEME, AUR_NETLOC, AUR_RPC_PATH, "", rpc_params, ""))
rsp, info = fetch_url(module, rpc_req)
# create a temporary file and copy content to do checksum-based replacement
if info['status'] != 200:
module.fail_json(msg="Request failed", status_code=info['status'], response=info['msg'], url=rpc_req)
pkg_info = json.load(rsp)
if pkg_info['resultcount'] < 1:
module.fail_json(msg="AUR query found no matching packages", package=pkg)
# check pkg cache before install
pkg_ver = pkg_info['results']['Version']
pkg_path = query_package_dir(module, pkg, pkg_ver)
if not pkg_path:
pkg_path = make_package(module, pkg, None)
else:
pkg_path = make_package(module, pkg, pkg_file)
if module.params['as_deps']:
params = '-U --asdeps %s' % pkg_path
else:
params = '-U %s' % pkg_path
cmd = "pacman %s --noconfirm" % (params)
rc, stdout, stderr = module.run_command(cmd, check_rc=False)
if rc != 0:
module.fail_json(msg="failed to install package", pkg=pkg, stderr=stderr)
module.exit_json(changed=True, msg="installed package", pkg=pkg)
def update_qsl(url, params):
"""Add or update a URL query string"""
if HAS_URLPARSE:
url_parts = list(urlparse(url))
query = dict(parse_qsl(url_parts[4]))
query.update(params)
url_parts[4] = urlencode(query)
return urlunparse(url_parts)
elif "?" in url:
return url + "&" + "&".join(
["%s=%s" % (k, v) for k, v in params.items()])
else:
return url + "?" + "&".join(
["%s=%s" % (k, v) for k, v in params.items()])
def prepare_aur_dir(module, build_dir, pkg):
pkg_file_basename = "%s.tar.gz" % pkg
pkg_file_dest = os.path.join(build_dir, pkg_file_basename)
aur_req = urlunparse((AUR_SCHEME, AUR_NETLOC, "%s/%s" % (AUR_SNAPSHOT_PATH, pkg_file_basename), "", "", ""))
rsp, info = fetch_url(module, aur_req)
if info['status'] != 200:
module.fail_json(msg="Request failed", url=aur_req, status_code=info['status'], response=info['msg'])
# save response to archive
f = open(pkg_file_dest, 'wb')
try:
shutil.copyfileobj(rsp, f)
except (Exception, err):
os.remove(pkg_file_dest)
module.fail_json(msg="failed to create package archive file: %s" % str(err))
f.close()
rsp.close()
extract_pkg(module, pkg, pkg_file_dest, build_dir)
def format_url(use_ssl, hostname, port, path=''):
""" Format url based on ssl flag, hostname, port and path
Args:
use_ssl (bool): is ssl enabled
hostname (str): hostname
port (str): port
path (str): url path
Returns:
str: The generated url string
"""
scheme = 'https' if use_ssl else 'http'
netloc = hostname
if (use_ssl and port != '443') or (not use_ssl and port != '80'):
netloc += ":%s" % port
try:
url = urlparse.urlunparse((scheme, netloc, path, '', '', ''))
except AttributeError:
# pylint: disable=undefined-variable
url = urlunparse((scheme, netloc, path, '', '', ''))
return url
def open_url(url,
data=None,
headers=None,
method=None,
use_proxy=True,
force=False,
last_mod_time=None,
timeout=10,
validate_certs=True,
url_username=None,
url_password=None,
http_agent=None,
force_basic_auth=False,
follow_redirects='urllib2',
client_cert=None,
client_key=None,
cookies=None):
'''
Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3)
Does not require the module environment
'''
handlers = []
ssl_handler = maybe_add_ssl_handler(url, validate_certs)
if ssl_handler:
handlers.append(ssl_handler)
# FIXME: change the following to use the generic_urlparse function
# to remove the indexed references for 'parsed'
parsed = urlparse(url)
if parsed[0] != 'ftp':
username = url_username
if headers is None:
headers = {}
if username:
password = url_password
netloc = parsed[1]
elif '@' in parsed[1]:
credentials, netloc = parsed[1].split('@', 1)
if ':' in credentials:
username, password = credentials.split(':', 1)
else:
username = credentials
password = ''
parsed = list(parsed)
parsed[1] = netloc
# reconstruct url without credentials
url = urlunparse(parsed)
if username and not force_basic_auth:
passman = urllib_request.HTTPPasswordMgrWithDefaultRealm()
# this creates a password manager
passman.add_password(None, netloc, username, password)
# because we have put None at the start it will always
# use this username/password combination for urls
# for which `theurl` is a super-url
authhandler = urllib_request.HTTPBasicAuthHandler(passman)
digest_authhandler = urllib_request.HTTPDigestAuthHandler(passman)
# create the AuthHandler
handlers.append(authhandler)
handlers.append(digest_authhandler)
elif username and force_basic_auth:
headers["Authorization"] = basic_auth_header(username, password)
else:
try:
rc = netrc.netrc(os.environ.get('NETRC'))
login = rc.authenticators(parsed[1])
except IOError:
login = None
if login:
username, _, password = login
if username and password:
headers["Authorization"] = basic_auth_header(
username, password)
if not use_proxy:
proxyhandler = urllib_request.ProxyHandler({})
handlers.append(proxyhandler)
if HAS_SSLCONTEXT and not validate_certs:
# In 2.7.9, the default context validates certificates
context = SSLContext(ssl.PROTOCOL_SSLv23)
context.options |= ssl.OP_NO_SSLv2
context.options |= ssl.OP_NO_SSLv3
context.verify_mode = ssl.CERT_NONE
context.check_hostname = False
handlers.append(
HTTPSClientAuthHandler(client_cert=client_cert,
client_key=client_key,
context=context))
elif client_cert:
handlers.append(
HTTPSClientAuthHandler(client_cert=client_cert,
client_key=client_key))
# pre-2.6 versions of python cannot use the custom https
# handler, since the socket class is lacking create_connection.
# Some python builds lack HTTPS support.
if hasattr(socket, 'create_connection') and CustomHTTPSHandler:
handlers.append(CustomHTTPSHandler)
handlers.append(RedirectHandlerFactory(follow_redirects, validate_certs))
# add some nicer cookie handling
if cookies is not None:
handlers.append(urllib_request.HTTPCookieProcessor(cookies))
opener = urllib_request.build_opener(*handlers)
urllib_request.install_opener(opener)
data = to_bytes(data, nonstring='passthru')
if method:
if method.upper() not in ('OPTIONS', 'GET', 'HEAD', 'POST', 'PUT',
'DELETE', 'TRACE', 'CONNECT', 'PATCH'):
raise ConnectionError('invalid HTTP request method; %s' %
method.upper())
request = RequestWithMethod(url, method.upper(), data)
else:
request = urllib_request.Request(url, data)
# add the custom agent header, to help prevent issues
# with sites that block the default urllib agent string
if http_agent:
request.add_header('User-agent', http_agent)
# Cache control
# Either we directly force a cache refresh
if force:
request.add_header('cache-control', 'no-cache')
# or we do it if the original is more recent than our copy
elif last_mod_time:
tstamp = last_mod_time.strftime('%a, %d %b %Y %H:%M:%S +0000')
request.add_header('If-Modified-Since', tstamp)
# user defined headers now, which may override things we've set above
if headers:
if not isinstance(headers, dict):
raise ValueError("headers provided to fetch_url() must be a dict")
for header in headers:
request.add_header(header, headers[header])
urlopen_args = [request, None]
if sys.version_info >= (2, 6, 0):
# urlopen in python prior to 2.6.0 did not
# have a timeout parameter
urlopen_args.append(timeout)
r = urllib_request.urlopen(*urlopen_args)
return r
def open_url(url, data=None, headers=None, method=None, use_proxy=True,
force=False, last_mod_time=None, timeout=10, validate_certs=True,
url_username=None, url_password=None, http_agent=None,
force_basic_auth=False, follow_redirects='urllib2'):
'''
Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3)
Does not require the module environment
'''
handlers = []
ssl_handler = maybe_add_ssl_handler(url, validate_certs)
if ssl_handler:
handlers.append(ssl_handler)
# FIXME: change the following to use the generic_urlparse function
# to remove the indexed references for 'parsed'
parsed = urlparse(url)
if parsed[0] != 'ftp':
username = url_username
if headers is None:
headers = {}
if username:
password = url_password
netloc = parsed[1]
elif '@' in parsed[1]:
credentials, netloc = parsed[1].split('@', 1)
if ':' in credentials:
username, password = credentials.split(':', 1)
else:
username = credentials
password = ''
parsed = list(parsed)
parsed[1] = netloc
# reconstruct url without credentials
url = urlunparse(parsed)
if username and not force_basic_auth:
passman = urllib_request.HTTPPasswordMgrWithDefaultRealm()
# this creates a password manager
passman.add_password(None, netloc, username, password)
# because we have put None at the start it will always
# use this username/password combination for urls
# for which `theurl` is a super-url
authhandler = urllib_request.HTTPBasicAuthHandler(passman)
# create the AuthHandler
handlers.append(authhandler)
elif username and force_basic_auth:
headers["Authorization"] = basic_auth_header(username, password)
else:
try:
rc = netrc.netrc(os.environ.get('NETRC'))
login = rc.authenticators(parsed[1])
except IOError:
login = None
if login:
username, _, password = login
if username and password:
headers["Authorization"] = basic_auth_header(username, password)
if not use_proxy:
proxyhandler = urllib_request.ProxyHandler({})
handlers.append(proxyhandler)
if HAS_SSLCONTEXT and not validate_certs:
# In 2.7.9, the default context validates certificates
context = SSLContext(ssl.PROTOCOL_SSLv23)
context.options |= ssl.OP_NO_SSLv2
context.options |= ssl.OP_NO_SSLv3
context.verify_mode = ssl.CERT_NONE
context.check_hostname = False
handlers.append(urllib_request.HTTPSHandler(context=context))
# pre-2.6 versions of python cannot use the custom https
# handler, since the socket class is lacking create_connection.
# Some python builds lack HTTPS support.
if hasattr(socket, 'create_connection') and CustomHTTPSHandler:
handlers.append(CustomHTTPSHandler)
handlers.append(RedirectHandlerFactory(follow_redirects, validate_certs))
opener = urllib_request.build_opener(*handlers)
urllib_request.install_opener(opener)
if method:
if method.upper() not in ('OPTIONS','GET','HEAD','POST','PUT','DELETE','TRACE','CONNECT','PATCH'):
raise ConnectionError('invalid HTTP request method; %s' % method.upper())
request = RequestWithMethod(url, method.upper(), data)
else:
request = urllib_request.Request(url, data)
# add the custom agent header, to help prevent issues
# with sites that block the default urllib agent string
request.add_header('User-agent', http_agent)
# if we're ok with getting a 304, set the timestamp in the
# header, otherwise make sure we don't get a cached copy
if last_mod_time and not force:
tstamp = last_mod_time.strftime('%a, %d %b %Y %H:%M:%S +0000')
request.add_header('If-Modified-Since', tstamp)
else:
request.add_header('cache-control', 'no-cache')
# user defined headers now, which may override things we've set above
if headers:
if not isinstance(headers, dict):
raise ValueError("headers provided to fetch_url() must be a dict")
for header in headers:
request.add_header(header, headers[header])
urlopen_args = [request, None]
if sys.version_info >= (2,6,0):
# urlopen in python prior to 2.6.0 did not
# have a timeout parameter
urlopen_args.append(timeout)
r = urllib_request.urlopen(*urlopen_args)
return r
