def test_Request_fallback(urlopen_mock, install_opener_mock, mocker):
cookies = cookiejar.CookieJar()
request = Request(
headers={'foo': 'bar'},
use_proxy=False,
force=True,
timeout=100,
validate_certs=False,
url_username='******',
url_password='******',
http_agent='ansible-tests',
force_basic_auth=True,
follow_redirects='all',
client_cert='/tmp/client.pem',
client_key='/tmp/client.key',
cookies=cookies,
unix_socket='/foo/bar/baz.sock',
ca_path='/foo/bar/baz.pem',
)
fallback_mock = mocker.spy(request, '_fallback')
r = request.open('GET', 'https://ansible.com')
calls = [
call(None, False), # use_proxy
call(None, True), # force
call(None, 100), # timeout
call(None, False), # validate_certs
call(None, 'user'), # url_username
call(None, 'passwd'), # url_password
call(None, 'ansible-tests'), # http_agent
call(None, True), # force_basic_auth
call(None, 'all'), # follow_redirects
call(None, '/tmp/client.pem'), # client_cert
call(None, '/tmp/client.key'), # client_key
call(None, cookies), # cookies
call(None, '/foo/bar/baz.sock'), # unix_socket
call(None, '/foo/bar/baz.pem'), # ca_path
]
fallback_mock.assert_has_calls(calls)
assert fallback_mock.call_count == 14 # All but headers use fallback
args = urlopen_mock.call_args[0]
assert args[1] is None # data, this is handled in the Request not urlopen
assert args[2] == 100 # timeout
req = args[0]
assert req.headers == {
'Authorization': b'Basic dXNlcjpwYXNzd2Q=',
'Cache-control': 'no-cache',
'Foo': 'bar',
'User-agent': 'ansible-tests'
}
assert req.data is None
assert req.get_method() == 'GET'
def test_Request_open_custom_method(urlopen_mock, install_opener_mock):
r = Request().open('DELETE', 'https://ansible.com/')
args = urlopen_mock.call_args[0]
req = args[0]
assert isinstance(req, RequestWithMethod)
def test_Request_open_username_force_basic(urlopen_mock, install_opener_mock):
r = Request().open('GET',
'http://ansible.com/',
url_username='******',
url_password='******',
force_basic_auth=True)
opener = install_opener_mock.call_args[0][0]
handlers = opener.handlers
expected_handlers = (
urllib_request.HTTPBasicAuthHandler,
urllib_request.HTTPDigestAuthHandler,
)
found_handlers = []
for handler in handlers:
if isinstance(handler, expected_handlers):
found_handlers.append(handler)
assert len(found_handlers) == 0
args = urlopen_mock.call_args[0]
req = args[0]
assert req.headers.get('Authorization') == b'Basic dXNlcjpwYXNzd2Q='
def test_Request_open_ftp(urlopen_mock, install_opener_mock, mocker):
mocker.patch(
'ansible_collections.notstdlib.moveitallout.plugins.module_utils.urls.ParseResultDottedDict.as_list',
side_effect=AssertionError)
# Using ftp scheme should prevent the AssertionError side effect to fire
r = Request().open('GET', 'ftp://[email protected]/')
def test_Request_open_client_cert(urlopen_mock, install_opener_mock):
here = os.path.dirname(__file__)
client_cert = os.path.join(here, 'fixtures/client.pem')
client_key = os.path.join(here, 'fixtures/client.key')
r = Request().open('GET',
'https://ansible.com/',
client_cert=client_cert,
client_key=client_key)
opener = install_opener_mock.call_args[0][0]
handlers = opener.handlers
ssl_handler = None
for handler in handlers:
if isinstance(handler, HTTPSClientAuthHandler):
ssl_handler = handler
break
assert ssl_handler is not None
assert ssl_handler.client_cert == client_cert
assert ssl_handler.client_key == client_key
https_connection = ssl_handler._build_https_connection('ansible.com')
assert https_connection.key_file == client_key
assert https_connection.cert_file == client_cert
def test_Request_open(urlopen_mock, install_opener_mock):
r = Request().open('GET', 'https://ansible.com/')
args = urlopen_mock.call_args[0]
assert args[1] is None # data, this is handled in the Request not urlopen
assert args[2] == 10 # timeout
req = args[0]
assert req.headers == {}
assert req.data is None
assert req.get_method() == 'GET'
opener = install_opener_mock.call_args[0][0]
handlers = opener.handlers
if not HAS_SSLCONTEXT:
expected_handlers = (
SSLValidationHandler,
RedirectHandlerFactory(), # factory, get handler
)
else:
expected_handlers = (
RedirectHandlerFactory(), # factory, get handler
)
found_handlers = []
for handler in handlers:
if isinstance(handler, SSLValidationHandler
) or handler.__class__.__name__ == 'RedirectHandler':
found_handlers.append(handler)
assert len(found_handlers) == len(expected_handlers)
def test_Request_open_invalid_method(urlopen_mock, install_opener_mock):
r = Request().open('UNKNOWN', 'https://ansible.com/')
args = urlopen_mock.call_args[0]
req = args[0]
assert req.data is None
assert req.get_method() == 'UNKNOWN'
def test_Request_open_user_agent(urlopen_mock, install_opener_mock):
r = Request().open('GET',
'https://ansible.com/',
http_agent='ansible-tests')
args = urlopen_mock.call_args[0]
req = args[0]
assert req.headers.get('User-agent') == 'ansible-tests'
def test_Request_open_last_mod(urlopen_mock, install_opener_mock):
now = datetime.datetime.now()
r = Request().open('GET', 'https://ansible.com/', last_mod_time=now)
args = urlopen_mock.call_args[0]
req = args[0]
assert req.headers.get('If-modified-since') == now.strftime(
'%a, %d %b %Y %H:%M:%S -0000')
def test_methods(method, kwargs, mocker):
expected = method.upper()
open_mock = mocker.patch(
'ansible_collections.notstdlib.moveitallout.plugins.module_utils.urls.Request.open'
)
request = Request()
getattr(request, method)('https://ansible.com')
open_mock.assert_called_once_with(expected, 'https://ansible.com',
**kwargs)
def test_Request_open_netrc(urlopen_mock, install_opener_mock, monkeypatch):
here = os.path.dirname(__file__)
monkeypatch.setenv('NETRC', os.path.join(here, 'fixtures/netrc'))
r = Request().open('GET', 'http://ansible.com/')
args = urlopen_mock.call_args[0]
req = args[0]
assert req.headers.get('Authorization') == b'Basic dXNlcjpwYXNzd2Q='
r = Request().open('GET', 'http://foo.ansible.com/')
args = urlopen_mock.call_args[0]
req = args[0]
assert 'Authorization' not in req.headers
monkeypatch.setenv('NETRC', os.path.join(here,
'fixtures/netrc.nonexistant'))
r = Request().open('GET', 'http://ansible.com/')
args = urlopen_mock.call_args[0]
req = args[0]
assert 'Authorization' not in req.headers
def test_Request_open_force(urlopen_mock, install_opener_mock):
r = Request().open('GET',
'https://ansible.com/',
force=True,
last_mod_time=datetime.datetime.now())
args = urlopen_mock.call_args[0]
req = args[0]
assert req.headers.get('Cache-control') == 'no-cache'
assert 'If-modified-since' not in req.headers
def __init__(self, headers=None, use_proxy=True, force=False, timeout=120,
validate_certs=True, url_username=None, url_password=None,
http_agent=None, force_basic_auth=False, follow_redirects='urllib2',
client_cert=None, client_key=None, cookies=None):
self.request = Request(
headers=headers,
use_proxy=use_proxy,
force=force,
timeout=timeout,
validate_certs=validate_certs,
url_username=url_username,
url_password=url_password,
http_agent=http_agent,
force_basic_auth=force_basic_auth,
follow_redirects=follow_redirects,
client_cert=client_cert,
client_key=client_key,
cookies=cookies
)
self.last_url = None
def test_Request_open_http(urlopen_mock, install_opener_mock):
r = Request().open('GET', 'http://ansible.com/')
args = urlopen_mock.call_args[0]
opener = install_opener_mock.call_args[0][0]
handlers = opener.handlers
found_handlers = []
for handler in handlers:
if isinstance(handler, SSLValidationHandler):
found_handlers.append(handler)
assert len(found_handlers) == 0
def test_Request_open_no_proxy(urlopen_mock, install_opener_mock, mocker):
build_opener_mock = mocker.patch(
'ansible_collections.notstdlib.moveitallout.plugins.module_utils.urls.urllib_request.build_opener'
)
r = Request().open('GET', 'http://ansible.com/', use_proxy=False)
handlers = build_opener_mock.call_args[0]
found_handlers = []
for handler in handlers:
if isinstance(handler, urllib_request.ProxyHandler):
found_handlers.append(handler)
assert len(found_handlers) == 1
def test_Request_open_unix_socket(urlopen_mock, install_opener_mock):
r = Request().open('GET',
'http://ansible.com/',
unix_socket='/foo/bar/baz.sock')
args = urlopen_mock.call_args[0]
opener = install_opener_mock.call_args[0][0]
handlers = opener.handlers
found_handlers = []
for handler in handlers:
if isinstance(handler, UnixHTTPHandler):
found_handlers.append(handler)
assert len(found_handlers) == 1
def test_Request_open_cookies(urlopen_mock, install_opener_mock):
r = Request().open('GET',
'https://ansible.com/',
cookies=cookiejar.CookieJar())
opener = install_opener_mock.call_args[0][0]
handlers = opener.handlers
cookies_handler = None
for handler in handlers:
if isinstance(handler, urllib_request.HTTPCookieProcessor):
cookies_handler = handler
break
assert cookies_handler is not None
def test_Request_open_https_unix_socket(urlopen_mock, install_opener_mock):
r = Request().open('GET',
'https://ansible.com/',
unix_socket='/foo/bar/baz.sock')
args = urlopen_mock.call_args[0]
opener = install_opener_mock.call_args[0][0]
handlers = opener.handlers
found_handlers = []
for handler in handlers:
if isinstance(handler, HTTPSClientAuthHandler):
found_handlers.append(handler)
assert len(found_handlers) == 1
inst = found_handlers[0]._build_https_connection('foo')
assert isinstance(inst, UnixHTTPSConnection)
def test_Request_open_username_in_url(urlopen_mock, install_opener_mock):
r = Request().open('GET', 'http://[email protected]/')
opener = install_opener_mock.call_args[0][0]
handlers = opener.handlers
expected_handlers = (
urllib_request.HTTPBasicAuthHandler,
urllib_request.HTTPDigestAuthHandler,
)
found_handlers = []
for handler in handlers:
if isinstance(handler, expected_handlers):
found_handlers.append(handler)
assert found_handlers[0].passwd.passwd[None] == {
(('ansible.com', '/'), ): ('user2', '')
}
def test_Request_open_auth_in_netloc(urlopen_mock, install_opener_mock):
r = Request().open('GET', 'http://*****:*****@ansible.com/')
args = urlopen_mock.call_args[0]
req = args[0]
assert req.get_full_url() == 'http://ansible.com/'
opener = install_opener_mock.call_args[0][0]
handlers = opener.handlers
expected_handlers = (
urllib_request.HTTPBasicAuthHandler,
urllib_request.HTTPDigestAuthHandler,
)
found_handlers = []
for handler in handlers:
if isinstance(handler, expected_handlers):
found_handlers.append(handler)
assert len(found_handlers) == 2
def test_Request_open_no_validate_certs(urlopen_mock, install_opener_mock):
r = Request().open('GET', 'https://ansible.com/', validate_certs=False)
opener = install_opener_mock.call_args[0][0]
handlers = opener.handlers
ssl_handler = None
for handler in handlers:
if isinstance(handler, HTTPSClientAuthHandler):
ssl_handler = handler
break
assert ssl_handler is not None
inst = ssl_handler._build_https_connection('foo')
assert isinstance(inst, httplib.HTTPSConnection)
context = ssl_handler._context
assert context.protocol == ssl.PROTOCOL_SSLv23
if ssl.OP_NO_SSLv2:
assert context.options & ssl.OP_NO_SSLv2
assert context.options & ssl.OP_NO_SSLv3
assert context.verify_mode == ssl.CERT_NONE
assert context.check_hostname is False
def _set_config(self, name, **kwargs):
headers = {
"Content-Type": "application/json",
"Connection": "keep-alive",
}
self.request = Request(headers=headers, timeout=60)
configurators = [self._read_config_vars]
for configurator in configurators:
self._config = configurator(name, **kwargs)
if self._config:
break
if self._config is None:
raise SessionConfigurationException(
to_native("No Configuration Found."))
# set up auth if passed
entrust_api_user = self.get_config("entrust_api_user")
entrust_api_key = self.get_config("entrust_api_key")
if entrust_api_user and entrust_api_key:
self.request.url_username = entrust_api_user
self.request.url_password = entrust_api_key
else:
raise SessionConfigurationException(
to_native("User and key must be provided."))
# set up client certificate if passed (support all-in one or cert + key)
entrust_api_cert = self.get_config("entrust_api_cert")
entrust_api_cert_key = self.get_config("entrust_api_cert_key")
if entrust_api_cert:
self.request.client_cert = entrust_api_cert
if entrust_api_cert_key:
self.request.client_key = entrust_api_cert_key
else:
raise SessionConfigurationException(
to_native(
"Client certificate for authentication to the API must be provided."
))
# set up the spec
entrust_api_specification_path = self.get_config(
"entrust_api_specification_path")
if not entrust_api_specification_path.startswith(
"http") and not os.path.isfile(entrust_api_specification_path):
raise SessionConfigurationException(
to_native(
"OpenAPI specification was not found at location {0}.".
format(entrust_api_specification_path)))
if not valid_file_format.match(entrust_api_specification_path):
raise SessionConfigurationException(
to_native(
"OpenAPI specification filename must end in .json, .yml or .yaml"
))
self.verify = True
if entrust_api_specification_path.startswith("http"):
try:
http_response = Request().open(
method="GET", url=entrust_api_specification_path)
http_response_contents = http_response.read()
if entrust_api_specification_path.endswith(".json"):
self._spec = json.load(http_response_contents)
elif entrust_api_specification_path.endswith(
".yml") or entrust_api_specification_path.endswith(
".yaml"):
self._spec = yaml.safe_load(http_response_contents)
except HTTPError as e:
raise SessionConfigurationException(
to_native(
"Error downloading specification from address '{0}', received error code '{1}'"
.format(entrust_api_specification_path, e.getcode())))
else:
with open(entrust_api_specification_path) as f:
if ".json" in entrust_api_specification_path:
self._spec = json.load(f)
elif ".yml" in entrust_api_specification_path or ".yaml" in entrust_api_specification_path:
self._spec = yaml.safe_load(f)
def parse(self, inventory, loader, path, cache=True):
super(InventoryModule, self).parse(inventory, loader, path)
if not self.no_config_file_supplied and os.path.isfile(path):
self._read_config_data(path)
# Read inventory from tower server.
# Note the environment variables will be handled automatically by InventoryManager.
tower_host = self.get_option('host')
if not re.match('(?:http|https)://', tower_host):
tower_host = 'https://{tower_host}'.format(tower_host=tower_host)
request_handler = Request(
url_username=self.get_option('username'),
url_password=self.get_option('password'),
force_basic_auth=True,
validate_certs=self.get_option('validate_certs'))
# validate type of inventory_id because we allow two types as special case
inventory_id = self.get_option('inventory_id')
if isinstance(inventory_id, int):
inventory_id = to_text(inventory_id, nonstring='simplerepr')
else:
try:
inventory_id = ensure_type(inventory_id, 'str')
except ValueError as e:
raise AnsibleOptionsError(
'Invalid type for configuration option inventory_id, '
'not integer, and cannot convert to string: {err}'.format(
err=to_native(e)))
inventory_id = inventory_id.replace('/', '')
inventory_url = '/api/v2/inventories/{inv_id}/script/?hostvars=1&towervars=1&all=1'.format(
inv_id=inventory_id)
inventory_url = urljoin(tower_host, inventory_url)
inventory = self.make_request(request_handler, inventory_url)
# To start with, create all the groups.
for group_name in inventory:
if group_name != '_meta':
self.inventory.add_group(group_name)
# Then, create all hosts and add the host vars.
all_hosts = inventory['_meta']['hostvars']
for host_name, host_vars in six.iteritems(all_hosts):
self.inventory.add_host(host_name)
for var_name, var_value in six.iteritems(host_vars):
self.inventory.set_variable(host_name, var_name, var_value)
# Lastly, create to group-host and group-group relationships, and set group vars.
for group_name, group_content in six.iteritems(inventory):
if group_name != 'all' and group_name != '_meta':
# First add hosts to groups
for host_name in group_content.get('hosts', []):
self.inventory.add_host(host_name, group_name)
# Then add the parent-children group relationships.
for child_group_name in group_content.get('children', []):
self.inventory.add_child(group_name, child_group_name)
# Set the group vars. Note we should set group var for 'all', but not '_meta'.
if group_name != '_meta':
for var_name, var_value in six.iteritems(
group_content.get('vars', {})):
self.inventory.set_variable(group_name, var_name,
var_value)
# Fetch extra variables if told to do so
if self.get_option('include_metadata'):
config_url = urljoin(tower_host, '/api/v2/config/')
config_data = self.make_request(request_handler, config_url)
server_data = {}
server_data['license_type'] = config_data.get(
'license_info', {}).get('license_type', 'unknown')
for key in ('version', 'ansible_version'):
server_data[key] = config_data.get(key, 'unknown')
self.inventory.set_variable('all', 'tower_metadata', server_data)
# Clean up the inventory.
self.inventory.reconcile_inventory()
def test_Request_init_headers_not_dict(urlopen_mock, install_opener_mock):
with pytest.raises(ValueError):
Request(headers=['bob'])
def test_Request_open_headers_not_dict(urlopen_mock, install_opener_mock):
with pytest.raises(ValueError):
Request().open('GET', 'https://ansible.com/', headers=['bob'])
def test_Request_open_headers(urlopen_mock, install_opener_mock):
r = Request().open('GET', 'http://ansible.com/', headers={'Foo': 'bar'})
args = urlopen_mock.call_args[0]
req = args[0]
assert req.headers == {'Foo': 'bar'}
class iControlRestSession(object):
"""Represents a session that communicates with a BigIP.
This acts as a loose wrapper around Ansible's ``Request`` class. We're doing
this as interim work until we move to the httpapi connector.
"""
def __init__(self, headers=None, use_proxy=True, force=False, timeout=120,
validate_certs=True, url_username=None, url_password=None,
http_agent=None, force_basic_auth=False, follow_redirects='urllib2',
client_cert=None, client_key=None, cookies=None):
self.request = Request(
headers=headers,
use_proxy=use_proxy,
force=force,
timeout=timeout,
validate_certs=validate_certs,
url_username=url_username,
url_password=url_password,
http_agent=http_agent,
force_basic_auth=force_basic_auth,
follow_redirects=follow_redirects,
client_cert=client_cert,
client_key=client_key,
cookies=cookies
)
self.last_url = None
def get_headers(self, result):
try:
return dict(result.getheaders())
except AttributeError:
return result.headers
def update_response(self, response, result):
response.headers = self.get_headers(result)
response._content = result.read()
response.status = result.getcode()
response.url = result.geturl()
response.msg = "OK (%s bytes)" % response.headers.get('Content-Length', 'unknown')
def send(self, method, url, **kwargs):
response = Response()
# Set the last_url called
#
# This is used by the object destructor to erase the token when the
# ModuleManager exits and destroys the iControlRestSession object
self.last_url = url
body = None
data = kwargs.pop('data', None)
json = kwargs.pop('json', None)
if not data and json is not None:
self.request.headers['Content-Type'] = 'application/json'
body = _json.dumps(json)
if not isinstance(body, bytes):
body = body.encode('utf-8')
if data:
body = data
if body:
kwargs['data'] = body
try:
result = self.request.open(method, url, **kwargs)
except HTTPError as e:
# Catch HTTPError delivered from Ansible
#
# The structure of this object, in Ansible 2.8 is
#
# HttpError {
# args
# characters_written
# close
# code
# delete
# errno
# file
# filename
# filename2
# fp
# getcode
# geturl
# hdrs
# headers
# info
# msg
# name
# reason
# strerror
# url
# with_traceback
# }
self.update_response(response, e)
return response
self.update_response(response, result)
return response
def delete(self, url, **kwargs):
return self.send('DELETE', url, **kwargs)
def get(self, url, **kwargs):
return self.send('GET', url, **kwargs)
def patch(self, url, data=None, **kwargs):
return self.send('PATCH', url, data=data, **kwargs)
def post(self, url, data=None, **kwargs):
return self.send('POST', url, data=data, **kwargs)
def put(self, url, data=None, **kwargs):
return self.send('PUT', url, data=data, **kwargs)
def __del__(self):
if self.last_url is None:
return
token = self.request.headers.get('X-F5-Auth-Token', None)
if not token:
return
try:
p = generic_urlparse(urlparse(self.last_url))
uri = "https://{0}:{1}/mgmt/shared/authz/tokens/{2}".format(
p['hostname'], p['port'], token
)
self.delete(uri)
except ValueError:
pass