def test_fetch_url_connectionerror(open_url_mock, fake_assible_module):
open_url_mock.side_effect = ConnectionError('TESTS')
with pytest.raises(FailJson) as excinfo:
fetch_url(fake_assible_module, 'http://assible.com/')
assert excinfo.value.kwargs['msg'] == 'TESTS'
assert 'http://assible.com/' == excinfo.value.kwargs['url']
assert excinfo.value.kwargs['status'] == -1
open_url_mock.side_effect = ValueError('TESTS')
with pytest.raises(FailJson) as excinfo:
fetch_url(fake_assible_module, 'http://assible.com/')
assert excinfo.value.kwargs['msg'] == 'TESTS'
assert 'http://assible.com/' == excinfo.value.kwargs['url']
assert excinfo.value.kwargs['status'] == -1
def test_fetch_url(open_url_mock, fake_assible_module):
r, info = fetch_url(fake_assible_module, 'http://assible.com/')
dummy, kwargs = open_url_mock.call_args
open_url_mock.assert_called_once_with('http://assible.com/', client_cert=None, client_key=None, cookies=kwargs['cookies'], data=None,
follow_redirects='urllib2', force=False, force_basic_auth='', headers=None,
http_agent='assible-httpget', last_mod_time=None, method=None, timeout=10, url_password='', url_username='',
use_proxy=True, validate_certs=True, use_gssapi=False, unix_socket=None, ca_path=None)
def test_fetch_url_nossl(open_url_mock, fake_assible_module, mocker):
mocker.patch('assible.module_utils.urls.get_distribution', return_value='notredhat')
open_url_mock.side_effect = NoSSLError
with pytest.raises(FailJson) as excinfo:
fetch_url(fake_assible_module, 'http://assible.com/')
assert 'python-ssl' not in excinfo.value.kwargs['msg']
mocker.patch('assible.module_utils.urls.get_distribution', return_value='redhat')
open_url_mock.side_effect = NoSSLError
with pytest.raises(FailJson) as excinfo:
fetch_url(fake_assible_module, 'http://assible.com/')
assert 'python-ssl' in excinfo.value.kwargs['msg']
assert 'http://assible.com/' == excinfo.value.kwargs['url']
assert excinfo.value.kwargs['status'] == -1
def _get_ppa_info(self, owner_name, ppa_name):
lp_api = self.LP_API % (owner_name, ppa_name)
headers = dict(Accept='application/json')
response, info = fetch_url(self.module, lp_api, headers=headers)
if info['status'] != 200:
self.module.fail_json(
msg="failed to fetch PPA information, error was: %s" %
info['msg'])
return json.loads(to_native(response.read()))
def test_fetch_url_httperror(open_url_mock, fake_assible_module):
open_url_mock.side_effect = urllib_error.HTTPError(
'http://assible.com/',
500,
'Internal Server Error',
{'Content-Type': 'application/json'},
StringIO('TESTS')
)
r, info = fetch_url(fake_assible_module, 'http://assible.com/')
assert info == {'msg': 'HTTP Error 500: Internal Server Error', 'body': 'TESTS',
'status': 500, 'url': 'http://assible.com/', 'content-type': 'application/json'}
def download_key(module, url):
# FIXME: move get_url code to common, allow for in-memory D/L, support proxies
# and reuse here
if url is None:
module.fail_json(msg="needed a URL but was not specified")
try:
rsp, info = fetch_url(module, url)
if info['status'] != 200:
module.fail_json(msg="Failed to download key at %s: %s" %
(url, info['msg']))
return rsp.read()
except Exception:
module.fail_json(msg="error getting key id from url: %s" % url,
traceback=format_exc())
def test_fetch_url_cookies(mocker, fake_assible_module):
def make_cookies(*args, **kwargs):
cookies = kwargs['cookies']
r = MagicMock()
try:
r.headers = HTTPMessage()
add_header = r.headers.add_header
except TypeError:
# PY2
r.headers = HTTPMessage(StringIO())
add_header = r.headers.addheader
r.info.return_value = r.headers
for name, value in (('Foo', 'bar'), ('Baz', 'qux')):
cookie = Cookie(
version=0,
name=name,
value=value,
port=None,
port_specified=False,
domain="assible.com",
domain_specified=True,
domain_initial_dot=False,
path="/",
path_specified=True,
secure=False,
expires=None,
discard=False,
comment=None,
comment_url=None,
rest=None
)
cookies.set_cookie(cookie)
add_header('Set-Cookie', '%s=%s' % (name, value))
return r
mocker = mocker.patch('assible.module_utils.urls.open_url', new=make_cookies)
r, info = fetch_url(fake_assible_module, 'http://assible.com/')
assert info['cookies'] == {'Baz': 'qux', 'Foo': 'bar'}
# Python sorts cookies in order of most specific (ie. longest) path first
# items with the same path are reversed from response order
assert info['cookies_string'] == 'Baz=qux; Foo=bar'
# The key here has a `-` as opposed to what we see in the `uri` module that converts to `_`
# Note: this is response order, which differs from cookies_string
assert info['set-cookie'] == 'Foo=bar, Baz=qux'
def fetch_key(self, url):
"""Downloads a key from url, returns a valid path to a gpg key"""
rsp, info = fetch_url(self.module, url)
if info['status'] != 200:
self.module.fail_json(
msg="failed to fetch key at %s , error was: %s" %
(url, info['msg']))
key = rsp.read()
if not is_pubkey(key):
self.module.fail_json(msg="Not a public key: %s" % url)
tmpfd, tmpname = tempfile.mkstemp()
self.module.add_cleanup_file(tmpname)
tmpfile = os.fdopen(tmpfd, "w+b")
tmpfile.write(key)
tmpfile.close()
return tmpname
def test_fetch_url_params(open_url_mock, fake_assible_module):
fake_assible_module.params = {
'validate_certs': False,
'url_username': '******',
'url_password': '******',
'http_agent': 'assible-test',
'force_basic_auth': True,
'follow_redirects': 'all',
'client_cert': 'client.pem',
'client_key': 'client.key',
}
r, info = fetch_url(fake_assible_module, 'http://assible.com/')
dummy, kwargs = open_url_mock.call_args
open_url_mock.assert_called_once_with('http://assible.com/', client_cert='client.pem', client_key='client.key', cookies=kwargs['cookies'], data=None,
follow_redirects='all', force=False, force_basic_auth=True, headers=None,
http_agent='assible-test', last_mod_time=None, method=None, timeout=10, url_password='******', url_username='******',
use_proxy=True, validate_certs=False, use_gssapi=False, unix_socket=None, ca_path=None)
def url_get(module,
url,
dest,
use_proxy,
last_mod_time,
force,
timeout=10,
headers=None,
tmp_dest=''):
"""
Download data from the url and store in a temporary file.
Return (tempfile, info about the request)
"""
if module.check_mode:
method = 'HEAD'
else:
method = 'GET'
start = datetime.datetime.utcnow()
rsp, info = fetch_url(module,
url,
use_proxy=use_proxy,
force=force,
last_mod_time=last_mod_time,
timeout=timeout,
headers=headers,
method=method)
elapsed = (datetime.datetime.utcnow() - start).seconds
if info['status'] == 304:
module.exit_json(url=url,
dest=dest,
changed=False,
msg=info.get('msg', ''),
status_code=info['status'],
elapsed=elapsed)
# Exceptions in fetch_url may result in a status -1, the ensures a proper error to the user in all cases
if info['status'] == -1:
module.fail_json(msg=info['msg'], url=url, dest=dest, elapsed=elapsed)
if info['status'] != 200 and not url.startswith('file:/') and not (
url.startswith('ftp:/') and info.get('msg', '').startswith('OK')):
module.fail_json(msg="Request failed",
status_code=info['status'],
response=info['msg'],
url=url,
dest=dest,
elapsed=elapsed)
# create a temporary file and copy content to do checksum-based replacement
if tmp_dest:
# tmp_dest should be an existing dir
tmp_dest_is_dir = os.path.isdir(tmp_dest)
if not tmp_dest_is_dir:
if os.path.exists(tmp_dest):
module.fail_json(
msg="%s is a file but should be a directory." % tmp_dest,
elapsed=elapsed)
else:
module.fail_json(msg="%s directory does not exist." % tmp_dest,
elapsed=elapsed)
else:
tmp_dest = module.tmpdir
fd, tempname = tempfile.mkstemp(dir=tmp_dest)
f = os.fdopen(fd, 'wb')
try:
shutil.copyfileobj(rsp, f)
except Exception as e:
os.remove(tempname)
module.fail_json(msg="failed to create temporary content file: %s" %
to_native(e),
elapsed=elapsed,
exception=traceback.format_exc())
f.close()
rsp.close()
return tempname, info
def test_fetch_url_no_urlparse(mocker, fake_assible_module):
mocker.patch('assible.module_utils.urls.HAS_URLPARSE', new=False)
with pytest.raises(FailJson):
fetch_url(fake_assible_module, 'http://assible.com/')
def test_fetch_url_badstatusline(open_url_mock, fake_assible_module):
open_url_mock.side_effect = httplib.BadStatusLine('TESTS')
r, info = fetch_url(fake_assible_module, 'http://assible.com/')
assert info == {'msg': 'Connection failure: connection was closed before a valid response was received: TESTS', 'status': -1, 'url': 'http://assible.com/'}
def test_fetch_url_exception(open_url_mock, fake_assible_module):
open_url_mock.side_effect = Exception('TESTS')
r, info = fetch_url(fake_assible_module, 'http://assible.com/')
exception = info.pop('exception')
assert info == {'msg': 'An unknown error occurred: TESTS', 'status': -1, 'url': 'http://assible.com/'}
assert "Exception: TESTS" in exception
def test_fetch_url_socketerror(open_url_mock, fake_assible_module):
open_url_mock.side_effect = socket.error('TESTS')
r, info = fetch_url(fake_assible_module, 'http://assible.com/')
assert info == {'msg': 'Connection failure: TESTS', 'status': -1, 'url': 'http://assible.com/'}
def test_fetch_url_urlerror(open_url_mock, fake_assible_module):
open_url_mock.side_effect = urllib_error.URLError('TESTS')
r, info = fetch_url(fake_assible_module, 'http://assible.com/')
assert info == {'msg': 'Request failed: <urlopen error TESTS>', 'status': -1, 'url': 'http://assible.com/'}
def uri(module, url, dest, body, body_format, method, headers, socket_timeout):
# is dest is set and is a directory, let's check if we get redirected and
# set the filename from that url
redirected = False
redir_info = {}
r = {}
src = module.params['src']
if src:
try:
headers.update({
'Content-Length': os.stat(src).st_size
})
data = open(src, 'rb')
except OSError:
module.fail_json(msg='Unable to open source file %s' % src, elapsed=0)
else:
data = body
kwargs = {}
if dest is not None:
# Stash follow_redirects, in this block we don't want to follow
# we'll reset back to the supplied value soon
follow_redirects = module.params['follow_redirects']
module.params['follow_redirects'] = False
if os.path.isdir(dest):
# first check if we are redirected to a file download
_, redir_info = fetch_url(module, url, data=body,
headers=headers,
method=method,
timeout=socket_timeout, unix_socket=module.params['unix_socket'])
# if we are redirected, update the url with the location header,
# and update dest with the new url filename
if redir_info['status'] in (301, 302, 303, 307):
url = redir_info['location']
redirected = True
dest = os.path.join(dest, url_filename(url))
# if destination file already exist, only download if file newer
if os.path.exists(dest):
kwargs['last_mod_time'] = datetime.datetime.utcfromtimestamp(os.path.getmtime(dest))
# Reset follow_redirects back to the stashed value
module.params['follow_redirects'] = follow_redirects
resp, info = fetch_url(module, url, data=data, headers=headers,
method=method, timeout=socket_timeout, unix_socket=module.params['unix_socket'],
**kwargs)
try:
content = resp.read()
except AttributeError:
# there was no content, but the error read()
# may have been stored in the info as 'body'
content = info.pop('body', '')
if src:
# Try to close the open file handle
try:
data.close()
except Exception:
pass
r['redirected'] = redirected or info['url'] != url
r.update(redir_info)
r.update(info)
return r, content, dest