def redirect_request(self, req, fp, code, msg, hdrs, newurl):
handler = maybe_add_ssl_handler(newurl, validate_certs)
if handler:
urllib_request._opener.add_handler(handler)
if follow_redirects == 'urllib2':
return urllib_request.HTTPRedirectHandler.redirect_request(
self, req, fp, code, msg, hdrs, newurl)
elif follow_redirects in ['no', 'none', False]:
raise urllib_error.HTTPError(newurl, code, msg, hdrs, fp)
do_redirect = False
if follow_redirects in ['all', 'yes', True]:
do_redirect = (code >= 300 and code < 400)
elif follow_redirects == 'safe':
m = req.get_method()
do_redirect = (code >= 300 and code < 400
and m in ('GET', 'HEAD'))
if do_redirect:
# be conciliant with URIs containing a space
newurl = newurl.replace(' ', '%20')
newheaders = dict(
(k, v) for k, v in req.headers.items()
if k.lower() not in ("content-length", "content-type"))
return urllib_request.Request(
newurl,
headers=newheaders,
origin_req_host=req.get_origin_req_host(),
unverifiable=True)
else:
raise urllib_error.HTTPError(req.get_full_url(), code, msg,
hdrs, fp)
def test_initialise_unknown(monkeypatch):
mock_open = MagicMock()
mock_open.side_effect = [
urllib_error.HTTPError('https://galaxy.ansible.com/api/', 500, 'msg', {}, StringIO(u'{"msg":"raw error"}')),
urllib_error.HTTPError('https://galaxy.ansible.com/api/api/', 500, 'msg', {}, StringIO(u'{"msg":"raw error"}')),
]
monkeypatch.setattr(galaxy_api, 'open_url', mock_open)
api = GalaxyAPI(None, "test", "https://galaxy.ansible.com/api/", token=GalaxyToken(token='my_token'))
expected = "Error when finding available api versions from test (%s) (HTTP Code: 500, Message: msg)" \
% api.api_server
with pytest.raises(AnsibleError, match=re.escape(expected)):
api.authenticate("github_token")
def test_initialise_automation_hub(monkeypatch):
mock_open = MagicMock()
mock_open.side_effect = [
urllib_error.HTTPError('https://galaxy.ansible.com/api', 401, 'msg',
{}, StringIO()),
# AH won't return v1 but we do for authenticate() to work.
StringIO(u'{"available_versions":{"v1":"/api/v1","v3":"/api/v3"}}'),
StringIO(u'{"token":"my token"}'),
]
monkeypatch.setattr(galaxy_api, 'open_url', mock_open)
api = GalaxyAPI(None,
"test",
"https://galaxy.ansible.com",
token=GalaxyToken(token='my_token'))
actual = api.authenticate("github_token")
assert len(api.available_api_versions) == 2
assert api.available_api_versions['v1'] == u'/api/v1'
assert api.available_api_versions['v3'] == u'/api/v3'
assert actual == {u'token': u'my token'}
assert mock_open.call_count == 3
assert mock_open.mock_calls[0][1][0] == 'https://galaxy.ansible.com/api'
assert mock_open.mock_calls[0][2]['headers'] == {
'Authorization': 'Token my_token'
}
assert mock_open.mock_calls[1][1][0] == 'https://galaxy.ansible.com/api'
assert mock_open.mock_calls[1][2]['headers'] == {
'Authorization': 'Bearer my_token'
}
assert mock_open.mock_calls[2][1][
0] == 'https://galaxy.ansible.com/api/v1/tokens/'
assert mock_open.mock_calls[2][2]['data'] == 'github_token=github_token'
def test_publish_failure_v3_with_json_info_409_conflict(
galaxy_server, collection_artifact, monkeypatch):
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v3': '/api/v3'}
monkeypatch.setattr(collection, 'get_available_api_versions',
mock_avail_ver)
artifact_path, mock_open = collection_artifact
error_response = {
"errors": [
{
"code": "conflict.collection_exists",
"detail":
'Collection "testing-ansible_testing_content-4.0.4" already exists.',
"title": "Conflict.",
"status": "409",
},
]
}
return_content = StringIO(to_text(json.dumps(error_response)))
mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com',
409, 'msg', {},
return_content)
expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \
'(HTTP Code: 409, Message: Collection "testing-ansible_testing_content-4.0.4"' \
' already exists. Code: conflict.collection_exists)'
with pytest.raises(AnsibleError, match=re.escape(expected)):
collection.publish_collection(artifact_path, galaxy_server, True, 0)
def test_build_requirement_from_name_second_server(monkeypatch):
galaxy_server = 'https://galaxy-dev.ansible.com'
json_str = artifact_versions_json('namespace', 'collection',
['1.0.1', '1.0.2', '1.0.3'],
galaxy_server)
mock_open = MagicMock()
mock_open.side_effect = (urllib_error.HTTPError(
'https://galaxy.server.com', 404, 'msg', {}, None), StringIO(json_str))
monkeypatch.setattr(collection, 'open_url', mock_open)
actual = collection.CollectionRequirement.from_name(
'namespace.collection', ['https://broken.com/', galaxy_server],
'>1.0.1', False, True)
assert actual.namespace == u'namespace'
assert actual.name == u'collection'
assert actual.b_path is None
assert actual.source == to_text(galaxy_server)
assert actual.skip is False
assert actual.versions == set([u'1.0.2', u'1.0.3'])
assert actual.latest_version == u'1.0.3'
assert actual.dependencies is None
assert mock_open.call_count == 2
assert mock_open.mock_calls[0][1][
0] == u"https://broken.com/api/v2/collections/namespace/collection/versions/"
assert mock_open.mock_calls[0][2] == {'validate_certs': False}
assert mock_open.mock_calls[1][1][
0] == u"%s/api/v2/collections/namespace/collection/versions/" % galaxy_server
assert mock_open.mock_calls[1][2] == {'validate_certs': False}
def test_build_requirement_from_name_401_unauthorized(galaxy_server,
monkeypatch,
tmp_path_factory):
mock_open = MagicMock()
mock_open.side_effect = api.GalaxyError(
urllib_error.HTTPError('https://galaxy.server.com', 401, 'msg', {},
StringIO()), "error")
monkeypatch.setattr(galaxy_server, 'get_collection_versions', mock_open)
test_dir = to_bytes(
tmp_path_factory.mktemp('test-ÅÑŚÌβŁÈ Collections Input'))
concrete_artifact_cm = collection.concrete_artifact_manager.ConcreteArtifactsManager(
test_dir, validate_certs=False)
cli = GalaxyCLI(args=[
'ansible-galaxy', 'collection', 'install',
'namespace.collection:>1.0.1'
])
requirements = cli._require_one_of_collections_requirements(
['namespace.collection'], None,
artifacts_manager=concrete_artifact_cm)['collections']
expected = "error (HTTP Code: 401, Message: msg)"
with pytest.raises(api.GalaxyError, match=re.escape(expected)):
collection._resolve_depenency_map(requirements,
[galaxy_server, galaxy_server],
concrete_artifact_cm, None, False,
False, False)
def test_build_requirement_from_name_second_server(galaxy_server, monkeypatch):
mock_get_versions = MagicMock()
mock_get_versions.return_value = ['1.0.1', '1.0.2', '1.0.3']
monkeypatch.setattr(galaxy_server, 'get_collection_versions',
mock_get_versions)
broken_server = copy.copy(galaxy_server)
broken_server.api_server = 'https://broken.com/'
mock_404 = MagicMock()
mock_404.side_effect = api.GalaxyError(
urllib_error.HTTPError('https://galaxy.server.com', 404, 'msg', {},
StringIO()), "custom msg")
monkeypatch.setattr(broken_server, 'get_collection_versions', mock_404)
actual = collection.CollectionRequirement.from_name(
'namespace.collection', [broken_server, galaxy_server], '>1.0.1',
False, True)
assert actual.namespace == u'namespace'
assert actual.name == u'collection'
assert actual.b_path is None
# assert actual.api == galaxy_server
assert actual.skip is False
assert actual.versions == set([u'1.0.2', u'1.0.3'])
assert actual.latest_version == u'1.0.3'
assert actual.dependencies == {}
assert mock_404.call_count == 1
assert mock_404.mock_calls[0][1] == ('namespace', 'collection')
assert mock_get_versions.call_count == 1
assert mock_get_versions.mock_calls[0][1] == ('namespace', 'collection')
def test_publish_failure(galaxy_server, collection_artifact):
artifact_path, mock_open = collection_artifact
mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 500, 'msg', {}, StringIO())
expected = 'Error when publishing collection (HTTP Code: 500, Message: Unknown error returned by Galaxy ' \
'server. Code: Unknown)'
with pytest.raises(AnsibleError, match=re.escape(expected)):
collection.publish_collection(artifact_path, galaxy_server, True, 0)
def test_publish_failure_with_json_info(galaxy_server, collection_artifact):
artifact_path, mock_open = collection_artifact
return_content = StringIO(u'{"message":"Galaxy error message","code":"GWE002"}')
mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 503, 'msg', {}, return_content)
expected = 'Error when publishing collection (HTTP Code: 503, Message: Galaxy error message Code: GWE002)'
with pytest.raises(AnsibleError, match=re.escape(expected)):
collection.publish_collection(artifact_path, galaxy_server, True, 0)
def test_build_requirement_from_name_401_unauthorized(galaxy_server, monkeypatch):
mock_open = MagicMock()
mock_open.side_effect = api.GalaxyError(urllib_error.HTTPError('https://galaxy.server.com', 401, 'msg', {},
StringIO()), "error")
monkeypatch.setattr(galaxy_server, 'get_collection_versions', mock_open)
expected = "error (HTTP Code: 401, Message: Unknown error returned by Galaxy server.)"
with pytest.raises(api.GalaxyError, match=re.escape(expected)):
collection.CollectionRequirement.from_name('namespace.collection', [galaxy_server, galaxy_server], '*', False)
def test_build_requirement_from_name_missing(galaxy_server, monkeypatch):
mock_open = MagicMock()
mock_open.side_effect = api.GalaxyError(urllib_error.HTTPError('https://galaxy.server.com', 404, 'msg', {},
StringIO()), "")
monkeypatch.setattr(galaxy_server, 'get_collection_versions', mock_open)
expected = "Failed to find collection namespace.collection:*"
with pytest.raises(AnsibleError, match=expected):
collection.CollectionRequirement.from_name('namespace.collection', [galaxy_server, galaxy_server], '*', False,
True)
def test_publish_failure(api_version, collection_url, response, expected, collection_artifact, monkeypatch):
api = get_test_galaxy_api('https://galaxy.server.com/api/', api_version)
expected_url = '%s/api/%s/%s' % (api.api_server, api_version, collection_url)
mock_open = MagicMock()
mock_open.side_effect = urllib_error.HTTPError(expected_url, 500, 'msg', {},
StringIO(to_text(json.dumps(response))))
monkeypatch.setattr(galaxy_api, 'open_url', mock_open)
with pytest.raises(GalaxyError, match=re.escape(to_native(expected % api.api_server))):
api.publish_collection(collection_artifact)
def test_build_requirement_from_name_missing(monkeypatch):
mock_open = MagicMock()
mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com',
404, 'msg', {}, None)
monkeypatch.setattr(collection, 'open_url', mock_open)
expected = "Failed to find collection namespace.collection:*"
with pytest.raises(AnsibleError, match=expected):
collection.CollectionRequirement.from_name(
'namespace.collection',
['https://broken.com/', 'https://broken2.com'], '*', False, True)
def test_publish_failure(galaxy_server, collection_artifact, monkeypatch):
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v2': '/api/v2'}
monkeypatch.setattr(collection, 'get_available_api_versions',
mock_avail_ver)
artifact_path, mock_open = collection_artifact
mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com',
500, 'msg', {}, StringIO())
expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \
'(HTTP Code: 500, Message: Unknown error returned by Galaxy ' \
'server. Code: Unknown)'
with pytest.raises(AnsibleError, match=re.escape(expected)):
collection.publish_collection(artifact_path, galaxy_server, True, 0)
def test_build_requirement_from_name_missing(galaxy_server, monkeypatch):
mock_open = MagicMock()
mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com',
404, 'msg', {}, None)
monkeypatch.setattr(collection, 'open_url', mock_open)
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v2': '/api/v2'}
monkeypatch.setattr(collection, 'get_available_api_versions',
mock_avail_ver)
expected = "Failed to find collection namespace.collection:*"
with pytest.raises(AnsibleError, match=expected):
collection.CollectionRequirement.from_name(
'namespace.collection', [galaxy_server, galaxy_server], '*', False,
True)
def test_get_available_api_versions_v3_auth_required_without_auth(
galaxy_server, collection_artifact, monkeypatch):
# mock_avail_ver = MagicMock()
# mock_avail_ver.side_effect = {api_version: '/api/%s' % api_version}
# monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
error_response = {
'code': 'unauthorized',
'detail': 'The request was not authorized'
}
artifact_path, mock_open = collection_artifact
return_content = StringIO(to_text(json.dumps(error_response)))
mock_open.side_effect = urllib_error.HTTPError(
'https://galaxy.server.com', 401, 'msg',
{'WWW-Authenticate': 'Bearer'}, return_content)
with pytest.raises(AnsibleError):
collection.get_available_api_versions(galaxy_server)
def test_get_available_api_versions_v3_auth_required_with_auth_on_retry(
galaxy_server, collection_artifact, monkeypatch):
# mock_avail_ver = MagicMock()
# mock_avail_ver.side_effect = {api_version: '/api/%s' % api_version}
# monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
error_obj = {
'code': 'unauthorized',
'detail': 'The request was not authorized'
}
success_obj = {
"description":
"GALAXY REST API",
"current_version":
"v1",
"available_versions": {
"v3": "/api/v3/"
},
"server_version":
"3.2.4",
"version_name":
"Doin' it Right",
"team_members": [
"chouseknecht", "cutwater", "alikins", "newswangerd", "awcrosby",
"tima", "gregdek"
]
}
artifact_path, mock_open = collection_artifact
error_response = StringIO(to_text(json.dumps(error_obj)))
success_response = StringIO(to_text(json.dumps(success_obj)))
mock_open.side_effect = [
urllib_error.HTTPError('https://galaxy.server.com', 401, 'msg',
{'WWW-Authenticate': 'Bearer'}, error_response),
success_response,
]
try:
res = collection.get_available_api_versions(galaxy_server)
except AnsibleError as err:
print(err)
raise
assert res == {'v3': '/api/v3/'}
def test_publish_failure_with_json_info(galaxy_server, collection_artifact,
monkeypatch):
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v2': '/api/v2'}
monkeypatch.setattr(collection, 'get_available_api_versions',
mock_avail_ver)
artifact_path, mock_open = collection_artifact
return_content = StringIO(
u'{"message":"Galaxy error message","code":"GWE002"}')
mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com',
503, 'msg', {},
return_content)
expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \
'(HTTP Code: 503, Message: Galaxy error message Code: GWE002)'
with pytest.raises(AnsibleError, match=re.escape(expected)):
collection.publish_collection(artifact_path, galaxy_server, True, 0)
def test_publish_failure_v3_with_json_info_multiple_errors(
galaxy_server, collection_artifact, monkeypatch):
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v3': '/api/v3'}
monkeypatch.setattr(collection, 'get_available_api_versions',
mock_avail_ver)
artifact_path, mock_open = collection_artifact
error_response = {
"errors": [
{
"code": "conflict.collection_exists",
"detail":
'Collection "mynamespace-mycollection-4.1.1" already exists.',
"title": "Conflict.",
"status": "400",
},
{
"code": "quantum_improbability",
"title": "Random(?) quantum improbability.",
"source": {
"parameter": "the_arrow_of_time"
},
"meta": {
"remediation": "Try again before"
}
},
]
}
return_content = StringIO(to_text(json.dumps(error_response)))
mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com',
400, 'msg', {},
return_content)
expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \
'(HTTP Code: 400, Message: Collection "mynamespace-mycollection-4.1.1"' \
' already exists. Code: conflict.collection_exists),' \
' (HTTP Code: 400, Message: Random(?) quantum improbability. Code: quantum_improbability)'
with pytest.raises(AnsibleError, match=re.escape(expected)):
collection.publish_collection(artifact_path, galaxy_server, True, 0)
def test_build_requirement_from_name_second_server(api_version, exp_api_url,
galaxy_server, monkeypatch):
mock_avail_ver = MagicMock()
avail_api_versions = {api_version: '/api/%s' % api_version}
mock_avail_ver.return_value = avail_api_versions
monkeypatch.setattr(collection, 'get_available_api_versions',
mock_avail_ver)
json_str = artifact_versions_json('namespace', 'collection',
['1.0.1', '1.0.2', '1.0.3'],
galaxy_server, avail_api_versions)
mock_open = MagicMock()
mock_open.side_effect = (urllib_error.HTTPError(
'https://galaxy.server.com', 404, 'msg', {}, None), StringIO(json_str))
monkeypatch.setattr(collection, 'open_url', mock_open)
broken_server = copy.copy(galaxy_server)
broken_server.api_server = 'https://broken.com/'
actual = collection.CollectionRequirement.from_name(
'namespace.collection', [broken_server, galaxy_server], '>1.0.1',
False, True)
assert actual.namespace == u'namespace'
assert actual.name == u'collection'
assert actual.b_path is None
# assert actual.api == galaxy_server
assert actual.skip is False
assert actual.versions == set([u'1.0.2', u'1.0.3'])
assert actual.latest_version == u'1.0.3'
assert actual.dependencies is None
assert mock_open.call_count == 2
assert mock_open.mock_calls[0][1][
0] == u"https://broken.com%s" % exp_api_url
assert mock_open.mock_calls[1][1][0] == u"%s%s" % (
galaxy_server.api_server, exp_api_url)
assert mock_open.mock_calls[1][2] == {
'validate_certs': True,
"headers": {}
}
def test_build_requirement_from_name_401_unauthorized(api_version,
errors_to_return,
expected, galaxy_server,
monkeypatch):
mock_avail_ver = MagicMock()
available_api_versions = {api_version: '/api/%s' % api_version}
mock_avail_ver.return_value = available_api_versions
monkeypatch.setattr(collection, 'get_available_api_versions',
mock_avail_ver)
json_str = error_json(galaxy_server,
errors_to_return=errors_to_return,
available_api_versions=available_api_versions)
mock_open = MagicMock()
monkeypatch.setattr(collection, 'open_url', mock_open)
mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com',
401, 'msg', {},
StringIO(json_str))
with pytest.raises(AnsibleError, match=expected):
collection.CollectionRequirement.from_name(
'namespace.collection', [galaxy_server, galaxy_server], '*', False)
def test_cache_flaky_pagination(cache_dir, monkeypatch):
responses = get_collection_versions()
cache_file = os.path.join(cache_dir, 'api.json')
api = get_test_galaxy_api('https://galaxy.server.com/api/', 'v2', no_cache=False)
# First attempt, fail midway through
mock_open = MagicMock(
side_effect=[
StringIO(to_text(json.dumps(responses[0]))),
StringIO(to_text(json.dumps(responses[1]))),
urllib_error.HTTPError(responses[1]['next'], 500, 'Error', {}, StringIO()),
StringIO(to_text(json.dumps(responses[3]))),
]
)
monkeypatch.setattr(galaxy_api, 'open_url', mock_open)
expected = (
r'Error when getting available collection versions for namespace\.collection '
r'from test \(https://galaxy\.server\.com/api/\) '
r'\(HTTP Code: 500, Message: Error Code: Unknown\)'
)
with pytest.raises(GalaxyError, match=expected):
api.get_collection_versions('namespace', 'collection')
with open(cache_file) as fd:
final_cache = json.loads(fd.read())
assert final_cache == {
'version': 1,
'galaxy.server.com:': {
'modified': {
'namespace.collection': responses[0]['modified']
}
}
}
# Reset API
api = get_test_galaxy_api('https://galaxy.server.com/api/', 'v2', no_cache=False)
# Second attempt is successful so cache should be populated
mock_open = MagicMock(
side_effect=[
StringIO(to_text(json.dumps(r)))
for r in responses
]
)
monkeypatch.setattr(galaxy_api, 'open_url', mock_open)
actual_versions = api.get_collection_versions('namespace', 'collection')
assert actual_versions == [u'1.0.0', u'1.0.1', u'1.0.2', u'1.0.3', u'1.0.4', u'1.0.5']
with open(cache_file) as fd:
final_cache = json.loads(fd.read())
cached_server = final_cache['galaxy.server.com:']
cached_collection = cached_server['/api/v2/collections/namespace/collection/versions/']
cached_versions = [r['version'] for r in cached_collection['results']]
assert cached_versions == actual_versions
def redirect_request(self, req, fp, code, msg, hdrs, newurl):
handler = maybe_add_ssl_handler(newurl, validate_certs)
if handler:
urllib_request._opener.add_handler(handler)
# Preserve urllib2 compatibility
if follow_redirects == 'urllib2':
return urllib_request.HTTPRedirectHandler.redirect_request(
self, req, fp, code, msg, hdrs, newurl)
# Handle disabled redirects
elif follow_redirects in ['no', 'none', False]:
raise urllib_error.HTTPError(newurl, code, msg, hdrs, fp)
method = req.get_method()
# Handle non-redirect HTTP status or invalid follow_redirects
if follow_redirects in ['all', 'yes', True]:
if code < 300 or code >= 400:
raise urllib_error.HTTPError(req.get_full_url(), code, msg,
hdrs, fp)
elif follow_redirects == 'safe':
if code < 300 or code >= 400 or method not in ('GET', 'HEAD'):
raise urllib_error.HTTPError(req.get_full_url(), code, msg,
hdrs, fp)
else:
raise urllib_error.HTTPError(req.get_full_url(), code, msg,
hdrs, fp)
try:
# Python 2-3.3
data = req.get_data()
origin_req_host = req.get_origin_req_host()
except AttributeError:
# Python 3.4+
data = req.data
origin_req_host = req.origin_req_host
# Be conciliant with URIs containing a space
newurl = newurl.replace(' ', '%20')
# Suport redirect with payload and original headers
if code in (307, 308):
# Preserve payload and headers
headers = req.headers
else:
# Do not preserve payload and filter headers
data = None
headers = dict(
(k, v) for k, v in req.headers.items()
if k.lower() not in ("content-length", "content-type",
"transfer-encoding"))
# http://tools.ietf.org/html/rfc7231#section-6.4.4
if code == 303 and method != 'HEAD':
method = 'GET'
# Do what the browsers do, despite standards...
# First, turn 302s into GETs.
if code == 302 and method != 'HEAD':
method = 'GET'
# Second, if a POST is responded to with a 301, turn it into a GET.
if code == 301 and method == 'POST':
method = 'GET'
return RequestWithMethod(
newurl,
method=method,
headers=headers,
data=data,
origin_req_host=origin_req_host,
unverifiable=True,
)
