def redirect_request(self, req, fp, code, msg, hdrs, newurl): handler = maybe_add_ssl_handler(newurl, validate_certs) if handler: urllib_request._opener.add_handler(handler) if follow_redirects == 'urllib2': return urllib_request.HTTPRedirectHandler.redirect_request( self, req, fp, code, msg, hdrs, newurl) elif follow_redirects in ['no', 'none', False]: raise urllib_error.HTTPError(newurl, code, msg, hdrs, fp) do_redirect = False if follow_redirects in ['all', 'yes', True]: do_redirect = (code >= 300 and code < 400) elif follow_redirects == 'safe': m = req.get_method() do_redirect = (code >= 300 and code < 400 and m in ('GET', 'HEAD')) if do_redirect: # be conciliant with URIs containing a space newurl = newurl.replace(' ', '%20') newheaders = dict( (k, v) for k, v in req.headers.items() if k.lower() not in ("content-length", "content-type")) return urllib_request.Request( newurl, headers=newheaders, origin_req_host=req.get_origin_req_host(), unverifiable=True) else: raise urllib_error.HTTPError(req.get_full_url(), code, msg, hdrs, fp)
def test_initialise_unknown(monkeypatch): mock_open = MagicMock() mock_open.side_effect = [ urllib_error.HTTPError('https://galaxy.ansible.com/api/', 500, 'msg', {}, StringIO(u'{"msg":"raw error"}')), urllib_error.HTTPError('https://galaxy.ansible.com/api/api/', 500, 'msg', {}, StringIO(u'{"msg":"raw error"}')), ] monkeypatch.setattr(galaxy_api, 'open_url', mock_open) api = GalaxyAPI(None, "test", "https://galaxy.ansible.com/api/", token=GalaxyToken(token='my_token')) expected = "Error when finding available api versions from test (%s) (HTTP Code: 500, Message: msg)" \ % api.api_server with pytest.raises(AnsibleError, match=re.escape(expected)): api.authenticate("github_token")
def test_initialise_automation_hub(monkeypatch): mock_open = MagicMock() mock_open.side_effect = [ urllib_error.HTTPError('https://galaxy.ansible.com/api', 401, 'msg', {}, StringIO()), # AH won't return v1 but we do for authenticate() to work. StringIO(u'{"available_versions":{"v1":"/api/v1","v3":"/api/v3"}}'), StringIO(u'{"token":"my token"}'), ] monkeypatch.setattr(galaxy_api, 'open_url', mock_open) api = GalaxyAPI(None, "test", "https://galaxy.ansible.com", token=GalaxyToken(token='my_token')) actual = api.authenticate("github_token") assert len(api.available_api_versions) == 2 assert api.available_api_versions['v1'] == u'/api/v1' assert api.available_api_versions['v3'] == u'/api/v3' assert actual == {u'token': u'my token'} assert mock_open.call_count == 3 assert mock_open.mock_calls[0][1][0] == 'https://galaxy.ansible.com/api' assert mock_open.mock_calls[0][2]['headers'] == { 'Authorization': 'Token my_token' } assert mock_open.mock_calls[1][1][0] == 'https://galaxy.ansible.com/api' assert mock_open.mock_calls[1][2]['headers'] == { 'Authorization': 'Bearer my_token' } assert mock_open.mock_calls[2][1][ 0] == 'https://galaxy.ansible.com/api/v1/tokens/' assert mock_open.mock_calls[2][2]['data'] == 'github_token=github_token'
def test_publish_failure_v3_with_json_info_409_conflict( galaxy_server, collection_artifact, monkeypatch): mock_avail_ver = MagicMock() mock_avail_ver.return_value = {'v3': '/api/v3'} monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) artifact_path, mock_open = collection_artifact error_response = { "errors": [ { "code": "conflict.collection_exists", "detail": 'Collection "testing-ansible_testing_content-4.0.4" already exists.', "title": "Conflict.", "status": "409", }, ] } return_content = StringIO(to_text(json.dumps(error_response))) mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 409, 'msg', {}, return_content) expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \ '(HTTP Code: 409, Message: Collection "testing-ansible_testing_content-4.0.4"' \ ' already exists. Code: conflict.collection_exists)' with pytest.raises(AnsibleError, match=re.escape(expected)): collection.publish_collection(artifact_path, galaxy_server, True, 0)
def test_build_requirement_from_name_second_server(monkeypatch): galaxy_server = 'https://galaxy-dev.ansible.com' json_str = artifact_versions_json('namespace', 'collection', ['1.0.1', '1.0.2', '1.0.3'], galaxy_server) mock_open = MagicMock() mock_open.side_effect = (urllib_error.HTTPError( 'https://galaxy.server.com', 404, 'msg', {}, None), StringIO(json_str)) monkeypatch.setattr(collection, 'open_url', mock_open) actual = collection.CollectionRequirement.from_name( 'namespace.collection', ['https://broken.com/', galaxy_server], '>1.0.1', False, True) assert actual.namespace == u'namespace' assert actual.name == u'collection' assert actual.b_path is None assert actual.source == to_text(galaxy_server) assert actual.skip is False assert actual.versions == set([u'1.0.2', u'1.0.3']) assert actual.latest_version == u'1.0.3' assert actual.dependencies is None assert mock_open.call_count == 2 assert mock_open.mock_calls[0][1][ 0] == u"https://broken.com/api/v2/collections/namespace/collection/versions/" assert mock_open.mock_calls[0][2] == {'validate_certs': False} assert mock_open.mock_calls[1][1][ 0] == u"%s/api/v2/collections/namespace/collection/versions/" % galaxy_server assert mock_open.mock_calls[1][2] == {'validate_certs': False}
def test_build_requirement_from_name_401_unauthorized(galaxy_server, monkeypatch, tmp_path_factory): mock_open = MagicMock() mock_open.side_effect = api.GalaxyError( urllib_error.HTTPError('https://galaxy.server.com', 401, 'msg', {}, StringIO()), "error") monkeypatch.setattr(galaxy_server, 'get_collection_versions', mock_open) test_dir = to_bytes( tmp_path_factory.mktemp('test-ÅÑŚÌβŁÈ Collections Input')) concrete_artifact_cm = collection.concrete_artifact_manager.ConcreteArtifactsManager( test_dir, validate_certs=False) cli = GalaxyCLI(args=[ 'ansible-galaxy', 'collection', 'install', 'namespace.collection:>1.0.1' ]) requirements = cli._require_one_of_collections_requirements( ['namespace.collection'], None, artifacts_manager=concrete_artifact_cm)['collections'] expected = "error (HTTP Code: 401, Message: msg)" with pytest.raises(api.GalaxyError, match=re.escape(expected)): collection._resolve_depenency_map(requirements, [galaxy_server, galaxy_server], concrete_artifact_cm, None, False, False, False)
def test_build_requirement_from_name_second_server(galaxy_server, monkeypatch): mock_get_versions = MagicMock() mock_get_versions.return_value = ['1.0.1', '1.0.2', '1.0.3'] monkeypatch.setattr(galaxy_server, 'get_collection_versions', mock_get_versions) broken_server = copy.copy(galaxy_server) broken_server.api_server = 'https://broken.com/' mock_404 = MagicMock() mock_404.side_effect = api.GalaxyError( urllib_error.HTTPError('https://galaxy.server.com', 404, 'msg', {}, StringIO()), "custom msg") monkeypatch.setattr(broken_server, 'get_collection_versions', mock_404) actual = collection.CollectionRequirement.from_name( 'namespace.collection', [broken_server, galaxy_server], '>1.0.1', False, True) assert actual.namespace == u'namespace' assert actual.name == u'collection' assert actual.b_path is None # assert actual.api == galaxy_server assert actual.skip is False assert actual.versions == set([u'1.0.2', u'1.0.3']) assert actual.latest_version == u'1.0.3' assert actual.dependencies == {} assert mock_404.call_count == 1 assert mock_404.mock_calls[0][1] == ('namespace', 'collection') assert mock_get_versions.call_count == 1 assert mock_get_versions.mock_calls[0][1] == ('namespace', 'collection')
def test_publish_failure(galaxy_server, collection_artifact): artifact_path, mock_open = collection_artifact mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 500, 'msg', {}, StringIO()) expected = 'Error when publishing collection (HTTP Code: 500, Message: Unknown error returned by Galaxy ' \ 'server. Code: Unknown)' with pytest.raises(AnsibleError, match=re.escape(expected)): collection.publish_collection(artifact_path, galaxy_server, True, 0)
def test_publish_failure_with_json_info(galaxy_server, collection_artifact): artifact_path, mock_open = collection_artifact return_content = StringIO(u'{"message":"Galaxy error message","code":"GWE002"}') mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 503, 'msg', {}, return_content) expected = 'Error when publishing collection (HTTP Code: 503, Message: Galaxy error message Code: GWE002)' with pytest.raises(AnsibleError, match=re.escape(expected)): collection.publish_collection(artifact_path, galaxy_server, True, 0)
def test_build_requirement_from_name_401_unauthorized(galaxy_server, monkeypatch): mock_open = MagicMock() mock_open.side_effect = api.GalaxyError(urllib_error.HTTPError('https://galaxy.server.com', 401, 'msg', {}, StringIO()), "error") monkeypatch.setattr(galaxy_server, 'get_collection_versions', mock_open) expected = "error (HTTP Code: 401, Message: Unknown error returned by Galaxy server.)" with pytest.raises(api.GalaxyError, match=re.escape(expected)): collection.CollectionRequirement.from_name('namespace.collection', [galaxy_server, galaxy_server], '*', False)
def test_build_requirement_from_name_missing(galaxy_server, monkeypatch): mock_open = MagicMock() mock_open.side_effect = api.GalaxyError(urllib_error.HTTPError('https://galaxy.server.com', 404, 'msg', {}, StringIO()), "") monkeypatch.setattr(galaxy_server, 'get_collection_versions', mock_open) expected = "Failed to find collection namespace.collection:*" with pytest.raises(AnsibleError, match=expected): collection.CollectionRequirement.from_name('namespace.collection', [galaxy_server, galaxy_server], '*', False, True)
def test_publish_failure(api_version, collection_url, response, expected, collection_artifact, monkeypatch): api = get_test_galaxy_api('https://galaxy.server.com/api/', api_version) expected_url = '%s/api/%s/%s' % (api.api_server, api_version, collection_url) mock_open = MagicMock() mock_open.side_effect = urllib_error.HTTPError(expected_url, 500, 'msg', {}, StringIO(to_text(json.dumps(response)))) monkeypatch.setattr(galaxy_api, 'open_url', mock_open) with pytest.raises(GalaxyError, match=re.escape(to_native(expected % api.api_server))): api.publish_collection(collection_artifact)
def test_build_requirement_from_name_missing(monkeypatch): mock_open = MagicMock() mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 404, 'msg', {}, None) monkeypatch.setattr(collection, 'open_url', mock_open) expected = "Failed to find collection namespace.collection:*" with pytest.raises(AnsibleError, match=expected): collection.CollectionRequirement.from_name( 'namespace.collection', ['https://broken.com/', 'https://broken2.com'], '*', False, True)
def test_publish_failure(galaxy_server, collection_artifact, monkeypatch): mock_avail_ver = MagicMock() mock_avail_ver.return_value = {'v2': '/api/v2'} monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) artifact_path, mock_open = collection_artifact mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 500, 'msg', {}, StringIO()) expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \ '(HTTP Code: 500, Message: Unknown error returned by Galaxy ' \ 'server. Code: Unknown)' with pytest.raises(AnsibleError, match=re.escape(expected)): collection.publish_collection(artifact_path, galaxy_server, True, 0)
def test_build_requirement_from_name_missing(galaxy_server, monkeypatch): mock_open = MagicMock() mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 404, 'msg', {}, None) monkeypatch.setattr(collection, 'open_url', mock_open) mock_avail_ver = MagicMock() mock_avail_ver.return_value = {'v2': '/api/v2'} monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) expected = "Failed to find collection namespace.collection:*" with pytest.raises(AnsibleError, match=expected): collection.CollectionRequirement.from_name( 'namespace.collection', [galaxy_server, galaxy_server], '*', False, True)
def test_get_available_api_versions_v3_auth_required_without_auth( galaxy_server, collection_artifact, monkeypatch): # mock_avail_ver = MagicMock() # mock_avail_ver.side_effect = {api_version: '/api/%s' % api_version} # monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) error_response = { 'code': 'unauthorized', 'detail': 'The request was not authorized' } artifact_path, mock_open = collection_artifact return_content = StringIO(to_text(json.dumps(error_response))) mock_open.side_effect = urllib_error.HTTPError( 'https://galaxy.server.com', 401, 'msg', {'WWW-Authenticate': 'Bearer'}, return_content) with pytest.raises(AnsibleError): collection.get_available_api_versions(galaxy_server)
def test_get_available_api_versions_v3_auth_required_with_auth_on_retry( galaxy_server, collection_artifact, monkeypatch): # mock_avail_ver = MagicMock() # mock_avail_ver.side_effect = {api_version: '/api/%s' % api_version} # monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) error_obj = { 'code': 'unauthorized', 'detail': 'The request was not authorized' } success_obj = { "description": "GALAXY REST API", "current_version": "v1", "available_versions": { "v3": "/api/v3/" }, "server_version": "3.2.4", "version_name": "Doin' it Right", "team_members": [ "chouseknecht", "cutwater", "alikins", "newswangerd", "awcrosby", "tima", "gregdek" ] } artifact_path, mock_open = collection_artifact error_response = StringIO(to_text(json.dumps(error_obj))) success_response = StringIO(to_text(json.dumps(success_obj))) mock_open.side_effect = [ urllib_error.HTTPError('https://galaxy.server.com', 401, 'msg', {'WWW-Authenticate': 'Bearer'}, error_response), success_response, ] try: res = collection.get_available_api_versions(galaxy_server) except AnsibleError as err: print(err) raise assert res == {'v3': '/api/v3/'}
def test_publish_failure_with_json_info(galaxy_server, collection_artifact, monkeypatch): mock_avail_ver = MagicMock() mock_avail_ver.return_value = {'v2': '/api/v2'} monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) artifact_path, mock_open = collection_artifact return_content = StringIO( u'{"message":"Galaxy error message","code":"GWE002"}') mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 503, 'msg', {}, return_content) expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \ '(HTTP Code: 503, Message: Galaxy error message Code: GWE002)' with pytest.raises(AnsibleError, match=re.escape(expected)): collection.publish_collection(artifact_path, galaxy_server, True, 0)
def test_publish_failure_v3_with_json_info_multiple_errors( galaxy_server, collection_artifact, monkeypatch): mock_avail_ver = MagicMock() mock_avail_ver.return_value = {'v3': '/api/v3'} monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) artifact_path, mock_open = collection_artifact error_response = { "errors": [ { "code": "conflict.collection_exists", "detail": 'Collection "mynamespace-mycollection-4.1.1" already exists.', "title": "Conflict.", "status": "400", }, { "code": "quantum_improbability", "title": "Random(?) quantum improbability.", "source": { "parameter": "the_arrow_of_time" }, "meta": { "remediation": "Try again before" } }, ] } return_content = StringIO(to_text(json.dumps(error_response))) mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 400, 'msg', {}, return_content) expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \ '(HTTP Code: 400, Message: Collection "mynamespace-mycollection-4.1.1"' \ ' already exists. Code: conflict.collection_exists),' \ ' (HTTP Code: 400, Message: Random(?) quantum improbability. Code: quantum_improbability)' with pytest.raises(AnsibleError, match=re.escape(expected)): collection.publish_collection(artifact_path, galaxy_server, True, 0)
def test_build_requirement_from_name_second_server(api_version, exp_api_url, galaxy_server, monkeypatch): mock_avail_ver = MagicMock() avail_api_versions = {api_version: '/api/%s' % api_version} mock_avail_ver.return_value = avail_api_versions monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) json_str = artifact_versions_json('namespace', 'collection', ['1.0.1', '1.0.2', '1.0.3'], galaxy_server, avail_api_versions) mock_open = MagicMock() mock_open.side_effect = (urllib_error.HTTPError( 'https://galaxy.server.com', 404, 'msg', {}, None), StringIO(json_str)) monkeypatch.setattr(collection, 'open_url', mock_open) broken_server = copy.copy(galaxy_server) broken_server.api_server = 'https://broken.com/' actual = collection.CollectionRequirement.from_name( 'namespace.collection', [broken_server, galaxy_server], '>1.0.1', False, True) assert actual.namespace == u'namespace' assert actual.name == u'collection' assert actual.b_path is None # assert actual.api == galaxy_server assert actual.skip is False assert actual.versions == set([u'1.0.2', u'1.0.3']) assert actual.latest_version == u'1.0.3' assert actual.dependencies is None assert mock_open.call_count == 2 assert mock_open.mock_calls[0][1][ 0] == u"https://broken.com%s" % exp_api_url assert mock_open.mock_calls[1][1][0] == u"%s%s" % ( galaxy_server.api_server, exp_api_url) assert mock_open.mock_calls[1][2] == { 'validate_certs': True, "headers": {} }
def test_build_requirement_from_name_401_unauthorized(api_version, errors_to_return, expected, galaxy_server, monkeypatch): mock_avail_ver = MagicMock() available_api_versions = {api_version: '/api/%s' % api_version} mock_avail_ver.return_value = available_api_versions monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) json_str = error_json(galaxy_server, errors_to_return=errors_to_return, available_api_versions=available_api_versions) mock_open = MagicMock() monkeypatch.setattr(collection, 'open_url', mock_open) mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 401, 'msg', {}, StringIO(json_str)) with pytest.raises(AnsibleError, match=expected): collection.CollectionRequirement.from_name( 'namespace.collection', [galaxy_server, galaxy_server], '*', False)
def test_cache_flaky_pagination(cache_dir, monkeypatch): responses = get_collection_versions() cache_file = os.path.join(cache_dir, 'api.json') api = get_test_galaxy_api('https://galaxy.server.com/api/', 'v2', no_cache=False) # First attempt, fail midway through mock_open = MagicMock( side_effect=[ StringIO(to_text(json.dumps(responses[0]))), StringIO(to_text(json.dumps(responses[1]))), urllib_error.HTTPError(responses[1]['next'], 500, 'Error', {}, StringIO()), StringIO(to_text(json.dumps(responses[3]))), ] ) monkeypatch.setattr(galaxy_api, 'open_url', mock_open) expected = ( r'Error when getting available collection versions for namespace\.collection ' r'from test \(https://galaxy\.server\.com/api/\) ' r'\(HTTP Code: 500, Message: Error Code: Unknown\)' ) with pytest.raises(GalaxyError, match=expected): api.get_collection_versions('namespace', 'collection') with open(cache_file) as fd: final_cache = json.loads(fd.read()) assert final_cache == { 'version': 1, 'galaxy.server.com:': { 'modified': { 'namespace.collection': responses[0]['modified'] } } } # Reset API api = get_test_galaxy_api('https://galaxy.server.com/api/', 'v2', no_cache=False) # Second attempt is successful so cache should be populated mock_open = MagicMock( side_effect=[ StringIO(to_text(json.dumps(r))) for r in responses ] ) monkeypatch.setattr(galaxy_api, 'open_url', mock_open) actual_versions = api.get_collection_versions('namespace', 'collection') assert actual_versions == [u'1.0.0', u'1.0.1', u'1.0.2', u'1.0.3', u'1.0.4', u'1.0.5'] with open(cache_file) as fd: final_cache = json.loads(fd.read()) cached_server = final_cache['galaxy.server.com:'] cached_collection = cached_server['/api/v2/collections/namespace/collection/versions/'] cached_versions = [r['version'] for r in cached_collection['results']] assert cached_versions == actual_versions
def redirect_request(self, req, fp, code, msg, hdrs, newurl): handler = maybe_add_ssl_handler(newurl, validate_certs) if handler: urllib_request._opener.add_handler(handler) # Preserve urllib2 compatibility if follow_redirects == 'urllib2': return urllib_request.HTTPRedirectHandler.redirect_request( self, req, fp, code, msg, hdrs, newurl) # Handle disabled redirects elif follow_redirects in ['no', 'none', False]: raise urllib_error.HTTPError(newurl, code, msg, hdrs, fp) method = req.get_method() # Handle non-redirect HTTP status or invalid follow_redirects if follow_redirects in ['all', 'yes', True]: if code < 300 or code >= 400: raise urllib_error.HTTPError(req.get_full_url(), code, msg, hdrs, fp) elif follow_redirects == 'safe': if code < 300 or code >= 400 or method not in ('GET', 'HEAD'): raise urllib_error.HTTPError(req.get_full_url(), code, msg, hdrs, fp) else: raise urllib_error.HTTPError(req.get_full_url(), code, msg, hdrs, fp) try: # Python 2-3.3 data = req.get_data() origin_req_host = req.get_origin_req_host() except AttributeError: # Python 3.4+ data = req.data origin_req_host = req.origin_req_host # Be conciliant with URIs containing a space newurl = newurl.replace(' ', '%20') # Suport redirect with payload and original headers if code in (307, 308): # Preserve payload and headers headers = req.headers else: # Do not preserve payload and filter headers data = None headers = dict( (k, v) for k, v in req.headers.items() if k.lower() not in ("content-length", "content-type", "transfer-encoding")) # http://tools.ietf.org/html/rfc7231#section-6.4.4 if code == 303 and method != 'HEAD': method = 'GET' # Do what the browsers do, despite standards... # First, turn 302s into GETs. if code == 302 and method != 'HEAD': method = 'GET' # Second, if a POST is responded to with a 301, turn it into a GET. if code == 301 and method == 'POST': method = 'GET' return RequestWithMethod( newurl, method=method, headers=headers, data=data, origin_req_host=origin_req_host, unverifiable=True, )