def test_register_model_403_error(get_project, get_repository):
"""Verify HTTP 403 is converted to a user-friendly error.
Depending on environment configuration, this can happen when attempting to
find a repository.
See: https://github.com/sassoftware/python-sasctl/issues/39
"""
from six.moves.urllib.error import HTTPError
from sasctl.exceptions import AuthorizationError
from sasctl.tasks import register_model
get_project.return_value = {'name': 'Project Name'}
get_repository.side_effect = HTTPError(None, 403, None, None, None)
# HTTP 403 error when getting repository should throw a user-friendly
# AuthorizationError
with pytest.raises(AuthorizationError):
register_model(None, 'model name', 'project name')
# All other errors should be bubbled up
get_repository.side_effect = HTTPError(None, 404, None, None, None)
with pytest.raises(HTTPError):
register_model(None, 'model name', 'project name')
def parse(self, content, p=None):
'''parses a URL or a response-string to doc tree'''
#HACK. With CSW data need to check for JSON/masked 429s first
try:
if p == 'fromstring':
etp = self._parse_f(content) #parse using string method
else:
etp = self._parse_p(
content, p) #parse normally or using provided parser
except HTTPError as he:
raise #but this won't happen because LXML pushes HTTP errors up to IO errors
except (IOError, http.client.IncompleteRead) as ioe:
#if re.search('failed to load HTTP resource', ioe.message): #No longer works on Py3
ioem = str(ioe) if PYVER3 else ioe.message
if re.search('failed to load HTTP resource', ioem):
raise HTTPError(
content, 429,
'IOE. Possible 429 Rate Limiting Error. ' + ioem, None,
None)
if re.search('IncompleteRead', ioem):
raise HTTPError(content, 418, 'IOE. Cannot read. ' + ioem,
None, None)
raise HTTPError(content, 404, 'IOE. Probable HTTP Error. ' + ioem,
None, None)
except etree.XMLSyntaxError as xse:
if re.search('Document is empty', str(xse)):
raise LXMLSyntaxException(
'Response from server is empty') from xse
except Exception as e:
raise
return etp
def parse_usdl_from_url(url):
cache_key = '_usdl_info/' + url
usdl_info = cache.get(cache_key)
if usdl_info is not None:
if isinstance(usdl_info, Exception):
raise usdl_info
else:
return usdl_info
headers = {
"Accept": "text/plain; application/rdf+xml; text/turtle; text/n3"
}
try:
response = requests.get(url, headers=headers)
if response.status_code != 200:
raise HTTPError(response.url, response.status_code,
response.reason, None, None)
content_type = parse_mime_type(response.headers.get('content-type'))[0]
parser = USDLParser(response.content, content_type)
usdl_info = parser.parse()
except (requests.ConnectionError, URLError, USDLParseException) as e:
cache.set(cache_key, e, 2 * 60 * 60)
raise
cache.set(cache_key, usdl_info)
return usdl_info
def full_text_search(self, store, search_string, options):
url = urljoin(
self._marketplace_uri,
"search/offerings/fulltext/" + urlquote_plus(search_string))
response = requests.get(url,
auth=HTTPBasicAuth(self._user, self._passwd))
if response.status_code != 200:
raise HTTPError(response.url, response.status_code,
response.reason, None, None)
parsed_body = etree.fromstring(response.content)
jobs = []
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
for res in parsed_body.xpath(SEARCH_RESULT_XPATH):
service = res.xpath(SEARCH_SERVICE_XPATH)[0]
url = service.xpath(URL_XPATH)[0].text
service_store = res.xpath(SEARCH_STORE_XPATH)[0].get('name')
if store != '' and store != service_store:
continue
try:
parsed_usdl = parse_usdl_from_url(url)
except:
continue
jobs += self._parse_offering(res.get('name'), url, parsed_usdl,
service_store, options, executor)
return {'resources': tuple(job.result() for job in jobs)}
def get_offering_info(self, store, id, options):
url = urljoin(
self._marketplace_uri,
"offering/store/%(store)s/offering/%(offering_id)s" % {
"store": urlquote(store),
"offering_id": urlquote(id)
})
response = requests.get(url,
auth=HTTPBasicAuth(self._user, self._passwd))
if response.status_code != 200:
raise HTTPError(response.url, response.status_code,
response.reason, None, None)
parsed_body = etree.fromstring(response.content)
url = parsed_body.xpath(URL_XPATH)[0].text
parsed_usdl = parse_usdl_from_url(url)
self.get_store(store)
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
jobs = self._parse_offering(id, url, parsed_usdl, store, options,
executor)
concurrent.futures.wait(jobs)
return tuple(job.result() for job in jobs)
def get_all_services_from_store(self, store, **options):
url = urljoin(self._marketplace_uri,
"offering/store/" + urlquote(store) + "/offerings")
response = requests.get(url,
auth=HTTPBasicAuth(self._user, self._passwd))
if response.status_code != 200:
raise HTTPError(response.url, response.status_code,
response.reason, None, None)
parsed_body = etree.fromstring(response.content)
jobs = []
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
for res in parsed_body.xpath(RESOURCE_XPATH):
url = res.xpath(URL_XPATH)[0].text
try:
parsed_usdl = parse_usdl_from_url(url)
except:
continue
jobs += self._parse_offering(res.get('name'), url, parsed_usdl,
store, options, executor)
concurrent.futures.wait(jobs)
return {'resources': tuple(job.result() for job in jobs)}
def test_download_failed_HTTPError(self, mock_urlopen):
mock_urlopen.side_effect = HTTPError(None, None, None, None, None)
fake_request = urllib.request.Request('http://fakeurl.com')
self.assertRaises(self.glance.RetryableError,
self.glance._download_tarball_and_verify,
fake_request, 'fake_staging_path')
def request(verb, path, session=None, raw=False, **kwargs):
session = session or current_session()
if session is None:
raise TypeError('No `Session` instance found.')
response = session.request(verb, path, **kwargs)
if 400 <= response.status_code <= 599:
raise HTTPError(response.url, response.status_code, response.text,
response.headers, None)
try:
if raw:
return response.json()
else:
obj = _unwrap(response.json())
# ETag is required to update any object
# May not be returned on all responses (e.g. listing multiple objects)
if isinstance(obj, RestObj):
setattr(obj, '_headers', response.headers)
return obj
except ValueError:
return response.text
def __init__(self, name):
"""
@param name: URL to be opened
@keyword additional_headers: additional HTTP request headers to be added to the call
"""
try:
# Note the removal of the fragment ID. This is necessary, per the HTTP spec
req = Request(url=name.split('#')[0])
req.add_header('Accept', 'text/html, application/xhtml+xml')
self.data = url_opener.open(req)
self.headers = self.data.info()
if URIOpener.CONTENT_LOCATION in self.headers:
self.location = urljoin(
self.data.geturl(),
self.headers[URIOpener.CONTENT_LOCATION])
else:
self.location = name
except HTTPError:
e = sys.exc_info()[1]
from pyMicrodata import HTTPError
msg = BaseHTTPRequestHandler.responses[e.code]
raise HTTPError('%s' % msg[1], e.code)
except Exception:
e = sys.exc_info()[1]
from pyMicrodata import MicrodataError
raise MicrodataError('%s' % e)
def fake_urlopen(req, **kw):
"""
Behave like six.moves.urllib.request.urlopen().
Return the contents of local fixture files on disk instead.
"""
o = urlparse(req.get_full_url())
localfile = os.path.join(FIXTURES_DIR, o.netloc, o.path[1:])
# Try reading the file, and handle some special cases if we get an error.
try:
with open(localfile):
pass
except IOError as e:
# Raise HTTP 404 errors for non-existent files.
if e.errno == errno.ENOENT:
url = req.get_full_url()
headers = HTTPMessage(StringIO(u''))
raise HTTPError(url, 404, 'Not Found', headers, None)
# If URL looked like a directory ("/"), open the file instead.
elif e.errno == errno.ENOTDIR:
localfile = localfile.rstrip('/')
# If localfile's a directory, look for a matching ".body" file instead.
elif e.errno == errno.EISDIR:
localfile = localfile.rstrip('/') + '.body'
else:
raise
response = urlopen('file://' + localfile)
return response
def request(self, method, url, *args, **kwargs):
if method != 'GET':
raise HTTPError('url', '405', 'Method not allowed', None, None)
parsed_url = urlparse(url)
final_path = os.path.normpath(
os.path.join(self.base_path, unquote(parsed_url.path[1:])))
if final_path.startswith(
self.base_path) and os.path.isfile(final_path):
f = codecs.open(final_path, 'rb')
contents = f.read()
f.close()
return {
'headers': {
'Content-Type':
mimetypes.guess_type(final_path, strict=False)[0],
'Content-Length':
len(contents),
},
'content': contents,
}
else:
return {'status_code': 404, 'reason': 'Not Found'}
def test_pd_api_missing(self, mock_urlopen):
mock_urlopen.side_effect = HTTPError('/', 404, 'Not Found', None,
BytesIO(b'{}'))
response = self.endpoint._pd_api('/test')
self.assertIsNone(response)
def test_app_http_error_retry_fail(self, urlopen_mock):
urlopen_mock.side_effect = HTTPError("http://mock.test", 500,
"HTTP 500 whatever", None, None)
with httmock.HTTMock(*_swagger_spec_mock_):
with self.assertRaises(APIException):
self.app = EsiApp(cache_prefix='esipy_test')
self.assertEqual(urlopen_mock.call_count, 3)
def _request_response(self, method, url, **kwargs):
"""Internal method to perform request and verify reponse.
Parameters
----------
method : `str`
name of the method to use (e.g. ``'GET'``).
url : `str`
remote URL to query.
**kwargs
other keyword arguments are passed to
:meth:`http.client.HTTPConnection.request`.
Returns
-------
response : `str`
reponse from server query
Raises
------
RuntimeError
if query is unsuccessful
"""
self.request(method, url, **kwargs)
response = self.getresponse()
if response.status != 200:
raise HTTPError(url, response.status, response.reason,
response.getheaders(), response.fp)
return response
def test_request_http_error(self):
self.mock_urlopen.side_effect = HTTPError(
self.url, 500, 'broke', {}, build_file_mock('oops'),
)
response = client.request(self.url)
expected = client.Response(500, 'broke', 'oops')
assert_equal(response, expected)
def test_get_version_from_bad_url_open(self, urlopen_mock):
urlopen_mock.side_effect = HTTPError(None, None, None, None, None)
base_image = vmimage.ImageProviderBase(version='[0-9]+', build=None, arch=None)
with self.assertRaises(vmimage.ImageProviderError) as exc:
base_image.get_version()
self.assertIn('Cannot open', exc.exception.args[0])
def __call__(self): # pragma: no cover
from six.moves.urllib.error import HTTPError
raise HTTPError(
'http://nohost/plone/internal_server_error',
500,
'InternalServerError',
{},
None
)
raise HTTPError
def test_get_attachments_404(self):
importer = tracker.GitHubTrackerImporter()
extractor = mock.Mock()
extractor.urlopen.side_effect = HTTPError('url', 404, 'mock', None,
None)
body = 'hello\n' \
'\r\n'
new_body, attachments = importer._get_attachments(extractor, body)
self.assertIsNotNone(attachments[0])
assert not hasattr(attachments[0], 'file')
def __call__(self):
from six.moves.urllib.error import HTTPError
raise HTTPError(
"http://nohost/plone/internal_server_error",
500,
"InternalServerError",
{},
None,
)
def test_httperror_raises_error(self, mocked_urlopen, urllib_requester):
url = 'http://test.com/api/1'
code = 777
msg = 'msg'
hdrs = 'hdrs'
fp = MockFile(url)
mocked_urlopen.side_effect = HTTPError(url, code, msg, hdrs, fp)
with pytest.raises(WykopAPIError):
urllib_requester.make_request(url)
def test_connection_error(self):
# Test connection error
self.http_mock.side_effect = HTTPError('http://example.cz/error/', 500,
'Error message',
{'Content-Type': 'text/plain'},
StringIO('BODY'))
response = self.fetcher.fetch('http://example.cz/error/')
expected = fetchers.HTTPResponse('http://example.cz/error/', 500,
{'Content-Type': 'text/plain'},
'BODY')
assertResponse(expected, response)
def test_request_http_error(self, _):
self.mock_urlopen.side_effect = HTTPError(
self.url,
500,
'broke',
mock.Mock(get_content_charset=mock.Mock(return_value='utf-8'), ),
build_file_mock(b'oops'),
)
response = client.request(self.url)
expected = client.Response(500, 'broke', 'oops')
assert_equal(response, expected)
def _mock_urlopen(url):
# print "Mock opening", url
package = url.replace('https://pypi.org/simple/', '')
if package not in mock_pypi_available:
raise HTTPError(
url, 404,
'HTTP Error 404: Not Found (%s does not have any releases)'
% package, None, None)
else:
answer = ' '.join(mock_pypi_available)
return StringIO(answer)
def get_store_info(self, store):
url = urljoin(self._marketplace_uri,
"registration/store/" + urlquote(store))
response = requests.get(url,
auth=HTTPBasicAuth(self._user, self._passwd))
if response.status_code != 200:
raise HTTPError(response.url, response.status_code,
response.reason, None, None)
parsed_body = etree.fromstring(response.content)
return self._process_store_info(parsed_body)
def mock_urlopen(url, context=None):
""" Return an on-disk JSON file's contents for a given url. """
filename = os.path.basename(url)
if not filename.endswith('.json'):
# This is not parsed; it just needs to be any 200 OK response.
return StringIO()
try:
f = open(
os.path.join(self.compose_path, "compose", "metadata",
filename), 'r')
except IOError as e:
raise HTTPError(404, e)
return f
def query_times(protocol, server, ifo, name, version, request, start, end):
flag = '%s:%s:%d' % (ifo, name, version)
span = SegmentList([Segment(start, end)])
if flag not in result:
raise HTTPError('test-url/', 404, 'Not found', None, None)
return {
'ifo': ifo,
'name': name,
'version': version,
'known': list(map(tuple, result[flag].known & span)),
'active': list(map(tuple, result[flag].active & span)),
'query_information': {},
'metadata': kwargs,
}, 'BOGUS_QUERY_STRING'
def test_app_http_error_retry_ok(self, urlopen_mock):
http_error = HTTPError("http://mock.test", 500, "HTTP 500 whatever",
None, None)
# this will make the function raise exception / return the value
# in this given order
side_effect_results = [
http_error, http_error,
open(TestEsiApp.ESI_META_SWAGGER)
]
urlopen_mock.side_effect = side_effect_results
with httmock.HTTMock(*_swagger_spec_mock_):
EsiApp(cache_prefix='esipy_test')
self.assertEqual(urlopen_mock.call_count, 3)
def test_request_raises_error(urlopen, request):
exception_data = MagicMock()
exception_data.read.return_value.decode.return_value = '{"error": 1}'
urlopen.side_effect = HTTPError('', '', '', '', exception_data)
ygg = YggdrasilCore()
assert not urlopen.called
assert not request.called
res = ygg._ygg_req('/test', {'a': 'b'})
# Read the response
assert exception_data.read.called
assert exception_data.read.return_value.decode.called
assert res == {'error': 1}
def test_galaxy_api_lookup_repo_by_name_500_json_not_dict(mocker, galaxy_api):
mocker.patch('ansible_galaxy.rest_api.open_url',
side_effect=HTTPError(url='http://whatever',
code=500,
msg='Stuff broke.',
hdrs={},
fp=io.StringIO(initial_value=u'[]')))
try:
galaxy_api.lookup_repo_by_name('some-test-namespace', 'some-test-name')
except exceptions.GalaxyClientError as e:
log.exception(e)
log.debug(e)
return
assert False, 'Excepted to get a HTTPError(code=500) here but did not.'
def redirect_request(self, req, fp, code, msg, headers, newurl):
newurl = newurl.replace(' ', '%20')
if code in (301, 307):
return Request(newurl,
data=req.get_data(),
headers=req.headers,
origin_req_host=req.get_origin_req_host(),
unverifiable=True)
elif code in (302, 303):
newheaders = dict((k, v) for k, v in req.headers.items()
if k.lower() not in ("content-length", "content-type"))
return Request(newurl,
headers=newheaders,
origin_req_host=req.get_origin_req_host(),
unverifiable=True)
else:
raise HTTPError(req.get_full_url(), code, msg, headers, fp)
