def test_register_model_403_error(get_project, get_repository): """Verify HTTP 403 is converted to a user-friendly error. Depending on environment configuration, this can happen when attempting to find a repository. See: https://github.com/sassoftware/python-sasctl/issues/39 """ from six.moves.urllib.error import HTTPError from sasctl.exceptions import AuthorizationError from sasctl.tasks import register_model get_project.return_value = {'name': 'Project Name'} get_repository.side_effect = HTTPError(None, 403, None, None, None) # HTTP 403 error when getting repository should throw a user-friendly # AuthorizationError with pytest.raises(AuthorizationError): register_model(None, 'model name', 'project name') # All other errors should be bubbled up get_repository.side_effect = HTTPError(None, 404, None, None, None) with pytest.raises(HTTPError): register_model(None, 'model name', 'project name')
def parse(self, content, p=None): '''parses a URL or a response-string to doc tree''' #HACK. With CSW data need to check for JSON/masked 429s first try: if p == 'fromstring': etp = self._parse_f(content) #parse using string method else: etp = self._parse_p( content, p) #parse normally or using provided parser except HTTPError as he: raise #but this won't happen because LXML pushes HTTP errors up to IO errors except (IOError, http.client.IncompleteRead) as ioe: #if re.search('failed to load HTTP resource', ioe.message): #No longer works on Py3 ioem = str(ioe) if PYVER3 else ioe.message if re.search('failed to load HTTP resource', ioem): raise HTTPError( content, 429, 'IOE. Possible 429 Rate Limiting Error. ' + ioem, None, None) if re.search('IncompleteRead', ioem): raise HTTPError(content, 418, 'IOE. Cannot read. ' + ioem, None, None) raise HTTPError(content, 404, 'IOE. Probable HTTP Error. ' + ioem, None, None) except etree.XMLSyntaxError as xse: if re.search('Document is empty', str(xse)): raise LXMLSyntaxException( 'Response from server is empty') from xse except Exception as e: raise return etp
def parse_usdl_from_url(url): cache_key = '_usdl_info/' + url usdl_info = cache.get(cache_key) if usdl_info is not None: if isinstance(usdl_info, Exception): raise usdl_info else: return usdl_info headers = { "Accept": "text/plain; application/rdf+xml; text/turtle; text/n3" } try: response = requests.get(url, headers=headers) if response.status_code != 200: raise HTTPError(response.url, response.status_code, response.reason, None, None) content_type = parse_mime_type(response.headers.get('content-type'))[0] parser = USDLParser(response.content, content_type) usdl_info = parser.parse() except (requests.ConnectionError, URLError, USDLParseException) as e: cache.set(cache_key, e, 2 * 60 * 60) raise cache.set(cache_key, usdl_info) return usdl_info
def full_text_search(self, store, search_string, options): url = urljoin( self._marketplace_uri, "search/offerings/fulltext/" + urlquote_plus(search_string)) response = requests.get(url, auth=HTTPBasicAuth(self._user, self._passwd)) if response.status_code != 200: raise HTTPError(response.url, response.status_code, response.reason, None, None) parsed_body = etree.fromstring(response.content) jobs = [] with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: for res in parsed_body.xpath(SEARCH_RESULT_XPATH): service = res.xpath(SEARCH_SERVICE_XPATH)[0] url = service.xpath(URL_XPATH)[0].text service_store = res.xpath(SEARCH_STORE_XPATH)[0].get('name') if store != '' and store != service_store: continue try: parsed_usdl = parse_usdl_from_url(url) except: continue jobs += self._parse_offering(res.get('name'), url, parsed_usdl, service_store, options, executor) return {'resources': tuple(job.result() for job in jobs)}
def get_offering_info(self, store, id, options): url = urljoin( self._marketplace_uri, "offering/store/%(store)s/offering/%(offering_id)s" % { "store": urlquote(store), "offering_id": urlquote(id) }) response = requests.get(url, auth=HTTPBasicAuth(self._user, self._passwd)) if response.status_code != 200: raise HTTPError(response.url, response.status_code, response.reason, None, None) parsed_body = etree.fromstring(response.content) url = parsed_body.xpath(URL_XPATH)[0].text parsed_usdl = parse_usdl_from_url(url) self.get_store(store) with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: jobs = self._parse_offering(id, url, parsed_usdl, store, options, executor) concurrent.futures.wait(jobs) return tuple(job.result() for job in jobs)
def get_all_services_from_store(self, store, **options): url = urljoin(self._marketplace_uri, "offering/store/" + urlquote(store) + "/offerings") response = requests.get(url, auth=HTTPBasicAuth(self._user, self._passwd)) if response.status_code != 200: raise HTTPError(response.url, response.status_code, response.reason, None, None) parsed_body = etree.fromstring(response.content) jobs = [] with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: for res in parsed_body.xpath(RESOURCE_XPATH): url = res.xpath(URL_XPATH)[0].text try: parsed_usdl = parse_usdl_from_url(url) except: continue jobs += self._parse_offering(res.get('name'), url, parsed_usdl, store, options, executor) concurrent.futures.wait(jobs) return {'resources': tuple(job.result() for job in jobs)}
def test_download_failed_HTTPError(self, mock_urlopen): mock_urlopen.side_effect = HTTPError(None, None, None, None, None) fake_request = urllib.request.Request('http://fakeurl.com') self.assertRaises(self.glance.RetryableError, self.glance._download_tarball_and_verify, fake_request, 'fake_staging_path')
def request(verb, path, session=None, raw=False, **kwargs): session = session or current_session() if session is None: raise TypeError('No `Session` instance found.') response = session.request(verb, path, **kwargs) if 400 <= response.status_code <= 599: raise HTTPError(response.url, response.status_code, response.text, response.headers, None) try: if raw: return response.json() else: obj = _unwrap(response.json()) # ETag is required to update any object # May not be returned on all responses (e.g. listing multiple objects) if isinstance(obj, RestObj): setattr(obj, '_headers', response.headers) return obj except ValueError: return response.text
def __init__(self, name): """ @param name: URL to be opened @keyword additional_headers: additional HTTP request headers to be added to the call """ try: # Note the removal of the fragment ID. This is necessary, per the HTTP spec req = Request(url=name.split('#')[0]) req.add_header('Accept', 'text/html, application/xhtml+xml') self.data = url_opener.open(req) self.headers = self.data.info() if URIOpener.CONTENT_LOCATION in self.headers: self.location = urljoin( self.data.geturl(), self.headers[URIOpener.CONTENT_LOCATION]) else: self.location = name except HTTPError: e = sys.exc_info()[1] from pyMicrodata import HTTPError msg = BaseHTTPRequestHandler.responses[e.code] raise HTTPError('%s' % msg[1], e.code) except Exception: e = sys.exc_info()[1] from pyMicrodata import MicrodataError raise MicrodataError('%s' % e)
def fake_urlopen(req, **kw): """ Behave like six.moves.urllib.request.urlopen(). Return the contents of local fixture files on disk instead. """ o = urlparse(req.get_full_url()) localfile = os.path.join(FIXTURES_DIR, o.netloc, o.path[1:]) # Try reading the file, and handle some special cases if we get an error. try: with open(localfile): pass except IOError as e: # Raise HTTP 404 errors for non-existent files. if e.errno == errno.ENOENT: url = req.get_full_url() headers = HTTPMessage(StringIO(u'')) raise HTTPError(url, 404, 'Not Found', headers, None) # If URL looked like a directory ("/"), open the file instead. elif e.errno == errno.ENOTDIR: localfile = localfile.rstrip('/') # If localfile's a directory, look for a matching ".body" file instead. elif e.errno == errno.EISDIR: localfile = localfile.rstrip('/') + '.body' else: raise response = urlopen('file://' + localfile) return response
def request(self, method, url, *args, **kwargs): if method != 'GET': raise HTTPError('url', '405', 'Method not allowed', None, None) parsed_url = urlparse(url) final_path = os.path.normpath( os.path.join(self.base_path, unquote(parsed_url.path[1:]))) if final_path.startswith( self.base_path) and os.path.isfile(final_path): f = codecs.open(final_path, 'rb') contents = f.read() f.close() return { 'headers': { 'Content-Type': mimetypes.guess_type(final_path, strict=False)[0], 'Content-Length': len(contents), }, 'content': contents, } else: return {'status_code': 404, 'reason': 'Not Found'}
def test_pd_api_missing(self, mock_urlopen): mock_urlopen.side_effect = HTTPError('/', 404, 'Not Found', None, BytesIO(b'{}')) response = self.endpoint._pd_api('/test') self.assertIsNone(response)
def test_app_http_error_retry_fail(self, urlopen_mock): urlopen_mock.side_effect = HTTPError("http://mock.test", 500, "HTTP 500 whatever", None, None) with httmock.HTTMock(*_swagger_spec_mock_): with self.assertRaises(APIException): self.app = EsiApp(cache_prefix='esipy_test') self.assertEqual(urlopen_mock.call_count, 3)
def _request_response(self, method, url, **kwargs): """Internal method to perform request and verify reponse. Parameters ---------- method : `str` name of the method to use (e.g. ``'GET'``). url : `str` remote URL to query. **kwargs other keyword arguments are passed to :meth:`http.client.HTTPConnection.request`. Returns ------- response : `str` reponse from server query Raises ------ RuntimeError if query is unsuccessful """ self.request(method, url, **kwargs) response = self.getresponse() if response.status != 200: raise HTTPError(url, response.status, response.reason, response.getheaders(), response.fp) return response
def test_request_http_error(self): self.mock_urlopen.side_effect = HTTPError( self.url, 500, 'broke', {}, build_file_mock('oops'), ) response = client.request(self.url) expected = client.Response(500, 'broke', 'oops') assert_equal(response, expected)
def test_get_version_from_bad_url_open(self, urlopen_mock): urlopen_mock.side_effect = HTTPError(None, None, None, None, None) base_image = vmimage.ImageProviderBase(version='[0-9]+', build=None, arch=None) with self.assertRaises(vmimage.ImageProviderError) as exc: base_image.get_version() self.assertIn('Cannot open', exc.exception.args[0])
def __call__(self): # pragma: no cover from six.moves.urllib.error import HTTPError raise HTTPError( 'http://nohost/plone/internal_server_error', 500, 'InternalServerError', {}, None ) raise HTTPError
def test_get_attachments_404(self): importer = tracker.GitHubTrackerImporter() extractor = mock.Mock() extractor.urlopen.side_effect = HTTPError('url', 404, 'mock', None, None) body = 'hello\n' \ '![cdbpzjc5ex4](https://f.cloud.github.com/assets/979771/1027411/a393ab5e-0e70-11e3-8a38-b93a3df904cf.jpg)\r\n' new_body, attachments = importer._get_attachments(extractor, body) self.assertIsNotNone(attachments[0]) assert not hasattr(attachments[0], 'file')
def __call__(self): from six.moves.urllib.error import HTTPError raise HTTPError( "http://nohost/plone/internal_server_error", 500, "InternalServerError", {}, None, )
def test_httperror_raises_error(self, mocked_urlopen, urllib_requester): url = 'http://test.com/api/1' code = 777 msg = 'msg' hdrs = 'hdrs' fp = MockFile(url) mocked_urlopen.side_effect = HTTPError(url, code, msg, hdrs, fp) with pytest.raises(WykopAPIError): urllib_requester.make_request(url)
def test_connection_error(self): # Test connection error self.http_mock.side_effect = HTTPError('http://example.cz/error/', 500, 'Error message', {'Content-Type': 'text/plain'}, StringIO('BODY')) response = self.fetcher.fetch('http://example.cz/error/') expected = fetchers.HTTPResponse('http://example.cz/error/', 500, {'Content-Type': 'text/plain'}, 'BODY') assertResponse(expected, response)
def test_request_http_error(self, _): self.mock_urlopen.side_effect = HTTPError( self.url, 500, 'broke', mock.Mock(get_content_charset=mock.Mock(return_value='utf-8'), ), build_file_mock(b'oops'), ) response = client.request(self.url) expected = client.Response(500, 'broke', 'oops') assert_equal(response, expected)
def _mock_urlopen(url): # print "Mock opening", url package = url.replace('https://pypi.org/simple/', '') if package not in mock_pypi_available: raise HTTPError( url, 404, 'HTTP Error 404: Not Found (%s does not have any releases)' % package, None, None) else: answer = ' '.join(mock_pypi_available) return StringIO(answer)
def get_store_info(self, store): url = urljoin(self._marketplace_uri, "registration/store/" + urlquote(store)) response = requests.get(url, auth=HTTPBasicAuth(self._user, self._passwd)) if response.status_code != 200: raise HTTPError(response.url, response.status_code, response.reason, None, None) parsed_body = etree.fromstring(response.content) return self._process_store_info(parsed_body)
def mock_urlopen(url, context=None): """ Return an on-disk JSON file's contents for a given url. """ filename = os.path.basename(url) if not filename.endswith('.json'): # This is not parsed; it just needs to be any 200 OK response. return StringIO() try: f = open( os.path.join(self.compose_path, "compose", "metadata", filename), 'r') except IOError as e: raise HTTPError(404, e) return f
def query_times(protocol, server, ifo, name, version, request, start, end): flag = '%s:%s:%d' % (ifo, name, version) span = SegmentList([Segment(start, end)]) if flag not in result: raise HTTPError('test-url/', 404, 'Not found', None, None) return { 'ifo': ifo, 'name': name, 'version': version, 'known': list(map(tuple, result[flag].known & span)), 'active': list(map(tuple, result[flag].active & span)), 'query_information': {}, 'metadata': kwargs, }, 'BOGUS_QUERY_STRING'
def test_app_http_error_retry_ok(self, urlopen_mock): http_error = HTTPError("http://mock.test", 500, "HTTP 500 whatever", None, None) # this will make the function raise exception / return the value # in this given order side_effect_results = [ http_error, http_error, open(TestEsiApp.ESI_META_SWAGGER) ] urlopen_mock.side_effect = side_effect_results with httmock.HTTMock(*_swagger_spec_mock_): EsiApp(cache_prefix='esipy_test') self.assertEqual(urlopen_mock.call_count, 3)
def test_request_raises_error(urlopen, request): exception_data = MagicMock() exception_data.read.return_value.decode.return_value = '{"error": 1}' urlopen.side_effect = HTTPError('', '', '', '', exception_data) ygg = YggdrasilCore() assert not urlopen.called assert not request.called res = ygg._ygg_req('/test', {'a': 'b'}) # Read the response assert exception_data.read.called assert exception_data.read.return_value.decode.called assert res == {'error': 1}
def test_galaxy_api_lookup_repo_by_name_500_json_not_dict(mocker, galaxy_api): mocker.patch('ansible_galaxy.rest_api.open_url', side_effect=HTTPError(url='http://whatever', code=500, msg='Stuff broke.', hdrs={}, fp=io.StringIO(initial_value=u'[]'))) try: galaxy_api.lookup_repo_by_name('some-test-namespace', 'some-test-name') except exceptions.GalaxyClientError as e: log.exception(e) log.debug(e) return assert False, 'Excepted to get a HTTPError(code=500) here but did not.'
def redirect_request(self, req, fp, code, msg, headers, newurl): newurl = newurl.replace(' ', '%20') if code in (301, 307): return Request(newurl, data=req.get_data(), headers=req.headers, origin_req_host=req.get_origin_req_host(), unverifiable=True) elif code in (302, 303): newheaders = dict((k, v) for k, v in req.headers.items() if k.lower() not in ("content-length", "content-type")) return Request(newurl, headers=newheaders, origin_req_host=req.get_origin_req_host(), unverifiable=True) else: raise HTTPError(req.get_full_url(), code, msg, headers, fp)