def get_soup(source: str, headers: dict = None, verify: bool = True, from_encoding: str = None, timeout=20) -> BeautifulSoup: """Get soup from website. Args: source (str): Website url. headers (dict, optional): Headers to be used for request. Defaults to general one. verify (bool, optional): Verify source URL. Defaults to True. from_encoding (str, optional): Encoding to use. Defaults to None. timeout (int, optional): If no response is received after `timeout` seconds, exception is raied. Defaults to 20. Returns: BeautifulSoup: Website soup. """ if headers is None: headers = get_headers() try: response = requests.get(source, headers=headers, verify=verify, timeout=timeout) except Exception as err: raise err if not response.ok: raise HTTPError("Web {} not found! {response.content}") content = response.content return BeautifulSoup(content, "html.parser", from_encoding=from_encoding)
def test_application_error(self): ''' Facebook errors often look like 500s Its a silly system, but we need to support it This is actually an application error ''' from io import StringIO graph = self.guy.graph() with mock.patch('urllib2.build_opener') as patched: from urllib.error import HTTPError opener = mock.MagicMock() response = StringIO('''{ "error": { "type": "OAuthException", "message": "Error validating access token: USER_ID has not authorized application APP_ID" } }''') opener.open.side_effect = HTTPError( 'bla', 500, 'bla', 'bla', response) patched.return_value = opener def make_request(): graph.get('me') self.assertRaises(facebook_exceptions.OAuthException, make_request)
def _http_request(url): ''' Perform an HTTP request and return request ''' log('Request URL: {url}', url=url) filename = url.split('/')[-1] for retry in (False, True): try: req = urlopen(url) log('Response code: {code}', code=req.getcode()) if 400 <= req.getcode() < 600: raise HTTPError('HTTP %s Error for url: %s' % (req.getcode(), url), response=req) break except HTTPError: Dialog().ok(localize(30004), localize( 30013, filename=filename)) # Failed to retrieve file return None except BadStatusLine: if retry: Dialog().ok( localize(30004), localize(30013, filename=filename)) # Failed to retrieve file return None return req
def _http_request(url): """Perform an HTTP request and return request""" try: # Python 3 from urllib.error import HTTPError from urllib.request import urlopen except ImportError: # Python 2 from urllib2 import HTTPError, urlopen log('Request URL: {url}', url=url) filename = url.split('/')[-1] try: req = urlopen(url, timeout=5) log('Response code: {code}', code=req.getcode()) if 400 <= req.getcode() < 600: raise HTTPError('HTTP %s Error for url: %s' % (req.getcode(), url), response=req) except HTTPError: ok_dialog(localize(30004), localize(30013, filename=filename)) # Failed to retrieve file return None return req
def mock_raise_httperror(*args, **kwargs): print("mock_raise_httperror\nargs: %s\nkwargs: %s" % (args, kwargs)) raise HTTPError(url="http://fake.come", code=101, msg="Fake HTTPError from Mock", hdrs="Foo", fp=StringIO("Bar"))
def side_effect(*args, **kwargs): response = StringIO(''' <title>Facebook | Error</title> Sorry, something went wrong. ''') http_exception = HTTPError('bla', 505, 'bla', 'bla', response) raise http_exception
def get_url(url, user=None, password=None): """ Fetch a url and return the content as a byte array :param url: the url to go and fetch :param user: optional http username to apply to the request :param password: optional http password to apply to the request :return: byte array containing the file. """ retry = 0 max_retry = int(os.getenv("DOWNLOAD_RETRY", "3")) min_delay = int(os.getenv("DOWNLOAD_MIN_WAIT", "60")) max_delay = int(os.getenv("DOWNLOAD_MAX_WAIT", "6000")) while retry < max_retry: retry += 1 r = requests.get(url, auth=(user, password)) if not r.ok: if r.status_code == 429: delay = randint(min_delay, max_delay) logging.error( f"Too many requests. {r.status_code} {r.content.decode('utf-8')}" ) logging.info(f"sleeping for {delay} seconds") sleep(delay) logging.info("trying again...") else: logging.error( f"could not make request {r.status_code} {r.content.decode('utf-8')}" ) raise HTTPError(f"could not make request {r.status_code}") else: return r
def mock_raise_503_httperror(*args, **kwargs): print("mock_raise_httperror\nargs: %s\nkwargs: %s" % (args, kwargs)) raise HTTPError(url="http://fake.come", code=503, msg="Service unavailable", hdrs="Foo", fp=StringIO("Bar"))
def test_client_parse_error_file(): dbbuddy = Db.DbBuddy() client = Db.GenericClient(dbbuddy) assert not client.parse_error_file() assert not dbbuddy.failures client.http_errors_file.write("Casp9\n%s\n//\n" % HTTPError( "101", "Fake HTTPError from Mock", "Foo", "Bar", StringIO("Baz"))) client.http_errors_file.write("Inx1\n%s\n//\n" % URLError("Fake URLError from Mock")) assert client.parse_error_file() == '''Casp9 HTTP Error Fake HTTPError from Mock: Foo Inx1 <urlopen error Fake URLError from Mock> ''' assert len(dbbuddy.failures) == 2 # Repeat to make sure that the same error is not added again client.http_errors_file.write("Inx1\n%s\n//\n" % URLError("Fake URLError from Mock")) assert not client.parse_error_file() assert len(dbbuddy.failures) == 2
def import_source_locations( vial_http: urllib3.connectionpool.ConnectionPool, import_run_id: str, import_locations: Iterable[load.ImportSourceLocation], import_batch_size: int = 500, ) -> None: """Import source locations""" for import_locations_batch in misc.batch(import_locations, import_batch_size): encoded_ndjson = "\n".join( [loc.json(exclude_none=True) for loc in import_locations_batch]) rsp = vial_http.request( "POST", f"/api/importSourceLocations?import_run_id={import_run_id}", headers={ **vial_http.headers, "Content-Type": "application/x-ndjson" }, body=encoded_ndjson.encode("utf-8"), ) if rsp.status != 200: raise HTTPError( f"/api/importSourceLocations?import_run_id={import_run_id}", rsp.status, rsp.data[:100], dict(rsp.headers), None, )
def test_response_code(self, mocker, test_image_url, code, reason): side_effect = HTTPError(test_image_url, code, reason, {}, None) mocker.patch(make_mock_target("misc", "urlopen"), side_effect=side_effect) with pytest.raises(RuntimeError): misc.download_file(test_image_url)
def check_response(response): res_json = xmltodict.parse(response.text) if response.status_code != 200: raise HTTPError(response.url, response.status_code, res_json['ResponseStatus']['subStatusCode'], None, None) return res_json
def download_from_url(url, dest_file): """ Attempt to download file specified by url to 'dest_file' Raises: WrongFileTypeException when content-type is not in the supported types or cannot be derived from the URL FileExceptionsException If the filename (derived from the URL) already exists in the destination directory. HTTPError ... """ # Don't download files multiple times! if pathexists(dest_file): raise FileExistsException('%s already downloaded.' % dest_file.split('/')[-1]) response = request(url) info = response.info() actual_url = response.url if actual_url == 'http://i.imgur.com/removed.png': raise HTTPError(actual_url, 404, "Imgur suggests the image was removed", None, None) # Work out file type either from the response or the url. if 'content-type' in list(info.keys()): filetype = info['content-type'] elif url.endswith('.jpg') or url.endswith('.jpeg'): filetype = 'image/jpeg' elif url.endswith('.png'): filetype = 'image/png' elif url.endswith('.gif'): filetype = 'image/gif' elif url.endswith('.mp4'): filetype = 'video/mp4' elif url.endswith('.webm'): filetype = 'video/webm' else: filetype = 'unknown' # Only try to download acceptable image types if filetype not in [ 'image/jpeg', 'image/png', 'image/gif', 'video/webm', 'video/mp4' ]: raise WrongFileTypeException('WRONG FILE TYPE: %s has type: %s!' % (url, filetype)) filedata = response.read() filehandle = open(dest_file, 'wb') filehandle.write(filedata) filehandle.close()
def test_handler_handles_http_error(mocker, event): """Ensures any HTTP client errors get handled""" mocker.patch( "service.urlopen", side_effect=HTTPError("http://example.com", 500, "Internal Error", {}, None), ) assert handler(event, None) is False
def test_http_error(): with patch("urllib.request.urlopen") as mock: mock.return_value.__enter__.return_value.read.side_effect = HTTPError( "https://itunes.apple.com/search", 503, "Service Unavailable", {}, None) with pytest.raises(Exception) as exc: http.get("https://itunes.apple.com/search", {"term": "python"}) assert str(exc.value) == "HTTP error 503: Service Unavailable"
async def raise_bytes_exception(scope: Scope, _info: Info, _matches: RouteMatches, _content: Content) -> HttpResponse: """A request handler which raises an exception with bytes content""" raise HTTPError(make_url(scope), 401, b'Unauthorized - bytes', [(b'content-type', b'text/plain')], None) # pylint: disable=unreachable return 204
def test_401(self): obj = BlazeMeterUploader() obj.engine = EngineEmul() mock = BZMock(obj._user) mock.mock_get.update({ 'https://a.blazemeter.com/api/v4/web/version': HTTPError(None, None, None, None, None, ), }) self.assertRaises(HTTPError, obj.prepare)
def check_response(response): if response.status_code != 200: raise HTTPError(response.url, response.status_code, msg=response.text, hdrs=response.headers, fp="") return True
def test_git_get_tag_release_description_fails_must_raise_http_error(self, mock_git_get_tag_release_description, mock_git_create_merge_request, mock_git_accept_merge_request): mock_git_get_tag_release_description.side_effect = HTTPError('url', 'cde', 'msg', 'hdrs', 'fp') with self.assertRaises(HTTPError): create_auto_merge_request('gitlab_endpoint', 'gitlab_token', 'project_id', 'source_branch', 'target_branch', ['user1'], 'tag_name')
def test_404(self): with mock.patch('pandas.read_html') as readmock: readmock.side_effect = HTTPError('http://example.org', 404, "fake error message", None, None) wf_module = fetch(url='http://example.org') result = render(wf_module) self.assertEqual(result, ProcessResult(error='Page not found (404)'))
def test_remote_vocab_error(MockGraph): mock_graph = Mock() mock_graph.parse.side_effect = HTTPError('http://example.org/foo/', 503, '', {}, None) MockGraph.return_value = mock_graph # failure to retrieve the vocabulary over HTTP should # raise a ValidationError pytest.raises(ValidationError, get_vocabulary, 'http://example.org/foo/')
def test_scrape_raises_http_error(): with mock.patch('scraper.urlopen') as urlopen_mock: urlopen_mock.side_effect = HTTPError('http://example.org', 404, 'Not found', {}, mock.Mock()) with pytest.raises(HTTPError) as exc: scraper.scrape('http://example.org') assert exc.value.code == 404 assert exc.value.msg == 'Not found'
def raise_for_status(self) -> None: """Raises :class:`HTTPError`, if one occurred.""" if 400 <= self.status_code: from urllib.error import HTTPError reason_text = self._get_reason_text() http_error_msg = f'{self.status_code} {"Client" if self.status_code < 500 else "Server"} Error: {reason_text} for url: {self.url}' raise HTTPError(self.url, self.status_code, http_error_msg, self.headers, None)
def test_api_req_error(self): self.mocks['api_req'].side_effect = HTTPError(None, 403, None, None, None) with pytest.raises(HTTPError): self.set_assignee() self.assert_api_req_call() self.mocks['post_comment'].assert_not_called()
def test_get_version_from_bad_url_open(self, urlopen_mock): urlopen_mock.side_effect = HTTPError(None, None, None, None, None) base_image = vmimage.ImageProviderBase(version="[0-9]+", build=None, arch=None) with self.assertRaises(vmimage.ImageProviderError) as exc: base_image.get_version() self.assertIn("Cannot open", exc.exception.args[0])
def test_value_error_raise_http_error(monkeypatch): monkeypatch.setattr( urllib.request, "urlopen", MagicMock(side_effect=HTTPError(404, "NF", "", "", "")), ) with pytest.raises(ValueError, match="Unreachable https://NF"): count_dots_on_i("https://NF")
def test_reload_unexpected_url(self): """ If the error URL is not the root, the error is propagated. """ error = HTTPError(self.api.url, 503, "Service Unavailable", {}, None) error.url = "/foo" self.fakes.jenkins.responses[urljoin(self.api.url, "reload")] = error self.assertRaises(HTTPError, self.api.reload)
def login(request, next_page=None, required=False): """Forwards to CAS login URL or verifies CAS ticket Modified locally: honour next=??? in query string, don't deliver a message, catch HTTPError and IOError, generate LogEntry """ if not next_page and 'next' in request.GET: next_page = request.GET['next'] if not next_page: next_page = _redirect_url(request) if request.user.is_authenticated: #message = "You are logged in as %s." % request.user.username #messages.success(request, message) return HttpResponseRedirect(next_page) ticket = request.GET.get('ticket') service = _service_url(request, next_page) if ticket: from django.contrib import auth try: user = auth.authenticate(ticket=ticket, service=service, request=request) except IOError as e: # Here we want to catch only: connection reset, timeouts, name or service unknown if e.errno in [104, 110, 'socket error']: user = None # HTTPError is a type of OSError, which IOError is an alias for. # Sometimes, the CAS server seems to just return a 500 internal server error. Let's handle that the # same way as the above case. elif isinstance(e, HTTPError): if e.code == 500: user = None else: # Any other HTTPError should bubble up and let us know something horrible has happened. raise HTTPError( "Got an HTTP Error when authenticating. The error is: {0!s}." .format(e)) else: raise IOError("The errno is %r: %s." % (e.errno, str(e))) except ParseError: user = None if user is not None: auth.login(request, user) #LOG EVENT# l = LogEntry(userid=user.username, description=("logged in as %s from %s") % (user.username, ip.get_ip(request)), related_object=user) l.save() return HttpResponseRedirect(next_page) elif settings.CAS_RETRY_LOGIN or required: return HttpResponseRedirect(_login_url(service)) else: error = "<h1>Forbidden</h1><p>Login failed.</p>" return HttpResponseForbidden(error) else: return HttpResponseRedirect(_login_url(service))
def __init__(self, geokey, geoauth, loc, code, searchtype='R', mset=EH_STANDARD, \ nset=EN_STANDARD, fuzzy=FUZ_STANDARD, max_return=None): self.location = loc if type(code) != str: raise TypeError(typerr_msg) if searchtype not in SEARCH_TYPES: raise GeoError(searchtype) extra_args = {'fuzzy': fuzzy} if searchtype == 'RF' else {} rows = max_return if max_return is not None else 1 if searchtype in ['E', 'EH', 'EN', 'EH EN', 'EN EH']: extra_args['name_equals'] = loc if max_return is None: rows = 10 search_res = geonames(loc, key=geokey, auth=geoauth, country=code, maxRows=rows, **extra_args) if search_res.status_code == 200: if search_res: if searchtype in ['R', 'RF', 'E']: self.result = search_res[0] return if searchtype in ['EH EN', 'EH', 'RIH']: fcl = mset else: fcl = nset if searchtype in ['RIH', 'RIN']: if search_res[0].feature_class in fcl and loc in search_res[ 0].address: self.result = search_res[0] else: self.result = None return for res in search_res: if res.feature_class in fcl: self.result = res return if searchtype in ['EH EN', 'EN EH']: self.result = search_res[0] else: self.result = None else: self.result = None else: raise HTTPError(search_res.url, search_res.status_code, 'GeoNames request failed', search_res.headers, None)
def size(self, name): conn = self._get_connection() conn.request('HEAD', self._location + self._get_name(name)) res = conn.getresponse() conn.close() if res.status != 200: raise HTTPError(self._location + name, res.status, res.reason, res.msg, res.fp) return res.getheader('Content-Length')