def __init__(self): self.__logger = Logger() self._request_exceptions = [type(item) for item in [requests.ConnectionError(), requests.HTTPError(), requests.TooManyRedirects(), requests.Timeout(), requests.TooManyRedirects(), requests.RequestException(), requests.ConnectTimeout(), requests.ReadTimeout()]] self._system_errors = [type(item) for item in [KeyError(), AttributeError(), IndexError(), ZeroDivisionError(), SystemError(), ValueError(), AssertionError()]] self._file_errors = [type(item) for item in [FileExistsError(), FileNotFoundError()]] self._database_errors = [type(item) for item in [sqlite3.Error(), sqlite3.DataError(), sqlite3.ProgrammingError(), sqlite3.DatabaseError(), sqlite3.NotSupportedError(), sqlite3.IntegrityError(), sqlite3.InterfaceError(), sqlite3.InternalError(), sqlite3.OperationalError()]] self._speech_recognizer_errors = [type(item) for item in [sr.RequestError(), sr.UnknownValueError(), sr.WaitTimeoutError(), sr.RequestError()]] self.__logger.info('ExceptionsHandler was successfully initialized.', __name__)
def test_update_notable_too_many_redirect(self, mocked_requests_post): """ Test update notable with wrong URL or connection issue""" print("Test update_notable returns TooManyRedirects\n") try: sim_status = 1 sim_content = '<response><sessionKey>' + self.simSessionKey + '</sessionKey></response>' mocked_requests_post.return_value = self._generateResponse( sim_content, 200) splnk_utils = splunk_utils.SplunkUtils(host=self.fake_host, port=self.fake_port, username=self.fake_username, password=self.fake_password, verify=self.verify) mocked_requests_post.side_effect = requests.TooManyRedirects( Mock(status=404), "Ambiguous excetpion.") ret = splnk_utils.update_notable(event_id=self.simEventId, comment=self.simComment, status=sim_status, cafile=self.verify) # # request post throws exception # assert False except splunk_utils.RequestError as e: assert True
def test_to_many_redirects_exception(self): with mock.patch('requests.get') as requests_get_mock: requests_get_mock.side_effect = requests.TooManyRedirects() with StdoutCapture() as output: self.api_caller.get_resource("/resource", DiscoveryContainer()) self.assertIn( "ERROR::Get url=http://1.2.3.4:567/resource Error <class 'requests.exceptions.TooManyRedirects'>:;", output.raw)
def test_add_threat_intel_item_errors(self, mocked_requests_post): # 1. Connect successfully sim_content = '<response><sessionKey>' + self.simSessionKey + '</sessionKey></response>' mocked_requests_post.return_value = self._generateResponse( sim_content, 200) splnk_utils = splunk_utils.SplunkUtils(host=self.fake_host, port=self.fake_port, username=self.fake_username, password=self.fake_password, verify=self.verify) # 2. Simulate wrong intel type try: splnk_utils.add_threat_intel_item("Fake type", {}, False) except splunk_utils.RequestError as e: print("Fake intel type causes exception as expected.") assert (True) # 3. Simulate RequestException threat_type = "ip_intel" mocked_requests_post.side_effect = requests.RequestException( Mock(status=404), "Ambiguous excetpion.") try: splnk_utils.add_threat_intel_item(threat_type, {}, False) except splunk_utils.RequestError: assert True # 4. Simulate ConnectionError mocked_requests_post.side_effect = requests.ConnectionError( Mock(status=404), "Ambiguous excetpion.") try: splnk_utils.add_threat_intel_item(threat_type, {}, False) except splunk_utils.RequestError: assert True # 5. Simulate HttpError mocked_requests_post.side_effect = requests.HTTPError( Mock(status=404), "Ambiguous excetpion.") try: splnk_utils.add_threat_intel_item(threat_type, {}, False) except splunk_utils.RequestError: assert True # 6. Simulate URLRequired mocked_requests_post.side_effect = requests.URLRequired( Mock(status=404), "Ambiguous excetpion.") try: splnk_utils.add_threat_intel_item(threat_type, {}, False) except splunk_utils.RequestError: assert True # 7. Simulate TooManyRedirects mocked_requests_post.side_effect = requests.TooManyRedirects( Mock(status=404), "Ambiguous excetpion.") try: splnk_utils.add_threat_intel_item(threat_type, {}, False) except splunk_utils.RequestError: assert True
def test_it_raises_an_exception_on_requests_error(self) -> None: self._request_mock.side_effect = requests.TooManyRedirects( 'Exceeded 10 redirects.') with self.assertRaises(HttpException) as exception_ctx: self._client.request('GET', '/error') self.assertEqual( 'Requests error: None: Exceeded 10 redirects. ' '(see https://requests.kennethreitz.org/en/master/api/#exceptions) for /error', str(exception_ctx.exception))
def get_story(new): """Return a story of the given ID.""" url = URLS['item'].format(new) try: data = req.get(url) except req.ConnectionError: raise except req.Timeout: raise req.Timeout('A timeout problem occurred.') except req.TooManyRedirects: raise req.TooManyRedirects('The request exceeds the configured number\ of maximum redirections.') else: return data.json()
def _request(self, url_suffix, **kwargs): url = '{base}{url_token}{url_suffix}'.format(base=self.base_url, url_suffix=url_suffix, url_token=self.url_token) kwargs['timeout'] = kwargs.get('timeout', self.school.timeout) res = self._http.request(url=url, proxies=self._proxy, allow_redirects=False, **kwargs) res.raise_for_status() if res.status_code == 302: raise requests.TooManyRedirects(response=res) return res
def Spider(self, limit): page = 1 webAppList = [] while page < limit: url = r"https://play.google.com/store/apps/category/GAME/collection/topselling_new_free?hl=en" try: source_Code = requests.get(url, timeout=10) raw_text = source_Code.text soup = BeautifulSoup(raw_text, "html.parser") self.FetchAppLinks(soup, webAppList) except requests.ConnectionError as connError: logger.log("Connection Error while connecting to Play store: ", url, " Error: ", connError) except requests.HTTPError as httpError: logger.log("Invalid HTTP response to Play store: ", url, " Error: ", httpError) except requests.Timeout() as requestTimeoutError: logger.log("Time-out to connect to Play store: ", url, " Error: ", requestTimeoutError) except requests.TooManyRedirects() as redirectsError: logger.log("Too many redirects for connection to Play store: ", url, " Error: ", redirectsError) #except HTMLParseError as htmlParsingError: # logger.log("HTMLParse Error: ", htmlParsingError) except Exception as e: logger.log("Execpetion occured at Func Spider: ", e) except requests.exceptions.Timeout as timeoutException: logger.log("Timeout Exception", timeoutException) page += 1 return webAppList
def test_consultar_cep(self, mock_api_call): # Criamos uma requisicao que sera bem sucedida param = { 'text': self.response_xml, 'ok': True, 'status_code': 200, } mock_api_call.return_value = mock.MagicMock(**param) self.assertDictEqual(consultar_cep('70002900'), self.expected_address) self.assertDictEqual(consultar_cep('70002900', ambiente=HOMOLOGACAO), self.expected_address) self.assertDictEqual(consultar_cep('70002900', ambiente=PRODUCAO), self.expected_address) # Aqui, construÃmos uma requisição que retornará erro param = { 'text': self.response_xml_error, 'ok': False, } mock_api_call.return_value = mock.MagicMock(**param) self.assertRaises(CEPInvalido, consultar_cep, '1232710') mock_api_call.return_value = mock.MagicMock(**param) self.assertRaises(CEPInvalido, consultar_cep, '00000000') self.assertRaises(KeyError, consultar_cep, '37503130', ambiente=3) # Verificamos as demais excecoes mock_api_call.side_effect = requests.ConnectTimeout() self.assertRaises(Timeout, consultar_cep, '37503130') mock_api_call.side_effect = requests.ConnectionError('', '') self.assertRaises(FalhaNaConexao, consultar_cep, '37503130') mock_api_call.side_effect = requests.TooManyRedirects() self.assertRaises(MultiploRedirecionamento, consultar_cep, '37503130') mock_api_call.side_effect = requests.RequestException() self.assertRaises(ExcecaoPyCEPCorreios, consultar_cep, '37503130')
# Disable logging for a cleaner testing output import logging logging.disable(logging.CRITICAL) # Some exception handling we'll use REQUEST_EXCEPTIONS = ( requests.ConnectionError( 0, 'requests.ConnectionError() not handled'), requests.RequestException( 0, 'requests.RequestException() not handled'), requests.HTTPError( 0, 'requests.HTTPError() not handled'), requests.ReadTimeout( 0, 'requests.ReadTimeout() not handled'), requests.TooManyRedirects( 0, 'requests.TooManyRedirects() not handled'), ) @mock.patch('requests.get') @mock.patch('requests.post') def test_config_http(mock_post, mock_get): """ API: ConfigHTTP() object """ # Define our good:// url class GoodNotification(NotifyBase): def __init__(self, *args, **kwargs): super(GoodNotification, self).__init__(*args, **kwargs)
class TestCanvasAPIError: @pytest.mark.parametrize( "status,expected_status,expected_exception_class,expected_exception_string", [ # A 401 Unauthorized response from Canvas, because our access token was # expired or deleted. ( 401, "401 Unauthorized", CanvasAPIAccessTokenError, "401 Client Error: Unauthorized for url: https://example.com/", ), # A 400 Bad Request response from Canvas, because we sent an invalid # parameter or something. ( 400, "400 Bad Request", CanvasAPIServerError, "400 Client Error: Bad Request for url: https://example.com/", ), # An unexpected error response from Canvas. ( 500, "500 Internal Server Error", CanvasAPIServerError, "500 Server Error: Internal Server Error for url: https://example.com/", ), ], ) def test_it_raises_the_right_subclass_for_different_Canvas_responses( self, status, expected_status, expected_exception_class, expected_exception_string, ): cause = self._requests_exception(status=status) raised_exception = self.assert_raises(cause, expected_exception_class) assert raised_exception.__cause__ == cause assert raised_exception.response == cause.response assert raised_exception.details == { "exception": expected_exception_string, "validation_errors": None, "response": {"status": expected_status, "body": '{"foo": "bar"}'}, } @pytest.mark.parametrize( "cause,expected_exception_string", [ (requests.RequestException(), "RequestException()"), (requests.ConnectionError(), "ConnectionError()"), (requests.TooManyRedirects(), "TooManyRedirects()"), (requests.ConnectTimeout(), "ConnectTimeout()"), (requests.ReadTimeout(), "ReadTimeout()"), ], ) def test_it_raises_CanvasAPIServerError_for_all_other_requests_errors( self, cause, expected_exception_string ): raised_exception = self.assert_raises(cause, CanvasAPIServerError) assert raised_exception.__cause__ == cause # For these kinds of errors no response (either successful or # unsuccessful) was ever received from Canvas (for example: the network # request timed out) so there's nothing to set as the response # property. assert raised_exception.response is None assert raised_exception.details == { "exception": expected_exception_string, "response": None, "validation_errors": None, } def test_it_raises_CanvasAPIServerError_for_a_successful_but_invalid_response( self, canvas_api_invalid_response ): cause = ValidationError("The response was invalid.") cause.response = canvas_api_invalid_response cause.response.body = "x" * 1000 # pylint:disable=no-member raised_exception = self.assert_raises(cause, CanvasAPIServerError) assert raised_exception.__cause__ == cause assert raised_exception.response == canvas_api_invalid_response assert raised_exception.details == { "exception": "Unable to process the contained instructions", "response": {"body": "Invalid", "status": "200 OK"}, "validation_errors": "The response was invalid.", } def test_it_raises_truncates_the_body_if_it_is_very_long( self, canvas_api_long_response ): # Make the response very long... cause = CanvasAPIServerError("The response was invalid.") cause.response = canvas_api_long_response raised_exception = self.assert_raises(cause, CanvasAPIServerError) body = raised_exception.details["response"]["body"] assert len(body) == 153 assert body.endswith("...") def assert_raises(self, cause, expected_exception_class): with pytest.raises( expected_exception_class, match="Calling the Canvas API failed" ) as exc_info: CanvasAPIError.raise_from(cause) return exc_info.value @pytest.fixture def canvas_api_long_response(self): """Return a successful (200 OK) response with a long body.""" httpretty.register_uri( httpretty.GET, "https://example.com", priority=1, status=200, body="x" * 2000, ) return requests.get("https://example.com") @pytest.fixture def canvas_api_invalid_response(self): """Return a successful (200 OK) response with an invalid body.""" httpretty.register_uri( httpretty.GET, "https://example.com", priority=1, status=200, body="Invalid" ) return requests.get("https://example.com") @staticmethod def _requests_exception(**kwargs): # pylint:disable=inconsistent-return-statements httpretty.register_uri( httpretty.GET, "https://example.com", priority=1, body=json.dumps({"foo": "bar"}), **kwargs ) response = requests.get("https://example.com") try: response.raise_for_status() except requests.RequestException as err: return err
class TestHTTPService: @pytest.mark.parametrize("method", ["GET", "PUT", "POST", "PATCH", "DELETE"]) def test_it_sends_the_request_and_returns_the_response(self, svc, method, url): httpretty.register_uri(method, url, body="test_response") response = svc.request(method, url) assert response.status_code == 200 assert response.text == "test_response" assert httpretty.last_request() == Any.object.with_attrs( {"url": url, "method": method} ) @pytest.mark.parametrize("method", ["GET", "PUT", "POST", "PATCH", "DELETE"]) def test_convenience_methods(self, svc, url, method): httpretty.register_uri(method, url, body="test_response") getattr(svc, method.lower())(url) assert httpretty.last_request() == Any.object.with_attrs( {"url": url, "method": method} ) def test_it_sends_request_params(self, svc, url): svc.request("GET", url, params={"test_param": "test_value"}) assert httpretty.last_request() == Any.object.with_attrs( {"url": f"{url}?test_param=test_value"} ) def test_it_sends_request_data(self, svc, url): svc.request("GET", url, data={"test_key": "test_value"}) assert httpretty.last_request() == Any.object.with_attrs( {"body": b"test_key=test_value"} ) def test_it_sends_request_json(self, svc, url): svc.request("GET", url, json={"test_key": "test_value"}) assert httpretty.last_request() == Any.object.with_attrs( {"body": b'{"test_key": "test_value"}'} ) def test_it_sends_request_headers(self, svc, url): svc.request("GET", url, headers={"HEADER_KEY": "HEADER_VALUE"}) assert httpretty.last_request().headers["HEADER_KEY"] == "HEADER_VALUE" def test_it_sends_request_auth(self, svc, url): svc.request("GET", url, auth=("user", "pass")) assert httpretty.last_request().headers["Authorization"] == "Basic dXNlcjpwYXNz" def test_it_uses_custom_timeouts(self, session): svc = HTTPService(session) svc.request("GET", "https://example.com", timeout=3) assert session.request.call_args[1]["timeout"] == 3 def test_it_passes_arbitrary_kwargs_to_requests(self, svc): session = Mock() svc = HTTPService(session) svc.request("GET", "https://example.com", foo="bar") assert session.request.call_args[1]["foo"] == "bar" @pytest.mark.parametrize( "exception", [ requests.ConnectionError(), requests.HTTPError(), requests.ReadTimeout(), requests.TooManyRedirects(), ], ) def test_it_raises_if_sending_the_request_fails(self, exception, session): session.request.side_effect = exception svc = HTTPService(session) with pytest.raises(UpstreamServiceError) as exc_info: svc.request("GET", "https://example.com") assert exc_info.value.response is None assert exc_info.value.__cause__ == exception @pytest.mark.parametrize("status", [400, 401, 403, 404, 500]) def test_it_raises_if_the_response_is_an_error(self, svc, url, status): httpretty.register_uri("GET", url, status=status) with pytest.raises(UnhandledUpstreamException) as exc_info: svc.request("GET", url) assert isinstance(exc_info.value.__cause__, requests.HTTPError) assert exc_info.value.response == Any.instance_of(requests.Response).with_attrs( {"status_code": status} ) @pytest.mark.parametrize("status", [400, 401, 403, 404, 500]) def test_it_doesnt_raise_if_the_response_is_an_error_when_disabled( self, svc, url, status ): httpretty.register_uri("GET", url, status=status) response = svc.request("GET", url, raise_for_status=False) assert response == Any.instance_of(requests.Response).with_attrs( {"status_code": status} ) @pytest.mark.parametrize("request_exception,expected_exception", EXCEPTION_MAP) def test_request_with_error_translator_defaults( self, error_translator, session, request_exception, expected_exception, url, ): svc = HTTPService(session, error_translator) session.request.side_effect = request_exception with pytest.raises(expected_exception): svc.request("GET", url) @pytest.mark.parametrize("generator_exception,expected_exception", EXCEPTION_MAP) def test_stream_with_error_translator_defaults( self, error_translator, session, generator_exception, expected_exception, url, ): svc = HTTPService(session, error_translator) def explode(): yield b"response-part-1" raise generator_exception session.request.return_value.iter_content.return_value = explode() with pytest.raises(expected_exception): list(svc.stream("GET", url)) def test_with_custom_exceptions(self, error_translator, session, url): svc = HTTPService(session, error_translator) session.request.side_effect = FileNotFoundError with pytest.raises(NotADirectoryError): svc.request("GET", url) def test_request_with_regular_exceptions(self, error_translator, session, url): svc = HTTPService(session, error_translator) session.request.side_effect = ValueError with pytest.raises(ValueError): svc.request("GET", url) def test_stream_with_regular_exceptions(self, error_translator, session, url): svc = HTTPService(session, error_translator) session.request.return_value.iter_content.side_effect = ValueError with pytest.raises(ValueError): list(svc.stream("GET", url)) @pytest.mark.parametrize( "input_bytes,output", ( ([b"longer_than_chunk_size"], [b"longer_than_chunk_size"]), ([b"chunksize"], [b"chunksize"]), ([b"chunk", b"size"], [b"chunksize"]), ([b"too", b"smol"], [b"toosmol"]), ), ) def test_stream_bytes(self, svc, response, input_bytes, output): # pylint:disable=protected-access response.iter_content.return_value = input_bytes results = svc._stream_bytes(response, min_chunk_size=9) assert list(results) == output response.iter_content.assert_called_once_with(chunk_size=9) @pytest.fixture def url(self): """Return the URL that we'll be sending test requests to.""" return "https://example.com/example" @pytest.fixture(autouse=True) def test_response(self, url): httpretty.register_uri( "GET", url, body='{"test_response_key": "test_response_value"}', priority=-1 ) @pytest.fixture def session(self): return create_autospec(requests.Session, instance=True, spec_set=True) @pytest.fixture def error_translator(self, session): def error_mapper_callable(err): if isinstance(err, FileNotFoundError): return NotADirectoryError("Translated") return None return error_mapper_callable @pytest.fixture def svc(self): return HTTPService() @pytest.fixture def response(self): return create_autospec(Response, instance=True, spec_set=True)
class AppriseURLTester(object): # Some exception handling we'll use req_exceptions = ( requests.ConnectionError(0, 'requests.ConnectionError() not handled'), requests.RequestException(0, 'requests.RequestException() not handled'), requests.HTTPError(0, 'requests.HTTPError() not handled'), requests.ReadTimeout(0, 'requests.ReadTimeout() not handled'), requests.TooManyRedirects(0, 'requests.TooManyRedirects() not handled'), ) # Attachment Testing Directory __test_var_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'var') # Our URLs we'll test against __tests = [] # Define how many characters exist per line row = 80 # Some variables we use to control the data we work with body_len = 1024 title_len = 1024 def __init__(self, tests=None, *args, **kwargs): """ Our initialization """ # Create a large body and title with random data self.body = ''.join( choice(str_alpha + str_num + ' ') for _ in range(self.body_len)) self.body = '\r\n'.join([ self.body[i:i + self.row] for i in range(0, len(self.body), self.row) ]) # Create our title using random data self.title = ''.join( choice(str_alpha + str_num) for _ in range(self.title_len)) if tests: self.__tests = tests def add(self, url, meta): """ Adds a test suite to our object """ self.__tests.append({ 'url': url, 'meta': meta, }) def run_all(self): """ Run all of our tests """ # iterate over our dictionary and test it out for (url, meta) in self.__tests: self.run(url, meta) @mock.patch('requests.get') @mock.patch('requests.post') def run(self, url, meta, mock_post, mock_get): """ Run a specific test """ # Disable Throttling to speed testing plugins.NotifyBase.request_rate_per_sec = 0 # Our expected instance instance = meta.get('instance', None) # Our expected server objects _self = meta.get('self', None) # Our expected Query response (True, False, or exception type) response = meta.get('response', True) # Our expected privacy url # Don't set this if don't need to check it's value privacy_url = meta.get('privacy_url') # Our regular expression url_matches = meta.get('url_matches') # Allow us to force the server response code to be something other then # the defaults requests_response_code = meta.get( 'requests_response_code', requests.codes.ok if response else requests.codes.not_found, ) # Allow us to force the server response text to be something other then # the defaults requests_response_text = meta.get('requests_response_text') if not isinstance(requests_response_text, six.string_types): # Convert to string requests_response_text = dumps(requests_response_text) # Whether or not we should include an image with our request; unless # otherwise specified, we assume that images are to be included include_image = meta.get('include_image', True) if include_image: # a default asset asset = AppriseAsset() else: # Disable images asset = AppriseAsset(image_path_mask=False, image_url_mask=False) asset.image_url_logo = None test_requests_exceptions = meta.get('test_requests_exceptions', False) # Mock our request object robj = mock.Mock() robj.content = u'' mock_get.return_value = robj mock_post.return_value = robj if test_requests_exceptions is False: # Handle our default response mock_post.return_value.status_code = requests_response_code mock_get.return_value.status_code = requests_response_code # Handle our default text response mock_get.return_value.content = requests_response_text mock_post.return_value.content = requests_response_text mock_get.return_value.text = requests_response_text mock_post.return_value.text = requests_response_text # Ensure there is no side effect set mock_post.side_effect = None mock_get.side_effect = None else: # Handle exception testing; first we turn the boolean flag # into a list of exceptions test_requests_exceptions = self.req_exceptions try: # We can now instantiate our object: obj = Apprise.instantiate(url, asset=asset, suppress_exceptions=False) except Exception as e: # Handle our exception if instance is None: print('%s %s' % (url, str(e))) raise e if not isinstance(e, instance): print('%s %s' % (url, str(e))) raise e # We're okay if we get here return if obj is None: if instance is not None: # We're done (assuming this is what we were # expecting) print("{} didn't instantiate itself " "(we expected it to be a {})".format(url, instance)) assert False # We're done because we got the results we expected return if instance is None: # Expected None but didn't get it print('%s instantiated %s (but expected None)' % (url, str(obj))) assert False if not isinstance(obj, instance): print('%s instantiated %s (but expected %s)' % (url, type(instance), str(obj))) assert False if isinstance(obj, plugins.NotifyBase): # Ensure we are not performing any type of thorttling obj.request_rate_per_sec = 0 # We loaded okay; now lets make sure we can reverse # this url assert isinstance(obj.url(), six.string_types) is True # Test url() with privacy=True assert isinstance(obj.url(privacy=True), six.string_types) is True # Some Simple Invalid Instance Testing assert instance.parse_url(None) is None assert instance.parse_url(object) is None assert instance.parse_url(42) is None if privacy_url: # Assess that our privacy url is as expected if not obj.url(privacy=True).startswith(privacy_url): raise AssertionError( "Privacy URL: '{}' != expected '{}'".format( obj.url(privacy=True)[:len(privacy_url)], privacy_url)) if url_matches: # Assess that our URL matches a set regex assert re.search(url_matches, obj.url()) # Instantiate the exact same object again using the URL # from the one that was already created properly obj_cmp = Apprise.instantiate(obj.url()) # Our object should be the same instance as what we had # originally expected above. if not isinstance(obj_cmp, plugins.NotifyBase): # Assert messages are hard to trace back with the # way these tests work. Just printing before # throwing our assertion failure makes things # easier to debug later on print('TEST FAIL: {} regenerated as {}'.format(url, obj.url())) assert False # Tidy our object del obj_cmp if _self: # Iterate over our expected entries inside of our # object for key, val in self.items(): # Test that our object has the desired key assert hasattr(key, obj) is True assert getattr(key, obj) == val try: self.__notify(url, obj, meta, asset) except AssertionError: # Don't mess with these entries print('%s AssertionError' % url) raise # Tidy our object and allow any possible defined deconstructors to # be executed. del obj @mock.patch('requests.get') @mock.patch('requests.post') @mock.patch('requests.head') @mock.patch('requests.put') @mock.patch('requests.delete') def __notify(self, url, obj, meta, asset, mock_del, mock_put, mock_head, mock_post, mock_get): """ Perform notification testing against object specified """ # # Prepare our options # # Allow notification type override, otherwise default to INFO notify_type = meta.get('notify_type', NotifyType.INFO) # Whether or not we're testing exceptions or not test_requests_exceptions = meta.get('test_requests_exceptions', False) # Our expected Query response (True, False, or exception type) response = meta.get('response', True) # Our expected Notify response (True or False) notify_response = meta.get('notify_response', response) # Our expected Notify Attachment response (True or False) attach_response = meta.get('attach_response', notify_response) # Test attachments # Don't set this if don't need to check it's value check_attachments = meta.get('check_attachments', True) # Allow us to force the server response code to be something other then # the defaults requests_response_code = meta.get( 'requests_response_code', requests.codes.ok if response else requests.codes.not_found, ) # Allow us to force the server response text to be something other then # the defaults requests_response_text = meta.get('requests_response_text') if not isinstance(requests_response_text, six.string_types): # Convert to string requests_response_text = dumps(requests_response_text) # A request robj = mock.Mock() robj.content = u'' mock_get.return_value = robj mock_post.return_value = robj mock_head.return_value = robj mock_del.return_value = robj mock_put.return_value = robj if test_requests_exceptions is False: # Handle our default response mock_put.return_value.status_code = requests_response_code mock_head.return_value.status_code = requests_response_code mock_del.return_value.status_code = requests_response_code mock_post.return_value.status_code = requests_response_code mock_get.return_value.status_code = requests_response_code # Handle our default text response mock_get.return_value.content = requests_response_text mock_post.return_value.content = requests_response_text mock_del.return_value.content = requests_response_text mock_put.return_value.content = requests_response_text mock_head.return_value.content = requests_response_text mock_get.return_value.text = requests_response_text mock_post.return_value.text = requests_response_text mock_put.return_value.text = requests_response_text mock_del.return_value.text = requests_response_text mock_head.return_value.text = requests_response_text # Ensure there is no side effect set mock_post.side_effect = None mock_del.side_effect = None mock_put.side_effect = None mock_head.side_effect = None mock_get.side_effect = None else: # Handle exception testing; first we turn the boolean flag # into a list of exceptions test_requests_exceptions = self.req_exceptions try: if test_requests_exceptions is False: # Disable throttling obj.request_rate_per_sec = 0 # check that we're as expected assert obj.notify(body=self.body, title=self.title, notify_type=notify_type) == notify_response # check that this doesn't change using different overflow # methods assert obj.notify( body=self.body, title=self.title, notify_type=notify_type, overflow=OverflowMode.UPSTREAM) == notify_response assert obj.notify( body=self.body, title=self.title, notify_type=notify_type, overflow=OverflowMode.TRUNCATE) == notify_response assert obj.notify( body=self.body, title=self.title, notify_type=notify_type, overflow=OverflowMode.SPLIT) == notify_response # # Handle varations of the Asset Object missing fields # # First make a backup app_id = asset.app_id app_desc = asset.app_desc # now clear records asset.app_id = None asset.app_desc = None # Notify should still work assert obj.notify(body=self.body, title=self.title, notify_type=notify_type) == notify_response # App ID only asset.app_id = app_id asset.app_desc = None # Notify should still work assert obj.notify(body=self.body, title=self.title, notify_type=notify_type) == notify_response # App Desc only asset.app_id = None asset.app_desc = app_desc # Notify should still work assert obj.notify(body=self.body, title=self.title, notify_type=notify_type) == notify_response # Restore asset.app_id = app_id asset.app_desc = app_desc if check_attachments: # Test single attachment support; even if the service # doesn't support attachments, it should still # gracefully ignore the data attach = os.path.join(self.__test_var_dir, 'apprise-test.gif') assert obj.notify(body=self.body, title=self.title, notify_type=notify_type, attach=attach) == attach_response # Same results should apply to a list of attachments attach = AppriseAttachment(( os.path.join(self.__test_var_dir, 'apprise-test.gif'), os.path.join(self.__test_var_dir, 'apprise-test.png'), os.path.join(self.__test_var_dir, 'apprise-test.jpeg'), )) assert obj.notify(body=self.body, title=self.title, notify_type=notify_type, attach=attach) == attach_response else: # Disable throttling obj.request_rate_per_sec = 0 for _exception in self.req_exceptions: mock_post.side_effect = _exception mock_head.side_effect = _exception mock_del.side_effect = _exception mock_put.side_effect = _exception mock_get.side_effect = _exception try: assert obj.notify(body=self.body, title=self.title, notify_type=NotifyType.INFO) is False except AssertionError: # Don't mess with these entries raise except Exception: # We can't handle this exception type raise except AssertionError: # Don't mess with these entries raise except Exception as e: # Check that we were expecting this exception to happen try: if not isinstance(e, response): raise e except TypeError: print('%s Unhandled response %s' % (url, type(e))) raise e # # Do the test again but without a title defined # try: if test_requests_exceptions is False: # check that we're as expected assert obj.notify(body='body', notify_type=notify_type) \ == notify_response else: for _exception in self.req_exceptions: mock_post.side_effect = _exception mock_del.side_effect = _exception mock_put.side_effect = _exception mock_head.side_effect = _exception mock_get.side_effect = _exception try: assert obj.notify(body=self.body, notify_type=NotifyType.INFO) is False except AssertionError: # Don't mess with these entries raise except Exception: # We can't handle this exception type raise except AssertionError: # Don't mess with these entries raise except Exception as e: # Check that we were expecting this exception to happen if not isinstance(e, response): raise e return True
def download_resource(self, url_info, reference_job=None): if reference_job and url_info.use_cached: content = reference_job.crawled_content.get(url_info.s3_key) if content is not None: return content, True fetched = sec_since_epoch_to_http_date( url_info.fetched) if url_info.fetched else None response = self.download(url_info, fetched=fetched, etag=url_info.etag) if response.status_code == 304: content = reference_job.crawled_content.get(url_info.s3_key) if content is not None: return content, True response = self.download(url_info) url_info.http_status = response.status_code url_info.response_headers = response.headers url_info.fetcher = self.id url_info.fetched = time.time() response.raise_for_status() if response.status_code in [300, 301, 302, 303, 307, 308]: if url_info.num_redirects >= MAX_REDIRECTS: raise requests.TooManyRedirects('Exceeded %s redirects.' % MAX_REDIRECTS) new_raw_url = urljoin(url_info.raw_url, response.headers['location']) if url_info.raw_url == new_raw_url: # URL redirects back to itself return None, None new_url_info = UrlInfo(new_raw_url) self.url_transformer.transform(new_url_info) if url_info.id == new_url_info.id: # Only the scheme changed (usually http -> https) so just fetch that URL right away instead url_info.change_scheme(extract_scheme(new_raw_url)) return self.download_resource(url_info, reference_job) new_url_info.referrer_id = url_info.id new_url_info.num_redirects = url_info.num_redirects + 1 new_url_info.discovered = time.time() url_info.redirects_to = new_url_info.id self.save_redirected_url(url_info) # Log the redirect # Calling self.process_url() on the new URL can cause unbounded recursion so return None here. Redirects # from robots.txt files are usually useless anyway. It is either a domain level redirect or an error page. if url_info.url.endswith('/robots.txt'): return None, None if self.process_url(new_url_info): self.mark_url_seen( new_url_info ) # local only because new_url_info isn't added to the frontier. self.add_to_frontier([new_url_info]) # The body of non-300 type responses is usually empty. if response.status_code != 300: return None, None return response.content, False
def get(self, *args, **kwargs): raise requests.TooManyRedirects("Fake Too Many Redirects Error!")
text = htmlPages.text soup = BeautifulSoup(text,"html.parser") self.FetchAptoideLinks(soup,appList) except requests.ConnectionError as connError: logger.log("Connection Error while connecting to Play store: ", urls," Error: ", connError) except requests.HTTPError as httpError: logger.log("Invalid HTTP response to Play store: ", urls, " Error: ", httpError) except requests.Timeout() as requestTimeoutError: logger.log("Time-out to connect to Play store: ", urls, " Error: ", requestTimeoutError) except requests.TooManyRedirects() as redirectsError: logger.log("Too many redirects for connection to Play store: ", urls, " Error: ", redirectsError) except Exception as e: logger.log("Excpetion occured at Func Spider: ", e) except requests.exceptions.Timeout as timeoutException: logger.log("Timeout Exception", timeoutException) return appList def FetchAptoideLinks(self,soup,appList): for link in soup.findAll("span",{"class":"bundle-item__info__span--big"}): hreff = link.find('a').get('href')