def __init__(self):
self.__logger = Logger()
self._request_exceptions = [type(item) for item in [requests.ConnectionError(), requests.HTTPError(),
requests.TooManyRedirects(), requests.Timeout(),
requests.TooManyRedirects(),
requests.RequestException(), requests.ConnectTimeout(),
requests.ReadTimeout()]]
self._system_errors = [type(item) for item in [KeyError(), AttributeError(), IndexError(),
ZeroDivisionError(), SystemError(), ValueError(),
AssertionError()]]
self._file_errors = [type(item) for item in [FileExistsError(), FileNotFoundError()]]
self._database_errors = [type(item) for item in [sqlite3.Error(), sqlite3.DataError(),
sqlite3.ProgrammingError(), sqlite3.DatabaseError(),
sqlite3.NotSupportedError(), sqlite3.IntegrityError(),
sqlite3.InterfaceError(), sqlite3.InternalError(),
sqlite3.OperationalError()]]
self._speech_recognizer_errors = [type(item) for item in
[sr.RequestError(), sr.UnknownValueError(), sr.WaitTimeoutError(),
sr.RequestError()]]
self.__logger.info('ExceptionsHandler was successfully initialized.', __name__)
def test_update_notable_too_many_redirect(self, mocked_requests_post):
""" Test update notable with wrong URL or connection issue"""
print("Test update_notable returns TooManyRedirects\n")
try:
sim_status = 1
sim_content = '<response><sessionKey>' + self.simSessionKey + '</sessionKey></response>'
mocked_requests_post.return_value = self._generateResponse(
sim_content, 200)
splnk_utils = splunk_utils.SplunkUtils(host=self.fake_host,
port=self.fake_port,
username=self.fake_username,
password=self.fake_password,
verify=self.verify)
mocked_requests_post.side_effect = requests.TooManyRedirects(
Mock(status=404), "Ambiguous excetpion.")
ret = splnk_utils.update_notable(event_id=self.simEventId,
comment=self.simComment,
status=sim_status,
cafile=self.verify)
#
# request post throws exception
#
assert False
except splunk_utils.RequestError as e:
assert True
def test_to_many_redirects_exception(self):
with mock.patch('requests.get') as requests_get_mock:
requests_get_mock.side_effect = requests.TooManyRedirects()
with StdoutCapture() as output:
self.api_caller.get_resource("/resource", DiscoveryContainer())
self.assertIn(
"ERROR::Get url=http://1.2.3.4:567/resource Error <class 'requests.exceptions.TooManyRedirects'>:;",
output.raw)
def test_add_threat_intel_item_errors(self, mocked_requests_post):
# 1. Connect successfully
sim_content = '<response><sessionKey>' + self.simSessionKey + '</sessionKey></response>'
mocked_requests_post.return_value = self._generateResponse(
sim_content, 200)
splnk_utils = splunk_utils.SplunkUtils(host=self.fake_host,
port=self.fake_port,
username=self.fake_username,
password=self.fake_password,
verify=self.verify)
# 2. Simulate wrong intel type
try:
splnk_utils.add_threat_intel_item("Fake type", {}, False)
except splunk_utils.RequestError as e:
print("Fake intel type causes exception as expected.")
assert (True)
# 3. Simulate RequestException
threat_type = "ip_intel"
mocked_requests_post.side_effect = requests.RequestException(
Mock(status=404), "Ambiguous excetpion.")
try:
splnk_utils.add_threat_intel_item(threat_type, {}, False)
except splunk_utils.RequestError:
assert True
# 4. Simulate ConnectionError
mocked_requests_post.side_effect = requests.ConnectionError(
Mock(status=404), "Ambiguous excetpion.")
try:
splnk_utils.add_threat_intel_item(threat_type, {}, False)
except splunk_utils.RequestError:
assert True
# 5. Simulate HttpError
mocked_requests_post.side_effect = requests.HTTPError(
Mock(status=404), "Ambiguous excetpion.")
try:
splnk_utils.add_threat_intel_item(threat_type, {}, False)
except splunk_utils.RequestError:
assert True
# 6. Simulate URLRequired
mocked_requests_post.side_effect = requests.URLRequired(
Mock(status=404), "Ambiguous excetpion.")
try:
splnk_utils.add_threat_intel_item(threat_type, {}, False)
except splunk_utils.RequestError:
assert True
# 7. Simulate TooManyRedirects
mocked_requests_post.side_effect = requests.TooManyRedirects(
Mock(status=404), "Ambiguous excetpion.")
try:
splnk_utils.add_threat_intel_item(threat_type, {}, False)
except splunk_utils.RequestError:
assert True
def test_it_raises_an_exception_on_requests_error(self) -> None:
self._request_mock.side_effect = requests.TooManyRedirects(
'Exceeded 10 redirects.')
with self.assertRaises(HttpException) as exception_ctx:
self._client.request('GET', '/error')
self.assertEqual(
'Requests error: None: Exceeded 10 redirects. '
'(see https://requests.kennethreitz.org/en/master/api/#exceptions) for /error',
str(exception_ctx.exception))
def get_story(new):
"""Return a story of the given ID."""
url = URLS['item'].format(new)
try:
data = req.get(url)
except req.ConnectionError:
raise
except req.Timeout:
raise req.Timeout('A timeout problem occurred.')
except req.TooManyRedirects:
raise req.TooManyRedirects('The request exceeds the configured number\
of maximum redirections.')
else:
return data.json()
def _request(self, url_suffix, **kwargs):
url = '{base}{url_token}{url_suffix}'.format(base=self.base_url,
url_suffix=url_suffix,
url_token=self.url_token)
kwargs['timeout'] = kwargs.get('timeout', self.school.timeout)
res = self._http.request(url=url,
proxies=self._proxy,
allow_redirects=False,
**kwargs)
res.raise_for_status()
if res.status_code == 302:
raise requests.TooManyRedirects(response=res)
return res
def Spider(self, limit):
page = 1
webAppList = []
while page < limit:
url = r"https://play.google.com/store/apps/category/GAME/collection/topselling_new_free?hl=en"
try:
source_Code = requests.get(url, timeout=10)
raw_text = source_Code.text
soup = BeautifulSoup(raw_text, "html.parser")
self.FetchAppLinks(soup, webAppList)
except requests.ConnectionError as connError:
logger.log("Connection Error while connecting to Play store: ",
url, " Error: ", connError)
except requests.HTTPError as httpError:
logger.log("Invalid HTTP response to Play store: ", url,
" Error: ", httpError)
except requests.Timeout() as requestTimeoutError:
logger.log("Time-out to connect to Play store: ", url,
" Error: ", requestTimeoutError)
except requests.TooManyRedirects() as redirectsError:
logger.log("Too many redirects for connection to Play store: ",
url, " Error: ", redirectsError)
#except HTMLParseError as htmlParsingError:
# logger.log("HTMLParse Error: ", htmlParsingError)
except Exception as e:
logger.log("Execpetion occured at Func Spider: ", e)
except requests.exceptions.Timeout as timeoutException:
logger.log("Timeout Exception", timeoutException)
page += 1
return webAppList
def test_consultar_cep(self, mock_api_call):
# Criamos uma requisicao que sera bem sucedida
param = {
'text': self.response_xml,
'ok': True,
'status_code': 200,
}
mock_api_call.return_value = mock.MagicMock(**param)
self.assertDictEqual(consultar_cep('70002900'), self.expected_address)
self.assertDictEqual(consultar_cep('70002900', ambiente=HOMOLOGACAO),
self.expected_address)
self.assertDictEqual(consultar_cep('70002900', ambiente=PRODUCAO),
self.expected_address)
# Aqui, construímos uma requisição que retornará erro
param = {
'text': self.response_xml_error,
'ok': False,
}
mock_api_call.return_value = mock.MagicMock(**param)
self.assertRaises(CEPInvalido, consultar_cep, '1232710')
mock_api_call.return_value = mock.MagicMock(**param)
self.assertRaises(CEPInvalido, consultar_cep, '00000000')
self.assertRaises(KeyError, consultar_cep, '37503130', ambiente=3)
# Verificamos as demais excecoes
mock_api_call.side_effect = requests.ConnectTimeout()
self.assertRaises(Timeout, consultar_cep, '37503130')
mock_api_call.side_effect = requests.ConnectionError('', '')
self.assertRaises(FalhaNaConexao, consultar_cep, '37503130')
mock_api_call.side_effect = requests.TooManyRedirects()
self.assertRaises(MultiploRedirecionamento, consultar_cep, '37503130')
mock_api_call.side_effect = requests.RequestException()
self.assertRaises(ExcecaoPyCEPCorreios, consultar_cep, '37503130')
# Disable logging for a cleaner testing output
import logging
logging.disable(logging.CRITICAL)
# Some exception handling we'll use
REQUEST_EXCEPTIONS = (
requests.ConnectionError(
0, 'requests.ConnectionError() not handled'),
requests.RequestException(
0, 'requests.RequestException() not handled'),
requests.HTTPError(
0, 'requests.HTTPError() not handled'),
requests.ReadTimeout(
0, 'requests.ReadTimeout() not handled'),
requests.TooManyRedirects(
0, 'requests.TooManyRedirects() not handled'),
)
@mock.patch('requests.get')
@mock.patch('requests.post')
def test_config_http(mock_post, mock_get):
"""
API: ConfigHTTP() object
"""
# Define our good:// url
class GoodNotification(NotifyBase):
def __init__(self, *args, **kwargs):
super(GoodNotification, self).__init__(*args, **kwargs)
class TestCanvasAPIError:
@pytest.mark.parametrize(
"status,expected_status,expected_exception_class,expected_exception_string",
[
# A 401 Unauthorized response from Canvas, because our access token was
# expired or deleted.
(
401,
"401 Unauthorized",
CanvasAPIAccessTokenError,
"401 Client Error: Unauthorized for url: https://example.com/",
),
# A 400 Bad Request response from Canvas, because we sent an invalid
# parameter or something.
(
400,
"400 Bad Request",
CanvasAPIServerError,
"400 Client Error: Bad Request for url: https://example.com/",
),
# An unexpected error response from Canvas.
(
500,
"500 Internal Server Error",
CanvasAPIServerError,
"500 Server Error: Internal Server Error for url: https://example.com/",
),
],
)
def test_it_raises_the_right_subclass_for_different_Canvas_responses(
self,
status,
expected_status,
expected_exception_class,
expected_exception_string,
):
cause = self._requests_exception(status=status)
raised_exception = self.assert_raises(cause, expected_exception_class)
assert raised_exception.__cause__ == cause
assert raised_exception.response == cause.response
assert raised_exception.details == {
"exception": expected_exception_string,
"validation_errors": None,
"response": {"status": expected_status, "body": '{"foo": "bar"}'},
}
@pytest.mark.parametrize(
"cause,expected_exception_string",
[
(requests.RequestException(), "RequestException()"),
(requests.ConnectionError(), "ConnectionError()"),
(requests.TooManyRedirects(), "TooManyRedirects()"),
(requests.ConnectTimeout(), "ConnectTimeout()"),
(requests.ReadTimeout(), "ReadTimeout()"),
],
)
def test_it_raises_CanvasAPIServerError_for_all_other_requests_errors(
self, cause, expected_exception_string
):
raised_exception = self.assert_raises(cause, CanvasAPIServerError)
assert raised_exception.__cause__ == cause
# For these kinds of errors no response (either successful or
# unsuccessful) was ever received from Canvas (for example: the network
# request timed out) so there's nothing to set as the response
# property.
assert raised_exception.response is None
assert raised_exception.details == {
"exception": expected_exception_string,
"response": None,
"validation_errors": None,
}
def test_it_raises_CanvasAPIServerError_for_a_successful_but_invalid_response(
self, canvas_api_invalid_response
):
cause = ValidationError("The response was invalid.")
cause.response = canvas_api_invalid_response
cause.response.body = "x" * 1000 # pylint:disable=no-member
raised_exception = self.assert_raises(cause, CanvasAPIServerError)
assert raised_exception.__cause__ == cause
assert raised_exception.response == canvas_api_invalid_response
assert raised_exception.details == {
"exception": "Unable to process the contained instructions",
"response": {"body": "Invalid", "status": "200 OK"},
"validation_errors": "The response was invalid.",
}
def test_it_raises_truncates_the_body_if_it_is_very_long(
self, canvas_api_long_response
):
# Make the response very long...
cause = CanvasAPIServerError("The response was invalid.")
cause.response = canvas_api_long_response
raised_exception = self.assert_raises(cause, CanvasAPIServerError)
body = raised_exception.details["response"]["body"]
assert len(body) == 153
assert body.endswith("...")
def assert_raises(self, cause, expected_exception_class):
with pytest.raises(
expected_exception_class, match="Calling the Canvas API failed"
) as exc_info:
CanvasAPIError.raise_from(cause)
return exc_info.value
@pytest.fixture
def canvas_api_long_response(self):
"""Return a successful (200 OK) response with a long body."""
httpretty.register_uri(
httpretty.GET,
"https://example.com",
priority=1,
status=200,
body="x" * 2000,
)
return requests.get("https://example.com")
@pytest.fixture
def canvas_api_invalid_response(self):
"""Return a successful (200 OK) response with an invalid body."""
httpretty.register_uri(
httpretty.GET, "https://example.com", priority=1, status=200, body="Invalid"
)
return requests.get("https://example.com")
@staticmethod
def _requests_exception(**kwargs): # pylint:disable=inconsistent-return-statements
httpretty.register_uri(
httpretty.GET,
"https://example.com",
priority=1,
body=json.dumps({"foo": "bar"}),
**kwargs
)
response = requests.get("https://example.com")
try:
response.raise_for_status()
except requests.RequestException as err:
return err
class TestHTTPService:
@pytest.mark.parametrize("method", ["GET", "PUT", "POST", "PATCH", "DELETE"])
def test_it_sends_the_request_and_returns_the_response(self, svc, method, url):
httpretty.register_uri(method, url, body="test_response")
response = svc.request(method, url)
assert response.status_code == 200
assert response.text == "test_response"
assert httpretty.last_request() == Any.object.with_attrs(
{"url": url, "method": method}
)
@pytest.mark.parametrize("method", ["GET", "PUT", "POST", "PATCH", "DELETE"])
def test_convenience_methods(self, svc, url, method):
httpretty.register_uri(method, url, body="test_response")
getattr(svc, method.lower())(url)
assert httpretty.last_request() == Any.object.with_attrs(
{"url": url, "method": method}
)
def test_it_sends_request_params(self, svc, url):
svc.request("GET", url, params={"test_param": "test_value"})
assert httpretty.last_request() == Any.object.with_attrs(
{"url": f"{url}?test_param=test_value"}
)
def test_it_sends_request_data(self, svc, url):
svc.request("GET", url, data={"test_key": "test_value"})
assert httpretty.last_request() == Any.object.with_attrs(
{"body": b"test_key=test_value"}
)
def test_it_sends_request_json(self, svc, url):
svc.request("GET", url, json={"test_key": "test_value"})
assert httpretty.last_request() == Any.object.with_attrs(
{"body": b'{"test_key": "test_value"}'}
)
def test_it_sends_request_headers(self, svc, url):
svc.request("GET", url, headers={"HEADER_KEY": "HEADER_VALUE"})
assert httpretty.last_request().headers["HEADER_KEY"] == "HEADER_VALUE"
def test_it_sends_request_auth(self, svc, url):
svc.request("GET", url, auth=("user", "pass"))
assert httpretty.last_request().headers["Authorization"] == "Basic dXNlcjpwYXNz"
def test_it_uses_custom_timeouts(self, session):
svc = HTTPService(session)
svc.request("GET", "https://example.com", timeout=3)
assert session.request.call_args[1]["timeout"] == 3
def test_it_passes_arbitrary_kwargs_to_requests(self, svc):
session = Mock()
svc = HTTPService(session)
svc.request("GET", "https://example.com", foo="bar")
assert session.request.call_args[1]["foo"] == "bar"
@pytest.mark.parametrize(
"exception",
[
requests.ConnectionError(),
requests.HTTPError(),
requests.ReadTimeout(),
requests.TooManyRedirects(),
],
)
def test_it_raises_if_sending_the_request_fails(self, exception, session):
session.request.side_effect = exception
svc = HTTPService(session)
with pytest.raises(UpstreamServiceError) as exc_info:
svc.request("GET", "https://example.com")
assert exc_info.value.response is None
assert exc_info.value.__cause__ == exception
@pytest.mark.parametrize("status", [400, 401, 403, 404, 500])
def test_it_raises_if_the_response_is_an_error(self, svc, url, status):
httpretty.register_uri("GET", url, status=status)
with pytest.raises(UnhandledUpstreamException) as exc_info:
svc.request("GET", url)
assert isinstance(exc_info.value.__cause__, requests.HTTPError)
assert exc_info.value.response == Any.instance_of(requests.Response).with_attrs(
{"status_code": status}
)
@pytest.mark.parametrize("status", [400, 401, 403, 404, 500])
def test_it_doesnt_raise_if_the_response_is_an_error_when_disabled(
self, svc, url, status
):
httpretty.register_uri("GET", url, status=status)
response = svc.request("GET", url, raise_for_status=False)
assert response == Any.instance_of(requests.Response).with_attrs(
{"status_code": status}
)
@pytest.mark.parametrize("request_exception,expected_exception", EXCEPTION_MAP)
def test_request_with_error_translator_defaults(
self,
error_translator,
session,
request_exception,
expected_exception,
url,
):
svc = HTTPService(session, error_translator)
session.request.side_effect = request_exception
with pytest.raises(expected_exception):
svc.request("GET", url)
@pytest.mark.parametrize("generator_exception,expected_exception", EXCEPTION_MAP)
def test_stream_with_error_translator_defaults(
self,
error_translator,
session,
generator_exception,
expected_exception,
url,
):
svc = HTTPService(session, error_translator)
def explode():
yield b"response-part-1"
raise generator_exception
session.request.return_value.iter_content.return_value = explode()
with pytest.raises(expected_exception):
list(svc.stream("GET", url))
def test_with_custom_exceptions(self, error_translator, session, url):
svc = HTTPService(session, error_translator)
session.request.side_effect = FileNotFoundError
with pytest.raises(NotADirectoryError):
svc.request("GET", url)
def test_request_with_regular_exceptions(self, error_translator, session, url):
svc = HTTPService(session, error_translator)
session.request.side_effect = ValueError
with pytest.raises(ValueError):
svc.request("GET", url)
def test_stream_with_regular_exceptions(self, error_translator, session, url):
svc = HTTPService(session, error_translator)
session.request.return_value.iter_content.side_effect = ValueError
with pytest.raises(ValueError):
list(svc.stream("GET", url))
@pytest.mark.parametrize(
"input_bytes,output",
(
([b"longer_than_chunk_size"], [b"longer_than_chunk_size"]),
([b"chunksize"], [b"chunksize"]),
([b"chunk", b"size"], [b"chunksize"]),
([b"too", b"smol"], [b"toosmol"]),
),
)
def test_stream_bytes(self, svc, response, input_bytes, output):
# pylint:disable=protected-access
response.iter_content.return_value = input_bytes
results = svc._stream_bytes(response, min_chunk_size=9)
assert list(results) == output
response.iter_content.assert_called_once_with(chunk_size=9)
@pytest.fixture
def url(self):
"""Return the URL that we'll be sending test requests to."""
return "https://example.com/example"
@pytest.fixture(autouse=True)
def test_response(self, url):
httpretty.register_uri(
"GET", url, body='{"test_response_key": "test_response_value"}', priority=-1
)
@pytest.fixture
def session(self):
return create_autospec(requests.Session, instance=True, spec_set=True)
@pytest.fixture
def error_translator(self, session):
def error_mapper_callable(err):
if isinstance(err, FileNotFoundError):
return NotADirectoryError("Translated")
return None
return error_mapper_callable
@pytest.fixture
def svc(self):
return HTTPService()
@pytest.fixture
def response(self):
return create_autospec(Response, instance=True, spec_set=True)
class AppriseURLTester(object):
# Some exception handling we'll use
req_exceptions = (
requests.ConnectionError(0, 'requests.ConnectionError() not handled'),
requests.RequestException(0,
'requests.RequestException() not handled'),
requests.HTTPError(0, 'requests.HTTPError() not handled'),
requests.ReadTimeout(0, 'requests.ReadTimeout() not handled'),
requests.TooManyRedirects(0,
'requests.TooManyRedirects() not handled'),
)
# Attachment Testing Directory
__test_var_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)),
'var')
# Our URLs we'll test against
__tests = []
# Define how many characters exist per line
row = 80
# Some variables we use to control the data we work with
body_len = 1024
title_len = 1024
def __init__(self, tests=None, *args, **kwargs):
"""
Our initialization
"""
# Create a large body and title with random data
self.body = ''.join(
choice(str_alpha + str_num + ' ') for _ in range(self.body_len))
self.body = '\r\n'.join([
self.body[i:i + self.row]
for i in range(0, len(self.body), self.row)
])
# Create our title using random data
self.title = ''.join(
choice(str_alpha + str_num) for _ in range(self.title_len))
if tests:
self.__tests = tests
def add(self, url, meta):
"""
Adds a test suite to our object
"""
self.__tests.append({
'url': url,
'meta': meta,
})
def run_all(self):
"""
Run all of our tests
"""
# iterate over our dictionary and test it out
for (url, meta) in self.__tests:
self.run(url, meta)
@mock.patch('requests.get')
@mock.patch('requests.post')
def run(self, url, meta, mock_post, mock_get):
"""
Run a specific test
"""
# Disable Throttling to speed testing
plugins.NotifyBase.request_rate_per_sec = 0
# Our expected instance
instance = meta.get('instance', None)
# Our expected server objects
_self = meta.get('self', None)
# Our expected Query response (True, False, or exception type)
response = meta.get('response', True)
# Our expected privacy url
# Don't set this if don't need to check it's value
privacy_url = meta.get('privacy_url')
# Our regular expression
url_matches = meta.get('url_matches')
# Allow us to force the server response code to be something other then
# the defaults
requests_response_code = meta.get(
'requests_response_code',
requests.codes.ok if response else requests.codes.not_found,
)
# Allow us to force the server response text to be something other then
# the defaults
requests_response_text = meta.get('requests_response_text')
if not isinstance(requests_response_text, six.string_types):
# Convert to string
requests_response_text = dumps(requests_response_text)
# Whether or not we should include an image with our request; unless
# otherwise specified, we assume that images are to be included
include_image = meta.get('include_image', True)
if include_image:
# a default asset
asset = AppriseAsset()
else:
# Disable images
asset = AppriseAsset(image_path_mask=False, image_url_mask=False)
asset.image_url_logo = None
test_requests_exceptions = meta.get('test_requests_exceptions', False)
# Mock our request object
robj = mock.Mock()
robj.content = u''
mock_get.return_value = robj
mock_post.return_value = robj
if test_requests_exceptions is False:
# Handle our default response
mock_post.return_value.status_code = requests_response_code
mock_get.return_value.status_code = requests_response_code
# Handle our default text response
mock_get.return_value.content = requests_response_text
mock_post.return_value.content = requests_response_text
mock_get.return_value.text = requests_response_text
mock_post.return_value.text = requests_response_text
# Ensure there is no side effect set
mock_post.side_effect = None
mock_get.side_effect = None
else:
# Handle exception testing; first we turn the boolean flag
# into a list of exceptions
test_requests_exceptions = self.req_exceptions
try:
# We can now instantiate our object:
obj = Apprise.instantiate(url,
asset=asset,
suppress_exceptions=False)
except Exception as e:
# Handle our exception
if instance is None:
print('%s %s' % (url, str(e)))
raise e
if not isinstance(e, instance):
print('%s %s' % (url, str(e)))
raise e
# We're okay if we get here
return
if obj is None:
if instance is not None:
# We're done (assuming this is what we were
# expecting)
print("{} didn't instantiate itself "
"(we expected it to be a {})".format(url, instance))
assert False
# We're done because we got the results we expected
return
if instance is None:
# Expected None but didn't get it
print('%s instantiated %s (but expected None)' % (url, str(obj)))
assert False
if not isinstance(obj, instance):
print('%s instantiated %s (but expected %s)' %
(url, type(instance), str(obj)))
assert False
if isinstance(obj, plugins.NotifyBase):
# Ensure we are not performing any type of thorttling
obj.request_rate_per_sec = 0
# We loaded okay; now lets make sure we can reverse
# this url
assert isinstance(obj.url(), six.string_types) is True
# Test url() with privacy=True
assert isinstance(obj.url(privacy=True), six.string_types) is True
# Some Simple Invalid Instance Testing
assert instance.parse_url(None) is None
assert instance.parse_url(object) is None
assert instance.parse_url(42) is None
if privacy_url:
# Assess that our privacy url is as expected
if not obj.url(privacy=True).startswith(privacy_url):
raise AssertionError(
"Privacy URL: '{}' != expected '{}'".format(
obj.url(privacy=True)[:len(privacy_url)],
privacy_url))
if url_matches:
# Assess that our URL matches a set regex
assert re.search(url_matches, obj.url())
# Instantiate the exact same object again using the URL
# from the one that was already created properly
obj_cmp = Apprise.instantiate(obj.url())
# Our object should be the same instance as what we had
# originally expected above.
if not isinstance(obj_cmp, plugins.NotifyBase):
# Assert messages are hard to trace back with the
# way these tests work. Just printing before
# throwing our assertion failure makes things
# easier to debug later on
print('TEST FAIL: {} regenerated as {}'.format(url, obj.url()))
assert False
# Tidy our object
del obj_cmp
if _self:
# Iterate over our expected entries inside of our
# object
for key, val in self.items():
# Test that our object has the desired key
assert hasattr(key, obj) is True
assert getattr(key, obj) == val
try:
self.__notify(url, obj, meta, asset)
except AssertionError:
# Don't mess with these entries
print('%s AssertionError' % url)
raise
# Tidy our object and allow any possible defined deconstructors to
# be executed.
del obj
@mock.patch('requests.get')
@mock.patch('requests.post')
@mock.patch('requests.head')
@mock.patch('requests.put')
@mock.patch('requests.delete')
def __notify(self, url, obj, meta, asset, mock_del, mock_put, mock_head,
mock_post, mock_get):
"""
Perform notification testing against object specified
"""
#
# Prepare our options
#
# Allow notification type override, otherwise default to INFO
notify_type = meta.get('notify_type', NotifyType.INFO)
# Whether or not we're testing exceptions or not
test_requests_exceptions = meta.get('test_requests_exceptions', False)
# Our expected Query response (True, False, or exception type)
response = meta.get('response', True)
# Our expected Notify response (True or False)
notify_response = meta.get('notify_response', response)
# Our expected Notify Attachment response (True or False)
attach_response = meta.get('attach_response', notify_response)
# Test attachments
# Don't set this if don't need to check it's value
check_attachments = meta.get('check_attachments', True)
# Allow us to force the server response code to be something other then
# the defaults
requests_response_code = meta.get(
'requests_response_code',
requests.codes.ok if response else requests.codes.not_found,
)
# Allow us to force the server response text to be something other then
# the defaults
requests_response_text = meta.get('requests_response_text')
if not isinstance(requests_response_text, six.string_types):
# Convert to string
requests_response_text = dumps(requests_response_text)
# A request
robj = mock.Mock()
robj.content = u''
mock_get.return_value = robj
mock_post.return_value = robj
mock_head.return_value = robj
mock_del.return_value = robj
mock_put.return_value = robj
if test_requests_exceptions is False:
# Handle our default response
mock_put.return_value.status_code = requests_response_code
mock_head.return_value.status_code = requests_response_code
mock_del.return_value.status_code = requests_response_code
mock_post.return_value.status_code = requests_response_code
mock_get.return_value.status_code = requests_response_code
# Handle our default text response
mock_get.return_value.content = requests_response_text
mock_post.return_value.content = requests_response_text
mock_del.return_value.content = requests_response_text
mock_put.return_value.content = requests_response_text
mock_head.return_value.content = requests_response_text
mock_get.return_value.text = requests_response_text
mock_post.return_value.text = requests_response_text
mock_put.return_value.text = requests_response_text
mock_del.return_value.text = requests_response_text
mock_head.return_value.text = requests_response_text
# Ensure there is no side effect set
mock_post.side_effect = None
mock_del.side_effect = None
mock_put.side_effect = None
mock_head.side_effect = None
mock_get.side_effect = None
else:
# Handle exception testing; first we turn the boolean flag
# into a list of exceptions
test_requests_exceptions = self.req_exceptions
try:
if test_requests_exceptions is False:
# Disable throttling
obj.request_rate_per_sec = 0
# check that we're as expected
assert obj.notify(body=self.body,
title=self.title,
notify_type=notify_type) == notify_response
# check that this doesn't change using different overflow
# methods
assert obj.notify(
body=self.body,
title=self.title,
notify_type=notify_type,
overflow=OverflowMode.UPSTREAM) == notify_response
assert obj.notify(
body=self.body,
title=self.title,
notify_type=notify_type,
overflow=OverflowMode.TRUNCATE) == notify_response
assert obj.notify(
body=self.body,
title=self.title,
notify_type=notify_type,
overflow=OverflowMode.SPLIT) == notify_response
#
# Handle varations of the Asset Object missing fields
#
# First make a backup
app_id = asset.app_id
app_desc = asset.app_desc
# now clear records
asset.app_id = None
asset.app_desc = None
# Notify should still work
assert obj.notify(body=self.body,
title=self.title,
notify_type=notify_type) == notify_response
# App ID only
asset.app_id = app_id
asset.app_desc = None
# Notify should still work
assert obj.notify(body=self.body,
title=self.title,
notify_type=notify_type) == notify_response
# App Desc only
asset.app_id = None
asset.app_desc = app_desc
# Notify should still work
assert obj.notify(body=self.body,
title=self.title,
notify_type=notify_type) == notify_response
# Restore
asset.app_id = app_id
asset.app_desc = app_desc
if check_attachments:
# Test single attachment support; even if the service
# doesn't support attachments, it should still
# gracefully ignore the data
attach = os.path.join(self.__test_var_dir,
'apprise-test.gif')
assert obj.notify(body=self.body,
title=self.title,
notify_type=notify_type,
attach=attach) == attach_response
# Same results should apply to a list of attachments
attach = AppriseAttachment((
os.path.join(self.__test_var_dir, 'apprise-test.gif'),
os.path.join(self.__test_var_dir, 'apprise-test.png'),
os.path.join(self.__test_var_dir, 'apprise-test.jpeg'),
))
assert obj.notify(body=self.body,
title=self.title,
notify_type=notify_type,
attach=attach) == attach_response
else:
# Disable throttling
obj.request_rate_per_sec = 0
for _exception in self.req_exceptions:
mock_post.side_effect = _exception
mock_head.side_effect = _exception
mock_del.side_effect = _exception
mock_put.side_effect = _exception
mock_get.side_effect = _exception
try:
assert obj.notify(body=self.body,
title=self.title,
notify_type=NotifyType.INFO) is False
except AssertionError:
# Don't mess with these entries
raise
except Exception:
# We can't handle this exception type
raise
except AssertionError:
# Don't mess with these entries
raise
except Exception as e:
# Check that we were expecting this exception to happen
try:
if not isinstance(e, response):
raise e
except TypeError:
print('%s Unhandled response %s' % (url, type(e)))
raise e
#
# Do the test again but without a title defined
#
try:
if test_requests_exceptions is False:
# check that we're as expected
assert obj.notify(body='body', notify_type=notify_type) \
== notify_response
else:
for _exception in self.req_exceptions:
mock_post.side_effect = _exception
mock_del.side_effect = _exception
mock_put.side_effect = _exception
mock_head.side_effect = _exception
mock_get.side_effect = _exception
try:
assert obj.notify(body=self.body,
notify_type=NotifyType.INFO) is False
except AssertionError:
# Don't mess with these entries
raise
except Exception:
# We can't handle this exception type
raise
except AssertionError:
# Don't mess with these entries
raise
except Exception as e:
# Check that we were expecting this exception to happen
if not isinstance(e, response):
raise e
return True
def download_resource(self, url_info, reference_job=None):
if reference_job and url_info.use_cached:
content = reference_job.crawled_content.get(url_info.s3_key)
if content is not None:
return content, True
fetched = sec_since_epoch_to_http_date(
url_info.fetched) if url_info.fetched else None
response = self.download(url_info, fetched=fetched, etag=url_info.etag)
if response.status_code == 304:
content = reference_job.crawled_content.get(url_info.s3_key)
if content is not None:
return content, True
response = self.download(url_info)
url_info.http_status = response.status_code
url_info.response_headers = response.headers
url_info.fetcher = self.id
url_info.fetched = time.time()
response.raise_for_status()
if response.status_code in [300, 301, 302, 303, 307, 308]:
if url_info.num_redirects >= MAX_REDIRECTS:
raise requests.TooManyRedirects('Exceeded %s redirects.' %
MAX_REDIRECTS)
new_raw_url = urljoin(url_info.raw_url,
response.headers['location'])
if url_info.raw_url == new_raw_url: # URL redirects back to itself
return None, None
new_url_info = UrlInfo(new_raw_url)
self.url_transformer.transform(new_url_info)
if url_info.id == new_url_info.id:
# Only the scheme changed (usually http -> https) so just fetch that URL right away instead
url_info.change_scheme(extract_scheme(new_raw_url))
return self.download_resource(url_info, reference_job)
new_url_info.referrer_id = url_info.id
new_url_info.num_redirects = url_info.num_redirects + 1
new_url_info.discovered = time.time()
url_info.redirects_to = new_url_info.id
self.save_redirected_url(url_info) # Log the redirect
# Calling self.process_url() on the new URL can cause unbounded recursion so return None here. Redirects
# from robots.txt files are usually useless anyway. It is either a domain level redirect or an error page.
if url_info.url.endswith('/robots.txt'):
return None, None
if self.process_url(new_url_info):
self.mark_url_seen(
new_url_info
) # local only because new_url_info isn't added to the frontier.
self.add_to_frontier([new_url_info])
# The body of non-300 type responses is usually empty.
if response.status_code != 300:
return None, None
return response.content, False
def get(self, *args, **kwargs):
raise requests.TooManyRedirects("Fake Too Many Redirects Error!")
text = htmlPages.text
soup = BeautifulSoup(text,"html.parser")
self.FetchAptoideLinks(soup,appList)
except requests.ConnectionError as connError:
logger.log("Connection Error while connecting to Play store: ", urls," Error: ", connError)
except requests.HTTPError as httpError:
logger.log("Invalid HTTP response to Play store: ", urls, " Error: ", httpError)
except requests.Timeout() as requestTimeoutError:
logger.log("Time-out to connect to Play store: ", urls, " Error: ", requestTimeoutError)
except requests.TooManyRedirects() as redirectsError:
logger.log("Too many redirects for connection to Play store: ", urls, " Error: ", redirectsError)
except Exception as e:
logger.log("Excpetion occured at Func Spider: ", e)
except requests.exceptions.Timeout as timeoutException:
logger.log("Timeout Exception", timeoutException)
return appList
def FetchAptoideLinks(self,soup,appList):
for link in soup.findAll("span",{"class":"bundle-item__info__span--big"}):
hreff = link.find('a').get('href')
