Example #1
0
    def test_multi_get_max_retry(self):
        """Tests the case when the number of the maximum retries is reached, due to the unsuccessful responses.
        `grequests.map` is called 3 times (based on `max_retry`), each time there is only one successful response.
        Eventually the call to `multi_get` returns the responses among which one is unsuccessful (`None`).
        """
        number_of_requests = 4
        query_params = [{'John Fitzgerald': 'Tom Hardy'}] * number_of_requests
        responses_to_calls = [
            self.mock_ok_responses(number_of_requests),
            self.mock_ok_responses(number_of_requests - 1),
            self.mock_ok_responses(number_of_requests - 2)
        ]
        # mock unsuccessful responses to the first call to grequests.map
        self.mock_unsuccessful_responses(responses_to_calls[0][0:3])
        # mock unsuccessful responses to the second call to grequests.map
        self.mock_unsuccessful_responses(responses_to_calls[1][1:3])
        # mock unsuccessful response to the third call to grequests.map
        self.mock_unsuccessful_response(responses_to_calls[2][1])
        grequests.map = MagicMock()
        grequests.map.side_effect = responses_to_calls

        actual_responses = MultiRequest(max_retry=3).multi_get(
            'example.com', query_params)

        T.assert_equal(3, grequests.map.call_count)
        T.assert_is(None, actual_responses[2])
Example #2
0
    def test_multi_get_max_retry(self):
        """Tests the case when the number of the maximum retries is reached, due to the unsuccessful responses.
        Request is repeated 3 times (based on `max_retry`), each time there is only one successful response.
        Eventually the call to `multi_get` returns the responses among which one is unsuccessful (`None`).
        """
        number_of_requests = 4
        query_params = [{'John Fitzgerald': 'Tom Hardy'}] * number_of_requests
        responses_to_calls = [
            self.mock_ok_responses(number_of_requests),
            self.mock_ok_responses(number_of_requests - 1),
            self.mock_ok_responses(number_of_requests - 2),
        ]
        # mock unsuccessful responses to the first call
        self.mock_unsuccessful_responses(responses_to_calls[0][0:3])
        # mock unsuccessful responses to the second call
        self.mock_unsuccessful_responses(responses_to_calls[1][1:3])
        # mock unsuccessful response to the third call
        self.mock_unsuccessful_response(responses_to_calls[2][1])
        get_mock = self.mock_request_futures(
            chain.from_iterable(responses_to_calls))

        actual_responses = MultiRequest(max_retry=3).multi_get(
            'example.com', query_params)

        T.assert_equal(get_mock.call_count, 9)
        T.assert_is(actual_responses[2], None)
Example #3
0
    def __init__(self, api_key, cache_file_name=None):
        auth_header = {'Authorization': 'Bearer {0}'.format(api_key)}
        self._requests = MultiRequest(default_headers=auth_header,
                                      max_requests=12,
                                      rate_limit=30)

        # Create an ApiCache if instructed to
        self._cache = ApiCache(cache_file_name) if cache_file_name else None
Example #4
0
 def test_multi_get_drop_404s(self):
     responses_to_calls = self.mock_ok_responses(3)
     self.mock_not_found_response(responses_to_calls[1])
     query_params = [{'Hugh Glass': 'Leonardo DiCaprio'}] * 3
     get_mock = self.mock_request_futures(responses_to_calls)
     result = MultiRequest(drop_404s=True).multi_get(
         'example.org', query_params)
     T.assert_equal(get_mock.call_count, 3)
     T.assert_is(result[1], None)
    def __init__(self, cache_file_name=None):
        """Establishes basic HTTP params and loads a cache.

        Args:
            cache_file_name: String file name of cache.
        """

        # TODO - lookup request rate limit
        # By observation, ShadowServer can be quite slow, so give it 90 seconds before it times out.
        self._requests = MultiRequest(max_requests=2, req_timeout=90.0)

        # Create an ApiCache if instructed to
        self._cache = ApiCache(cache_file_name) if cache_file_name else None
Example #6
0
    def test_multi_get_none_response(self):
        """Tests the behavior of the `multi_get()` method when one of the responses from `grequests.map` is `None`."""
        number_of_requests = 10
        query_params = [{'Jim Bridger': 'Will Poulter'}] * number_of_requests
        responses = self.mock_ok_responses(number_of_requests)
        responses[3] = None
        self.mock_grequests_map(responses)

        actual_responses = MultiRequest(max_retry=1).multi_get(
            'example.com', query_params)

        T.assert_equals(10, len(actual_responses))
        T.assert_is(None, actual_responses[3])
Example #7
0
    def test_multi_get_access_forbidden(self):
        """Tests the exception handling in the cases when a request returns "403 Forbidden"."""
        number_of_requests = 20
        query_params = [{
            'Hugh Glass': 'Leonardo DiCaprio'
        }] * number_of_requests
        responses = self.mock_ok_responses(number_of_requests)
        self.mock_forbidden_response(responses[13])
        self.mock_grequests_map(responses)

        with T.assert_raises_such_that(
                InvalidRequestError,
                lambda e: T.assert_equal(str(e), 'Access forbidden')):
            MultiRequest().multi_get('example.com', query_params)
Example #8
0
    def __init__(self, api_key, resources_per_req=25, cache_file_name=None):
        """Establishes basic HTTP params and loads a cache.

        Args:
            api_key: VirusTotal API key
            resources_per_req: Maximum number of resources (hashes, URLs)
                to be send in a single request
            cache_file_name: String file name of cache.
        """
        self._api_key = api_key
        self._resources_per_req = resources_per_req
        self._requests = MultiRequest()

        # Create an ApiCache if instructed to
        self._cache = ApiCache(cache_file_name) if cache_file_name else None
Example #9
0
    def __init__(self, cache_file_name=None, update_cache=True):
        """Establishes basic HTTP params and loads a cache.

        Args:
            cache_file_name: String file name of cache.
            update_cache: Determines whether cache should be written out back to the disk when closing it.
                          Default is `True`.
        """

        # TODO - lookup request rate limit
        # By observation, ShadowServer can be quite slow, so give it 90 seconds before it times out.
        self._requests = MultiRequest(max_requests=2, req_timeout=90.0)

        # Create an ApiCache if instructed to
        self._cache = ApiCache(cache_file_name, update_cache) if cache_file_name else None
Example #10
0
    def __init__(self, api_key, resources_per_req=25, cache_file_name=None, update_cache=True, req_timeout=None):
        """Establishes basic HTTP params and loads a cache.

        Args:
            api_key: VirusTotal API key
            resources_per_req: Maximum number of resources (hashes, URLs)
                to be send in a single request
            cache_file_name: String file name of cache.
            update_cache: Determines whether cache should be written out back to the disk when closing it.
                          Default is `True`.
            req_timeout: Maximum number of seconds to wait without reading a response byte before deciding an error has occurred.
                         Default is None.
        """
        self._api_key = api_key
        self._resources_per_req = resources_per_req
        self._requests = MultiRequest(req_timeout=req_timeout)

        # Create an ApiCache if instructed to
        self._cache = ApiCache(cache_file_name, update_cache) if cache_file_name else None
Example #11
0
    def test_multi_get_response_to_json(self):
        """Tests the exception handling in the cases when the response was supposed to return JSON but did not."""
        number_of_requests = 5
        query_params = [{
            'Andrew Henry': 'Domhnall Gleeson'
        }] * number_of_requests
        responses = self.mock_ok_responses(number_of_requests)
        self.mock_json_convertion_error(responses[3])
        self.mock_grequests_map(responses)
        logging.warning = MagicMock()

        actual_responses = MultiRequest().multi_get('example.com',
                                                    query_params)

        T.assert_equals(5, len(actual_responses))
        T.assert_is(None, actual_responses[3])
        logging.warning.called_once_with(
            'Expected response in JSON format from example.com/movie/TheRevenant but the actual response text is: This is not JSON'
        )
Example #12
0
    def __init__(self,
                 api_key,
                 cache_file_name=None,
                 update_cache=True,
                 req_timeout=None):
        """Establishes basic HTTP params and loads a cache.

        Args:
            api_key: VirusTotal API key
            cache_file_name: String file name of cache.
            update_cache: Determines whether cache should be written out back to the disk when closing it.
                          Default is `True`.
            req_timeout: Maximum number of seconds to wait without reading a response byte before deciding an error has occurred.
                         Default is None.
        """
        self._requests = MultiRequest(req_timeout=req_timeout,
                                      default_headers={'x-apikey': api_key},
                                      drop_404s=True)

        # Create an ApiCache if instructed to
        self._cache = ApiCache(cache_file_name,
                               update_cache) if cache_file_name else None
Example #13
0
    def __init__(self,
                 api_key,
                 resources_per_req=25,
                 cache_file_name=None,
                 update_cache=True):
        """Establishes basic HTTP params and loads a cache.

        Args:
            api_key: VirusTotal API key
            resources_per_req: Maximum number of resources (hashes, URLs)
                to be send in a single request
            cache_file_name: String file name of cache.
            update_cache: Determines whether cache should be written out back to the disk when closing it.
                          Default is `True`.
        """
        self._api_key = api_key
        self._resources_per_req = resources_per_req
        self._requests = MultiRequest()

        # Create an ApiCache if instructed to
        self._cache = ApiCache(cache_file_name,
                               update_cache) if cache_file_name else None
Example #14
0
    def test_multi_get_retry_only_unsuccessful_requests(self):
        """Tests whether only the unsuccessful requests are passed to the consequitive calls to `grequests.map()`.
        The calls to `grequests.map()` return 3 unsuccessful responses to the first call and then 2 unsuccessful responses to the second.
        The third (and the last) call to `grequests.map()` returns successful responses only.
        """
        responses_to_calls = [
            self.mock_ok_responses(10),
            self.mock_ok_responses(3),
            self.mock_ok_responses(2)
        ]
        # mock unsuccessful responses to the first call to grequests.map
        unsuccessful_responses_first_call = [
            responses_to_calls[0][2],
            responses_to_calls[0][3],
            responses_to_calls[0][5],
        ]
        self.mock_unsuccessful_responses(unsuccessful_responses_first_call)
        # mock unsuccessful responses to the second call to grequests.map
        unsuccessful_responses_second_call = [
            responses_to_calls[1][0],
            responses_to_calls[1][2],
        ]
        self.mock_unsuccessful_responses(unsuccessful_responses_second_call)
        grequests.map = MagicMock()
        grequests.map.side_effect = responses_to_calls

        query_params = [
            {
                'Max Rockatansky': 'Tom Hardy'
            },
            {
                'Imperator Furiosa': 'Charlize Theron'
            },
            {
                'Nux': 'Nicholas Hoult'
            },
            {
                'Immortan Joe': 'Hugh Keays-Byrne'
            },
            {
                'Slit': 'Josh Helman'
            },
            {
                'Rictus Erectus': 'Nathan Jones'
            },
            {
                'Toast the Knowing': 'Zoë Kravitz'
            },
            {
                'The Splendid Angharad': 'Rosie Huntington-Whiteley'
            },
            {
                'Capable': 'Riley Keough'
            },
            {
                'The Dag': 'Abbey Lee'
            },
        ]

        MultiRequest().multi_get('example.com', query_params)

        T.assert_equals(3, grequests.map.call_count)
        # assert that only the failed requests from the first call to grequests.map are passed in the second call
        second_call = grequests.map.call_args_list[1]
        self.assert_only_unsuccessful_requests(
            second_call, unsuccessful_responses_first_call)
        # assert that only the failed requests from the second call to grequests.map are passed in the third call
        third_call = grequests.map.call_args_list[2]
        self.assert_only_unsuccessful_requests(
            third_call, unsuccessful_responses_second_call)
Example #15
0
    def test_multi_get_retry_only_unsuccessful_requests(self):
        """Tests whether only the unsuccessful requests are passed to the consequitive request calls.
        3 unsuccessful responses to the first request batch and then 2 unsuccessful responses to the second.
        The third (and the last) returns successful responses only.
        """
        responses_to_calls = [
            self.mock_ok_responses(10),
            self.mock_ok_responses(3),
            self.mock_ok_responses(2),
        ]
        # mock unsuccessful responses to the first call
        unsuccessful_responses_first_call = [
            responses_to_calls[0][2],
            responses_to_calls[0][3],
            responses_to_calls[0][5],
        ]
        self.mock_unsuccessful_responses(unsuccessful_responses_first_call)
        # mock unsuccessful responses to the second call
        unsuccessful_responses_second_call = [
            responses_to_calls[1][0],
            responses_to_calls[1][2],
        ]
        self.mock_unsuccessful_responses(unsuccessful_responses_second_call)
        mock_get = self.mock_request_futures(
            chain.from_iterable(responses_to_calls))

        query_params = [
            {
                'Max Rockatansky': 'Tom Hardy'
            },
            {
                'Imperator Furiosa': 'Charlize Theron'
            },
            {
                'Nux': 'Nicholas Hoult'
            },
            {
                'Immortan Joe': 'Hugh Keays-Byrne'
            },
            {
                'Slit': 'Josh Helman'
            },
            {
                'Rictus Erectus': 'Nathan Jones'
            },
            {
                'Toast the Knowing': 'Zoë Kravitz'
            },
            {
                'The Splendid Angharad': 'Rosie Huntington-Whiteley'
            },
            {
                'Capable': 'Riley Keough'
            },
            {
                'The Dag': 'Abbey Lee'
            },
        ]

        MultiRequest().multi_get('example.com', query_params)
        T.assert_equal(mock_get.call_count, 15)  # 10 + 3 + 2
        call_params = [
            kwargs['params'] for args, kwargs in mock_get.call_args_list
        ]
        # Assert retries
        call_params_keys = [list(cp.keys())[0] for cp in call_params]
        T.assert_equal(call_params_keys.count('Nux'), 3)
        T.assert_equal(call_params_keys.count('Immortan Joe'), 2)
        T.assert_equal(call_params_keys.count('Rictus Erectus'), 3)