Exemple #1
0
    def test_ignore_download_exception(self, mocker):
        url = 'http://example.com'

        socket_errors = ((socket.timeout(), 'ignore_timeout'), )

        urllib_errors = (
            (URLError(reason=socket.timeout(),
                      filename=None), 'ignore_timeout'),
            (URLError(reason=socket.gaierror(),
                      filename=None), 'ignore_network'),
            (URLError(reason=OSError(), filename=None), 'ignore_network'),
        )

        http_status_codes = (403, 404, 500)
        http_errors = []
        for code in http_status_codes:
            http_errors.append((
                HTTPError(url=url, code=code, msg='msg', hdrs={}, fp=None),
                'ignore_{}'.format(code),
            ))

        errors_and_flags = (tuple(socket_errors) + tuple(urllib_errors) +
                            tuple(http_errors))

        grabber = Grabber('http://example.com')

        for e, flag in errors_and_flags:
            # Ignore error
            setattr(grabber, flag, True)
            assert grabber.ignore_download_exception(e) is True

            # Don't ignore error
            setattr(grabber, flag, False)
            assert grabber.ignore_download_exception(e) is False
Exemple #2
0
    def test_download_image__exception_ignored(self, mocker):
        correct_url = 'http://example.com'
        now = datetime.now()
        e = Exception()

        dummy_request = {'requested_at': now, 'url': correct_url}
        dummy_downloader = mocker.Mock(side_effect=e)

        grabber = Grabber('http://bad-url.com')
        grabber.get_download_callable = mocker.Mock(
            grabber.get_download_callable,
            autospec=True,
            return_value=dummy_downloader,
        )
        grabber.ignore_download_exception = mocker.Mock(
            grabber.ignore_download_exception,
            autospec=True,
            return_value=True)

        expected_result = {
            'requested_at': now,
            'url': correct_url,
            'image': None,
            'error': e
        }

        result = grabber.download_image(dummy_request)

        assert result == expected_result
        grabber.get_download_callable.assert_called_once_with()
        dummy_downloader.assert_called_once_with(correct_url, grabber)
        grabber.ignore_download_exception.assert_called_once_with(e)
Exemple #3
0
    def test_download_image__exception_raised(self, mocker):
        correct_url = 'http://example.com'
        now = datetime.now()
        e = Exception()

        dummy_request = {'requested_at': now, 'url': correct_url}
        dummy_downloader = mocker.Mock(side_effect=e)

        grabber = Grabber('http://bad-url.com')
        grabber.get_download_callable = mocker.Mock(
            grabber.get_download_callable,
            autospec=True,
            return_value=dummy_downloader,
        )
        grabber.ignore_download_exception = mocker.Mock(
            grabber.ignore_download_exception,
            autospec=True,
            return_value=False)

        with pytest.raises(Exception):
            grabber.download_image(dummy_request)

        grabber.get_download_callable.assert_called_once_with()
        dummy_downloader.assert_called_once_with(correct_url, grabber)
        grabber.ignore_download_exception.assert_called_once_with(e)
Exemple #4
0
 def test_ignore_download_exception__ignores_general_exceptions(self):
     grabber = Grabber('http://example.com')
     e = Exception()
     assert grabber.ignore_download_exception(e) is False