def test_fetch_with_timeout(self): """ Test that the report state is failed and that the baseurl can be tried again. """ # requests.ConnectionError def timeout(*args, **kwargs): raise Timeout() with mock.patch('nectar.downloaders.threaded._logger') as mock_logger: URL = 'http://pulpproject.org/primary.xml' req = DownloadRequest(URL, StringIO()) session = threaded.build_session(self.config) session.get = timeout report = self.downloader._fetch(req, session) self.assertEqual(report.state, report.DOWNLOAD_FAILED) self.assertNotIn('pulpproject.org', self.downloader.failed_netlocs) session2 = threaded.build_session(self.config) session2.get = mock.MagicMock() report2 = self.downloader._fetch(req, session2) self.assertEqual(report2.state, report2.DOWNLOAD_FAILED) self.assertEqual(session2.get.call_count, 1) expected_log_message = "Request Timeout - Connection with " \ "http://pulpproject.org/primary.xml timed out." log_calls = [ mock_call[1][0] for mock_call in mock_logger.mock_calls ] self.assertIn(expected_log_message, log_calls)
def test_fetch_with_connection_error(self): """ Test that the report state is failed and that the baseurl is not tried again. """ # requests.ConnectionError def connection_error(*args, **kwargs): raise ConnectionError() with mock.patch('nectar.downloaders.threaded._logger') as mock_logger: URL = 'http://pulpproject.org/primary.xml' req = DownloadRequest(URL, StringIO()) session = threaded.build_session(self.config) session.get = connection_error try: report = self.downloader._fetch(req, session) except ConnectionError: raise AssertionError("ConnectionError should be raised") self.assertEqual(report.state, report.DOWNLOAD_FAILED) self.assertIn('pulpproject.org', self.downloader.failed_netlocs) session2 = threaded.build_session(self.config) session2.get = mock.MagicMock() report2 = self.downloader._fetch(req, session2) self.assertEqual(report2.state, report2.DOWNLOAD_FAILED) self.assertEqual(session2.get.call_count, 0) expected_log_message = "Connection Error - http://pulpproject.org/primary.xml " \ "could not be reached." log_calls = [mock_call[1][0] for mock_call in mock_logger.mock_calls] self.assertIn(expected_log_message, log_calls)
def test_fetch_with_timeout(self): """ Test that the report state is failed and that the baseurl can be tried again. """ # requests.ConnectionError def timeout(*args, **kwargs): raise Timeout() with mock.patch('nectar.downloaders.threaded._logger') as mock_logger: URL = 'http://pulpproject.org/primary.xml' req = DownloadRequest(URL, StringIO()) session = threaded.build_session(self.config) session.get = timeout report = self.downloader._fetch(req, session) self.assertEqual(report.state, report.DOWNLOAD_FAILED) self.assertNotIn('pulpproject.org', self.downloader.failed_netlocs) session2 = threaded.build_session(self.config) session2.get = mock.MagicMock() report2 = self.downloader._fetch(req, session2) self.assertEqual(report2.state, report2.DOWNLOAD_FAILED) self.assertEqual(session2.get.call_count, 1) expected_log_message = "Request Timeout - Connection with " \ "http://pulpproject.org/primary.xml timed out." log_calls = [mock_call[1][0] for mock_call in mock_logger.mock_calls] self.assertIn(expected_log_message, log_calls)
def test_build_session(self): kwargs = {'basic_auth_username': '******', 'basic_auth_password': '******', 'ssl_validation': False, 'ssl_client_cert_path': os.path.join(_find_data_directory(), 'pki/bogus/cert.pem'), 'ssl_client_key_path': os.path.join(_find_data_directory(), 'pki/bogus/key.pem'), 'proxy_url': 'https://invalid-proxy.com', 'proxy_port': 1234, 'proxy_username': '******', 'proxy_password': '******'} proxy_host = urllib.splithost(urllib.splittype(kwargs['proxy_url'])[1])[0] cfg = config.DownloaderConfig(**kwargs) session = threaded.build_session(cfg) self.assertEqual(session.stream, True) self.assertEqual(session.auth, (kwargs['basic_auth_username'], kwargs['basic_auth_password'])) self.assertEqual(session.cert, (kwargs['ssl_client_cert_path'], kwargs['ssl_client_key_path'])) self.assertEqual(session.proxies, {'http': 'https://%s:%s@%s:%d' % (kwargs['proxy_username'], kwargs['proxy_password'], proxy_host, kwargs['proxy_port']), 'https': 'http://%s:%s@%s:%d' % (kwargs['proxy_username'], kwargs['proxy_password'], proxy_host, kwargs['proxy_port'])})
def test_empty_string_proxy_username(self): """ Yoram Hekma submitted a patch[0] that ensured that an empty string in the proxy username would not count as the user supplying a username. This test ensures that behavior is tested. [0] https://github.com/pulp/nectar/pull/47 """ kwargs = { 'proxy_url': 'https://invalid-proxy.com', 'proxy_port': 1234, 'proxy_username': '', 'proxy_password': '' } proxy_host = urllib.splithost( urllib.splittype(kwargs['proxy_url'])[1])[0] cfg = config.DownloaderConfig(**kwargs) session = threaded.build_session(cfg) self.assertEqual(session.stream, True) self.assertFalse(hasattr(session.auth, 'proxy_username')) self.assertFalse(hasattr(session.auth, 'proxy_password')) # Since the user provided the empty string for the proxy username, the username and password # should be missing in the session proxies. self.assertEqual( session.proxies, { 'http': 'https://%s:%d' % (proxy_host, kwargs['proxy_port']), 'https': 'https://%s:%d' % (proxy_host, kwargs['proxy_port']) })
def test_empty_string_proxy_username(self): """ Yoram Hekma submitted a patch[0] that ensured that an empty string in the proxy username would not count as the user supplying a username. This test ensures that behavior is tested. [0] https://github.com/pulp/nectar/pull/47 """ kwargs = {'proxy_url': 'https://invalid-proxy.com', 'proxy_port': 1234, 'proxy_username': '', 'proxy_password': ''} proxy_host = urllib.splithost(urllib.splittype(kwargs['proxy_url'])[1])[0] cfg = config.DownloaderConfig(**kwargs) session = threaded.build_session(cfg) self.assertEqual(session.stream, True) self.assertFalse(hasattr(session.auth, 'proxy_username')) self.assertFalse(hasattr(session.auth, 'proxy_password')) # Since the user provided the empty string for the proxy username, the username and password # should be missing in the session proxies. self.assertEqual(session.proxies, {'http': 'https://%s:%d' % (proxy_host, kwargs['proxy_port']), 'https': 'https://%s:%d' % (proxy_host, kwargs['proxy_port'])})
def test_fetch_with_timeout(self): """ Test that the report state is failed and that the baseurl can be tried again. """ with mock.patch('nectar.downloaders.threaded._logger') as mock_logger: URL = 'http://fakeurl/primary.xml' req = DownloadRequest(URL, StringIO()) self.session.get.side_effect = Timeout report = self.downloader._fetch(req) self.assertEqual(report.state, report.DOWNLOAD_FAILED) self.assertNotIn('fakeurl', self.downloader.failed_netlocs) session2 = threaded.build_session(self.config) session2.get = mock.MagicMock() report2 = self.downloader._fetch(req) self.assertEqual(report2.state, report2.DOWNLOAD_FAILED) self.assertEqual(self.session.get.call_count, 2) expected_log_message = "Request Timeout - Connection with " \ "http://fakeurl/primary.xml timed out." log_calls = [mock_call[1][0] for mock_call in mock_logger.mock_calls] self.assertIn(expected_log_message, log_calls)
def test_fetch_with_timeout(self): """ Test that the report state is failed and that the baseurl can be tried again. """ with mock.patch('nectar.downloaders.threaded._logger') as mock_logger: URL = 'http://fakeurl/primary.xml' req = DownloadRequest(URL, StringIO()) self.session.get.side_effect = Timeout report = self.downloader._fetch(req) self.assertEqual(report.state, report.DOWNLOAD_FAILED) self.assertNotIn('fakeurl', self.downloader.failed_netlocs) session2 = threaded.build_session(self.config) session2.get = mock.MagicMock() report2 = self.downloader._fetch(req) self.assertEqual(report2.state, report2.DOWNLOAD_FAILED) self.assertEqual(self.session.get.call_count, 2) expected_log_message = "Request Timeout - Connection with " \ "http://fakeurl/primary.xml timed out." log_calls = [ mock_call[1][0] for mock_call in mock_logger.mock_calls ] self.assertIn(expected_log_message, log_calls)
def test_fetch_with_connection_error_badstatusline(self): """ Test that the baseurl is tried again if ConnectionError reason BadStatusLine happened. """ # requests.ConnectionError def connection_error(*args, **kwargs): raise ConnectionError('Connection aborted.', httplib.BadStatusLine("''",)) with mock.patch('nectar.downloaders.threaded._logger') as mock_logger: URL = 'http://pulpproject.org/primary.xml' req = DownloadRequest(URL, StringIO()) session = threaded.build_session(self.config) session.get = mock.MagicMock() session.get.side_effect = connection_error self.downloader._fetch(req, session) self.assertEqual(session.get.call_count, 2) expected_log_msg = ['Download of http://pulpproject.org/primary.xml failed. Re-trying.', 'Re-trying http://pulpproject.org/primary.xml due to remote server ' 'connection failure.', 'Download of http://pulpproject.org/primary.xml failed. Re-trying.', 'Download of http://pulpproject.org/primary.xml failed and reached ' 'maximum retries'] log_calls = [mock_call[1][0] for mock_call in mock_logger.mock_calls] self.assertEqual(expected_log_msg, log_calls)
def test_configure_session(self): kwargs = { 'basic_auth_username': '******', 'basic_auth_password': '******', 'headers': { 'pulp-header': 'awesome!' }, 'ssl_validation': False, 'ssl_client_cert_path': os.path.join(_find_data_directory(), 'pki/bogus/cert.pem'), 'ssl_client_key_path': os.path.join(_find_data_directory(), 'pki/bogus/key.pem'), 'proxy_url': 'https://invalid-proxy.com', 'proxy_port': 1234, 'proxy_username': '******', 'proxy_password': '******' } proxy_host = urllib.splithost( urllib.splittype(kwargs['proxy_url'])[1])[0] cfg = config.DownloaderConfig(**kwargs) session = threaded.build_session(cfg) self.assertEqual(session.stream, True) # other headers get added by the requests library, so we'll just check # for the one we added self.assertEqual(session.headers.get('pulp-header'), 'awesome!') self.assertEqual(session.auth.username, kwargs['basic_auth_username']) self.assertEqual(session.auth.password, kwargs['basic_auth_password']) self.assertEqual(session.auth.proxy_username, kwargs['proxy_username']) self.assertEqual(session.auth.proxy_password, kwargs['proxy_password']) self.assertEqual( session.cert, (kwargs['ssl_client_cert_path'], kwargs['ssl_client_key_path'])) # test proxy username and passwod are url encoded before sending the request self.assertEqual( session.proxies, { 'http': 'https://%s:%s@%s:%d' % (urllib.quote(kwargs['proxy_username']), urllib.quote(kwargs['proxy_password']), proxy_host, kwargs['proxy_port']), 'https': 'https://%s:%s@%s:%d' % (urllib.quote(kwargs['proxy_username']), urllib.quote(kwargs['proxy_password']), proxy_host, kwargs['proxy_port']) })
def test_request_headers(self): URL = 'http://pulpproject.org/robots.txt' req = DownloadRequest(URL, StringIO(), headers={'pulp_header': 'awesome!'}) response = Response() response.status_code = httplib.OK response.raw = StringIO('abc') session = threaded.build_session(self.config) session.get = mock.MagicMock(return_value=response, spec_set=session.get) self.downloader._fetch(req, session) session.get.assert_called_once_with(URL, headers={'pulp_header': 'awesome!'})
def test_wrong_content_encoding(self): URL = 'http://pulpproject.org/primary.xml.gz' req = DownloadRequest(URL, StringIO()) response = Response() response.status_code = httplib.OK response.raw = StringIO('abc') session = threaded.build_session(self.config) session.get = mock.MagicMock(return_value=response, spec_set=session.get) report = self.downloader._fetch(req, session) self.assertEqual(report.state, DOWNLOAD_SUCCEEDED) self.assertEqual(report.bytes_downloaded, 3) session.get.assert_called_once_with(URL, headers={'accept-encoding':''})
def test_request_headers(self): URL = 'http://pulpproject.org/robots.txt' req = DownloadRequest(URL, StringIO(), headers={'pulp_header': 'awesome!'}) response = Response() response.status_code = httplib.OK response.raw = StringIO('abc') session = threaded.build_session(self.config) session.get = mock.MagicMock(return_value=response, spec_set=session.get) self.downloader._fetch(req, session) session.get.assert_called_once_with( URL, headers={'pulp_header': 'awesome!'})
def test_normal_content_encoding(self): URL = 'http://pulpproject.org/primary.xml' req = DownloadRequest(URL, StringIO()) response = Response() response.status_code = httplib.OK response.iter_content = mock.MagicMock(return_value=['abc'], spec_set=response.iter_content) session = threaded.build_session(self.config) session.get = mock.MagicMock(return_value=response, spec_set=session.get) report = self.downloader._fetch(req, session) self.assertEqual(report.state, DOWNLOAD_SUCCEEDED) self.assertEqual(report.bytes_downloaded, 3) # passing "None" for headers lets the requests library add whatever # headers it thinks are appropriate. session.get.assert_called_once_with(URL, headers=None)
def test_wrong_content_encoding(self): URL = 'http://pulpproject.org/primary.xml.gz' req = DownloadRequest(URL, StringIO()) response = Response() response.status_code = httplib.OK response.raw = StringIO('abc') session = threaded.build_session(self.config) session.get = mock.MagicMock(return_value=response, spec_set=session.get) report = self.downloader._fetch(req, session) self.assertEqual(report.state, report.DOWNLOAD_SUCCEEDED) self.assertEqual(report.bytes_downloaded, 3) session.get.assert_called_once_with(URL, headers={'accept-encoding': ''})
def test_response_headers(self): """ Make sure that whatever headers come back on the response get added to the report. """ URL = 'http://pulpproject.org/robots.txt' req = DownloadRequest(URL, StringIO(), headers={'pulp_header': 'awesome!'}) response = Response() response.status_code = httplib.OK response.headers = {'content-length': '1024'} response.raw = StringIO('abc') session = threaded.build_session(self.config) session.get = mock.MagicMock(return_value=response, spec_set=session.get) report = self.downloader._fetch(req, session) self.assertEqual(report.headers['content-length'], '1024')
def test_normal_content_encoding(self): URL = 'http://pulpproject.org/primary.xml' req = DownloadRequest(URL, StringIO()) response = Response() response.status_code = httplib.OK response.iter_content = mock.MagicMock(return_value=['abc'], spec_set=response.iter_content) session = threaded.build_session(self.config) session.get = mock.MagicMock(return_value=response, spec_set=session.get) report = self.downloader._fetch(req, session) self.assertEqual(report.state, report.DOWNLOAD_SUCCEEDED) self.assertEqual(report.bytes_downloaded, 3) # passing "None" for headers lets the requests library add whatever # headers it thinks are appropriate. session.get.assert_called_once_with(URL, headers={})
def test_response_headers(self): """ Make sure that whatever headers come back on the response get added to the report. """ URL = 'http://pulpproject.org/robots.txt' req = DownloadRequest(URL, StringIO(), headers={'pulp_header': 'awesome!'}) response = Response() response.status_code = httplib.OK response.headers = {'content-length': '1024'} response.raw = StringIO('abc') session = threaded.build_session(self.config) session.get = mock.MagicMock(return_value=response, spec_set=session.get) report = self.downloader._fetch(req, session) self.assertEqual(report.headers['content-length'], '1024')
def test_configure_session(self): kwargs = {'basic_auth_username': '******', 'basic_auth_password': '******', 'headers': {'pulp-header': 'awesome!'}, 'ssl_validation': False, 'ssl_client_cert_path': os.path.join(_find_data_directory(), 'pki/bogus/cert.pem'), 'ssl_client_key_path': os.path.join(_find_data_directory(), 'pki/bogus/key.pem'), 'proxy_url': 'https://invalid-proxy.com', 'proxy_port': 1234, 'proxy_username': '******', 'proxy_password': '******'} proxy_host = urllib.splithost(urllib.splittype(kwargs['proxy_url'])[1])[0] cfg = config.DownloaderConfig(**kwargs) session = threaded.build_session(cfg) self.assertEqual(session.stream, True) # other headers get added by the requests library, so we'll just check # for the one we added self.assertEqual(session.headers.get('pulp-header'), 'awesome!') self.assertEqual(session.auth.username, kwargs['basic_auth_username']) self.assertEqual(session.auth.password, kwargs['basic_auth_password']) self.assertEqual(session.auth.proxy_username, kwargs['proxy_username']) self.assertEqual(session.auth.proxy_password, kwargs['proxy_password']) self.assertEqual(session.cert, (kwargs['ssl_client_cert_path'], kwargs['ssl_client_key_path'])) # test proxy username and passwod are url encoded before sending the request self.assertEqual(session.proxies, {'http': 'https://%s:%s@%s:%d' % (urllib.quote(kwargs['proxy_username']), urllib.quote(kwargs['proxy_password']), proxy_host, kwargs['proxy_port']), 'https': 'https://%s:%s@%s:%d' % (urllib.quote(kwargs['proxy_username']), urllib.quote(kwargs['proxy_password']), proxy_host, kwargs['proxy_port'])})