class HttpDownloaderTests(base_downloader.BaseDownloaderTests):
def setUp(self):
super(HttpDownloaderTests, self).setUp()
self.config.repo_plugin_config[constants.CONFIG_FEED] = TEST_SOURCE
self.downloader = HttpDownloader(self.repo, None, self.config, self.mock_cancelled_callback)
@mock.patch('pycurl.Curl')
def test_retrieve_metadata(self, mock_curl_constructor):
# Setup
mock_curl = mock.MagicMock()
mock_curl.getinfo.return_value = 200 # simulate a successful download
mock_curl_constructor.return_value = mock_curl
# Test
docs = self.downloader.retrieve_metadata(self.mock_progress_report)
# Verify
self.assertEqual(1, len(docs))
# Progress indicators
self.assertEqual(self.mock_progress_report.metadata_query_finished_count, 1)
self.assertEqual(self.mock_progress_report.metadata_query_total_count, 1)
self.assertEqual(2, self.mock_progress_report.update_progress.call_count)
@mock.patch('pycurl.Curl')
def test_retrieve_metadata_multiple_queries(self, mock_curl_constructor):
# Setup
mock_curl = mock.MagicMock()
mock_curl.getinfo.return_value = 200 # simulate a successful download
mock_curl_constructor.return_value = mock_curl
self.config.repo_plugin_config[constants.CONFIG_QUERIES] = ['a', ['b', 'c']]
# Test
docs = self.downloader.retrieve_metadata(self.mock_progress_report)
# Verify
self.assertEqual(2, len(docs))
# Progress indicators
self.assertEqual(self.mock_progress_report.metadata_query_finished_count, 2)
self.assertEqual(self.mock_progress_report.metadata_query_total_count, 2)
self.assertEqual(3, self.mock_progress_report.update_progress.call_count)
@mock.patch('pycurl.Curl')
def test_retrieve_metadata_with_error(self, mock_curl_constructor):
# Setup
mock_curl = mock.MagicMock()
mock_curl.getinfo.return_value = 404 # simulate an error
mock_curl_constructor.return_value = mock_curl
# Test
try:
self.downloader.retrieve_metadata(self.mock_progress_report)
self.fail()
except exceptions.FileNotFoundException, e:
expected = TEST_SOURCE + constants.REPO_METADATA_FILENAME
self.assertEqual(expected, e.location)
def setUp(self):
self.working_dir = tempfile.mkdtemp(prefix='http-downloader-tests')
self.repo = Repository('test-repo', working_dir=self.working_dir)
repo_config = {
constants.CONFIG_FEED: TEST_SOURCE,
}
self.config = PluginCallConfiguration({}, repo_config)
self.mock_cancelled_callback = mock.MagicMock().is_cancelled
self.mock_cancelled_callback.return_value = False
self.downloader = HttpDownloader(self.repo, None, self.config,
self.mock_cancelled_callback)
self.mock_progress_report = mock.MagicMock()
def setUp(self):
self.working_dir = tempfile.mkdtemp(prefix='http-downloader-tests')
self.repo = Repository('test-repo', working_dir=self.working_dir)
repo_config = {
constants.CONFIG_FEED: TEST_SOURCE,
}
self.config = PluginCallConfiguration({}, repo_config)
self.mock_cancelled_callback = mock.MagicMock().is_cancelled
self.mock_cancelled_callback.return_value = False
self.downloader = HttpDownloader(self.repo, None, self.config, self.mock_cancelled_callback)
self.mock_progress_report = mock.MagicMock()
def setUp(self):
super(HttpDownloaderTests, self).setUp()
self.config.repo_plugin_config[constants.CONFIG_FEED] = TEST_SOURCE
self.downloader = HttpDownloader(self.repo, None, self.config)
class HttpDownloaderTests(base_downloader.BaseDownloaderTests):
def setUp(self):
super(HttpDownloaderTests, self).setUp()
self.config.repo_plugin_config[constants.CONFIG_FEED] = TEST_SOURCE
self.downloader = HttpDownloader(self.repo, None, self.config)
@mock.patch('nectar.config.DownloaderConfig.finalize')
@mock.patch('nectar.downloaders.threaded.HTTPThreadedDownloader.download')
def test_retrieve_metadata(self, mock_downloader_download, mock_finalize):
docs = self.downloader.retrieve_metadata(self.mock_progress_report)
self.assertEqual(len(docs), 1)
self.assertEqual(mock_downloader_download.call_count, 1)
mock_finalize.assert_called_once()
@mock.patch('nectar.downloaders.threaded.HTTPThreadedDownloader.download')
def test_retrieve_metadata_multiple_queries(self,
mock_downloader_download):
self.config.repo_plugin_config[constants.CONFIG_QUERIES] = [
'a', ['b', 'c']
]
docs = self.downloader.retrieve_metadata(self.mock_progress_report)
self.assertEqual(2, len(docs))
self.assertEqual(mock_downloader_download.call_count, 1)
@mock.patch(
'pulp_puppet.plugins.importers.downloaders.web.HTTPMetadataDownloadEventListener'
)
@mock.patch('nectar.downloaders.threaded.HTTPThreadedDownloader.download')
def test_retrieve_metadata_with_error(self, mock_downloader_download,
mock_listener_constructor):
# Setup
mock_listener = mock.MagicMock()
report = DownloadReport(None, None)
report.error_msg = 'oops'
mock_listener.failed_reports = [report]
mock_listener_constructor.return_value = mock_listener
# Test
try:
self.downloader.retrieve_metadata(self.mock_progress_report)
self.fail()
except exceptions.FileRetrievalException:
pass
@mock.patch('nectar.config.DownloaderConfig.finalize')
@mock.patch('nectar.downloaders.threaded.HTTPThreadedDownloader.download')
def test_retrieve_module(self, mock_downloader_download, mock_finalize):
try:
stored_filename = self.downloader.retrieve_module(
self.mock_progress_report, self.module)
except:
self.fail()
mock_downloader_download.assert_called_once()
mock_finalize.assert_called_once()
@mock.patch(
'pulp_puppet.plugins.importers.downloaders.web.HTTPModuleDownloadEventListener'
)
@mock.patch('nectar.downloaders.threaded.HTTPThreadedDownloader.download')
def test_retrieve_module_missing_module(self, mock_downloader_download,
mock_listener_constructor):
# Setup
mock_listener = mock.MagicMock()
report = DownloadReport(None, None)
report.error_msg = 'oops'
mock_listener.failed_reports = [report]
mock_listener_constructor.return_value = mock_listener
# Test
try:
self.downloader.retrieve_module(self.mock_progress_report,
self.module)
self.fail()
except exceptions.FileRetrievalException:
expected_filename = web._create_download_tmp_dir(self.working_dir)
expected_filename = os.path.join(expected_filename,
self.module.filename())
self.assertFalse(os.path.exists(os.path.join(expected_filename)))
@mock.patch('nectar.downloaders.threaded.HTTPThreadedDownloader.download')
def test_cleanup_module(self, mock_downloader_download):
stored_filename = self.downloader.retrieve_module(
self.mock_progress_report, self.module)
self.downloader.cleanup_module(self.module)
self.assertTrue(not os.path.exists(stored_filename))
def test_create_metadata_download_urls(self):
# Setup
self.config.repo_plugin_config[constants.CONFIG_QUERIES] = [
'a', ['b', 'c']
]
# Test
urls = self.downloader._create_metadata_download_urls()
# Verify
self.assertEqual(2, len(urls))
self.assertEqual(urls[0], TEST_SOURCE + 'modules.json?q=a')
self.assertEqual(urls[1], TEST_SOURCE + 'modules.json?q=b&q=c')
def test_create_metadata_download_urls_no_queries(self):
# Test
urls = self.downloader._create_metadata_download_urls()
# Verify
self.assertEqual(1, len(urls))
self.assertEqual(urls[0], TEST_SOURCE + 'modules.json')
def test_create_module_url(self):
# Test
# Strip the trailing / off to make sure that branch is followed
self.config.repo_plugin_config[
constants.CONFIG_FEED] = TEST_SOURCE[:-1]
url = self.downloader._create_module_url(self.module)
# Verify
expected = TEST_SOURCE + \
constants.HOSTED_MODULE_FILE_RELATIVE_PATH % (self.module.author[0], self.module.author) + \
self.module.filename()
self.assertEqual(url, expected)
def test_create_download_tmp_dir(self):
# Test
created = web._create_download_tmp_dir(self.working_dir)
# Verify
self.assertTrue(os.path.exists(created))
self.assertEqual(created,
os.path.join(self.working_dir, web.DOWNLOAD_TMP_DIR))
class LiveHttpDownloaderTests(unittest.TestCase):
def setUp(self):
self.working_dir = tempfile.mkdtemp(prefix='http-downloader-tests')
self.repo = Repository('test-repo', working_dir=self.working_dir)
repo_config = {
constants.CONFIG_FEED: TEST_SOURCE,
}
self.config = PluginCallConfiguration({}, repo_config)
self.mock_cancelled_callback = mock.MagicMock().is_cancelled
self.mock_cancelled_callback.return_value = False
self.downloader = HttpDownloader(self.repo, None, self.config,
self.mock_cancelled_callback)
self.mock_progress_report = mock.MagicMock()
def tearDown(self):
if os.path.exists(self.working_dir):
shutil.rmtree(self.working_dir)
def test_retrieve_metadata(self):
docs = self._run_metadata_test()
self.assertEqual(1, len(docs))
def test_retrieve_metadata_with_vague_query(self):
self.config.repo_plugin_config[constants.CONFIG_QUERIES] = ['httpd']
docs = self._run_metadata_test()
self.assertEqual(1, len(docs))
def test_retrieve_metadata_with_specific_query(self):
self.config.repo_plugin_config[constants.CONFIG_QUERIES] = [
'thias/php'
]
docs = self._run_metadata_test()
self.assertEqual(1, len(docs))
def test_retrieve_metadata_with_multiple_specific_queries(self):
self.config.repo_plugin_config[constants.CONFIG_QUERIES] = [
'thias/php', 'larstobi/dns'
]
docs = self._run_metadata_test()
self.assertEqual(2, len(docs))
def test_retrieve_module(self):
module = model.Module('php', '0.2.1', 'thias')
# Test
self.downloader.retrieve_module(self.mock_progress_report, module)
# Verify
module_dir = os.path.join(self.working_dir, DOWNLOAD_TMP_DIR)
expected_file = os.path.join(module_dir, module.filename())
self.assertTrue(os.path.exists(expected_file))
# Extract the metadata to make sure the tar is valid and we can open it
metadata.extract_metadata(module, expected_file, self.working_dir)
# Spot check that something from the metadata was stuffed into the module
self.assertTrue(module.checksums is not None)
def _run_metadata_test(self):
# Test
docs = self.downloader.retrieve_metadata(self.mock_progress_report)
# Verify
parsed = model.RepositoryMetadata()
for d in docs:
parsed.update_from_json(d)
print('Number of Modules: %s' % len(parsed.modules))
return docs
def setUp(self):
super(HttpDownloaderTests, self).setUp()
self.config.repo_plugin_config[constants.CONFIG_FEED] = TEST_SOURCE
self.downloader = HttpDownloader(self.repo, None, self.config, self.mock_cancelled_callback)
class LiveHttpDownloaderTests(unittest.TestCase):
def setUp(self):
self.working_dir = tempfile.mkdtemp(prefix='http-downloader-tests')
self.repo = Repository('test-repo', working_dir=self.working_dir)
repo_config = {
constants.CONFIG_FEED: TEST_SOURCE,
}
self.config = PluginCallConfiguration({}, repo_config)
self.mock_cancelled_callback = mock.MagicMock().is_cancelled
self.mock_cancelled_callback.return_value = False
self.downloader = HttpDownloader(self.repo, None, self.config, self.mock_cancelled_callback)
self.mock_progress_report = mock.MagicMock()
def tearDown(self):
if os.path.exists(self.working_dir):
shutil.rmtree(self.working_dir)
def test_retrieve_metadata(self):
docs = self._run_metadata_test()
self.assertEqual(1, len(docs))
def test_retrieve_metadata_with_vague_query(self):
self.config.repo_plugin_config[constants.CONFIG_QUERIES] = ['httpd']
docs = self._run_metadata_test()
self.assertEqual(1, len(docs))
def test_retrieve_metadata_with_specific_query(self):
self.config.repo_plugin_config[constants.CONFIG_QUERIES] = ['thias/php']
docs = self._run_metadata_test()
self.assertEqual(1, len(docs))
def test_retrieve_metadata_with_multiple_specific_queries(self):
self.config.repo_plugin_config[constants.CONFIG_QUERIES] = ['thias/php', 'larstobi/dns']
docs = self._run_metadata_test()
self.assertEqual(2, len(docs))
def test_retrieve_module(self):
module = model.Module('php', '0.2.1', 'thias')
# Test
self.downloader.retrieve_module(self.mock_progress_report, module)
# Verify
module_dir = os.path.join(self.working_dir, DOWNLOAD_TMP_DIR)
expected_file = os.path.join(module_dir, module.filename())
self.assertTrue(os.path.exists(expected_file))
# Extract the metadata to make sure the tar is valid and we can open it
metadata.extract_metadata(module, expected_file, self.working_dir)
# Spot check that something from the metadata was stuffed into the module
self.assertTrue(module.checksums is not None)
def _run_metadata_test(self):
# Test
docs = self.downloader.retrieve_metadata(self.mock_progress_report)
# Verify
parsed = model.RepositoryMetadata()
for d in docs:
parsed.update_from_json(d)
print('Number of Modules: %s' % len(parsed.modules))
return docs
class HttpDownloaderTests(base_downloader.BaseDownloaderTests):
def setUp(self):
super(HttpDownloaderTests, self).setUp()
self.config.repo_plugin_config[constants.CONFIG_FEED] = TEST_SOURCE
self.downloader = HttpDownloader(self.repo, None, self.config)
@mock.patch('nectar.config.DownloaderConfig.finalize')
@mock.patch('nectar.downloaders.threaded.HTTPThreadedDownloader.download')
def test_retrieve_metadata(self, mock_downloader_download, mock_finalize):
docs = self.downloader.retrieve_metadata(self.mock_progress_report)
self.assertEqual(len(docs), 1)
self.assertEqual(mock_downloader_download.call_count, 1)
mock_finalize.assert_called_once()
@mock.patch('nectar.downloaders.threaded.HTTPThreadedDownloader.download')
def test_retrieve_metadata_multiple_queries(self, mock_downloader_download):
self.config.repo_plugin_config[constants.CONFIG_QUERIES] = ['a', ['b', 'c']]
docs = self.downloader.retrieve_metadata(self.mock_progress_report)
self.assertEqual(2, len(docs))
self.assertEqual(mock_downloader_download.call_count, 1)
@mock.patch('pulp_puppet.plugins.importers.downloaders.web.HTTPMetadataDownloadEventListener')
@mock.patch('nectar.downloaders.threaded.HTTPThreadedDownloader.download')
def test_retrieve_metadata_with_error(self, mock_downloader_download, mock_listener_constructor):
# Setup
mock_listener = mock.MagicMock()
report = DownloadReport(None, None)
report.error_msg = 'oops'
mock_listener.failed_reports = [report]
mock_listener_constructor.return_value = mock_listener
# Test
try:
self.downloader.retrieve_metadata(self.mock_progress_report)
self.fail()
except exceptions.FileRetrievalException:
pass
@mock.patch('nectar.config.DownloaderConfig.finalize')
@mock.patch('nectar.downloaders.threaded.HTTPThreadedDownloader.download')
def test_retrieve_module(self, mock_downloader_download, mock_finalize):
try:
stored_filename = self.downloader.retrieve_module(self.mock_progress_report, self.module)
except:
self.fail()
mock_downloader_download.assert_called_once()
mock_finalize.assert_called_once()
@mock.patch('pulp_puppet.plugins.importers.downloaders.web.HTTPModuleDownloadEventListener')
@mock.patch('nectar.downloaders.threaded.HTTPThreadedDownloader.download')
def test_retrieve_module_missing_module(self, mock_downloader_download, mock_listener_constructor):
# Setup
mock_listener = mock.MagicMock()
report = DownloadReport(None, None)
report.error_msg = 'oops'
mock_listener.failed_reports = [report]
mock_listener_constructor.return_value = mock_listener
# Test
try:
self.downloader.retrieve_module(self.mock_progress_report, self.module)
self.fail()
except exceptions.FileRetrievalException:
expected_filename = web._create_download_tmp_dir(self.working_dir)
expected_filename = os.path.join(expected_filename, self.module.filename())
self.assertFalse(os.path.exists(os.path.join(expected_filename)))
@mock.patch('nectar.downloaders.threaded.HTTPThreadedDownloader.download')
def test_cleanup_module(self, mock_downloader_download):
stored_filename = self.downloader.retrieve_module(self.mock_progress_report, self.module)
self.downloader.cleanup_module(self.module)
self.assertTrue(not os.path.exists(stored_filename))
def test_create_metadata_download_urls(self):
# Setup
self.config.repo_plugin_config[constants.CONFIG_QUERIES] = ['a', ['b', 'c']]
# Test
urls = self.downloader._create_metadata_download_urls()
# Verify
self.assertEqual(2, len(urls))
self.assertEqual(urls[0], TEST_SOURCE + 'modules.json?q=a')
self.assertEqual(urls[1], TEST_SOURCE + 'modules.json?q=b&q=c')
def test_create_metadata_download_urls_no_queries(self):
# Test
urls = self.downloader._create_metadata_download_urls()
# Verify
self.assertEqual(1, len(urls))
self.assertEqual(urls[0], TEST_SOURCE + 'modules.json')
def test_create_module_url(self):
# Test
# Strip the trailing / off to make sure that branch is followed
self.config.repo_plugin_config[constants.CONFIG_FEED] = TEST_SOURCE[:-1]
url = self.downloader._create_module_url(self.module)
# Verify
expected = TEST_SOURCE + \
constants.HOSTED_MODULE_FILE_RELATIVE_PATH % (self.module.author[0], self.module.author) + \
self.module.filename()
self.assertEqual(url, expected)
def test_create_download_tmp_dir(self):
# Test
created = web._create_download_tmp_dir(self.working_dir)
# Verify
self.assertTrue(os.path.exists(created))
self.assertEqual(created, os.path.join(self.working_dir, web.DOWNLOAD_TMP_DIR))
class HttpDownloaderTests(base_downloader.BaseDownloaderTests):
def setUp(self):
super(HttpDownloaderTests, self).setUp()
self.config.repo_plugin_config[constants.CONFIG_FEED] = TEST_SOURCE
self.downloader = HttpDownloader(self.repo, None, self.config,
self.mock_cancelled_callback)
@mock.patch('pycurl.Curl')
def test_retrieve_metadata(self, mock_curl_constructor):
# Setup
mock_curl = mock.MagicMock()
mock_curl.getinfo.return_value = 200 # simulate a successful download
mock_curl_constructor.return_value = mock_curl
# Test
docs = self.downloader.retrieve_metadata(self.mock_progress_report)
# Verify
self.assertEqual(1, len(docs))
# Progress indicators
self.assertEqual(
self.mock_progress_report.metadata_query_finished_count, 1)
self.assertEqual(self.mock_progress_report.metadata_query_total_count,
1)
self.assertEqual(2,
self.mock_progress_report.update_progress.call_count)
@mock.patch('pycurl.Curl')
def test_retrieve_metadata_multiple_queries(self, mock_curl_constructor):
# Setup
mock_curl = mock.MagicMock()
mock_curl.getinfo.return_value = 200 # simulate a successful download
mock_curl_constructor.return_value = mock_curl
self.config.repo_plugin_config[constants.CONFIG_QUERIES] = [
'a', ['b', 'c']
]
# Test
docs = self.downloader.retrieve_metadata(self.mock_progress_report)
# Verify
self.assertEqual(2, len(docs))
# Progress indicators
self.assertEqual(
self.mock_progress_report.metadata_query_finished_count, 2)
self.assertEqual(self.mock_progress_report.metadata_query_total_count,
2)
self.assertEqual(3,
self.mock_progress_report.update_progress.call_count)
@mock.patch('pycurl.Curl')
def test_retrieve_metadata_with_error(self, mock_curl_constructor):
# Setup
mock_curl = mock.MagicMock()
mock_curl.getinfo.return_value = 404 # simulate an error
mock_curl_constructor.return_value = mock_curl
# Test
try:
self.downloader.retrieve_metadata(self.mock_progress_report)
self.fail()
except exceptions.FileNotFoundException, e:
expected = TEST_SOURCE + constants.REPO_METADATA_FILENAME
self.assertEqual(expected, e.location)