def test_refresh_cancel_in_plugin(self, mock_plugin, *unused): container = ContentContainer(path=self.tmp_dir) event = CancelEvent(3) report = container.refresh(event, force=True) plugin = mock_plugin.return_value[0] self.assertEqual(plugin.refresh.call_count, 1) self.assertEqual(len(report), 1)
def test_download_cancelled_in_failed(self, mock_started, mock_cancel): request_list = [] for n in range(0, 5): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request( TYPE_ID, unit_key, 'http://unit-city/unit_%d' % n, os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = HTTPThreadedDownloader(DownloaderConfig()) container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() event = CancelEvent(2) report = container.download(event, downloader, request_list) self.assertTrue(mock_started.called) self.assertTrue(mock_cancel.called) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 0) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 5)
def test_download_fail_completely(self): request_list = [] _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10) shutil.rmtree(_dir) _dir = self.populate_content(PRIMARY, 0, 20) # primary for n in range(0, 10): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request( TYPE_ID, unit_key, 'http://redhat.com/%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = HTTPThreadedDownloader(DownloaderConfig()) listener = MockListener() container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() event = Event() container.download(event, downloader, request_list, listener) # primary for i in range(0, len(request_list)): request = request_list[i] self.assertFalse(request.downloaded) self.assertEqual(len(request.errors), 1) self.assertEqual(listener.download_started.call_count, len(request_list)) self.assertEqual(listener.download_succeeded.call_count, 0) self.assertEqual(listener.download_failed.call_count, len(request_list))
def test_refresh_cancel_in_plugin(self, mock_plugin, *unused): container = ContentContainer(path=self.tmp_dir) event = CancelEvent(3) report = container.refresh(event, force=True) plugin = mock_plugin.return_value[0] self.assertEqual(plugin.refresh.call_count, 1) self.assertEqual(len(report), 1)
def test_download_cancelled_in_failed(self, mock_started, mock_cancel): request_list = [] for n in range(0, 5): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request(TYPE_ID, unit_key, 'http://unit-city/unit_%d' % n, os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = HTTPThreadedDownloader(DownloaderConfig()) container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() event = CancelEvent(2) report = container.download(event, downloader, request_list) self.assertTrue(mock_started.called) self.assertTrue(mock_cancel.called) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 0) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 5)
def test_download_with_errors(self): request_list = [] _dir, cataloged = self.populate_catalog(ORPHANED, 0, 10) _dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10) shutil.rmtree(_dir) _dir = self.populate_content(PRIMARY, 0, 20) # unit-world for n in range(0, 10): request = Request(cataloged[n].type_id, cataloged[n].unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) # primary for n in range(11, 20): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request(TYPE_ID, unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) listener = Mock() container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() event = Event() # test report = container.download(event, downloader, request_list, listener) # validation # unit-world for i in range(0, 10): request = request_list[i] self.assertTrue(request.downloaded, msg='URL: %s' % request.url) self.assertEqual(len(request.errors), 1) with open(request.destination) as fp: s = fp.read() self.assertTrue(UNDERGROUND in s) # primary for i in range(11, len(request_list)): request = request_list[i] self.assertTrue(request.downloaded, msg='URL: %s' % request.url) self.assertEqual(len(request.errors), 0) with open(request.destination) as fp: s = fp.read() self.assertTrue(PRIMARY in s) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 3) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0) self.assertEqual(report.downloads[UNDERGROUND].total_succeeded, 10) self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 10)
def test_download(self): request_list = [] _dir, cataloged = self.populate_catalog(ORPHANED, 0, 10) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10) _dir = self.populate_content(PRIMARY, 0, 20) # unit-world for n in range(0, 10): request = Request( cataloged[n].type_id, cataloged[n].unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) # primary for n in range(11, 20): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request( TYPE_ID, unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) listener = MockListener() container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() event = Event() report = container.download(event, downloader, request_list, listener) # unit-world for i in range(0, 10): request = request_list[i] self.assertTrue(request.downloaded) self.assertEqual(len(request.errors), 0) with open(request.destination) as fp: s = fp.read() self.assertTrue(UNIT_WORLD in s) # primary for i in range(11, len(request_list)): request = request_list[i] self.assertTrue(request.downloaded) self.assertEqual(len(request.errors), 0) with open(request.destination) as fp: s = fp.read() self.assertTrue(PRIMARY in s) self.assertEqual(listener.download_started.call_count, len(request_list)) self.assertEqual(listener.download_succeeded.call_count, len(request_list)) self.assertEqual(listener.download_failed.call_count, 0) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 2) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 10) self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 0)
def test_download_with_errors(self): request_list = [] _dir, cataloged = self.populate_catalog(ORPHANED, 0, 10) _dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10) shutil.rmtree(_dir) _dir = self.populate_content(PRIMARY, 0, 20) # unit-world for n in range(0, 10): request = Request( cataloged[n].type_id, cataloged[n].unit_key, "file://%s/unit_%d" % (_dir, n), os.path.join(self.downloaded, "unit_%d" % n), ) request_list.append(request) # primary for n in range(11, 20): unit_key = {"name": "unit_%d" % n, "version": "1.0.%d" % n, "release": "1", "checksum": str(uuid4())} request = Request( TYPE_ID, unit_key, "file://%s/unit_%d" % (_dir, n), os.path.join(self.downloaded, "unit_%d" % n) ) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) listener = Mock() container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() event = Event() # test report = container.download(event, downloader, request_list, listener) # validation # unit-world for i in range(0, 10): request = request_list[i] self.assertTrue(request.downloaded, msg="URL: %s" % request.url) self.assertEqual(len(request.errors), 1) with open(request.destination) as fp: s = fp.read() self.assertTrue(UNDERGROUND in s) # primary for i in range(11, len(request_list)): request = request_list[i] self.assertTrue(request.downloaded, msg="URL: %s" % request.url) self.assertEqual(len(request.errors), 0) with open(request.destination) as fp: s = fp.read() self.assertTrue(PRIMARY in s) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 3) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0) self.assertEqual(report.downloads[UNDERGROUND].total_succeeded, 10) self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 10)
def test_refresh_exception(self, mock_refresh): container = ContentContainer(path=self.tmp_dir) event = Event() report = container.refresh(event, force=True) self.assertEqual(len(report), 2) for r in report: self.assertFalse(r.succeeded) self.assertEqual(r.added_count, 0) self.assertEqual(r.deleted_count, 0) self.assertEqual(len(r.errors), 1) collection = ContentCatalog.get_collection() self.assertEqual(mock_refresh.call_count, 2) self.assertEqual(collection.find().count(), 0)
def test_refresh_exception(self, mock_refresh): container = ContentContainer(path=self.tmp_dir) event = Event() report = container.refresh(event, force=True) self.assertEqual(len(report), 2) for r in report: self.assertFalse(r.succeeded) self.assertEqual(r.added_count, 0) self.assertEqual(r.deleted_count, 0) self.assertEqual(len(r.errors), 1) collection = ContentCatalog.get_collection() self.assertEqual(mock_refresh.call_count, 2) self.assertEqual(collection.find().count(), 0)
def test_refresh_failure(self, mock_plugin): container = ContentContainer(path=self.tmp_dir) event = Event() report = container.refresh(event, force=True) self.assertEqual(len(report), 5) for r in report: self.assertFalse(r.succeeded) self.assertEqual(r.added_count, 0) self.assertEqual(r.deleted_count, 0) self.assertEqual(len(r.errors), 1) plugin = mock_plugin.return_value[0] collection = ContentCatalog.get_collection() self.assertEqual(plugin.refresh.call_count, 5) self.assertEqual(collection.find().count(), 0)
def test_refresh_failure(self, mock_plugin): container = ContentContainer(path=self.tmp_dir) event = Event() report = container.refresh(event, force=True) self.assertEqual(len(report), 5) for r in report: self.assertFalse(r.succeeded) self.assertEqual(r.added_count, 0) self.assertEqual(r.deleted_count, 0) self.assertEqual(len(r.errors), 1) plugin = mock_plugin.return_value[0] collection = ContentCatalog.get_collection() self.assertEqual(plugin.refresh.call_count, 5) self.assertEqual(collection.find().count(), 0)
def test_download_with_errors(self): request_list = [] _dir, cataloged = self.populate_catalog(ORPHANED, 0, 1000) _dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 1000) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 1000) shutil.rmtree(_dir) _dir = self.populate_content(PRIMARY, 0, 2000) # unit-world for n in range(0, 1000): request = Request( cataloged[n].type_id, cataloged[n].unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) # primary for n in range(1001, 2000): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request( TYPE_ID, unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) event = Event() threshold = len(request_list) * 0.10 # cancel after 10% started listener = TestListener(event, threshold) container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() # test report = container.download(event, downloader, request_list, listener) # validation self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 2) self.assertTrue(0 < report.downloads[UNDERGROUND].total_succeeded < 500) self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0) self.assertTrue(0 < report.downloads[UNIT_WORLD].total_failed < 1000)
def test_download_with_unsupported_url(self): request_list = [] _dir, cataloged = self.populate_catalog(UNSUPPORTED_PROTOCOL, 0, 10) _dir = self.populate_content(PRIMARY, 0, 20) # unit-world for n in range(0, 10): request = Request(cataloged[n].type_id, cataloged[n].unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) # primary for n in range(11, 20): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request(TYPE_ID, unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) listener = MockListener() container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() event = Event() report = container.download(event, downloader, request_list, listener) for i in range(0, len(request_list)): request = request_list[i] self.assertTrue(request.downloaded) self.assertEqual(len(request.errors), 0) with open(request.destination) as fp: s = fp.read() self.assertTrue(PRIMARY in s) self.assertEqual(listener.download_started.call_count, len(request_list)) self.assertEqual(listener.download_succeeded.call_count, len(request_list)) self.assertEqual(listener.download_failed.call_count, 0) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 19) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0)
def test_refresh(self, mock_plugin): container = ContentContainer(path=self.tmp_dir) event = Event() report = container.refresh(event, force=True) plugin = mock_plugin.return_value[0] self.assertEqual(plugin.refresh.call_count, 5) self.assertEqual(len(report), 5) for r in report: self.assertTrue(r.succeeded) self.assertEqual(r.added_count, 100) self.assertEqual(r.deleted_count, 0) calls = iter(plugin.refresh.call_args_list) for source in ContentSource.load_all(self.tmp_dir).values(): for url in source.urls: args = calls.next()[0] self.assertTrue(isinstance(args[0], CatalogerConduit)) self.assertEqual(args[1], source.descriptor) self.assertEqual(args[2], url)
def test_refresh(self, mock_plugin): container = ContentContainer(path=self.tmp_dir) event = Event() report = container.refresh(event, force=True) plugin = mock_plugin.return_value[0] self.assertEqual(plugin.refresh.call_count, 5) self.assertEqual(len(report), 5) for r in report: self.assertTrue(r.succeeded) self.assertEqual(r.added_count, 100) self.assertEqual(r.deleted_count, 0) calls = iter(plugin.refresh.call_args_list) for source in ContentSource.load_all(self.tmp_dir).values(): for url in source.urls: args = calls.next()[0] self.assertTrue(isinstance(args[0], CatalogerConduit)) self.assertEqual(args[1], source.descriptor) self.assertEqual(args[2], url)
def test_download_with_errors(self): request_list = [] _dir, cataloged = self.populate_catalog(ORPHANED, 0, 1000) _dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 1000) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 1000) shutil.rmtree(_dir) _dir = self.populate_content(PRIMARY, 0, 2000) # unit-world for n in range(0, 1000): request = Request(cataloged[n].type_id, cataloged[n].unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) # primary for n in range(1001, 2000): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request(TYPE_ID, unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) event = Event() threshold = len(request_list) * 0.10 # cancel after 10% started listener = TestListener(event, threshold) container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() # test report = container.download(event, downloader, request_list, listener) # validation self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 2) self.assertTrue( 0 < report.downloads[UNDERGROUND].total_succeeded < 500) self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0) self.assertTrue(0 < report.downloads[UNIT_WORLD].total_failed < 1000)
def test_download_cancelled_in_started(self, mock_cancel): request_list = [] _dir = self.populate_content(PRIMARY, 0, 5) for n in range(0, 5): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request( TYPE_ID, unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() event = CancelEvent(2) container.download(event, downloader, request_list) self.assertTrue(mock_cancel.called)
def test_download_fail_completely(self): request_list = [] _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10) shutil.rmtree(_dir) _dir = self.populate_content(PRIMARY, 0, 20) # primary for n in range(0, 10): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request(TYPE_ID, unit_key, 'http://redhat.com/%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = HTTPThreadedDownloader(DownloaderConfig()) listener = MockListener() container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() event = Event() report = container.download(event, downloader, request_list, listener) # primary for i in range(0, len(request_list)): request = request_list[i] self.assertFalse(request.downloaded) self.assertEqual(len(request.errors), 1) self.assertEqual(listener.download_started.call_count, len(request_list)) self.assertEqual(listener.download_succeeded.call_count, 0) self.assertEqual(listener.download_failed.call_count, len(request_list)) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 0) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 10)
def test_download(self): request_list = [] _dir, cataloged = self.populate_catalog(ORPHANED, 0, 1000) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 1000) _dir = self.populate_content(PRIMARY, 0, 2000) # unit-world for n in range(0, 1000): request = Request( cataloged[n].type_id, cataloged[n].unit_key, "file://%s/unit_%d" % (_dir, n), os.path.join(self.downloaded, "unit_%d" % n), ) request_list.append(request) # primary for n in range(1001, 2000): unit_key = {"name": "unit_%d" % n, "version": "1.0.%d" % n, "release": "1", "checksum": str(uuid4())} request = Request( TYPE_ID, unit_key, "file://%s/unit_%d" % (_dir, n), os.path.join(self.downloaded, "unit_%d" % n) ) request_list.append(request) event = Event() threshold = len(request_list) * 0.80 # cancel after 80% started downloader = LocalFileDownloader(DownloaderConfig()) listener = TestListener(event, threshold) container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() # test report = container.download(event, downloader, request_list, listener) # validation self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 2) self.assertTrue(0 < report.downloads[PRIMARY_ID].total_succeeded < 999) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 1000) self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 0)