def test_download_cancelled_in_failed(self, mock_started, mock_cancel): request_list = [] for n in range(0, 5): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request(TYPE_ID, unit_key, 'http://unit-city/unit_%d' % n, os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = HTTPThreadedDownloader(DownloaderConfig()) container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() event = CancelEvent(2) report = container.download(event, downloader, request_list) self.assertTrue(mock_started.called) self.assertTrue(mock_cancel.called) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 0) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 5)
def test_refresh_cancel_in_plugin(self, mock_plugin, *unused): container = ContentContainer(path=self.tmp_dir) event = CancelEvent(3) report = container.refresh(event, force=True) plugin = mock_plugin.return_value[0] self.assertEqual(plugin.refresh.call_count, 1) self.assertEqual(len(report), 1)
def test_download_with_errors(self): request_list = [] _dir, cataloged = self.populate_catalog(ORPHANED, 0, 10) _dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10) shutil.rmtree(_dir) _dir = self.populate_content(PRIMARY, 0, 20) # unit-world for n in range(0, 10): request = Request(cataloged[n].type_id, cataloged[n].unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) # primary for n in range(11, 20): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request(TYPE_ID, unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) listener = Mock() container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() event = Event() # test report = container.download(event, downloader, request_list, listener) # validation # unit-world for i in range(0, 10): request = request_list[i] self.assertTrue(request.downloaded, msg='URL: %s' % request.url) self.assertEqual(len(request.errors), 1) with open(request.destination) as fp: s = fp.read() self.assertTrue(UNDERGROUND in s) # primary for i in range(11, len(request_list)): request = request_list[i] self.assertTrue(request.downloaded, msg='URL: %s' % request.url) self.assertEqual(len(request.errors), 0) with open(request.destination) as fp: s = fp.read() self.assertTrue(PRIMARY in s) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 3) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0) self.assertEqual(report.downloads[UNDERGROUND].total_succeeded, 10) self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 10)
def test_download_cancelled_in_download(self): container = ContentContainer(path=self.tmp_dir) container.collated = Mock() event = CancelEvent(1) report = container.download(event, None, []) self.assertFalse(container.collated.called) self.assertEqual(report.total_passes, 0) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 0)
def test_download_cancelled_during_refreshing(self): downloader = LocalFileDownloader(DownloaderConfig()) container = ContentContainer(path=self.tmp_dir) container.collated = Mock() event = CancelEvent(1) report = container.download(event, downloader, []) self.assertFalse(container.collated.called) self.assertEqual(report.total_passes, 0) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 0)
def test_refresh_exception(self, mock_refresh): container = ContentContainer(path=self.tmp_dir) event = Event() report = container.refresh(event, force=True) self.assertEqual(len(report), 2) for r in report: self.assertFalse(r.succeeded) self.assertEqual(r.added_count, 0) self.assertEqual(r.deleted_count, 0) self.assertEqual(len(r.errors), 1) collection = ContentCatalog.get_collection() self.assertEqual(mock_refresh.call_count, 2) self.assertEqual(collection.find().count(), 0)
def test_refresh_failure(self, mock_plugin): container = ContentContainer(path=self.tmp_dir) event = Event() report = container.refresh(event, force=True) self.assertEqual(len(report), 5) for r in report: self.assertFalse(r.succeeded) self.assertEqual(r.added_count, 0) self.assertEqual(r.deleted_count, 0) self.assertEqual(len(r.errors), 1) plugin = mock_plugin.return_value[0] collection = ContentCatalog.get_collection() self.assertEqual(plugin.refresh.call_count, 5) self.assertEqual(collection.find().count(), 0)
def test_download_with_unsupported_url(self): request_list = [] _dir, cataloged = self.populate_catalog(UNSUPPORTED_PROTOCOL, 0, 10) _dir = self.populate_content(PRIMARY, 0, 20) # unit-world for n in range(0, 10): request = Request(cataloged[n].type_id, cataloged[n].unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) # primary for n in range(11, 20): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request(TYPE_ID, unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) listener = MockListener() container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() event = Event() report = container.download(event, downloader, request_list, listener) for i in range(0, len(request_list)): request = request_list[i] self.assertTrue(request.downloaded) self.assertEqual(len(request.errors), 0) with open(request.destination) as fp: s = fp.read() self.assertTrue(PRIMARY in s) self.assertEqual(listener.download_started.call_count, len(request_list)) self.assertEqual(listener.download_succeeded.call_count, len(request_list)) self.assertEqual(listener.download_failed.call_count, 0) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 19) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0)
def test_purge_orphans(self): _dir, cataloged = self.populate_catalog(ORPHANED, 0, 10) _dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10) collection = ContentCatalog.get_collection() self.assertEqual(collection.find().count(), 30) container = ContentContainer(path=self.tmp_dir) container.purge_orphans() self.assertEqual(collection.find().count(), 20) self.assertEqual(collection.find({'source_id': ORPHANED}).count(), 0) self.assertEqual( collection.find({ 'source_id': UNDERGROUND }).count(), 10) self.assertEqual( collection.find({ 'source_id': UNIT_WORLD }).count(), 10)
def test_refresh(self, mock_plugin): container = ContentContainer(path=self.tmp_dir) event = Event() report = container.refresh(event, force=True) plugin = mock_plugin.return_value[0] self.assertEqual(plugin.refresh.call_count, 5) self.assertEqual(len(report), 5) for r in report: self.assertTrue(r.succeeded) self.assertEqual(r.added_count, 100) self.assertEqual(r.deleted_count, 0) calls = iter(plugin.refresh.call_args_list) for source in ContentSource.load_all(self.tmp_dir).values(): for url in source.urls: args = calls.next()[0] self.assertTrue(isinstance(args[0], CatalogerConduit)) self.assertEqual(args[1], source.descriptor) self.assertEqual(args[2], url)
def test_download_with_errors(self): request_list = [] _dir, cataloged = self.populate_catalog(ORPHANED, 0, 1000) _dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 1000) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 1000) shutil.rmtree(_dir) _dir = self.populate_content(PRIMARY, 0, 2000) # unit-world for n in range(0, 1000): request = Request(cataloged[n].type_id, cataloged[n].unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) # primary for n in range(1001, 2000): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request(TYPE_ID, unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) event = Event() threshold = len(request_list) * 0.10 # cancel after 10% started listener = TestListener(event, threshold) container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() # test report = container.download(event, downloader, request_list, listener) # validation self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 2) self.assertTrue( 0 < report.downloads[UNDERGROUND].total_succeeded < 500) self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0) self.assertTrue(0 < report.downloads[UNIT_WORLD].total_failed < 1000)
def _add_units(self, request, unit_inventory): """ Determine the list of units contained in the parent inventory but are not contained in the child inventory and add them. For each unit, this is performed in the following steps: 1. Download the file (if defined) associated with the unit. 2. Add the unit to the child inventory. 3. Associate the unit to the repository. The unit is added only: 1. If no file is associated with unit. 2. The file associated with the unit is successfully downloaded. For units with files, the unit is added to the inventory as part of the unit download manager callback. :param request: A synchronization request. :type request: SyncRequest :param unit_inventory: The inventory of both parent and child content units. :type unit_inventory: UnitInventory """ download_list = [] units = unit_inventory.units_on_parent_only() request.progress.begin_adding_units(len(units)) listener = ContentDownloadListener(self, request) for unit, unit_ref in units: if request.cancelled(): return self._reset_storage_path(unit) if not self._needs_download(unit): # unit has no file associated self.add_unit(request, unit_ref.fetch()) continue unit_path, destination = self._path_and_destination(unit) unit_URL = pathlib.url_join(unit_inventory.base_URL, unit_path) _request = listener.create_request(unit_URL, destination, unit, unit_ref) download_list.append(_request) if request.cancelled(): return container = ContentContainer() request.summary.sources = \ container.download(request.cancel_event, request.downloader, download_list, listener) request.summary.errors.extend(listener.error_list)
def test_download_fail_completely(self): request_list = [] _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10) shutil.rmtree(_dir) _dir = self.populate_content(PRIMARY, 0, 20) # primary for n in range(0, 10): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request(TYPE_ID, unit_key, 'http://redhat.com/%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = HTTPThreadedDownloader(DownloaderConfig()) listener = MockListener() container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() event = Event() report = container.download(event, downloader, request_list, listener) # primary for i in range(0, len(request_list)): request = request_list[i] self.assertFalse(request.downloaded) self.assertEqual(len(request.errors), 1) self.assertEqual(listener.download_started.call_count, len(request_list)) self.assertEqual(listener.download_succeeded.call_count, 0) self.assertEqual(listener.download_failed.call_count, len(request_list)) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 0) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 10)