def test_dict(self): report = DownloadReport() report.downloads['s1'] = DownloadDetails() report.downloads['s2'] = DownloadDetails() expected = { 'total_sources': 0, 'downloads': { 's1': {'total_failed': 0, 'total_succeeded': 0}, 's2': {'total_failed': 0, 'total_succeeded': 0} }, } self.assertEqual(report.dict(), expected)
def test_update_rendering(self): repo_ids = ['repo_%d' % n for n in range(0, 3)] handler_report = ContentReport() summary_report = SummaryReport() summary_report.setup([{'repo_id': r} for r in repo_ids]) for r in summary_report.repository.values(): r.action = RepositoryReport.ADDED download_report = DownloadReport() download_report.downloads[PRIMARY_ID] = DownloadDetails() download_report.downloads['content-world'] = DownloadDetails() r.sources = download_report.dict() handler_report.set_succeeded(details=summary_report.dict()) renderer = UpdateRenderer(self.context.prompt, handler_report.dict()) renderer.render() self.assertEqual(len(self.recorder.lines), 59)
def __call__(self): """ Begin processing the batch of requests. Download files using available alternate content sources. An attempt is made to satisfy each download request using the alternate content sources in the order specified by priority. The specified downloader is designated as the primary source and is used in the event that the request cannot be completed using alternate sources. :return: The download report. :rtype: DownloadReport """ report = DownloadReport() report.total_sources = len(self.sources) for request in self.requests: event = Started(request) event(self.listener) request.find_sources(self.primary, self.sources) for source, url in request.sources: details = report.downloads.setdefault(source.id, DownloadDetails()) try: self._download(url, request.destination, source) details.total_succeeded += 1 request.downloaded = True event = Succeeded(request) event(self.listener) break except DownloadFailed, df: request.errors.append(str(df)) details.total_failed += 1 if request.downloaded: continue event = Failed(request) event(self.listener)
def __call__(self): """ Begin processing the batch of requests. Download files using available alternate content sources. An attempt is made to satisfy each download request using the alternate content sources in the order specified by priority. The specified downloader is designated as the primary source and is used in the event that the request cannot be completed using alternate sources. :return: The download report. :rtype: DownloadReport """ count = 0 report = DownloadReport() report.total_sources = len(self.sources) try: for request in self.requests: request.find_sources(self.primary, self.sources) self.dispatch(request) count += 1 finally: self.in_progress.wait(count) for queue in self.queues.values(): queue.put(None) queue.halt() for queue in self.queues.values(): queue.join() for source_id, queue in self.queues.items(): listener = queue.downloader.event_listener downloads = report.downloads.setdefault(source_id, DownloadDetails()) downloads.total_succeeded += listener.total_succeeded downloads.total_failed += listener.total_failed return report
def test_dict(self): report = SummaryReport() report.errors.append(NodeError(1, A=1, B=2)) report.errors.append(NodeError(2, A=10, B=20)) report.sources.total_passes = 1 report.sources.total_sources = 10 details = DownloadDetails() details.total_succeeded = 98 details.total_failed = 2 report.sources.downloads['content-world'] = details details = DownloadDetails() details.total_succeeded = 999999 details.total_failed = 0 report.sources.downloads['content-galaxy'] = details # test d = report.dict() # validation expected = { 'sources': { 'downloads': { 'content-world': { 'total_failed': 2, 'total_succeeded': 98 }, 'content-galaxy': { 'total_failed': 0, 'total_succeeded': 999999 } }, 'total_passes': 1, 'total_sources': 10 }, 'errors': [{ 'details': { 'A': 1, 'B': 2 }, 'error_id': 1 }, { 'details': { 'A': 10, 'B': 20 }, 'error_id': 2 }] } self.assertEqual(d, expected)
def test_dict(self): report = SummaryReport() report.errors.append(NodeError(1, A=1, B=2)) report.errors.append(NodeError(2, A=10, B=20)) report.sources.total_passes = 1 report.sources.total_sources = 10 details = DownloadDetails() details.total_succeeded = 98 details.total_failed = 2 report.sources.downloads['content-world'] = details details = DownloadDetails() details.total_succeeded = 999999 details.total_failed = 0 report.sources.downloads['content-galaxy'] = details # test d = report.dict() # validation expected = { 'sources': { 'downloads': { 'content-world': {'total_failed': 2, 'total_succeeded': 98}, 'content-galaxy': {'total_failed': 0, 'total_succeeded': 999999} }, 'total_sources': 10 }, 'errors': [ {'details': {'A': 1, 'B': 2}, 'error_id': 1}, {'details': {'A': 10, 'B': 20}, 'error_id': 2} ] } self.assertEqual(d, expected)
def download(self, cancel_event, downloader, request_list, listener=None): """ Download files using available alternate content sources. An attempt is made to satisfy each download request using the alternate content sources in the order specified by priority. The specified downloader is designated as the primary source and is used in the event that the request cannot be completed using alternate sources. :param cancel_event: An event that indicates the download has been canceled. :type cancel_event: threading.Event :param downloader: A primary nectar downloader. Used to download the requested content unit when it cannot be achieved using alternate content sources. :type downloader: nectar.downloaders.base.Downloader :param request_list: A list of pulp.server.content.sources.model.Request. :type request_list: list :param listener: An optional download request listener. :type listener: Listener :return: A download report. :rtype: DownloadReport """ self.refresh(cancel_event) report = DownloadReport() primary = PrimarySource(downloader) for request in request_list: request.find_sources(primary, self.sources) report.total_sources = len(self.sources) while not cancel_event.isSet(): collated = self.collated(request_list) if not collated: # Either we have exhausted our content sources or all # of the requests have been satisfied. break report.total_passes += 1 for source, nectar_list in collated.items(): downloader = source.get_downloader() nectar_listener = NectarListener(cancel_event, downloader, listener) downloader.event_listener = nectar_listener downloader.download(nectar_list) downloads = report.downloads.setdefault( source.id, DownloadDetails()) downloads.total_succeeded += nectar_listener.total_succeeded downloads.total_failed += nectar_listener.total_failed if cancel_event.isSet(): break return report
def test_dict(self): details = DownloadDetails() self.assertEqual(details.dict(), { 'total_failed': 0, 'total_succeeded': 0 })
def test_construction(self): details = DownloadDetails() self.assertEqual(details.total_succeeded, 0) self.assertEqual(details.total_failed, 0)
def test_dict(self): details = DownloadDetails() self.assertEqual(details.dict(), {'total_failed': 0, 'total_succeeded': 0})