def __call__(self): """ Begin processing the batch of requests. Download files using available alternate content sources. An attempt is made to satisfy each download request using the alternate content sources in the order specified by priority. The specified downloader is designated as the primary source and is used in the event that the request cannot be completed using alternate sources. :return: The download report. :rtype: DownloadReport """ count = 0 report = DownloadReport() report.total_sources = len(self.sources) try: for request in self.requests: request.find_sources(self.primary, self.sources) self.dispatch(request) count += 1 finally: self.in_progress.wait(count) for queue in self.queues.values(): queue.put(None) queue.halt() for queue in self.queues.values(): queue.join() for source_id, queue in self.queues.items(): listener = queue.downloader.event_listener downloads = report.downloads.setdefault(source_id, DownloadDetails()) downloads.total_succeeded += listener.total_succeeded downloads.total_failed += listener.total_failed return report
def __call__(self): """ Begin processing the batch of requests. Download files using available alternate content sources. An attempt is made to satisfy each download request using the alternate content sources in the order specified by priority. The specified downloader is designated as the primary source and is used in the event that the request cannot be completed using alternate sources. :return: The download report. :rtype: DownloadReport """ report = DownloadReport() report.total_sources = len(self.sources) for request in self.requests: event = Started(request) event(self.listener) request.find_sources(self.primary, self.sources) for source, url in request.sources: details = report.downloads.setdefault(source.id, DownloadDetails()) try: self._download(url, request.destination, source) details.total_succeeded += 1 request.downloaded = True event = Succeeded(request) event(self.listener) break except DownloadFailed, df: request.errors.append(str(df)) details.total_failed += 1 if request.downloaded: continue event = Failed(request) event(self.listener)
def test_dict(self): report = DownloadReport() report.downloads['s1'] = DownloadDetails() report.downloads['s2'] = DownloadDetails() expected = { 'total_sources': 0, 'downloads': { 's1': {'total_failed': 0, 'total_succeeded': 0}, 's2': {'total_failed': 0, 'total_succeeded': 0} }, } self.assertEqual(report.dict(), expected)
def test_dict(self): report = DownloadReport() report.downloads["s1"] = DownloadDetails() report.downloads["s2"] = DownloadDetails() expected = { "total_sources": 0, "downloads": { "s1": {"total_failed": 0, "total_succeeded": 0}, "s2": {"total_failed": 0, "total_succeeded": 0}, }, } self.assertEqual(report.dict(), expected)
def test_update_rendering(self): repo_ids = ['repo_%d' % n for n in range(0, 3)] handler_report = ContentReport() summary_report = SummaryReport() summary_report.setup([{'repo_id': r} for r in repo_ids]) for r in summary_report.repository.values(): r.action = RepositoryReport.ADDED download_report = DownloadReport() download_report.downloads[PRIMARY_ID] = DownloadDetails() download_report.downloads['content-world'] = DownloadDetails() r.sources = download_report.dict() handler_report.set_succeeded(details=summary_report.dict()) renderer = UpdateRenderer(self.context.prompt, handler_report.dict()) renderer.render() self.assertEqual(len(self.recorder.lines), 59)
def download(self, cancel_event, downloader, request_list, listener=None): """ Download files using available alternate content sources. An attempt is made to satisfy each download request using the alternate content sources in the order specified by priority. The specified downloader is designated as the primary source and is used in the event that the request cannot be completed using alternate sources. :param cancel_event: An event that indicates the download has been canceled. :type cancel_event: threading.Event :param downloader: A primary nectar downloader. Used to download the requested content unit when it cannot be achieved using alternate content sources. :type downloader: nectar.downloaders.base.Downloader :param request_list: A list of pulp.server.content.sources.model.Request. :type request_list: list :param listener: An optional download request listener. :type listener: Listener :return: A download report. :rtype: DownloadReport """ self.refresh(cancel_event) report = DownloadReport() primary = PrimarySource(downloader) for request in request_list: request.find_sources(primary, self.sources) report.total_sources = len(self.sources) while not cancel_event.isSet(): collated = self.collated(request_list) if not collated: # Either we have exhausted our content sources or all # of the requests have been satisfied. break report.total_passes += 1 for source, nectar_list in collated.items(): downloader = source.get_downloader() nectar_listener = NectarListener(cancel_event, downloader, listener) downloader.event_listener = nectar_listener downloader.download(nectar_list) downloads = report.downloads.setdefault( source.id, DownloadDetails()) downloads.total_succeeded += nectar_listener.total_succeeded downloads.total_failed += nectar_listener.total_failed if cancel_event.isSet(): break return report
def download(self, cancel_event, downloader, request_list, listener=None): """ Download files using available alternate content sources. An attempt is made to satisfy each download request using the alternate content sources in the order specified by priority. The specified downloader is designated as the primary source and is used in the event that the request cannot be completed using alternate sources. :param cancel_event: An event that indicates the download has been canceled. :type cancel_event: threading.Event :param downloader: A primary nectar downloader. Used to download the requested content unit when it cannot be achieved using alternate content sources. :type downloader: nectar.downloaders.base.Downloader :param request_list: A list of pulp.server.content.sources.model.Request. :type request_list: list :param listener: An optional download request listener. :type listener: Listener :return: A download report. :rtype: DownloadReport """ self.refresh(cancel_event) report = DownloadReport() primary = PrimarySource(downloader) for request in request_list: request.find_sources(primary, self.sources) report.total_sources = len(self.sources) while not cancel_event.isSet(): collated = self.collated(request_list) if not collated: # Either we have exhausted our content sources or all # of the requests have been satisfied. break report.total_passes += 1 for source, nectar_list in collated.items(): downloader = source.get_downloader() nectar_listener = NectarListener(cancel_event, downloader, listener) downloader.event_listener = nectar_listener downloader.download(nectar_list) downloads = report.downloads.setdefault(source.id, DownloadDetails()) downloads.total_succeeded += nectar_listener.total_succeeded downloads.total_failed += nectar_listener.total_failed if cancel_event.isSet(): break return report
class SummaryReport(object): """ A report that provides both summary and details regarding the importing of content units associated with a repository. :ivar errors: List of errors. :type errors: ErrorList :ivar sources: The content sources container statistics. :type sources: DownloadReport """ def __init__(self): self.errors = ErrorList() self.sources = DownloadReport() def dict(self): """ Dictionary representation. :return: A dictionary representation. :rtype: dict """ return dict(errors=[e.dict() for e in self.errors], sources=self.sources.dict())
def test_construction(self): report = DownloadReport() self.assertEqual(report.total_passes, 0) self.assertEqual(report.total_sources, 0) self.assertEqual(report.downloads, {})
def __init__(self): self.errors = ErrorList() self.sources = DownloadReport()