def test_dict(self):
report = DownloadReport()
report.downloads['s1'] = DownloadDetails()
report.downloads['s2'] = DownloadDetails()
expected = {
'total_sources': 0,
'downloads': {
's1': {'total_failed': 0, 'total_succeeded': 0},
's2': {'total_failed': 0, 'total_succeeded': 0}
},
}
self.assertEqual(report.dict(), expected)
def test_update_rendering(self):
repo_ids = ['repo_%d' % n for n in range(0, 3)]
handler_report = ContentReport()
summary_report = SummaryReport()
summary_report.setup([{'repo_id': r} for r in repo_ids])
for r in summary_report.repository.values():
r.action = RepositoryReport.ADDED
download_report = DownloadReport()
download_report.downloads[PRIMARY_ID] = DownloadDetails()
download_report.downloads['content-world'] = DownloadDetails()
r.sources = download_report.dict()
handler_report.set_succeeded(details=summary_report.dict())
renderer = UpdateRenderer(self.context.prompt, handler_report.dict())
renderer.render()
self.assertEqual(len(self.recorder.lines), 59)
def __call__(self):
"""
Begin processing the batch of requests.
Download files using available alternate content sources.
An attempt is made to satisfy each download request using the alternate
content sources in the order specified by priority. The specified
downloader is designated as the primary source and is used in the event that
the request cannot be completed using alternate sources.
:return: The download report.
:rtype: DownloadReport
"""
report = DownloadReport()
report.total_sources = len(self.sources)
for request in self.requests:
event = Started(request)
event(self.listener)
request.find_sources(self.primary, self.sources)
for source, url in request.sources:
details = report.downloads.setdefault(source.id,
DownloadDetails())
try:
self._download(url, request.destination, source)
details.total_succeeded += 1
request.downloaded = True
event = Succeeded(request)
event(self.listener)
break
except DownloadFailed, df:
request.errors.append(str(df))
details.total_failed += 1
if request.downloaded:
continue
event = Failed(request)
event(self.listener)
def __call__(self):
"""
Begin processing the batch of requests.
Download files using available alternate content sources.
An attempt is made to satisfy each download request using the alternate
content sources in the order specified by priority. The specified
downloader is designated as the primary source and is used in the event that
the request cannot be completed using alternate sources.
:return: The download report.
:rtype: DownloadReport
"""
count = 0
report = DownloadReport()
report.total_sources = len(self.sources)
try:
for request in self.requests:
request.find_sources(self.primary, self.sources)
self.dispatch(request)
count += 1
finally:
self.in_progress.wait(count)
for queue in self.queues.values():
queue.put(None)
queue.halt()
for queue in self.queues.values():
queue.join()
for source_id, queue in self.queues.items():
listener = queue.downloader.event_listener
downloads = report.downloads.setdefault(source_id, DownloadDetails())
downloads.total_succeeded += listener.total_succeeded
downloads.total_failed += listener.total_failed
return report
def test_dict(self):
report = SummaryReport()
report.errors.append(NodeError(1, A=1, B=2))
report.errors.append(NodeError(2, A=10, B=20))
report.sources.total_passes = 1
report.sources.total_sources = 10
details = DownloadDetails()
details.total_succeeded = 98
details.total_failed = 2
report.sources.downloads['content-world'] = details
details = DownloadDetails()
details.total_succeeded = 999999
details.total_failed = 0
report.sources.downloads['content-galaxy'] = details
# test
d = report.dict()
# validation
expected = {
'sources': {
'downloads': {
'content-world': {
'total_failed': 2,
'total_succeeded': 98
},
'content-galaxy': {
'total_failed': 0,
'total_succeeded': 999999
}
},
'total_passes': 1,
'total_sources': 10
},
'errors': [{
'details': {
'A': 1,
'B': 2
},
'error_id': 1
}, {
'details': {
'A': 10,
'B': 20
},
'error_id': 2
}]
}
self.assertEqual(d, expected)
def test_dict(self):
report = SummaryReport()
report.errors.append(NodeError(1, A=1, B=2))
report.errors.append(NodeError(2, A=10, B=20))
report.sources.total_passes = 1
report.sources.total_sources = 10
details = DownloadDetails()
details.total_succeeded = 98
details.total_failed = 2
report.sources.downloads['content-world'] = details
details = DownloadDetails()
details.total_succeeded = 999999
details.total_failed = 0
report.sources.downloads['content-galaxy'] = details
# test
d = report.dict()
# validation
expected = {
'sources': {
'downloads': {
'content-world': {'total_failed': 2, 'total_succeeded': 98},
'content-galaxy': {'total_failed': 0, 'total_succeeded': 999999}
},
'total_sources': 10
},
'errors': [
{'details': {'A': 1, 'B': 2}, 'error_id': 1},
{'details': {'A': 10, 'B': 20}, 'error_id': 2}
]
}
self.assertEqual(d, expected)
def download(self, cancel_event, downloader, request_list, listener=None):
"""
Download files using available alternate content sources.
An attempt is made to satisfy each download request using the alternate
content sources in the order specified by priority. The specified
downloader is designated as the primary source and is used in the event that
the request cannot be completed using alternate sources.
:param cancel_event: An event that indicates the download has been canceled.
:type cancel_event: threading.Event
:param downloader: A primary nectar downloader. Used to download the
requested content unit when it cannot be achieved using alternate
content sources.
:type downloader: nectar.downloaders.base.Downloader
:param request_list: A list of pulp.server.content.sources.model.Request.
:type request_list: list
:param listener: An optional download request listener.
:type listener: Listener
:return: A download report.
:rtype: DownloadReport
"""
self.refresh(cancel_event)
report = DownloadReport()
primary = PrimarySource(downloader)
for request in request_list:
request.find_sources(primary, self.sources)
report.total_sources = len(self.sources)
while not cancel_event.isSet():
collated = self.collated(request_list)
if not collated:
# Either we have exhausted our content sources or all
# of the requests have been satisfied.
break
report.total_passes += 1
for source, nectar_list in collated.items():
downloader = source.get_downloader()
nectar_listener = NectarListener(cancel_event, downloader,
listener)
downloader.event_listener = nectar_listener
downloader.download(nectar_list)
downloads = report.downloads.setdefault(
source.id, DownloadDetails())
downloads.total_succeeded += nectar_listener.total_succeeded
downloads.total_failed += nectar_listener.total_failed
if cancel_event.isSet():
break
return report
def test_dict(self):
details = DownloadDetails()
self.assertEqual(details.dict(), {
'total_failed': 0,
'total_succeeded': 0
})
def test_construction(self):
details = DownloadDetails()
self.assertEqual(details.total_succeeded, 0)
self.assertEqual(details.total_failed, 0)
def test_dict(self):
details = DownloadDetails()
self.assertEqual(details.dict(), {'total_failed': 0, 'total_succeeded': 0})