def __call__(self):
"""
Begin processing the batch of requests.
Download files using available alternate content sources.
An attempt is made to satisfy each download request using the alternate
content sources in the order specified by priority. The specified
downloader is designated as the primary source and is used in the event that
the request cannot be completed using alternate sources.
:return: The download report.
:rtype: DownloadReport
"""
count = 0
report = DownloadReport()
report.total_sources = len(self.sources)
try:
for request in self.requests:
request.find_sources(self.primary, self.sources)
self.dispatch(request)
count += 1
finally:
self.in_progress.wait(count)
for queue in self.queues.values():
queue.put(None)
queue.halt()
for queue in self.queues.values():
queue.join()
for source_id, queue in self.queues.items():
listener = queue.downloader.event_listener
downloads = report.downloads.setdefault(source_id, DownloadDetails())
downloads.total_succeeded += listener.total_succeeded
downloads.total_failed += listener.total_failed
return report
def __call__(self):
"""
Begin processing the batch of requests.
Download files using available alternate content sources.
An attempt is made to satisfy each download request using the alternate
content sources in the order specified by priority. The specified
downloader is designated as the primary source and is used in the event that
the request cannot be completed using alternate sources.
:return: The download report.
:rtype: DownloadReport
"""
report = DownloadReport()
report.total_sources = len(self.sources)
for request in self.requests:
event = Started(request)
event(self.listener)
request.find_sources(self.primary, self.sources)
for source, url in request.sources:
details = report.downloads.setdefault(source.id, DownloadDetails())
try:
self._download(url, request.destination, source)
details.total_succeeded += 1
request.downloaded = True
event = Succeeded(request)
event(self.listener)
break
except DownloadFailed, df:
request.errors.append(str(df))
details.total_failed += 1
if request.downloaded:
continue
event = Failed(request)
event(self.listener)
def __call__(self):
"""
Begin processing the batch of requests.
Download files using available alternate content sources.
An attempt is made to satisfy each download request using the alternate
content sources in the order specified by priority. The specified
downloader is designated as the primary source and is used in the event that
the request cannot be completed using alternate sources.
:return: The download report.
:rtype: DownloadReport
"""
report = DownloadReport()
report.total_sources = len(self.sources)
for request in self.requests:
event = Started(request)
event(self.listener)
request.find_sources(self.primary, self.sources)
for source, url in request.sources:
details = report.downloads.setdefault(source.id,
DownloadDetails())
try:
self._download(url, request.destination, source)
details.total_succeeded += 1
request.downloaded = True
event = Succeeded(request)
event(self.listener)
break
except DownloadFailed, df:
request.errors.append(str(df))
details.total_failed += 1
if request.downloaded:
continue
event = Failed(request)
event(self.listener)
def __call__(self):
"""
Begin processing the batch of requests.
Download files using available alternate content sources.
An attempt is made to satisfy each download request using the alternate
content sources in the order specified by priority. The specified
downloader is designated as the primary source and is used in the event that
the request cannot be completed using alternate sources.
:return: The download report.
:rtype: DownloadReport
"""
count = 0
report = DownloadReport()
report.total_sources = len(self.sources)
try:
for request in self.requests:
request.find_sources(self.primary, self.sources)
self.dispatch(request)
count += 1
finally:
self.in_progress.wait(count)
for queue in self.queues.values():
queue.put(None)
queue.halt()
for queue in self.queues.values():
queue.join()
for source_id, queue in self.queues.items():
listener = queue.downloader.event_listener
downloads = report.downloads.setdefault(source_id, DownloadDetails())
downloads.total_succeeded += listener.total_succeeded
downloads.total_failed += listener.total_failed
return report
def test_dict(self):
report = DownloadReport()
report.downloads['s1'] = DownloadDetails()
report.downloads['s2'] = DownloadDetails()
expected = {
'total_sources': 0,
'downloads': {
's1': {'total_failed': 0, 'total_succeeded': 0},
's2': {'total_failed': 0, 'total_succeeded': 0}
},
}
self.assertEqual(report.dict(), expected)
def test_dict(self):
report = DownloadReport()
report.downloads['s1'] = DownloadDetails()
report.downloads['s2'] = DownloadDetails()
expected = {
'total_sources': 0,
'downloads': {
's1': {'total_failed': 0, 'total_succeeded': 0},
's2': {'total_failed': 0, 'total_succeeded': 0}
},
}
self.assertEqual(report.dict(), expected)
def test_dict(self):
report = DownloadReport()
report.downloads["s1"] = DownloadDetails()
report.downloads["s2"] = DownloadDetails()
expected = {
"total_sources": 0,
"downloads": {
"s1": {"total_failed": 0, "total_succeeded": 0},
"s2": {"total_failed": 0, "total_succeeded": 0},
},
}
self.assertEqual(report.dict(), expected)
def test_update_rendering(self):
repo_ids = ['repo_%d' % n for n in range(0, 3)]
handler_report = ContentReport()
summary_report = SummaryReport()
summary_report.setup([{'repo_id': r} for r in repo_ids])
for r in summary_report.repository.values():
r.action = RepositoryReport.ADDED
download_report = DownloadReport()
download_report.downloads[PRIMARY_ID] = DownloadDetails()
download_report.downloads['content-world'] = DownloadDetails()
r.sources = download_report.dict()
handler_report.set_succeeded(details=summary_report.dict())
renderer = UpdateRenderer(self.context.prompt, handler_report.dict())
renderer.render()
self.assertEqual(len(self.recorder.lines), 59)
def test_update_rendering(self):
repo_ids = ['repo_%d' % n for n in range(0, 3)]
handler_report = ContentReport()
summary_report = SummaryReport()
summary_report.setup([{'repo_id': r} for r in repo_ids])
for r in summary_report.repository.values():
r.action = RepositoryReport.ADDED
download_report = DownloadReport()
download_report.downloads[PRIMARY_ID] = DownloadDetails()
download_report.downloads['content-world'] = DownloadDetails()
r.sources = download_report.dict()
handler_report.set_succeeded(details=summary_report.dict())
renderer = UpdateRenderer(self.context.prompt, handler_report.dict())
renderer.render()
self.assertEqual(len(self.recorder.lines), 59)
def download(self, cancel_event, downloader, request_list, listener=None):
"""
Download files using available alternate content sources.
An attempt is made to satisfy each download request using the alternate
content sources in the order specified by priority. The specified
downloader is designated as the primary source and is used in the event that
the request cannot be completed using alternate sources.
:param cancel_event: An event that indicates the download has been canceled.
:type cancel_event: threading.Event
:param downloader: A primary nectar downloader. Used to download the
requested content unit when it cannot be achieved using alternate
content sources.
:type downloader: nectar.downloaders.base.Downloader
:param request_list: A list of pulp.server.content.sources.model.Request.
:type request_list: list
:param listener: An optional download request listener.
:type listener: Listener
:return: A download report.
:rtype: DownloadReport
"""
self.refresh(cancel_event)
report = DownloadReport()
primary = PrimarySource(downloader)
for request in request_list:
request.find_sources(primary, self.sources)
report.total_sources = len(self.sources)
while not cancel_event.isSet():
collated = self.collated(request_list)
if not collated:
# Either we have exhausted our content sources or all
# of the requests have been satisfied.
break
report.total_passes += 1
for source, nectar_list in collated.items():
downloader = source.get_downloader()
nectar_listener = NectarListener(cancel_event, downloader,
listener)
downloader.event_listener = nectar_listener
downloader.download(nectar_list)
downloads = report.downloads.setdefault(
source.id, DownloadDetails())
downloads.total_succeeded += nectar_listener.total_succeeded
downloads.total_failed += nectar_listener.total_failed
if cancel_event.isSet():
break
return report
def download(self, cancel_event, downloader, request_list, listener=None):
"""
Download files using available alternate content sources.
An attempt is made to satisfy each download request using the alternate
content sources in the order specified by priority. The specified
downloader is designated as the primary source and is used in the event that
the request cannot be completed using alternate sources.
:param cancel_event: An event that indicates the download has been canceled.
:type cancel_event: threading.Event
:param downloader: A primary nectar downloader. Used to download the
requested content unit when it cannot be achieved using alternate
content sources.
:type downloader: nectar.downloaders.base.Downloader
:param request_list: A list of pulp.server.content.sources.model.Request.
:type request_list: list
:param listener: An optional download request listener.
:type listener: Listener
:return: A download report.
:rtype: DownloadReport
"""
self.refresh(cancel_event)
report = DownloadReport()
primary = PrimarySource(downloader)
for request in request_list:
request.find_sources(primary, self.sources)
report.total_sources = len(self.sources)
while not cancel_event.isSet():
collated = self.collated(request_list)
if not collated:
# Either we have exhausted our content sources or all
# of the requests have been satisfied.
break
report.total_passes += 1
for source, nectar_list in collated.items():
downloader = source.get_downloader()
nectar_listener = NectarListener(cancel_event, downloader, listener)
downloader.event_listener = nectar_listener
downloader.download(nectar_list)
downloads = report.downloads.setdefault(source.id, DownloadDetails())
downloads.total_succeeded += nectar_listener.total_succeeded
downloads.total_failed += nectar_listener.total_failed
if cancel_event.isSet():
break
return report
class SummaryReport(object):
"""
A report that provides both summary and details regarding the importing
of content units associated with a repository.
:ivar errors: List of errors.
:type errors: ErrorList
:ivar sources: The content sources container statistics.
:type sources: DownloadReport
"""
def __init__(self):
self.errors = ErrorList()
self.sources = DownloadReport()
def dict(self):
"""
Dictionary representation.
:return: A dictionary representation.
:rtype: dict
"""
return dict(errors=[e.dict() for e in self.errors],
sources=self.sources.dict())
class SummaryReport(object):
"""
A report that provides both summary and details regarding the importing
of content units associated with a repository.
:ivar errors: List of errors.
:type errors: ErrorList
:ivar sources: The content sources container statistics.
:type sources: DownloadReport
"""
def __init__(self):
self.errors = ErrorList()
self.sources = DownloadReport()
def dict(self):
"""
Dictionary representation.
:return: A dictionary representation.
:rtype: dict
"""
return dict(errors=[e.dict() for e in self.errors], sources=self.sources.dict())
def test_construction(self):
report = DownloadReport()
self.assertEqual(report.total_passes, 0)
self.assertEqual(report.total_sources, 0)
self.assertEqual(report.downloads, {})
def __init__(self):
self.errors = ErrorList()
self.sources = DownloadReport()
def __init__(self):
self.errors = ErrorList()
self.sources = DownloadReport()
