def test_find_sources(self, fake_manager):
type_id = 'test_1'
unit_key = 1
destination = '/tmp/123'
url = 'http://redhat.com/repository'
primary = PrimarySource(None)
alternatives = dict([(s, ContentSource(s, d)) for s, d in DESCRIPTOR])
fake_manager().find.return_value = CATALOG
# test
request = Request(type_id, unit_key, url, destination)
request.find_sources(primary, alternatives)
# validation
# validate sources sorted by priority with the primary last.
# should only have matched on s-1 and s-3.
self.assertEqual(len(request.sources), 5)
self.assertEqual(request.sources[0][0].id, 's-3')
self.assertEqual(request.sources[0][1], CATALOG[2][constants.URL])
self.assertEqual(request.sources[1][0].id, 's-3')
self.assertEqual(request.sources[1][1], CATALOG[3][constants.URL])
self.assertEqual(request.sources[2][0].id, 's-1')
self.assertEqual(request.sources[2][1], CATALOG[0][constants.URL])
self.assertEqual(request.sources[3][0].id, 's-1')
self.assertEqual(request.sources[3][1], CATALOG[1][constants.URL])
self.assertEqual(request.sources[4][0].id, primary.id)
self.assertEqual(request.sources[4][1], url)
def download(self, downloader, requests, listener=None):
"""
Download files using available alternate content sources.
An attempt is made to satisfy each download request using the alternate
content sources in the order specified by priority. The specified
downloader is designated as the primary source and is used in the event that
the request cannot be completed using alternate sources.
:param downloader: A primary nectar downloader. Used to download the
requested content unit when it cannot be achieved using alternate content sources.
:type downloader: nectar.downloaders.base.Downloader
:param requests: An iterable of pulp.server.content.sources.model.Request.
:type requests: iterable
:param listener: An optional download request listener.
:type listener: Listener
:return: A download report.
:rtype: DownloadReport
"""
self.refresh()
primary = PrimarySource(downloader)
if self.threaded:
method = Threaded
else:
method = Serial
batch = method(primary, self, requests, listener)
report = batch()
return report
def test_download(self, fake_load):
sources = []
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.get_downloader = Mock()
sources.append(s)
fake_load.return_value = sources
request_list = []
for n in range(6):
r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
r.find_sources = Mock(return_value=sources[n % 3:])
request_list.append(r)
collated = [{
sources[0]: ['nectar-1'],
sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
sources[2]: ['nectar-5', 'nectar-6']
}, {}]
fake_collated = Mock(side_effect=collated)
fake_listener = Mock()
canceled = FakeEvent()
fake_primary = PrimarySource(Mock())
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = fake_collated
report = container.download(canceled, fake_primary, request_list,
fake_listener)
# validation
container.refresh.assert_called_with(canceled)
for r in request_list:
r.find_sources.assert_called_with(fake_primary, container.sources)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, len(sources))
self.assertEqual(len(report.downloads), 3)
for source in sources:
self.assertEqual(report.downloads[source.id].total_succeeded, 0)
self.assertEqual(report.downloads[source.id].total_failed, 0)
for source in sources:
source.get_downloader.assert_called_with()
downloader = source.get_downloader()
listener = downloader.event_listener
self.assertEqual(listener.cancel_event, canceled)
self.assertEqual(listener.downloader, downloader)
self.assertEqual(listener.listener, fake_listener)
downloader.download.assert_called_with(collated[0][source])
def test_download_canceled_after_collated(self, fake_load):
sources = []
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.get_downloader = Mock()
sources.append(s)
fake_load.return_value = sources
request_list = []
for n in range(6):
r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
r.find_sources = Mock(return_value=sources[n % 3:])
request_list.append(r)
collated = [{
sources[0]: ['nectar-1'],
sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
sources[2]: ['nectar-5', 'nectar-6']
}, {}]
fake_collated = Mock(side_effect=collated)
fake_listener = Mock()
canceled = Mock()
canceled.isSet.side_effect = [False, True, True]
fake_primary = PrimarySource(Mock())
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = fake_collated
report = container.download(canceled, fake_primary, request_list,
fake_listener)
# validation
container.refresh.assert_called_with(canceled)
for r in request_list:
r.find_sources.assert_called_with(fake_primary, container.sources)
called = 0
for s in sources:
if s.get_downloader.called:
called += 1
self.assertEqual(called, 1)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, len(sources))
self.assertEqual(len(report.downloads), 1)
self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0)
self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def download(self, cancel_event, downloader, request_list, listener=None):
"""
Download files using available alternate content sources.
An attempt is made to satisfy each download request using the alternate
content sources in the order specified by priority. The specified
downloader is designated as the primary source and is used in the event that
the request cannot be completed using alternate sources.
:param cancel_event: An event that indicates the download has been canceled.
:type cancel_event: threading.Event
:param downloader: A primary nectar downloader. Used to download the
requested content unit when it cannot be achieved using alternate
content sources.
:type downloader: nectar.downloaders.base.Downloader
:param request_list: A list of pulp.server.content.sources.model.Request.
:type request_list: list
:param listener: An optional download request listener.
:type listener: Listener
:return: A download report.
:rtype: DownloadReport
"""
self.refresh(cancel_event)
report = DownloadReport()
primary = PrimarySource(downloader)
for request in request_list:
request.find_sources(primary, self.sources)
report.total_sources = len(self.sources)
while not cancel_event.isSet():
collated = self.collated(request_list)
if not collated:
# Either we have exhausted our content sources or all
# of the requests have been satisfied.
break
report.total_passes += 1
for source, nectar_list in collated.items():
downloader = source.get_downloader()
nectar_listener = NectarListener(cancel_event, downloader,
listener)
downloader.event_listener = nectar_listener
downloader.download(nectar_list)
downloads = report.downloads.setdefault(
source.id, DownloadDetails())
downloads.total_succeeded += nectar_listener.total_succeeded
downloads.total_failed += nectar_listener.total_failed
if cancel_event.isSet():
break
return report
def test_priority(self):
primary = PrimarySource(None)
self.assertEqual(primary.priority, sys.maxint)
def test_refresh(self):
# just added for coverage
primary = PrimarySource(None)
primary.refresh(None)
def test_downloader(self):
downloader = Mock()
primary = PrimarySource(downloader)
self.assertEqual(primary.get_downloader(), downloader)
def test_construction(self):
downloader = Mock()
primary = PrimarySource(downloader)
self.assertEqual(primary.id, PRIMARY_ID)
self.assertEqual(primary._downloader, downloader)
def test_refresh(self):
# just added for coverage
primary = PrimarySource(None)
primary.refresh(None)
def test_downloader(self):
downloader = Mock()
primary = PrimarySource(downloader)
self.assertEqual(primary.get_downloader(), downloader)
def test_max_concurrent(self):
primary = PrimarySource(None)
self.assertEqual(primary.max_concurrent, int(DEFAULT[constants.MAX_CONCURRENT]))
def test_session_has_attribute(self):
downloader = Mock()
primary = PrimarySource(downloader)
self.assertEqual(primary.session, downloader.session)
def test_session_no_attribute(self):
primary = PrimarySource(None)
self.assertEqual(primary.session, None)