def test_download_cancelled_in_failed(self, mock_started, mock_cancel):
request_list = []
for n in range(0, 5):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(
TYPE_ID,
unit_key,
'http://unit-city/unit_%d' % n,
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = HTTPThreadedDownloader(DownloaderConfig())
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
event = CancelEvent(2)
report = container.download(event, downloader, request_list)
self.assertTrue(mock_started.called)
self.assertTrue(mock_cancel.called)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 1)
self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 0)
self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 5)
def test_refresh_cancel_in_plugin(self, mock_plugin, *unused):
container = ContentContainer(path=self.tmp_dir)
event = CancelEvent(3)
report = container.refresh(event, force=True)
plugin = mock_plugin.return_value[0]
self.assertEqual(plugin.refresh.call_count, 1)
self.assertEqual(len(report), 1)
def test_download_fail_completely(self):
request_list = []
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10)
shutil.rmtree(_dir)
_dir = self.populate_content(PRIMARY, 0, 20)
# primary
for n in range(0, 10):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(
TYPE_ID,
unit_key,
'http://redhat.com/%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = HTTPThreadedDownloader(DownloaderConfig())
listener = MockListener()
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
event = Event()
container.download(event, downloader, request_list, listener)
# primary
for i in range(0, len(request_list)):
request = request_list[i]
self.assertFalse(request.downloaded)
self.assertEqual(len(request.errors), 1)
self.assertEqual(listener.download_started.call_count, len(request_list))
self.assertEqual(listener.download_succeeded.call_count, 0)
self.assertEqual(listener.download_failed.call_count, len(request_list))
def test_download_cancelled_during_refreshing(self):
downloader = LocalFileDownloader(DownloaderConfig())
container = ContentContainer(path=self.tmp_dir)
container.collated = Mock()
event = CancelEvent(1)
container.download(event, downloader, [])
self.assertFalse(container.collated.called)
def test_download(self):
request_list = []
_dir, cataloged = self.populate_catalog(ORPHANED, 0, 10)
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10)
_dir = self.populate_content(PRIMARY, 0, 20)
# unit-world
for n in range(0, 10):
request = Request(
cataloged[n].type_id,
cataloged[n].unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
# primary
for n in range(11, 20):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(
TYPE_ID,
unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = LocalFileDownloader(DownloaderConfig())
listener = MockListener()
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
event = Event()
report = container.download(event, downloader, request_list, listener)
# unit-world
for i in range(0, 10):
request = request_list[i]
self.assertTrue(request.downloaded)
self.assertEqual(len(request.errors), 0)
with open(request.destination) as fp:
s = fp.read()
self.assertTrue(UNIT_WORLD in s)
# primary
for i in range(11, len(request_list)):
request = request_list[i]
self.assertTrue(request.downloaded)
self.assertEqual(len(request.errors), 0)
with open(request.destination) as fp:
s = fp.read()
self.assertTrue(PRIMARY in s)
self.assertEqual(listener.download_started.call_count, len(request_list))
self.assertEqual(listener.download_succeeded.call_count, len(request_list))
self.assertEqual(listener.download_failed.call_count, 0)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 2)
self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9)
self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0)
self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 10)
self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 0)
def test_download_with_errors(self):
request_list = []
_dir, cataloged = self.populate_catalog(ORPHANED, 0, 10)
_dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10)
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10)
shutil.rmtree(_dir)
_dir = self.populate_content(PRIMARY, 0, 20)
# unit-world
for n in range(0, 10):
request = Request(
cataloged[n].type_id,
cataloged[n].unit_key,
"file://%s/unit_%d" % (_dir, n),
os.path.join(self.downloaded, "unit_%d" % n),
)
request_list.append(request)
# primary
for n in range(11, 20):
unit_key = {"name": "unit_%d" % n, "version": "1.0.%d" % n, "release": "1", "checksum": str(uuid4())}
request = Request(
TYPE_ID, unit_key, "file://%s/unit_%d" % (_dir, n), os.path.join(self.downloaded, "unit_%d" % n)
)
request_list.append(request)
downloader = LocalFileDownloader(DownloaderConfig())
listener = Mock()
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
event = Event()
# test
report = container.download(event, downloader, request_list, listener)
# validation
# unit-world
for i in range(0, 10):
request = request_list[i]
self.assertTrue(request.downloaded, msg="URL: %s" % request.url)
self.assertEqual(len(request.errors), 1)
with open(request.destination) as fp:
s = fp.read()
self.assertTrue(UNDERGROUND in s)
# primary
for i in range(11, len(request_list)):
request = request_list[i]
self.assertTrue(request.downloaded, msg="URL: %s" % request.url)
self.assertEqual(len(request.errors), 0)
with open(request.destination) as fp:
s = fp.read()
self.assertTrue(PRIMARY in s)
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 3)
self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9)
self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0)
self.assertEqual(report.downloads[UNDERGROUND].total_succeeded, 10)
self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0)
self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0)
self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 10)
def test_download_cancelled_in_download(self):
container = ContentContainer(path=self.tmp_dir)
container.collated = Mock()
event = CancelEvent(1)
report = container.download(event, None, [])
self.assertFalse(container.collated.called)
self.assertEqual(report.total_passes, 0)
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 0)
def test_download_cancelled_during_refreshing(self):
downloader = LocalFileDownloader(DownloaderConfig())
container = ContentContainer(path=self.tmp_dir)
container.collated = Mock()
event = CancelEvent(1)
report = container.download(event, downloader, [])
self.assertFalse(container.collated.called)
self.assertEqual(report.total_passes, 0)
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 0)
def test_download(self):
request_list = []
_dir, cataloged = self.populate_catalog(ORPHANED, 0, 10)
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10)
_dir = self.populate_content(PRIMARY, 0, 20)
# unit-world
for n in range(0, 10):
request = Request(cataloged[n].type_id, cataloged[n].unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
# primary
for n in range(11, 20):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(TYPE_ID, unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = LocalFileDownloader(DownloaderConfig())
listener = Mock()
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
event = Event()
# test
report = container.download(event, downloader, request_list, listener)
# validation
# unit-world
for i in range(0, 10):
request = request_list[i]
self.assertTrue(request.downloaded)
self.assertEqual(len(request.errors), 0)
with open(request.destination) as fp:
s = fp.read()
self.assertTrue(UNIT_WORLD in s)
# primary
for i in range(11, len(request_list)):
request = request_list[i]
self.assertTrue(request.downloaded)
self.assertEqual(len(request.errors), 0)
with open(request.destination) as fp:
s = fp.read()
self.assertTrue(PRIMARY in s)
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 2)
self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9)
self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0)
self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 10)
self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 0)
def test_purge_orphans(self):
_dir, cataloged = self.populate_catalog(ORPHANED, 0, 10)
_dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10)
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10)
collection = ContentCatalog.get_collection()
self.assertEqual(collection.find().count(), 30)
container = ContentContainer(path=self.tmp_dir)
container.purge_orphans()
self.assertEqual(collection.find().count(), 20)
self.assertEqual(collection.find({'source_id': ORPHANED}).count(), 0)
self.assertEqual(collection.find({'source_id': UNDERGROUND}).count(), 10)
self.assertEqual(collection.find({'source_id': UNIT_WORLD}).count(), 10)
def test_refresh_exception(self, mock_refresh):
container = ContentContainer(path=self.tmp_dir)
event = Event()
report = container.refresh(event, force=True)
self.assertEqual(len(report), 2)
for r in report:
self.assertFalse(r.succeeded)
self.assertEqual(r.added_count, 0)
self.assertEqual(r.deleted_count, 0)
self.assertEqual(len(r.errors), 1)
collection = ContentCatalog.get_collection()
self.assertEqual(mock_refresh.call_count, 2)
self.assertEqual(collection.find().count(), 0)
def test_refresh_exception(self, mock_refresh):
container = ContentContainer(path=self.tmp_dir)
event = Event()
report = container.refresh(event, force=True)
self.assertEqual(len(report), 2)
for r in report:
self.assertFalse(r.succeeded)
self.assertEqual(r.added_count, 0)
self.assertEqual(r.deleted_count, 0)
self.assertEqual(len(r.errors), 1)
collection = ContentCatalog.get_collection()
self.assertEqual(mock_refresh.call_count, 2)
self.assertEqual(collection.find().count(), 0)
def test_refresh_failure(self, mock_plugin):
container = ContentContainer(path=self.tmp_dir)
event = Event()
report = container.refresh(event, force=True)
self.assertEqual(len(report), 5)
for r in report:
self.assertFalse(r.succeeded)
self.assertEqual(r.added_count, 0)
self.assertEqual(r.deleted_count, 0)
self.assertEqual(len(r.errors), 1)
plugin = mock_plugin.return_value[0]
collection = ContentCatalog.get_collection()
self.assertEqual(plugin.refresh.call_count, 5)
self.assertEqual(collection.find().count(), 0)
def test_refresh_failure(self, mock_plugin):
container = ContentContainer(path=self.tmp_dir)
event = Event()
report = container.refresh(event, force=True)
self.assertEqual(len(report), 5)
for r in report:
self.assertFalse(r.succeeded)
self.assertEqual(r.added_count, 0)
self.assertEqual(r.deleted_count, 0)
self.assertEqual(len(r.errors), 1)
plugin = mock_plugin.return_value[0]
collection = ContentCatalog.get_collection()
self.assertEqual(plugin.refresh.call_count, 5)
self.assertEqual(collection.find().count(), 0)
def test_download_with_errors(self):
request_list = []
_dir, cataloged = self.populate_catalog(ORPHANED, 0, 1000)
_dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 1000)
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 1000)
shutil.rmtree(_dir)
_dir = self.populate_content(PRIMARY, 0, 2000)
# unit-world
for n in range(0, 1000):
request = Request(
cataloged[n].type_id,
cataloged[n].unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
# primary
for n in range(1001, 2000):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(
TYPE_ID,
unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = LocalFileDownloader(DownloaderConfig())
event = Event()
threshold = len(request_list) * 0.10 # cancel after 10% started
listener = TestListener(event, threshold)
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
# test
report = container.download(event, downloader, request_list, listener)
# validation
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 2)
self.assertTrue(0 < report.downloads[UNDERGROUND].total_succeeded < 500)
self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0)
self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0)
self.assertTrue(0 < report.downloads[UNIT_WORLD].total_failed < 1000)
def test_refresh(self, mock_plugin):
container = ContentContainer(path=self.tmp_dir)
event = Event()
report = container.refresh(event, force=True)
plugin = mock_plugin.return_value[0]
self.assertEqual(plugin.refresh.call_count, 5)
self.assertEqual(len(report), 5)
for r in report:
self.assertTrue(r.succeeded)
self.assertEqual(r.added_count, 100)
self.assertEqual(r.deleted_count, 0)
calls = iter(plugin.refresh.call_args_list)
for source in ContentSource.load_all(self.tmp_dir).values():
for url in source.urls:
args = calls.next()[0]
self.assertTrue(isinstance(args[0], CatalogerConduit))
self.assertEqual(args[1], source.descriptor)
self.assertEqual(args[2], url)
def test_refresh(self, mock_plugin):
container = ContentContainer(path=self.tmp_dir)
event = Event()
report = container.refresh(event, force=True)
plugin = mock_plugin.return_value[0]
self.assertEqual(plugin.refresh.call_count, 5)
self.assertEqual(len(report), 5)
for r in report:
self.assertTrue(r.succeeded)
self.assertEqual(r.added_count, 100)
self.assertEqual(r.deleted_count, 0)
calls = iter(plugin.refresh.call_args_list)
for source in ContentSource.load_all(self.tmp_dir).values():
for url in source.urls:
args = calls.next()[0]
self.assertTrue(isinstance(args[0], CatalogerConduit))
self.assertEqual(args[1], source.descriptor)
self.assertEqual(args[2], url)
def test_purge_orphans(self):
_dir, cataloged = self.populate_catalog(ORPHANED, 0, 10)
_dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10)
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10)
collection = ContentCatalog.get_collection()
self.assertEqual(collection.find().count(), 30)
container = ContentContainer(path=self.tmp_dir)
container.purge_orphans()
self.assertEqual(collection.find().count(), 20)
self.assertEqual(collection.find({'source_id': ORPHANED}).count(), 0)
self.assertEqual(
collection.find({
'source_id': UNDERGROUND
}).count(), 10)
self.assertEqual(
collection.find({
'source_id': UNIT_WORLD
}).count(), 10)
def test_download_with_errors(self):
request_list = []
_dir, cataloged = self.populate_catalog(ORPHANED, 0, 1000)
_dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 1000)
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 1000)
shutil.rmtree(_dir)
_dir = self.populate_content(PRIMARY, 0, 2000)
# unit-world
for n in range(0, 1000):
request = Request(cataloged[n].type_id, cataloged[n].unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
# primary
for n in range(1001, 2000):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(TYPE_ID, unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = LocalFileDownloader(DownloaderConfig())
event = Event()
threshold = len(request_list) * 0.10 # cancel after 10% started
listener = TestListener(event, threshold)
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
# test
report = container.download(event, downloader, request_list, listener)
# validation
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 2)
self.assertTrue(
0 < report.downloads[UNDERGROUND].total_succeeded < 500)
self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0)
self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0)
self.assertTrue(0 < report.downloads[UNIT_WORLD].total_failed < 1000)
def _add_units(self, request, unit_inventory):
"""
Determine the list of units contained in the parent inventory
but are not contained in the child inventory and add them.
For each unit, this is performed in the following steps:
1. Download the file (if defined) associated with the unit.
2. Add the unit to the child inventory.
3. Associate the unit to the repository.
The unit is added only:
1. If no file is associated with unit.
2. The file associated with the unit is successfully downloaded.
For units with files, the unit is added to the inventory as part of the
unit download manager callback.
:param request: A synchronization request.
:type request: SyncRequest
:param unit_inventory: The inventory of both parent and child content units.
:type unit_inventory: UnitInventory
"""
download_list = []
units = unit_inventory.units_on_parent_only()
request.progress.begin_adding_units(len(units))
listener = ContentDownloadListener(self, request)
for unit, unit_ref in units:
if request.cancelled():
return
self._reset_storage_path(unit)
if not self._needs_download(unit):
# unit has no file associated
self.add_unit(request, unit_ref.fetch())
continue
unit_path, destination = self._path_and_destination(unit)
unit_URL = pathlib.url_join(unit_inventory.base_URL, unit_path)
_request = listener.create_request(unit_URL, destination, unit, unit_ref)
download_list.append(_request)
if request.cancelled():
return
container = ContentContainer()
request.summary.sources = \
container.download(request.cancel_event, request.downloader, download_list, listener)
request.summary.errors.extend(listener.error_list)
def test_download_cancelled_in_started(self, mock_cancel):
request_list = []
_dir = self.populate_content(PRIMARY, 0, 5)
for n in range(0, 5):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(
TYPE_ID,
unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = LocalFileDownloader(DownloaderConfig())
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
event = CancelEvent(2)
container.download(event, downloader, request_list)
self.assertTrue(mock_cancel.called)
def test_download_fail_completely(self):
request_list = []
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10)
shutil.rmtree(_dir)
_dir = self.populate_content(PRIMARY, 0, 20)
# primary
for n in range(0, 10):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(TYPE_ID, unit_key,
'http://redhat.com/%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = HTTPThreadedDownloader(DownloaderConfig())
listener = MockListener()
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
event = Event()
report = container.download(event, downloader, request_list, listener)
# primary
for i in range(0, len(request_list)):
request = request_list[i]
self.assertFalse(request.downloaded)
self.assertEqual(len(request.errors), 1)
self.assertEqual(listener.download_started.call_count,
len(request_list))
self.assertEqual(listener.download_succeeded.call_count, 0)
self.assertEqual(listener.download_failed.call_count,
len(request_list))
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 1)
self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 0)
self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 10)
def test_download(self):
request_list = []
_dir, cataloged = self.populate_catalog(ORPHANED, 0, 1000)
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 1000)
_dir = self.populate_content(PRIMARY, 0, 2000)
# unit-world
for n in range(0, 1000):
request = Request(
cataloged[n].type_id,
cataloged[n].unit_key,
"file://%s/unit_%d" % (_dir, n),
os.path.join(self.downloaded, "unit_%d" % n),
)
request_list.append(request)
# primary
for n in range(1001, 2000):
unit_key = {"name": "unit_%d" % n, "version": "1.0.%d" % n, "release": "1", "checksum": str(uuid4())}
request = Request(
TYPE_ID, unit_key, "file://%s/unit_%d" % (_dir, n), os.path.join(self.downloaded, "unit_%d" % n)
)
request_list.append(request)
event = Event()
threshold = len(request_list) * 0.80 # cancel after 80% started
downloader = LocalFileDownloader(DownloaderConfig())
listener = TestListener(event, threshold)
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
# test
report = container.download(event, downloader, request_list, listener)
# validation
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 2)
self.assertTrue(0 < report.downloads[PRIMARY_ID].total_succeeded < 999)
self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0)
self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 1000)
self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 0)
def __init__(self, base_url, nectar_conf, units, dst_dir, listener):
"""
:param base_url: The repository base url.
:type base_url: str
:param units: An iterable of units to download.
:type units: iterable
:param dst_dir: The absolute path to where the packages are to be downloaded.
:type dst_dir: str
:param listener: A nectar listener.
:type listener: nectar.listener.DownloadListener
"""
self.base_url = base_url
self.units = units
self.dst_dir = dst_dir
self.listener = ContainerListener(listener)
self.primary = create_downloader(base_url, nectar_conf)
self.container = ContentContainer()
self.canceled = Event()
class Packages(object):
"""
Package downloader.
:ivar base_url: The repository base url.
:type base_url: str
:ivar units: An iterable of units to download.
:type units: iterable
:ivar dst_dir: The absolute path to where the packages are to be downloaded.
:type dst_dir: str
:ivar listener: A nectar listener.
:type listener: nectar.listener.DownloadListener
:ivar primary: The primary nectar downloader.
:type primary: nectar.downloaders.base.Downloader
:ivar container: A content container.
:type container: ContentContainer
:ivar canceled: An event that signals the running download has been canceled.
:type canceled: threading.Event
"""
def __init__(self, base_url, nectar_conf, units, dst_dir, listener):
"""
:param base_url: The repository base url.
:type base_url: str
:param units: An iterable of units to download.
:type units: iterable
:param dst_dir: The absolute path to where the packages are to be downloaded.
:type dst_dir: str
:param listener: A nectar listener.
:type listener: nectar.listener.DownloadListener
"""
self.base_url = base_url
self.units = units
self.dst_dir = dst_dir
self.listener = ContainerListener(listener)
self.primary = create_downloader(base_url, nectar_conf)
self.container = ContentContainer()
self.canceled = Event()
@property
def downloader(self):
"""
Provided only for API comparability.
:return: self
"""
return self
def get_requests(self):
"""
Get requests for the units requested to be downloaded.
:return: An iterable of: Request
:rtype: iterable
"""
for unit in self.units:
if unit.metadata.get('base_url'):
url = urljoin(unit.metadata.get('base_url'), unit.download_path)
else:
url = urljoin(self.base_url, unit.download_path)
file_name = os.path.basename(unit.relative_path)
destination = os.path.join(self.dst_dir, file_name)
request = Request(
type_id=unit.TYPE,
unit_key=unit.unit_key,
url=url,
destination=destination)
request.data = unit
yield request
def download_packages(self):
"""
Download packages using alternate content source container.
"""
report = self.container.download(
self.canceled, self.primary, self.get_requests(), self.listener)
log.info(CONTAINER_REPORT, dict(r=report.dict(), u=self.base_url))
def cancel(self):
"""
Cancel a running download.
"""
self.canceled.set()
def test_download_cancelled_in_download(self):
container = ContentContainer(path=self.tmp_dir)
container.collated = Mock()
event = CancelEvent(1)
container.download(event, None, [])
self.assertFalse(container.collated.called)