def _download(self, request, entry, responder): """ Download the file. :param request: The original twisted client HTTP request being handled by the streamer. :type request: twisted.web.server.Request :param entry: The catalog entry to download. :type entry: pulp.server.db.model.LazyCatalogEntry :param responder: The file-like object that nectar should write to. :type responder: Responder :return: The download report. :rtype: nectar.report.DownloadReport """ downloader = None try: unit = self._get_unit(entry) downloader = self._get_downloader(request, entry) alt_request = ContainerRequest(entry.unit_type_id, unit.unit_key, entry.url, responder) listener = downloader.event_listener container = ContentContainer(threaded=False) container.download(downloader, [alt_request], listener) if listener.succeeded_reports: return listener.succeeded_reports[0] else: raise DownloadFailed() finally: try: downloader.config.finalize() except Exception: if logger.isEnabledFor(logging.DEBUG): logger.exception(_('finalize() failed.'))
def _update_units(self, request, unit_inventory): """ Update units that have been updated on the parent since added or last updated in the child inventory. :param request: A synchronization request. :type request: SyncRequest :param unit_inventory: The inventory of both parent and child content units. :type unit_inventory: UnitInventory """ download_list = [] units = unit_inventory.updated_units() listener = ContentDownloadListener(self, request) for unit, unit_ref in units: storage_path = unit[constants.STORAGE_PATH] if storage_path: self._reset_storage_path(unit) unit_url, destination = self._url_and_destination(unit_inventory.base_URL, unit) _request = listener.create_request(unit_url, destination, unit, unit_ref) download_list.append(_request) else: unit = unit_ref.fetch() self.add_unit(request, unit) if not download_list: return container = ContentContainer() request.summary.sources = container.download( request.downloader, download_list, listener) request.summary.errors.extend(listener.error_list)
def test_download_to_stream(self): request_list = [] _dir, cataloged = self.populate_catalog(ORPHANED, 0, 10) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10) _dir = self.populate_content(PRIMARY, 0, 20) # unit-world for n in range(0, 10): request = Request( cataloged[n].type_id, cataloged[n].unit_key, 'file://%s/unit_%d' % (_dir, n), StringIO()) request_list.append(request) # primary for n in range(11, 20): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request( TYPE_ID, unit_key, 'file://%s/unit_%d' % (_dir, n), StringIO()) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) listener = Mock() container = ContentContainer(path=self.tmp_dir) container.threaded = False container.refresh = Mock() # test report = container.download(downloader, request_list, listener) # validation # unit-world for i in range(0, 10): request = request_list[i] self.assertTrue(request.downloaded) self.assertEqual(len(request.errors), 0) fp = request.destination s = fp.getvalue() self.assertTrue(UNIT_WORLD in s) # primary for i in range(11, len(request_list)): request = request_list[i] self.assertTrue(request.downloaded) self.assertEqual(len(request.errors), 0) fp = request.destination s = fp.getvalue() self.assertTrue(PRIMARY in s) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 2) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 10) self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 0)
def test_download(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [ { sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {} ] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = FakeEvent() fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 3) for source in sources: self.assertEqual(report.downloads[source.id].total_succeeded, 0) self.assertEqual(report.downloads[source.id].total_failed, 0) for source in sources: source.get_downloader.assert_called_with() downloader = source.get_downloader() listener = downloader.event_listener self.assertEqual(listener.cancel_event, canceled) self.assertEqual(listener.downloader, downloader) self.assertEqual(listener.listener, fake_listener) downloader.download.assert_called_with(collated[0][source])
def test_download(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [{ sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {}] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = FakeEvent() fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 3) for source in sources: self.assertEqual(report.downloads[source.id].total_succeeded, 0) self.assertEqual(report.downloads[source.id].total_failed, 0) for source in sources: source.get_downloader.assert_called_with() downloader = source.get_downloader() listener = downloader.event_listener self.assertEqual(listener.cancel_event, canceled) self.assertEqual(listener.downloader, downloader) self.assertEqual(listener.listener, fake_listener) downloader.download.assert_called_with(collated[0][source])
def test_download_canceled_after_collated(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [ { sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {} ] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = Mock() canceled.isSet.side_effect = [False, True, True] fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) called = 0 for s in sources: if s.get_downloader.called: called += 1 self.assertEqual(called, 1) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0) self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def test_download_canceled_after_collated(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [{ sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {}] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = Mock() canceled.isSet.side_effect = [False, True, True] fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) called = 0 for s in sources: if s.get_downloader.called: called += 1 self.assertEqual(called, 1) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0) self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def test_download_canceled_before_collated(self, fake_load): fake_load.return_value = [] canceled = FakeEvent() canceled.set() # test container = ContentContainer('') container.refresh = Mock() container.collated = Mock() report = container.download(canceled, None, [], None) container.refresh.assert_called_with(canceled) self.assertFalse(container.collated.called) self.assertEqual(report.total_passes, 0) self.assertEqual(report.total_sources, 0) self.assertEqual(len(report.downloads), 0)
def _add_units(self, request, unit_inventory): """ Determine the list of units contained in the parent inventory but are not contained in the child inventory and add them. For each unit, this is performed in the following steps: 1. Download the file (if defined) associated with the unit. 2. Add the unit to the child inventory. 3. Associate the unit to the repository. The unit is added only: 1. If no file is associated with unit. 2. The file associated with the unit is successfully downloaded. For units with files, the unit is added to the inventory as part of the unit download manager callback. :param request: A synchronization request. :type request: SyncRequest :param unit_inventory: The inventory of both parent and child content units. :type unit_inventory: UnitInventory """ download_list = [] units = unit_inventory.units_on_parent_only() request.progress.begin_adding_units(len(units)) listener = ContentDownloadListener(self, request) for unit, unit_ref in units: if request.cancelled(): return self._reset_storage_path(unit) if not self._needs_download(unit): # unit has no file associated self.add_unit(request, unit_ref.fetch()) continue unit_path, destination = self._path_and_destination(unit) unit_URL = pathlib.url_join(unit_inventory.base_URL, unit_path) _request = listener.create_request(unit_URL, destination, unit, unit_ref) download_list.append(_request) if request.cancelled(): return container = ContentContainer() request.summary.sources = \ container.download(request.cancel_event, request.downloader, download_list, listener) request.summary.errors.extend(listener.error_list)
def test_threaded_download(self, fake_load, fake_refresh, fake_primary, fake_batch): path = Mock() downloader = Mock() requests = Mock() listener = Mock() _batch = Mock() _batch.download.return_value = 123 fake_batch.return_value = _batch # test container = ContentContainer(path) report = container.download(downloader, requests, listener) # validation fake_load.assert_called_with(path) fake_refresh.assert_called_with() fake_primary.assert_called_with(downloader) fake_batch.assert_called_with(fake_primary(), container, requests, listener) _batch.assert_called_with() self.assertEqual(report, _batch.return_value)
def test_download(self, fake_load, fake_refresh, fake_primary, fake_batch): path = Mock() downloader = Mock() requests = Mock() listener = Mock() canceled = Mock() canceled.is_set.return_value = False _batch = Mock() _batch.download.return_value = 123 fake_batch.return_value = _batch # test container = ContentContainer(path) report = container.download(canceled, downloader, requests, listener) # validation fake_load.assert_called_with(path) fake_refresh.assert_called_with(canceled) fake_primary.assert_called_with(downloader) fake_batch.assert_called_with(canceled, fake_primary(), fake_load(), requests, listener) fake_batch().download.assert_called_with() self.assertEqual(report, _batch.download.return_value)
def test_download_with_errors(self): request_list = [] _dir, cataloged = self.populate_catalog(ORPHANED, 0, 10) _dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10) shutil.rmtree(_dir) _dir = self.populate_content(PRIMARY, 0, 20) # unit-world for n in range(0, 10): request = Request( cataloged[n].type_id, cataloged[n].unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) # primary for n in range(11, 20): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request( TYPE_ID, unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) listener = Mock() container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() # test report = container.download(downloader, request_list, listener) # validation # unit-world for i in range(0, 10): request = request_list[i] self.assertTrue(request.downloaded, msg='URL: %s' % request.url) self.assertEqual(len(request.errors), 1) with open(request.destination) as fp: s = fp.read() self.assertTrue(UNDERGROUND in s) # primary for i in range(11, len(request_list)): request = request_list[i] self.assertTrue(request.downloaded, msg='URL: %s' % request.url) self.assertEqual(len(request.errors), 0) with open(request.destination) as fp: s = fp.read() self.assertTrue(PRIMARY in s) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 3) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0) self.assertEqual(report.downloads[UNDERGROUND].total_succeeded, 10) self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 10)
class Packages(object): """ Package downloader. :ivar base_url: The repository base url. :type base_url: str :ivar units: An iterable of units to download. :type units: iterable :ivar dst_dir: The absolute path to where the packages are to be downloaded. :type dst_dir: str :ivar listener: A nectar listener. :type listener: nectar.listener.DownloadListener :ivar primary: The primary nectar downloader. :type primary: nectar.downloaders.base.Downloader :ivar container: A content container. :type container: ContentContainer :ivar url_modify: Optional URL modifier. :type url_modify: pulp_rpm.plugins.importers.yum.utils.RepoURLModifier """ def __init__(self, base_url, nectar_conf, units, dst_dir, listener, url_modify=None): """ :param base_url: The repository base url. :type base_url: str :param units: An iterable of units to download. :type units: iterable :param dst_dir: The absolute path to where the packages are to be downloaded. :type dst_dir: str :param listener: A nectar listener. :type listener: nectar.listener.DownloadListener :param url_modify: Optional URL modifier :type url_modify: pulp_rpm.plugins.importers.yum.utils.RepoURLModifier """ self.base_url = base_url self.units = units self.dst_dir = dst_dir self.listener = ContainerListener(listener) self.primary = create_downloader(base_url, nectar_conf) self.container = ContentContainer() self.url_modify = url_modify or RepoURLModifier() @property def downloader(self): """ Provided only for API comparability. :return: self """ return self def get_requests(self): """ Get requests for the units requested to be downloaded. :return: An iterable of: Request :rtype: iterable """ for unit in self.units: base_url = unit.base_url or self.base_url url = self.url_modify(base_url, path_append=unit.filename) destination = os.path.join(self.dst_dir, unit.filename) request = Request( type_id=unit.type_id, unit_key=unit.unit_key, url=url, destination=destination) request.data = unit yield request def download_packages(self): """ Download packages using alternate content source container. """ report = self.container.download(self.primary, self.get_requests(), self.listener) _log.info(CONTAINER_REPORT, dict(r=report.dict(), u=self.base_url))
class Packages(object): """ Package downloader. :ivar base_url: The repository base url. :type base_url: str :ivar units: An iterable of units to download. :type units: iterable :ivar dst_dir: The absolute path to where the packages are to be downloaded. :type dst_dir: str :ivar listener: A nectar listener. :type listener: nectar.listener.DownloadListener :ivar primary: The primary nectar downloader. :type primary: nectar.downloaders.base.Downloader :ivar container: A content container. :type container: ContentContainer :ivar url_modify: Optional URL modifier. :type url_modify: pulp_rpm.plugins.importers.yum.utils.RepoURLModifier """ def __init__(self, base_url, nectar_conf, units, dst_dir, listener, url_modify=None): """ :param base_url: The repository base url. :type base_url: str :param units: An iterable of units to download. :type units: iterable :param dst_dir: The absolute path to where the packages are to be downloaded. :type dst_dir: str :param listener: A nectar listener. :type listener: nectar.listener.DownloadListener :param url_modify: Optional URL modifier :type url_modify: pulp_rpm.plugins.importers.yum.utils.RepoURLModifier """ self.base_url = base_url self.units = units self.dst_dir = dst_dir self.listener = ContainerListener(listener) self.primary = create_downloader(base_url, nectar_conf) self.container = ContentContainer() self.url_modify = url_modify or RepoURLModifier() @property def downloader(self): """ Provided only for API comparability. :return: self """ return self def get_requests(self): """ Get requests for the units requested to be downloaded. :return: An iterable of: Request :rtype: iterable """ for unit in self.units: base_url = unit.base_url or self.base_url url = self.url_modify(base_url, path_append=unit.download_path) destination = os.path.join(self.dst_dir, unit.filename) request = Request(type_id=unit.type_id, unit_key=unit.unit_key, url=url, destination=destination) request.data = unit yield request def download_packages(self): """ Download packages using alternate content source container. """ report = self.container.download(self.primary, self.get_requests(), self.listener) _log.info(CONTAINER_REPORT, dict(r=report.dict(), u=self.base_url))
class Packages(object): """ Package downloader. :ivar base_url: The repository base url. :type base_url: str :ivar units: An iterable of units to download. :type units: iterable :ivar dst_dir: The absolute path to where the packages are to be downloaded. :type dst_dir: str :ivar listener: A nectar listener. :type listener: nectar.listener.DownloadListener :ivar primary: The primary nectar downloader. :type primary: nectar.downloaders.base.Downloader :ivar container: A content container. :type container: ContentContainer :ivar canceled: An event that signals the running download has been canceled. :type canceled: threading.Event """ def __init__(self, base_url, nectar_conf, units, dst_dir, listener): """ :param base_url: The repository base url. :type base_url: str :param units: An iterable of units to download. :type units: iterable :param dst_dir: The absolute path to where the packages are to be downloaded. :type dst_dir: str :param listener: A nectar listener. :type listener: nectar.listener.DownloadListener """ self.base_url = base_url self.units = units self.dst_dir = dst_dir self.listener = ContainerListener(listener) self.primary = create_downloader(base_url, nectar_conf) self.container = ContentContainer() self.canceled = Event() @property def downloader(self): """ Provided only for API comparability. :return: self """ return self def get_requests(self): """ Get requests for the units requested to be downloaded. :return: An iterable of: Request :rtype: iterable """ for unit in self.units: if unit.metadata.get('base_url'): url = urljoin(unit.metadata.get('base_url'), unit.download_path) else: url = urljoin(self.base_url, unit.download_path) file_name = os.path.basename(unit.relative_path) destination = os.path.join(self.dst_dir, file_name) request = Request( type_id=unit.TYPE, unit_key=unit.unit_key, url=url, destination=destination) request.data = unit yield request def download_packages(self): """ Download packages using alternate content source container. """ report = self.container.download( self.canceled, self.primary, self.get_requests(), self.listener) _log.info(CONTAINER_REPORT, dict(r=report.dict(), u=self.base_url)) def cancel(self): """ Cancel a running download. """ self.canceled.set()