def __init__(self, synchronizer, downloader): """ :param synchronizer: The object performing the synchronization. :type synchronizer: SynchronizeWithDirectory :param downloader: A nectar downloader. :type downloader: nectar.downoaders.base.Downloader """ AggregatingEventListener.__init__(self) self.synchronizer = synchronizer self.downloader = downloader downloader.event_listener = self
def __init__(self, repo_url, dst_dir, nectar_config, url_modify=None): """ :param repo_url: URL for the base of a yum repository :type repo_url: basestring :param dst_dir: full path to a destination to which files should be downloaded :type dst_dir: basestring :param nectar_config: download config for nectar :type nectar_config: nectar.config.DownloaderConfig :param url_modify: Optional URL modifier :type url_modify: pulp_rpm.plugins.importers.yum.utils.RepoURLModifier """ super(MetadataFiles, self).__init__() self._url_modify = url_modify or utils.RepoURLModifier() self.repo_url = self._url_modify(repo_url) self.dst_dir = dst_dir self.event_listener = AggregatingEventListener() self.downloader = nectar_factory.create_downloader( self.repo_url, nectar_config, self.event_listener) self.revision = None self.metadata = {} self.dbs = {}
def get_treefile(feed, tmp_dir, nectar_config): """ Download the treefile and return its full path on disk, or None if not found :param feed: URL to the repository :type feed: str :param tmp_dir: full path to the temporary directory being used :type tmp_dir: str :param nectar_config: download config to be used by nectar :type nectar_config: nectar.config.DownloaderConfig :return: full path to treefile on disk, or None if not found :rtype: str or NoneType """ for filename in constants.TREE_INFO_LIST: path = os.path.join(tmp_dir, filename) url = os.path.join(feed, filename) request = DownloadRequest(url, path) listener = AggregatingEventListener() downloader = nectar_factory.create_downloader(feed, nectar_config, listener) downloader.download([request]) if len(listener.succeeded_reports) == 1: # bz 1095829 strip_treeinfo_repomd(path) return path
def __init__(self, name, download_config, registry_url, working_dir): """ Initialize the V2Repository. :param name: name of a docker repository :type name: basestring :param download_config: download configuration object :type download_config: nectar.config.DownloaderConfig :param registry_url: URL for the docker registry :type registry_url: basestring :param working_dir: full path to the directory where files should be saved :type working_dir: basestring """ # Docker's registry aligns non-namespaced images to the library namespace. # if we have a docker registry image, and no namespace, add the library # namespace to the image name. if '/' not in name and re.search(r'registry[-,\w]*.docker.io', registry_url, re.IGNORECASE): self.name = "library/" + name else: self.name = name self.download_config = download_config self.registry_url = registry_url self.downloader = HTTPThreadedDownloader(self.download_config, AggregatingEventListener()) self.working_dir = working_dir self.token = None
def get_distribution_file(feed, tmp_dir, nectar_config): """ Download the pulp_distribution.xml and return its full path on disk, or None if not found :param feed: URL to the repository :type feed: str :param tmp_dir: full path to the temporary directory being used :type tmp_dir: str :param nectar_config: download config to be used by nectar :type nectar_config: nectar.config.DownloaderConfig :return: full path to distribution file on disk, or None if not found :rtype: str or NoneType """ filename = constants.DISTRIBUTION_XML path = os.path.join(tmp_dir, filename) url = os.path.join(feed, filename) request = DownloadRequest(url, path) listener = AggregatingEventListener() downloader = nectar_factory.create_downloader(feed, nectar_config, listener) downloader.download([request]) if len(listener.succeeded_reports) == 1: return path return None
def test_does_fires_events(self): # collect the success event in this listener if one is fired listener = AggregatingEventListener() downloader = LyingDownloader(DownloaderConfig(), listener) downloader.download_one(self.request, events=True) self.assertEqual(len(listener.succeeded_reports), 1)
def test_copy(self): config = DownloaderConfig() listener = AggregatingEventListener() downloader = local.LocalFileDownloader(config, listener) request_list = self._make_requests(DATA_FILES) downloader.download(request_list) self.assertEqual(len(listener.succeeded_reports), len(request_list)) self.assertEqual(len(listener.failed_reports), 0)
def test_symbolic_link_download(self): config = DownloaderConfig(use_sym_links=True) listener = AggregatingEventListener() downloader = local.LocalFileDownloader(config, listener) request_list = self._make_requests(DATA_FILES[:1]) downloader.download(request_list) self.assertEqual(len(listener.succeeded_reports), 1) self.assertEqual(len(listener.failed_reports), 0) self.assertTrue(os.path.islink(request_list[0].destination))
def test_destination_bad_permissions(self): config = DownloaderConfig(use_sym_links=True) listener = AggregatingEventListener() downloader = local.LocalFileDownloader(config, listener) request_list = self._make_requests(DATA_DIR[:1]) request_list[0].destination = '/' + DATA_FILES[0] downloader.download(request_list) self.assertEqual(len(listener.succeeded_reports), 0) self.assertEqual(len(listener.failed_reports), 1)
def test_source_bad_permissions(self): config = DownloaderConfig(use_sym_links=True) listener = AggregatingEventListener() downloader = local.LocalFileDownloader(config, listener) request = DownloadRequest('file://root/no', os.path.join(self.dest_dir, 'doesnt.even.matter')) downloader.download([request]) self.assertEqual(len(listener.succeeded_reports), 0) self.assertEqual(len(listener.failed_reports), 1)
def test_unsupported_url_scheme(self): config = DownloaderConfig(use_sym_links=True) listener = AggregatingEventListener() downloader = local.LocalFileDownloader(config, listener) request = DownloadRequest('http://thiswontwork.com', os.path.join(self.dest_dir, 'doesnt.even.matter')) downloader.download([request]) self.assertEqual(len(listener.succeeded_reports), 0) self.assertEqual(len(listener.failed_reports), 1)
def test_unlinkable_destination(self): config = DownloaderConfig(use_hard_links=True) listener = AggregatingEventListener() downloader = local.LocalFileDownloader(config, listener) request_list = self._make_requests(DATA_DIR[:1]) request_list[0].destination = StringIO() downloader.download(request_list) self.assertEqual(len(listener.succeeded_reports), 0) self.assertEqual(len(listener.failed_reports), 1)
def test_calls_download_method(self): config = DownloaderConfig() listener = AggregatingEventListener() downloader = local.LocalFileDownloader(config, listener) request = DownloadRequest('http://foo', StringIO()) report = DownloadReport.from_download_request(request) # mock _copy, which is the default function to which requests are passed with mock.patch.object(downloader, '_copy') as mock_method: mock_method.return_value = report ret = downloader._download_one(request) self.assertEqual(ret, report) mock_method.assert_called_once_with(request)
def test_copy_download(self): config = DownloaderConfig() listener = AggregatingEventListener() downloader = local.LocalFileDownloader(config, listener) request_list = self._make_requests(DATA_FILES[:1]) downloader.download(request_list) self.assertEqual(len(listener.succeeded_reports), 1) self.assertEqual(len(listener.failed_reports), 0) src_stat = os.stat(os.path.join(DATA_DIR, DATA_FILES[0])) dst_stat = os.stat(request_list[0].destination) self.assertEqual(src_stat.st_size, dst_stat.st_size) self.assertNotEqual(src_stat.st_ino, dst_stat.st_ino)
def fetch_units(self): """ Fetch the units file referenced in the manifest. :raise ManifestDownloadError: on downloading errors. :raise HTTPError: on URL errors. :raise ValueError: on json decoding errors """ base_url = self.url.rsplit('/', 1)[0] url = pathlib.join(base_url, UNITS_FILE_NAME) destination = pathlib.join(os.path.dirname(self.path), UNITS_FILE_NAME) request = DownloadRequest(str(url), destination) listener = AggregatingEventListener() self.downloader.event_listener = listener self.downloader.download([request]) if listener.failed_reports: report = listener.failed_reports[0] raise ManifestDownloadError(self.url, report.error_msg)
def fetch(self): """ Fetch the manifest file using the specified URL. :raise ManifestDownloadError: on downloading errors. :raise HTTPError: on URL errors. :raise ValueError: on json decoding errors """ listener = AggregatingEventListener() destination = os.path.join(os.path.dirname(self.path), '.' + MANIFEST_FILE_NAME) request = DownloadRequest(self.url, destination) self.downloader.event_listener = listener self.downloader.download([request]) if listener.failed_reports: report = listener.failed_reports[0] raise ManifestDownloadError(self.url, report.error_msg) self.read(destination)
def get_distribution_file(self, tmp_dir): """ Download the pulp_distribution.xml and return its full path on disk, or None if not found :param tmp_dir: The absolute path to the temporary directory :type tmp_dir: str :return: The absolute path to distribution file on disk, or None if not found :rtype: str or None """ filename = constants.DISTRIBUTION_XML path = os.path.join(tmp_dir, filename) url = os.path.join(self.feed, filename) request = DownloadRequest(url, path) listener = AggregatingEventListener() downloader = nectar_factory.create_downloader(self.feed, self.nectar_config, listener) downloader.download([request]) if len(listener.succeeded_reports) == 1: return path return None
def test__common_link_when_source_not_found(self, mock_logger): """ Test that nectar properly handles an attempt to link local files that do not exist. """ config = DownloaderConfig(use_sym_links=True) listener = AggregatingEventListener() downloader = local.LocalFileDownloader(config, listener) request = DownloadRequest('file://i/dont/exist', os.path.join(self.dest_dir, 'doesnt.even.matter')) downloader.download([request]) self.assertEqual(len(listener.succeeded_reports), 0) self.assertEqual(len(listener.failed_reports), 1) debug_messages = ''.join([mock_call[1][0][1] for mock_call in mock_logger.debug.mock_calls]) self.assertTrue('No such file or directory' in debug_messages) self.assertEqual(mock_logger.exception.call_count, 0)
def get_treefile(self, tmp_dir): """ Download the treefile and return its full path on disk, or None if not found :param tmp_dir: The absolute path to the temporary directory :type tmp_dir: str :return: The absolute path to treefile on disk, or None if not found :rtype: str or None """ for filename in constants.TREE_INFO_LIST: path = os.path.join(tmp_dir, filename) url = os.path.join(self.feed, filename) request = DownloadRequest(url, path) listener = AggregatingEventListener() downloader = nectar_factory.create_downloader(self.feed, self.nectar_config, listener) downloader.download([request]) if len(listener.succeeded_reports) == 1: # bz 1095829 self.strip_treeinfo_repomd(path) return path
def __init__(self, repo_url, dst_dir, nectar_config): """ :param repo_url: URL for the base of a yum repository :type repo_url: basestring :param dst_dir: full path to a destination to which files should be downloaded :type dst_dir: basestring :param nectar_config: download config for nectar :type nectar_config: nectar.config.DownloaderConfig """ super(MetadataFiles, self).__init__() self.repo_url = repo_url self.dst_dir = dst_dir self.event_listener = AggregatingEventListener() self.downloader = nectar_factory.create_downloader( repo_url, nectar_config, self.event_listener) self.revision = None self.metadata = {} self.dbs = {}
def __init__(self, name, download_config, registry_url, working_dir): """ :param name: name of a docker repository :type name: basestring :param download_config: download configuration object :type download_config: nectar.config.DownloaderConfig :param registry_url: URL for the docker registry :type registry_url: basestring :param working_dir: full path to the directory where files should be saved :type working_dir: basestring """ self.name = name self.download_config = download_config self.registry_url = registry_url self.listener = AggregatingEventListener() self.downloader = HTTPThreadedDownloader(self.download_config, self.listener) self.working_dir = working_dir self.token = None self.endpoint = None