def __init__(self, synchronizer, downloader):
"""
:param synchronizer: The object performing the synchronization.
:type synchronizer: SynchronizeWithDirectory
:param downloader: A nectar downloader.
:type downloader: nectar.downoaders.base.Downloader
"""
AggregatingEventListener.__init__(self)
self.synchronizer = synchronizer
self.downloader = downloader
downloader.event_listener = self
def __init__(self, synchronizer, downloader):
"""
:param synchronizer: The object performing the synchronization.
:type synchronizer: SynchronizeWithDirectory
:param downloader: A nectar downloader.
:type downloader: nectar.downoaders.base.Downloader
"""
AggregatingEventListener.__init__(self)
self.synchronizer = synchronizer
self.downloader = downloader
downloader.event_listener = self
def __init__(self, repo_url, dst_dir, nectar_config, url_modify=None):
"""
:param repo_url: URL for the base of a yum repository
:type repo_url: basestring
:param dst_dir: full path to a destination to which files
should be downloaded
:type dst_dir: basestring
:param nectar_config: download config for nectar
:type nectar_config: nectar.config.DownloaderConfig
:param url_modify: Optional URL modifier
:type url_modify: pulp_rpm.plugins.importers.yum.utils.RepoURLModifier
"""
super(MetadataFiles, self).__init__()
self._url_modify = url_modify or utils.RepoURLModifier()
self.repo_url = self._url_modify(repo_url)
self.dst_dir = dst_dir
self.event_listener = AggregatingEventListener()
self.downloader = nectar_factory.create_downloader(
self.repo_url, nectar_config, self.event_listener)
self.revision = None
self.metadata = {}
self.dbs = {}
def get_treefile(feed, tmp_dir, nectar_config):
"""
Download the treefile and return its full path on disk, or None if not found
:param feed: URL to the repository
:type feed: str
:param tmp_dir: full path to the temporary directory being used
:type tmp_dir: str
:param nectar_config: download config to be used by nectar
:type nectar_config: nectar.config.DownloaderConfig
:return: full path to treefile on disk, or None if not found
:rtype: str or NoneType
"""
for filename in constants.TREE_INFO_LIST:
path = os.path.join(tmp_dir, filename)
url = os.path.join(feed, filename)
request = DownloadRequest(url, path)
listener = AggregatingEventListener()
downloader = nectar_factory.create_downloader(feed, nectar_config,
listener)
downloader.download([request])
if len(listener.succeeded_reports) == 1:
# bz 1095829
strip_treeinfo_repomd(path)
return path
def __init__(self, name, download_config, registry_url, working_dir):
"""
Initialize the V2Repository.
:param name: name of a docker repository
:type name: basestring
:param download_config: download configuration object
:type download_config: nectar.config.DownloaderConfig
:param registry_url: URL for the docker registry
:type registry_url: basestring
:param working_dir: full path to the directory where files should
be saved
:type working_dir: basestring
"""
# Docker's registry aligns non-namespaced images to the library namespace.
# if we have a docker registry image, and no namespace, add the library
# namespace to the image name.
if '/' not in name and re.search(r'registry[-,\w]*.docker.io',
registry_url, re.IGNORECASE):
self.name = "library/" + name
else:
self.name = name
self.download_config = download_config
self.registry_url = registry_url
self.downloader = HTTPThreadedDownloader(self.download_config,
AggregatingEventListener())
self.working_dir = working_dir
self.token = None
def get_distribution_file(feed, tmp_dir, nectar_config):
"""
Download the pulp_distribution.xml and return its full path on disk, or None if not found
:param feed: URL to the repository
:type feed: str
:param tmp_dir: full path to the temporary directory being used
:type tmp_dir: str
:param nectar_config: download config to be used by nectar
:type nectar_config: nectar.config.DownloaderConfig
:return: full path to distribution file on disk, or None if not found
:rtype: str or NoneType
"""
filename = constants.DISTRIBUTION_XML
path = os.path.join(tmp_dir, filename)
url = os.path.join(feed, filename)
request = DownloadRequest(url, path)
listener = AggregatingEventListener()
downloader = nectar_factory.create_downloader(feed, nectar_config,
listener)
downloader.download([request])
if len(listener.succeeded_reports) == 1:
return path
return None
def test_does_fires_events(self):
# collect the success event in this listener if one is fired
listener = AggregatingEventListener()
downloader = LyingDownloader(DownloaderConfig(), listener)
downloader.download_one(self.request, events=True)
self.assertEqual(len(listener.succeeded_reports), 1)
def test_copy(self):
config = DownloaderConfig()
listener = AggregatingEventListener()
downloader = local.LocalFileDownloader(config, listener)
request_list = self._make_requests(DATA_FILES)
downloader.download(request_list)
self.assertEqual(len(listener.succeeded_reports), len(request_list))
self.assertEqual(len(listener.failed_reports), 0)
def test_symbolic_link_download(self):
config = DownloaderConfig(use_sym_links=True)
listener = AggregatingEventListener()
downloader = local.LocalFileDownloader(config, listener)
request_list = self._make_requests(DATA_FILES[:1])
downloader.download(request_list)
self.assertEqual(len(listener.succeeded_reports), 1)
self.assertEqual(len(listener.failed_reports), 0)
self.assertTrue(os.path.islink(request_list[0].destination))
def test_destination_bad_permissions(self):
config = DownloaderConfig(use_sym_links=True)
listener = AggregatingEventListener()
downloader = local.LocalFileDownloader(config, listener)
request_list = self._make_requests(DATA_DIR[:1])
request_list[0].destination = '/' + DATA_FILES[0]
downloader.download(request_list)
self.assertEqual(len(listener.succeeded_reports), 0)
self.assertEqual(len(listener.failed_reports), 1)
def test_source_bad_permissions(self):
config = DownloaderConfig(use_sym_links=True)
listener = AggregatingEventListener()
downloader = local.LocalFileDownloader(config, listener)
request = DownloadRequest('file://root/no',
os.path.join(self.dest_dir, 'doesnt.even.matter'))
downloader.download([request])
self.assertEqual(len(listener.succeeded_reports), 0)
self.assertEqual(len(listener.failed_reports), 1)
def test_unsupported_url_scheme(self):
config = DownloaderConfig(use_sym_links=True)
listener = AggregatingEventListener()
downloader = local.LocalFileDownloader(config, listener)
request = DownloadRequest('http://thiswontwork.com',
os.path.join(self.dest_dir, 'doesnt.even.matter'))
downloader.download([request])
self.assertEqual(len(listener.succeeded_reports), 0)
self.assertEqual(len(listener.failed_reports), 1)
def test_unlinkable_destination(self):
config = DownloaderConfig(use_hard_links=True)
listener = AggregatingEventListener()
downloader = local.LocalFileDownloader(config, listener)
request_list = self._make_requests(DATA_DIR[:1])
request_list[0].destination = StringIO()
downloader.download(request_list)
self.assertEqual(len(listener.succeeded_reports), 0)
self.assertEqual(len(listener.failed_reports), 1)
def test_calls_download_method(self):
config = DownloaderConfig()
listener = AggregatingEventListener()
downloader = local.LocalFileDownloader(config, listener)
request = DownloadRequest('http://foo', StringIO())
report = DownloadReport.from_download_request(request)
# mock _copy, which is the default function to which requests are passed
with mock.patch.object(downloader, '_copy') as mock_method:
mock_method.return_value = report
ret = downloader._download_one(request)
self.assertEqual(ret, report)
mock_method.assert_called_once_with(request)
def test_copy_download(self):
config = DownloaderConfig()
listener = AggregatingEventListener()
downloader = local.LocalFileDownloader(config, listener)
request_list = self._make_requests(DATA_FILES[:1])
downloader.download(request_list)
self.assertEqual(len(listener.succeeded_reports), 1)
self.assertEqual(len(listener.failed_reports), 0)
src_stat = os.stat(os.path.join(DATA_DIR, DATA_FILES[0]))
dst_stat = os.stat(request_list[0].destination)
self.assertEqual(src_stat.st_size, dst_stat.st_size)
self.assertNotEqual(src_stat.st_ino, dst_stat.st_ino)
def fetch_units(self):
"""
Fetch the units file referenced in the manifest.
:raise ManifestDownloadError: on downloading errors.
:raise HTTPError: on URL errors.
:raise ValueError: on json decoding errors
"""
base_url = self.url.rsplit('/', 1)[0]
url = pathlib.join(base_url, UNITS_FILE_NAME)
destination = pathlib.join(os.path.dirname(self.path), UNITS_FILE_NAME)
request = DownloadRequest(str(url), destination)
listener = AggregatingEventListener()
self.downloader.event_listener = listener
self.downloader.download([request])
if listener.failed_reports:
report = listener.failed_reports[0]
raise ManifestDownloadError(self.url, report.error_msg)
def fetch(self):
"""
Fetch the manifest file using the specified URL.
:raise ManifestDownloadError: on downloading errors.
:raise HTTPError: on URL errors.
:raise ValueError: on json decoding errors
"""
listener = AggregatingEventListener()
destination = os.path.join(os.path.dirname(self.path),
'.' + MANIFEST_FILE_NAME)
request = DownloadRequest(self.url, destination)
self.downloader.event_listener = listener
self.downloader.download([request])
if listener.failed_reports:
report = listener.failed_reports[0]
raise ManifestDownloadError(self.url, report.error_msg)
self.read(destination)
def get_distribution_file(self, tmp_dir):
"""
Download the pulp_distribution.xml and return its full path on disk, or None if not found
:param tmp_dir: The absolute path to the temporary directory
:type tmp_dir: str
:return: The absolute path to distribution file on disk, or None if not found
:rtype: str or None
"""
filename = constants.DISTRIBUTION_XML
path = os.path.join(tmp_dir, filename)
url = os.path.join(self.feed, filename)
request = DownloadRequest(url, path)
listener = AggregatingEventListener()
downloader = nectar_factory.create_downloader(self.feed, self.nectar_config, listener)
downloader.download([request])
if len(listener.succeeded_reports) == 1:
return path
return None
def test__common_link_when_source_not_found(self, mock_logger):
"""
Test that nectar properly handles an attempt to link local files
that do not exist.
"""
config = DownloaderConfig(use_sym_links=True)
listener = AggregatingEventListener()
downloader = local.LocalFileDownloader(config, listener)
request = DownloadRequest('file://i/dont/exist',
os.path.join(self.dest_dir, 'doesnt.even.matter'))
downloader.download([request])
self.assertEqual(len(listener.succeeded_reports), 0)
self.assertEqual(len(listener.failed_reports), 1)
debug_messages = ''.join([mock_call[1][0][1] for mock_call in mock_logger.debug.mock_calls])
self.assertTrue('No such file or directory' in debug_messages)
self.assertEqual(mock_logger.exception.call_count, 0)
def get_treefile(self, tmp_dir):
"""
Download the treefile and return its full path on disk, or None if not found
:param tmp_dir: The absolute path to the temporary directory
:type tmp_dir: str
:return: The absolute path to treefile on disk, or None if not found
:rtype: str or None
"""
for filename in constants.TREE_INFO_LIST:
path = os.path.join(tmp_dir, filename)
url = os.path.join(self.feed, filename)
request = DownloadRequest(url, path)
listener = AggregatingEventListener()
downloader = nectar_factory.create_downloader(self.feed, self.nectar_config, listener)
downloader.download([request])
if len(listener.succeeded_reports) == 1:
# bz 1095829
self.strip_treeinfo_repomd(path)
return path
def __init__(self, repo_url, dst_dir, nectar_config):
"""
:param repo_url: URL for the base of a yum repository
:type repo_url: basestring
:param dst_dir: full path to a destination to which files
should be downloaded
:type dst_dir: basestring
:param nectar_config: download config for nectar
:type nectar_config: nectar.config.DownloaderConfig
"""
super(MetadataFiles, self).__init__()
self.repo_url = repo_url
self.dst_dir = dst_dir
self.event_listener = AggregatingEventListener()
self.downloader = nectar_factory.create_downloader(
repo_url, nectar_config, self.event_listener)
self.revision = None
self.metadata = {}
self.dbs = {}
def __init__(self, name, download_config, registry_url, working_dir):
"""
:param name: name of a docker repository
:type name: basestring
:param download_config: download configuration object
:type download_config: nectar.config.DownloaderConfig
:param registry_url: URL for the docker registry
:type registry_url: basestring
:param working_dir: full path to the directory where files should
be saved
:type working_dir: basestring
"""
self.name = name
self.download_config = download_config
self.registry_url = registry_url
self.listener = AggregatingEventListener()
self.downloader = HTTPThreadedDownloader(self.download_config,
self.listener)
self.working_dir = working_dir
self.token = None
self.endpoint = None