def get_treefile(feed, tmp_dir, nectar_config):
"""
Download the treefile and return its full path on disk, or None if not found
:param feed: URL to the repository
:type feed: str
:param tmp_dir: full path to the temporary directory being used
:type tmp_dir: str
:param nectar_config: download config to be used by nectar
:type nectar_config: nectar.config.DownloaderConfig
:return: full path to treefile on disk, or None if not found
:rtype: str or NoneType
"""
for filename in constants.TREE_INFO_LIST:
path = os.path.join(tmp_dir, filename)
url = os.path.join(feed, filename)
request = DownloadRequest(url, path)
listener = AggregatingEventListener()
downloader = nectar_factory.create_downloader(feed, nectar_config, listener)
downloader.download([request])
if len(listener.succeeded_reports) == 1:
# bz 1095829
strip_treeinfo_repomd(path)
return path
def download_files(self, tmp_dir, files):
"""
Download distribution files.
:param tmp_dir: The absolute to where the downloaded files.
:type tmp_dir: str
:param files: List of distribution dictionary files.
:type files: list
:return: generator of: (destination, location)
The *destination* is the absolute path to the downloaded file.
The *location* is the relative path within the tmp_dir.
:rtype: generator
:raise DownloadFailed: if any of the downloads fail.
"""
listener = DistFileListener(self)
self.progress_report.set_initial_values(len(files))
downloader = nectar_factory.create_downloader(self.feed,
self.nectar_config,
listener)
requests = (self.file_to_download_request(f, tmp_dir) for f in files)
downloader.download(requests)
if len(listener.failed_reports):
_logger.error(_('some distro file downloads failed'))
self.progress_report['state'] = constants.STATE_FAILED
self.progress_report['error_details'] = [
(fail.url, fail.error_report)
for fail in listener.failed_reports
]
raise DownloadFailed()
for report in listener.succeeded_reports:
location = os.path.relpath(report.destination, tmp_dir)
yield report.destination, location
def _parse_as_mirrorlist(self, feed):
"""
Treats the provided feed as mirrorlist. Parses its content and extracts
urls to sync.
:param feed: feed that should be treated as mirrorlist
:type: str
:return: list the URLs received from the mirrorlist
:rtype: list
"""
url_file = StringIO()
downloader = nectar_factory.create_downloader(feed, self.nectar_config)
request = DownloadRequest(feed, url_file)
downloader.download_one(request)
url_file.seek(0)
url_parse = url_file.read().split('\n')
repo_url = []
# Due to the fact, that format of mirrorlist can be different, this regex
# matches the cases when the url is not commented out and does not have any
# punctuation characters in front.
pattern = re.compile("(^|^[\w\s=]+\s)((http(s)?)://.*)")
for line in url_parse:
for match in re.finditer(pattern, line):
repo_url.append(match.group(2))
random.shuffle(repo_url)
return repo_url
def __init__(self,
base_url,
nectar_conf,
units,
dst_dir,
listener,
url_modify=None):
"""
:param base_url: The repository base url.
:type base_url: str
:param units: An iterable of units to download.
:type units: iterable
:param dst_dir: The absolute path to where the packages are to be downloaded.
:type dst_dir: str
:param listener: A nectar listener.
:type listener: nectar.listener.DownloadListener
:param url_modify: Optional URL modifier
:type url_modify: pulp_rpm.plugins.importers.yum.utils.RepoURLModifier
"""
self.base_url = base_url
self.units = units
self.dst_dir = dst_dir
self.listener = ContainerListener(listener)
self.primary = create_downloader(base_url, nectar_conf)
self.container = ContentContainer()
self.url_modify = url_modify or RepoURLModifier()
def __init__(self, repo_url, dst_dir, nectar_config, url_modify=None):
"""
:param repo_url: URL for the base of a yum repository
:type repo_url: basestring
:param dst_dir: full path to a destination to which files
should be downloaded
:type dst_dir: basestring
:param nectar_config: download config for nectar
:type nectar_config: nectar.config.DownloaderConfig
:param url_modify: Optional URL modifier
:type url_modify: pulp_rpm.plugins.importers.yum.utils.RepoURLModifier
"""
super(MetadataFiles, self).__init__()
self._url_modify = url_modify or utils.RepoURLModifier()
self.repo_url = self._url_modify(repo_url)
self.dst_dir = dst_dir
self.event_listener = AggregatingEventListener()
self.downloader = nectar_factory.create_downloader(
self.repo_url, nectar_config, self.event_listener)
self.revision = None
self.metadata = {}
self.dbs = {}
def _parse_as_mirrorlist(self, feed):
"""
Treats the provided feed as mirrorlist. Parses its content and extracts
urls to sync.
:param feed: feed that should be treated as mirrorlist
:type: str
:return: list the URLs received from the mirrorlist
:rtype: list
"""
url_file = StringIO()
downloader = nectar_factory.create_downloader(feed, self.nectar_config)
request = DownloadRequest(feed, url_file)
downloader.download_one(request)
url_file.seek(0)
url_parse = url_file.read().split('\n')
repo_url = []
# Due to the fact, that format of mirrorlist can be different, this regex
# matches the cases when the url is not commented out and does not have any
# punctuation characters in front.
pattern = re.compile("(^|^[\w\s=]+\s)((http(s)?)://.*)")
for line in url_parse:
for match in re.finditer(pattern, line):
repo_url.append(match.group(2))
random.shuffle(repo_url)
return repo_url
def download_files(self, tmp_dir, files):
"""
Download distribution files.
:param tmp_dir: The absolute to where the downloaded files.
:type tmp_dir: str
:param files: List of distribution dictionary files.
:type files: list
:return: generator of: (destination, location)
The *destination* is the absolute path to the downloaded file.
The *location* is the relative path within the tmp_dir.
:rtype: generator
:raise DownloadFailed: if any of the downloads fail.
"""
listener = DistFileListener(self)
self.progress_report.set_initial_values(len(files))
downloader = nectar_factory.create_downloader(self.feed, self.nectar_config, listener)
requests = (self.file_to_download_request(f, tmp_dir) for f in files)
downloader.download(requests)
if len(listener.failed_reports):
_logger.error(_('some distro file downloads failed'))
self.progress_report['state'] = constants.STATE_FAILED
self.progress_report['error_details'] = [
(fail.url, fail.error_report) for fail in listener.failed_reports
]
raise DownloadFailed()
for report in listener.succeeded_reports:
location = os.path.relpath(report.destination, tmp_dir)
yield report.destination, location
def __init__(self, repo_url, dst_dir, nectar_config, url_modify=None):
"""
:param repo_url: URL for the base of a yum repository
:type repo_url: basestring
:param dst_dir: full path to a destination to which files
should be downloaded
:type dst_dir: basestring
:param nectar_config: download config for nectar
:type nectar_config: nectar.config.DownloaderConfig
:param url_modify: Optional URL modifier
:type url_modify: pulp_rpm.plugins.importers.yum.utils.RepoURLModifier
"""
super(MetadataFiles, self).__init__()
self._url_modify = url_modify or utils.RepoURLModifier()
self.repo_url = self._url_modify(repo_url)
self.dst_dir = dst_dir
self.event_listener = AggregatingEventListener()
self.downloader = nectar_factory.create_downloader(self.repo_url, nectar_config,
self.event_listener)
self.revision = None
self.metadata = {}
self.dbs = {}
self.rpm_count = None
def get_treefile(feed, tmp_dir, nectar_config):
"""
Download the treefile and return its full path on disk, or None if not found
:param feed: URL to the repository
:type feed: str
:param tmp_dir: full path to the temporary directory being used
:type tmp_dir: str
:param nectar_config: download config to be used by nectar
:type nectar_config: nectar.config.DownloaderConfig
:return: full path to treefile on disk, or None if not found
:rtype: str or NoneType
"""
for filename in constants.TREE_INFO_LIST:
path = os.path.join(tmp_dir, filename)
url = os.path.join(feed, filename)
request = DownloadRequest(url, path)
listener = AggregatingEventListener()
downloader = nectar_factory.create_downloader(feed, nectar_config,
listener)
downloader.download([request])
if len(listener.succeeded_reports) == 1:
# bz 1095829
strip_treeinfo_repomd(path)
return path
def get_distribution_file(feed, tmp_dir, nectar_config):
"""
Download the pulp_distribution.xml and return its full path on disk, or None if not found
:param feed: URL to the repository
:type feed: str
:param tmp_dir: full path to the temporary directory being used
:type tmp_dir: str
:param nectar_config: download config to be used by nectar
:type nectar_config: nectar.config.DownloaderConfig
:return: full path to distribution file on disk, or None if not found
:rtype: str or NoneType
"""
filename = constants.DISTRIBUTION_XML
path = os.path.join(tmp_dir, filename)
url = os.path.join(feed, filename)
request = DownloadRequest(url, path)
listener = AggregatingEventListener()
downloader = nectar_factory.create_downloader(feed, nectar_config,
listener)
downloader.download([request])
if len(listener.succeeded_reports) == 1:
return path
return None
def get_distribution_file(feed, tmp_dir, nectar_config):
"""
Download the pulp_distribution.xml and return its full path on disk, or None if not found
:param feed: URL to the repository
:type feed: str
:param tmp_dir: full path to the temporary directory being used
:type tmp_dir: str
:param nectar_config: download config to be used by nectar
:type nectar_config: nectar.config.DownloaderConfig
:return: full path to distribution file on disk, or None if not found
:rtype: str or NoneType
"""
filename = constants.DISTRIBUTION_XML
path = os.path.join(tmp_dir, filename)
url = os.path.join(feed, filename)
request = DownloadRequest(url, path)
listener = AggregatingEventListener()
downloader = nectar_factory.create_downloader(feed, nectar_config, listener)
downloader.download([request])
if len(listener.succeeded_reports) == 1:
return path
return None
def __init__(self, repo_url, nectar_config, package_model_iterator, dst_dir,
event_listener=None):
self.repo_url = repo_url
self.package_model_iterator = package_model_iterator
self.dst_dir = dst_dir
self.downloader = nectar_factory.create_downloader(repo_url, nectar_config,
event_listener)
def sync(sync_conduit, feed, working_dir, nectar_config, report, progress_callback):
"""
Look for a distribution in the target repo and sync it if found
:param sync_conduit: conduit provided by the platform
:type sync_conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param feed: URL of the yum repo being sync'd
:type feed: basestring
:param working_dir: full path to the directory to which files
should be downloaded
:type working_dir: basestring
:param nectar_config: download config to be used by nectar
:type nectar_config: nectar.config.DownloaderConfig
:param report: progress report object
:type report: pulp_rpm.plugins.importers.yum.report.DistributionReport
:param progress_callback: function that takes no arguments but induces
the current progress report to be sent.
"""
# this temporary dir will hopefully be moved to the unit's storage path
# if all downloads go well. If not, it will be deleted below, ensuring a
# complete cleanup
tmp_dir = tempfile.mkdtemp(dir=working_dir)
try:
treefile_path = get_treefile(feed, tmp_dir, nectar_config)
if not treefile_path:
_LOGGER.debug('no treefile found')
report['state'] = constants.STATE_COMPLETE
return
try:
model, files = parse_treefile(treefile_path)
except ValueError:
_LOGGER.error('could not parse treefile')
report['state'] = constants.STATE_FAILED
return
report.set_initial_values(len(files))
listener = DistroFileListener(report, progress_callback)
downloader = nectar_factory.create_downloader(feed, nectar_config, listener)
_LOGGER.debug('downloading distribution files')
downloader.download(file_to_download_request(f, feed, tmp_dir) for f in files)
if len(listener.failed_reports) == 0:
unit = sync_conduit.init_unit(ids.TYPE_ID_DISTRO, model.unit_key, model.metadata, model.relative_path)
model.process_download_reports(listener.succeeded_reports)
# remove pre-existing dir
shutil.rmtree(unit.storage_path, ignore_errors=True)
shutil.move(tmp_dir, unit.storage_path)
# mkdtemp is very paranoid, so we'll change to more sensible perms
os.chmod(unit.storage_path, 0o775)
sync_conduit.save_unit(unit)
else:
_LOGGER.error('some distro file downloads failed')
report['state'] = constants.STATE_FAILED
report['error_details'] = [(fail.url, fail.error_report) for fail in listener.failed_reports]
return
report['state'] = constants.STATE_COMPLETE
finally:
shutil.rmtree(tmp_dir, ignore_errors=True)
def __init__(self,
repo_url,
nectar_config,
package_model_iterator,
dst_dir,
event_listener=None):
self.repo_url = repo_url
self.package_model_iterator = package_model_iterator
self.dst_dir = dst_dir
self.downloader = nectar_factory.create_downloader(
repo_url, nectar_config, event_listener)
def get_downloader(self, conduit, config, url):
"""
Get object suitable for downloading content published
in the content catalog by a content source.
:param conduit: Access to pulp platform API.
:type conduit: pulp.server.plugins.conduits.cataloger.CatalogerConduit
:param config: The content source configuration.
:type config: dict
:param url: The URL for the content source.
:type url: str
:return: A configured downloader.
:rtype: nectar.downloaders.base.Downloader
"""
return nectar_factory.create_downloader(url, self.nectar_config(config))
def get_downloader(self, conduit, config, url):
"""
Get object suitable for downloading content published
in the content catalog by a content source.
:param conduit: Access to pulp platform API.
:type conduit: pulp.server.plugins.conduits.cataloger.CatalogerConduit
:param config: The content source configuration.
:type config: dict
:param url: The URL for the content source.
:type url: str
:return: A configured downloader.
:rtype: nectar.downloaders.base.Downloader
"""
return nectar_factory.create_downloader(url,
self.nectar_config(config))
def __init__(self, base_url, nectar_conf, units, dst_dir, listener):
"""
:param base_url: The repository base url.
:type base_url: str
:param units: An iterable of units to download.
:type units: iterable
:param dst_dir: The absolute path to where the packages are to be downloaded.
:type dst_dir: str
:param listener: A nectar listener.
:type listener: nectar.listener.DownloadListener
"""
self.base_url = base_url
self.units = units
self.dst_dir = dst_dir
self.listener = ContainerListener(listener)
self.primary = create_downloader(base_url, nectar_conf)
self.container = ContentContainer()
self.canceled = Event()
def get_distribution_file(self, tmp_dir):
"""
Download the pulp_distribution.xml and return its full path on disk, or None if not found
:param tmp_dir: The absolute path to the temporary directory
:type tmp_dir: str
:return: The absolute path to distribution file on disk, or None if not found
:rtype: str or None
"""
filename = constants.DISTRIBUTION_XML
path = os.path.join(tmp_dir, filename)
url = os.path.join(self.feed, filename)
request = DownloadRequest(url, path)
listener = AggregatingEventListener()
downloader = nectar_factory.create_downloader(self.feed, self.nectar_config, listener)
downloader.download([request])
if len(listener.succeeded_reports) == 1:
return path
return None
def get_distribution_file(self, tmp_dir):
"""
Download the pulp_distribution.xml and return its full path on disk, or None if not found
:param tmp_dir: The absolute path to the temporary directory
:type tmp_dir: str
:return: The absolute path to distribution file on disk, or None if not found
:rtype: str or None
"""
filename = constants.DISTRIBUTION_XML
path = os.path.join(tmp_dir, filename)
url = os.path.join(self.feed, filename)
request = DownloadRequest(url, path)
listener = AggregatingEventListener()
downloader = nectar_factory.create_downloader(self.feed, self.nectar_config, listener)
downloader.download([request])
if len(listener.succeeded_reports) == 1:
return path
return None
def __init__(self, repo_url, dst_dir, nectar_config):
"""
:param repo_url: URL for the base of a yum repository
:type repo_url: basestring
:param dst_dir: full path to a destination to which files
should be downloaded
:type dst_dir: basestring
:param nectar_config: download config for nectar
:type nectar_config: nectar.config.DownloaderConfig
"""
super(MetadataFiles, self).__init__()
self.repo_url = repo_url
self.dst_dir = dst_dir
self.event_listener = AggregatingEventListener()
self.downloader = nectar_factory.create_downloader(repo_url, nectar_config, self.event_listener)
self.revision = None
self.metadata = {}
self.dbs = {}
def __init__(self, base_url, nectar_conf, units, dst_dir, listener, url_modify=None):
"""
:param base_url: The repository base url.
:type base_url: str
:param units: An iterable of units to download.
:type units: iterable
:param dst_dir: The absolute path to where the packages are to be downloaded.
:type dst_dir: str
:param listener: A nectar listener.
:type listener: nectar.listener.DownloadListener
:param url_modify: Optional URL modifier
:type url_modify: pulp_rpm.plugins.importers.yum.utils.RepoURLModifier
"""
self.base_url = base_url
self.units = units
self.dst_dir = dst_dir
self.listener = ContainerListener(listener)
self.primary = create_downloader(base_url, nectar_conf)
self.container = ContentContainer()
self.url_modify = url_modify or RepoURLModifier()
def get_treefile(self, tmp_dir):
"""
Download the treefile and return its full path on disk, or None if not found
:param tmp_dir: The absolute path to the temporary directory
:type tmp_dir: str
:return: The absolute path to treefile on disk, or None if not found
:rtype: str or None
"""
for filename in constants.TREE_INFO_LIST:
path = os.path.join(tmp_dir, filename)
url = os.path.join(self.feed, filename)
request = DownloadRequest(url, path)
listener = AggregatingEventListener()
downloader = nectar_factory.create_downloader(self.feed, self.nectar_config, listener)
downloader.download([request])
if len(listener.succeeded_reports) == 1:
# bz 1095829
self.strip_treeinfo_repomd(path)
return path
def get_treefile(self, tmp_dir):
"""
Download the treefile and return its full path on disk, or None if not found
:param tmp_dir: The absolute path to the temporary directory
:type tmp_dir: str
:return: The absolute path to treefile on disk, or None if not found
:rtype: str or None
"""
for filename in constants.TREE_INFO_LIST:
path = os.path.join(tmp_dir, filename)
url = os.path.join(self.feed, filename)
request = DownloadRequest(url, path)
listener = AggregatingEventListener()
downloader = nectar_factory.create_downloader(self.feed, self.nectar_config, listener)
downloader.download([request])
if len(listener.succeeded_reports) == 1:
# bz 1095829
self.strip_treeinfo_repomd(path)
return path
def __init__(self, repo_url, dst_dir, nectar_config):
"""
:param repo_url: URL for the base of a yum repository
:type repo_url: basestring
:param dst_dir: full path to a destination to which files
should be downloaded
:type dst_dir: basestring
:param nectar_config: download config for nectar
:type nectar_config: nectar.config.DownloaderConfig
"""
super(MetadataFiles, self).__init__()
self.repo_url = repo_url
self.dst_dir = dst_dir
self.event_listener = AggregatingEventListener()
self.downloader = nectar_factory.create_downloader(
repo_url, nectar_config, self.event_listener)
self.revision = None
self.metadata = {}
self.dbs = {}
def test_https_url(self):
downloader = nectar_factory.create_downloader('https://foo',
self.mock_config,
self.mock_event_listener)
self.assertTrue(isinstance(downloader, HTTPThreadedDownloader))
def test_file_url(self):
downloader = nectar_factory.create_downloader('file:///foo',
self.mock_config,
self.mock_event_listener)
self.assertTrue(isinstance(downloader, LocalFileDownloader))
def sync(sync_conduit, feed, working_dir, nectar_config, report, progress_callback):
"""
Look for a distribution in the target repo and sync it if found
:param sync_conduit: conduit provided by the platform
:type sync_conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param feed: URL of the yum repo being sync'd
:type feed: basestring
:param working_dir: full path to the directory to which files
should be downloaded
:type working_dir: basestring
:param nectar_config: download config to be used by nectar
:type nectar_config: nectar.config.DownloaderConfig
:param report: progress report object
:type report: pulp_rpm.plugins.importers.yum.report.DistributionReport
:param progress_callback: function that takes no arguments but induces
the current progress report to be sent.
"""
# this temporary dir will hopefully be moved to the unit's storage path
# if all downloads go well. If not, it will be deleted below, ensuring a
# complete cleanup
tmp_dir = tempfile.mkdtemp(dir=working_dir)
try:
treefile_path = get_treefile(feed, tmp_dir, nectar_config)
if not treefile_path:
_LOGGER.debug('no treefile found')
return
try:
model, files = parse_treefile(treefile_path)
except ValueError:
_LOGGER.error('could not parse treefile')
report['state'] = constants.STATE_FAILED
return
distribution_type_criteria = UnitAssociationCriteria(type_ids=[ids.TYPE_ID_DISTRO])
existing_units = sync_conduit.get_units(criteria=distribution_type_criteria)
# skip this whole process if the upstream treeinfo file hasn't changed
if len(existing_units) == 1 and existing_distribution_is_current(existing_units[0], model):
_LOGGER.debug('upstream distribution unchanged; skipping')
return
# Get any errors
dist_files = process_distribution(feed, tmp_dir, nectar_config, model, report)
files.extend(dist_files)
report.set_initial_values(len(files))
listener = DistroFileListener(report, progress_callback)
downloader = nectar_factory.create_downloader(feed, nectar_config, listener)
_LOGGER.debug('downloading distribution files')
downloader.download(file_to_download_request(f, feed, tmp_dir) for f in files)
if len(listener.failed_reports) == 0:
unit = sync_conduit.init_unit(ids.TYPE_ID_DISTRO, model.unit_key, model.metadata,
model.relative_path)
model.process_download_reports(listener.succeeded_reports)
# remove pre-existing dir
shutil.rmtree(unit.storage_path, ignore_errors=True)
pulp_copytree(tmp_dir, unit.storage_path)
# mkdtemp is very paranoid, so we'll change to more sensible perms
os.chmod(unit.storage_path, 0o775)
sync_conduit.save_unit(unit)
# find any old distribution units and remove them. See BZ #1150714
for existing_unit in existing_units:
if existing_unit != unit:
_LOGGER.info("Removing out-of-date distribution unit %s for repo %s" %
(existing_unit.unit_key, sync_conduit.repo_id))
sync_conduit.remove_unit(existing_unit)
else:
_LOGGER.error('some distro file downloads failed')
report['state'] = constants.STATE_FAILED
report['error_details'] = [(fail.url, fail.error_report) for fail in
listener.failed_reports]
return
finally:
shutil.rmtree(tmp_dir, ignore_errors=True)
def sync(repo, sync_conduit, feed, working_dir, nectar_config, report, progress_callback):
"""
Look for a distribution in the target repo and sync it if found
:param repo: The repository that is the target of the sync
:type repo: pulp.server.db.model.Repository
:param sync_conduit: conduit provided by the platform
:type sync_conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param feed: URL of the yum repo being sync'd
:type feed: basestring
:param working_dir: full path to the directory to which files
should be downloaded
:type working_dir: basestring
:param nectar_config: download config to be used by nectar
:type nectar_config: nectar.config.DownloaderConfig
:param report: progress report object
:type report: pulp_rpm.plugins.importers.yum.report.DistributionReport
:param progress_callback: function that takes no arguments but induces
the current progress report to be sent.
"""
tmp_dir = tempfile.mkdtemp(dir=working_dir)
try:
treefile_path = get_treefile(feed, tmp_dir, nectar_config)
if not treefile_path:
_LOGGER.debug('no treefile found')
return
try:
model, files = parse_treefile(treefile_path)
except ValueError:
_LOGGER.error('could not parse treefile')
report['state'] = constants.STATE_FAILED
return
existing_units = repo_controller.find_repo_content_units(
repo, repo_content_unit_q=mongoengine.Q(unit_type_id=ids.TYPE_ID_DISTRO),
yield_content_unit=True)
existing_units = list(existing_units)
# skip this whole process if the upstream treeinfo file hasn't changed
if len(existing_units) == 1 and existing_distribution_is_current(existing_units[0], model):
_LOGGER.debug('upstream distribution unchanged; skipping')
return
# Get any errors
dist_files = process_distribution(feed, tmp_dir, nectar_config, model, report)
files.extend(dist_files)
report.set_initial_values(len(files))
listener = DistroFileListener(report, progress_callback)
downloader = nectar_factory.create_downloader(feed, nectar_config, listener)
_LOGGER.debug('downloading distribution files')
downloader.download(file_to_download_request(f, feed, tmp_dir) for f in files)
if len(listener.failed_reports) == 0:
model.set_content(tmp_dir)
model.save()
# The save sets the content path, which is needed to generate the download_reports
# Long term this should be done by a serializer
model.process_download_reports(listener.succeeded_reports)
model.save()
repo_controller.associate_single_unit(repo, model)
# find any old distribution units and remove them. See BZ #1150714
for existing_unit in existing_units:
if existing_unit != model:
_LOGGER.info("Removing out-of-date distribution unit %s for repo %s" %
(existing_unit.unit_key, sync_conduit.repo_id))
platform_models.RepositoryContentUnit.objects(
repo_id=sync_conduit.repo_id,
unit_type_id=models.Distribution._content_type_id.default,
unit_id=existing_unit.id).delete()
else:
_LOGGER.error('some distro file downloads failed')
report['state'] = constants.STATE_FAILED
report['error_details'] = [(fail.url, fail.error_report) for fail in
listener.failed_reports]
return
finally:
shutil.rmtree(tmp_dir, ignore_errors=True)
def sync(sync_conduit, feed, working_dir, nectar_config, report,
progress_callback):
"""
Look for a distribution in the target repo and sync it if found
:param sync_conduit: conduit provided by the platform
:type sync_conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param feed: URL of the yum repo being sync'd
:type feed: basestring
:param working_dir: full path to the directory to which files
should be downloaded
:type working_dir: basestring
:param nectar_config: download config to be used by nectar
:type nectar_config: nectar.config.DownloaderConfig
:param report: progress report object
:type report: pulp_rpm.plugins.importers.yum.report.DistributionReport
:param progress_callback: function that takes no arguments but induces
the current progress report to be sent.
"""
# this temporary dir will hopefully be moved to the unit's storage path
# if all downloads go well. If not, it will be deleted below, ensuring a
# complete cleanup
tmp_dir = tempfile.mkdtemp(dir=working_dir)
try:
treefile_path = get_treefile(feed, tmp_dir, nectar_config)
if not treefile_path:
_LOGGER.debug('no treefile found')
return
try:
model, files = parse_treefile(treefile_path)
except ValueError:
_LOGGER.error('could not parse treefile')
report['state'] = constants.STATE_FAILED
return
distribution_type_criteria = UnitAssociationCriteria(
type_ids=[ids.TYPE_ID_DISTRO])
existing_units = sync_conduit.get_units(
criteria=distribution_type_criteria)
# skip this whole process if the upstream treeinfo file hasn't changed
if len(existing_units) == 1 and existing_distribution_is_current(
existing_units[0], model):
_LOGGER.debug('upstream distribution unchanged; skipping')
return
# Get any errors
dist_files = process_distribution(feed, tmp_dir, nectar_config, model,
report)
files.extend(dist_files)
report.set_initial_values(len(files))
listener = DistroFileListener(report, progress_callback)
downloader = nectar_factory.create_downloader(feed, nectar_config,
listener)
_LOGGER.debug('downloading distribution files')
downloader.download(
file_to_download_request(f, feed, tmp_dir) for f in files)
if len(listener.failed_reports) == 0:
unit = sync_conduit.init_unit(ids.TYPE_ID_DISTRO, model.unit_key,
model.metadata, model.relative_path)
model.process_download_reports(listener.succeeded_reports)
# remove pre-existing dir
shutil.rmtree(unit.storage_path, ignore_errors=True)
pulp_copytree(tmp_dir, unit.storage_path)
# mkdtemp is very paranoid, so we'll change to more sensible perms
os.chmod(unit.storage_path, 0o775)
sync_conduit.save_unit(unit)
# find any old distribution units and remove them. See BZ #1150714
for existing_unit in existing_units:
if existing_unit != unit:
_LOGGER.info(
"Removing out-of-date distribution unit %s for repo %s"
% (existing_unit.unit_key, sync_conduit.repo_id))
sync_conduit.remove_unit(existing_unit)
else:
_LOGGER.error('some distro file downloads failed')
report['state'] = constants.STATE_FAILED
report['error_details'] = [(fail.url, fail.error_report)
for fail in listener.failed_reports]
return
finally:
shutil.rmtree(tmp_dir, ignore_errors=True)
def test_file_url(self):
downloader = nectar_factory.create_downloader('file:///foo', self.mock_config,
self.mock_event_listener)
self.assertTrue(isinstance(downloader, HTTPCurlDownloader))
def test_https_url(self):
downloader = nectar_factory.create_downloader('https://foo', self.mock_config,
self.mock_event_listener)
self.assertTrue(isinstance(downloader, HTTPThreadedDownloader))
