def _decide_rpms_to_download(self, metadata_files):
"""
Decide which RPMs should be downloaded based on the repo metadata and on
the importer config.
:param metadata_files: instance of MetadataFiles
:type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles
:return: tuple of (set(RPM.NAMEDTUPLEs), number of RPMs, total size in bytes)
:rtype: tuple
"""
if ids.TYPE_ID_RPM in self.config.get(constants.CONFIG_SKIP, []):
_logger.debug('skipping RPM sync')
return set(), 0, 0
primary_file_handle = metadata_files.get_metadata_file_handle(
primary.METADATA_FILE_NAME)
try:
# scan through all the metadata to decide which packages to download
package_info_generator = packages.package_list_generator(
primary_file_handle, primary.PACKAGE_TAG,
primary.process_package_element)
wanted = self._identify_wanted_versions(package_info_generator)
# check for the units that are not in the repo, but exist on the server
# and associate them to the repo
to_download = existing.check_all_and_associate(
wanted.iterkeys(), self.conduit, self.download_deferred)
count = len(to_download)
size = 0
for unit in to_download:
size += wanted[unit]
return to_download, count, size
finally:
primary_file_handle.close()
def _decide_rpms_to_download(self, metadata_files):
"""
Decide which RPMs should be downloaded based on the repo metadata and on
the importer config.
:param metadata_files: instance of MetadataFiles
:type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles
:return: tuple of (set(RPM.NAMEDTUPLEs), number of RPMs, total size in bytes)
:rtype: tuple
"""
if ids.TYPE_ID_RPM in self.config.get(constants.CONFIG_SKIP, []):
_logger.debug('skipping RPM sync')
return set(), 0, 0
primary_file_handle = metadata_files.get_metadata_file_handle(primary.METADATA_FILE_NAME)
try:
# scan through all the metadata to decide which packages to download
package_info_generator = packages.package_list_generator(
primary_file_handle, primary.PACKAGE_TAG, primary.process_package_element)
wanted = self._identify_wanted_versions(package_info_generator)
# check for the units that are not in the repo, but exist on the server
# and associate them to the repo
to_download = existing.check_all_and_associate(
wanted.iterkeys(), self.conduit, self.download_deferred)
count = len(to_download)
size = 0
for unit in to_download:
size += wanted[unit]
return to_download, count, size
finally:
primary_file_handle.close()
def test_drpms_check_all_and_associate_negative(self, mock_isfile, mock_save, mock_search_all_units):
mock_search_all_units.return_value = []
mock_isfile.return_value = True
units = model_factory.drpm_models(3)
input_units = set([unit.as_named_tuple for unit in units])
result = check_all_and_associate(input_units, self.conduit)
self.assertEqual(len(list(result)), 3)
def test_drpms_check_all_and_associate_positive(self, mock_isfile, mock_save, mock_search_all_units):
units = model_factory.drpm_models(3)
mock_search_all_units.return_value = units
mock_isfile.return_value = True
input_units = set([unit.as_named_tuple for unit in units])
for unit in units:
unit.metadata['filename'] = 'test-filename'
unit.storage_path = "existing_storage_path"
result = check_all_and_associate(input_units, self.conduit)
self.assertEqual(len(list(result)), 0)
def _decide_drpms_to_download(self, metadata_files):
"""
Decide which DRPMs should be downloaded based on the repo metadata and on
the importer config.
:param metadata_files: instance of MetadataFiles
:type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles
:return: tuple of (set(DRPM.NAMEDTUPLEs), number of DRPMs, total size in bytes)
:rtype: tuple
"""
if models.DRPM.TYPE in self.call_config.get(constants.CONFIG_SKIP, []):
_logger.debug('skipping DRPM sync')
return set(), 0, 0
to_download = set()
count = 0
size = 0
# multiple options for deltainfo files depending on the distribution
# so we have to go through all of them
for metadata_file_name in presto.METADATA_FILE_NAMES:
presto_file_handle = metadata_files.get_metadata_file_handle(
metadata_file_name)
if presto_file_handle:
try:
package_info_generator = packages.package_list_generator(
presto_file_handle, presto.PACKAGE_TAG,
presto.process_package_element)
wanted = self._identify_wanted_versions(
package_info_generator)
# check for the units that are already in the repo
not_found_in_the_repo = existing.check_repo(
wanted.iterkeys(), self.sync_conduit.get_units)
# check for the units that are not in the repo, but exist on the server
# and associate them to the repo
to_download = existing.check_all_and_associate(
not_found_in_the_repo, self.sync_conduit)
count += len(to_download)
for unit in to_download:
size += wanted[unit]
finally:
presto_file_handle.close()
return to_download, count, size
def _decide_drpms_to_download(self, metadata_files):
"""
Decide which DRPMs should be downloaded based on the repo metadata and on
the importer config.
:param metadata_files: instance of MetadataFiles
:type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles
:return: tuple of (set(DRPM.NAMEDTUPLEs), number of DRPMs, total size in bytes)
:rtype: tuple
"""
if models.DRPM.TYPE in self.call_config.get(constants.CONFIG_SKIP, []):
_logger.debug('skipping DRPM sync')
return set(), 0, 0
to_download = set()
count = 0
size = 0
# multiple options for deltainfo files depending on the distribution
# so we have to go through all of them
for metadata_file_name in presto.METADATA_FILE_NAMES:
presto_file_handle = metadata_files.get_metadata_file_handle(metadata_file_name)
if presto_file_handle:
try:
package_info_generator = packages.package_list_generator(
presto_file_handle,
presto.PACKAGE_TAG,
presto.process_package_element)
wanted = self._identify_wanted_versions(package_info_generator)
# check for the units that are already in the repo
not_found_in_the_repo = existing.check_repo(wanted.iterkeys(),
self.sync_conduit.get_units)
# check for the units that are not in the repo, but exist on the server
# and associate them to the repo
to_download = existing.check_all_and_associate(not_found_in_the_repo,
self.sync_conduit)
count += len(to_download)
for unit in to_download:
size += wanted[unit]
finally:
presto_file_handle.close()
return to_download, count, size
def _decide_rpms_to_download(self, metadata_files, catalog):
"""
Decide which RPMs should be downloaded based on the repo metadata and on
the importer config.
:param metadata_files: instance of MetadataFiles
:type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles
:param catalog: Deferred downloading catalog.
:type catalog: PackageCatalog
:return: tuple of (set(RPM.NAMEDTUPLEs), number of RPMs, total size in bytes)
:rtype: tuple
:raises PulpCodedException: if there is some inconsistency in metadata
"""
if ids.TYPE_ID_RPM in self.config.get(constants.CONFIG_SKIP, []):
_logger.debug('skipping RPM sync')
return set(), 0, 0
primary_file_handle = metadata_files.get_metadata_file_handle(primary.METADATA_FILE_NAME)
try:
# scan through all the metadata to decide which packages to download
package_info_generator = packages.package_list_generator(
primary_file_handle, primary.PACKAGE_TAG, primary.process_package_element)
# count packages while iterating over primary metadata and deciding on wanted packages
counter = itertools.count()
wrapped_generator = itertools.imap(lambda x, y: x, package_info_generator, counter)
wanted = self._identify_wanted_versions(wrapped_generator)
primary_rpm_count = counter.next()
if primary_rpm_count != metadata_files.rpm_count:
reason = 'metadata is missing for some packages in filelists.xml and in other.xml'
raise PulpCodedException(error_code=error_codes.RPM1015, reason=reason)
# check for the units that are not in the repo, but exist on the server
# and associate them to the repo
to_download = existing.check_all_and_associate(
wanted, self.conduit, self.config, self.download_deferred, catalog)
count = len(to_download)
size = 0
for unit in to_download:
size += wanted[unit].size
return to_download, count, size
finally:
primary_file_handle.close()