def _decide_drpms_to_download(self, metadata_files):
"""
Decide which DRPMs should be downloaded based on the repo metadata and on
the importer config.
:param metadata_files: instance of MetadataFiles
:type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles
:return: tuple of (set(DRPM.NAMEDTUPLEs), number of DRPMs, total size in bytes)
:rtype: tuple
"""
if models.DRPM.TYPE in self.call_config.get(constants.CONFIG_SKIP, []):
_LOGGER.debug('skipping DRPM sync')
return set(), 0, 0
presto_file_handle = metadata_files.get_metadata_file_handle(presto.METADATA_FILE_NAME)
if presto_file_handle:
try:
package_info_generator = packages.package_list_generator(presto_file_handle,
presto.PACKAGE_TAG,
presto.process_package_element)
wanted = self._identify_wanted_versions(package_info_generator)
to_download = existing.check_repo(wanted.iterkeys(), self.sync_conduit.get_units)
count = len(to_download)
size = 0
for unit in to_download:
size += wanted[unit]
finally:
presto_file_handle.close()
else:
to_download = set()
count = 0
size = 0
return to_download, count, size
def _decide_rpms_to_download(self, metadata_files):
"""
Decide which RPMs should be downloaded based on the repo metadata and on
the importer config.
:param metadata_files: instance of MetadataFiles
:type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles
:return: tuple of (set(RPM.NAMEDTUPLEs), number of RPMs, total size in bytes)
:rtype: tuple
"""
if models.RPM.TYPE in self.call_config.get(constants.CONFIG_SKIP, []):
_logger.debug('skipping RPM sync')
return set(), 0, 0
primary_file_handle = metadata_files.get_metadata_file_handle(primary.METADATA_FILE_NAME)
try:
# scan through all the metadata to decide which packages to download
package_info_generator = packages.package_list_generator(primary_file_handle,
primary.PACKAGE_TAG,
primary.process_package_element)
wanted = self._identify_wanted_versions(package_info_generator)
# check for the units that are already in the repo
not_found_in_the_repo = existing.check_repo(wanted.iterkeys(),
self.sync_conduit.get_units)
# check for the units that are not in the repo, but exist on the server
# and associate them to the repo
to_download = existing.check_all_and_associate(not_found_in_the_repo,
self.sync_conduit)
count = len(to_download)
size = 0
for unit in to_download:
size += wanted[unit]
return to_download, count, size
finally:
primary_file_handle.close()
def save_fileless_units(self, file_handle, tag, process_func, mutable_type=False,
additive_type=False):
"""
Generic method for saving units parsed from a repo metadata file where
the units do not have files to store on disk. For example, groups.
:param file_handle: open file-like object containing metadata
:type file_handle: file
:param tag: XML tag that identifies each unit
:type tag: basestring
:param process_func: function that processes each unit and returns
a dict representing that unit's attribute names
and values. The function must take one parameter,
which is an ElementTree instance
:type process_func: function
:param mutable_type: iff True, each unit will be saved regardless of
whether it already exists in the repo. this is
useful for units like group and category which
don't have a version, but could change
:type mutable_type: bool
:param additive_type: iff True, units will be updated instead of
replaced. For example, if you wanted to save an
errata and concatenate its package list with an
existing errata, you'd set this. Note that mutable_type
and additive_type are mutually exclusive.
:type additive_type: bool
"""
if mutable_type and additive_type:
raise PulpCodedException(message="The mutable_type and additive_type arguments for "
"this method are mutually exclusive.")
# iterate through the file and determine what we want to have
package_info_generator = packages.package_list_generator(file_handle,
tag,
process_func)
# if units aren't mutable, we don't need to attempt saving units that
# we already have
if not mutable_type and not additive_type:
wanted = (model.as_named_tuple for model in package_info_generator)
# given what we want, filter out what we already have
to_save = existing.check_repo(wanted, self.sync_conduit.get_units)
# rewind, iterate again through the file, and save what we need
file_handle.seek(0)
all_packages = packages.package_list_generator(file_handle,
tag,
process_func)
package_info_generator = (model for model in all_packages if
model.as_named_tuple in to_save)
for model in package_info_generator:
unit = self.sync_conduit.init_unit(model.TYPE, model.unit_key, model.metadata, None)
if additive_type:
existing_unit = self.sync_conduit.find_unit_by_unit_key(model.TYPE, model.unit_key)
if existing_unit:
unit = self._concatenate_units(existing_unit, unit)
self.sync_conduit.save_unit(unit)
def _decide_drpms_to_download(self, metadata_files):
"""
Decide which DRPMs should be downloaded based on the repo metadata and on
the importer config.
:param metadata_files: instance of MetadataFiles
:type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles
:return: tuple of (set(DRPM.NAMEDTUPLEs), number of DRPMs, total size in bytes)
:rtype: tuple
"""
if models.DRPM.TYPE in self.call_config.get(constants.CONFIG_SKIP, []):
_logger.debug('skipping DRPM sync')
return set(), 0, 0
to_download = set()
count = 0
size = 0
# multiple options for deltainfo files depending on the distribution
# so we have to go through all of them
for metadata_file_name in presto.METADATA_FILE_NAMES:
presto_file_handle = metadata_files.get_metadata_file_handle(
metadata_file_name)
if presto_file_handle:
try:
package_info_generator = packages.package_list_generator(
presto_file_handle, presto.PACKAGE_TAG,
presto.process_package_element)
wanted = self._identify_wanted_versions(
package_info_generator)
# check for the units that are already in the repo
not_found_in_the_repo = existing.check_repo(
wanted.iterkeys(), self.sync_conduit.get_units)
# check for the units that are not in the repo, but exist on the server
# and associate them to the repo
to_download = existing.check_all_and_associate(
not_found_in_the_repo, self.sync_conduit)
count += len(to_download)
for unit in to_download:
size += wanted[unit]
finally:
presto_file_handle.close()
return to_download, count, size
def _decide_drpms_to_download(self, metadata_files):
"""
Decide which DRPMs should be downloaded based on the repo metadata and on
the importer config.
:param metadata_files: instance of MetadataFiles
:type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles
:return: tuple of (set(DRPM.NAMEDTUPLEs), number of DRPMs, total size in bytes)
:rtype: tuple
"""
if models.DRPM.TYPE in self.call_config.get(constants.CONFIG_SKIP, []):
_logger.debug('skipping DRPM sync')
return set(), 0, 0
to_download = set()
count = 0
size = 0
# multiple options for deltainfo files depending on the distribution
# so we have to go through all of them
for metadata_file_name in presto.METADATA_FILE_NAMES:
presto_file_handle = metadata_files.get_metadata_file_handle(metadata_file_name)
if presto_file_handle:
try:
package_info_generator = packages.package_list_generator(
presto_file_handle,
presto.PACKAGE_TAG,
presto.process_package_element)
wanted = self._identify_wanted_versions(package_info_generator)
# check for the units that are already in the repo
not_found_in_the_repo = existing.check_repo(wanted.iterkeys(),
self.sync_conduit.get_units)
# check for the units that are not in the repo, but exist on the server
# and associate them to the repo
to_download = existing.check_all_and_associate(not_found_in_the_repo,
self.sync_conduit)
count += len(to_download)
for unit in to_download:
size += wanted[unit]
finally:
presto_file_handle.close()
return to_download, count, size
def _decide_rpms_to_download(self, metadata_files):
"""
Decide which RPMs should be downloaded based on the repo metadata and on
the importer config.
:param metadata_files: instance of MetadataFiles
:type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles
:return: tuple of (set(RPM.NAMEDTUPLEs), number of RPMs, total size in bytes)
:rtype: tuple
"""
if models.RPM.TYPE in self.call_config.get(constants.CONFIG_SKIP, []):
_logger.debug('skipping RPM sync')
return set(), 0, 0
primary_file_handle = metadata_files.get_metadata_file_handle(
primary.METADATA_FILE_NAME)
try:
# scan through all the metadata to decide which packages to download
package_info_generator = packages.package_list_generator(
primary_file_handle, primary.PACKAGE_TAG,
primary.process_package_element)
wanted = self._identify_wanted_versions(package_info_generator)
# check for the units that are already in the repo
not_found_in_the_repo = existing.check_repo(
wanted.iterkeys(), self.sync_conduit.get_units)
# check for the units that are not in the repo, but exist on the server
# and associate them to the repo
to_download = existing.check_all_and_associate(
not_found_in_the_repo, self.sync_conduit)
count = len(to_download)
size = 0
for unit in to_download:
size += wanted[unit]
return to_download, count, size
finally:
primary_file_handle.close()
def save_fileless_units(self, file_handle, tag, process_func, mutable_type=False,
additive_type=False):
"""
Generic method for saving units parsed from a repo metadata file where
the units do not have files to store on disk. For example, groups.
:param file_handle: open file-like object containing metadata
:type file_handle: file
:param tag: XML tag that identifies each unit
:type tag: basestring
:param process_func: function that processes each unit and returns
a dict representing that unit's attribute names
and values. The function must take one parameter,
which is an ElementTree instance
:type process_func: function
:param mutable_type: iff True, each unit will be saved regardless of
whether it already exists in the repo. this is
useful for units like group and category which
don't have a version, but could change
:type mutable_type: bool
:param additive_type: iff True, units will be updated instead of
replaced. For example, if you wanted to save an
errata and concatenate its package list with an
existing errata, you'd set this. Note that mutable_type
and additive_type are mutually exclusive.
:type additive_type: bool
"""
if mutable_type and additive_type:
raise PulpCodedException(message="The mutable_type and additive_type arguments for "
"this method are mutually exclusive.")
# iterate through the file and determine what we want to have
package_info_generator = packages.package_list_generator(file_handle,
tag,
process_func)
# if units aren't mutable, we don't need to attempt saving units that
# we already have
if not mutable_type and not additive_type:
wanted = (model.unit_key_as_named_tuple for model in package_info_generator)
# given what we want, filter out what we already have
to_save = existing.check_repo(wanted)
# rewind, iterate again through the file, and save what we need
file_handle.seek(0)
all_packages = packages.package_list_generator(file_handle,
tag,
process_func)
package_info_generator = \
(model for model in all_packages if model.unit_key_as_named_tuple in to_save)
for model in package_info_generator:
# Add repo_id to each collection of the pkglist of the new erratum
if isinstance(model, models.Errata):
for collection in model.pkglist:
collection['_pulp_repo_id'] = self.repo.repo_id
existing_unit = model.__class__.objects.filter(**model.unit_key).first()
if not existing_unit:
model.save()
else:
if additive_type:
model = self._concatenate_units(existing_unit, model)
model.save()
else:
# make sure the associate_unit call gets the existing unit
model = existing_unit
repo_controller.associate_single_unit(self.repo, model)
def save_fileless_units(self,
file_handle,
tag,
process_func,
mutable_type=False,
additive_type=False):
"""
Generic method for saving units parsed from a repo metadata file where
the units do not have files to store on disk. For example, groups.
:param file_handle: open file-like object containing metadata
:type file_handle: file
:param tag: XML tag that identifies each unit
:type tag: basestring
:param process_func: function that processes each unit and returns
a dict representing that unit's attribute names
and values. The function must take one parameter,
which is an ElementTree instance
:type process_func: function
:param mutable_type: iff True, each unit will be saved regardless of
whether it already exists in the repo. this is
useful for units like group and category which
don't have a version, but could change
:type mutable_type: bool
:param additive_type: iff True, units will be updated instead of
replaced. For example, if you wanted to save an
errata and concatenate its package list with an
existing errata, you'd set this. Note that mutable_type
and additive_type are mutually exclusive.
:type additive_type: bool
"""
if mutable_type and additive_type:
raise PulpCodedException(
message="The mutable_type and additive_type arguments for "
"this method are mutually exclusive.")
# iterate through the file and determine what we want to have
package_info_generator = packages.package_list_generator(
file_handle, tag, process_func)
# if units aren't mutable, we don't need to attempt saving units that
# we already have
if not mutable_type and not additive_type:
wanted = (model.unit_key_as_named_tuple
for model in package_info_generator)
# given what we want, filter out what we already have
to_save = existing.check_repo(wanted)
# rewind, iterate again through the file, and save what we need
file_handle.seek(0)
all_packages = packages.package_list_generator(
file_handle, tag, process_func)
package_info_generator = \
(model for model in all_packages if model.unit_key_as_named_tuple in to_save)
for model in package_info_generator:
existing_unit = model.__class__.objects.filter(
**model.unit_key).first()
if not existing_unit:
model.save()
else:
if additive_type:
model = self._concatenate_units(existing_unit, model)
model.save()
else:
# make sure the associate_unit call gets the existing unit
model = existing_unit
repo_controller.associate_single_unit(self.repo, model)
def save_fileless_units(self, file_handle, tag, process_func, mutable_type=False,
additive_type=False):
"""
Generic method for saving units parsed from a repo metadata file where
the units do not have files to store on disk. For example, groups.
:param file_handle: open file-like object containing metadata
:type file_handle: file
:param tag: XML tag that identifies each unit
:type tag: basestring
:param process_func: function that processes each unit and returns
a dict representing that unit's attribute names
and values. The function must take one parameter,
which is an ElementTree instance
:type process_func: function
:param mutable_type: iff True, each unit will be saved regardless of
whether it already exists in the repo. this is
useful for units like group and category which
don't have a version, but could change
:type mutable_type: bool
:param additive_type: iff True, units will be updated instead of
replaced. For example, if you wanted to save an
errata and concatenate its package list with an
existing errata, you'd set this. Note that mutable_type
and additive_type are mutually exclusive.
:type additive_type: bool
"""
if mutable_type and additive_type:
raise PulpCodedException(message="The mutable_type and additive_type arguments for "
"this method are mutually exclusive.")
# iterate through the file and determine what we want to have
package_info_generator = packages.package_list_generator(file_handle,
tag,
process_func)
# if units aren't mutable, we don't need to attempt saving units that
# we already have
if not mutable_type and not additive_type:
wanted = (model.unit_key_as_named_tuple for model in package_info_generator)
# given what we want, filter out what we already have
to_save = existing.check_repo(wanted)
# rewind, iterate again through the file, and save what we need
file_handle.seek(0)
all_packages = packages.package_list_generator(file_handle,
tag,
process_func)
package_info_generator = \
(model for model in all_packages if model.unit_key_as_named_tuple in to_save)
errata_could_not_be_merged_count = 0
for model in package_info_generator:
# Add repo_id to each collection of the pkglist of the new erratum
if isinstance(model, models.Errata):
for collection in model.pkglist:
collection['_pulp_repo_id'] = self.repo.repo_id
existing_unit = model.__class__.objects.filter(**model.unit_key).first()
if not existing_unit:
model.save()
else:
if additive_type:
try:
model = self._concatenate_units(existing_unit, model)
except ValueError:
# Sometimes Errata units cannot be merged and a ValueError is raised
# Count the errors and log them further down
if isinstance(model, models.Errata):
msg = _('The Errata "%(errata_id)s" could not be merged from the '
'remote repository. This is likely due to the existing or new '
'Errata not containing a valid `updated` field.')
_logger.debug(msg % {'errata_id': model.errata_id})
errata_could_not_be_merged_count += 1
continue
else:
raise
model.save()
else:
# make sure the associate_unit call gets the existing unit
model = existing_unit
repo_controller.associate_single_unit(self.repo, model)
if errata_could_not_be_merged_count != 0:
msg = _('There were %(count)d Errata units which could not be merged. This is likely '
'due to Errata in this repo not containing valid `updated` fields.')
_logger.warn(msg % {'count': errata_could_not_be_merged_count})
