def copy_rpms_by_name(names, import_conduit, copy_deps):
"""
Copy RPMs from source repo to destination repo by name
:param names: iterable of RPM names
:type names: iterable of basestring
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:return: set of pulp.plugins.model.Unit that were copied
:rtype: set
"""
to_copy = {}
search_dicts = ({'name': name} for name in names)
units = existing.get_existing_units(search_dicts, models.RPM.UNIT_KEY_NAMES,
models.RPM.TYPE,
import_conduit.get_source_units)
for unit in units:
model = models.RPM.from_package_info(unit.unit_key)
previous = to_copy.get(model.key_string_without_version)
if previous is None:
to_copy[model.key_string_without_version] = (model.complete_version_serialized, unit)
else:
to_copy[model.key_string_without_version] = max(((model.complete_version_serialized, unit), previous))
return copy_rpms((unit for v, unit in to_copy.itervalues()), import_conduit, copy_deps)
def get_rpms_to_copy_by_key(rpm_search_dicts, import_conduit):
"""
Errata specify NEVRA for the RPMs they reference. This method is useful for
taking those specifications and finding actual units available in the source
repository.
:param rpm_search_dicts: iterable of dicts that include a subset of rpm
unit key parameters
:type rpm_search_dicts: iterable
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:return: set of namedtuples needed by the dest repo
"""
# identify which RPMs are desired and store as named tuples
named_tuples = set()
for key in rpm_search_dicts:
# ignore checksum from updateinfo.xml
key['checksum'] = None
key['checksumtype'] = None
named_tuples.add(models.RPM.NAMEDTUPLE(**key))
# identify which of those RPMs already exist
existing_units = existing.get_existing_units(rpm_search_dicts, models.RPM.UNIT_KEY_NAMES,
models.RPM.TYPE, import_conduit.get_destination_units)
# remove units that already exist in the destination from the set of units
# we want to copy
for unit in existing_units:
unit_key = unit.unit_key.copy()
# ignore checksum from updateinfo.xml
unit_key['checksum'] = None
unit_key['checksumtype'] = None
named_tuples.discard(models.RPM.NAMEDTUPLE(**unit_key))
return named_tuples
def copy_rpms(units, import_conduit, copy_deps, solver=None):
"""
Copy RPMs from the source repo to the destination repo, and optionally copy
dependencies as well. Dependencies are resolved recursively.
:param units: iterable of Units
:type units: iterable of pulp.plugins.models.Unit
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:param copy_deps: if True, copies dependencies as specified in "Requires"
lines in the RPM metadata. Matches against NEVRAs
and Provides declarations that are found in the
source repository. Silently skips any dependencies
that cannot be resolved within the source repo.
:param solver: an object that can be used for dependency solving.
this is useful so that data can be cached in the
depsolving object and re-used by each iteration of
this method.
:type solver: pulp_rpm.plugins.importers.yum.depsolve.Solver
:return: set of pulp.plugins.models.Unit that were copied
:rtype: set
"""
unit_set = set()
for unit in units:
# we are passing in units that may have flattened "provides" metadata.
# This flattened field is not used by associate_unit().
import_conduit.associate_unit(unit)
unit_set.add(unit)
if copy_deps and unit_set:
if solver is None:
solver = depsolve.Solver(import_conduit.get_source_units)
# This returns units that have a flattened 'provides' metadata field
# for memory purposes (RHBZ #1185868)
deps = solver.find_dependent_rpms(unit_set)
# remove rpms already in the destination repo
existing_units = set(
existing.get_existing_units(
[dep.unit_key for dep in deps],
models.RPM.UNIT_KEY_NAMES,
models.RPM.TYPE,
import_conduit.get_destination_units,
)
)
# the hash comparison for Units is unit key + type_id, the metadata
# field is not used.
to_copy = deps - existing_units
_LOGGER.debug("Copying deps: %s" % str(sorted([x.unit_key["name"] for x in to_copy])))
if to_copy:
unit_set |= copy_rpms(to_copy, import_conduit, copy_deps, solver)
return unit_set
def filter_available_rpms(rpms, import_conduit):
"""
Given a series of RPM named tuples, return an iterable of those which are
available in the source repository
:param rpms: iterable of RPMs that are desired to be copied
:type rpms: iterable of pulp_rpm.common.models.RPM.NAMEDTUPLE
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:return: iterable of Units that should be copied
:return: iterable of pulp.plugins.model.Unit
"""
return existing.get_existing_units((_no_checksum_clean_unit_key(unit) for unit in rpms),
models.RPM.UNIT_KEY_NAMES, models.RPM.TYPE,
import_conduit.get_source_units)
def filter_available_rpms(rpms, import_conduit, repo):
"""
Given a series of RPM named tuples, return an iterable of those which are
available in the source repository
:param rpms: iterable of RPMs that are desired to be copied
:type rpms: iterable of pulp_rpm.plugins.db.models.RPM.NAMEDTUPLE
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:param repo: repository in which to search for RPMs
:type repo: pulp.server.db.model.Repository
:return: iterable of Units that should be copied
:return: iterable of pulp_rpm.plugins.db.models.RPM
"""
return existing.get_existing_units(
(_no_checksum_clean_unit_key(unit) for unit in rpms), models.RPM, repo)
def filter_available_rpms(rpms, import_conduit, repo):
"""
Given a series of RPM named tuples, return an iterable of those which are
available in the source repository
:param rpms: iterable of RPMs that are desired to be copied
:type rpms: iterable of pulp_rpm.plugins.db.models.RPM.NAMEDTUPLE
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:param repo: repository in which to search for RPMs
:type repo: pulp.server.db.model.Repository
:return: iterable of Units that should be copied
:return: iterable of pulp_rpm.plugins.db.models.RPM
"""
return existing.get_existing_units((_no_checksum_clean_unit_key(unit) for unit in rpms),
models.RPM, repo)
def get_rpms_to_copy_by_name(rpm_names, import_conduit):
"""
Groups reference names of RPMs. This method is useful for taking those names
and removing ones that already exist in the destination repository.
:param rpm_names: iterable of RPM names
:type rpm_names: iterable
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:return: set of names that don't already exist in the destination repo
:rtype: set
"""
search_dicts = ({'name': name} for name in rpm_names)
units = existing.get_existing_units(search_dicts, models.RPM.UNIT_KEY_NAMES,
models.RPM.TYPE, import_conduit.get_destination_units)
names = set(rpm_names)
for unit in units:
names.discard(unit.unit_key['name'])
return names
def get_rpms_to_copy_by_name(rpm_names, import_conduit, repo):
"""
Groups reference names of RPMs. This method is useful for taking those names
and removing ones that already exist in the destination repository.
:param rpm_names: iterable of RPM names
:type rpm_names: iterable
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:param repo: repository from which to get RPMs
:type repo: pulp.server.db.model.Repository
:return: set of names that don't already exist in the destination repo
:rtype: set
"""
search_dicts = ({'name': name} for name in rpm_names)
units = existing.get_existing_units(search_dicts, models.RPM, repo)
names = set(rpm_names)
for unit in units:
names.discard(unit.name)
return names
def get_rpms_to_copy_by_name(rpm_names, import_conduit, repo):
"""
Groups reference names of RPMs. This method is useful for taking those names
and removing ones that already exist in the destination repository.
:param rpm_names: iterable of RPM names
:type rpm_names: iterable
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:param repo: repository from which to get RPMs
:type repo: pulp.server.db.model.Repository
:return: set of names that don't already exist in the destination repo
:rtype: set
"""
search_dicts = ({'name': name} for name in rpm_names)
units = existing.get_existing_units(search_dicts, models.RPM, repo)
names = set(rpm_names)
for unit in units:
names.discard(unit.name)
return names
def copy_rpms(units, import_conduit, copy_deps):
"""
Copy RPMs from the source repo to the destination repo, and optionally copy
dependencies as well. Dependencies are resolved recursively.
:param units: iterable of Units
:type units: iterable of pulp.plugins.models.Unit
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:param copy_deps: if True, copies dependencies as specified in "Requires"
lines in the RPM metadata. Matches against NEVRAs
and Provides declarations that are found in the
source repository. Silently skips any dependencies
that cannot be resolved within the source repo.
:return: set of pulp.plugins.models.Unit that were copied
:rtype: set
"""
unit_set = set()
for unit in units:
import_conduit.associate_unit(unit)
unit_set.add(unit)
if copy_deps and unit_set:
deps = depsolve.find_dependent_rpms(unit_set, import_conduit.get_source_units)
# only consider deps that exist in the source repo
available_deps = set(filter_available_rpms(deps, import_conduit))
# remove rpms already in the destination repo
existing_units = set(existing.get_existing_units([dep.unit_key for dep in available_deps],
models.RPM.UNIT_KEY_NAMES, models.RPM.TYPE,
import_conduit.get_destination_units))
to_copy = available_deps - existing_units
_LOGGER.debug('Copying deps: %s' % str(sorted([x.unit_key['name'] for x in to_copy])))
if to_copy:
unit_set |= copy_rpms(to_copy, import_conduit, copy_deps)
return unit_set
def get_rpms_to_copy_by_key(rpm_search_dicts, import_conduit, repo):
"""
Errata specify NEVRA for the RPMs they reference. This method is useful for
taking those specifications and finding actual units available in the source
repository.
:param rpm_search_dicts: iterable of dicts that include a subset of rpm
unit key parameters
:type rpm_search_dicts: iterable
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:param repo: repository from which to get RPMs
:type repo: pulp.server.db.model.Repository
:return: set of namedtuples needed by the dest repo
:rtype: set
"""
# identify which RPMs are desired and store as named tuples
named_tuples = set()
for key in rpm_search_dicts:
# ignore checksum from updateinfo.xml
key['checksum'] = None
key['checksumtype'] = None
named_tuples.add(models.RPM.NAMED_TUPLE(**key))
# identify which of those RPMs already exist
existing_units = existing.get_existing_units(rpm_search_dicts, models.RPM,
repo)
# remove units that already exist in the destination from the set of units
# we want to copy
for unit in existing_units:
unit_key = unit.unit_key.copy()
# ignore checksum from updateinfo.xml
unit_key['checksum'] = None
unit_key['checksumtype'] = None
named_tuples.discard(models.RPM.NAMED_TUPLE(**unit_key))
return named_tuples
def copy_rpms(units,
source_repo,
dest_repo,
import_conduit,
copy_deps,
solver=None):
"""
Copy RPMs from the source repo to the destination repo, and optionally copy
dependencies as well. Dependencies are resolved recursively.
:param units: iterable of Units
:type units: iterable of pulp_rpm.plugins.db.models.RPM
:param source_repo: The repository we are copying units from.
:type source_repo: pulp.server.db.model.Repository
:param dest_repo: The repository we are copying units to
:type dest_repo: pulp.server.db.model.Repository
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:param copy_deps: if True, copies dependencies as specified in "Requires"
lines in the RPM metadata. Matches against NEVRAs
and Provides declarations that are found in the
source repository. Silently skips any dependencies
that cannot be resolved within the source repo.
:param solver: an object that can be used for dependency solving.
this is useful so that data can be cached in the
depsolving object and re-used by each iteration of
this method.
:type solver: pulp_rpm.plugins.importers.yum.depsolve.Solver
:return: set of pulp.plugins.models.Unit that were copied
:rtype: set
"""
unit_set = set()
for unit in units:
# we are passing in units that may have flattened "provides" metadata.
# This flattened field is not used by associate_single_unit().
repo_controller.associate_single_unit(dest_repo, unit)
unit_set.add(unit)
if copy_deps and unit_set:
if solver is None:
solver = depsolve.Solver(source_repo)
# This returns units that have a flattened 'provides' metadata field
# for memory purposes (RHBZ #1185868)
deps = solver.find_dependent_rpms(unit_set)
# remove rpms already in the destination repo
existing_units = set(
existing.get_existing_units([dep.unit_key for dep in deps],
models.RPM, dest_repo))
# the hash comparison for Units is unit key + type_id, the metadata
# field is not used.
to_copy = deps - existing_units
_LOGGER.debug('Copying deps: %s' %
str(sorted([x.name for x in to_copy])))
if to_copy:
unit_set |= copy_rpms(to_copy, source_repo, dest_repo,
import_conduit, copy_deps, solver)
return unit_set
def copy_rpms(units, source_repo, dest_repo, import_conduit, config, copy_deps, solver=None):
"""
Copy RPMs from the source repo to the destination repo, and optionally copy
dependencies as well. Dependencies are resolved recursively.
:param units: iterable of Units
:type units: iterable of pulp_rpm.plugins.db.models.RPM
:param source_repo: The repository we are copying units from.
:type source_repo: pulp.server.db.model.Repository
:param dest_repo: The repository we are copying units to
:type dest_repo: pulp.server.db.model.Repository
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:param config: configuration instance passed to the importer of the destination repo
:type config: pulp.plugins.config.PluginCallConfiguration
:param copy_deps: if True, copies dependencies as specified in "Requires"
lines in the RPM metadata. Matches against NEVRAs
and Provides declarations that are found in the
source repository. Silently skips any dependencies
that cannot be resolved within the source repo.
:param solver: an object that can be used for dependency solving.
this is useful so that data can be cached in the
depsolving object and re-used by each iteration of
this method.
:type solver: pulp_rpm.plugins.importers.yum.depsolve.Solver
:return: set of pulp.plugins.models.Unit that were copied
:rtype: set
"""
unit_set = set()
failed_signature_check = 0
for unit in units:
# we are passing in units that may have flattened "provides" metadata.
# This flattened field is not used by associate_single_unit().
if rpm_parse.signature_enabled(config):
if unit.downloaded:
try:
rpm_parse.filter_signature(unit, config)
except PulpCodedException as e:
_LOGGER.debug(e)
failed_signature_check += 1
continue
else:
continue
repo_controller.associate_single_unit(repository=dest_repo, unit=unit)
unit_set.add(unit)
if failed_signature_check:
_LOGGER.warning(_('%s packages failed signature filter and were not imported.'
% failed_signature_check))
if copy_deps and unit_set:
if solver is None:
solver = depsolve.Solver(source_repo)
# This returns units that have a flattened 'provides' metadata field
# for memory purposes (RHBZ #1185868)
deps = solver.find_dependent_rpms(unit_set)
# remove rpms already in the destination repo
existing_units = set(existing.get_existing_units([dep.unit_key for dep in deps],
models.RPM, dest_repo))
# the hash comparison for Units is unit key + type_id, the metadata
# field is not used.
to_copy = deps - existing_units
_LOGGER.debug('Copying deps: %s' % str(sorted([x.name for x in to_copy])))
if to_copy:
unit_set |= copy_rpms(to_copy, source_repo, dest_repo, import_conduit, config,
copy_deps, solver)
return unit_set