def resolve_advisories(version, previous_version):
"""
Decide which advisories to add to a repo version and which to remove, and adjust a repo version.
Advisory can be in 3 different states with relation to a repository version:
- in-memory and added before this function call, so it's a part of the current incomplete
repository version only
- in the db, it's been added in some previous repository version
- has no relation to any repository version because it's been created in this function as an
outcome of conflict resolution.
All 3 states need to be handled differently.
The in-db ones and newly created are straightforward, just remove/add in a standard way.
To remove in-memory ones (`content_pks_to_exclude`) from an incomplete repo version,
one needs to do it directly from RepositoryContent. They've never been a part of a repo
version, they are also not among the `content_pks_to_add` or `content_pks_to_remove` ones.
Args:
version (pulpcore.app.models.RepositoryVersion): current incomplete repository version
previous_version (pulpcore.app.models.RepositoryVersion): a version preceding
the current incomplete one
"""
content_pks_to_add = set()
content_pks_to_remove = set()
content_pks_to_exclude = set(
) # exclude from the set of content which is being added
# identify conflicting advisories
advisory_pulp_type = UpdateRecord.get_pulp_type()
current_advisories = UpdateRecord.objects.filter(
pk__in=version.content.filter(pulp_type=advisory_pulp_type))
added_advisories = current_advisories
advisory_conflicts = []
# check for any conflict
current_ids = [adv.id for adv in current_advisories]
if previous_version and len(current_ids) != len(set(current_ids)):
previous_advisories = UpdateRecord.objects.filter(
pk__in=previous_version.content.filter(
pulp_type=advisory_pulp_type))
previous_advisory_ids = set(
previous_advisories.values_list('id', flat=True))
# diff for querysets works fine but the result is not fully functional queryset,
# e.g. filtering doesn't work
added_advisories = current_advisories.difference(previous_advisories)
if len(list(added_advisories)) != len(set(added_advisories)):
raise AdvisoryConflict(
_('It is not possible to add two advisories of the same id to '
'a repository version.'))
added_advisory_ids = set(adv.id for adv in added_advisories)
advisory_conflicts = added_advisory_ids.intersection(
previous_advisory_ids)
added_advisory_pks = [adv.pk for adv in added_advisories]
for advisory_id in advisory_conflicts:
previous_advisory = previous_advisories.get(id=advisory_id)
added_advisory = UpdateRecord.objects.get(
id=advisory_id, pk__in=added_advisory_pks)
to_add, to_remove, to_exclude = resolve_advisory_conflict(
previous_advisory, added_advisory)
content_pks_to_add.update(to_add)
content_pks_to_remove.update(to_remove)
content_pks_to_exclude.update(to_exclude)
if content_pks_to_add:
version.add_content(Content.objects.filter(pk__in=content_pks_to_add))
if content_pks_to_remove:
version.remove_content(
Content.objects.filter(pk__in=content_pks_to_remove))
if content_pks_to_exclude:
RepositoryContent.objects.filter(repository=version.repository,
content_id__in=content_pks_to_exclude,
version_added=version).delete()
def find_children_of_content(content, src_repo_version):
"""Finds the content referenced directly by other content and returns it all together.
Finds RPMs referenced by Advisory/Errata content.
Args:
content (iterable): Content for which to resolve children
src_repo_version (pulpcore.models.RepositoryVersion): Source repo version
Returns: Queryset of Content objects that are children of the intial set of content
"""
# Content that were selected to be copied
advisory_ids = content.filter(
pulp_type=UpdateRecord.get_pulp_type()).only('pk')
packagecategory_ids = content.filter(
pulp_type=PackageCategory.get_pulp_type()).only('pk')
packageenvironment_ids = content.filter(
pulp_type=PackageEnvironment.get_pulp_type()).only('pk')
packagegroup_ids = content.filter(
pulp_type=PackageGroup.get_pulp_type()).only('pk')
# Content in the source repository version
package_ids = src_repo_version.content.filter(
pulp_type=Package.get_pulp_type()).only('pk')
module_ids = src_repo_version.content.filter(
pulp_type=Modulemd.get_pulp_type()).only('pk')
advisories = UpdateRecord.objects.filter(pk__in=advisory_ids)
packages = Package.objects.filter(pk__in=package_ids)
packagecategories = PackageCategory.objects.filter(
pk__in=packagecategory_ids)
packageenvironments = PackageEnvironment.objects.filter(
pk__in=packageenvironment_ids)
packagegroups = PackageGroup.objects.filter(pk__in=packagegroup_ids)
modules = Modulemd.objects.filter(pk__in=module_ids)
children = set()
for advisory in advisories:
# Find rpms referenced by Advisories/Errata
package_nevras = advisory.get_pkglist()
for nevra in package_nevras:
(name, epoch, version, release, arch) = nevra
try:
package = packages.get(name=name,
epoch=epoch,
version=version,
release=release,
arch=arch)
children.add(package.pk)
except Package.DoesNotExist:
raise
except MultipleObjectsReturned:
raise
module_nsvcas = advisory.get_module_list()
for nsvca in module_nsvcas:
(name, stream, version, context, arch) = nsvca
try:
module = modules.get(name=name,
stream=stream,
version=version,
context=context,
arch=arch)
children.add(module.pk)
except Modulemd.DoesNotExist:
raise
except MultipleObjectsReturned:
raise
# PackageCategories & PackageEnvironments resolution must go before PackageGroups
# TODO: refactor to be more effecient (lower number of queries)
for packagecategory in packagecategories.iterator():
for category_package_group in packagecategory.group_ids:
category_package_groups = PackageGroup.objects.filter(
name=category_package_group['name'],
pk__in=src_repo_version.content)
children.update(
[pkggroup.pk for pkggroup in category_package_groups])
packagegroups = packagegroups.union(category_package_groups)
for packageenvironment in packageenvironments.iterator():
for env_package_group in packageenvironment.group_ids:
env_package_groups = PackageGroup.objects.filter(
name=env_package_group['name'],
pk__in=src_repo_version.content)
children.update([envgroup.pk for envgroup in env_package_groups])
packagegroups = packagegroups.union(env_package_groups)
for optional_env_package_group in packageenvironment.option_ids:
opt_env_package_groups = PackageGroup.objects.filter(
name=optional_env_package_group['name'],
pk__in=src_repo_version.content)
children.update(
[optpkggroup.pk for optpkggroup in opt_env_package_groups])
packagegroups = packagegroups.union(opt_env_package_groups)
# Find rpms referenced by PackageGroups
for packagegroup in packagegroups.iterator():
group_package_names = [pkg['name'] for pkg in packagegroup.packages]
for pkg in group_package_names:
packages_by_name = [
pkg for pkg in Package.objects.with_age().filter(
name=pkg, pk__in=src_repo_version.content) if pkg.age == 1
]
for pkg in packages_by_name:
children.add(pkg.pk)
return Content.objects.filter(pk__in=children)
def resolve_advisories(version, previous_version):
"""
Decide which advisories to add to a repo version and which to remove, and adjust a repo version.
Args:
version (pulpcore.app.models.RepositoryVersion): current incomplete repository version
previous_version (pulpcore.app.models.RepositoryVersion): a version preceding
the current incomplete one
"""
content_pks_to_add = set()
content_pks_to_remove = set()
content_pks_to_exclude = set(
) # exclude from the set of content which is being added
# identify conflicting advisories
advisory_pulp_type = UpdateRecord.get_pulp_type()
current_advisories = UpdateRecord.objects.filter(
pk__in=version.content.filter(pulp_type=advisory_pulp_type))
added_advisories = current_advisories
advisory_conflicts = []
# check for IDs if any conflict
# e.g. mirror mode can already removed advisories with same ID
current_ids = [adv.id for adv in current_advisories]
if previous_version and len(current_ids) != len(set(current_ids)):
previous_advisories = UpdateRecord.objects.filter(
pk__in=previous_version.content.filter(
pulp_type=advisory_pulp_type))
previous_advisory_ids = set(
previous_advisories.values_list('id', flat=True))
# diff for querysets works fine but the result is not fully functional queryset,
# e.g. filtering doesn't work
added_advisories = current_advisories.difference(previous_advisories)
if len(list(added_advisories)) != len(set(added_advisories)):
raise AdvisoryConflict(
_('It is not possible to add two advisories of the same id to '
'a repository version.'))
added_advisory_ids = set(adv.id for adv in added_advisories)
advisory_conflicts = added_advisory_ids.intersection(
previous_advisory_ids)
added_advisory_pks = [adv.pk for adv in added_advisories]
for advisory_id in advisory_conflicts:
previous_advisory = previous_advisories.get(id=advisory_id)
added_advisory = UpdateRecord.objects.get(
id=advisory_id, pk__in=added_advisory_pks)
to_add, to_remove, to_exclude = resolve_advisory_conflict(
previous_advisory, added_advisory)
content_pks_to_add.update(to_add)
content_pks_to_remove.update(to_remove)
content_pks_to_exclude.update(to_exclude)
if content_pks_to_add:
version.add_content(Content.objects.filter(pk__in=content_pks_to_add))
if content_pks_to_remove:
version.remove_content(
Content.objects.filter(pk__in=content_pks_to_remove))
if content_pks_to_exclude:
RepositoryContent.objects.filter(repository=version.repository,
content_id__in=content_pks_to_exclude,
version_added=version).delete()
def find_children_of_content(content, src_repo_version):
"""Finds the content referenced directly by other content and returns it all together.
Finds RPMs referenced by Advisory/Errata content.
Args:
content (Queryset): Content for which to resolve children
src_repo_version (pulpcore.models.RepositoryVersion): Source repo version
Returns: Queryset of Content objects that are children of the intial set of content
"""
# Content that were selected to be copied
advisory_ids = content.filter(
pulp_type=UpdateRecord.get_pulp_type()).only("pk")
packagecategory_ids = content.filter(
pulp_type=PackageCategory.get_pulp_type()).only("pk")
packageenvironment_ids = content.filter(
pulp_type=PackageEnvironment.get_pulp_type()).only("pk")
packagegroup_ids = content.filter(
pulp_type=PackageGroup.get_pulp_type()).only("pk")
# Content in the source repository version
package_ids = src_repo_version.content.filter(
pulp_type=Package.get_pulp_type()).only("pk")
module_ids = src_repo_version.content.filter(
pulp_type=Modulemd.get_pulp_type()).only("pk")
advisories = UpdateRecord.objects.filter(pk__in=advisory_ids)
packages = Package.objects.filter(pk__in=package_ids)
packagecategories = PackageCategory.objects.filter(
pk__in=packagecategory_ids)
packageenvironments = PackageEnvironment.objects.filter(
pk__in=packageenvironment_ids)
packagegroups = PackageGroup.objects.filter(pk__in=packagegroup_ids)
modules = Modulemd.objects.filter(pk__in=module_ids)
children = set()
for advisory in advisories.iterator():
# Find rpms referenced by Advisories/Errata
package_nevras = advisory.get_pkglist()
advisory_package_q = Q(pk__in=[])
for nevra in package_nevras:
(name, epoch, version, release, arch) = nevra
advisory_package_q |= Q(name=name,
epoch=epoch,
version=version,
release=release,
arch=arch)
children.update(
packages.filter(advisory_package_q).values_list("pk", flat=True))
module_nsvcas = advisory.get_module_list()
advisory_module_q = Q(pk__in=[])
for nsvca in module_nsvcas:
(name, stream, version, context, arch) = nsvca
advisory_module_q |= Q(name=name,
stream=stream,
version=version,
context=context,
arch=arch)
children.update(
modules.filter(advisory_module_q).values_list("pk", flat=True))
# PackageCategories & PackageEnvironments resolution must go before PackageGroups
packagegroup_names = set()
for packagecategory in packagecategories.iterator():
for group_id in packagecategory.group_ids:
packagegroup_names.add(group_id["name"])
for packageenvironment in packageenvironments.iterator():
for group_id in packageenvironment.group_ids:
packagegroup_names.add(group_id["name"])
for group_id in packageenvironment.option_ids:
packagegroup_names.add(group_id["name"])
child_package_groups = PackageGroup.objects.filter(
name__in=packagegroup_names, pk__in=src_repo_version.content)
children.update([pkggroup.pk for pkggroup in child_package_groups])
packagegroups = packagegroups.union(child_package_groups)
# Find rpms referenced by PackageGroups
packagegroup_package_names = set()
for packagegroup in packagegroups.iterator():
packagegroup_package_names |= set(pkg["name"]
for pkg in packagegroup.packages)
# TODO: do modular/nonmodular need to be taken into account?
existing_package_names = (Package.objects.filter(
name__in=packagegroup_package_names,
pk__in=content,
).values_list("name", flat=True).distinct())
missing_package_names = packagegroup_package_names - set(
existing_package_names)
needed_packages = Package.objects.with_age().filter(
name__in=missing_package_names, pk__in=src_repo_version.content)
# Pick the latest version of each package available which isn't already present
# in the content set.
for pkg in needed_packages.iterator():
if pkg.age == 1:
children.add(pkg.pk)
return Content.objects.filter(pk__in=children)
def resolve_advisories(version, previous_version):
"""
Decide which advisories to add to a repo version and which to remove, and adjust a repo version.
Advisory can be in 3 different states with relation to a repository version:
- in-memory and added before this function call, so it's a part of the current incomplete
repository version only
- in the db, it's been added in some previous repository version
- has no relation to any repository version because it's been created in this function as an
outcome of conflict resolution.
All 3 states need to be handled differently.
The in-db ones and newly created are straightforward, just remove/add in a standard way.
To remove in-memory ones (`content_pks_to_exclude`) from an incomplete repo version,
one needs to do it directly from RepositoryContent. They've never been a part of a repo
version, they are also not among the `content_pks_to_add` or `content_pks_to_remove` ones.
Args:
version (pulpcore.app.models.RepositoryVersion): current incomplete repository version
previous_version (pulpcore.app.models.RepositoryVersion): a version preceding
the current incomplete one
"""
# identify conflicting advisories
advisory_pulp_type = UpdateRecord.get_pulp_type()
current_advisories = UpdateRecord.objects.filter(
pk__in=version.content.filter(pulp_type=advisory_pulp_type))
# check for any conflict
unique_advisory_ids = {adv.id for adv in current_advisories}
if len(current_advisories) == len(unique_advisory_ids):
# no conflicts
return
current_advisories_by_id = defaultdict(list)
for advisory in current_advisories:
current_advisories_by_id[advisory.id].append(advisory)
if previous_version:
previous_advisories = UpdateRecord.objects.filter(
pk__in=previous_version.content.filter(
pulp_type=advisory_pulp_type))
previous_advisory_ids = set(
previous_advisories.values_list("id", flat=True))
# diff for querysets works fine but the result is not fully functional queryset,
# e.g. filtering doesn't work
added_advisories = current_advisories.difference(previous_advisories)
added_advisories_by_id = defaultdict(list)
for advisory in added_advisories:
added_advisories_by_id[advisory.id].append(advisory)
else:
previous_advisory_ids = set()
added_advisories = current_advisories
added_advisories_by_id = current_advisories_by_id
# Conflicts can be in different places and behaviour differs based on that.
# `in_added`, when conflict happens in the added advisories, this is not allowed and
# should fail.
# `added_vs_previous`, a standard conflict between an advisory which is being added and the one
# in the preceding repo version. This should be resolved according to the heuristics,
# unless previous repo version has conflicts. In the latter case, the added advisory is picked.
advisory_id_conflicts = {"in_added": [], "added_vs_previous": []}
for advisory_id, advisories in current_advisories_by_id.items():
# we are only interested in conflicts where added advisory is present, we are not trying
# to fix old conflicts in the existing repo version. There is no real harm in htose,
# just confusing.
if len(advisories) > 1 and advisory_id in added_advisories_by_id:
# if the conflict is in added advisories (2+ advisories with the same id are being
# added), we need to collect such ids to fail later with
# a list of all conflicting advisories. No other processing of those is needed.
if len(added_advisories_by_id[advisory_id]) > 1:
advisory_id_conflicts["in_added"].append(advisory_id)
# a standard conflict is detected
elif advisory_id in previous_advisory_ids:
advisory_id_conflicts["added_vs_previous"].append(advisory_id)
if advisory_id_conflicts["in_added"]:
raise AdvisoryConflict(
_("It is not possible to add more than one advisory with the same id to a "
"repository version. Affected advisories: {}.".format(",".join(
advisory_id_conflicts["in_added"]))))
content_pks_to_add = set()
content_pks_to_remove = set()
content_pks_to_exclude = set(
) # exclude from the set of content which is being added
if advisory_id_conflicts["added_vs_previous"]:
for advisory_id in advisory_id_conflicts["added_vs_previous"]:
previous_advisory_qs = previous_advisories.filter(id=advisory_id)
# there can only be one added advisory at this point otherwise the AdvisoryConflict
# would have been raised by now
added_advisory = added_advisories_by_id[advisory_id][0]
added_advisory.touch()
if previous_advisory_qs.count() > 1:
# due to an old bug there could be N advisories with the same id in a repo,
# this is wrong and there may not be a good way to resolve those, so let's take a
# new one.
content_pks_to_add.update([added_advisory.pk])
content_pks_to_remove.update(
[adv.pk for adv in previous_advisory_qs])
else:
to_add, to_remove, to_exclude = resolve_advisory_conflict(
previous_advisory_qs.first(), added_advisory)
content_pks_to_add.update(to_add)
content_pks_to_remove.update(to_remove)
content_pks_to_exclude.update(to_exclude)
if content_pks_to_add:
version.add_content(Content.objects.filter(pk__in=content_pks_to_add))
if content_pks_to_remove:
version.remove_content(
Content.objects.filter(pk__in=content_pks_to_remove))
if content_pks_to_exclude:
RepositoryContent.objects.filter(
repository=version.repository,
content_id__in=content_pks_to_exclude,
version_added=version,
).delete()
def find_children_of_content(content, repository_version):
"""Finds the content referenced directly by other content and returns it all together.
Finds RPMs referenced by Advisory/Errata content.
Args:
content (iterable): Content for which to resolve children
repository_version (pulpcore.models.RepositoryVersion): Source repo version
Returns: Queryset of Content objects that are children of the intial set of content
"""
# Advisories that were selected to be copied
advisory_ids = content.filter(
pulp_type=UpdateRecord.get_pulp_type()).only('pk')
# All packages in the source repository version
package_ids = repository_version.content.filter(
pulp_type=Package.get_pulp_type()).only('pk')
# All modules in the source repository version
module_ids = repository_version.content.filter(
pulp_type=Modulemd.get_pulp_type()).only('pk')
advisories = UpdateRecord.objects.filter(pk__in=advisory_ids)
packages = Package.objects.filter(pk__in=package_ids)
modules = Modulemd.objects.filter(pk__in=module_ids)
children = set()
for advisory in advisories:
# Find rpms referenced by Advisories/Errata
package_nevras = advisory.get_pkglist()
for nevra in package_nevras:
(name, epoch, version, release, arch) = nevra
try:
package = packages.get(name=name,
epoch=epoch,
version=version,
release=release,
arch=arch)
children.add(package.pk)
except Package.DoesNotExist:
raise
except MultipleObjectsReturned:
raise
module_nsvcas = advisory.get_module_list()
for nsvca in module_nsvcas:
(name, stream, version, context, arch) = nsvca
try:
module = modules.get(name=name,
stream=stream,
version=version,
context=context,
arch=arch)
children.add(module.pk)
except Modulemd.DoesNotExist:
raise
except MultipleObjectsReturned:
raise
# TODO: Find rpms referenced by PackageGroups,
# PackageGroups referenced by PackageCategories, etc.
return Content.objects.filter(pk__in=children)