def resolve_advisories(version, previous_version): """ Decide which advisories to add to a repo version and which to remove, and adjust a repo version. Advisory can be in 3 different states with relation to a repository version: - in-memory and added before this function call, so it's a part of the current incomplete repository version only - in the db, it's been added in some previous repository version - has no relation to any repository version because it's been created in this function as an outcome of conflict resolution. All 3 states need to be handled differently. The in-db ones and newly created are straightforward, just remove/add in a standard way. To remove in-memory ones (`content_pks_to_exclude`) from an incomplete repo version, one needs to do it directly from RepositoryContent. They've never been a part of a repo version, they are also not among the `content_pks_to_add` or `content_pks_to_remove` ones. Args: version (pulpcore.app.models.RepositoryVersion): current incomplete repository version previous_version (pulpcore.app.models.RepositoryVersion): a version preceding the current incomplete one """ content_pks_to_add = set() content_pks_to_remove = set() content_pks_to_exclude = set( ) # exclude from the set of content which is being added # identify conflicting advisories advisory_pulp_type = UpdateRecord.get_pulp_type() current_advisories = UpdateRecord.objects.filter( pk__in=version.content.filter(pulp_type=advisory_pulp_type)) added_advisories = current_advisories advisory_conflicts = [] # check for any conflict current_ids = [adv.id for adv in current_advisories] if previous_version and len(current_ids) != len(set(current_ids)): previous_advisories = UpdateRecord.objects.filter( pk__in=previous_version.content.filter( pulp_type=advisory_pulp_type)) previous_advisory_ids = set( previous_advisories.values_list('id', flat=True)) # diff for querysets works fine but the result is not fully functional queryset, # e.g. filtering doesn't work added_advisories = current_advisories.difference(previous_advisories) if len(list(added_advisories)) != len(set(added_advisories)): raise AdvisoryConflict( _('It is not possible to add two advisories of the same id to ' 'a repository version.')) added_advisory_ids = set(adv.id for adv in added_advisories) advisory_conflicts = added_advisory_ids.intersection( previous_advisory_ids) added_advisory_pks = [adv.pk for adv in added_advisories] for advisory_id in advisory_conflicts: previous_advisory = previous_advisories.get(id=advisory_id) added_advisory = UpdateRecord.objects.get( id=advisory_id, pk__in=added_advisory_pks) to_add, to_remove, to_exclude = resolve_advisory_conflict( previous_advisory, added_advisory) content_pks_to_add.update(to_add) content_pks_to_remove.update(to_remove) content_pks_to_exclude.update(to_exclude) if content_pks_to_add: version.add_content(Content.objects.filter(pk__in=content_pks_to_add)) if content_pks_to_remove: version.remove_content( Content.objects.filter(pk__in=content_pks_to_remove)) if content_pks_to_exclude: RepositoryContent.objects.filter(repository=version.repository, content_id__in=content_pks_to_exclude, version_added=version).delete()
def find_children_of_content(content, src_repo_version): """Finds the content referenced directly by other content and returns it all together. Finds RPMs referenced by Advisory/Errata content. Args: content (iterable): Content for which to resolve children src_repo_version (pulpcore.models.RepositoryVersion): Source repo version Returns: Queryset of Content objects that are children of the intial set of content """ # Content that were selected to be copied advisory_ids = content.filter( pulp_type=UpdateRecord.get_pulp_type()).only('pk') packagecategory_ids = content.filter( pulp_type=PackageCategory.get_pulp_type()).only('pk') packageenvironment_ids = content.filter( pulp_type=PackageEnvironment.get_pulp_type()).only('pk') packagegroup_ids = content.filter( pulp_type=PackageGroup.get_pulp_type()).only('pk') # Content in the source repository version package_ids = src_repo_version.content.filter( pulp_type=Package.get_pulp_type()).only('pk') module_ids = src_repo_version.content.filter( pulp_type=Modulemd.get_pulp_type()).only('pk') advisories = UpdateRecord.objects.filter(pk__in=advisory_ids) packages = Package.objects.filter(pk__in=package_ids) packagecategories = PackageCategory.objects.filter( pk__in=packagecategory_ids) packageenvironments = PackageEnvironment.objects.filter( pk__in=packageenvironment_ids) packagegroups = PackageGroup.objects.filter(pk__in=packagegroup_ids) modules = Modulemd.objects.filter(pk__in=module_ids) children = set() for advisory in advisories: # Find rpms referenced by Advisories/Errata package_nevras = advisory.get_pkglist() for nevra in package_nevras: (name, epoch, version, release, arch) = nevra try: package = packages.get(name=name, epoch=epoch, version=version, release=release, arch=arch) children.add(package.pk) except Package.DoesNotExist: raise except MultipleObjectsReturned: raise module_nsvcas = advisory.get_module_list() for nsvca in module_nsvcas: (name, stream, version, context, arch) = nsvca try: module = modules.get(name=name, stream=stream, version=version, context=context, arch=arch) children.add(module.pk) except Modulemd.DoesNotExist: raise except MultipleObjectsReturned: raise # PackageCategories & PackageEnvironments resolution must go before PackageGroups # TODO: refactor to be more effecient (lower number of queries) for packagecategory in packagecategories.iterator(): for category_package_group in packagecategory.group_ids: category_package_groups = PackageGroup.objects.filter( name=category_package_group['name'], pk__in=src_repo_version.content) children.update( [pkggroup.pk for pkggroup in category_package_groups]) packagegroups = packagegroups.union(category_package_groups) for packageenvironment in packageenvironments.iterator(): for env_package_group in packageenvironment.group_ids: env_package_groups = PackageGroup.objects.filter( name=env_package_group['name'], pk__in=src_repo_version.content) children.update([envgroup.pk for envgroup in env_package_groups]) packagegroups = packagegroups.union(env_package_groups) for optional_env_package_group in packageenvironment.option_ids: opt_env_package_groups = PackageGroup.objects.filter( name=optional_env_package_group['name'], pk__in=src_repo_version.content) children.update( [optpkggroup.pk for optpkggroup in opt_env_package_groups]) packagegroups = packagegroups.union(opt_env_package_groups) # Find rpms referenced by PackageGroups for packagegroup in packagegroups.iterator(): group_package_names = [pkg['name'] for pkg in packagegroup.packages] for pkg in group_package_names: packages_by_name = [ pkg for pkg in Package.objects.with_age().filter( name=pkg, pk__in=src_repo_version.content) if pkg.age == 1 ] for pkg in packages_by_name: children.add(pkg.pk) return Content.objects.filter(pk__in=children)
def resolve_advisories(version, previous_version): """ Decide which advisories to add to a repo version and which to remove, and adjust a repo version. Args: version (pulpcore.app.models.RepositoryVersion): current incomplete repository version previous_version (pulpcore.app.models.RepositoryVersion): a version preceding the current incomplete one """ content_pks_to_add = set() content_pks_to_remove = set() content_pks_to_exclude = set( ) # exclude from the set of content which is being added # identify conflicting advisories advisory_pulp_type = UpdateRecord.get_pulp_type() current_advisories = UpdateRecord.objects.filter( pk__in=version.content.filter(pulp_type=advisory_pulp_type)) added_advisories = current_advisories advisory_conflicts = [] # check for IDs if any conflict # e.g. mirror mode can already removed advisories with same ID current_ids = [adv.id for adv in current_advisories] if previous_version and len(current_ids) != len(set(current_ids)): previous_advisories = UpdateRecord.objects.filter( pk__in=previous_version.content.filter( pulp_type=advisory_pulp_type)) previous_advisory_ids = set( previous_advisories.values_list('id', flat=True)) # diff for querysets works fine but the result is not fully functional queryset, # e.g. filtering doesn't work added_advisories = current_advisories.difference(previous_advisories) if len(list(added_advisories)) != len(set(added_advisories)): raise AdvisoryConflict( _('It is not possible to add two advisories of the same id to ' 'a repository version.')) added_advisory_ids = set(adv.id for adv in added_advisories) advisory_conflicts = added_advisory_ids.intersection( previous_advisory_ids) added_advisory_pks = [adv.pk for adv in added_advisories] for advisory_id in advisory_conflicts: previous_advisory = previous_advisories.get(id=advisory_id) added_advisory = UpdateRecord.objects.get( id=advisory_id, pk__in=added_advisory_pks) to_add, to_remove, to_exclude = resolve_advisory_conflict( previous_advisory, added_advisory) content_pks_to_add.update(to_add) content_pks_to_remove.update(to_remove) content_pks_to_exclude.update(to_exclude) if content_pks_to_add: version.add_content(Content.objects.filter(pk__in=content_pks_to_add)) if content_pks_to_remove: version.remove_content( Content.objects.filter(pk__in=content_pks_to_remove)) if content_pks_to_exclude: RepositoryContent.objects.filter(repository=version.repository, content_id__in=content_pks_to_exclude, version_added=version).delete()
def find_children_of_content(content, src_repo_version): """Finds the content referenced directly by other content and returns it all together. Finds RPMs referenced by Advisory/Errata content. Args: content (Queryset): Content for which to resolve children src_repo_version (pulpcore.models.RepositoryVersion): Source repo version Returns: Queryset of Content objects that are children of the intial set of content """ # Content that were selected to be copied advisory_ids = content.filter( pulp_type=UpdateRecord.get_pulp_type()).only("pk") packagecategory_ids = content.filter( pulp_type=PackageCategory.get_pulp_type()).only("pk") packageenvironment_ids = content.filter( pulp_type=PackageEnvironment.get_pulp_type()).only("pk") packagegroup_ids = content.filter( pulp_type=PackageGroup.get_pulp_type()).only("pk") # Content in the source repository version package_ids = src_repo_version.content.filter( pulp_type=Package.get_pulp_type()).only("pk") module_ids = src_repo_version.content.filter( pulp_type=Modulemd.get_pulp_type()).only("pk") advisories = UpdateRecord.objects.filter(pk__in=advisory_ids) packages = Package.objects.filter(pk__in=package_ids) packagecategories = PackageCategory.objects.filter( pk__in=packagecategory_ids) packageenvironments = PackageEnvironment.objects.filter( pk__in=packageenvironment_ids) packagegroups = PackageGroup.objects.filter(pk__in=packagegroup_ids) modules = Modulemd.objects.filter(pk__in=module_ids) children = set() for advisory in advisories.iterator(): # Find rpms referenced by Advisories/Errata package_nevras = advisory.get_pkglist() advisory_package_q = Q(pk__in=[]) for nevra in package_nevras: (name, epoch, version, release, arch) = nevra advisory_package_q |= Q(name=name, epoch=epoch, version=version, release=release, arch=arch) children.update( packages.filter(advisory_package_q).values_list("pk", flat=True)) module_nsvcas = advisory.get_module_list() advisory_module_q = Q(pk__in=[]) for nsvca in module_nsvcas: (name, stream, version, context, arch) = nsvca advisory_module_q |= Q(name=name, stream=stream, version=version, context=context, arch=arch) children.update( modules.filter(advisory_module_q).values_list("pk", flat=True)) # PackageCategories & PackageEnvironments resolution must go before PackageGroups packagegroup_names = set() for packagecategory in packagecategories.iterator(): for group_id in packagecategory.group_ids: packagegroup_names.add(group_id["name"]) for packageenvironment in packageenvironments.iterator(): for group_id in packageenvironment.group_ids: packagegroup_names.add(group_id["name"]) for group_id in packageenvironment.option_ids: packagegroup_names.add(group_id["name"]) child_package_groups = PackageGroup.objects.filter( name__in=packagegroup_names, pk__in=src_repo_version.content) children.update([pkggroup.pk for pkggroup in child_package_groups]) packagegroups = packagegroups.union(child_package_groups) # Find rpms referenced by PackageGroups packagegroup_package_names = set() for packagegroup in packagegroups.iterator(): packagegroup_package_names |= set(pkg["name"] for pkg in packagegroup.packages) # TODO: do modular/nonmodular need to be taken into account? existing_package_names = (Package.objects.filter( name__in=packagegroup_package_names, pk__in=content, ).values_list("name", flat=True).distinct()) missing_package_names = packagegroup_package_names - set( existing_package_names) needed_packages = Package.objects.with_age().filter( name__in=missing_package_names, pk__in=src_repo_version.content) # Pick the latest version of each package available which isn't already present # in the content set. for pkg in needed_packages.iterator(): if pkg.age == 1: children.add(pkg.pk) return Content.objects.filter(pk__in=children)
def resolve_advisories(version, previous_version): """ Decide which advisories to add to a repo version and which to remove, and adjust a repo version. Advisory can be in 3 different states with relation to a repository version: - in-memory and added before this function call, so it's a part of the current incomplete repository version only - in the db, it's been added in some previous repository version - has no relation to any repository version because it's been created in this function as an outcome of conflict resolution. All 3 states need to be handled differently. The in-db ones and newly created are straightforward, just remove/add in a standard way. To remove in-memory ones (`content_pks_to_exclude`) from an incomplete repo version, one needs to do it directly from RepositoryContent. They've never been a part of a repo version, they are also not among the `content_pks_to_add` or `content_pks_to_remove` ones. Args: version (pulpcore.app.models.RepositoryVersion): current incomplete repository version previous_version (pulpcore.app.models.RepositoryVersion): a version preceding the current incomplete one """ # identify conflicting advisories advisory_pulp_type = UpdateRecord.get_pulp_type() current_advisories = UpdateRecord.objects.filter( pk__in=version.content.filter(pulp_type=advisory_pulp_type)) # check for any conflict unique_advisory_ids = {adv.id for adv in current_advisories} if len(current_advisories) == len(unique_advisory_ids): # no conflicts return current_advisories_by_id = defaultdict(list) for advisory in current_advisories: current_advisories_by_id[advisory.id].append(advisory) if previous_version: previous_advisories = UpdateRecord.objects.filter( pk__in=previous_version.content.filter( pulp_type=advisory_pulp_type)) previous_advisory_ids = set( previous_advisories.values_list("id", flat=True)) # diff for querysets works fine but the result is not fully functional queryset, # e.g. filtering doesn't work added_advisories = current_advisories.difference(previous_advisories) added_advisories_by_id = defaultdict(list) for advisory in added_advisories: added_advisories_by_id[advisory.id].append(advisory) else: previous_advisory_ids = set() added_advisories = current_advisories added_advisories_by_id = current_advisories_by_id # Conflicts can be in different places and behaviour differs based on that. # `in_added`, when conflict happens in the added advisories, this is not allowed and # should fail. # `added_vs_previous`, a standard conflict between an advisory which is being added and the one # in the preceding repo version. This should be resolved according to the heuristics, # unless previous repo version has conflicts. In the latter case, the added advisory is picked. advisory_id_conflicts = {"in_added": [], "added_vs_previous": []} for advisory_id, advisories in current_advisories_by_id.items(): # we are only interested in conflicts where added advisory is present, we are not trying # to fix old conflicts in the existing repo version. There is no real harm in htose, # just confusing. if len(advisories) > 1 and advisory_id in added_advisories_by_id: # if the conflict is in added advisories (2+ advisories with the same id are being # added), we need to collect such ids to fail later with # a list of all conflicting advisories. No other processing of those is needed. if len(added_advisories_by_id[advisory_id]) > 1: advisory_id_conflicts["in_added"].append(advisory_id) # a standard conflict is detected elif advisory_id in previous_advisory_ids: advisory_id_conflicts["added_vs_previous"].append(advisory_id) if advisory_id_conflicts["in_added"]: raise AdvisoryConflict( _("It is not possible to add more than one advisory with the same id to a " "repository version. Affected advisories: {}.".format(",".join( advisory_id_conflicts["in_added"])))) content_pks_to_add = set() content_pks_to_remove = set() content_pks_to_exclude = set( ) # exclude from the set of content which is being added if advisory_id_conflicts["added_vs_previous"]: for advisory_id in advisory_id_conflicts["added_vs_previous"]: previous_advisory_qs = previous_advisories.filter(id=advisory_id) # there can only be one added advisory at this point otherwise the AdvisoryConflict # would have been raised by now added_advisory = added_advisories_by_id[advisory_id][0] added_advisory.touch() if previous_advisory_qs.count() > 1: # due to an old bug there could be N advisories with the same id in a repo, # this is wrong and there may not be a good way to resolve those, so let's take a # new one. content_pks_to_add.update([added_advisory.pk]) content_pks_to_remove.update( [adv.pk for adv in previous_advisory_qs]) else: to_add, to_remove, to_exclude = resolve_advisory_conflict( previous_advisory_qs.first(), added_advisory) content_pks_to_add.update(to_add) content_pks_to_remove.update(to_remove) content_pks_to_exclude.update(to_exclude) if content_pks_to_add: version.add_content(Content.objects.filter(pk__in=content_pks_to_add)) if content_pks_to_remove: version.remove_content( Content.objects.filter(pk__in=content_pks_to_remove)) if content_pks_to_exclude: RepositoryContent.objects.filter( repository=version.repository, content_id__in=content_pks_to_exclude, version_added=version, ).delete()
def find_children_of_content(content, repository_version): """Finds the content referenced directly by other content and returns it all together. Finds RPMs referenced by Advisory/Errata content. Args: content (iterable): Content for which to resolve children repository_version (pulpcore.models.RepositoryVersion): Source repo version Returns: Queryset of Content objects that are children of the intial set of content """ # Advisories that were selected to be copied advisory_ids = content.filter( pulp_type=UpdateRecord.get_pulp_type()).only('pk') # All packages in the source repository version package_ids = repository_version.content.filter( pulp_type=Package.get_pulp_type()).only('pk') # All modules in the source repository version module_ids = repository_version.content.filter( pulp_type=Modulemd.get_pulp_type()).only('pk') advisories = UpdateRecord.objects.filter(pk__in=advisory_ids) packages = Package.objects.filter(pk__in=package_ids) modules = Modulemd.objects.filter(pk__in=module_ids) children = set() for advisory in advisories: # Find rpms referenced by Advisories/Errata package_nevras = advisory.get_pkglist() for nevra in package_nevras: (name, epoch, version, release, arch) = nevra try: package = packages.get(name=name, epoch=epoch, version=version, release=release, arch=arch) children.add(package.pk) except Package.DoesNotExist: raise except MultipleObjectsReturned: raise module_nsvcas = advisory.get_module_list() for nsvca in module_nsvcas: (name, stream, version, context, arch) = nsvca try: module = modules.get(name=name, stream=stream, version=version, context=context, arch=arch) children.add(module.pk) except Modulemd.DoesNotExist: raise except MultipleObjectsReturned: raise # TODO: Find rpms referenced by PackageGroups, # PackageGroups referenced by PackageCategories, etc. return Content.objects.filter(pk__in=children)