def pkg_grp_to_libcomps(obj):
"""
Convert PackageGroup object to libcomps Group object.
Args:
obj(Pulp2PackageGroup): Pulp2PackageGroup
Returns:
group: libcomps.Group object
"""
group = libcomps.Group()
group.id = obj.package_group_id
group.default = obj.default
group.uservisible = obj.user_visible
group.display_order = obj.display_order
group.name = obj.name
group.desc = obj.description
group.packages = _list_to_pkglist(obj.packages)
group.biarchonly = obj.biarch_only
group.desc_by_lang = dict_to_strdict(obj.desc_by_lang)
group.name_by_lang = dict_to_strdict(obj.name_by_lang)
return group
def pkg_grp_to_libcomps(self):
"""
Convert PackageGroup object to libcomps Group object.
Returns:
group: libcomps.Group object
"""
group = libcomps.Group()
group.id = getattr(self, PULP_GROUP_ATTRS.ID)
group.default = getattr(self, PULP_GROUP_ATTRS.DEFAULT)
group.uservisible = getattr(self, PULP_GROUP_ATTRS.USER_VISIBLE)
group.display_order = getattr(self, PULP_GROUP_ATTRS.DISPLAY_ORDER)
group.name = getattr(self, PULP_GROUP_ATTRS.NAME)
group.desc = getattr(self, PULP_GROUP_ATTRS.DESCRIPTION)
group.packages = self.list_to_pkglist(
getattr(self, PULP_GROUP_ATTRS.PACKAGES))
group.biarchonly = getattr(self, PULP_GROUP_ATTRS.BIARCH_ONLY)
group.desc_by_lang = dict_to_strdict(
getattr(self, PULP_GROUP_ATTRS.DESC_BY_LANG))
group.name_by_lang = dict_to_strdict(
getattr(self, PULP_GROUP_ATTRS.NAME_BY_LANG))
return group
def pkg_cat_to_libcomps(self):
"""
Convert PackageCategory object to libcomps Category object.
Returns:
group: libcomps.Category object
"""
cat = libcomps.Category()
cat.id = getattr(self, PULP_CATEGORY_ATTRS.ID)
cat.name = getattr(self, PULP_CATEGORY_ATTRS.NAME)
cat.desc = getattr(self, PULP_CATEGORY_ATTRS.DESCRIPTION)
cat.display_order = getattr(self, PULP_CATEGORY_ATTRS.DISPLAY_ORDER)
cat.group_ids = list_to_idlist(getattr(self, PULP_CATEGORY_ATTRS.GROUP_IDS))
cat.desc_by_lang = dict_to_strdict(getattr(self, PULP_CATEGORY_ATTRS.DESC_BY_LANG))
cat.name_by_lang = dict_to_strdict(getattr(self, PULP_CATEGORY_ATTRS.NAME_BY_LANG))
return cat
def pkg_env_to_libcomps(self):
"""
Convert PackageEnvironment object to libcomps Environment object.
Returns:
group: libcomps.Environment object
"""
env = libcomps.Environment()
env.id = getattr(self, PULP_ENVIRONMENT_ATTRS.ID)
env.name = getattr(self, PULP_ENVIRONMENT_ATTRS.NAME)
env.desc = getattr(self, PULP_ENVIRONMENT_ATTRS.DESCRIPTION)
env.display_order = getattr(self, PULP_ENVIRONMENT_ATTRS.DISPLAY_ORDER)
env.group_ids = list_to_idlist(getattr(self, PULP_ENVIRONMENT_ATTRS.GROUP_IDS))
env.option_ids = list_to_idlist(getattr(self, PULP_ENVIRONMENT_ATTRS.OPTION_IDS))
env.desc_by_lang = dict_to_strdict(getattr(self, PULP_ENVIRONMENT_ATTRS.DESC_BY_LANG))
env.name_by_lang = dict_to_strdict(getattr(self, PULP_ENVIRONMENT_ATTRS.NAME_BY_LANG))
return env
def pkg_cat_to_libcomps(obj):
"""
Convert PackageCategory object to libcomps Category object.
Args:
obj(Pulp2PackageCategory): Pulp2PackageCategory
Returns:
group: libcomps.Category object
"""
cat = libcomps.Category()
cat.id = obj.package_category_id
cat.name = obj.name
cat.desc = obj.description
cat.display_order = obj.display_order
cat.group_ids = list_to_idlist(_packages_to_grplist(obj.packagegroupids))
cat.desc_by_lang = dict_to_strdict(obj.desc_by_lang)
cat.name_by_lang = dict_to_strdict(obj.name_by_lang)
return cat
def pkg_env_to_libcomps(obj):
"""
Convert PackageEnvironment object to libcomps Environment object.
Args:
obj(Pulp2PackageEnvironment): Pulp2PackageEnvironment
Returns:
group: libcomps.Environment object
"""
env = libcomps.Environment()
env.id = obj.package_environment_id
env.name = obj.name
env.desc = obj.description
env.display_order = obj.display_order
env.group_ids = list_to_idlist(_packages_to_grplist(obj.group_ids))
result = _packages_to_optionlist(obj.option_ids)
env.option_ids = list_to_idlist(result)
env.desc_by_lang = dict_to_strdict(obj.desc_by_lang)
env.name_by_lang = dict_to_strdict(obj.name_by_lang)
return env
def create_repomd_xml(
content,
publication,
checksum_types,
extra_repomdrecords,
sub_folder=None,
metadata_signing_service=None,
):
"""
Creates a repomd.xml file.
Args:
content(app.models.Content): content set
publication(pulpcore.plugin.models.Publication): the publication
extra_repomdrecords(list): list with data relative to repo metadata files
sub_folder(str): name of the folder for sub repos
metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService):
A reference to an associated signing service.
"""
cwd = os.getcwd()
repodata_path = REPODATA_PATH
has_modules = False
has_comps = False
package_checksum_type = checksum_types.get("package")
if sub_folder:
cwd = os.path.join(cwd, sub_folder)
repodata_path = os.path.join(sub_folder, repodata_path)
# Prepare metadata files
repomd_path = os.path.join(cwd, "repomd.xml")
pri_xml_path = os.path.join(cwd, "primary.xml.gz")
fil_xml_path = os.path.join(cwd, "filelists.xml.gz")
oth_xml_path = os.path.join(cwd, "other.xml.gz")
upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz")
mod_yml_path = os.path.join(cwd, "modules.yaml")
comps_xml_path = os.path.join(cwd, "comps.xml")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
if publication.sqlite_metadata:
pri_db_path = os.path.join(cwd, "primary.sqlite")
fil_db_path = os.path.join(cwd, "filelists.sqlite")
oth_db_path = os.path.join(cwd, "other.sqlite")
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
packages = Package.objects.filter(pk__in=content)
total_packages = packages.count()
pri_xml.set_num_of_pkgs(total_packages)
fil_xml.set_num_of_pkgs(total_packages)
oth_xml.set_num_of_pkgs(total_packages)
# We want to support publishing with a different checksum type than the one built-in to the
# package itself, so we need to get the correct checksums somehow if there is an override.
# We must also take into consideration that if the package has not been downloaded the only
# checksum that is available is the one built-in.
#
# Since this lookup goes from Package->Content->ContentArtifact->Artifact, performance is a
# challenge. We use ContentArtifact as our starting point because it enables us to work with
# simple foreign keys and avoid messing with the many-to-many relationship, which doesn't
# work with select_related() and performs poorly with prefetch_related(). This is fine
# because we know that Packages should only ever have one artifact per content.
contentartifact_qs = (
ContentArtifact.objects.filter(content__in=packages.only("pk")).
select_related(
# content__rpm_package is a bit of a hack, exploiting the way django sets up model
# inheritance, but it works and is unlikely to break. All content artifacts being
# accessed here have an associated Package since they originally came from the
# Package queryset.
"artifact",
"content__rpm_package",
).only("artifact", "content__rpm_package__checksum_type",
"content__rpm_package__pkgId"))
pkg_to_hash = {}
for ca in contentartifact_qs.iterator():
pkgid = None
if package_checksum_type:
package_checksum_type = package_checksum_type.lower()
pkgid = getattr(ca.artifact, package_checksum_type, None)
if pkgid:
pkg_to_hash[ca.content_id] = (package_checksum_type, pkgid)
else:
pkg_to_hash[ca.content_id] = (
ca.content.rpm_package.checksum_type,
ca.content.rpm_package.pkgId,
)
# Process all packages
for package in packages.iterator():
pkg = package.to_createrepo_c()
# rewrite the checksum and checksum type with the desired ones
(checksum, pkgId) = pkg_to_hash[package.pk]
pkg.checksum_type = checksum
pkg.pkgId = pkgId
pkg_filename = os.path.basename(package.location_href)
# this can cause an issue when two same RPM package names appears
# a/name1.rpm b/name1.rpm
pkg.location_href = os.path.join(PACKAGES_DIRECTORY,
pkg_filename[0].lower(), pkg_filename)
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
if publication.sqlite_metadata:
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=content).iterator():
upd_xml.add_chunk(
cr.xml_dump_updaterecord(update_record.to_createrepo_c()))
# Process modulemd and modulemd_defaults
with open(mod_yml_path, "ab") as mod_yml:
for modulemd in Modulemd.objects.filter(pk__in=content).iterator():
mod_yml.write(modulemd._artifacts.get().file.read())
has_modules = True
for default in ModulemdDefaults.objects.filter(
pk__in=content).iterator():
mod_yml.write(default._artifacts.get().file.read())
has_modules = True
# Process comps
comps = libcomps.Comps()
for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator():
group = pkg_grp.pkg_grp_to_libcomps()
comps.groups.append(group)
has_comps = True
for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator():
cat = pkg_cat.pkg_cat_to_libcomps()
comps.categories.append(cat)
has_comps = True
for pkg_env in PackageEnvironment.objects.filter(
pk__in=content).iterator():
env = pkg_env.pkg_env_to_libcomps()
comps.environments.append(env)
has_comps = True
for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator():
comps.langpacks = dict_to_strdict(pkg_lng.matches)
has_comps = True
comps.toxml_f(
comps_xml_path,
xml_options={
"default_explicit": True,
"empty_groups": True,
"uservisible_explicit": True
},
)
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
if publication.sqlite_metadata:
repomdrecords = [
("primary", pri_xml_path, pri_db),
("filelists", fil_xml_path, fil_db),
("other", oth_xml_path, oth_db),
("primary_db", pri_db_path, None),
("filelists_db", fil_db_path, None),
("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None),
]
else:
repomdrecords = [
("primary", pri_xml_path, None),
("filelists", fil_xml_path, None),
("other", oth_xml_path, None),
("updateinfo", upd_xml_path, None),
]
if has_modules:
repomdrecords.append(("modules", mod_yml_path, None))
if has_comps:
repomdrecords.append(("group", comps_xml_path, None))
repomdrecords.extend(extra_repomdrecords)
sqlite_files = ("primary_db", "filelists_db", "other_db")
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
checksum_type = get_checksum_type(name, checksum_types)
if name in sqlite_files:
record_bz = record.compress_and_fill(checksum_type, cr.BZ2)
record_bz.type = name
record_bz.rename_file()
path = record_bz.location_href.split("/")[-1]
repomd.set_record(record_bz)
else:
record.fill(checksum_type)
if db_to_update:
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
record.rename_file()
path = record.location_href.split("/")[-1]
repomd.set_record(record)
if sub_folder:
path = os.path.join(sub_folder, path)
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path, os.path.basename(path)),
publication=publication,
file=File(open(path, "rb")),
)
with open(repomd_path, "w") as repomd_f:
repomd_f.write(repomd.xml_dump())
if metadata_signing_service:
signing_service = AsciiArmoredDetachedSigningService.objects.get(
pk=metadata_signing_service.pk)
sign_results = signing_service.sign(repomd_path)
# publish a signed file
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(sign_results["file"])),
publication=publication,
file=File(open(sign_results["file"], "rb")),
)
# publish a detached signature
PublishedMetadata.create_from_file(
relative_path=os.path.join(
repodata_path, os.path.basename(sign_results["signature"])),
publication=publication,
file=File(open(sign_results["signature"], "rb")),
)
# publish a public key required for further verification
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(sign_results["key"])),
publication=publication,
file=File(open(sign_results["key"], "rb")),
)
else:
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(repomd_path)),
publication=publication,
file=File(open(repomd_path, "rb")),
)
def create_repomd_xml(content, publication, extra_repomdrecords, sub_folder=None):
"""
Creates a repomd.xml file.
Args:
content(app.models.Content): content set
publication(pulpcore.plugin.models.Publication): the publication
extra_repomdrecords(list): list with data relative to repo metadata files
sub_folder(str): name of the folder for sub repos
"""
cwd = os.getcwd()
repodata_path = REPODATA_PATH
has_modules = False
has_comps = False
if sub_folder:
cwd = os.path.join(cwd, sub_folder)
repodata_path = os.path.join(sub_folder, repodata_path)
# Prepare metadata files
repomd_path = os.path.join(cwd, "repomd.xml")
pri_xml_path = os.path.join(cwd, "primary.xml.gz")
fil_xml_path = os.path.join(cwd, "filelists.xml.gz")
oth_xml_path = os.path.join(cwd, "other.xml.gz")
pri_db_path = os.path.join(cwd, "primary.sqlite")
fil_db_path = os.path.join(cwd, "filelists.sqlite")
oth_db_path = os.path.join(cwd, "other.sqlite")
upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz")
mod_yml_path = os.path.join(cwd, "modules.yaml")
comps_xml_path = os.path.join(cwd, "comps.xml")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
packages = Package.objects.filter(pk__in=content)
total_packages = packages.count()
pri_xml.set_num_of_pkgs(total_packages)
fil_xml.set_num_of_pkgs(total_packages)
oth_xml.set_num_of_pkgs(total_packages)
# Process all packages
for package in packages.iterator():
pkg = package.to_createrepo_c()
pkg.location_href = package.contentartifact_set.only('relative_path').first().relative_path
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(pk__in=content).iterator():
upd_xml.add_chunk(cr.xml_dump_updaterecord(update_record.to_createrepo_c()))
# Process modulemd and modulemd_defaults
with open(mod_yml_path, 'ab') as mod_yml:
for modulemd in Modulemd.objects.filter(pk__in=content).iterator():
mod_yml.write(modulemd._artifacts.get().file.read())
has_modules = True
for default in ModulemdDefaults.objects.filter(pk__in=content).iterator():
mod_yml.write(default._artifacts.get().file.read())
has_modules = True
# Process comps
comps = libcomps.Comps()
for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator():
group = pkg_grp.pkg_grp_to_libcomps()
comps.groups.append(group)
has_comps = True
for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator():
cat = pkg_cat.pkg_cat_to_libcomps()
comps.categories.append(cat)
has_comps = True
for pkg_env in PackageEnvironment.objects.filter(pk__in=content).iterator():
env = pkg_env.pkg_env_to_libcomps()
comps.environments.append(env)
has_comps = True
for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator():
comps.langpacks = dict_to_strdict(pkg_lng.matches)
has_comps = True
comps.toxml_f(comps_xml_path, xml_options={"default_explicit": True,
"empty_groups": True,
"uservisible_explicit": True})
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
repomdrecords = [("primary", pri_xml_path, pri_db),
("filelists", fil_xml_path, fil_db),
("other", oth_xml_path, oth_db),
("primary_db", pri_db_path, None),
("filelists_db", fil_db_path, None),
("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None)]
if has_modules:
repomdrecords.append(("modules", mod_yml_path, None))
if has_comps:
repomdrecords.append(("group", comps_xml_path, None))
repomdrecords.extend(extra_repomdrecords)
sqlite_files = ("primary_db", "filelists_db", "other_db")
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
if name in sqlite_files:
record_bz = record.compress_and_fill(cr.SHA256, cr.BZ2)
record_bz.type = name
record_bz.rename_file()
path = record_bz.location_href.split('/')[-1]
repomd.set_record(record_bz)
else:
record.fill(cr.SHA256)
if (db_to_update):
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
record.rename_file()
path = record.location_href.split('/')[-1]
repomd.set_record(record)
if sub_folder:
path = os.path.join(sub_folder, path)
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path, os.path.basename(path)),
publication=publication,
file=File(open(path, 'rb'))
)
with open(repomd_path, "w") as repomd_f:
repomd_f.write(repomd.xml_dump())
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path, os.path.basename(repomd_path)),
publication=publication,
file=File(open(repomd_path, 'rb'))
)
def generate_repo_metadata(
content,
publication,
checksum_types,
extra_repomdrecords,
sub_folder=None,
metadata_signing_service=None,
):
"""
Creates a repomd.xml file.
Args:
content(app.models.Content): content set
publication(pulpcore.plugin.models.Publication): the publication
extra_repomdrecords(list): list with data relative to repo metadata files
sub_folder(str): name of the folder for sub repos
metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService):
A reference to an associated signing service.
"""
cwd = os.getcwd()
repodata_path = REPODATA_PATH
has_modules = False
has_comps = False
package_checksum_type = checksum_types.get("package")
if sub_folder:
cwd = os.path.join(cwd, sub_folder)
repodata_path = os.path.join(sub_folder, repodata_path)
if package_checksum_type and package_checksum_type not in settings.ALLOWED_CONTENT_CHECKSUMS:
raise ValueError(
"Repository contains disallowed package checksum type '{}', "
"thus can't be published. {}".format(package_checksum_type,
ALLOWED_CHECKSUM_ERROR_MSG))
# Prepare metadata files
repomd_path = os.path.join(cwd, "repomd.xml")
pri_xml_path = os.path.join(cwd, "primary.xml.gz")
fil_xml_path = os.path.join(cwd, "filelists.xml.gz")
oth_xml_path = os.path.join(cwd, "other.xml.gz")
upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz")
mod_yml_path = os.path.join(cwd, "modules.yaml")
comps_xml_path = os.path.join(cwd, "comps.xml")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
if publication.sqlite_metadata:
pri_db_path = os.path.join(cwd, "primary.sqlite")
fil_db_path = os.path.join(cwd, "filelists.sqlite")
oth_db_path = os.path.join(cwd, "other.sqlite")
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
packages = Package.objects.filter(pk__in=content)
# We want to support publishing with a different checksum type than the one built-in to the
# package itself, so we need to get the correct checksums somehow if there is an override.
# We must also take into consideration that if the package has not been downloaded the only
# checksum that is available is the one built-in.
#
# Since this lookup goes from Package->Content->ContentArtifact->Artifact, performance is a
# challenge. We use ContentArtifact as our starting point because it enables us to work with
# simple foreign keys and avoid messing with the many-to-many relationship, which doesn't
# work with select_related() and performs poorly with prefetch_related(). This is fine
# because we know that Packages should only ever have one artifact per content.
contentartifact_qs = (
ContentArtifact.objects.filter(content__in=packages.only("pk")).
select_related(
# content__rpm_package is a bit of a hack, exploiting the way django sets up model
# inheritance, but it works and is unlikely to break. All content artifacts being
# accessed here have an associated Package since they originally came from the
# Package queryset.
"artifact",
"content__rpm_package",
).only("artifact", "content__rpm_package__checksum_type",
"content__rpm_package__pkgId"))
pkg_to_hash = {}
for ca in contentartifact_qs.iterator():
if package_checksum_type:
package_checksum_type = package_checksum_type.lower()
pkgid = getattr(ca.artifact, package_checksum_type, None)
if not package_checksum_type or not pkgid:
if ca.content.rpm_package.checksum_type not in settings.ALLOWED_CONTENT_CHECKSUMS:
raise ValueError(
"Package {} as content unit {} contains forbidden checksum type '{}', "
"thus can't be published. {}".format(
ca.content.rpm_package.nevra,
ca.content.pk,
ca.content.rpm_package.checksum_type,
ALLOWED_CHECKSUM_ERROR_MSG,
))
package_checksum_type = ca.content.rpm_package.checksum_type
pkgid = ca.content.rpm_package.pkgId
pkg_to_hash[ca.content_id] = (package_checksum_type, pkgid)
# TODO: this is meant to be a !! *temporary* !! fix for
# https://github.com/pulp/pulp_rpm/issues/2407
pkg_pks_to_ignore = set()
latest_build_time_by_nevra = defaultdict(list)
for pkg in packages.only("pk", "name", "epoch", "version", "release",
"arch", "time_build").iterator():
latest_build_time_by_nevra[pkg.nevra].append((pkg.time_build, pkg.pk))
for nevra, pkg_data in latest_build_time_by_nevra.items():
# sort the packages by when they were built
if len(pkg_data) > 1:
pkg_data.sort(key=lambda p: p[0], reverse=True)
pkg_pks_to_ignore |= set(entry[1] for entry in pkg_data[1:])
log.warning(
"Duplicate packages found competing for NEVRA {nevra}, selected the one with "
"the most recent build time, excluding {others} others.".
format(nevra=nevra, others=len(pkg_data[1:])))
total_packages = packages.count() - len(pkg_pks_to_ignore)
pri_xml.set_num_of_pkgs(total_packages)
fil_xml.set_num_of_pkgs(total_packages)
oth_xml.set_num_of_pkgs(total_packages)
# Process all packages
for package in packages.order_by("name", "evr").iterator():
if package.pk in pkg_pks_to_ignore: # Temporary!
continue
pkg = package.to_createrepo_c()
# rewrite the checksum and checksum type with the desired ones
(checksum, pkgId) = pkg_to_hash[package.pk]
pkg.checksum_type = checksum
pkg.pkgId = pkgId
pkg_filename = os.path.basename(package.location_href)
# this can cause an issue when two same RPM package names appears
# a/name1.rpm b/name1.rpm
pkg.location_href = os.path.join(PACKAGES_DIRECTORY,
pkg_filename[0].lower(), pkg_filename)
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
if publication.sqlite_metadata:
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=content).iterator():
upd_xml.add_chunk(
cr.xml_dump_updaterecord(update_record.to_createrepo_c()))
# Process modulemd, modulemd_defaults and obsoletes
with open(mod_yml_path, "ab") as mod_yml:
for modulemd in Modulemd.objects.filter(pk__in=content).iterator():
mod_yml.write(modulemd.snippet.encode())
has_modules = True
for default in ModulemdDefaults.objects.filter(
pk__in=content).iterator():
mod_yml.write(default.snippet.encode())
has_modules = True
for obsolete in ModulemdObsolete.objects.filter(
pk__in=content).iterator():
mod_yml.write(obsolete.snippet.encode())
has_modules = True
# Process comps
comps = libcomps.Comps()
for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator():
group = pkg_grp.pkg_grp_to_libcomps()
comps.groups.append(group)
has_comps = True
for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator():
cat = pkg_cat.pkg_cat_to_libcomps()
comps.categories.append(cat)
has_comps = True
for pkg_env in PackageEnvironment.objects.filter(
pk__in=content).iterator():
env = pkg_env.pkg_env_to_libcomps()
comps.environments.append(env)
has_comps = True
for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator():
comps.langpacks = dict_to_strdict(pkg_lng.matches)
has_comps = True
comps.toxml_f(
comps_xml_path,
xml_options={
"default_explicit": True,
"empty_groups": True,
"empty_packages": True,
"uservisible_explicit": True,
},
)
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
# If the repository is empty, use a revision of 0
# See: https://pulp.plan.io/issues/9402
if not content.exists():
repomd.revision = "0"
if publication.sqlite_metadata:
repomdrecords = [
("primary", pri_xml_path, pri_db),
("filelists", fil_xml_path, fil_db),
("other", oth_xml_path, oth_db),
("primary_db", pri_db_path, None),
("filelists_db", fil_db_path, None),
("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None),
]
else:
repomdrecords = [
("primary", pri_xml_path, None),
("filelists", fil_xml_path, None),
("other", oth_xml_path, None),
("updateinfo", upd_xml_path, None),
]
if has_modules:
repomdrecords.append(("modules", mod_yml_path, None))
if has_comps:
repomdrecords.append(("group", comps_xml_path, None))
repomdrecords.extend(extra_repomdrecords)
sqlite_files = ("primary_db", "filelists_db", "other_db")
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
checksum_type = cr_checksum_type_from_string(
get_checksum_type(name,
checksum_types,
default=publication.metadata_checksum_type))
if name in sqlite_files:
record_bz = record.compress_and_fill(checksum_type, cr.BZ2)
record_bz.type = name
record_bz.rename_file()
path = record_bz.location_href.split("/")[-1]
repomd.set_record(record_bz)
else:
record.fill(checksum_type)
if db_to_update:
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
record.rename_file()
path = record.location_href.split("/")[-1]
repomd.set_record(record)
if sub_folder:
path = os.path.join(sub_folder, path)
with open(path, "rb") as repodata_fd:
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(path)),
publication=publication,
file=File(repodata_fd),
)
with open(repomd_path, "w") as repomd_f:
repomd_f.write(repomd.xml_dump())
if metadata_signing_service:
signing_service = AsciiArmoredDetachedSigningService.objects.get(
pk=metadata_signing_service)
sign_results = signing_service.sign(repomd_path)
# publish a signed file
with open(sign_results["file"], "rb") as signed_file_fd:
PublishedMetadata.create_from_file(
relative_path=os.path.join(
repodata_path, os.path.basename(sign_results["file"])),
publication=publication,
file=File(signed_file_fd),
)
# publish a detached signature
with open(sign_results["signature"], "rb") as signature_fd:
PublishedMetadata.create_from_file(
relative_path=os.path.join(
repodata_path,
os.path.basename(sign_results["signature"])),
publication=publication,
file=File(signature_fd),
)
# publish a public key required for further verification
pubkey_name = "repomd.xml.key"
with open(pubkey_name, "wb+") as f:
f.write(signing_service.public_key.encode("utf-8"))
f.flush()
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path, pubkey_name),
publication=publication,
file=File(f),
)
else:
with open(repomd_path, "rb") as repomd_fd:
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(repomd_path)),
publication=publication,
file=File(repomd_fd),
)
def create_repomd_xml(content,
publication,
checksum_types,
extra_repomdrecords,
sub_folder=None,
metadata_signing_service=None):
"""
Creates a repomd.xml file.
Args:
content(app.models.Content): content set
publication(pulpcore.plugin.models.Publication): the publication
extra_repomdrecords(list): list with data relative to repo metadata files
sub_folder(str): name of the folder for sub repos
metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService):
A reference to an associated signing service.
"""
cwd = os.getcwd()
repodata_path = REPODATA_PATH
has_modules = False
has_comps = False
package_checksum_type = checksum_types.get("package")
if sub_folder:
cwd = os.path.join(cwd, sub_folder)
repodata_path = os.path.join(sub_folder, repodata_path)
# Prepare metadata files
repomd_path = os.path.join(cwd, "repomd.xml")
pri_xml_path = os.path.join(cwd, "primary.xml.gz")
fil_xml_path = os.path.join(cwd, "filelists.xml.gz")
oth_xml_path = os.path.join(cwd, "other.xml.gz")
pri_db_path = os.path.join(cwd, "primary.sqlite")
fil_db_path = os.path.join(cwd, "filelists.sqlite")
oth_db_path = os.path.join(cwd, "other.sqlite")
upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz")
mod_yml_path = os.path.join(cwd, "modules.yaml")
comps_xml_path = os.path.join(cwd, "comps.xml")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
packages = Package.objects.filter(pk__in=content)
total_packages = packages.count()
pri_xml.set_num_of_pkgs(total_packages)
fil_xml.set_num_of_pkgs(total_packages)
oth_xml.set_num_of_pkgs(total_packages)
# Process all packages
for package in packages.iterator():
pkg = package.to_createrepo_c(package_checksum_type)
pkg_filename = os.path.basename(package.location_href)
# this can cause an issue when two same RPM package names appears
# a/name1.rpm b/name1.rpm
pkg.location_href = os.path.join(PACKAGES_DIRECTORY,
pkg_filename[0].lower(), pkg_filename)
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=content).iterator():
upd_xml.add_chunk(
cr.xml_dump_updaterecord(update_record.to_createrepo_c()))
# Process modulemd and modulemd_defaults
with open(mod_yml_path, 'ab') as mod_yml:
for modulemd in Modulemd.objects.filter(pk__in=content).iterator():
mod_yml.write(modulemd._artifacts.get().file.read())
has_modules = True
for default in ModulemdDefaults.objects.filter(
pk__in=content).iterator():
mod_yml.write(default._artifacts.get().file.read())
has_modules = True
# Process comps
comps = libcomps.Comps()
for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator():
group = pkg_grp.pkg_grp_to_libcomps()
comps.groups.append(group)
has_comps = True
for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator():
cat = pkg_cat.pkg_cat_to_libcomps()
comps.categories.append(cat)
has_comps = True
for pkg_env in PackageEnvironment.objects.filter(
pk__in=content).iterator():
env = pkg_env.pkg_env_to_libcomps()
comps.environments.append(env)
has_comps = True
for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator():
comps.langpacks = dict_to_strdict(pkg_lng.matches)
has_comps = True
comps.toxml_f(comps_xml_path,
xml_options={
"default_explicit": True,
"empty_groups": True,
"uservisible_explicit": True
})
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
repomdrecords = [("primary", pri_xml_path, pri_db),
("filelists", fil_xml_path, fil_db),
("other", oth_xml_path, oth_db),
("primary_db", pri_db_path, None),
("filelists_db", fil_db_path, None),
("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None)]
if has_modules:
repomdrecords.append(("modules", mod_yml_path, None))
if has_comps:
repomdrecords.append(("group", comps_xml_path, None))
repomdrecords.extend(extra_repomdrecords)
sqlite_files = ("primary_db", "filelists_db", "other_db")
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
checksum_type = get_checksum_type(name, checksum_types)
if name in sqlite_files:
record_bz = record.compress_and_fill(checksum_type, cr.BZ2)
record_bz.type = name
record_bz.rename_file()
path = record_bz.location_href.split('/')[-1]
repomd.set_record(record_bz)
else:
record.fill(checksum_type)
if (db_to_update):
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
record.rename_file()
path = record.location_href.split('/')[-1]
repomd.set_record(record)
if sub_folder:
path = os.path.join(sub_folder, path)
PublishedMetadata.create_from_file(relative_path=os.path.join(
repodata_path, os.path.basename(path)),
publication=publication,
file=File(open(path, 'rb')))
with open(repomd_path, "w") as repomd_f:
repomd_f.write(repomd.xml_dump())
if metadata_signing_service:
signing_service = AsciiArmoredDetachedSigningService.objects.get(
pk=metadata_signing_service.pk)
sign_results = signing_service.sign(repomd_path)
# publish a signed file
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(sign_results['file'])),
publication=publication,
file=File(open(sign_results['file'], 'rb')))
# publish a detached signature
PublishedMetadata.create_from_file(relative_path=os.path.join(
repodata_path, os.path.basename(sign_results['signature'])),
publication=publication,
file=File(
open(sign_results['signature'],
'rb')))
# publish a public key required for further verification
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(sign_results['key'])),
publication=publication,
file=File(open(sign_results['key'], 'rb')))
else:
PublishedMetadata.create_from_file(relative_path=os.path.join(
repodata_path, os.path.basename(repomd_path)),
publication=publication,
file=File(open(repomd_path, 'rb')))
