def test_xml_dump_updaterecord_no_updated_date(self): now = datetime.now() # Microseconds are always 0 in updateinfo now = datetime(now.year, now.month, now.day, now.hour, now.minute, now.second, 0) rec = cr.UpdateRecord() rec.fromstr = "from" rec.status = "status" rec.type = "type" rec.version = "version" rec.id = "id" rec.title = "title" rec.issued_date = now rec.rights = "rights" rec.release = "release" rec.pushcount = "pushcount" rec.severity = "severity" rec.summary = "summary" rec.description = "description" rec.solution = "solution" rec.reboot_suggested = True target_xml = \ """ <update from="from" status="status" type="type" version="version"> <id>id</id> <title>title</title> <issued date="%(now)s"/> <rights>rights</rights> <release>release</release> <pushcount>pushcount</pushcount> <severity>severity</severity> <summary>summary</summary> <description>description</description> <solution>solution</solution> <reboot_suggested>True</reboot_suggested> <references/> <pkglist/> </update> """ % {"now": now.strftime("%Y-%m-%d %H:%M:%S")} xml = cr.xml_dump_updaterecord(rec) self.assertEqual(xml, target_xml) # Setting it to None is the same as not setting it at all rec.updated_date = None xml = cr.xml_dump_updaterecord(rec) self.assertEqual(xml, target_xml)
def update_record_xml(update_record): """ Return xml for an UpdateRecord. Args: update_record (app.models.UpdateRecord): create xml from this record Returns: str: xml for the UpdateRecord """ rec = cr.UpdateRecord() rec.fromstr = update_record.fromstr rec.status = update_record.status rec.type = update_record.type rec.version = update_record.version rec.id = update_record.id rec.title = update_record.title rec.issued_date = parse_datetime(update_record.issued_date) rec.updated_date = parse_datetime(update_record.updated_date) rec.rights = update_record.rights rec.summary = update_record.summary rec.description = update_record.description for collection in update_record.collections.all(): col = cr.UpdateCollection() col.shortname = collection.shortname col.name = collection.name for package in collection.packages.all(): pkg = cr.UpdateCollectionPackage() pkg.name = package.name pkg.version = package.version pkg.release = package.release pkg.epoch = package.epoch pkg.arch = package.arch pkg.src = package.src pkg.filename = package.filename pkg.reboot_suggested = package.reboot_suggested if package.sum: pkg.sum = package.sum pkg.sum_type = int(package.sum_type or 0) col.append(pkg) rec.append_collection(col) for reference in update_record.references.all(): ref = cr.UpdateReference() ref.href = reference.href ref.id = reference.ref_id ref.type = reference.ref_type ref.title = reference.title rec.append_reference(ref) return cr.xml_dump_updaterecord(rec)
def update_record_xml(update_record): """ Return xml for an UpdateRecord. Args: update_record (app.models.UpdateRecord): create xml from this record Returns: str: xml for the UpdateRecord """ rec = cr.UpdateRecord() rec.fromstr = update_record.fromstr rec.status = update_record.status rec.type = update_record.type rec.version = update_record.version rec.id = update_record.id rec.title = update_record.title rec.issued_date = parse_datetime(update_record.issued_date) rec.updated_date = parse_datetime(update_record.updated_date) rec.rights = update_record.rights rec.summary = update_record.summary rec.description = update_record.description for collection in update_record.collections.all(): col = cr.UpdateCollection() col.shortname = collection.shortname col.name = collection.name for package in collection.packages.all(): pkg = cr.UpdateCollectionPackage() pkg.name = package.name pkg.version = package.version pkg.release = package.release pkg.epoch = package.epoch pkg.arch = package.arch pkg.src = package.src pkg.filename = package.filename pkg.reboot_suggested = package.reboot_suggested if package.sum: pkg.sum = package.sum pkg.sum_type = int(package.sum_type or 0) col.append(pkg) rec.append_collection(col) for reference in update_record.references.all(): ref = cr.UpdateReference() ref.href = reference.href ref.id = reference.ref_id ref.type = reference.ref_type ref.title = reference.title rec.append_reference(ref) return cr.xml_dump_updaterecord(rec)
def generate(): pkg = cr.UpdateCollectionPackage() pkg.name = "Foo" pkg.version = "1.2.3" pkg.release = "1" pkg.epoch = "0" pkg.arch = "noarch" pkg.src = "foo.src.rpm" pkg.filename = "foo-1.2.3-1.rpm" pkg.sum = "123456789" pkg.sum_type = cr.MD5 pkg.reboot_suggested = False col = cr.UpdateCollection() col.shortname = "Bar-product" col.name = "Bar Product" col.append(pkg) ref = cr.UpdateReference() ref.href = "http://foo.bar/foobar" ref.id = "123" ref.type = "self" ref.title = "Foo Update" rec = cr.UpdateRecord() rec.fromstr = "*****@*****.**" rec.status = "final" rec.type = "enhancement" rec.version = "1" rec.id = "UPDATE-1" rec.title = "Bar Product Update" rec.issued_date = "2014-08-14 00:00:00" rec.updated_date = "2014-08-14 00:00:00" rec.rights = "Copyright 2014 Bar Inc" rec.summary = "An update for Bar" rec.description = "Fixes a bug" rec.append_collection(col) rec.append_reference(ref) chunk = cr.xml_dump_updaterecord(rec) f = cr.UpdateInfoXmlFile(OUT_FILE) f.add_chunk(chunk) f.close() print "See the %s" % OUT_FILE
def generate(): pkg = cr.UpdateCollectionPackage() pkg.name = "Foo" pkg.version = "1.2.3" pkg.release = "1" pkg.epoch = "0" pkg.arch = "noarch" pkg.src = "foo.src.rpm" pkg.filename = "foo-1.2.3-1.rpm" pkg.sum = "123456789" pkg.sum_type = cr.MD5 pkg.reboot_suggested = False col = cr.UpdateCollection() col.shortname = "Bar-product" col.name = "Bar Product" col.append(pkg) ref = cr.UpdateReference() ref.href = "http://foo.bar/foobar" ref.id = "123" ref.type = "self" ref.title = "Foo Update" rec = cr.UpdateRecord() rec.fromstr = "*****@*****.**" rec.status = "final" rec.type = "enhancement" rec.version = "1" rec.id = "UPDATE-1" rec.title = "Bar Product Update" rec.issued_date = datetime.datetime(2014, 8, 14) rec.updated_date = datetime.datetime(2014, 8, 14) rec.rights = "Copyright 2014 Bar Inc" rec.summary = "An update for Bar" rec.description = "Fixes a bug" rec.append_collection(col) rec.append_reference(ref) chunk = cr.xml_dump_updaterecord(rec) f = cr.UpdateInfoXmlFile(OUT_FILE) f.add_chunk(chunk) f.close() print "See the %s" % OUT_FILE
def test_xml_dump_updaterecord(self): now = datetime.now() # Microseconds are always 0 in updateinfo now = datetime(now.year, now.month, now.day, now.hour, now.minute, now.second, 0) rec = cr.UpdateRecord() rec.fromstr = "from" rec.status = "status" rec.type = "type" rec.version = "version" rec.id = "id" rec.title = "title" rec.issued_date = now rec.updated_date = now rec.rights = "rights" rec.release = "release" rec.pushcount = "pushcount" rec.severity = "severity" rec.summary = "summary" rec.description = "description" rec.solution = "solution" xml = cr.xml_dump_updaterecord(rec) self.assertEqual( xml, """ <update from="from" status="status" type="type" version="version"> <id>id</id> <title>title</title> <issued date="%(now)s"/> <updated date="%(now)s"/> <rights>rights</rights> <release>release</release> <pushcount>pushcount</pushcount> <severity>severity</severity> <summary>summary</summary> <description>description</description> <solution>solution</solution> <references/> <pkglist/> </update> """ % {"now": now.strftime("%Y-%m-%d %H:%M:%S")})
def create_repomd_xml( content, publication, checksum_types, extra_repomdrecords, sub_folder=None, metadata_signing_service=None, ): """ Creates a repomd.xml file. Args: content(app.models.Content): content set publication(pulpcore.plugin.models.Publication): the publication extra_repomdrecords(list): list with data relative to repo metadata files sub_folder(str): name of the folder for sub repos metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService): A reference to an associated signing service. """ cwd = os.getcwd() repodata_path = REPODATA_PATH has_modules = False has_comps = False package_checksum_type = checksum_types.get("package") if sub_folder: cwd = os.path.join(cwd, sub_folder) repodata_path = os.path.join(sub_folder, repodata_path) # Prepare metadata files repomd_path = os.path.join(cwd, "repomd.xml") pri_xml_path = os.path.join(cwd, "primary.xml.gz") fil_xml_path = os.path.join(cwd, "filelists.xml.gz") oth_xml_path = os.path.join(cwd, "other.xml.gz") upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz") mod_yml_path = os.path.join(cwd, "modules.yaml") comps_xml_path = os.path.join(cwd, "comps.xml") pri_xml = cr.PrimaryXmlFile(pri_xml_path) fil_xml = cr.FilelistsXmlFile(fil_xml_path) oth_xml = cr.OtherXmlFile(oth_xml_path) upd_xml = cr.UpdateInfoXmlFile(upd_xml_path) if publication.sqlite_metadata: pri_db_path = os.path.join(cwd, "primary.sqlite") fil_db_path = os.path.join(cwd, "filelists.sqlite") oth_db_path = os.path.join(cwd, "other.sqlite") pri_db = cr.PrimarySqlite(pri_db_path) fil_db = cr.FilelistsSqlite(fil_db_path) oth_db = cr.OtherSqlite(oth_db_path) packages = Package.objects.filter(pk__in=content) total_packages = packages.count() pri_xml.set_num_of_pkgs(total_packages) fil_xml.set_num_of_pkgs(total_packages) oth_xml.set_num_of_pkgs(total_packages) # We want to support publishing with a different checksum type than the one built-in to the # package itself, so we need to get the correct checksums somehow if there is an override. # We must also take into consideration that if the package has not been downloaded the only # checksum that is available is the one built-in. # # Since this lookup goes from Package->Content->ContentArtifact->Artifact, performance is a # challenge. We use ContentArtifact as our starting point because it enables us to work with # simple foreign keys and avoid messing with the many-to-many relationship, which doesn't # work with select_related() and performs poorly with prefetch_related(). This is fine # because we know that Packages should only ever have one artifact per content. contentartifact_qs = ( ContentArtifact.objects.filter(content__in=packages.only("pk")). select_related( # content__rpm_package is a bit of a hack, exploiting the way django sets up model # inheritance, but it works and is unlikely to break. All content artifacts being # accessed here have an associated Package since they originally came from the # Package queryset. "artifact", "content__rpm_package", ).only("artifact", "content__rpm_package__checksum_type", "content__rpm_package__pkgId")) pkg_to_hash = {} for ca in contentartifact_qs.iterator(): pkgid = None if package_checksum_type: package_checksum_type = package_checksum_type.lower() pkgid = getattr(ca.artifact, package_checksum_type, None) if pkgid: pkg_to_hash[ca.content_id] = (package_checksum_type, pkgid) else: pkg_to_hash[ca.content_id] = ( ca.content.rpm_package.checksum_type, ca.content.rpm_package.pkgId, ) # Process all packages for package in packages.iterator(): pkg = package.to_createrepo_c() # rewrite the checksum and checksum type with the desired ones (checksum, pkgId) = pkg_to_hash[package.pk] pkg.checksum_type = checksum pkg.pkgId = pkgId pkg_filename = os.path.basename(package.location_href) # this can cause an issue when two same RPM package names appears # a/name1.rpm b/name1.rpm pkg.location_href = os.path.join(PACKAGES_DIRECTORY, pkg_filename[0].lower(), pkg_filename) pri_xml.add_pkg(pkg) fil_xml.add_pkg(pkg) oth_xml.add_pkg(pkg) if publication.sqlite_metadata: pri_db.add_pkg(pkg) fil_db.add_pkg(pkg) oth_db.add_pkg(pkg) # Process update records for update_record in UpdateRecord.objects.filter( pk__in=content).iterator(): upd_xml.add_chunk( cr.xml_dump_updaterecord(update_record.to_createrepo_c())) # Process modulemd and modulemd_defaults with open(mod_yml_path, "ab") as mod_yml: for modulemd in Modulemd.objects.filter(pk__in=content).iterator(): mod_yml.write(modulemd._artifacts.get().file.read()) has_modules = True for default in ModulemdDefaults.objects.filter( pk__in=content).iterator(): mod_yml.write(default._artifacts.get().file.read()) has_modules = True # Process comps comps = libcomps.Comps() for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator(): group = pkg_grp.pkg_grp_to_libcomps() comps.groups.append(group) has_comps = True for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator(): cat = pkg_cat.pkg_cat_to_libcomps() comps.categories.append(cat) has_comps = True for pkg_env in PackageEnvironment.objects.filter( pk__in=content).iterator(): env = pkg_env.pkg_env_to_libcomps() comps.environments.append(env) has_comps = True for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator(): comps.langpacks = dict_to_strdict(pkg_lng.matches) has_comps = True comps.toxml_f( comps_xml_path, xml_options={ "default_explicit": True, "empty_groups": True, "uservisible_explicit": True }, ) pri_xml.close() fil_xml.close() oth_xml.close() upd_xml.close() repomd = cr.Repomd() if publication.sqlite_metadata: repomdrecords = [ ("primary", pri_xml_path, pri_db), ("filelists", fil_xml_path, fil_db), ("other", oth_xml_path, oth_db), ("primary_db", pri_db_path, None), ("filelists_db", fil_db_path, None), ("other_db", oth_db_path, None), ("updateinfo", upd_xml_path, None), ] else: repomdrecords = [ ("primary", pri_xml_path, None), ("filelists", fil_xml_path, None), ("other", oth_xml_path, None), ("updateinfo", upd_xml_path, None), ] if has_modules: repomdrecords.append(("modules", mod_yml_path, None)) if has_comps: repomdrecords.append(("group", comps_xml_path, None)) repomdrecords.extend(extra_repomdrecords) sqlite_files = ("primary_db", "filelists_db", "other_db") for name, path, db_to_update in repomdrecords: record = cr.RepomdRecord(name, path) checksum_type = get_checksum_type(name, checksum_types) if name in sqlite_files: record_bz = record.compress_and_fill(checksum_type, cr.BZ2) record_bz.type = name record_bz.rename_file() path = record_bz.location_href.split("/")[-1] repomd.set_record(record_bz) else: record.fill(checksum_type) if db_to_update: db_to_update.dbinfo_update(record.checksum) db_to_update.close() record.rename_file() path = record.location_href.split("/")[-1] repomd.set_record(record) if sub_folder: path = os.path.join(sub_folder, path) PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(path)), publication=publication, file=File(open(path, "rb")), ) with open(repomd_path, "w") as repomd_f: repomd_f.write(repomd.xml_dump()) if metadata_signing_service: signing_service = AsciiArmoredDetachedSigningService.objects.get( pk=metadata_signing_service.pk) sign_results = signing_service.sign(repomd_path) # publish a signed file PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(sign_results["file"])), publication=publication, file=File(open(sign_results["file"], "rb")), ) # publish a detached signature PublishedMetadata.create_from_file( relative_path=os.path.join( repodata_path, os.path.basename(sign_results["signature"])), publication=publication, file=File(open(sign_results["signature"], "rb")), ) # publish a public key required for further verification PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(sign_results["key"])), publication=publication, file=File(open(sign_results["key"], "rb")), ) else: PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(repomd_path)), publication=publication, file=File(open(repomd_path, "rb")), )
def create_repomd_xml(content, publication, extra_repomdrecords, sub_folder=None): """ Creates a repomd.xml file. Args: content(app.models.Content): content set publication(pulpcore.plugin.models.Publication): the publication extra_repomdrecords(list): list with data relative to repo metadata files sub_folder(str): name of the folder for sub repos """ cwd = os.getcwd() repodata_path = REPODATA_PATH has_modules = False has_comps = False if sub_folder: cwd = os.path.join(cwd, sub_folder) repodata_path = os.path.join(sub_folder, repodata_path) # Prepare metadata files repomd_path = os.path.join(cwd, "repomd.xml") pri_xml_path = os.path.join(cwd, "primary.xml.gz") fil_xml_path = os.path.join(cwd, "filelists.xml.gz") oth_xml_path = os.path.join(cwd, "other.xml.gz") pri_db_path = os.path.join(cwd, "primary.sqlite") fil_db_path = os.path.join(cwd, "filelists.sqlite") oth_db_path = os.path.join(cwd, "other.sqlite") upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz") mod_yml_path = os.path.join(cwd, "modules.yaml") comps_xml_path = os.path.join(cwd, "comps.xml") pri_xml = cr.PrimaryXmlFile(pri_xml_path) fil_xml = cr.FilelistsXmlFile(fil_xml_path) oth_xml = cr.OtherXmlFile(oth_xml_path) pri_db = cr.PrimarySqlite(pri_db_path) fil_db = cr.FilelistsSqlite(fil_db_path) oth_db = cr.OtherSqlite(oth_db_path) upd_xml = cr.UpdateInfoXmlFile(upd_xml_path) packages = Package.objects.filter(pk__in=content) total_packages = packages.count() pri_xml.set_num_of_pkgs(total_packages) fil_xml.set_num_of_pkgs(total_packages) oth_xml.set_num_of_pkgs(total_packages) # Process all packages for package in packages.iterator(): pkg = package.to_createrepo_c() pkg.location_href = package.contentartifact_set.only('relative_path').first().relative_path pri_xml.add_pkg(pkg) fil_xml.add_pkg(pkg) oth_xml.add_pkg(pkg) pri_db.add_pkg(pkg) fil_db.add_pkg(pkg) oth_db.add_pkg(pkg) # Process update records for update_record in UpdateRecord.objects.filter(pk__in=content).iterator(): upd_xml.add_chunk(cr.xml_dump_updaterecord(update_record.to_createrepo_c())) # Process modulemd and modulemd_defaults with open(mod_yml_path, 'ab') as mod_yml: for modulemd in Modulemd.objects.filter(pk__in=content).iterator(): mod_yml.write(modulemd._artifacts.get().file.read()) has_modules = True for default in ModulemdDefaults.objects.filter(pk__in=content).iterator(): mod_yml.write(default._artifacts.get().file.read()) has_modules = True # Process comps comps = libcomps.Comps() for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator(): group = pkg_grp.pkg_grp_to_libcomps() comps.groups.append(group) has_comps = True for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator(): cat = pkg_cat.pkg_cat_to_libcomps() comps.categories.append(cat) has_comps = True for pkg_env in PackageEnvironment.objects.filter(pk__in=content).iterator(): env = pkg_env.pkg_env_to_libcomps() comps.environments.append(env) has_comps = True for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator(): comps.langpacks = dict_to_strdict(pkg_lng.matches) has_comps = True comps.toxml_f(comps_xml_path, xml_options={"default_explicit": True, "empty_groups": True, "uservisible_explicit": True}) pri_xml.close() fil_xml.close() oth_xml.close() upd_xml.close() repomd = cr.Repomd() repomdrecords = [("primary", pri_xml_path, pri_db), ("filelists", fil_xml_path, fil_db), ("other", oth_xml_path, oth_db), ("primary_db", pri_db_path, None), ("filelists_db", fil_db_path, None), ("other_db", oth_db_path, None), ("updateinfo", upd_xml_path, None)] if has_modules: repomdrecords.append(("modules", mod_yml_path, None)) if has_comps: repomdrecords.append(("group", comps_xml_path, None)) repomdrecords.extend(extra_repomdrecords) sqlite_files = ("primary_db", "filelists_db", "other_db") for name, path, db_to_update in repomdrecords: record = cr.RepomdRecord(name, path) if name in sqlite_files: record_bz = record.compress_and_fill(cr.SHA256, cr.BZ2) record_bz.type = name record_bz.rename_file() path = record_bz.location_href.split('/')[-1] repomd.set_record(record_bz) else: record.fill(cr.SHA256) if (db_to_update): db_to_update.dbinfo_update(record.checksum) db_to_update.close() record.rename_file() path = record.location_href.split('/')[-1] repomd.set_record(record) if sub_folder: path = os.path.join(sub_folder, path) PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(path)), publication=publication, file=File(open(path, 'rb')) ) with open(repomd_path, "w") as repomd_f: repomd_f.write(repomd.xml_dump()) PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(repomd_path)), publication=publication, file=File(open(repomd_path, 'rb')) )
def generate_repo_metadata( content, publication, checksum_types, extra_repomdrecords, sub_folder=None, metadata_signing_service=None, ): """ Creates a repomd.xml file. Args: content(app.models.Content): content set publication(pulpcore.plugin.models.Publication): the publication extra_repomdrecords(list): list with data relative to repo metadata files sub_folder(str): name of the folder for sub repos metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService): A reference to an associated signing service. """ cwd = os.getcwd() repodata_path = REPODATA_PATH has_modules = False has_comps = False package_checksum_type = checksum_types.get("package") if sub_folder: cwd = os.path.join(cwd, sub_folder) repodata_path = os.path.join(sub_folder, repodata_path) if package_checksum_type and package_checksum_type not in settings.ALLOWED_CONTENT_CHECKSUMS: raise ValueError( "Repository contains disallowed package checksum type '{}', " "thus can't be published. {}".format(package_checksum_type, ALLOWED_CHECKSUM_ERROR_MSG)) # Prepare metadata files repomd_path = os.path.join(cwd, "repomd.xml") pri_xml_path = os.path.join(cwd, "primary.xml.gz") fil_xml_path = os.path.join(cwd, "filelists.xml.gz") oth_xml_path = os.path.join(cwd, "other.xml.gz") upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz") mod_yml_path = os.path.join(cwd, "modules.yaml") comps_xml_path = os.path.join(cwd, "comps.xml") pri_xml = cr.PrimaryXmlFile(pri_xml_path) fil_xml = cr.FilelistsXmlFile(fil_xml_path) oth_xml = cr.OtherXmlFile(oth_xml_path) upd_xml = cr.UpdateInfoXmlFile(upd_xml_path) if publication.sqlite_metadata: pri_db_path = os.path.join(cwd, "primary.sqlite") fil_db_path = os.path.join(cwd, "filelists.sqlite") oth_db_path = os.path.join(cwd, "other.sqlite") pri_db = cr.PrimarySqlite(pri_db_path) fil_db = cr.FilelistsSqlite(fil_db_path) oth_db = cr.OtherSqlite(oth_db_path) packages = Package.objects.filter(pk__in=content) # We want to support publishing with a different checksum type than the one built-in to the # package itself, so we need to get the correct checksums somehow if there is an override. # We must also take into consideration that if the package has not been downloaded the only # checksum that is available is the one built-in. # # Since this lookup goes from Package->Content->ContentArtifact->Artifact, performance is a # challenge. We use ContentArtifact as our starting point because it enables us to work with # simple foreign keys and avoid messing with the many-to-many relationship, which doesn't # work with select_related() and performs poorly with prefetch_related(). This is fine # because we know that Packages should only ever have one artifact per content. contentartifact_qs = ( ContentArtifact.objects.filter(content__in=packages.only("pk")). select_related( # content__rpm_package is a bit of a hack, exploiting the way django sets up model # inheritance, but it works and is unlikely to break. All content artifacts being # accessed here have an associated Package since they originally came from the # Package queryset. "artifact", "content__rpm_package", ).only("artifact", "content__rpm_package__checksum_type", "content__rpm_package__pkgId")) pkg_to_hash = {} for ca in contentartifact_qs.iterator(): if package_checksum_type: package_checksum_type = package_checksum_type.lower() pkgid = getattr(ca.artifact, package_checksum_type, None) if not package_checksum_type or not pkgid: if ca.content.rpm_package.checksum_type not in settings.ALLOWED_CONTENT_CHECKSUMS: raise ValueError( "Package {} as content unit {} contains forbidden checksum type '{}', " "thus can't be published. {}".format( ca.content.rpm_package.nevra, ca.content.pk, ca.content.rpm_package.checksum_type, ALLOWED_CHECKSUM_ERROR_MSG, )) package_checksum_type = ca.content.rpm_package.checksum_type pkgid = ca.content.rpm_package.pkgId pkg_to_hash[ca.content_id] = (package_checksum_type, pkgid) # TODO: this is meant to be a !! *temporary* !! fix for # https://github.com/pulp/pulp_rpm/issues/2407 pkg_pks_to_ignore = set() latest_build_time_by_nevra = defaultdict(list) for pkg in packages.only("pk", "name", "epoch", "version", "release", "arch", "time_build").iterator(): latest_build_time_by_nevra[pkg.nevra].append((pkg.time_build, pkg.pk)) for nevra, pkg_data in latest_build_time_by_nevra.items(): # sort the packages by when they were built if len(pkg_data) > 1: pkg_data.sort(key=lambda p: p[0], reverse=True) pkg_pks_to_ignore |= set(entry[1] for entry in pkg_data[1:]) log.warning( "Duplicate packages found competing for NEVRA {nevra}, selected the one with " "the most recent build time, excluding {others} others.". format(nevra=nevra, others=len(pkg_data[1:]))) total_packages = packages.count() - len(pkg_pks_to_ignore) pri_xml.set_num_of_pkgs(total_packages) fil_xml.set_num_of_pkgs(total_packages) oth_xml.set_num_of_pkgs(total_packages) # Process all packages for package in packages.order_by("name", "evr").iterator(): if package.pk in pkg_pks_to_ignore: # Temporary! continue pkg = package.to_createrepo_c() # rewrite the checksum and checksum type with the desired ones (checksum, pkgId) = pkg_to_hash[package.pk] pkg.checksum_type = checksum pkg.pkgId = pkgId pkg_filename = os.path.basename(package.location_href) # this can cause an issue when two same RPM package names appears # a/name1.rpm b/name1.rpm pkg.location_href = os.path.join(PACKAGES_DIRECTORY, pkg_filename[0].lower(), pkg_filename) pri_xml.add_pkg(pkg) fil_xml.add_pkg(pkg) oth_xml.add_pkg(pkg) if publication.sqlite_metadata: pri_db.add_pkg(pkg) fil_db.add_pkg(pkg) oth_db.add_pkg(pkg) # Process update records for update_record in UpdateRecord.objects.filter( pk__in=content).iterator(): upd_xml.add_chunk( cr.xml_dump_updaterecord(update_record.to_createrepo_c())) # Process modulemd, modulemd_defaults and obsoletes with open(mod_yml_path, "ab") as mod_yml: for modulemd in Modulemd.objects.filter(pk__in=content).iterator(): mod_yml.write(modulemd.snippet.encode()) has_modules = True for default in ModulemdDefaults.objects.filter( pk__in=content).iterator(): mod_yml.write(default.snippet.encode()) has_modules = True for obsolete in ModulemdObsolete.objects.filter( pk__in=content).iterator(): mod_yml.write(obsolete.snippet.encode()) has_modules = True # Process comps comps = libcomps.Comps() for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator(): group = pkg_grp.pkg_grp_to_libcomps() comps.groups.append(group) has_comps = True for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator(): cat = pkg_cat.pkg_cat_to_libcomps() comps.categories.append(cat) has_comps = True for pkg_env in PackageEnvironment.objects.filter( pk__in=content).iterator(): env = pkg_env.pkg_env_to_libcomps() comps.environments.append(env) has_comps = True for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator(): comps.langpacks = dict_to_strdict(pkg_lng.matches) has_comps = True comps.toxml_f( comps_xml_path, xml_options={ "default_explicit": True, "empty_groups": True, "empty_packages": True, "uservisible_explicit": True, }, ) pri_xml.close() fil_xml.close() oth_xml.close() upd_xml.close() repomd = cr.Repomd() # If the repository is empty, use a revision of 0 # See: https://pulp.plan.io/issues/9402 if not content.exists(): repomd.revision = "0" if publication.sqlite_metadata: repomdrecords = [ ("primary", pri_xml_path, pri_db), ("filelists", fil_xml_path, fil_db), ("other", oth_xml_path, oth_db), ("primary_db", pri_db_path, None), ("filelists_db", fil_db_path, None), ("other_db", oth_db_path, None), ("updateinfo", upd_xml_path, None), ] else: repomdrecords = [ ("primary", pri_xml_path, None), ("filelists", fil_xml_path, None), ("other", oth_xml_path, None), ("updateinfo", upd_xml_path, None), ] if has_modules: repomdrecords.append(("modules", mod_yml_path, None)) if has_comps: repomdrecords.append(("group", comps_xml_path, None)) repomdrecords.extend(extra_repomdrecords) sqlite_files = ("primary_db", "filelists_db", "other_db") for name, path, db_to_update in repomdrecords: record = cr.RepomdRecord(name, path) checksum_type = cr_checksum_type_from_string( get_checksum_type(name, checksum_types, default=publication.metadata_checksum_type)) if name in sqlite_files: record_bz = record.compress_and_fill(checksum_type, cr.BZ2) record_bz.type = name record_bz.rename_file() path = record_bz.location_href.split("/")[-1] repomd.set_record(record_bz) else: record.fill(checksum_type) if db_to_update: db_to_update.dbinfo_update(record.checksum) db_to_update.close() record.rename_file() path = record.location_href.split("/")[-1] repomd.set_record(record) if sub_folder: path = os.path.join(sub_folder, path) with open(path, "rb") as repodata_fd: PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(path)), publication=publication, file=File(repodata_fd), ) with open(repomd_path, "w") as repomd_f: repomd_f.write(repomd.xml_dump()) if metadata_signing_service: signing_service = AsciiArmoredDetachedSigningService.objects.get( pk=metadata_signing_service) sign_results = signing_service.sign(repomd_path) # publish a signed file with open(sign_results["file"], "rb") as signed_file_fd: PublishedMetadata.create_from_file( relative_path=os.path.join( repodata_path, os.path.basename(sign_results["file"])), publication=publication, file=File(signed_file_fd), ) # publish a detached signature with open(sign_results["signature"], "rb") as signature_fd: PublishedMetadata.create_from_file( relative_path=os.path.join( repodata_path, os.path.basename(sign_results["signature"])), publication=publication, file=File(signature_fd), ) # publish a public key required for further verification pubkey_name = "repomd.xml.key" with open(pubkey_name, "wb+") as f: f.write(signing_service.public_key.encode("utf-8")) f.flush() PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, pubkey_name), publication=publication, file=File(f), ) else: with open(repomd_path, "rb") as repomd_fd: PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(repomd_path)), publication=publication, file=File(repomd_fd), )
def create_repomd_xml(content, publication, checksum_types, extra_repomdrecords, sub_folder=None, metadata_signing_service=None): """ Creates a repomd.xml file. Args: content(app.models.Content): content set publication(pulpcore.plugin.models.Publication): the publication extra_repomdrecords(list): list with data relative to repo metadata files sub_folder(str): name of the folder for sub repos metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService): A reference to an associated signing service. """ cwd = os.getcwd() repodata_path = REPODATA_PATH has_modules = False has_comps = False package_checksum_type = checksum_types.get("package") if sub_folder: cwd = os.path.join(cwd, sub_folder) repodata_path = os.path.join(sub_folder, repodata_path) # Prepare metadata files repomd_path = os.path.join(cwd, "repomd.xml") pri_xml_path = os.path.join(cwd, "primary.xml.gz") fil_xml_path = os.path.join(cwd, "filelists.xml.gz") oth_xml_path = os.path.join(cwd, "other.xml.gz") pri_db_path = os.path.join(cwd, "primary.sqlite") fil_db_path = os.path.join(cwd, "filelists.sqlite") oth_db_path = os.path.join(cwd, "other.sqlite") upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz") mod_yml_path = os.path.join(cwd, "modules.yaml") comps_xml_path = os.path.join(cwd, "comps.xml") pri_xml = cr.PrimaryXmlFile(pri_xml_path) fil_xml = cr.FilelistsXmlFile(fil_xml_path) oth_xml = cr.OtherXmlFile(oth_xml_path) pri_db = cr.PrimarySqlite(pri_db_path) fil_db = cr.FilelistsSqlite(fil_db_path) oth_db = cr.OtherSqlite(oth_db_path) upd_xml = cr.UpdateInfoXmlFile(upd_xml_path) packages = Package.objects.filter(pk__in=content) total_packages = packages.count() pri_xml.set_num_of_pkgs(total_packages) fil_xml.set_num_of_pkgs(total_packages) oth_xml.set_num_of_pkgs(total_packages) # Process all packages for package in packages.iterator(): pkg = package.to_createrepo_c(package_checksum_type) pkg_filename = os.path.basename(package.location_href) # this can cause an issue when two same RPM package names appears # a/name1.rpm b/name1.rpm pkg.location_href = os.path.join(PACKAGES_DIRECTORY, pkg_filename[0].lower(), pkg_filename) pri_xml.add_pkg(pkg) fil_xml.add_pkg(pkg) oth_xml.add_pkg(pkg) pri_db.add_pkg(pkg) fil_db.add_pkg(pkg) oth_db.add_pkg(pkg) # Process update records for update_record in UpdateRecord.objects.filter( pk__in=content).iterator(): upd_xml.add_chunk( cr.xml_dump_updaterecord(update_record.to_createrepo_c())) # Process modulemd and modulemd_defaults with open(mod_yml_path, 'ab') as mod_yml: for modulemd in Modulemd.objects.filter(pk__in=content).iterator(): mod_yml.write(modulemd._artifacts.get().file.read()) has_modules = True for default in ModulemdDefaults.objects.filter( pk__in=content).iterator(): mod_yml.write(default._artifacts.get().file.read()) has_modules = True # Process comps comps = libcomps.Comps() for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator(): group = pkg_grp.pkg_grp_to_libcomps() comps.groups.append(group) has_comps = True for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator(): cat = pkg_cat.pkg_cat_to_libcomps() comps.categories.append(cat) has_comps = True for pkg_env in PackageEnvironment.objects.filter( pk__in=content).iterator(): env = pkg_env.pkg_env_to_libcomps() comps.environments.append(env) has_comps = True for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator(): comps.langpacks = dict_to_strdict(pkg_lng.matches) has_comps = True comps.toxml_f(comps_xml_path, xml_options={ "default_explicit": True, "empty_groups": True, "uservisible_explicit": True }) pri_xml.close() fil_xml.close() oth_xml.close() upd_xml.close() repomd = cr.Repomd() repomdrecords = [("primary", pri_xml_path, pri_db), ("filelists", fil_xml_path, fil_db), ("other", oth_xml_path, oth_db), ("primary_db", pri_db_path, None), ("filelists_db", fil_db_path, None), ("other_db", oth_db_path, None), ("updateinfo", upd_xml_path, None)] if has_modules: repomdrecords.append(("modules", mod_yml_path, None)) if has_comps: repomdrecords.append(("group", comps_xml_path, None)) repomdrecords.extend(extra_repomdrecords) sqlite_files = ("primary_db", "filelists_db", "other_db") for name, path, db_to_update in repomdrecords: record = cr.RepomdRecord(name, path) checksum_type = get_checksum_type(name, checksum_types) if name in sqlite_files: record_bz = record.compress_and_fill(checksum_type, cr.BZ2) record_bz.type = name record_bz.rename_file() path = record_bz.location_href.split('/')[-1] repomd.set_record(record_bz) else: record.fill(checksum_type) if (db_to_update): db_to_update.dbinfo_update(record.checksum) db_to_update.close() record.rename_file() path = record.location_href.split('/')[-1] repomd.set_record(record) if sub_folder: path = os.path.join(sub_folder, path) PublishedMetadata.create_from_file(relative_path=os.path.join( repodata_path, os.path.basename(path)), publication=publication, file=File(open(path, 'rb'))) with open(repomd_path, "w") as repomd_f: repomd_f.write(repomd.xml_dump()) if metadata_signing_service: signing_service = AsciiArmoredDetachedSigningService.objects.get( pk=metadata_signing_service.pk) sign_results = signing_service.sign(repomd_path) # publish a signed file PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(sign_results['file'])), publication=publication, file=File(open(sign_results['file'], 'rb'))) # publish a detached signature PublishedMetadata.create_from_file(relative_path=os.path.join( repodata_path, os.path.basename(sign_results['signature'])), publication=publication, file=File( open(sign_results['signature'], 'rb'))) # publish a public key required for further verification PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(sign_results['key'])), publication=publication, file=File(open(sign_results['key'], 'rb'))) else: PublishedMetadata.create_from_file(relative_path=os.path.join( repodata_path, os.path.basename(repomd_path)), publication=publication, file=File(open(repomd_path, 'rb')))