def test_sqlite_primary_schema(self): path = os.path.join(self.tmpdir, "primary.db") cr.PrimarySqlite(path) self.assertTrue(os.path.isfile(path)) con = sqlite3.connect(path) # Check tables self.assertEqual( con.execute( """select name from sqlite_master where type="table";"""). fetchall(), [ (u'db_info', ), (u'packages', ), (u'files', ), (u'requires', ), (u'provides', ), (u'conflicts', ), (u'obsoletes', ), (u'suggests', ), (u'enhances', ), (u'recommends', ), (u'supplements', ), ]) # Check indexes self.assertEqual( con.execute( """select name from sqlite_master where type="index";"""). fetchall(), [ (u'packagename', ), (u'packageId', ), (u'filenames', ), (u'pkgfiles', ), (u'pkgrequires', ), (u'requiresname', ), (u'pkgprovides', ), (u'providesname', ), (u'pkgconflicts', ), (u'pkgobsoletes', ), (u'pkgsuggests', ), (u'pkgenhances', ), (u'pkgrecommends', ), (u'pkgsupplements', ), ]) # Check triggers self.assertEqual( con.execute( """select name from sqlite_master where type="trigger";"""). fetchall(), [(u'removals', )])
def test_sqlite_basic_operations(self): db_pri = cr.Sqlite(self.tmpdir + "/primary.db", cr.DB_PRIMARY) self.assertTrue(db_pri) self.assertTrue(os.path.isfile(self.tmpdir + "/primary.db")) db_pri = cr.PrimarySqlite(self.tmpdir + "/primary2.db") self.assertTrue(db_pri) self.assertTrue(os.path.isfile(self.tmpdir + "/primary2.db")) db_fil = cr.Sqlite(self.tmpdir + "/filelists.db", cr.DB_FILELISTS) self.assertTrue(db_fil) self.assertTrue(os.path.isfile(self.tmpdir + "/filelists.db")) db_fil = cr.FilelistsSqlite(self.tmpdir + "/filelists2.db") self.assertTrue(db_fil) self.assertTrue(os.path.isfile(self.tmpdir + "/filelists2.db")) db_oth = cr.Sqlite(self.tmpdir + "/other.db", cr.DB_OTHER) self.assertTrue(db_oth) self.assertTrue(os.path.isfile(self.tmpdir + "/other.db")) db_oth = cr.OtherSqlite(self.tmpdir + "/other2.db") self.assertTrue(db_oth) self.assertTrue(os.path.isfile(self.tmpdir + "/other2.db"))
def publish(repository_version_pk): """ Create a Publication based on a RepositoryVersion. Args: repository_version_pk (str): Create a publication from this repository version. """ repository_version = RepositoryVersion.objects.get( pk=repository_version_pk) log.info( _('Publishing: repository={repo}, version={version}').format( repo=repository_version.repository.name, version=repository_version.number, )) with WorkingDirectory(): with RpmPublication.create(repository_version) as publication: packages = populate(publication) # Prepare metadata files repomd_path = os.path.join(os.getcwd(), "repomd.xml") pri_xml_path = os.path.join(os.getcwd(), "primary.xml.gz") fil_xml_path = os.path.join(os.getcwd(), "filelists.xml.gz") oth_xml_path = os.path.join(os.getcwd(), "other.xml.gz") pri_db_path = os.path.join(os.getcwd(), "primary.sqlite") fil_db_path = os.path.join(os.getcwd(), "filelists.sqlite") oth_db_path = os.path.join(os.getcwd(), "other.sqlite") upd_xml_path = os.path.join(os.getcwd(), "updateinfo.xml.gz") pri_xml = cr.PrimaryXmlFile(pri_xml_path) fil_xml = cr.FilelistsXmlFile(fil_xml_path) oth_xml = cr.OtherXmlFile(oth_xml_path) pri_db = cr.PrimarySqlite(pri_db_path) fil_db = cr.FilelistsSqlite(fil_db_path) oth_db = cr.OtherSqlite(oth_db_path) upd_xml = cr.UpdateInfoXmlFile(upd_xml_path) pri_xml.set_num_of_pkgs(len(packages)) fil_xml.set_num_of_pkgs(len(packages)) oth_xml.set_num_of_pkgs(len(packages)) # Process all packages for package in packages: pkg = package.to_createrepo_c() pkg.location_href = package.contentartifact_set.first( ).relative_path pri_xml.add_pkg(pkg) fil_xml.add_pkg(pkg) oth_xml.add_pkg(pkg) pri_db.add_pkg(pkg) fil_db.add_pkg(pkg) oth_db.add_pkg(pkg) # Process update records for update_record in UpdateRecord.objects.filter( pk__in=publication.repository_version.content): upd_xml.add_chunk(update_record_xml(update_record)) pri_xml.close() fil_xml.close() oth_xml.close() upd_xml.close() repomd = cr.Repomd() repomdrecords = (("primary", pri_xml_path, pri_db), ("filelists", fil_xml_path, fil_db), ("other", oth_xml_path, oth_db), ("primary_db", pri_db_path, None), ("filelists_db", fil_db_path, None), ("other_db", oth_db_path, None), ("updateinfo", upd_xml_path, None)) sqlite_files = ("primary_db", "filelists_db", "other_db") for name, path, db_to_update in repomdrecords: record = cr.RepomdRecord(name, path) if name in sqlite_files: record_bz = record.compress_and_fill(cr.SHA256, cr.BZ2) record_bz.type = name record_bz.rename_file() path = record_bz.location_href.split('/')[-1] repomd.set_record(record_bz) else: record.fill(cr.SHA256) if (db_to_update): db_to_update.dbinfo_update(record.checksum) db_to_update.close() record.rename_file() path = record.location_href.split('/')[-1] repomd.set_record(record) metadata = PublishedMetadata( relative_path=os.path.join(REPODATA_PATH, os.path.basename(path)), publication=publication, file=File(open(os.path.basename(path), 'rb'))) metadata.save() with open(repomd_path, "w") as repomd_f: repomd_f.write(repomd.xml_dump()) metadata = PublishedMetadata( relative_path=os.path.join(REPODATA_PATH, os.path.basename(repomd_path)), publication=publication, file=File(open(os.path.basename(repomd_path), 'rb'))) metadata.save()
def create_repomd_xml( content, publication, checksum_types, extra_repomdrecords, sub_folder=None, metadata_signing_service=None, ): """ Creates a repomd.xml file. Args: content(app.models.Content): content set publication(pulpcore.plugin.models.Publication): the publication extra_repomdrecords(list): list with data relative to repo metadata files sub_folder(str): name of the folder for sub repos metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService): A reference to an associated signing service. """ cwd = os.getcwd() repodata_path = REPODATA_PATH has_modules = False has_comps = False package_checksum_type = checksum_types.get("package") if sub_folder: cwd = os.path.join(cwd, sub_folder) repodata_path = os.path.join(sub_folder, repodata_path) # Prepare metadata files repomd_path = os.path.join(cwd, "repomd.xml") pri_xml_path = os.path.join(cwd, "primary.xml.gz") fil_xml_path = os.path.join(cwd, "filelists.xml.gz") oth_xml_path = os.path.join(cwd, "other.xml.gz") upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz") mod_yml_path = os.path.join(cwd, "modules.yaml") comps_xml_path = os.path.join(cwd, "comps.xml") pri_xml = cr.PrimaryXmlFile(pri_xml_path) fil_xml = cr.FilelistsXmlFile(fil_xml_path) oth_xml = cr.OtherXmlFile(oth_xml_path) upd_xml = cr.UpdateInfoXmlFile(upd_xml_path) if publication.sqlite_metadata: pri_db_path = os.path.join(cwd, "primary.sqlite") fil_db_path = os.path.join(cwd, "filelists.sqlite") oth_db_path = os.path.join(cwd, "other.sqlite") pri_db = cr.PrimarySqlite(pri_db_path) fil_db = cr.FilelistsSqlite(fil_db_path) oth_db = cr.OtherSqlite(oth_db_path) packages = Package.objects.filter(pk__in=content) total_packages = packages.count() pri_xml.set_num_of_pkgs(total_packages) fil_xml.set_num_of_pkgs(total_packages) oth_xml.set_num_of_pkgs(total_packages) # We want to support publishing with a different checksum type than the one built-in to the # package itself, so we need to get the correct checksums somehow if there is an override. # We must also take into consideration that if the package has not been downloaded the only # checksum that is available is the one built-in. # # Since this lookup goes from Package->Content->ContentArtifact->Artifact, performance is a # challenge. We use ContentArtifact as our starting point because it enables us to work with # simple foreign keys and avoid messing with the many-to-many relationship, which doesn't # work with select_related() and performs poorly with prefetch_related(). This is fine # because we know that Packages should only ever have one artifact per content. contentartifact_qs = ( ContentArtifact.objects.filter(content__in=packages.only("pk")). select_related( # content__rpm_package is a bit of a hack, exploiting the way django sets up model # inheritance, but it works and is unlikely to break. All content artifacts being # accessed here have an associated Package since they originally came from the # Package queryset. "artifact", "content__rpm_package", ).only("artifact", "content__rpm_package__checksum_type", "content__rpm_package__pkgId")) pkg_to_hash = {} for ca in contentartifact_qs.iterator(): pkgid = None if package_checksum_type: package_checksum_type = package_checksum_type.lower() pkgid = getattr(ca.artifact, package_checksum_type, None) if pkgid: pkg_to_hash[ca.content_id] = (package_checksum_type, pkgid) else: pkg_to_hash[ca.content_id] = ( ca.content.rpm_package.checksum_type, ca.content.rpm_package.pkgId, ) # Process all packages for package in packages.iterator(): pkg = package.to_createrepo_c() # rewrite the checksum and checksum type with the desired ones (checksum, pkgId) = pkg_to_hash[package.pk] pkg.checksum_type = checksum pkg.pkgId = pkgId pkg_filename = os.path.basename(package.location_href) # this can cause an issue when two same RPM package names appears # a/name1.rpm b/name1.rpm pkg.location_href = os.path.join(PACKAGES_DIRECTORY, pkg_filename[0].lower(), pkg_filename) pri_xml.add_pkg(pkg) fil_xml.add_pkg(pkg) oth_xml.add_pkg(pkg) if publication.sqlite_metadata: pri_db.add_pkg(pkg) fil_db.add_pkg(pkg) oth_db.add_pkg(pkg) # Process update records for update_record in UpdateRecord.objects.filter( pk__in=content).iterator(): upd_xml.add_chunk( cr.xml_dump_updaterecord(update_record.to_createrepo_c())) # Process modulemd and modulemd_defaults with open(mod_yml_path, "ab") as mod_yml: for modulemd in Modulemd.objects.filter(pk__in=content).iterator(): mod_yml.write(modulemd._artifacts.get().file.read()) has_modules = True for default in ModulemdDefaults.objects.filter( pk__in=content).iterator(): mod_yml.write(default._artifacts.get().file.read()) has_modules = True # Process comps comps = libcomps.Comps() for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator(): group = pkg_grp.pkg_grp_to_libcomps() comps.groups.append(group) has_comps = True for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator(): cat = pkg_cat.pkg_cat_to_libcomps() comps.categories.append(cat) has_comps = True for pkg_env in PackageEnvironment.objects.filter( pk__in=content).iterator(): env = pkg_env.pkg_env_to_libcomps() comps.environments.append(env) has_comps = True for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator(): comps.langpacks = dict_to_strdict(pkg_lng.matches) has_comps = True comps.toxml_f( comps_xml_path, xml_options={ "default_explicit": True, "empty_groups": True, "uservisible_explicit": True }, ) pri_xml.close() fil_xml.close() oth_xml.close() upd_xml.close() repomd = cr.Repomd() if publication.sqlite_metadata: repomdrecords = [ ("primary", pri_xml_path, pri_db), ("filelists", fil_xml_path, fil_db), ("other", oth_xml_path, oth_db), ("primary_db", pri_db_path, None), ("filelists_db", fil_db_path, None), ("other_db", oth_db_path, None), ("updateinfo", upd_xml_path, None), ] else: repomdrecords = [ ("primary", pri_xml_path, None), ("filelists", fil_xml_path, None), ("other", oth_xml_path, None), ("updateinfo", upd_xml_path, None), ] if has_modules: repomdrecords.append(("modules", mod_yml_path, None)) if has_comps: repomdrecords.append(("group", comps_xml_path, None)) repomdrecords.extend(extra_repomdrecords) sqlite_files = ("primary_db", "filelists_db", "other_db") for name, path, db_to_update in repomdrecords: record = cr.RepomdRecord(name, path) checksum_type = get_checksum_type(name, checksum_types) if name in sqlite_files: record_bz = record.compress_and_fill(checksum_type, cr.BZ2) record_bz.type = name record_bz.rename_file() path = record_bz.location_href.split("/")[-1] repomd.set_record(record_bz) else: record.fill(checksum_type) if db_to_update: db_to_update.dbinfo_update(record.checksum) db_to_update.close() record.rename_file() path = record.location_href.split("/")[-1] repomd.set_record(record) if sub_folder: path = os.path.join(sub_folder, path) PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(path)), publication=publication, file=File(open(path, "rb")), ) with open(repomd_path, "w") as repomd_f: repomd_f.write(repomd.xml_dump()) if metadata_signing_service: signing_service = AsciiArmoredDetachedSigningService.objects.get( pk=metadata_signing_service.pk) sign_results = signing_service.sign(repomd_path) # publish a signed file PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(sign_results["file"])), publication=publication, file=File(open(sign_results["file"], "rb")), ) # publish a detached signature PublishedMetadata.create_from_file( relative_path=os.path.join( repodata_path, os.path.basename(sign_results["signature"])), publication=publication, file=File(open(sign_results["signature"], "rb")), ) # publish a public key required for further verification PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(sign_results["key"])), publication=publication, file=File(open(sign_results["key"], "rb")), ) else: PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(repomd_path)), publication=publication, file=File(open(repomd_path, "rb")), )
def create_rempomd_xml(packages, publication, extra_repomdrecords, sub_folder=None): """ Creates a repomd.xml file. Args: packages(app.models.Package): set of packages publication(pulpcore.plugin.models.Publication): the publication extra_repomdrecords(list): list with data relative to repo metadata files sub_folder(str): name of the folder for sub repos """ cwd = os.getcwd() repodata_path = REPODATA_PATH has_modules = False if sub_folder: cwd = os.path.join(cwd, sub_folder) repodata_path = os.path.join(sub_folder, repodata_path) # Prepare metadata files repomd_path = os.path.join(cwd, "repomd.xml") pri_xml_path = os.path.join(cwd, "primary.xml.gz") fil_xml_path = os.path.join(cwd, "filelists.xml.gz") oth_xml_path = os.path.join(cwd, "other.xml.gz") pri_db_path = os.path.join(cwd, "primary.sqlite") fil_db_path = os.path.join(cwd, "filelists.sqlite") oth_db_path = os.path.join(cwd, "other.sqlite") upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz") mod_yml_path = os.path.join(cwd, "modules.yaml") pri_xml = cr.PrimaryXmlFile(pri_xml_path) fil_xml = cr.FilelistsXmlFile(fil_xml_path) oth_xml = cr.OtherXmlFile(oth_xml_path) pri_db = cr.PrimarySqlite(pri_db_path) fil_db = cr.FilelistsSqlite(fil_db_path) oth_db = cr.OtherSqlite(oth_db_path) upd_xml = cr.UpdateInfoXmlFile(upd_xml_path) pri_xml.set_num_of_pkgs(len(packages)) fil_xml.set_num_of_pkgs(len(packages)) oth_xml.set_num_of_pkgs(len(packages)) # Process all packages for package in packages: pkg = package.to_createrepo_c() pkg.location_href = package.contentartifact_set.first().relative_path pri_xml.add_pkg(pkg) fil_xml.add_pkg(pkg) oth_xml.add_pkg(pkg) pri_db.add_pkg(pkg) fil_db.add_pkg(pkg) oth_db.add_pkg(pkg) # Process update records for update_record in UpdateRecord.objects.filter( pk__in=publication.repository_version.content): upd_xml.add_chunk(update_record_xml(update_record)) # Process modulemd and modulemd_defaults with open(mod_yml_path, 'ab') as mod_yml: for modulemd in Modulemd.objects.filter( pk__in=publication.repository_version.content): mod_yml.write(modulemd._artifacts.get().file.read()) has_modules = True for default in ModulemdDefaults.objects.filter( pk__in=publication.repository_version.content): mod_yml.write(default._artifacts.get().file.read()) has_modules = True pri_xml.close() fil_xml.close() oth_xml.close() upd_xml.close() repomd = cr.Repomd() repomdrecords = [("primary", pri_xml_path, pri_db), ("filelists", fil_xml_path, fil_db), ("other", oth_xml_path, oth_db), ("primary_db", pri_db_path, None), ("filelists_db", fil_db_path, None), ("other_db", oth_db_path, None), ("updateinfo", upd_xml_path, None)] if has_modules: repomdrecords.append(("modules", mod_yml_path, None)) repomdrecords.extend(extra_repomdrecords) sqlite_files = ("primary_db", "filelists_db", "other_db") for name, path, db_to_update in repomdrecords: record = cr.RepomdRecord(name, path) if name in sqlite_files: record_bz = record.compress_and_fill(cr.SHA256, cr.BZ2) record_bz.type = name record_bz.rename_file() path = record_bz.location_href.split('/')[-1] repomd.set_record(record_bz) elif name == "modules": record_md = record.compress_and_fill(cr.SHA256, cr.GZ) record_md.type = name record_md.rename_file() path = record_md.location_href.split('/')[-1] repomd.set_record(record_md) else: record.fill(cr.SHA256) if (db_to_update): db_to_update.dbinfo_update(record.checksum) db_to_update.close() record.rename_file() path = record.location_href.split('/')[-1] repomd.set_record(record) if sub_folder: path = os.path.join(sub_folder, path) PublishedMetadata.create_from_file(relative_path=os.path.join( repodata_path, os.path.basename(path)), publication=publication, file=File(open(path, 'rb'))) with open(repomd_path, "w") as repomd_f: repomd_f.write(repomd.xml_dump()) PublishedMetadata.create_from_file(relative_path=os.path.join( repodata_path, os.path.basename(repomd_path)), publication=publication, file=File(open(repomd_path, 'rb')))
def create_repomd_xml(content, publication, extra_repomdrecords, sub_folder=None): """ Creates a repomd.xml file. Args: content(app.models.Content): content set publication(pulpcore.plugin.models.Publication): the publication extra_repomdrecords(list): list with data relative to repo metadata files sub_folder(str): name of the folder for sub repos """ cwd = os.getcwd() repodata_path = REPODATA_PATH has_modules = False has_comps = False if sub_folder: cwd = os.path.join(cwd, sub_folder) repodata_path = os.path.join(sub_folder, repodata_path) # Prepare metadata files repomd_path = os.path.join(cwd, "repomd.xml") pri_xml_path = os.path.join(cwd, "primary.xml.gz") fil_xml_path = os.path.join(cwd, "filelists.xml.gz") oth_xml_path = os.path.join(cwd, "other.xml.gz") pri_db_path = os.path.join(cwd, "primary.sqlite") fil_db_path = os.path.join(cwd, "filelists.sqlite") oth_db_path = os.path.join(cwd, "other.sqlite") upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz") mod_yml_path = os.path.join(cwd, "modules.yaml") comps_xml_path = os.path.join(cwd, "comps.xml") pri_xml = cr.PrimaryXmlFile(pri_xml_path) fil_xml = cr.FilelistsXmlFile(fil_xml_path) oth_xml = cr.OtherXmlFile(oth_xml_path) pri_db = cr.PrimarySqlite(pri_db_path) fil_db = cr.FilelistsSqlite(fil_db_path) oth_db = cr.OtherSqlite(oth_db_path) upd_xml = cr.UpdateInfoXmlFile(upd_xml_path) packages = Package.objects.filter(pk__in=content) total_packages = packages.count() pri_xml.set_num_of_pkgs(total_packages) fil_xml.set_num_of_pkgs(total_packages) oth_xml.set_num_of_pkgs(total_packages) # Process all packages for package in packages.iterator(): pkg = package.to_createrepo_c() pkg.location_href = package.contentartifact_set.only('relative_path').first().relative_path pri_xml.add_pkg(pkg) fil_xml.add_pkg(pkg) oth_xml.add_pkg(pkg) pri_db.add_pkg(pkg) fil_db.add_pkg(pkg) oth_db.add_pkg(pkg) # Process update records for update_record in UpdateRecord.objects.filter(pk__in=content).iterator(): upd_xml.add_chunk(cr.xml_dump_updaterecord(update_record.to_createrepo_c())) # Process modulemd and modulemd_defaults with open(mod_yml_path, 'ab') as mod_yml: for modulemd in Modulemd.objects.filter(pk__in=content).iterator(): mod_yml.write(modulemd._artifacts.get().file.read()) has_modules = True for default in ModulemdDefaults.objects.filter(pk__in=content).iterator(): mod_yml.write(default._artifacts.get().file.read()) has_modules = True # Process comps comps = libcomps.Comps() for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator(): group = pkg_grp.pkg_grp_to_libcomps() comps.groups.append(group) has_comps = True for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator(): cat = pkg_cat.pkg_cat_to_libcomps() comps.categories.append(cat) has_comps = True for pkg_env in PackageEnvironment.objects.filter(pk__in=content).iterator(): env = pkg_env.pkg_env_to_libcomps() comps.environments.append(env) has_comps = True for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator(): comps.langpacks = dict_to_strdict(pkg_lng.matches) has_comps = True comps.toxml_f(comps_xml_path, xml_options={"default_explicit": True, "empty_groups": True, "uservisible_explicit": True}) pri_xml.close() fil_xml.close() oth_xml.close() upd_xml.close() repomd = cr.Repomd() repomdrecords = [("primary", pri_xml_path, pri_db), ("filelists", fil_xml_path, fil_db), ("other", oth_xml_path, oth_db), ("primary_db", pri_db_path, None), ("filelists_db", fil_db_path, None), ("other_db", oth_db_path, None), ("updateinfo", upd_xml_path, None)] if has_modules: repomdrecords.append(("modules", mod_yml_path, None)) if has_comps: repomdrecords.append(("group", comps_xml_path, None)) repomdrecords.extend(extra_repomdrecords) sqlite_files = ("primary_db", "filelists_db", "other_db") for name, path, db_to_update in repomdrecords: record = cr.RepomdRecord(name, path) if name in sqlite_files: record_bz = record.compress_and_fill(cr.SHA256, cr.BZ2) record_bz.type = name record_bz.rename_file() path = record_bz.location_href.split('/')[-1] repomd.set_record(record_bz) else: record.fill(cr.SHA256) if (db_to_update): db_to_update.dbinfo_update(record.checksum) db_to_update.close() record.rename_file() path = record.location_href.split('/')[-1] repomd.set_record(record) if sub_folder: path = os.path.join(sub_folder, path) PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(path)), publication=publication, file=File(open(path, 'rb')) ) with open(repomd_path, "w") as repomd_f: repomd_f.write(repomd.xml_dump()) PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(repomd_path)), publication=publication, file=File(open(repomd_path, 'rb')) )
def generate_repo_metadata( content, publication, checksum_types, extra_repomdrecords, sub_folder=None, metadata_signing_service=None, ): """ Creates a repomd.xml file. Args: content(app.models.Content): content set publication(pulpcore.plugin.models.Publication): the publication extra_repomdrecords(list): list with data relative to repo metadata files sub_folder(str): name of the folder for sub repos metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService): A reference to an associated signing service. """ cwd = os.getcwd() repodata_path = REPODATA_PATH has_modules = False has_comps = False package_checksum_type = checksum_types.get("package") if sub_folder: cwd = os.path.join(cwd, sub_folder) repodata_path = os.path.join(sub_folder, repodata_path) if package_checksum_type and package_checksum_type not in settings.ALLOWED_CONTENT_CHECKSUMS: raise ValueError( "Repository contains disallowed package checksum type '{}', " "thus can't be published. {}".format(package_checksum_type, ALLOWED_CHECKSUM_ERROR_MSG)) # Prepare metadata files repomd_path = os.path.join(cwd, "repomd.xml") pri_xml_path = os.path.join(cwd, "primary.xml.gz") fil_xml_path = os.path.join(cwd, "filelists.xml.gz") oth_xml_path = os.path.join(cwd, "other.xml.gz") upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz") mod_yml_path = os.path.join(cwd, "modules.yaml") comps_xml_path = os.path.join(cwd, "comps.xml") pri_xml = cr.PrimaryXmlFile(pri_xml_path) fil_xml = cr.FilelistsXmlFile(fil_xml_path) oth_xml = cr.OtherXmlFile(oth_xml_path) upd_xml = cr.UpdateInfoXmlFile(upd_xml_path) if publication.sqlite_metadata: pri_db_path = os.path.join(cwd, "primary.sqlite") fil_db_path = os.path.join(cwd, "filelists.sqlite") oth_db_path = os.path.join(cwd, "other.sqlite") pri_db = cr.PrimarySqlite(pri_db_path) fil_db = cr.FilelistsSqlite(fil_db_path) oth_db = cr.OtherSqlite(oth_db_path) packages = Package.objects.filter(pk__in=content) # We want to support publishing with a different checksum type than the one built-in to the # package itself, so we need to get the correct checksums somehow if there is an override. # We must also take into consideration that if the package has not been downloaded the only # checksum that is available is the one built-in. # # Since this lookup goes from Package->Content->ContentArtifact->Artifact, performance is a # challenge. We use ContentArtifact as our starting point because it enables us to work with # simple foreign keys and avoid messing with the many-to-many relationship, which doesn't # work with select_related() and performs poorly with prefetch_related(). This is fine # because we know that Packages should only ever have one artifact per content. contentartifact_qs = ( ContentArtifact.objects.filter(content__in=packages.only("pk")). select_related( # content__rpm_package is a bit of a hack, exploiting the way django sets up model # inheritance, but it works and is unlikely to break. All content artifacts being # accessed here have an associated Package since they originally came from the # Package queryset. "artifact", "content__rpm_package", ).only("artifact", "content__rpm_package__checksum_type", "content__rpm_package__pkgId")) pkg_to_hash = {} for ca in contentartifact_qs.iterator(): if package_checksum_type: package_checksum_type = package_checksum_type.lower() pkgid = getattr(ca.artifact, package_checksum_type, None) if not package_checksum_type or not pkgid: if ca.content.rpm_package.checksum_type not in settings.ALLOWED_CONTENT_CHECKSUMS: raise ValueError( "Package {} as content unit {} contains forbidden checksum type '{}', " "thus can't be published. {}".format( ca.content.rpm_package.nevra, ca.content.pk, ca.content.rpm_package.checksum_type, ALLOWED_CHECKSUM_ERROR_MSG, )) package_checksum_type = ca.content.rpm_package.checksum_type pkgid = ca.content.rpm_package.pkgId pkg_to_hash[ca.content_id] = (package_checksum_type, pkgid) # TODO: this is meant to be a !! *temporary* !! fix for # https://github.com/pulp/pulp_rpm/issues/2407 pkg_pks_to_ignore = set() latest_build_time_by_nevra = defaultdict(list) for pkg in packages.only("pk", "name", "epoch", "version", "release", "arch", "time_build").iterator(): latest_build_time_by_nevra[pkg.nevra].append((pkg.time_build, pkg.pk)) for nevra, pkg_data in latest_build_time_by_nevra.items(): # sort the packages by when they were built if len(pkg_data) > 1: pkg_data.sort(key=lambda p: p[0], reverse=True) pkg_pks_to_ignore |= set(entry[1] for entry in pkg_data[1:]) log.warning( "Duplicate packages found competing for NEVRA {nevra}, selected the one with " "the most recent build time, excluding {others} others.". format(nevra=nevra, others=len(pkg_data[1:]))) total_packages = packages.count() - len(pkg_pks_to_ignore) pri_xml.set_num_of_pkgs(total_packages) fil_xml.set_num_of_pkgs(total_packages) oth_xml.set_num_of_pkgs(total_packages) # Process all packages for package in packages.order_by("name", "evr").iterator(): if package.pk in pkg_pks_to_ignore: # Temporary! continue pkg = package.to_createrepo_c() # rewrite the checksum and checksum type with the desired ones (checksum, pkgId) = pkg_to_hash[package.pk] pkg.checksum_type = checksum pkg.pkgId = pkgId pkg_filename = os.path.basename(package.location_href) # this can cause an issue when two same RPM package names appears # a/name1.rpm b/name1.rpm pkg.location_href = os.path.join(PACKAGES_DIRECTORY, pkg_filename[0].lower(), pkg_filename) pri_xml.add_pkg(pkg) fil_xml.add_pkg(pkg) oth_xml.add_pkg(pkg) if publication.sqlite_metadata: pri_db.add_pkg(pkg) fil_db.add_pkg(pkg) oth_db.add_pkg(pkg) # Process update records for update_record in UpdateRecord.objects.filter( pk__in=content).iterator(): upd_xml.add_chunk( cr.xml_dump_updaterecord(update_record.to_createrepo_c())) # Process modulemd, modulemd_defaults and obsoletes with open(mod_yml_path, "ab") as mod_yml: for modulemd in Modulemd.objects.filter(pk__in=content).iterator(): mod_yml.write(modulemd.snippet.encode()) has_modules = True for default in ModulemdDefaults.objects.filter( pk__in=content).iterator(): mod_yml.write(default.snippet.encode()) has_modules = True for obsolete in ModulemdObsolete.objects.filter( pk__in=content).iterator(): mod_yml.write(obsolete.snippet.encode()) has_modules = True # Process comps comps = libcomps.Comps() for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator(): group = pkg_grp.pkg_grp_to_libcomps() comps.groups.append(group) has_comps = True for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator(): cat = pkg_cat.pkg_cat_to_libcomps() comps.categories.append(cat) has_comps = True for pkg_env in PackageEnvironment.objects.filter( pk__in=content).iterator(): env = pkg_env.pkg_env_to_libcomps() comps.environments.append(env) has_comps = True for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator(): comps.langpacks = dict_to_strdict(pkg_lng.matches) has_comps = True comps.toxml_f( comps_xml_path, xml_options={ "default_explicit": True, "empty_groups": True, "empty_packages": True, "uservisible_explicit": True, }, ) pri_xml.close() fil_xml.close() oth_xml.close() upd_xml.close() repomd = cr.Repomd() # If the repository is empty, use a revision of 0 # See: https://pulp.plan.io/issues/9402 if not content.exists(): repomd.revision = "0" if publication.sqlite_metadata: repomdrecords = [ ("primary", pri_xml_path, pri_db), ("filelists", fil_xml_path, fil_db), ("other", oth_xml_path, oth_db), ("primary_db", pri_db_path, None), ("filelists_db", fil_db_path, None), ("other_db", oth_db_path, None), ("updateinfo", upd_xml_path, None), ] else: repomdrecords = [ ("primary", pri_xml_path, None), ("filelists", fil_xml_path, None), ("other", oth_xml_path, None), ("updateinfo", upd_xml_path, None), ] if has_modules: repomdrecords.append(("modules", mod_yml_path, None)) if has_comps: repomdrecords.append(("group", comps_xml_path, None)) repomdrecords.extend(extra_repomdrecords) sqlite_files = ("primary_db", "filelists_db", "other_db") for name, path, db_to_update in repomdrecords: record = cr.RepomdRecord(name, path) checksum_type = cr_checksum_type_from_string( get_checksum_type(name, checksum_types, default=publication.metadata_checksum_type)) if name in sqlite_files: record_bz = record.compress_and_fill(checksum_type, cr.BZ2) record_bz.type = name record_bz.rename_file() path = record_bz.location_href.split("/")[-1] repomd.set_record(record_bz) else: record.fill(checksum_type) if db_to_update: db_to_update.dbinfo_update(record.checksum) db_to_update.close() record.rename_file() path = record.location_href.split("/")[-1] repomd.set_record(record) if sub_folder: path = os.path.join(sub_folder, path) with open(path, "rb") as repodata_fd: PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(path)), publication=publication, file=File(repodata_fd), ) with open(repomd_path, "w") as repomd_f: repomd_f.write(repomd.xml_dump()) if metadata_signing_service: signing_service = AsciiArmoredDetachedSigningService.objects.get( pk=metadata_signing_service) sign_results = signing_service.sign(repomd_path) # publish a signed file with open(sign_results["file"], "rb") as signed_file_fd: PublishedMetadata.create_from_file( relative_path=os.path.join( repodata_path, os.path.basename(sign_results["file"])), publication=publication, file=File(signed_file_fd), ) # publish a detached signature with open(sign_results["signature"], "rb") as signature_fd: PublishedMetadata.create_from_file( relative_path=os.path.join( repodata_path, os.path.basename(sign_results["signature"])), publication=publication, file=File(signature_fd), ) # publish a public key required for further verification pubkey_name = "repomd.xml.key" with open(pubkey_name, "wb+") as f: f.write(signing_service.public_key.encode("utf-8")) f.flush() PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, pubkey_name), publication=publication, file=File(f), ) else: with open(repomd_path, "rb") as repomd_fd: PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(repomd_path)), publication=publication, file=File(repomd_fd), )
def create_repomd_xml(content, publication, checksum_types, extra_repomdrecords, sub_folder=None, metadata_signing_service=None): """ Creates a repomd.xml file. Args: content(app.models.Content): content set publication(pulpcore.plugin.models.Publication): the publication extra_repomdrecords(list): list with data relative to repo metadata files sub_folder(str): name of the folder for sub repos metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService): A reference to an associated signing service. """ cwd = os.getcwd() repodata_path = REPODATA_PATH has_modules = False has_comps = False package_checksum_type = checksum_types.get("package") if sub_folder: cwd = os.path.join(cwd, sub_folder) repodata_path = os.path.join(sub_folder, repodata_path) # Prepare metadata files repomd_path = os.path.join(cwd, "repomd.xml") pri_xml_path = os.path.join(cwd, "primary.xml.gz") fil_xml_path = os.path.join(cwd, "filelists.xml.gz") oth_xml_path = os.path.join(cwd, "other.xml.gz") pri_db_path = os.path.join(cwd, "primary.sqlite") fil_db_path = os.path.join(cwd, "filelists.sqlite") oth_db_path = os.path.join(cwd, "other.sqlite") upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz") mod_yml_path = os.path.join(cwd, "modules.yaml") comps_xml_path = os.path.join(cwd, "comps.xml") pri_xml = cr.PrimaryXmlFile(pri_xml_path) fil_xml = cr.FilelistsXmlFile(fil_xml_path) oth_xml = cr.OtherXmlFile(oth_xml_path) pri_db = cr.PrimarySqlite(pri_db_path) fil_db = cr.FilelistsSqlite(fil_db_path) oth_db = cr.OtherSqlite(oth_db_path) upd_xml = cr.UpdateInfoXmlFile(upd_xml_path) packages = Package.objects.filter(pk__in=content) total_packages = packages.count() pri_xml.set_num_of_pkgs(total_packages) fil_xml.set_num_of_pkgs(total_packages) oth_xml.set_num_of_pkgs(total_packages) # Process all packages for package in packages.iterator(): pkg = package.to_createrepo_c(package_checksum_type) pkg_filename = os.path.basename(package.location_href) # this can cause an issue when two same RPM package names appears # a/name1.rpm b/name1.rpm pkg.location_href = os.path.join(PACKAGES_DIRECTORY, pkg_filename[0].lower(), pkg_filename) pri_xml.add_pkg(pkg) fil_xml.add_pkg(pkg) oth_xml.add_pkg(pkg) pri_db.add_pkg(pkg) fil_db.add_pkg(pkg) oth_db.add_pkg(pkg) # Process update records for update_record in UpdateRecord.objects.filter( pk__in=content).iterator(): upd_xml.add_chunk( cr.xml_dump_updaterecord(update_record.to_createrepo_c())) # Process modulemd and modulemd_defaults with open(mod_yml_path, 'ab') as mod_yml: for modulemd in Modulemd.objects.filter(pk__in=content).iterator(): mod_yml.write(modulemd._artifacts.get().file.read()) has_modules = True for default in ModulemdDefaults.objects.filter( pk__in=content).iterator(): mod_yml.write(default._artifacts.get().file.read()) has_modules = True # Process comps comps = libcomps.Comps() for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator(): group = pkg_grp.pkg_grp_to_libcomps() comps.groups.append(group) has_comps = True for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator(): cat = pkg_cat.pkg_cat_to_libcomps() comps.categories.append(cat) has_comps = True for pkg_env in PackageEnvironment.objects.filter( pk__in=content).iterator(): env = pkg_env.pkg_env_to_libcomps() comps.environments.append(env) has_comps = True for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator(): comps.langpacks = dict_to_strdict(pkg_lng.matches) has_comps = True comps.toxml_f(comps_xml_path, xml_options={ "default_explicit": True, "empty_groups": True, "uservisible_explicit": True }) pri_xml.close() fil_xml.close() oth_xml.close() upd_xml.close() repomd = cr.Repomd() repomdrecords = [("primary", pri_xml_path, pri_db), ("filelists", fil_xml_path, fil_db), ("other", oth_xml_path, oth_db), ("primary_db", pri_db_path, None), ("filelists_db", fil_db_path, None), ("other_db", oth_db_path, None), ("updateinfo", upd_xml_path, None)] if has_modules: repomdrecords.append(("modules", mod_yml_path, None)) if has_comps: repomdrecords.append(("group", comps_xml_path, None)) repomdrecords.extend(extra_repomdrecords) sqlite_files = ("primary_db", "filelists_db", "other_db") for name, path, db_to_update in repomdrecords: record = cr.RepomdRecord(name, path) checksum_type = get_checksum_type(name, checksum_types) if name in sqlite_files: record_bz = record.compress_and_fill(checksum_type, cr.BZ2) record_bz.type = name record_bz.rename_file() path = record_bz.location_href.split('/')[-1] repomd.set_record(record_bz) else: record.fill(checksum_type) if (db_to_update): db_to_update.dbinfo_update(record.checksum) db_to_update.close() record.rename_file() path = record.location_href.split('/')[-1] repomd.set_record(record) if sub_folder: path = os.path.join(sub_folder, path) PublishedMetadata.create_from_file(relative_path=os.path.join( repodata_path, os.path.basename(path)), publication=publication, file=File(open(path, 'rb'))) with open(repomd_path, "w") as repomd_f: repomd_f.write(repomd.xml_dump()) if metadata_signing_service: signing_service = AsciiArmoredDetachedSigningService.objects.get( pk=metadata_signing_service.pk) sign_results = signing_service.sign(repomd_path) # publish a signed file PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(sign_results['file'])), publication=publication, file=File(open(sign_results['file'], 'rb'))) # publish a detached signature PublishedMetadata.create_from_file(relative_path=os.path.join( repodata_path, os.path.basename(sign_results['signature'])), publication=publication, file=File( open(sign_results['signature'], 'rb'))) # publish a public key required for further verification PublishedMetadata.create_from_file( relative_path=os.path.join(repodata_path, os.path.basename(sign_results['key'])), publication=publication, file=File(open(sign_results['key'], 'rb'))) else: PublishedMetadata.create_from_file(relative_path=os.path.join( repodata_path, os.path.basename(repomd_path)), publication=publication, file=File(open(repomd_path, 'rb')))
def build_metadata(self): staging = tempfile.mkdtemp(prefix='yumsync-', suffix='-metadata') if self.checksum == 'sha' or self.checksum == 'sha1': sumtype = 'sha' else: sumtype = 'sha256' repodata_path = os.path.join(staging, 'repodata') os.mkdir(repodata_path) # Prepare metadata files repomd_path = os.path.join(repodata_path, "repomd.xml") pri_xml_path = os.path.join(repodata_path, "primary.xml.gz") fil_xml_path = os.path.join(repodata_path, "filelists.xml.gz") oth_xml_path = os.path.join(repodata_path, "other.xml.gz") pri_db_path = os.path.join(repodata_path, "primary.sqlite") fil_db_path = os.path.join(repodata_path, "filelists.sqlite") oth_db_path = os.path.join(repodata_path, "other.sqlite") # Related python objects pri_xml = createrepo.PrimaryXmlFile(pri_xml_path) fil_xml = createrepo.FilelistsXmlFile(fil_xml_path) oth_xml = createrepo.OtherXmlFile(oth_xml_path) pri_db = createrepo.PrimarySqlite(pri_db_path) fil_db = createrepo.FilelistsSqlite(fil_db_path) oth_db = createrepo.OtherSqlite(oth_db_path) # Set package list if self.local_dir and self.link_type == "individual_symlink" and self.version_dir: pkg_list = [ ( os.path.join(self.version_package_dir,"{}".format(pkg)), os.path.join("packages","{}".format(pkg)) ) for pkg in self._packages] else: pkg_list = [ ( os.path.join(self.package_dir,"{}".format(pkg)), os.path.join("packages","{}".format(pkg)) )for pkg in self._packages] pri_xml.set_num_of_pkgs(len(pkg_list)) fil_xml.set_num_of_pkgs(len(pkg_list)) oth_xml.set_num_of_pkgs(len(pkg_list)) # Process all packages in // if possible self.metadata_progress = 0 self.total_pkgs = len(pkg_list) metadata_mutex = Lock() def collect_result(future): self.metadata_progress += 1 self._callback('repo_metadata', int((self.metadata_progress+1)*100//self.total_pkgs)) def process_pkg(filename, href): pkg = createrepo.package_from_rpm(filename) pkg.location_href = href return pkg try: from concurrent.futures import ThreadPoolExecutor parallelize = True except: parallelize = False if parallelize: with ThreadPoolExecutor(max_workers=self._workers) as executor: futures = [] for filename in pkg_list: future = executor.submit(process_pkg, filename[0], filename[1]) future.add_done_callback(collect_result) futures.append(future) for future in futures: try: pkg = future.result(10) except Exception as exc: logging.exception("Thread generated an exception") else: pri_xml.add_pkg(pkg) fil_xml.add_pkg(pkg) oth_xml.add_pkg(pkg) pri_db.add_pkg(pkg) fil_db.add_pkg(pkg) oth_db.add_pkg(pkg) else: for idx, filename in enumerate(pkg_list): process_pkg(filename[0], filename[1]) collect_result(None) pri_xml.close() fil_xml.close() oth_xml.close() # Note: DBs are still open! We have to calculate checksums of xml files # and insert them to the databases first! self._callback('repo_metadata', 'building') # Prepare repomd.xml repomd = createrepo.Repomd() # Order is important ! repomdrecords = (("primary", pri_xml_path, pri_db, False), ("filelists", fil_xml_path, fil_db, False), ("other", oth_xml_path, oth_db, False), ("primary_db", pri_db_path, None, True), ("filelists_db", fil_db_path, None, True), ("other_db", oth_db_path, None, True)) for name, path, db_to_update, compress in repomdrecords: record = createrepo.RepomdRecord(name, path) if compress: record.compress_and_fill(createrepo.SHA256, createrepo.XZ_COMPRESSION) else: record.fill(createrepo.SHA256) if (db_to_update): db_to_update.dbinfo_update(record.checksum) db_to_update.close() repomd.set_record(record) if self._repomd: for md_type, md_content in six.iteritems(self._repomd): md_file = os.path.join(repodata_path, md_type[1]) with open(md_file, 'w') as f: f.write(md_content) record = createrepo.RepomdRecord(md_type[0], md_file) record.fill(createrepo.SHA256) repomd.set_record(record) open(repomd_path, "w").write(repomd.xml_dump()) return staging
def do_repodata(path): # Prepare repodata/ directory repodata_path = os.path.join(path, "repodata") if os.path.exists(repodata_path): x = 0 while True: new_repodata_path = "%s_%s" % (repodata_path, x) if not os.path.exists(new_repodata_path): shutil.move(repodata_path, new_repodata_path) break x += 1 os.mkdir(repodata_path) # Prepare metadata files repomd_path = os.path.join(repodata_path, "repomd.xml") pri_xml_path = os.path.join(repodata_path, "primary.xml.gz") fil_xml_path = os.path.join(repodata_path, "filelists.xml.gz") oth_xml_path = os.path.join(repodata_path, "other.xml.gz") pri_db_path = os.path.join(repodata_path, "primary.sqlite") fil_db_path = os.path.join(repodata_path, "filelists.sqlite") oth_db_path = os.path.join(repodata_path, "other.sqlite") pri_xml = cr.PrimaryXmlFile(pri_xml_path) fil_xml = cr.FilelistsXmlFile(fil_xml_path) oth_xml = cr.OtherXmlFile(oth_xml_path) pri_db = cr.PrimarySqlite(pri_db_path) fil_db = cr.FilelistsSqlite(fil_db_path) oth_db = cr.OtherSqlite(oth_db_path) # List directory and prepare list of files to process pkg_list = [] for filename in os.listdir(path): filename = os.path.join(path, filename) if os.path.isfile(filename) and filename.endswith(".rpm"): pkg_list.append(filename) pri_xml.set_num_of_pkgs(len(pkg_list)) fil_xml.set_num_of_pkgs(len(pkg_list)) oth_xml.set_num_of_pkgs(len(pkg_list)) # Process all packages for filename in pkg_list: pkg = cr.package_from_rpm(filename) pkg.location_href = os.path.basename(filename) print "Processing: %s" % pkg.nevra() pri_xml.add_pkg(pkg) fil_xml.add_pkg(pkg) oth_xml.add_pkg(pkg) pri_db.add_pkg(pkg) fil_db.add_pkg(pkg) oth_db.add_pkg(pkg) pri_xml.close() fil_xml.close() oth_xml.close() # Note: DBs are still open! We have to calculate checksums of xml files # and insert them to the databases first! # Prepare repomd.xml repomd = cr.Repomd() # Add records into the repomd.xml repomdrecords = (("primary", pri_xml_path, pri_db), ("filelists", fil_xml_path, fil_db), ("other", oth_xml_path, oth_db), ("primary_db", pri_db_path, None), ("filelists_db", fil_db_path, None), ("other_db", oth_db_path, None)) for name, path, db_to_update in repomdrecords: record = cr.RepomdRecord(name, path) record.fill(cr.SHA256) if (db_to_update): db_to_update.dbinfo_update(record.checksum) db_to_update.close() repomd.set_record(record) # Write repomd.xml open(repomd_path, "w").write(repomd.xml_dump())
def publish(publisher_pk, repository_version_pk): """ Use provided publisher to create a Publication based on a RepositoryVersion. Args: publisher_pk (str): Use the publish settings provided by this publisher. repository_version_pk (str): Create a publication from this repository version. """ publisher = RpmPublisher.objects.get(pk=publisher_pk) repository_version = RepositoryVersion.objects.get( pk=repository_version_pk) log.info( _('Publishing: repository={repo}, version={version}, publisher={publisher}' ).format( repo=repository_version.repository.name, version=repository_version.number, publisher=publisher.name, )) with WorkingDirectory(): with Publication.create(repository_version, publisher) as publication: populate(publication) # Prepare metadata files repomd_path = os.path.join(os.getcwd(), "repomd.xml") pri_xml_path = os.path.join(os.getcwd(), "primary.xml.gz") fil_xml_path = os.path.join(os.getcwd(), "filelists.xml.gz") oth_xml_path = os.path.join(os.getcwd(), "other.xml.gz") pri_db_path = os.path.join(os.getcwd(), "primary.sqlite") fil_db_path = os.path.join(os.getcwd(), "filelists.sqlite") oth_db_path = os.path.join(os.getcwd(), "other.sqlite") upd_xml_path = os.path.join(os.getcwd(), "updateinfo.xml.gz") pri_xml = cr.PrimaryXmlFile(pri_xml_path) fil_xml = cr.FilelistsXmlFile(fil_xml_path) oth_xml = cr.OtherXmlFile(oth_xml_path) pri_db = cr.PrimarySqlite(pri_db_path) fil_db = cr.FilelistsSqlite(fil_db_path) oth_db = cr.OtherSqlite(oth_db_path) upd_xml = cr.UpdateInfoXmlFile(upd_xml_path) artifacts = publication.published_artifact.all() pri_xml.set_num_of_pkgs(len(artifacts)) fil_xml.set_num_of_pkgs(len(artifacts)) oth_xml.set_num_of_pkgs(len(artifacts)) # Process all packages for artifact in artifacts: # TODO: pass attributes from db rather than use the filesystem pkg = cr.package_from_rpm( artifact.content_artifact.artifact.file.path) pkg.location_href = artifact.content_artifact.relative_path pri_xml.add_pkg(pkg) fil_xml.add_pkg(pkg) oth_xml.add_pkg(pkg) pri_db.add_pkg(pkg) fil_db.add_pkg(pkg) oth_db.add_pkg(pkg) # Process update records for update_record in UpdateRecord.objects.filter( pk__in=publication.repository_version.content): upd_xml.add_chunk(update_record_xml(update_record)) pri_xml.close() fil_xml.close() oth_xml.close() upd_xml.close() repomd = cr.Repomd() repomdrecords = (("primary", pri_xml_path, pri_db), ("filelists", fil_xml_path, fil_db), ("other", oth_xml_path, oth_db), ("primary_db", pri_db_path, None), ("filelists_db", fil_db_path, None), ("other_db", oth_db_path, None), ("updateinfo", upd_xml_path, None)) for name, path, db_to_update in repomdrecords: record = cr.RepomdRecord(name, path) record.fill(cr.SHA256) if (db_to_update): db_to_update.dbinfo_update(record.checksum) db_to_update.close() repomd.set_record(record) metadata = PublishedMetadata( relative_path=os.path.join(REPODATA_PATH, os.path.basename(path)), publication=publication, file=File(open(os.path.basename(path), 'rb'))) metadata.save() open(repomd_path, "w").write(repomd.xml_dump()) metadata = PublishedMetadata( relative_path=os.path.join(REPODATA_PATH, os.path.basename(repomd_path)), publication=publication, file=File(open(os.path.basename(repomd_path), 'rb'))) metadata.save()