def test_contentstat(self):
pkg = cr.package_from_rpm(PKG_ARCHER_PATH)
self.assertTrue(pkg)
pkg.time_file = 1
pkg.time_build = 1
cs = cr.ContentStat(cr.SHA256)
self.assertEqual(cs.size, 0)
self.assertEqual(cs.checksum_type, cr.SHA256)
self.assertEqual(cs.checksum, None)
path = os.path.join(self.tmpdir, "primary.xml.gz")
f = cr.PrimaryXmlFile(path, cr.GZ_COMPRESSION, cs)
self.assertTrue(f)
self.assertTrue(os.path.isfile(path))
f.add_pkg(pkg)
f.close()
self.assertTrue(os.path.isfile(path))
self.assertEqual(cs.size, 2668)
self.assertEqual(cs.checksum_type, cr.SHA256)
self.assertEqual(cs.checksum, "67bc6282915fad80dc11f3d7c3210977a0bde"\
"05a762256d86083c2447d425776")
def test_xmlfile_add_chunk(self):
chunk = " <chunk>Some XML chunk</chunk>\n"
# Primary
path = os.path.join(self.tmpdir, "primary.xml")
f = cr.PrimaryXmlFile(path, cr.NO_COMPRESSION)
self.assertTrue(f)
self.assertTrue(os.path.isfile(path))
f.add_chunk(chunk)
self.assertRaises(TypeError, f.add_chunk, None)
self.assertRaises(TypeError, f.add_chunk, 123)
self.assertRaises(TypeError, f.add_chunk, [1])
self.assertRaises(TypeError, f.add_chunk, ["foo"])
f.close()
self.assertTrue(os.path.isfile(path))
self.assertEqual(
open(path).read(), """<?xml version="1.0" encoding="UTF-8"?>
<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="0">
<chunk>Some XML chunk</chunk>
</metadata>""")
# Filelists
path = os.path.join(self.tmpdir, "filelists.xml")
f = cr.FilelistsXmlFile(path, cr.NO_COMPRESSION)
self.assertTrue(f)
self.assertTrue(os.path.isfile(path))
f.add_chunk(chunk)
self.assertRaises(TypeError, f.add_chunk, None)
self.assertRaises(TypeError, f.add_chunk, 123)
self.assertRaises(TypeError, f.add_chunk, [1])
self.assertRaises(TypeError, f.add_chunk, ["foo"])
f.close()
self.assertTrue(os.path.isfile(path))
self.assertEqual(
open(path).read(), """<?xml version="1.0" encoding="UTF-8"?>
<filelists xmlns="http://linux.duke.edu/metadata/filelists" packages="0">
<chunk>Some XML chunk</chunk>
</filelists>""")
# Other
path = os.path.join(self.tmpdir, "other.xml")
f = cr.OtherXmlFile(path, cr.NO_COMPRESSION)
self.assertTrue(f)
self.assertTrue(os.path.isfile(path))
f.add_chunk(chunk)
self.assertRaises(TypeError, f.add_chunk, None)
self.assertRaises(TypeError, f.add_chunk, 123)
self.assertRaises(TypeError, f.add_chunk, [1])
self.assertRaises(TypeError, f.add_chunk, ["foo"])
f.close()
self.assertTrue(os.path.isfile(path))
self.assertEqual(
open(path).read(), """<?xml version="1.0" encoding="UTF-8"?>
<otherdata xmlns="http://linux.duke.edu/metadata/other" packages="0">
<chunk>Some XML chunk</chunk>
</otherdata>""")
def test_xmlfile_no_compression(self):
path = os.path.join(self.tmpdir, "primary.xml")
f = cr.PrimaryXmlFile(path, cr.NO_COMPRESSION)
self.assertTrue(f)
self.assertTrue(os.path.isfile(path))
f.close()
with open(path) as primary:
self.assertEqual(primary.read(),
"""<?xml version="1.0" encoding="UTF-8"?>
<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="0">
</metadata>""")
def test_xmlfile_gz_compression(self):
path = os.path.join(self.tmpdir, "primary.xml.gz")
f = cr.PrimaryXmlFile(path, cr.GZ_COMPRESSION)
self.assertTrue(f)
self.assertTrue(os.path.isfile(path))
f.close()
import gzip
content = gzip.open(path).read().decode('utf-8')
self.assertEqual(
content, """<?xml version="1.0" encoding="UTF-8"?>
<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="0">
</metadata>""")
def test_xmlfile_set_num_of_pkgs(self):
path = os.path.join(self.tmpdir, "primary.xml")
f = cr.PrimaryXmlFile(path, cr.NO_COMPRESSION)
self.assertTrue(f)
self.assertTrue(os.path.isfile(path))
f.set_num_of_pkgs(22)
f.close()
content = open(path).read()
self.assertEqual(
content, """<?xml version="1.0" encoding="UTF-8"?>
<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="22">
</metadata>""")
def test_xmlfile_xz_compression(self):
path = os.path.join(self.tmpdir, "primary.xml.xz")
f = cr.PrimaryXmlFile(path, cr.XZ_COMPRESSION)
self.assertTrue(f)
self.assertTrue(os.path.isfile(path))
f.close()
import subprocess
with subprocess.Popen(["unxz", "--stdout", path], stdout=subprocess.PIPE) as p:
content = p.stdout.read().decode('utf-8')
self.assertEqual(content,
"""<?xml version="1.0" encoding="UTF-8"?>
<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="0">
</metadata>""")
def test_xmlfile_operations_on_closed_file(self):
# Already closed file
path = os.path.join(self.tmpdir, "primary.xml.gz")
pkg = cr.package_from_rpm(PKG_ARCHER_PATH)
self.assertTrue(pkg)
f = cr.PrimaryXmlFile(path, cr.GZ_COMPRESSION)
self.assertTrue(f)
self.assertTrue(os.path.isfile(path))
f.close()
self.assertRaises(cr.CreaterepoCError, f.set_num_of_pkgs, 1)
self.assertRaises(cr.CreaterepoCError, f.add_pkg, pkg)
self.assertRaises(cr.CreaterepoCError, f.add_chunk, "<chunk>text</chunk>")
self.assertEqual("<createrepo_c.XmlFile Closed object>", f.__str__())
f.close() # No error should be raised
del(f) # No error should be raised
def test_contentstat_ref_in_xmlfile(self):
"""Test if reference is saved properly"""
pkg = cr.package_from_rpm(PKG_ARCHER_PATH)
self.assertTrue(pkg)
pkg.time_file = 1
pkg.time_build = 1
cs = cr.ContentStat(cr.SHA256)
self.assertEqual(cs.size, 0)
self.assertEqual(cs.checksum_type, cr.SHA256)
self.assertEqual(cs.checksum, None)
path = os.path.join(self.tmpdir, "primary.xml.gz")
f = cr.PrimaryXmlFile(path, cr.GZ_COMPRESSION, cs)
self.assertTrue(f)
self.assertTrue(os.path.isfile(path))
del cs
f.add_pkg(pkg)
f.close()
self.assertTrue(os.path.isfile(path))
def create_repomd_xml(content, publication, extra_repomdrecords, sub_folder=None):
"""
Creates a repomd.xml file.
Args:
content(app.models.Content): content set
publication(pulpcore.plugin.models.Publication): the publication
extra_repomdrecords(list): list with data relative to repo metadata files
sub_folder(str): name of the folder for sub repos
"""
cwd = os.getcwd()
repodata_path = REPODATA_PATH
has_modules = False
has_comps = False
if sub_folder:
cwd = os.path.join(cwd, sub_folder)
repodata_path = os.path.join(sub_folder, repodata_path)
# Prepare metadata files
repomd_path = os.path.join(cwd, "repomd.xml")
pri_xml_path = os.path.join(cwd, "primary.xml.gz")
fil_xml_path = os.path.join(cwd, "filelists.xml.gz")
oth_xml_path = os.path.join(cwd, "other.xml.gz")
pri_db_path = os.path.join(cwd, "primary.sqlite")
fil_db_path = os.path.join(cwd, "filelists.sqlite")
oth_db_path = os.path.join(cwd, "other.sqlite")
upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz")
mod_yml_path = os.path.join(cwd, "modules.yaml")
comps_xml_path = os.path.join(cwd, "comps.xml")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
packages = Package.objects.filter(pk__in=content)
total_packages = packages.count()
pri_xml.set_num_of_pkgs(total_packages)
fil_xml.set_num_of_pkgs(total_packages)
oth_xml.set_num_of_pkgs(total_packages)
# Process all packages
for package in packages.iterator():
pkg = package.to_createrepo_c()
pkg.location_href = package.contentartifact_set.only('relative_path').first().relative_path
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(pk__in=content).iterator():
upd_xml.add_chunk(cr.xml_dump_updaterecord(update_record.to_createrepo_c()))
# Process modulemd and modulemd_defaults
with open(mod_yml_path, 'ab') as mod_yml:
for modulemd in Modulemd.objects.filter(pk__in=content).iterator():
mod_yml.write(modulemd._artifacts.get().file.read())
has_modules = True
for default in ModulemdDefaults.objects.filter(pk__in=content).iterator():
mod_yml.write(default._artifacts.get().file.read())
has_modules = True
# Process comps
comps = libcomps.Comps()
for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator():
group = pkg_grp.pkg_grp_to_libcomps()
comps.groups.append(group)
has_comps = True
for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator():
cat = pkg_cat.pkg_cat_to_libcomps()
comps.categories.append(cat)
has_comps = True
for pkg_env in PackageEnvironment.objects.filter(pk__in=content).iterator():
env = pkg_env.pkg_env_to_libcomps()
comps.environments.append(env)
has_comps = True
for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator():
comps.langpacks = dict_to_strdict(pkg_lng.matches)
has_comps = True
comps.toxml_f(comps_xml_path, xml_options={"default_explicit": True,
"empty_groups": True,
"uservisible_explicit": True})
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
repomdrecords = [("primary", pri_xml_path, pri_db),
("filelists", fil_xml_path, fil_db),
("other", oth_xml_path, oth_db),
("primary_db", pri_db_path, None),
("filelists_db", fil_db_path, None),
("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None)]
if has_modules:
repomdrecords.append(("modules", mod_yml_path, None))
if has_comps:
repomdrecords.append(("group", comps_xml_path, None))
repomdrecords.extend(extra_repomdrecords)
sqlite_files = ("primary_db", "filelists_db", "other_db")
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
if name in sqlite_files:
record_bz = record.compress_and_fill(cr.SHA256, cr.BZ2)
record_bz.type = name
record_bz.rename_file()
path = record_bz.location_href.split('/')[-1]
repomd.set_record(record_bz)
else:
record.fill(cr.SHA256)
if (db_to_update):
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
record.rename_file()
path = record.location_href.split('/')[-1]
repomd.set_record(record)
if sub_folder:
path = os.path.join(sub_folder, path)
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path, os.path.basename(path)),
publication=publication,
file=File(open(path, 'rb'))
)
with open(repomd_path, "w") as repomd_f:
repomd_f.write(repomd.xml_dump())
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path, os.path.basename(repomd_path)),
publication=publication,
file=File(open(repomd_path, 'rb'))
)
def test_xmlfile_add_pkg(self):
pkg = cr.package_from_rpm(PKG_ARCHER_PATH)
self.assertTrue(pkg)
pkg.time_file = 111
# Primary
path = os.path.join(self.tmpdir, "primary.xml")
f = cr.PrimaryXmlFile(path, cr.NO_COMPRESSION)
self.assertTrue(f)
self.assertTrue(os.path.isfile(path))
f.add_pkg(pkg)
self.assertRaises(TypeError, f.add_pkg, None)
self.assertRaises(TypeError, f.add_pkg, 123)
self.assertRaises(TypeError, f.add_pkg, "foo")
self.assertRaises(TypeError, f.add_pkg, [456])
f.close()
self.assertTrue(os.path.isfile(path))
with open(path) as primary:
self.assertEqual(primary.read(),
"""<?xml version="1.0" encoding="UTF-8"?>
<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="0">
<package type="rpm">
<name>Archer</name>
<arch>x86_64</arch>
<version epoch="2" ver="3.4.5" rel="6"/>
<checksum type="sha256" pkgid="YES">4e0b775220c67f0f2c1fd2177e626b9c863a098130224ff09778ede25cea9a9e</checksum>
<summary>Complex package.</summary>
<description>Archer package</description>
<packager>Sterling Archer</packager>
<url>http://soo_complex_package.eu/</url>
<time file="111" build="1365416480"/>
<size package="3101" installed="0" archive="544"/>
<location href=""/>
<format>
<rpm:license>GPL</rpm:license>
<rpm:vendor>ISIS</rpm:vendor>
<rpm:group>Development/Tools</rpm:group>
<rpm:buildhost>localhost.localdomain</rpm:buildhost>
<rpm:sourcerpm>Archer-3.4.5-6.src.rpm</rpm:sourcerpm>
<rpm:header-range start="280" end="2865"/>
<rpm:provides>
<rpm:entry name="bara" flags="LE" epoch="0" ver="22"/>
<rpm:entry name="barb" flags="GE" epoch="0" ver="11.22.33" rel="44"/>
<rpm:entry name="barc" flags="EQ" epoch="0" ver="33"/>
<rpm:entry name="bard" flags="LT" epoch="0" ver="44"/>
<rpm:entry name="bare" flags="GT" epoch="0" ver="55"/>
<rpm:entry name="Archer" flags="EQ" epoch="2" ver="3.4.5" rel="6"/>
<rpm:entry name="Archer(x86-64)" flags="EQ" epoch="2" ver="3.4.5" rel="6"/>
</rpm:provides>
<rpm:requires>
<rpm:entry name="fooa" flags="LE" epoch="0" ver="2"/>
<rpm:entry name="foob" flags="GE" epoch="0" ver="1.0.0" rel="1"/>
<rpm:entry name="fooc" flags="EQ" epoch="0" ver="3"/>
<rpm:entry name="food" flags="LT" epoch="0" ver="4"/>
<rpm:entry name="fooe" flags="GT" epoch="0" ver="5"/>
<rpm:entry name="foof" flags="EQ" epoch="0" ver="6" pre="1"/>
</rpm:requires>
<rpm:conflicts>
<rpm:entry name="bba" flags="LE" epoch="0" ver="2222"/>
<rpm:entry name="bbb" flags="GE" epoch="0" ver="1111.2222.3333" rel="4444"/>
<rpm:entry name="bbc" flags="EQ" epoch="0" ver="3333"/>
<rpm:entry name="bbd" flags="LT" epoch="0" ver="4444"/>
<rpm:entry name="bbe" flags="GT" epoch="0" ver="5555"/>
</rpm:conflicts>
<rpm:obsoletes>
<rpm:entry name="aaa" flags="LE" epoch="0" ver="222"/>
<rpm:entry name="aab" flags="GE" epoch="0" ver="111.2.3" rel="4"/>
<rpm:entry name="aac" flags="EQ" epoch="0" ver="333"/>
<rpm:entry name="aad" flags="LT" epoch="0" ver="444"/>
<rpm:entry name="aae" flags="GT" epoch="0" ver="555"/>
</rpm:obsoletes>
<file>/usr/bin/complex_a</file>
</format>
</package>
</metadata>""")
# Filelists
path = os.path.join(self.tmpdir, "filelists.xml")
f = cr.FilelistsXmlFile(path, cr.NO_COMPRESSION)
self.assertTrue(f)
self.assertTrue(os.path.isfile(path))
f.add_pkg(pkg)
self.assertRaises(TypeError, f.add_pkg, None)
self.assertRaises(TypeError, f.add_pkg, 123)
self.assertRaises(TypeError, f.add_pkg, "foo")
self.assertRaises(TypeError, f.add_pkg, [456])
f.close()
self.assertTrue(os.path.isfile(path))
with open(path) as filelists:
self.assertEqual(filelists.read(),
"""<?xml version="1.0" encoding="UTF-8"?>
<filelists xmlns="http://linux.duke.edu/metadata/filelists" packages="0">
<package pkgid="4e0b775220c67f0f2c1fd2177e626b9c863a098130224ff09778ede25cea9a9e" name="Archer" arch="x86_64">
<version epoch="2" ver="3.4.5" rel="6"/>
<file>/usr/bin/complex_a</file>
<file type="dir">/usr/share/doc/Archer-3.4.5</file>
<file>/usr/share/doc/Archer-3.4.5/README</file>
</package>
</filelists>""")
# Other
path = os.path.join(self.tmpdir, "other.xml")
f = cr.OtherXmlFile(path, cr.NO_COMPRESSION)
self.assertTrue(f)
self.assertTrue(os.path.isfile(path))
f.add_pkg(pkg)
self.assertRaises(TypeError, f.add_pkg, None)
self.assertRaises(TypeError, f.add_pkg, 123)
self.assertRaises(TypeError, f.add_pkg, "foo")
self.assertRaises(TypeError, f.add_pkg, [456])
f.close()
self.assertTrue(os.path.isfile(path))
with open(path) as other:
self.assertEqual(other.read(),
"""<?xml version="1.0" encoding="UTF-8"?>
<otherdata xmlns="http://linux.duke.edu/metadata/other" packages="0">
<package pkgid="4e0b775220c67f0f2c1fd2177e626b9c863a098130224ff09778ede25cea9a9e" name="Archer" arch="x86_64">
<version epoch="2" ver="3.4.5" rel="6"/>
<changelog author="Tomas Mlcoch <[email protected]> - 1.1.1-1" date="1334664000">- First changelog.</changelog>
<changelog author="Tomas Mlcoch <[email protected]> - 2.2.2-2" date="1334750400">- That was totally ninja!</changelog>
<changelog author="Tomas Mlcoch <[email protected]> - 3.3.3-3" date="1365422400">- 3. changelog.</changelog>
</package>
</otherdata>""")
def create_rempomd_xml(packages,
publication,
extra_repomdrecords,
sub_folder=None):
"""
Creates a repomd.xml file.
Args:
packages(app.models.Package): set of packages
publication(pulpcore.plugin.models.Publication): the publication
extra_repomdrecords(list): list with data relative to repo metadata files
sub_folder(str): name of the folder for sub repos
"""
cwd = os.getcwd()
repodata_path = REPODATA_PATH
has_modules = False
if sub_folder:
cwd = os.path.join(cwd, sub_folder)
repodata_path = os.path.join(sub_folder, repodata_path)
# Prepare metadata files
repomd_path = os.path.join(cwd, "repomd.xml")
pri_xml_path = os.path.join(cwd, "primary.xml.gz")
fil_xml_path = os.path.join(cwd, "filelists.xml.gz")
oth_xml_path = os.path.join(cwd, "other.xml.gz")
pri_db_path = os.path.join(cwd, "primary.sqlite")
fil_db_path = os.path.join(cwd, "filelists.sqlite")
oth_db_path = os.path.join(cwd, "other.sqlite")
upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz")
mod_yml_path = os.path.join(cwd, "modules.yaml")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
pri_xml.set_num_of_pkgs(len(packages))
fil_xml.set_num_of_pkgs(len(packages))
oth_xml.set_num_of_pkgs(len(packages))
# Process all packages
for package in packages:
pkg = package.to_createrepo_c()
pkg.location_href = package.contentartifact_set.first().relative_path
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=publication.repository_version.content):
upd_xml.add_chunk(update_record_xml(update_record))
# Process modulemd and modulemd_defaults
with open(mod_yml_path, 'ab') as mod_yml:
for modulemd in Modulemd.objects.filter(
pk__in=publication.repository_version.content):
mod_yml.write(modulemd._artifacts.get().file.read())
has_modules = True
for default in ModulemdDefaults.objects.filter(
pk__in=publication.repository_version.content):
mod_yml.write(default._artifacts.get().file.read())
has_modules = True
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
repomdrecords = [("primary", pri_xml_path, pri_db),
("filelists", fil_xml_path, fil_db),
("other", oth_xml_path, oth_db),
("primary_db", pri_db_path, None),
("filelists_db", fil_db_path, None),
("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None)]
if has_modules:
repomdrecords.append(("modules", mod_yml_path, None))
repomdrecords.extend(extra_repomdrecords)
sqlite_files = ("primary_db", "filelists_db", "other_db")
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
if name in sqlite_files:
record_bz = record.compress_and_fill(cr.SHA256, cr.BZ2)
record_bz.type = name
record_bz.rename_file()
path = record_bz.location_href.split('/')[-1]
repomd.set_record(record_bz)
elif name == "modules":
record_md = record.compress_and_fill(cr.SHA256, cr.GZ)
record_md.type = name
record_md.rename_file()
path = record_md.location_href.split('/')[-1]
repomd.set_record(record_md)
else:
record.fill(cr.SHA256)
if (db_to_update):
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
record.rename_file()
path = record.location_href.split('/')[-1]
repomd.set_record(record)
if sub_folder:
path = os.path.join(sub_folder, path)
PublishedMetadata.create_from_file(relative_path=os.path.join(
repodata_path, os.path.basename(path)),
publication=publication,
file=File(open(path, 'rb')))
with open(repomd_path, "w") as repomd_f:
repomd_f.write(repomd.xml_dump())
PublishedMetadata.create_from_file(relative_path=os.path.join(
repodata_path, os.path.basename(repomd_path)),
publication=publication,
file=File(open(repomd_path, 'rb')))
def create_repomd_xml(content,
publication,
checksum_types,
extra_repomdrecords,
sub_folder=None,
metadata_signing_service=None):
"""
Creates a repomd.xml file.
Args:
content(app.models.Content): content set
publication(pulpcore.plugin.models.Publication): the publication
extra_repomdrecords(list): list with data relative to repo metadata files
sub_folder(str): name of the folder for sub repos
metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService):
A reference to an associated signing service.
"""
cwd = os.getcwd()
repodata_path = REPODATA_PATH
has_modules = False
has_comps = False
package_checksum_type = checksum_types.get("package")
if sub_folder:
cwd = os.path.join(cwd, sub_folder)
repodata_path = os.path.join(sub_folder, repodata_path)
# Prepare metadata files
repomd_path = os.path.join(cwd, "repomd.xml")
pri_xml_path = os.path.join(cwd, "primary.xml.gz")
fil_xml_path = os.path.join(cwd, "filelists.xml.gz")
oth_xml_path = os.path.join(cwd, "other.xml.gz")
pri_db_path = os.path.join(cwd, "primary.sqlite")
fil_db_path = os.path.join(cwd, "filelists.sqlite")
oth_db_path = os.path.join(cwd, "other.sqlite")
upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz")
mod_yml_path = os.path.join(cwd, "modules.yaml")
comps_xml_path = os.path.join(cwd, "comps.xml")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
packages = Package.objects.filter(pk__in=content)
total_packages = packages.count()
pri_xml.set_num_of_pkgs(total_packages)
fil_xml.set_num_of_pkgs(total_packages)
oth_xml.set_num_of_pkgs(total_packages)
# Process all packages
for package in packages.iterator():
pkg = package.to_createrepo_c(package_checksum_type)
pkg_filename = os.path.basename(package.location_href)
# this can cause an issue when two same RPM package names appears
# a/name1.rpm b/name1.rpm
pkg.location_href = os.path.join(PACKAGES_DIRECTORY,
pkg_filename[0].lower(), pkg_filename)
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=content).iterator():
upd_xml.add_chunk(
cr.xml_dump_updaterecord(update_record.to_createrepo_c()))
# Process modulemd and modulemd_defaults
with open(mod_yml_path, 'ab') as mod_yml:
for modulemd in Modulemd.objects.filter(pk__in=content).iterator():
mod_yml.write(modulemd._artifacts.get().file.read())
has_modules = True
for default in ModulemdDefaults.objects.filter(
pk__in=content).iterator():
mod_yml.write(default._artifacts.get().file.read())
has_modules = True
# Process comps
comps = libcomps.Comps()
for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator():
group = pkg_grp.pkg_grp_to_libcomps()
comps.groups.append(group)
has_comps = True
for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator():
cat = pkg_cat.pkg_cat_to_libcomps()
comps.categories.append(cat)
has_comps = True
for pkg_env in PackageEnvironment.objects.filter(
pk__in=content).iterator():
env = pkg_env.pkg_env_to_libcomps()
comps.environments.append(env)
has_comps = True
for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator():
comps.langpacks = dict_to_strdict(pkg_lng.matches)
has_comps = True
comps.toxml_f(comps_xml_path,
xml_options={
"default_explicit": True,
"empty_groups": True,
"uservisible_explicit": True
})
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
repomdrecords = [("primary", pri_xml_path, pri_db),
("filelists", fil_xml_path, fil_db),
("other", oth_xml_path, oth_db),
("primary_db", pri_db_path, None),
("filelists_db", fil_db_path, None),
("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None)]
if has_modules:
repomdrecords.append(("modules", mod_yml_path, None))
if has_comps:
repomdrecords.append(("group", comps_xml_path, None))
repomdrecords.extend(extra_repomdrecords)
sqlite_files = ("primary_db", "filelists_db", "other_db")
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
checksum_type = get_checksum_type(name, checksum_types)
if name in sqlite_files:
record_bz = record.compress_and_fill(checksum_type, cr.BZ2)
record_bz.type = name
record_bz.rename_file()
path = record_bz.location_href.split('/')[-1]
repomd.set_record(record_bz)
else:
record.fill(checksum_type)
if (db_to_update):
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
record.rename_file()
path = record.location_href.split('/')[-1]
repomd.set_record(record)
if sub_folder:
path = os.path.join(sub_folder, path)
PublishedMetadata.create_from_file(relative_path=os.path.join(
repodata_path, os.path.basename(path)),
publication=publication,
file=File(open(path, 'rb')))
with open(repomd_path, "w") as repomd_f:
repomd_f.write(repomd.xml_dump())
if metadata_signing_service:
signing_service = AsciiArmoredDetachedSigningService.objects.get(
pk=metadata_signing_service.pk)
sign_results = signing_service.sign(repomd_path)
# publish a signed file
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(sign_results['file'])),
publication=publication,
file=File(open(sign_results['file'], 'rb')))
# publish a detached signature
PublishedMetadata.create_from_file(relative_path=os.path.join(
repodata_path, os.path.basename(sign_results['signature'])),
publication=publication,
file=File(
open(sign_results['signature'],
'rb')))
# publish a public key required for further verification
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(sign_results['key'])),
publication=publication,
file=File(open(sign_results['key'], 'rb')))
else:
PublishedMetadata.create_from_file(relative_path=os.path.join(
repodata_path, os.path.basename(repomd_path)),
publication=publication,
file=File(open(repomd_path, 'rb')))
def build_metadata(self):
staging = tempfile.mkdtemp(prefix='yumsync-', suffix='-metadata')
if self.checksum == 'sha' or self.checksum == 'sha1':
sumtype = 'sha'
else:
sumtype = 'sha256'
repodata_path = os.path.join(staging, 'repodata')
os.mkdir(repodata_path)
# Prepare metadata files
repomd_path = os.path.join(repodata_path, "repomd.xml")
pri_xml_path = os.path.join(repodata_path, "primary.xml.gz")
fil_xml_path = os.path.join(repodata_path, "filelists.xml.gz")
oth_xml_path = os.path.join(repodata_path, "other.xml.gz")
pri_db_path = os.path.join(repodata_path, "primary.sqlite")
fil_db_path = os.path.join(repodata_path, "filelists.sqlite")
oth_db_path = os.path.join(repodata_path, "other.sqlite")
# Related python objects
pri_xml = createrepo.PrimaryXmlFile(pri_xml_path)
fil_xml = createrepo.FilelistsXmlFile(fil_xml_path)
oth_xml = createrepo.OtherXmlFile(oth_xml_path)
pri_db = createrepo.PrimarySqlite(pri_db_path)
fil_db = createrepo.FilelistsSqlite(fil_db_path)
oth_db = createrepo.OtherSqlite(oth_db_path)
# Set package list
if self.local_dir and self.link_type == "individual_symlink" and self.version_dir:
pkg_list = [
(
os.path.join(self.version_package_dir,"{}".format(pkg)),
os.path.join("packages","{}".format(pkg))
) for pkg in self._packages]
else:
pkg_list = [
(
os.path.join(self.package_dir,"{}".format(pkg)),
os.path.join("packages","{}".format(pkg))
)for pkg in self._packages]
pri_xml.set_num_of_pkgs(len(pkg_list))
fil_xml.set_num_of_pkgs(len(pkg_list))
oth_xml.set_num_of_pkgs(len(pkg_list))
# Process all packages in // if possible
self.metadata_progress = 0
self.total_pkgs = len(pkg_list)
metadata_mutex = Lock()
def collect_result(future):
self.metadata_progress += 1
self._callback('repo_metadata', int((self.metadata_progress+1)*100//self.total_pkgs))
def process_pkg(filename, href):
pkg = createrepo.package_from_rpm(filename)
pkg.location_href = href
return pkg
try:
from concurrent.futures import ThreadPoolExecutor
parallelize = True
except:
parallelize = False
if parallelize:
with ThreadPoolExecutor(max_workers=self._workers) as executor:
futures = []
for filename in pkg_list:
future = executor.submit(process_pkg, filename[0], filename[1])
future.add_done_callback(collect_result)
futures.append(future)
for future in futures:
try:
pkg = future.result(10)
except Exception as exc:
logging.exception("Thread generated an exception")
else:
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
else:
for idx, filename in enumerate(pkg_list):
process_pkg(filename[0], filename[1])
collect_result(None)
pri_xml.close()
fil_xml.close()
oth_xml.close()
# Note: DBs are still open! We have to calculate checksums of xml files
# and insert them to the databases first!
self._callback('repo_metadata', 'building')
# Prepare repomd.xml
repomd = createrepo.Repomd()
# Order is important !
repomdrecords = (("primary", pri_xml_path, pri_db, False),
("filelists", fil_xml_path, fil_db, False),
("other", oth_xml_path, oth_db, False),
("primary_db", pri_db_path, None, True),
("filelists_db", fil_db_path, None, True),
("other_db", oth_db_path, None, True))
for name, path, db_to_update, compress in repomdrecords:
record = createrepo.RepomdRecord(name, path)
if compress:
record.compress_and_fill(createrepo.SHA256, createrepo.XZ_COMPRESSION)
else:
record.fill(createrepo.SHA256)
if (db_to_update):
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
repomd.set_record(record)
if self._repomd:
for md_type, md_content in six.iteritems(self._repomd):
md_file = os.path.join(repodata_path, md_type[1])
with open(md_file, 'w') as f:
f.write(md_content)
record = createrepo.RepomdRecord(md_type[0], md_file)
record.fill(createrepo.SHA256)
repomd.set_record(record)
open(repomd_path, "w").write(repomd.xml_dump())
return staging
def do_repodata(path):
# Prepare repodata/ directory
repodata_path = os.path.join(path, "repodata")
if os.path.exists(repodata_path):
x = 0
while True:
new_repodata_path = "%s_%s" % (repodata_path, x)
if not os.path.exists(new_repodata_path):
shutil.move(repodata_path, new_repodata_path)
break
x += 1
os.mkdir(repodata_path)
# Prepare metadata files
repomd_path = os.path.join(repodata_path, "repomd.xml")
pri_xml_path = os.path.join(repodata_path, "primary.xml.gz")
fil_xml_path = os.path.join(repodata_path, "filelists.xml.gz")
oth_xml_path = os.path.join(repodata_path, "other.xml.gz")
pri_db_path = os.path.join(repodata_path, "primary.sqlite")
fil_db_path = os.path.join(repodata_path, "filelists.sqlite")
oth_db_path = os.path.join(repodata_path, "other.sqlite")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
# List directory and prepare list of files to process
pkg_list = []
for filename in os.listdir(path):
filename = os.path.join(path, filename)
if os.path.isfile(filename) and filename.endswith(".rpm"):
pkg_list.append(filename)
pri_xml.set_num_of_pkgs(len(pkg_list))
fil_xml.set_num_of_pkgs(len(pkg_list))
oth_xml.set_num_of_pkgs(len(pkg_list))
# Process all packages
for filename in pkg_list:
pkg = cr.package_from_rpm(filename)
pkg.location_href = os.path.basename(filename)
print "Processing: %s" % pkg.nevra()
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
pri_xml.close()
fil_xml.close()
oth_xml.close()
# Note: DBs are still open! We have to calculate checksums of xml files
# and insert them to the databases first!
# Prepare repomd.xml
repomd = cr.Repomd()
# Add records into the repomd.xml
repomdrecords = (("primary", pri_xml_path, pri_db),
("filelists", fil_xml_path, fil_db),
("other", oth_xml_path, oth_db),
("primary_db", pri_db_path, None),
("filelists_db", fil_db_path, None),
("other_db", oth_db_path, None))
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
record.fill(cr.SHA256)
if (db_to_update):
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
repomd.set_record(record)
# Write repomd.xml
open(repomd_path, "w").write(repomd.xml_dump())
def create_repomd_xml(
content,
publication,
checksum_types,
extra_repomdrecords,
sub_folder=None,
metadata_signing_service=None,
):
"""
Creates a repomd.xml file.
Args:
content(app.models.Content): content set
publication(pulpcore.plugin.models.Publication): the publication
extra_repomdrecords(list): list with data relative to repo metadata files
sub_folder(str): name of the folder for sub repos
metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService):
A reference to an associated signing service.
"""
cwd = os.getcwd()
repodata_path = REPODATA_PATH
has_modules = False
has_comps = False
package_checksum_type = checksum_types.get("package")
if sub_folder:
cwd = os.path.join(cwd, sub_folder)
repodata_path = os.path.join(sub_folder, repodata_path)
# Prepare metadata files
repomd_path = os.path.join(cwd, "repomd.xml")
pri_xml_path = os.path.join(cwd, "primary.xml.gz")
fil_xml_path = os.path.join(cwd, "filelists.xml.gz")
oth_xml_path = os.path.join(cwd, "other.xml.gz")
upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz")
mod_yml_path = os.path.join(cwd, "modules.yaml")
comps_xml_path = os.path.join(cwd, "comps.xml")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
if publication.sqlite_metadata:
pri_db_path = os.path.join(cwd, "primary.sqlite")
fil_db_path = os.path.join(cwd, "filelists.sqlite")
oth_db_path = os.path.join(cwd, "other.sqlite")
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
packages = Package.objects.filter(pk__in=content)
total_packages = packages.count()
pri_xml.set_num_of_pkgs(total_packages)
fil_xml.set_num_of_pkgs(total_packages)
oth_xml.set_num_of_pkgs(total_packages)
# We want to support publishing with a different checksum type than the one built-in to the
# package itself, so we need to get the correct checksums somehow if there is an override.
# We must also take into consideration that if the package has not been downloaded the only
# checksum that is available is the one built-in.
#
# Since this lookup goes from Package->Content->ContentArtifact->Artifact, performance is a
# challenge. We use ContentArtifact as our starting point because it enables us to work with
# simple foreign keys and avoid messing with the many-to-many relationship, which doesn't
# work with select_related() and performs poorly with prefetch_related(). This is fine
# because we know that Packages should only ever have one artifact per content.
contentartifact_qs = (
ContentArtifact.objects.filter(content__in=packages.only("pk")).
select_related(
# content__rpm_package is a bit of a hack, exploiting the way django sets up model
# inheritance, but it works and is unlikely to break. All content artifacts being
# accessed here have an associated Package since they originally came from the
# Package queryset.
"artifact",
"content__rpm_package",
).only("artifact", "content__rpm_package__checksum_type",
"content__rpm_package__pkgId"))
pkg_to_hash = {}
for ca in contentartifact_qs.iterator():
pkgid = None
if package_checksum_type:
package_checksum_type = package_checksum_type.lower()
pkgid = getattr(ca.artifact, package_checksum_type, None)
if pkgid:
pkg_to_hash[ca.content_id] = (package_checksum_type, pkgid)
else:
pkg_to_hash[ca.content_id] = (
ca.content.rpm_package.checksum_type,
ca.content.rpm_package.pkgId,
)
# Process all packages
for package in packages.iterator():
pkg = package.to_createrepo_c()
# rewrite the checksum and checksum type with the desired ones
(checksum, pkgId) = pkg_to_hash[package.pk]
pkg.checksum_type = checksum
pkg.pkgId = pkgId
pkg_filename = os.path.basename(package.location_href)
# this can cause an issue when two same RPM package names appears
# a/name1.rpm b/name1.rpm
pkg.location_href = os.path.join(PACKAGES_DIRECTORY,
pkg_filename[0].lower(), pkg_filename)
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
if publication.sqlite_metadata:
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=content).iterator():
upd_xml.add_chunk(
cr.xml_dump_updaterecord(update_record.to_createrepo_c()))
# Process modulemd and modulemd_defaults
with open(mod_yml_path, "ab") as mod_yml:
for modulemd in Modulemd.objects.filter(pk__in=content).iterator():
mod_yml.write(modulemd._artifacts.get().file.read())
has_modules = True
for default in ModulemdDefaults.objects.filter(
pk__in=content).iterator():
mod_yml.write(default._artifacts.get().file.read())
has_modules = True
# Process comps
comps = libcomps.Comps()
for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator():
group = pkg_grp.pkg_grp_to_libcomps()
comps.groups.append(group)
has_comps = True
for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator():
cat = pkg_cat.pkg_cat_to_libcomps()
comps.categories.append(cat)
has_comps = True
for pkg_env in PackageEnvironment.objects.filter(
pk__in=content).iterator():
env = pkg_env.pkg_env_to_libcomps()
comps.environments.append(env)
has_comps = True
for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator():
comps.langpacks = dict_to_strdict(pkg_lng.matches)
has_comps = True
comps.toxml_f(
comps_xml_path,
xml_options={
"default_explicit": True,
"empty_groups": True,
"uservisible_explicit": True
},
)
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
if publication.sqlite_metadata:
repomdrecords = [
("primary", pri_xml_path, pri_db),
("filelists", fil_xml_path, fil_db),
("other", oth_xml_path, oth_db),
("primary_db", pri_db_path, None),
("filelists_db", fil_db_path, None),
("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None),
]
else:
repomdrecords = [
("primary", pri_xml_path, None),
("filelists", fil_xml_path, None),
("other", oth_xml_path, None),
("updateinfo", upd_xml_path, None),
]
if has_modules:
repomdrecords.append(("modules", mod_yml_path, None))
if has_comps:
repomdrecords.append(("group", comps_xml_path, None))
repomdrecords.extend(extra_repomdrecords)
sqlite_files = ("primary_db", "filelists_db", "other_db")
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
checksum_type = get_checksum_type(name, checksum_types)
if name in sqlite_files:
record_bz = record.compress_and_fill(checksum_type, cr.BZ2)
record_bz.type = name
record_bz.rename_file()
path = record_bz.location_href.split("/")[-1]
repomd.set_record(record_bz)
else:
record.fill(checksum_type)
if db_to_update:
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
record.rename_file()
path = record.location_href.split("/")[-1]
repomd.set_record(record)
if sub_folder:
path = os.path.join(sub_folder, path)
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path, os.path.basename(path)),
publication=publication,
file=File(open(path, "rb")),
)
with open(repomd_path, "w") as repomd_f:
repomd_f.write(repomd.xml_dump())
if metadata_signing_service:
signing_service = AsciiArmoredDetachedSigningService.objects.get(
pk=metadata_signing_service.pk)
sign_results = signing_service.sign(repomd_path)
# publish a signed file
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(sign_results["file"])),
publication=publication,
file=File(open(sign_results["file"], "rb")),
)
# publish a detached signature
PublishedMetadata.create_from_file(
relative_path=os.path.join(
repodata_path, os.path.basename(sign_results["signature"])),
publication=publication,
file=File(open(sign_results["signature"], "rb")),
)
# publish a public key required for further verification
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(sign_results["key"])),
publication=publication,
file=File(open(sign_results["key"], "rb")),
)
else:
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(repomd_path)),
publication=publication,
file=File(open(repomd_path, "rb")),
)
def generate_repo_metadata(
content,
publication,
checksum_types,
extra_repomdrecords,
sub_folder=None,
metadata_signing_service=None,
):
"""
Creates a repomd.xml file.
Args:
content(app.models.Content): content set
publication(pulpcore.plugin.models.Publication): the publication
extra_repomdrecords(list): list with data relative to repo metadata files
sub_folder(str): name of the folder for sub repos
metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService):
A reference to an associated signing service.
"""
cwd = os.getcwd()
repodata_path = REPODATA_PATH
has_modules = False
has_comps = False
package_checksum_type = checksum_types.get("package")
if sub_folder:
cwd = os.path.join(cwd, sub_folder)
repodata_path = os.path.join(sub_folder, repodata_path)
if package_checksum_type and package_checksum_type not in settings.ALLOWED_CONTENT_CHECKSUMS:
raise ValueError(
"Repository contains disallowed package checksum type '{}', "
"thus can't be published. {}".format(package_checksum_type,
ALLOWED_CHECKSUM_ERROR_MSG))
# Prepare metadata files
repomd_path = os.path.join(cwd, "repomd.xml")
pri_xml_path = os.path.join(cwd, "primary.xml.gz")
fil_xml_path = os.path.join(cwd, "filelists.xml.gz")
oth_xml_path = os.path.join(cwd, "other.xml.gz")
upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz")
mod_yml_path = os.path.join(cwd, "modules.yaml")
comps_xml_path = os.path.join(cwd, "comps.xml")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
if publication.sqlite_metadata:
pri_db_path = os.path.join(cwd, "primary.sqlite")
fil_db_path = os.path.join(cwd, "filelists.sqlite")
oth_db_path = os.path.join(cwd, "other.sqlite")
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
packages = Package.objects.filter(pk__in=content)
# We want to support publishing with a different checksum type than the one built-in to the
# package itself, so we need to get the correct checksums somehow if there is an override.
# We must also take into consideration that if the package has not been downloaded the only
# checksum that is available is the one built-in.
#
# Since this lookup goes from Package->Content->ContentArtifact->Artifact, performance is a
# challenge. We use ContentArtifact as our starting point because it enables us to work with
# simple foreign keys and avoid messing with the many-to-many relationship, which doesn't
# work with select_related() and performs poorly with prefetch_related(). This is fine
# because we know that Packages should only ever have one artifact per content.
contentartifact_qs = (
ContentArtifact.objects.filter(content__in=packages.only("pk")).
select_related(
# content__rpm_package is a bit of a hack, exploiting the way django sets up model
# inheritance, but it works and is unlikely to break. All content artifacts being
# accessed here have an associated Package since they originally came from the
# Package queryset.
"artifact",
"content__rpm_package",
).only("artifact", "content__rpm_package__checksum_type",
"content__rpm_package__pkgId"))
pkg_to_hash = {}
for ca in contentartifact_qs.iterator():
if package_checksum_type:
package_checksum_type = package_checksum_type.lower()
pkgid = getattr(ca.artifact, package_checksum_type, None)
if not package_checksum_type or not pkgid:
if ca.content.rpm_package.checksum_type not in settings.ALLOWED_CONTENT_CHECKSUMS:
raise ValueError(
"Package {} as content unit {} contains forbidden checksum type '{}', "
"thus can't be published. {}".format(
ca.content.rpm_package.nevra,
ca.content.pk,
ca.content.rpm_package.checksum_type,
ALLOWED_CHECKSUM_ERROR_MSG,
))
package_checksum_type = ca.content.rpm_package.checksum_type
pkgid = ca.content.rpm_package.pkgId
pkg_to_hash[ca.content_id] = (package_checksum_type, pkgid)
# TODO: this is meant to be a !! *temporary* !! fix for
# https://github.com/pulp/pulp_rpm/issues/2407
pkg_pks_to_ignore = set()
latest_build_time_by_nevra = defaultdict(list)
for pkg in packages.only("pk", "name", "epoch", "version", "release",
"arch", "time_build").iterator():
latest_build_time_by_nevra[pkg.nevra].append((pkg.time_build, pkg.pk))
for nevra, pkg_data in latest_build_time_by_nevra.items():
# sort the packages by when they were built
if len(pkg_data) > 1:
pkg_data.sort(key=lambda p: p[0], reverse=True)
pkg_pks_to_ignore |= set(entry[1] for entry in pkg_data[1:])
log.warning(
"Duplicate packages found competing for NEVRA {nevra}, selected the one with "
"the most recent build time, excluding {others} others.".
format(nevra=nevra, others=len(pkg_data[1:])))
total_packages = packages.count() - len(pkg_pks_to_ignore)
pri_xml.set_num_of_pkgs(total_packages)
fil_xml.set_num_of_pkgs(total_packages)
oth_xml.set_num_of_pkgs(total_packages)
# Process all packages
for package in packages.order_by("name", "evr").iterator():
if package.pk in pkg_pks_to_ignore: # Temporary!
continue
pkg = package.to_createrepo_c()
# rewrite the checksum and checksum type with the desired ones
(checksum, pkgId) = pkg_to_hash[package.pk]
pkg.checksum_type = checksum
pkg.pkgId = pkgId
pkg_filename = os.path.basename(package.location_href)
# this can cause an issue when two same RPM package names appears
# a/name1.rpm b/name1.rpm
pkg.location_href = os.path.join(PACKAGES_DIRECTORY,
pkg_filename[0].lower(), pkg_filename)
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
if publication.sqlite_metadata:
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=content).iterator():
upd_xml.add_chunk(
cr.xml_dump_updaterecord(update_record.to_createrepo_c()))
# Process modulemd, modulemd_defaults and obsoletes
with open(mod_yml_path, "ab") as mod_yml:
for modulemd in Modulemd.objects.filter(pk__in=content).iterator():
mod_yml.write(modulemd.snippet.encode())
has_modules = True
for default in ModulemdDefaults.objects.filter(
pk__in=content).iterator():
mod_yml.write(default.snippet.encode())
has_modules = True
for obsolete in ModulemdObsolete.objects.filter(
pk__in=content).iterator():
mod_yml.write(obsolete.snippet.encode())
has_modules = True
# Process comps
comps = libcomps.Comps()
for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator():
group = pkg_grp.pkg_grp_to_libcomps()
comps.groups.append(group)
has_comps = True
for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator():
cat = pkg_cat.pkg_cat_to_libcomps()
comps.categories.append(cat)
has_comps = True
for pkg_env in PackageEnvironment.objects.filter(
pk__in=content).iterator():
env = pkg_env.pkg_env_to_libcomps()
comps.environments.append(env)
has_comps = True
for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator():
comps.langpacks = dict_to_strdict(pkg_lng.matches)
has_comps = True
comps.toxml_f(
comps_xml_path,
xml_options={
"default_explicit": True,
"empty_groups": True,
"empty_packages": True,
"uservisible_explicit": True,
},
)
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
# If the repository is empty, use a revision of 0
# See: https://pulp.plan.io/issues/9402
if not content.exists():
repomd.revision = "0"
if publication.sqlite_metadata:
repomdrecords = [
("primary", pri_xml_path, pri_db),
("filelists", fil_xml_path, fil_db),
("other", oth_xml_path, oth_db),
("primary_db", pri_db_path, None),
("filelists_db", fil_db_path, None),
("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None),
]
else:
repomdrecords = [
("primary", pri_xml_path, None),
("filelists", fil_xml_path, None),
("other", oth_xml_path, None),
("updateinfo", upd_xml_path, None),
]
if has_modules:
repomdrecords.append(("modules", mod_yml_path, None))
if has_comps:
repomdrecords.append(("group", comps_xml_path, None))
repomdrecords.extend(extra_repomdrecords)
sqlite_files = ("primary_db", "filelists_db", "other_db")
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
checksum_type = cr_checksum_type_from_string(
get_checksum_type(name,
checksum_types,
default=publication.metadata_checksum_type))
if name in sqlite_files:
record_bz = record.compress_and_fill(checksum_type, cr.BZ2)
record_bz.type = name
record_bz.rename_file()
path = record_bz.location_href.split("/")[-1]
repomd.set_record(record_bz)
else:
record.fill(checksum_type)
if db_to_update:
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
record.rename_file()
path = record.location_href.split("/")[-1]
repomd.set_record(record)
if sub_folder:
path = os.path.join(sub_folder, path)
with open(path, "rb") as repodata_fd:
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(path)),
publication=publication,
file=File(repodata_fd),
)
with open(repomd_path, "w") as repomd_f:
repomd_f.write(repomd.xml_dump())
if metadata_signing_service:
signing_service = AsciiArmoredDetachedSigningService.objects.get(
pk=metadata_signing_service)
sign_results = signing_service.sign(repomd_path)
# publish a signed file
with open(sign_results["file"], "rb") as signed_file_fd:
PublishedMetadata.create_from_file(
relative_path=os.path.join(
repodata_path, os.path.basename(sign_results["file"])),
publication=publication,
file=File(signed_file_fd),
)
# publish a detached signature
with open(sign_results["signature"], "rb") as signature_fd:
PublishedMetadata.create_from_file(
relative_path=os.path.join(
repodata_path,
os.path.basename(sign_results["signature"])),
publication=publication,
file=File(signature_fd),
)
# publish a public key required for further verification
pubkey_name = "repomd.xml.key"
with open(pubkey_name, "wb+") as f:
f.write(signing_service.public_key.encode("utf-8"))
f.flush()
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path, pubkey_name),
publication=publication,
file=File(f),
)
else:
with open(repomd_path, "rb") as repomd_fd:
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(repomd_path)),
publication=publication,
file=File(repomd_fd),
)
def publish(repository_version_pk):
"""
Create a Publication based on a RepositoryVersion.
Args:
repository_version_pk (str): Create a publication from this repository version.
"""
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_('Publishing: repository={repo}, version={version}').format(
repo=repository_version.repository.name,
version=repository_version.number,
))
with WorkingDirectory():
with RpmPublication.create(repository_version) as publication:
packages = populate(publication)
# Prepare metadata files
repomd_path = os.path.join(os.getcwd(), "repomd.xml")
pri_xml_path = os.path.join(os.getcwd(), "primary.xml.gz")
fil_xml_path = os.path.join(os.getcwd(), "filelists.xml.gz")
oth_xml_path = os.path.join(os.getcwd(), "other.xml.gz")
pri_db_path = os.path.join(os.getcwd(), "primary.sqlite")
fil_db_path = os.path.join(os.getcwd(), "filelists.sqlite")
oth_db_path = os.path.join(os.getcwd(), "other.sqlite")
upd_xml_path = os.path.join(os.getcwd(), "updateinfo.xml.gz")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
pri_xml.set_num_of_pkgs(len(packages))
fil_xml.set_num_of_pkgs(len(packages))
oth_xml.set_num_of_pkgs(len(packages))
# Process all packages
for package in packages:
pkg = package.to_createrepo_c()
pkg.location_href = package.contentartifact_set.first(
).relative_path
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=publication.repository_version.content):
upd_xml.add_chunk(update_record_xml(update_record))
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
repomdrecords = (("primary", pri_xml_path,
pri_db), ("filelists", fil_xml_path, fil_db),
("other", oth_xml_path,
oth_db), ("primary_db", pri_db_path, None),
("filelists_db", fil_db_path,
None), ("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None))
sqlite_files = ("primary_db", "filelists_db", "other_db")
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
if name in sqlite_files:
record_bz = record.compress_and_fill(cr.SHA256, cr.BZ2)
record_bz.type = name
record_bz.rename_file()
path = record_bz.location_href.split('/')[-1]
repomd.set_record(record_bz)
else:
record.fill(cr.SHA256)
if (db_to_update):
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
record.rename_file()
path = record.location_href.split('/')[-1]
repomd.set_record(record)
metadata = PublishedMetadata(
relative_path=os.path.join(REPODATA_PATH,
os.path.basename(path)),
publication=publication,
file=File(open(os.path.basename(path), 'rb')))
metadata.save()
with open(repomd_path, "w") as repomd_f:
repomd_f.write(repomd.xml_dump())
metadata = PublishedMetadata(
relative_path=os.path.join(REPODATA_PATH,
os.path.basename(repomd_path)),
publication=publication,
file=File(open(os.path.basename(repomd_path), 'rb')))
metadata.save()
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = RpmPublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_('Publishing: repository={repo}, version={version}, publisher={publisher}'
).format(
repo=repository_version.repository.name,
version=repository_version.number,
publisher=publisher.name,
))
with WorkingDirectory():
with Publication.create(repository_version, publisher) as publication:
populate(publication)
# Prepare metadata files
repomd_path = os.path.join(os.getcwd(), "repomd.xml")
pri_xml_path = os.path.join(os.getcwd(), "primary.xml.gz")
fil_xml_path = os.path.join(os.getcwd(), "filelists.xml.gz")
oth_xml_path = os.path.join(os.getcwd(), "other.xml.gz")
pri_db_path = os.path.join(os.getcwd(), "primary.sqlite")
fil_db_path = os.path.join(os.getcwd(), "filelists.sqlite")
oth_db_path = os.path.join(os.getcwd(), "other.sqlite")
upd_xml_path = os.path.join(os.getcwd(), "updateinfo.xml.gz")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
artifacts = publication.published_artifact.all()
pri_xml.set_num_of_pkgs(len(artifacts))
fil_xml.set_num_of_pkgs(len(artifacts))
oth_xml.set_num_of_pkgs(len(artifacts))
# Process all packages
for artifact in artifacts:
# TODO: pass attributes from db rather than use the filesystem
pkg = cr.package_from_rpm(
artifact.content_artifact.artifact.file.path)
pkg.location_href = artifact.content_artifact.relative_path
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=publication.repository_version.content):
upd_xml.add_chunk(update_record_xml(update_record))
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
repomdrecords = (("primary", pri_xml_path,
pri_db), ("filelists", fil_xml_path, fil_db),
("other", oth_xml_path,
oth_db), ("primary_db", pri_db_path, None),
("filelists_db", fil_db_path,
None), ("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None))
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
record.fill(cr.SHA256)
if (db_to_update):
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
repomd.set_record(record)
metadata = PublishedMetadata(
relative_path=os.path.join(REPODATA_PATH,
os.path.basename(path)),
publication=publication,
file=File(open(os.path.basename(path), 'rb')))
metadata.save()
open(repomd_path, "w").write(repomd.xml_dump())
metadata = PublishedMetadata(
relative_path=os.path.join(REPODATA_PATH,
os.path.basename(repomd_path)),
publication=publication,
file=File(open(os.path.basename(repomd_path), 'rb')))
metadata.save()
