def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = FilePublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(pk=repository_version_pk)
log.info(
_('Publishing: repository=%(repository)s, version=%(version)d, publisher=%(publisher)s'),
{
'repository': repository_version.repository.name,
'version': repository_version.number,
'publisher': publisher.name,
})
with WorkingDirectory():
with Publication.create(repository_version, publisher) as publication:
manifest = Manifest('PULP_MANIFEST')
manifest.write(populate(publication))
metadata = PublishedMetadata(
relative_path=os.path.basename(manifest.relative_path),
publication=publication,
file=File(open(manifest.relative_path, 'rb')))
metadata.save()
log.info(
_('Publication: %(publication)s created'),
{
'publication': publication.pk
})
def publish(repository_version_pk):
"""
Create a Publication based on a RepositoryVersion.
Args:
repository_version_pk (str): Create a publication from this repository version.
"""
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_("Publishing: repository=%(repository)s, version=%(version)d"),
{
"repository": repository_version.repository.name,
"version": repository_version.number
},
)
with WorkingDirectory():
with CookbookPublication.create(repository_version) as publication:
check_repo_version_constraint(publication)
universe = Universe("__universe__")
universe.write(populate(publication))
metadata = PublishedMetadata(
relative_path=os.path.basename(universe.relative_path),
publication=publication,
file=File(open(universe.relative_path, "rb")),
)
metadata.save()
log.info(_("Publication: %(publication)s created"),
{"publication": publication.pk})
def publish(repository_version_pk):
"""
Create a Publication based on a RepositoryVersion.
Args:
repository_version_pk (str): Create a publication from this repository version.
"""
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_("Publishing: repository=%(repository)s, version=%(version)d"),
{
"repository": repository_version.repository.name,
"version": repository_version.number
},
)
with tempfile.TemporaryDirectory("."):
with ProgressReport(message="Publishing Content",
code="publishing.content") as progress_report:
with CookbookPublication.create(repository_version) as publication:
universe = Universe("__universe__")
universe.write(
populate(publication, progress_report=progress_report))
PublishedMetadata.create_from_file(
relative_path=os.path.basename(universe.relative_path),
publication=publication,
file=File(open(universe.relative_path, "rb")),
)
log.info(_("Publication: %(publication)s created"),
{"publication": publication.pk})
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = FilePublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(pk=repository_version_pk)
log.info(_('Publishing: repository={repo}, version={ver}, publisher={pub}').format(
repo=repository_version.repository.name,
ver=repository_version.number,
pub=publisher.name,
))
with WorkingDirectory():
with Publication.create(repository_version, publisher, pass_through=True) as publication:
manifest = Manifest(publisher.manifest)
manifest.write(populate(publication))
metadata = PublishedMetadata(
relative_path=os.path.basename(manifest.relative_path),
publication=publication,
file=File(open(manifest.relative_path, 'rb')))
metadata.save()
log.info(_('Publication: {publication} created').format(publication=publication.pk))
async def migrate_to_pulp3(cls, pulp2distributor, repo_version):
"""
Migrate distributor to Pulp 3.
Args:
pulp2distributor(Pulp2ditributor): Pre-migrated pulp2 distributor to migrate
Return:
publication and distribution: FilePublication and FileDistribution in Pulp3
created(bool): True if Distribution has just been created; False if Distribution
is an existing one
"""
if not repo_version:
repo_version = pulp2distributor.pulp2_repository.pulp3_repository_version
publication = repo_version.publication_set.first()
if not publication:
# create publication
with FilePublication.create(repo_version,
pass_through=True) as publication:
manifest = Manifest('PULP_MANIFEST')
manifest.write(populate(publication))
PublishedMetadata.create_from_file(file=File(
open(manifest.relative_path, "rb")),
publication=publication)
# create distribution
pulp2_config = pulp2distributor.pulp2_config
base_config = cls.parse_base_config(pulp2distributor, pulp2_config)
base_config['base_path'] = pulp2_config.get(
'relative_url', pulp2distributor.pulp2_repository.pulp2_repo_id)
base_config['publication'] = publication
distribution, created = FileDistribution.objects.update_or_create(
**base_config)
return publication, distribution, created
def finish(self):
# Publish Packages files
for component in self.components.values():
component.finish()
# Publish Release file
self.release["Components"] = " ".join(self.components.keys())
release_dir = os.path.join("dists", self.distribution.strip("/"))
release_path = os.path.join(release_dir, "Release")
os.makedirs(os.path.dirname(release_path), exist_ok=True)
with open(release_path, "wb") as release_file:
self.release.dump(release_file)
release_metadata = PublishedMetadata.create_from_file(
publication=self.publication,
file=File(open(release_path, "rb")),
)
release_metadata.save()
if self.signing_service:
signed = self.signing_service.sign(release_path)
for signature_file in signed["signatures"].values():
file_name = os.path.basename(signature_file)
relative_path = os.path.join(release_dir, file_name)
metadata = PublishedMetadata.create_from_file(
publication=self.publication,
file=File(open(signature_file, "rb")),
relative_path=relative_path,
)
metadata.save()
def populate(self):
"""
Populate a publication.
Create published artifacts for a publication.
"""
content = self.publication.repository_version.content
self.repomdrecords = self.prepare_metadata_files(content)
self.publish_artifacts(content)
distribution_trees = DistributionTree.objects.filter(
pk__in=content
).prefetch_related(
"addons",
"variants",
"addons__repository",
"variants__repository",
"contentartifact_set"
)
for distribution_tree in distribution_trees:
self.handle_sub_repos(distribution_tree)
treeinfo_file = create_treeinfo(distribution_tree)
PublishedMetadata.create_from_file(
publication=self.publication,
file=File(open(treeinfo_file.name, 'rb'))
)
for name, content in self.sub_repos:
os.mkdir(name)
setattr(self, f"{name}_content", content)
setattr(self, f"{name}_repomdrecords", self.prepare_metadata_files(content, name))
def populate(self):
"""
Populate a publication.
Create published artifacts for a publication.
"""
publication = self.publication
main_content = publication.repository_version.content
distribution_trees = DistributionTree.objects.filter(
pk__in=publication.repository_version.content).prefetch_related(
"addons", "variants", "addons__repository",
"variants__repository", "contentartifact_set")
for distribution_tree in distribution_trees:
for content_artifact in distribution_tree.contentartifact_set.all(
):
self.published_artifacts.append(
PublishedArtifact(
relative_path=content_artifact.relative_path,
publication=publication,
content_artifact=content_artifact))
for addon in distribution_tree.addons.all():
repository_version = addon.repository.latest_version()
if repository_version and repository_version.content != main_content:
self.sub_repos.append(
(addon.addon_id, repository_version.content))
for variant in distribution_tree.variants.all():
repository_version = variant.repository.latest_version()
if repository_version and repository_version.content != main_content:
self.sub_repos.append(
(variant.variant_id, repository_version.content))
treeinfo_file = create_treeinfo(distribution_tree)
PublishedMetadata.create_from_file(
publication=publication,
file=File(open(treeinfo_file.name, 'rb')))
self.packages = self.get_packages(main_content)
self.repomdrecords = self.prepare_metadata_files(main_content)
all_packages = self.packages
for name, content in self.sub_repos:
os.mkdir(name)
sub_repo_packages = self.get_packages(content)
all_packages = all_packages | sub_repo_packages
setattr(self, f"{name}_packages", sub_repo_packages)
setattr(self, f"{name}_repomdrecords",
self.prepare_metadata_files(content, name))
for package in all_packages.distinct():
for content_artifact in package.contentartifact_set.all():
self.published_artifacts.append(
PublishedArtifact(
relative_path=content_artifact.relative_path,
publication=self.publication,
content_artifact=content_artifact))
PublishedArtifact.objects.bulk_create(self.published_artifacts)
def publish(manifest, repository_version_pk):
"""
Create a Publication based on a RepositoryVersion.
Args:
manifest (str): Filename to use for manifest file.
repository_version_pk (str): Create a publication from this repository version.
"""
repo_version = RepositoryVersion.objects.get(pk=repository_version_pk)
log.info(
_("Publishing: repository={repo}, version={ver}, manifest={manifest}").
format(repo=repo_version.repository.name,
ver=repo_version.number,
manifest=manifest))
with WorkingDirectory():
with FilePublication.create(repo_version,
pass_through=True) as publication:
manifest = Manifest(manifest)
manifest.write(populate(publication))
PublishedMetadata.create_from_file(file=File(
open(manifest.relative_path, "rb")),
publication=publication)
log.info(
_("Publication: {publication} created").format(
publication=publication.pk))
def handle_sub_repos(self, distribution_tree):
"""
Get sub-repo content and publish them.
Args:
distribution_tree (pulp_rpm.models.DistributionTree): A distribution_tree object.
"""
original_treeinfo_content_artifact = distribution_tree.contentartifact_set.get(
relative_path__in=[".treeinfo", "treeinfo"]
)
artifact_file = storage.open(original_treeinfo_content_artifact.artifact.file.name)
with tempfile.NamedTemporaryFile("wb", dir=".") as temp_file:
shutil.copyfileobj(artifact_file, temp_file)
temp_file.flush()
treeinfo = PulpTreeInfo()
treeinfo.load(f=temp_file.name)
treeinfo_data = TreeinfoData(treeinfo.parsed_sections())
# rewrite the treeinfo file such that the variant repository and package location
# is a relative subtree
treeinfo.rewrite_subrepo_paths(treeinfo_data)
# TODO: better way to do this?
main_variant = treeinfo.original_parser._sections.get("general", {}).get(
"variant", None
)
treeinfo_file = tempfile.NamedTemporaryFile(dir=".")
treeinfo.dump(treeinfo_file.name, main_variant=main_variant)
with open(treeinfo_file.name, "rb") as treeinfo_fd:
PublishedMetadata.create_from_file(
relative_path=original_treeinfo_content_artifact.relative_path,
publication=self.publication,
file=File(treeinfo_fd),
)
artifact_file.close()
relations = ["addon", "variant"]
for relation in relations:
addons_or_variants = getattr(distribution_tree, f"{relation}s").all()
for addon_or_variant in addons_or_variants:
if not addon_or_variant.repository:
# a variant of the main repo
continue
repository = addon_or_variant.repository.cast()
repository_version = repository.latest_version()
if repository_version and repository.user_hidden:
addon_or_variant_id = getattr(addon_or_variant, f"{relation}_id")
self.sub_repos.append(
(
addon_or_variant_id,
repository_version.content,
repository.original_checksum_types,
)
)
def publish(self):
"""
Publish the repository.
"""
manifest = Manifest('PULP_MANIFEST')
manifest.write(self._publish())
metadata = PublishedMetadata(
relative_path=os.path.basename(manifest.path),
publication=self.publication,
file=File(open(manifest.path, 'rb')))
metadata.save()
def _publish_specs(specs, relative_path, publication):
write_specs(specs, relative_path)
with open(relative_path, "rb") as f_in:
with gzip.open(relative_path + ".gz", "wb") as f_out:
shutil.copyfileobj(f_in, f_out)
specs_metadata = PublishedMetadata.create_from_file(
publication=publication, file=File(open(relative_path, "rb")))
specs_metadata.save()
specs_metadata_gz = PublishedMetadata.create_from_file(
publication=publication, file=File(open(relative_path + ".gz", "rb")))
specs_metadata_gz.save()
def handle_sub_repos(self, distribution_tree):
"""
Get sub-repo content and publish them.
Args:
distribution_tree (pulp_rpm.models.DistributionTree): A distribution_tree object.
"""
original_treeinfo_content_artifact = distribution_tree.contentartifact_set.get(
relative_path__in=[".treeinfo", "treeinfo"])
artifact_file = storage.open(
original_treeinfo_content_artifact.artifact.file.name)
with NamedTemporaryFile("wb") as temp_file:
shutil.copyfileobj(artifact_file, temp_file)
temp_file.flush()
treeinfo = PulpTreeInfo()
treeinfo.load(f=temp_file.name)
treeinfo_parsed = treeinfo.parsed_sections()
treeinfodata = TreeinfoData(treeinfo_parsed)
for variant in treeinfo.variants.get_variants():
variant.paths.repository = treeinfodata.variants[
variant.id]["repository"]
variant.paths.packages = treeinfodata.variants[
variant.id]["packages"]
treeinfo_file = NamedTemporaryFile()
treeinfo.dump(treeinfo_file.name)
PublishedMetadata.create_from_file(
relative_path=original_treeinfo_content_artifact.relative_path,
publication=self.publication,
file=File(open(treeinfo_file.name, "rb")),
)
relations = ["addon", "variant"]
for relation in relations:
addons_or_variants = getattr(distribution_tree,
f"{relation}s").all()
for addon_or_variant in addons_or_variants:
if not addon_or_variant.repository:
# a variant of the main repo
continue
repository = addon_or_variant.repository.cast()
repository_version = repository.latest_version()
if repository_version and repository.sub_repo:
addon_or_variant_id = getattr(addon_or_variant,
f"{relation}_id")
self.sub_repos.append((
addon_or_variant_id,
repository_version.content,
repository.original_checksum_types,
))
def finish(self):
# Publish Packages files
for (package_index_file, package_index_path) in self.package_index_files.values():
package_index_file.close()
gz_package_index_path = _zip_file(package_index_path)
package_index = PublishedMetadata.create_from_file(
publication=self.parent.publication, file=File(open(package_index_path, "rb"))
)
package_index.save()
gz_package_index = PublishedMetadata.create_from_file(
publication=self.parent.publication, file=File(open(gz_package_index_path, "rb"))
)
gz_package_index.save()
self.parent.add_metadata(package_index)
self.parent.add_metadata(gz_package_index)
def publish(self):
"""
Publish the repository.
"""
with ProgressBar(message=_("Publishing repository metadata"), total=1) as bar:
manifest_name = 'PULP_MANIFEST'
with open(manifest_name, 'w+') as fp:
for entry in self._publish():
fp.write(entry)
fp.write('\n')
metadata = PublishedMetadata(
relative_path=os.path.basename(manifest_name),
publication=self.publication,
file=File(open(manifest_name, 'rb')))
metadata.save()
bar.increment()
def synchronize(remote_pk, repository_pk, mirror):
"""
Sync content from the remote repository.
Create a new version of the repository that is synchronized with the remote.
Args:
remote_pk (str): The remote PK.
repository_pk (str): The repository PK.
mirror (bool): True for mirror mode, False for additive.
Raises:
ValueError: If the remote does not specify a URL to sync.
"""
remote = FileRemote.objects.get(pk=remote_pk)
repository = FileRepository.objects.get(pk=repository_pk)
if not remote.url:
raise ValueError(
_("A remote must have a url specified to synchronize."))
first_stage = FileFirstStage(remote)
dv = DeclarativeVersion(first_stage, repository, mirror=mirror)
rv = dv.create()
if rv and mirror:
# TODO: this is awful, we really should rewrite the DeclarativeVersion API to
# accomodate this use case
global metadata_files
with FilePublication.create(rv, pass_through=True) as publication:
(mdfile_path, relative_path) = metadata_files.pop()
PublishedMetadata.create_from_file(
file=File(open(mdfile_path, "rb")),
relative_path=relative_path,
publication=publication,
)
publication.manifest = relative_path
publication.save()
log.info(
_("Publication: {publication} created").format(
publication=publication.pk))
return rv
def _publish_specs(specs, relative_path, publication):
write_specs(specs, relative_path)
with open(relative_path, 'rb') as f_in:
with gzip.open(relative_path + '.gz', 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
specs_metadata = PublishedMetadata(relative_path=relative_path,
publication=publication,
file=File(open(relative_path, 'rb')))
specs_metadata.save()
specs_metadata_gz = PublishedMetadata(relative_path=relative_path + '.gz',
publication=publication,
file=File(
open(relative_path + '.gz',
'rb')))
specs_metadata_gz.save()
def publish_chart_content(publication):
"""
Create published artifacts and metadata for a publication
Args:
publication (ChartPublication): The publication to store
"""
entries = {}
for content in ChartContent.objects.filter(
pk__in=publication.repository_version.content).order_by(
'name', '-created'):
artifacts = content.contentartifact_set.all()
for artifact in artifacts:
published = PublishedArtifact(relative_path=artifact.relative_path,
publication=publication,
content_artifact=artifact)
published.save()
entry = {
'apiVersion': 'v1',
'created': content.created.isoformat(),
'description': content.description,
'digest': content.digest,
'icon': content.icon,
'keywords': content.keywords,
'name': content.name,
'urls': [artifact.relative_path for artifact in artifacts],
'version': content.version
}
if content.name not in entries:
entries[content.name] = []
# Strip away empty keys when building metadata
entries[content.name].append(
{k: v
for k, v in entry.items() if (v is not None and v != [])})
doc = {
'apiVersion': 'v1',
'entries': entries,
'generated': timezone.now().isoformat()
}
with open('index.yaml', 'w') as index:
index.write(yaml.dump(doc))
index = PublishedMetadata.create_from_file(publication=publication,
file=File(
open('index.yaml', 'rb')))
index.save()
def publish(repository_version_pk):
"""
Create a Publication based on a RepositoryVersion.
Args:
repository_version_pk (str): Create a publication from this repository version.
"""
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_('Publishing: repository={repo}, version={version}').format(
repo=repository_version.repository.name,
version=repository_version.number,
))
with WorkingDirectory():
with RpmPublication.create(repository_version) as publication:
packages = populate(publication)
# Prepare metadata files
repomd_path = os.path.join(os.getcwd(), "repomd.xml")
pri_xml_path = os.path.join(os.getcwd(), "primary.xml.gz")
fil_xml_path = os.path.join(os.getcwd(), "filelists.xml.gz")
oth_xml_path = os.path.join(os.getcwd(), "other.xml.gz")
pri_db_path = os.path.join(os.getcwd(), "primary.sqlite")
fil_db_path = os.path.join(os.getcwd(), "filelists.sqlite")
oth_db_path = os.path.join(os.getcwd(), "other.sqlite")
upd_xml_path = os.path.join(os.getcwd(), "updateinfo.xml.gz")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
pri_xml.set_num_of_pkgs(len(packages))
fil_xml.set_num_of_pkgs(len(packages))
oth_xml.set_num_of_pkgs(len(packages))
# Process all packages
for package in packages:
pkg = package.to_createrepo_c()
pkg.location_href = package.contentartifact_set.first(
).relative_path
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=publication.repository_version.content):
upd_xml.add_chunk(update_record_xml(update_record))
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
repomdrecords = (("primary", pri_xml_path,
pri_db), ("filelists", fil_xml_path, fil_db),
("other", oth_xml_path,
oth_db), ("primary_db", pri_db_path, None),
("filelists_db", fil_db_path,
None), ("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None))
sqlite_files = ("primary_db", "filelists_db", "other_db")
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
if name in sqlite_files:
record_bz = record.compress_and_fill(cr.SHA256, cr.BZ2)
record_bz.type = name
record_bz.rename_file()
path = record_bz.location_href.split('/')[-1]
repomd.set_record(record_bz)
else:
record.fill(cr.SHA256)
if (db_to_update):
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
record.rename_file()
path = record.location_href.split('/')[-1]
repomd.set_record(record)
metadata = PublishedMetadata(
relative_path=os.path.join(REPODATA_PATH,
os.path.basename(path)),
publication=publication,
file=File(open(os.path.basename(path), 'rb')))
metadata.save()
with open(repomd_path, "w") as repomd_f:
repomd_f.write(repomd.xml_dump())
metadata = PublishedMetadata(
relative_path=os.path.join(REPODATA_PATH,
os.path.basename(repomd_path)),
publication=publication,
file=File(open(os.path.basename(repomd_path), 'rb')))
metadata.save()
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = RpmPublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_('Publishing: repository={repo}, version={version}, publisher={publisher}'
).format(
repo=repository_version.repository.name,
version=repository_version.number,
publisher=publisher.name,
))
with WorkingDirectory():
with Publication.create(repository_version, publisher) as publication:
populate(publication)
# Prepare metadata files
repomd_path = os.path.join(os.getcwd(), "repomd.xml")
pri_xml_path = os.path.join(os.getcwd(), "primary.xml.gz")
fil_xml_path = os.path.join(os.getcwd(), "filelists.xml.gz")
oth_xml_path = os.path.join(os.getcwd(), "other.xml.gz")
pri_db_path = os.path.join(os.getcwd(), "primary.sqlite")
fil_db_path = os.path.join(os.getcwd(), "filelists.sqlite")
oth_db_path = os.path.join(os.getcwd(), "other.sqlite")
upd_xml_path = os.path.join(os.getcwd(), "updateinfo.xml.gz")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
artifacts = publication.published_artifact.all()
pri_xml.set_num_of_pkgs(len(artifacts))
fil_xml.set_num_of_pkgs(len(artifacts))
oth_xml.set_num_of_pkgs(len(artifacts))
# Process all packages
for artifact in artifacts:
# TODO: pass attributes from db rather than use the filesystem
pkg = cr.package_from_rpm(
artifact.content_artifact.artifact.file.path)
pkg.location_href = artifact.content_artifact.relative_path
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=publication.repository_version.content):
upd_xml.add_chunk(update_record_xml(update_record))
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
repomdrecords = (("primary", pri_xml_path,
pri_db), ("filelists", fil_xml_path, fil_db),
("other", oth_xml_path,
oth_db), ("primary_db", pri_db_path, None),
("filelists_db", fil_db_path,
None), ("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None))
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
record.fill(cr.SHA256)
if (db_to_update):
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
repomd.set_record(record)
metadata = PublishedMetadata(
relative_path=os.path.join(REPODATA_PATH,
os.path.basename(path)),
publication=publication,
file=File(open(os.path.basename(path), 'rb')))
metadata.save()
open(repomd_path, "w").write(repomd.xml_dump())
metadata = PublishedMetadata(
relative_path=os.path.join(REPODATA_PATH,
os.path.basename(repomd_path)),
publication=publication,
file=File(open(os.path.basename(repomd_path), 'rb')))
metadata.save()
def create_repomd_xml(
content,
publication,
checksum_types,
extra_repomdrecords,
sub_folder=None,
metadata_signing_service=None,
):
"""
Creates a repomd.xml file.
Args:
content(app.models.Content): content set
publication(pulpcore.plugin.models.Publication): the publication
extra_repomdrecords(list): list with data relative to repo metadata files
sub_folder(str): name of the folder for sub repos
metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService):
A reference to an associated signing service.
"""
cwd = os.getcwd()
repodata_path = REPODATA_PATH
has_modules = False
has_comps = False
package_checksum_type = checksum_types.get("package")
if sub_folder:
cwd = os.path.join(cwd, sub_folder)
repodata_path = os.path.join(sub_folder, repodata_path)
# Prepare metadata files
repomd_path = os.path.join(cwd, "repomd.xml")
pri_xml_path = os.path.join(cwd, "primary.xml.gz")
fil_xml_path = os.path.join(cwd, "filelists.xml.gz")
oth_xml_path = os.path.join(cwd, "other.xml.gz")
upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz")
mod_yml_path = os.path.join(cwd, "modules.yaml")
comps_xml_path = os.path.join(cwd, "comps.xml")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
if publication.sqlite_metadata:
pri_db_path = os.path.join(cwd, "primary.sqlite")
fil_db_path = os.path.join(cwd, "filelists.sqlite")
oth_db_path = os.path.join(cwd, "other.sqlite")
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
packages = Package.objects.filter(pk__in=content)
total_packages = packages.count()
pri_xml.set_num_of_pkgs(total_packages)
fil_xml.set_num_of_pkgs(total_packages)
oth_xml.set_num_of_pkgs(total_packages)
# We want to support publishing with a different checksum type than the one built-in to the
# package itself, so we need to get the correct checksums somehow if there is an override.
# We must also take into consideration that if the package has not been downloaded the only
# checksum that is available is the one built-in.
#
# Since this lookup goes from Package->Content->ContentArtifact->Artifact, performance is a
# challenge. We use ContentArtifact as our starting point because it enables us to work with
# simple foreign keys and avoid messing with the many-to-many relationship, which doesn't
# work with select_related() and performs poorly with prefetch_related(). This is fine
# because we know that Packages should only ever have one artifact per content.
contentartifact_qs = (
ContentArtifact.objects.filter(content__in=packages.only("pk")).
select_related(
# content__rpm_package is a bit of a hack, exploiting the way django sets up model
# inheritance, but it works and is unlikely to break. All content artifacts being
# accessed here have an associated Package since they originally came from the
# Package queryset.
"artifact",
"content__rpm_package",
).only("artifact", "content__rpm_package__checksum_type",
"content__rpm_package__pkgId"))
pkg_to_hash = {}
for ca in contentartifact_qs.iterator():
pkgid = None
if package_checksum_type:
package_checksum_type = package_checksum_type.lower()
pkgid = getattr(ca.artifact, package_checksum_type, None)
if pkgid:
pkg_to_hash[ca.content_id] = (package_checksum_type, pkgid)
else:
pkg_to_hash[ca.content_id] = (
ca.content.rpm_package.checksum_type,
ca.content.rpm_package.pkgId,
)
# Process all packages
for package in packages.iterator():
pkg = package.to_createrepo_c()
# rewrite the checksum and checksum type with the desired ones
(checksum, pkgId) = pkg_to_hash[package.pk]
pkg.checksum_type = checksum
pkg.pkgId = pkgId
pkg_filename = os.path.basename(package.location_href)
# this can cause an issue when two same RPM package names appears
# a/name1.rpm b/name1.rpm
pkg.location_href = os.path.join(PACKAGES_DIRECTORY,
pkg_filename[0].lower(), pkg_filename)
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
if publication.sqlite_metadata:
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=content).iterator():
upd_xml.add_chunk(
cr.xml_dump_updaterecord(update_record.to_createrepo_c()))
# Process modulemd and modulemd_defaults
with open(mod_yml_path, "ab") as mod_yml:
for modulemd in Modulemd.objects.filter(pk__in=content).iterator():
mod_yml.write(modulemd._artifacts.get().file.read())
has_modules = True
for default in ModulemdDefaults.objects.filter(
pk__in=content).iterator():
mod_yml.write(default._artifacts.get().file.read())
has_modules = True
# Process comps
comps = libcomps.Comps()
for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator():
group = pkg_grp.pkg_grp_to_libcomps()
comps.groups.append(group)
has_comps = True
for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator():
cat = pkg_cat.pkg_cat_to_libcomps()
comps.categories.append(cat)
has_comps = True
for pkg_env in PackageEnvironment.objects.filter(
pk__in=content).iterator():
env = pkg_env.pkg_env_to_libcomps()
comps.environments.append(env)
has_comps = True
for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator():
comps.langpacks = dict_to_strdict(pkg_lng.matches)
has_comps = True
comps.toxml_f(
comps_xml_path,
xml_options={
"default_explicit": True,
"empty_groups": True,
"uservisible_explicit": True
},
)
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
if publication.sqlite_metadata:
repomdrecords = [
("primary", pri_xml_path, pri_db),
("filelists", fil_xml_path, fil_db),
("other", oth_xml_path, oth_db),
("primary_db", pri_db_path, None),
("filelists_db", fil_db_path, None),
("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None),
]
else:
repomdrecords = [
("primary", pri_xml_path, None),
("filelists", fil_xml_path, None),
("other", oth_xml_path, None),
("updateinfo", upd_xml_path, None),
]
if has_modules:
repomdrecords.append(("modules", mod_yml_path, None))
if has_comps:
repomdrecords.append(("group", comps_xml_path, None))
repomdrecords.extend(extra_repomdrecords)
sqlite_files = ("primary_db", "filelists_db", "other_db")
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
checksum_type = get_checksum_type(name, checksum_types)
if name in sqlite_files:
record_bz = record.compress_and_fill(checksum_type, cr.BZ2)
record_bz.type = name
record_bz.rename_file()
path = record_bz.location_href.split("/")[-1]
repomd.set_record(record_bz)
else:
record.fill(checksum_type)
if db_to_update:
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
record.rename_file()
path = record.location_href.split("/")[-1]
repomd.set_record(record)
if sub_folder:
path = os.path.join(sub_folder, path)
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path, os.path.basename(path)),
publication=publication,
file=File(open(path, "rb")),
)
with open(repomd_path, "w") as repomd_f:
repomd_f.write(repomd.xml_dump())
if metadata_signing_service:
signing_service = AsciiArmoredDetachedSigningService.objects.get(
pk=metadata_signing_service.pk)
sign_results = signing_service.sign(repomd_path)
# publish a signed file
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(sign_results["file"])),
publication=publication,
file=File(open(sign_results["file"], "rb")),
)
# publish a detached signature
PublishedMetadata.create_from_file(
relative_path=os.path.join(
repodata_path, os.path.basename(sign_results["signature"])),
publication=publication,
file=File(open(sign_results["signature"], "rb")),
)
# publish a public key required for further verification
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(sign_results["key"])),
publication=publication,
file=File(open(sign_results["key"], "rb")),
)
else:
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(repomd_path)),
publication=publication,
file=File(open(repomd_path, "rb")),
)
def publish(repository_version_pk):
"""
Create a Publication based on a RepositoryVersion.
Args:
repository_version_pk (str): Create a publication from this repository version.
"""
repository_version = RepositoryVersion.objects.get(pk=repository_version_pk)
log.info(_('Publishing: repository={repo}, version={version}').format(
repo=repository_version.repository.name,
version=repository_version.number,
))
with WorkingDirectory():
with RpmPublication.create(repository_version) as publication:
populate(publication)
# Prepare metadata files
repomd_path = os.path.join(os.getcwd(), "repomd.xml")
pri_xml_path = os.path.join(os.getcwd(), "primary.xml.gz")
fil_xml_path = os.path.join(os.getcwd(), "filelists.xml.gz")
oth_xml_path = os.path.join(os.getcwd(), "other.xml.gz")
pri_db_path = os.path.join(os.getcwd(), "primary.sqlite")
fil_db_path = os.path.join(os.getcwd(), "filelists.sqlite")
oth_db_path = os.path.join(os.getcwd(), "other.sqlite")
upd_xml_path = os.path.join(os.getcwd(), "updateinfo.xml.gz")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
artifacts = publication.published_artifact.all()
pri_xml.set_num_of_pkgs(len(artifacts))
fil_xml.set_num_of_pkgs(len(artifacts))
oth_xml.set_num_of_pkgs(len(artifacts))
# Process all packages
for artifact in artifacts:
pkg = artifact.content_artifact.content.cast().to_createrepo_c()
pkg.location_href = artifact.content_artifact.relative_path
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=publication.repository_version.content):
upd_xml.add_chunk(update_record_xml(update_record))
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
repomdrecords = (("primary", pri_xml_path, pri_db),
("filelists", fil_xml_path, fil_db),
("other", oth_xml_path, oth_db),
("primary_db", pri_db_path, None),
("filelists_db", fil_db_path, None),
("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None))
sqlite_files = ("primary_db", "filelists_db", "other_db")
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
if name in sqlite_files:
record_bz = record.compress_and_fill(cr.SHA256, cr.BZ2)
record_bz.type = name
record_bz.rename_file()
path = record_bz.location_href.split('/')[-1]
repomd.set_record(record_bz)
else:
record.fill(cr.SHA256)
if (db_to_update):
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
record.rename_file()
path = record.location_href.split('/')[-1]
repomd.set_record(record)
metadata = PublishedMetadata(
relative_path=os.path.join(REPODATA_PATH, os.path.basename(path)),
publication=publication,
file=File(open(os.path.basename(path), 'rb'))
)
metadata.save()
with open(repomd_path, "w") as repomd_f:
repomd_f.write(repomd.xml_dump())
metadata = PublishedMetadata(
relative_path=os.path.join(REPODATA_PATH, os.path.basename(repomd_path)),
publication=publication,
file=File(open(os.path.basename(repomd_path), 'rb'))
)
metadata.save()
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = RpmPublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(pk=repository_version_pk)
log.info(_('Publishing: repository={repo}, version={version}, publisher={publisher}').format(
repo=repository_version.repository.name,
version=repository_version.number,
publisher=publisher.name,
))
with WorkingDirectory():
with Publication.create(repository_version, publisher) as publication:
populate(publication)
# Prepare metadata files
repomd_path = os.path.join(os.getcwd(), "repomd.xml")
pri_xml_path = os.path.join(os.getcwd(), "primary.xml.gz")
fil_xml_path = os.path.join(os.getcwd(), "filelists.xml.gz")
oth_xml_path = os.path.join(os.getcwd(), "other.xml.gz")
pri_db_path = os.path.join(os.getcwd(), "primary.sqlite")
fil_db_path = os.path.join(os.getcwd(), "filelists.sqlite")
oth_db_path = os.path.join(os.getcwd(), "other.sqlite")
upd_xml_path = os.path.join(os.getcwd(), "updateinfo.xml.gz")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
artifacts = publication.published_artifact.all()
pri_xml.set_num_of_pkgs(len(artifacts))
fil_xml.set_num_of_pkgs(len(artifacts))
oth_xml.set_num_of_pkgs(len(artifacts))
# Process all packages
for artifact in artifacts:
# TODO: pass attributes from db rather than use the filesystem
pkg = cr.package_from_rpm(artifact.content_artifact.artifact.file.path)
pkg.location_href = artifact.content_artifact.relative_path
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=publication.repository_version.content):
upd_xml.add_chunk(update_record_xml(update_record))
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
repomdrecords = (("primary", pri_xml_path, pri_db),
("filelists", fil_xml_path, fil_db),
("other", oth_xml_path, oth_db),
("primary_db", pri_db_path, None),
("filelists_db", fil_db_path, None),
("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None))
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
record.fill(cr.SHA256)
if (db_to_update):
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
repomd.set_record(record)
metadata = PublishedMetadata(
relative_path=os.path.join(REPODATA_PATH, os.path.basename(path)),
publication=publication,
file=File(open(os.path.basename(path), 'rb'))
)
metadata.save()
open(repomd_path, "w").write(repomd.xml_dump())
metadata = PublishedMetadata(
relative_path=os.path.join(REPODATA_PATH, os.path.basename(repomd_path)),
publication=publication,
file=File(open(os.path.basename(repomd_path), 'rb'))
)
metadata.save()
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = DebPublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(pk=repository_version_pk)
log.info(_('Publishing: repository={repo}, version={ver}, publisher={pub}').format(
repo=repository_version.repository.name,
ver=repository_version.number,
pub=publisher.name
))
with WorkingDirectory():
with Publication.create(repository_version, publisher, pass_through=False) as publication:
if publisher.simple:
repository = repository_version.repository
release = deb822.Release()
# TODO: release['Label']
release['Codename'] = 'default'
release['Components'] = 'all'
release['Architectures'] = ''
if repository.description:
release['Description'] = repository.description
release['MD5sum'] = []
release['SHA1'] = []
release['SHA256'] = []
release['SHA512'] = []
package_index_files = {}
for package in Package.objects.filter(
pk__in=repository_version.content.order_by('-_created')
):
published_artifact = PublishedArtifact(
relative_path=package.filename(),
publication=publication,
content_artifact=package.contentartifact_set.get(),
)
published_artifact.save()
if package.architecture not in package_index_files:
package_index_path = os.path.join(
'dists',
'default',
'all',
'binary-{}'.format(package.architecture),
'Packages',
)
os.makedirs(os.path.dirname(
package_index_path), exist_ok=True)
package_index_files[package.architecture] = (
open(package_index_path, 'wb'), package_index_path)
package.to822('all').dump(
package_index_files[package.architecture][0])
package_index_files[package.architecture][0].write(b'\n')
for package_index_file, package_index_path in package_index_files.values():
package_index_file.close()
gz_package_index_path = _zip_file(package_index_path)
_add_to_release(release, package_index_path)
_add_to_release(release, gz_package_index_path)
package_index = PublishedMetadata(
relative_path=package_index_path,
publication=publication,
file=File(open(package_index_path, 'rb')),
)
package_index.save()
gz_package_index = PublishedMetadata(
relative_path=gz_package_index_path,
publication=publication,
file=File(open(gz_package_index_path, 'rb')),
)
gz_package_index.save()
release['Architectures'] = ', '.join(package_index_files.keys())
release_path = os.path.join('dists', 'default', 'Release')
os.makedirs(os.path.dirname(release_path), exist_ok=True)
with open(release_path, 'wb') as release_file:
release.dump(release_file)
release_metadata = PublishedMetadata(
relative_path=release_path,
publication=publication,
file=File(open(release_path, 'rb')),
)
release_metadata.save()
if publisher.structured:
raise NotImplementedError(
"Structured publishing is not yet implemented.")
log.info(_('Publication: {publication} created').format(publication=publication.pk))
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = DebPublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_('Publishing: repository={repo}, version={ver}, publisher={pub}').
format(repo=repository_version.repository.name,
ver=repository_version.number,
pub=publisher.name))
with WorkingDirectory():
with Publication.create(repository_version,
publisher,
pass_through=False) as publication:
if publisher.simple:
repository = repository_version.repository
release = deb822.Release()
# TODO: release['Label']
release['Codename'] = 'default'
release['Components'] = 'all'
release['Architectures'] = ''
if repository.description:
release['Description'] = repository.description
release['MD5sum'] = []
release['SHA1'] = []
release['SHA256'] = []
release['SHA512'] = []
package_index_files = {}
for package in Package.objects.filter(
pk__in=repository_version.content.order_by(
'-_created')):
published_artifact = PublishedArtifact(
relative_path=package.filename(),
publication=publication,
content_artifact=package.contentartifact_set.get(),
)
published_artifact.save()
if package.architecture not in package_index_files:
package_index_path = os.path.join(
'dists',
'default',
'all',
'binary-{}'.format(package.architecture),
'Packages',
)
os.makedirs(os.path.dirname(package_index_path),
exist_ok=True)
package_index_files[package.architecture] = (open(
package_index_path, 'wb'), package_index_path)
package.to822('all').dump(
package_index_files[package.architecture][0])
package_index_files[package.architecture][0].write(b'\n')
for package_index_file, package_index_path in package_index_files.values(
):
package_index_file.close()
gz_package_index_path = _zip_file(package_index_path)
_add_to_release(release, package_index_path)
_add_to_release(release, gz_package_index_path)
package_index = PublishedMetadata(
relative_path=package_index_path,
publication=publication,
file=File(open(package_index_path, 'rb')),
)
package_index.save()
gz_package_index = PublishedMetadata(
relative_path=gz_package_index_path,
publication=publication,
file=File(open(gz_package_index_path, 'rb')),
)
gz_package_index.save()
release['Architectures'] = ', '.join(
package_index_files.keys())
release_path = os.path.join('dists', 'default', 'Release')
os.makedirs(os.path.dirname(release_path), exist_ok=True)
with open(release_path, 'wb') as release_file:
release.dump(release_file)
release_metadata = PublishedMetadata(
relative_path=release_path,
publication=publication,
file=File(open(release_path, 'rb')),
)
release_metadata.save()
if publisher.structured:
raise NotImplementedError(
"Structured publishing is not yet implemented.")
log.info(
_('Publication: {publication} created').format(
publication=publication.pk))
def create_repomd_xml(content, publication, extra_repomdrecords, sub_folder=None):
"""
Creates a repomd.xml file.
Args:
content(app.models.Content): content set
publication(pulpcore.plugin.models.Publication): the publication
extra_repomdrecords(list): list with data relative to repo metadata files
sub_folder(str): name of the folder for sub repos
"""
cwd = os.getcwd()
repodata_path = REPODATA_PATH
has_modules = False
has_comps = False
if sub_folder:
cwd = os.path.join(cwd, sub_folder)
repodata_path = os.path.join(sub_folder, repodata_path)
# Prepare metadata files
repomd_path = os.path.join(cwd, "repomd.xml")
pri_xml_path = os.path.join(cwd, "primary.xml.gz")
fil_xml_path = os.path.join(cwd, "filelists.xml.gz")
oth_xml_path = os.path.join(cwd, "other.xml.gz")
pri_db_path = os.path.join(cwd, "primary.sqlite")
fil_db_path = os.path.join(cwd, "filelists.sqlite")
oth_db_path = os.path.join(cwd, "other.sqlite")
upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz")
mod_yml_path = os.path.join(cwd, "modules.yaml")
comps_xml_path = os.path.join(cwd, "comps.xml")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
packages = Package.objects.filter(pk__in=content)
total_packages = packages.count()
pri_xml.set_num_of_pkgs(total_packages)
fil_xml.set_num_of_pkgs(total_packages)
oth_xml.set_num_of_pkgs(total_packages)
# Process all packages
for package in packages.iterator():
pkg = package.to_createrepo_c()
pkg.location_href = package.contentartifact_set.only('relative_path').first().relative_path
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(pk__in=content).iterator():
upd_xml.add_chunk(cr.xml_dump_updaterecord(update_record.to_createrepo_c()))
# Process modulemd and modulemd_defaults
with open(mod_yml_path, 'ab') as mod_yml:
for modulemd in Modulemd.objects.filter(pk__in=content).iterator():
mod_yml.write(modulemd._artifacts.get().file.read())
has_modules = True
for default in ModulemdDefaults.objects.filter(pk__in=content).iterator():
mod_yml.write(default._artifacts.get().file.read())
has_modules = True
# Process comps
comps = libcomps.Comps()
for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator():
group = pkg_grp.pkg_grp_to_libcomps()
comps.groups.append(group)
has_comps = True
for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator():
cat = pkg_cat.pkg_cat_to_libcomps()
comps.categories.append(cat)
has_comps = True
for pkg_env in PackageEnvironment.objects.filter(pk__in=content).iterator():
env = pkg_env.pkg_env_to_libcomps()
comps.environments.append(env)
has_comps = True
for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator():
comps.langpacks = dict_to_strdict(pkg_lng.matches)
has_comps = True
comps.toxml_f(comps_xml_path, xml_options={"default_explicit": True,
"empty_groups": True,
"uservisible_explicit": True})
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
repomdrecords = [("primary", pri_xml_path, pri_db),
("filelists", fil_xml_path, fil_db),
("other", oth_xml_path, oth_db),
("primary_db", pri_db_path, None),
("filelists_db", fil_db_path, None),
("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None)]
if has_modules:
repomdrecords.append(("modules", mod_yml_path, None))
if has_comps:
repomdrecords.append(("group", comps_xml_path, None))
repomdrecords.extend(extra_repomdrecords)
sqlite_files = ("primary_db", "filelists_db", "other_db")
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
if name in sqlite_files:
record_bz = record.compress_and_fill(cr.SHA256, cr.BZ2)
record_bz.type = name
record_bz.rename_file()
path = record_bz.location_href.split('/')[-1]
repomd.set_record(record_bz)
else:
record.fill(cr.SHA256)
if (db_to_update):
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
record.rename_file()
path = record.location_href.split('/')[-1]
repomd.set_record(record)
if sub_folder:
path = os.path.join(sub_folder, path)
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path, os.path.basename(path)),
publication=publication,
file=File(open(path, 'rb'))
)
with open(repomd_path, "w") as repomd_f:
repomd_f.write(repomd.xml_dump())
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path, os.path.basename(repomd_path)),
publication=publication,
file=File(open(repomd_path, 'rb'))
)
def publish(repository_version_pk, simple=False, structured=False):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
repository_version_pk (str): Create a publication from this repository version.
simple (bool): Create a simple publication with all packages contained in default/all.
structured (bool): Create a structured publication with releases and components.
(Not yet implemented)
"""
repo_version = RepositoryVersion.objects.get(pk=repository_version_pk)
log.info(
_("Publishing: repository={repo}, version={ver}, simple={simple}, structured={structured}"
).format( # noqa
repo=repo_version.repository.name,
ver=repo_version.number,
simple=simple,
structured=structured,
))
with WorkingDirectory():
with DebPublication.create(repo_version,
pass_through=False) as publication:
publication.simple = simple
publication.structured = structured
if simple:
repository = repo_version.repository
release = deb822.Release()
# TODO: release['Label']
release["Codename"] = "default"
release["Components"] = "all"
release["Architectures"] = ""
if repository.description:
release["Description"] = repository.description
release["MD5sum"] = []
release["SHA1"] = []
release["SHA256"] = []
release["SHA512"] = []
package_index_files = {}
for package in Package.objects.filter(
pk__in=repo_version.content.order_by("-pulp_created")):
published_artifact = PublishedArtifact(
relative_path=package.filename(),
publication=publication,
content_artifact=package.contentartifact_set.get(),
)
published_artifact.save()
if package.architecture not in package_index_files:
package_index_path = os.path.join(
"dists",
"default",
"all",
"binary-{}".format(package.architecture),
"Packages",
)
os.makedirs(os.path.dirname(package_index_path),
exist_ok=True)
package_index_files[package.architecture] = (
open(package_index_path, "wb"),
package_index_path,
)
package_serializer = Package822Serializer(
package, context={"request": None})
package_serializer.to822("all").dump(
package_index_files[package.architecture][0])
package_index_files[package.architecture][0].write(b"\n")
for (package_index_file,
package_index_path) in package_index_files.values():
package_index_file.close()
gz_package_index_path = _zip_file(package_index_path)
_add_to_release(release, package_index_path)
_add_to_release(release, gz_package_index_path)
package_index = PublishedMetadata.create_from_file(
publication=publication,
file=File(open(package_index_path, "rb")))
package_index.save()
gz_package_index = PublishedMetadata.create_from_file(
publication=publication,
file=File(open(gz_package_index_path, "rb")))
gz_package_index.save()
release["Architectures"] = ", ".join(
package_index_files.keys())
release_path = os.path.join("dists", "default", "Release")
os.makedirs(os.path.dirname(release_path), exist_ok=True)
with open(release_path, "wb") as release_file:
release.dump(release_file)
release_metadata = PublishedMetadata.create_from_file(
publication=publication,
file=File(open(release_path, "rb")))
release_metadata.save()
if structured:
raise NotImplementedError(
"Structured publishing is not yet implemented.")
log.info(
_("Publication: {publication} created").format(
publication=publication.pk))
def create_repomd_xml(content,
publication,
checksum_types,
extra_repomdrecords,
sub_folder=None,
metadata_signing_service=None):
"""
Creates a repomd.xml file.
Args:
content(app.models.Content): content set
publication(pulpcore.plugin.models.Publication): the publication
extra_repomdrecords(list): list with data relative to repo metadata files
sub_folder(str): name of the folder for sub repos
metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService):
A reference to an associated signing service.
"""
cwd = os.getcwd()
repodata_path = REPODATA_PATH
has_modules = False
has_comps = False
package_checksum_type = checksum_types.get("package")
if sub_folder:
cwd = os.path.join(cwd, sub_folder)
repodata_path = os.path.join(sub_folder, repodata_path)
# Prepare metadata files
repomd_path = os.path.join(cwd, "repomd.xml")
pri_xml_path = os.path.join(cwd, "primary.xml.gz")
fil_xml_path = os.path.join(cwd, "filelists.xml.gz")
oth_xml_path = os.path.join(cwd, "other.xml.gz")
pri_db_path = os.path.join(cwd, "primary.sqlite")
fil_db_path = os.path.join(cwd, "filelists.sqlite")
oth_db_path = os.path.join(cwd, "other.sqlite")
upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz")
mod_yml_path = os.path.join(cwd, "modules.yaml")
comps_xml_path = os.path.join(cwd, "comps.xml")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
packages = Package.objects.filter(pk__in=content)
total_packages = packages.count()
pri_xml.set_num_of_pkgs(total_packages)
fil_xml.set_num_of_pkgs(total_packages)
oth_xml.set_num_of_pkgs(total_packages)
# Process all packages
for package in packages.iterator():
pkg = package.to_createrepo_c(package_checksum_type)
pkg_filename = os.path.basename(package.location_href)
# this can cause an issue when two same RPM package names appears
# a/name1.rpm b/name1.rpm
pkg.location_href = os.path.join(PACKAGES_DIRECTORY,
pkg_filename[0].lower(), pkg_filename)
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=content).iterator():
upd_xml.add_chunk(
cr.xml_dump_updaterecord(update_record.to_createrepo_c()))
# Process modulemd and modulemd_defaults
with open(mod_yml_path, 'ab') as mod_yml:
for modulemd in Modulemd.objects.filter(pk__in=content).iterator():
mod_yml.write(modulemd._artifacts.get().file.read())
has_modules = True
for default in ModulemdDefaults.objects.filter(
pk__in=content).iterator():
mod_yml.write(default._artifacts.get().file.read())
has_modules = True
# Process comps
comps = libcomps.Comps()
for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator():
group = pkg_grp.pkg_grp_to_libcomps()
comps.groups.append(group)
has_comps = True
for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator():
cat = pkg_cat.pkg_cat_to_libcomps()
comps.categories.append(cat)
has_comps = True
for pkg_env in PackageEnvironment.objects.filter(
pk__in=content).iterator():
env = pkg_env.pkg_env_to_libcomps()
comps.environments.append(env)
has_comps = True
for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator():
comps.langpacks = dict_to_strdict(pkg_lng.matches)
has_comps = True
comps.toxml_f(comps_xml_path,
xml_options={
"default_explicit": True,
"empty_groups": True,
"uservisible_explicit": True
})
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
repomdrecords = [("primary", pri_xml_path, pri_db),
("filelists", fil_xml_path, fil_db),
("other", oth_xml_path, oth_db),
("primary_db", pri_db_path, None),
("filelists_db", fil_db_path, None),
("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None)]
if has_modules:
repomdrecords.append(("modules", mod_yml_path, None))
if has_comps:
repomdrecords.append(("group", comps_xml_path, None))
repomdrecords.extend(extra_repomdrecords)
sqlite_files = ("primary_db", "filelists_db", "other_db")
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
checksum_type = get_checksum_type(name, checksum_types)
if name in sqlite_files:
record_bz = record.compress_and_fill(checksum_type, cr.BZ2)
record_bz.type = name
record_bz.rename_file()
path = record_bz.location_href.split('/')[-1]
repomd.set_record(record_bz)
else:
record.fill(checksum_type)
if (db_to_update):
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
record.rename_file()
path = record.location_href.split('/')[-1]
repomd.set_record(record)
if sub_folder:
path = os.path.join(sub_folder, path)
PublishedMetadata.create_from_file(relative_path=os.path.join(
repodata_path, os.path.basename(path)),
publication=publication,
file=File(open(path, 'rb')))
with open(repomd_path, "w") as repomd_f:
repomd_f.write(repomd.xml_dump())
if metadata_signing_service:
signing_service = AsciiArmoredDetachedSigningService.objects.get(
pk=metadata_signing_service.pk)
sign_results = signing_service.sign(repomd_path)
# publish a signed file
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(sign_results['file'])),
publication=publication,
file=File(open(sign_results['file'], 'rb')))
# publish a detached signature
PublishedMetadata.create_from_file(relative_path=os.path.join(
repodata_path, os.path.basename(sign_results['signature'])),
publication=publication,
file=File(
open(sign_results['signature'],
'rb')))
# publish a public key required for further verification
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(sign_results['key'])),
publication=publication,
file=File(open(sign_results['key'], 'rb')))
else:
PublishedMetadata.create_from_file(relative_path=os.path.join(
repodata_path, os.path.basename(repomd_path)),
publication=publication,
file=File(open(repomd_path, 'rb')))
def generate_repo_metadata(
content,
publication,
checksum_types,
extra_repomdrecords,
sub_folder=None,
metadata_signing_service=None,
):
"""
Creates a repomd.xml file.
Args:
content(app.models.Content): content set
publication(pulpcore.plugin.models.Publication): the publication
extra_repomdrecords(list): list with data relative to repo metadata files
sub_folder(str): name of the folder for sub repos
metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService):
A reference to an associated signing service.
"""
cwd = os.getcwd()
repodata_path = REPODATA_PATH
has_modules = False
has_comps = False
package_checksum_type = checksum_types.get("package")
if sub_folder:
cwd = os.path.join(cwd, sub_folder)
repodata_path = os.path.join(sub_folder, repodata_path)
if package_checksum_type and package_checksum_type not in settings.ALLOWED_CONTENT_CHECKSUMS:
raise ValueError(
"Repository contains disallowed package checksum type '{}', "
"thus can't be published. {}".format(package_checksum_type,
ALLOWED_CHECKSUM_ERROR_MSG))
# Prepare metadata files
repomd_path = os.path.join(cwd, "repomd.xml")
pri_xml_path = os.path.join(cwd, "primary.xml.gz")
fil_xml_path = os.path.join(cwd, "filelists.xml.gz")
oth_xml_path = os.path.join(cwd, "other.xml.gz")
upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz")
mod_yml_path = os.path.join(cwd, "modules.yaml")
comps_xml_path = os.path.join(cwd, "comps.xml")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
if publication.sqlite_metadata:
pri_db_path = os.path.join(cwd, "primary.sqlite")
fil_db_path = os.path.join(cwd, "filelists.sqlite")
oth_db_path = os.path.join(cwd, "other.sqlite")
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
packages = Package.objects.filter(pk__in=content)
# We want to support publishing with a different checksum type than the one built-in to the
# package itself, so we need to get the correct checksums somehow if there is an override.
# We must also take into consideration that if the package has not been downloaded the only
# checksum that is available is the one built-in.
#
# Since this lookup goes from Package->Content->ContentArtifact->Artifact, performance is a
# challenge. We use ContentArtifact as our starting point because it enables us to work with
# simple foreign keys and avoid messing with the many-to-many relationship, which doesn't
# work with select_related() and performs poorly with prefetch_related(). This is fine
# because we know that Packages should only ever have one artifact per content.
contentartifact_qs = (
ContentArtifact.objects.filter(content__in=packages.only("pk")).
select_related(
# content__rpm_package is a bit of a hack, exploiting the way django sets up model
# inheritance, but it works and is unlikely to break. All content artifacts being
# accessed here have an associated Package since they originally came from the
# Package queryset.
"artifact",
"content__rpm_package",
).only("artifact", "content__rpm_package__checksum_type",
"content__rpm_package__pkgId"))
pkg_to_hash = {}
for ca in contentartifact_qs.iterator():
if package_checksum_type:
package_checksum_type = package_checksum_type.lower()
pkgid = getattr(ca.artifact, package_checksum_type, None)
if not package_checksum_type or not pkgid:
if ca.content.rpm_package.checksum_type not in settings.ALLOWED_CONTENT_CHECKSUMS:
raise ValueError(
"Package {} as content unit {} contains forbidden checksum type '{}', "
"thus can't be published. {}".format(
ca.content.rpm_package.nevra,
ca.content.pk,
ca.content.rpm_package.checksum_type,
ALLOWED_CHECKSUM_ERROR_MSG,
))
package_checksum_type = ca.content.rpm_package.checksum_type
pkgid = ca.content.rpm_package.pkgId
pkg_to_hash[ca.content_id] = (package_checksum_type, pkgid)
# TODO: this is meant to be a !! *temporary* !! fix for
# https://github.com/pulp/pulp_rpm/issues/2407
pkg_pks_to_ignore = set()
latest_build_time_by_nevra = defaultdict(list)
for pkg in packages.only("pk", "name", "epoch", "version", "release",
"arch", "time_build").iterator():
latest_build_time_by_nevra[pkg.nevra].append((pkg.time_build, pkg.pk))
for nevra, pkg_data in latest_build_time_by_nevra.items():
# sort the packages by when they were built
if len(pkg_data) > 1:
pkg_data.sort(key=lambda p: p[0], reverse=True)
pkg_pks_to_ignore |= set(entry[1] for entry in pkg_data[1:])
log.warning(
"Duplicate packages found competing for NEVRA {nevra}, selected the one with "
"the most recent build time, excluding {others} others.".
format(nevra=nevra, others=len(pkg_data[1:])))
total_packages = packages.count() - len(pkg_pks_to_ignore)
pri_xml.set_num_of_pkgs(total_packages)
fil_xml.set_num_of_pkgs(total_packages)
oth_xml.set_num_of_pkgs(total_packages)
# Process all packages
for package in packages.order_by("name", "evr").iterator():
if package.pk in pkg_pks_to_ignore: # Temporary!
continue
pkg = package.to_createrepo_c()
# rewrite the checksum and checksum type with the desired ones
(checksum, pkgId) = pkg_to_hash[package.pk]
pkg.checksum_type = checksum
pkg.pkgId = pkgId
pkg_filename = os.path.basename(package.location_href)
# this can cause an issue when two same RPM package names appears
# a/name1.rpm b/name1.rpm
pkg.location_href = os.path.join(PACKAGES_DIRECTORY,
pkg_filename[0].lower(), pkg_filename)
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
if publication.sqlite_metadata:
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=content).iterator():
upd_xml.add_chunk(
cr.xml_dump_updaterecord(update_record.to_createrepo_c()))
# Process modulemd, modulemd_defaults and obsoletes
with open(mod_yml_path, "ab") as mod_yml:
for modulemd in Modulemd.objects.filter(pk__in=content).iterator():
mod_yml.write(modulemd.snippet.encode())
has_modules = True
for default in ModulemdDefaults.objects.filter(
pk__in=content).iterator():
mod_yml.write(default.snippet.encode())
has_modules = True
for obsolete in ModulemdObsolete.objects.filter(
pk__in=content).iterator():
mod_yml.write(obsolete.snippet.encode())
has_modules = True
# Process comps
comps = libcomps.Comps()
for pkg_grp in PackageGroup.objects.filter(pk__in=content).iterator():
group = pkg_grp.pkg_grp_to_libcomps()
comps.groups.append(group)
has_comps = True
for pkg_cat in PackageCategory.objects.filter(pk__in=content).iterator():
cat = pkg_cat.pkg_cat_to_libcomps()
comps.categories.append(cat)
has_comps = True
for pkg_env in PackageEnvironment.objects.filter(
pk__in=content).iterator():
env = pkg_env.pkg_env_to_libcomps()
comps.environments.append(env)
has_comps = True
for pkg_lng in PackageLangpacks.objects.filter(pk__in=content).iterator():
comps.langpacks = dict_to_strdict(pkg_lng.matches)
has_comps = True
comps.toxml_f(
comps_xml_path,
xml_options={
"default_explicit": True,
"empty_groups": True,
"empty_packages": True,
"uservisible_explicit": True,
},
)
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
# If the repository is empty, use a revision of 0
# See: https://pulp.plan.io/issues/9402
if not content.exists():
repomd.revision = "0"
if publication.sqlite_metadata:
repomdrecords = [
("primary", pri_xml_path, pri_db),
("filelists", fil_xml_path, fil_db),
("other", oth_xml_path, oth_db),
("primary_db", pri_db_path, None),
("filelists_db", fil_db_path, None),
("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None),
]
else:
repomdrecords = [
("primary", pri_xml_path, None),
("filelists", fil_xml_path, None),
("other", oth_xml_path, None),
("updateinfo", upd_xml_path, None),
]
if has_modules:
repomdrecords.append(("modules", mod_yml_path, None))
if has_comps:
repomdrecords.append(("group", comps_xml_path, None))
repomdrecords.extend(extra_repomdrecords)
sqlite_files = ("primary_db", "filelists_db", "other_db")
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
checksum_type = cr_checksum_type_from_string(
get_checksum_type(name,
checksum_types,
default=publication.metadata_checksum_type))
if name in sqlite_files:
record_bz = record.compress_and_fill(checksum_type, cr.BZ2)
record_bz.type = name
record_bz.rename_file()
path = record_bz.location_href.split("/")[-1]
repomd.set_record(record_bz)
else:
record.fill(checksum_type)
if db_to_update:
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
record.rename_file()
path = record.location_href.split("/")[-1]
repomd.set_record(record)
if sub_folder:
path = os.path.join(sub_folder, path)
with open(path, "rb") as repodata_fd:
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(path)),
publication=publication,
file=File(repodata_fd),
)
with open(repomd_path, "w") as repomd_f:
repomd_f.write(repomd.xml_dump())
if metadata_signing_service:
signing_service = AsciiArmoredDetachedSigningService.objects.get(
pk=metadata_signing_service)
sign_results = signing_service.sign(repomd_path)
# publish a signed file
with open(sign_results["file"], "rb") as signed_file_fd:
PublishedMetadata.create_from_file(
relative_path=os.path.join(
repodata_path, os.path.basename(sign_results["file"])),
publication=publication,
file=File(signed_file_fd),
)
# publish a detached signature
with open(sign_results["signature"], "rb") as signature_fd:
PublishedMetadata.create_from_file(
relative_path=os.path.join(
repodata_path,
os.path.basename(sign_results["signature"])),
publication=publication,
file=File(signature_fd),
)
# publish a public key required for further verification
pubkey_name = "repomd.xml.key"
with open(pubkey_name, "wb+") as f:
f.write(signing_service.public_key.encode("utf-8"))
f.flush()
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path, pubkey_name),
publication=publication,
file=File(f),
)
else:
with open(repomd_path, "rb") as repomd_fd:
PublishedMetadata.create_from_file(
relative_path=os.path.join(repodata_path,
os.path.basename(repomd_path)),
publication=publication,
file=File(repomd_fd),
)
def create_rempomd_xml(packages,
publication,
extra_repomdrecords,
sub_folder=None):
"""
Creates a repomd.xml file.
Args:
packages(app.models.Package): set of packages
publication(pulpcore.plugin.models.Publication): the publication
extra_repomdrecords(list): list with data relative to repo metadata files
sub_folder(str): name of the folder for sub repos
"""
cwd = os.getcwd()
repodata_path = REPODATA_PATH
has_modules = False
if sub_folder:
cwd = os.path.join(cwd, sub_folder)
repodata_path = os.path.join(sub_folder, repodata_path)
# Prepare metadata files
repomd_path = os.path.join(cwd, "repomd.xml")
pri_xml_path = os.path.join(cwd, "primary.xml.gz")
fil_xml_path = os.path.join(cwd, "filelists.xml.gz")
oth_xml_path = os.path.join(cwd, "other.xml.gz")
pri_db_path = os.path.join(cwd, "primary.sqlite")
fil_db_path = os.path.join(cwd, "filelists.sqlite")
oth_db_path = os.path.join(cwd, "other.sqlite")
upd_xml_path = os.path.join(cwd, "updateinfo.xml.gz")
mod_yml_path = os.path.join(cwd, "modules.yaml")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
pri_xml.set_num_of_pkgs(len(packages))
fil_xml.set_num_of_pkgs(len(packages))
oth_xml.set_num_of_pkgs(len(packages))
# Process all packages
for package in packages:
pkg = package.to_createrepo_c()
pkg.location_href = package.contentartifact_set.first().relative_path
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=publication.repository_version.content):
upd_xml.add_chunk(update_record_xml(update_record))
# Process modulemd and modulemd_defaults
with open(mod_yml_path, 'ab') as mod_yml:
for modulemd in Modulemd.objects.filter(
pk__in=publication.repository_version.content):
mod_yml.write(modulemd._artifacts.get().file.read())
has_modules = True
for default in ModulemdDefaults.objects.filter(
pk__in=publication.repository_version.content):
mod_yml.write(default._artifacts.get().file.read())
has_modules = True
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
repomdrecords = [("primary", pri_xml_path, pri_db),
("filelists", fil_xml_path, fil_db),
("other", oth_xml_path, oth_db),
("primary_db", pri_db_path, None),
("filelists_db", fil_db_path, None),
("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None)]
if has_modules:
repomdrecords.append(("modules", mod_yml_path, None))
repomdrecords.extend(extra_repomdrecords)
sqlite_files = ("primary_db", "filelists_db", "other_db")
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
if name in sqlite_files:
record_bz = record.compress_and_fill(cr.SHA256, cr.BZ2)
record_bz.type = name
record_bz.rename_file()
path = record_bz.location_href.split('/')[-1]
repomd.set_record(record_bz)
elif name == "modules":
record_md = record.compress_and_fill(cr.SHA256, cr.GZ)
record_md.type = name
record_md.rename_file()
path = record_md.location_href.split('/')[-1]
repomd.set_record(record_md)
else:
record.fill(cr.SHA256)
if (db_to_update):
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
record.rename_file()
path = record.location_href.split('/')[-1]
repomd.set_record(record)
if sub_folder:
path = os.path.join(sub_folder, path)
PublishedMetadata.create_from_file(relative_path=os.path.join(
repodata_path, os.path.basename(path)),
publication=publication,
file=File(open(path, 'rb')))
with open(repomd_path, "w") as repomd_f:
repomd_f.write(repomd.xml_dump())
PublishedMetadata.create_from_file(relative_path=os.path.join(
repodata_path, os.path.basename(repomd_path)),
publication=publication,
file=File(open(repomd_path, 'rb')))
