def publish_artifacts(self, content):
"""
Publish artifacts.
Args:
content (pulpcore.plugin.models.Content): content set.
"""
published_artifacts = []
for content_artifact in ContentArtifact.objects.filter(
content__in=content.exclude(pulp_type__in=[
RepoMetadataFile.get_pulp_type(),
Modulemd.get_pulp_type(),
ModulemdDefaults.get_pulp_type()
]).distinct()).iterator():
relative_path = content_artifact.relative_path
if content_artifact.content.pulp_type == Package.get_pulp_type():
relative_path = os.path.join(PACKAGES_DIRECTORY,
relative_path.lower()[0],
content_artifact.relative_path)
published_artifacts.append(
PublishedArtifact(relative_path=relative_path,
publication=self.publication,
content_artifact=content_artifact))
PublishedArtifact.objects.bulk_create(published_artifacts,
batch_size=2000)
def publish_artifacts(self, content, prefix=""):
"""
Publish artifacts.
Args:
content (pulpcore.plugin.models.Content): content set.
prefix (str): a relative path prefix for the published artifact
"""
published_artifacts = []
# Special case for Packages
contentartifact_qs = ContentArtifact.objects.filter(
content__in=content).filter(
content__pulp_type=Package.get_pulp_type())
for content_artifact in contentartifact_qs.values(
"pk", "relative_path").iterator():
relative_path = content_artifact["relative_path"]
relative_path = os.path.join(prefix, PACKAGES_DIRECTORY,
relative_path.lower()[0],
relative_path)
published_artifacts.append(
PublishedArtifact(
relative_path=relative_path,
publication=self.publication,
content_artifact_id=content_artifact["pk"],
))
# Handle everything else
is_treeinfo = Q(relative_path__in=["treeinfo", ".treeinfo"])
unpublishable_types = Q(content__pulp_type__in=[
RepoMetadataFile.get_pulp_type(),
Modulemd.get_pulp_type(),
ModulemdDefaults.get_pulp_type(),
# already dealt with
Package.get_pulp_type(),
])
contentartifact_qs = (ContentArtifact.objects.filter(
content__in=content).exclude(unpublishable_types).exclude(
is_treeinfo))
for content_artifact in contentartifact_qs.values(
"pk", "relative_path").iterator():
published_artifacts.append(
PublishedArtifact(
relative_path=content_artifact["relative_path"],
publication=self.publication,
content_artifact_id=content_artifact["pk"],
))
PublishedArtifact.objects.bulk_create(published_artifacts,
batch_size=2000)
def publish_artifacts(self, content, prefix=""):
"""
Publish artifacts.
Args:
content (pulpcore.plugin.models.Content): content set.
prefix (str): a relative path prefix for the published artifact
"""
published_artifacts = []
# Special case for Packages
contentartifact_qs = ContentArtifact.objects.filter(content__in=content).filter(
content__pulp_type=Package.get_pulp_type()
)
paths = set()
duplicated_paths = []
for content_artifact in contentartifact_qs.values("pk", "relative_path").iterator():
relative_path = content_artifact["relative_path"]
relative_path = os.path.join(
prefix, PACKAGES_DIRECTORY, relative_path.lower()[0], relative_path
)
#
# Some Suboptimal Repos have the 'same' artifact living in multiple places.
# Specifically, the same NEVRA, in more than once place, **with different checksums**
# (since if all that was different was location_href there would be only one
# ContentArtifact in the first place).
#
# pulp_rpm wants to publish a 'canonical' repository-layout, under which an RPM
# "name-version-release-arch" appears at "Packages/n/name-version-release-arch.rpm".
# Because the assumption is that Packages don't "own" their path, only the filename
# is kept as relative_path.
#
# In this case, we have to pick one - which is essentially what the rest of the RPM
# Ecosystem does when faced with the impossible. This code takes the first-found. We
# could implement something more complicated, if there are better options
# (choose by last-created maybe?)
#
# Note that this only impacts user-created publications, which produce the "standard"
# RPM layout of repo/Packages/f/foo.rpm. A publication created by mirror-sync retains
# whatever layout their "upstream" repo-metadata dictates.
#
if relative_path in paths:
duplicated_paths.append(f'{relative_path}:{content_artifact["pk"]}')
continue
else:
paths.add(relative_path)
published_artifacts.append(
PublishedArtifact(
relative_path=relative_path,
publication=self.publication,
content_artifact_id=content_artifact["pk"],
)
)
if duplicated_paths:
log.warning(
_("Duplicate paths found at publish : {problems} ").format(
problems="; ".join(duplicated_paths)
)
)
# Handle everything else
is_treeinfo = Q(relative_path__in=["treeinfo", ".treeinfo"])
unpublishable_types = Q(
content__pulp_type__in=[
RepoMetadataFile.get_pulp_type(),
Modulemd.get_pulp_type(),
ModulemdDefaults.get_pulp_type(),
# already dealt with
Package.get_pulp_type(),
]
)
contentartifact_qs = (
ContentArtifact.objects.filter(content__in=content)
.exclude(unpublishable_types)
.exclude(is_treeinfo)
)
for content_artifact in contentartifact_qs.values("pk", "relative_path").iterator():
published_artifacts.append(
PublishedArtifact(
relative_path=content_artifact["relative_path"],
publication=self.publication,
content_artifact_id=content_artifact["pk"],
)
)
PublishedArtifact.objects.bulk_create(published_artifacts, batch_size=2000)
def publish_artifacts(self, content, prefix=""):
"""
Publish artifacts.
Args:
content (pulpcore.plugin.models.Content): content set.
prefix (str): a relative path prefix for the published artifact
"""
published_artifacts = []
# Special case for Packages
contentartifact_qs = (ContentArtifact.objects.filter(
content__in=content).filter(
content__pulp_type=Package.get_pulp_type()).select_related(
"content__rpm_package__time_build"))
rel_path_mapping = defaultdict(list)
# Some Suboptimal Repos have the 'same' artifact living in multiple places.
# Specifically, the same NEVRA, in more than once place, **with different checksums**
# (since if all that was different was location_href there would be only one
# ContentArtifact in the first place).
#
# pulp_rpm wants to publish a 'canonical' repository-layout, under which an RPM
# "name-version-release-arch" appears at "Packages/n/name-version-release-arch.rpm".
# Because the assumption is that Packages don't "own" their path, only the filename
# is kept as relative_path.
#
# In this case, we have to pick one - which is essentially what the rest of the RPM
# Ecosystem does when faced with the impossible. This code takes the one with the
# most recent build time which is the same heuristic used by Yum/DNF/Zypper.
#
# Note that this only impacts user-created publications, which produce the "standard"
# RPM layout of repo/Packages/f/foo.rpm. A publication created by mirror-sync retains
# whatever layout their "upstream" repo-metadata dictates.
fields = ["pk", "relative_path", "content__rpm_package__time_build"]
for content_artifact in contentartifact_qs.values(*fields).iterator():
relative_path = content_artifact["relative_path"]
time_build = content_artifact["content__rpm_package__time_build"]
relative_path = os.path.join(prefix, PACKAGES_DIRECTORY,
relative_path.lower()[0],
relative_path)
rel_path_mapping[relative_path].append(
(content_artifact["pk"], time_build))
for rel_path, content_artifacts in rel_path_mapping.items():
# sort the content artifacts by when the package was built
if len(content_artifacts) > 1:
content_artifacts.sort(key=lambda p: p[1], reverse=True)
log.warning(
"Duplicate packages found competing for {path}, selected the one with "
"the most recent build time, excluding {others} others.".
format(path=rel_path, others=len(content_artifacts[1:])))
# Only add the first one (the one with the highest build time)
published_artifacts.append(
PublishedArtifact(
relative_path=rel_path,
publication=self.publication,
content_artifact_id=content_artifacts[0][0],
))
# Handle everything else
is_treeinfo = Q(relative_path__in=["treeinfo", ".treeinfo"])
unpublishable_types = Q(content__pulp_type__in=[
RepoMetadataFile.get_pulp_type(),
Modulemd.get_pulp_type(),
ModulemdDefaults.get_pulp_type(),
# already dealt with
Package.get_pulp_type(),
])
contentartifact_qs = (ContentArtifact.objects.filter(
content__in=content).exclude(unpublishable_types).exclude(
is_treeinfo))
for content_artifact in contentartifact_qs.values(
"pk", "relative_path").iterator():
published_artifacts.append(
PublishedArtifact(
relative_path=content_artifact["relative_path"],
publication=self.publication,
content_artifact_id=content_artifact["pk"],
))
PublishedArtifact.objects.bulk_create(published_artifacts,
batch_size=2000)
