def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a Publication from this repository version.
"""
publisher = GemPublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_('Publishing: repository=%(repository)s, version=%(version)d, publisher=%(publisher)s'
), {
'repository': repository_version.repository.name,
'version': repository_version.number,
'publisher': publisher.name,
})
with WorkingDirectory():
with Publication.create(repository_version, publisher) as publication:
specs = Specs()
latest_versions = {}
prerelease_specs = Specs()
for content in GemContent.objects.filter(
pk__in=publication.repository_version.content).order_by(
'-created'):
for content_artifact in content.contentartifact_set.all():
published_artifact = PublishedArtifact(
relative_path=content_artifact.relative_path,
publication=publication,
content_artifact=content_artifact)
published_artifact.save()
if re.fullmatch(r"[0-9.]*", content.version):
specs.append(Key(content.name, content.version))
old_ver = latest_versions.get(content.name)
if old_ver is None or version.parse(
old_ver) < version.parse(content.version):
latest_versions[content.name] = content.version
else:
prerelease_specs.append(Key(content.name, content.version))
latest_specs = Specs(
Key(name, version)
for name, version in latest_versions.items())
publish_specs(specs, 'specs.4.8', publication)
publish_specs(latest_specs, 'latest_specs.4.8', publication)
publish_specs(prerelease_specs, 'prerelease_specs.4.8',
publication)
log.info(_('Publication: %(publication)s created'),
{'publication': publication.pk})
def add_package(self, package):
published_artifact = PublishedArtifact(
relative_path=package.filename(self.name),
publication=self.parent.publication,
content_artifact=package.contentartifact_set.get(),
)
published_artifact.save()
package_serializer = Package822Serializer(package,
context={"request": None})
package_serializer.to822(self.name).dump(
self.package_index_files[package.architecture][0])
self.package_index_files[package.architecture][0].write(b"\n")
def publish(repository_version_pk):
"""
Create a Publication based on a RepositoryVersion.
Args:
repository_version_pk (str): Create a publication from this repository version.
"""
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_("Publishing: repository={repo}, version={ver}").format(
repo=repository_version.repository.name,
ver=repository_version.number))
with WorkingDirectory():
with GemPublication.create(repository_version) as publication:
specs = []
latest_versions = {}
prerelease_specs = []
for content in GemContent.objects.filter(
pk__in=publication.repository_version.content).order_by(
"-pulp_created"):
for content_artifact in content.contentartifact_set.all():
published_artifact = PublishedArtifact(
relative_path=content_artifact.relative_path,
publication=publication,
content_artifact=content_artifact,
)
published_artifact.save()
if re.fullmatch(r"[0-9.]*", content.version):
specs.append(Key(content.name, content.version))
old_ver = latest_versions.get(content.name)
if old_ver is None or version.parse(
old_ver) < version.parse(content.version):
latest_versions[content.name] = content.version
else:
prerelease_specs.append(Key(content.name, content.version))
latest_specs = [
Key(name, version)
for name, version in latest_versions.items()
]
_publish_specs(specs, "specs.4.8", publication)
_publish_specs(latest_specs, "latest_specs.4.8", publication)
_publish_specs(prerelease_specs, "prerelease_specs.4.8",
publication)
log.info(
_("Publication: {publication} created").format(
publication=publication.pk))
def publish_chart_content(publication):
"""
Create published artifacts and metadata for a publication
Args:
publication (ChartPublication): The publication to store
"""
entries = {}
for content in ChartContent.objects.filter(
pk__in=publication.repository_version.content).order_by(
'name', '-created'):
artifacts = content.contentartifact_set.all()
for artifact in artifacts:
published = PublishedArtifact(relative_path=artifact.relative_path,
publication=publication,
content_artifact=artifact)
published.save()
entry = {
'apiVersion': 'v1',
'created': content.created.isoformat(),
'description': content.description,
'digest': content.digest,
'icon': content.icon,
'keywords': content.keywords,
'name': content.name,
'urls': [artifact.relative_path for artifact in artifacts],
'version': content.version
}
if content.name not in entries:
entries[content.name] = []
# Strip away empty keys when building metadata
entries[content.name].append(
{k: v
for k, v in entry.items() if (v is not None and v != [])})
doc = {
'apiVersion': 'v1',
'entries': entries,
'generated': timezone.now().isoformat()
}
with open('index.yaml', 'w') as index:
index.write(yaml.dump(doc))
index = PublishedMetadata.create_from_file(publication=publication,
file=File(
open('index.yaml', 'rb')))
index.save()
def _publish(self):
"""
Create published artifacts and yield the manifest entry for each.
Yields:
Entry: The manifest entry.
"""
for content in self.publication.repo_version.content():
for content_artifact in content.contentartifact_set.all():
artifact = self._find_artifact(content_artifact)
published_artifact = PublishedArtifact(
relative_path=content_artifact.relative_path,
publication=self.publication,
content_artifact=content_artifact)
published_artifact.save()
entry = Entry(
path=content_artifact.relative_path,
digest=artifact.sha256,
size=artifact.size)
yield entry
def populate(publication):
"""
Populate a publication.
Create published artifacts for a publication.
Args:
publication (pulpcore.plugin.models.Publication): A Publication to populate.
"""
def find_artifact():
_artifact = content_artifact.artifact
if not _artifact:
_artifact = RemoteArtifact.objects.filter(content_artifact=content_artifact).first()
return _artifact
for package in Package.objects.filter(pk__in=publication.repository_version.content):
for content_artifact in package.contentartifact_set.all():
published_artifact = PublishedArtifact(
relative_path=content_artifact.relative_path,
publication=publication,
content_artifact=content_artifact)
published_artifact.save()
def _publish(self):
"""
Create published artifacts and yield the string representation to be written to the
PULP_MANIFEST file.
Yields:
String: The manifest entry for the published content
"""
repo_content = ExampleContent.objects.filter(repositories=self.repository)
with ProgressBar(message=_("Publishing ExampleContent"), total=repo_content.count()) as bar:
for content in repo_content:
for content_artifact in content.contentartifact_set.all():
artifact = self._find_artifact(content_artifact)
published_artifact = PublishedArtifact(
relative_path=content_artifact.relative_path,
publication=self.publication,
content_artifact=content_artifact)
published_artifact.save()
entry = "{},{},{}".format(content_artifact.relative_path,
artifact.sha256,
artifact.size)
yield entry
bar.increment()
def populate(publication):
"""
Populate a publication.
Create published artifacts for a publication.
Args:
publication (pulpcore.plugin.models.Publication): A Publication to populate.
"""
def find_artifact():
_artifact = content_artifact.artifact
if not _artifact:
_artifact = RemoteArtifact.objects.filter(content_artifact=content_artifact).first()
return _artifact
for package in Package.objects.filter(pk__in=publication.repository_version.content):
for content_artifact in package.contentartifact_set.all():
published_artifact = PublishedArtifact(
relative_path=content_artifact.relative_path,
publication=publication,
content_artifact=content_artifact)
published_artifact.save()
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = DebPublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_('Publishing: repository={repo}, version={ver}, publisher={pub}').
format(repo=repository_version.repository.name,
ver=repository_version.number,
pub=publisher.name))
with WorkingDirectory():
with Publication.create(repository_version,
publisher,
pass_through=False) as publication:
if publisher.simple:
repository = repository_version.repository
release = deb822.Release()
# TODO: release['Label']
release['Codename'] = 'default'
release['Components'] = 'all'
release['Architectures'] = ''
if repository.description:
release['Description'] = repository.description
release['MD5sum'] = []
release['SHA1'] = []
release['SHA256'] = []
release['SHA512'] = []
package_index_files = {}
for package in Package.objects.filter(
pk__in=repository_version.content.order_by(
'-_created')):
published_artifact = PublishedArtifact(
relative_path=package.filename(),
publication=publication,
content_artifact=package.contentartifact_set.get(),
)
published_artifact.save()
if package.architecture not in package_index_files:
package_index_path = os.path.join(
'dists',
'default',
'all',
'binary-{}'.format(package.architecture),
'Packages',
)
os.makedirs(os.path.dirname(package_index_path),
exist_ok=True)
package_index_files[package.architecture] = (open(
package_index_path, 'wb'), package_index_path)
package.to822('all').dump(
package_index_files[package.architecture][0])
package_index_files[package.architecture][0].write(b'\n')
for package_index_file, package_index_path in package_index_files.values(
):
package_index_file.close()
gz_package_index_path = _zip_file(package_index_path)
_add_to_release(release, package_index_path)
_add_to_release(release, gz_package_index_path)
package_index = PublishedMetadata(
relative_path=package_index_path,
publication=publication,
file=File(open(package_index_path, 'rb')),
)
package_index.save()
gz_package_index = PublishedMetadata(
relative_path=gz_package_index_path,
publication=publication,
file=File(open(gz_package_index_path, 'rb')),
)
gz_package_index.save()
release['Architectures'] = ', '.join(
package_index_files.keys())
release_path = os.path.join('dists', 'default', 'Release')
os.makedirs(os.path.dirname(release_path), exist_ok=True)
with open(release_path, 'wb') as release_file:
release.dump(release_file)
release_metadata = PublishedMetadata(
relative_path=release_path,
publication=publication,
file=File(open(release_path, 'rb')),
)
release_metadata.save()
if publisher.structured:
raise NotImplementedError(
"Structured publishing is not yet implemented.")
log.info(
_('Publication: {publication} created').format(
publication=publication.pk))
def publish(repository_version_pk, simple=False, structured=False):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
repository_version_pk (str): Create a publication from this repository version.
simple (bool): Create a simple publication with all packages contained in default/all.
structured (bool): Create a structured publication with releases and components.
(Not yet implemented)
"""
repo_version = RepositoryVersion.objects.get(pk=repository_version_pk)
log.info(
_("Publishing: repository={repo}, version={ver}, simple={simple}, structured={structured}"
).format( # noqa
repo=repo_version.repository.name,
ver=repo_version.number,
simple=simple,
structured=structured,
))
with WorkingDirectory():
with DebPublication.create(repo_version,
pass_through=False) as publication:
publication.simple = simple
publication.structured = structured
if simple:
repository = repo_version.repository
release = deb822.Release()
# TODO: release['Label']
release["Codename"] = "default"
release["Components"] = "all"
release["Architectures"] = ""
if repository.description:
release["Description"] = repository.description
release["MD5sum"] = []
release["SHA1"] = []
release["SHA256"] = []
release["SHA512"] = []
package_index_files = {}
for package in Package.objects.filter(
pk__in=repo_version.content.order_by("-pulp_created")):
published_artifact = PublishedArtifact(
relative_path=package.filename(),
publication=publication,
content_artifact=package.contentartifact_set.get(),
)
published_artifact.save()
if package.architecture not in package_index_files:
package_index_path = os.path.join(
"dists",
"default",
"all",
"binary-{}".format(package.architecture),
"Packages",
)
os.makedirs(os.path.dirname(package_index_path),
exist_ok=True)
package_index_files[package.architecture] = (
open(package_index_path, "wb"),
package_index_path,
)
package_serializer = Package822Serializer(
package, context={"request": None})
package_serializer.to822("all").dump(
package_index_files[package.architecture][0])
package_index_files[package.architecture][0].write(b"\n")
for (package_index_file,
package_index_path) in package_index_files.values():
package_index_file.close()
gz_package_index_path = _zip_file(package_index_path)
_add_to_release(release, package_index_path)
_add_to_release(release, gz_package_index_path)
package_index = PublishedMetadata.create_from_file(
publication=publication,
file=File(open(package_index_path, "rb")))
package_index.save()
gz_package_index = PublishedMetadata.create_from_file(
publication=publication,
file=File(open(gz_package_index_path, "rb")))
gz_package_index.save()
release["Architectures"] = ", ".join(
package_index_files.keys())
release_path = os.path.join("dists", "default", "Release")
os.makedirs(os.path.dirname(release_path), exist_ok=True)
with open(release_path, "wb") as release_file:
release.dump(release_file)
release_metadata = PublishedMetadata.create_from_file(
publication=publication,
file=File(open(release_path, "rb")))
release_metadata.save()
if structured:
raise NotImplementedError(
"Structured publishing is not yet implemented.")
log.info(
_("Publication: {publication} created").format(
publication=publication.pk))
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = DebPublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(pk=repository_version_pk)
log.info(_('Publishing: repository={repo}, version={ver}, publisher={pub}').format(
repo=repository_version.repository.name,
ver=repository_version.number,
pub=publisher.name
))
with WorkingDirectory():
with Publication.create(repository_version, publisher, pass_through=False) as publication:
if publisher.simple:
repository = repository_version.repository
release = deb822.Release()
# TODO: release['Label']
release['Codename'] = 'default'
release['Components'] = 'all'
release['Architectures'] = ''
if repository.description:
release['Description'] = repository.description
release['MD5sum'] = []
release['SHA1'] = []
release['SHA256'] = []
release['SHA512'] = []
package_index_files = {}
for package in Package.objects.filter(
pk__in=repository_version.content.order_by('-_created')
):
published_artifact = PublishedArtifact(
relative_path=package.filename(),
publication=publication,
content_artifact=package.contentartifact_set.get(),
)
published_artifact.save()
if package.architecture not in package_index_files:
package_index_path = os.path.join(
'dists',
'default',
'all',
'binary-{}'.format(package.architecture),
'Packages',
)
os.makedirs(os.path.dirname(
package_index_path), exist_ok=True)
package_index_files[package.architecture] = (
open(package_index_path, 'wb'), package_index_path)
package.to822('all').dump(
package_index_files[package.architecture][0])
package_index_files[package.architecture][0].write(b'\n')
for package_index_file, package_index_path in package_index_files.values():
package_index_file.close()
gz_package_index_path = _zip_file(package_index_path)
_add_to_release(release, package_index_path)
_add_to_release(release, gz_package_index_path)
package_index = PublishedMetadata(
relative_path=package_index_path,
publication=publication,
file=File(open(package_index_path, 'rb')),
)
package_index.save()
gz_package_index = PublishedMetadata(
relative_path=gz_package_index_path,
publication=publication,
file=File(open(gz_package_index_path, 'rb')),
)
gz_package_index.save()
release['Architectures'] = ', '.join(package_index_files.keys())
release_path = os.path.join('dists', 'default', 'Release')
os.makedirs(os.path.dirname(release_path), exist_ok=True)
with open(release_path, 'wb') as release_file:
release.dump(release_file)
release_metadata = PublishedMetadata(
relative_path=release_path,
publication=publication,
file=File(open(release_path, 'rb')),
)
release_metadata.save()
if publisher.structured:
raise NotImplementedError(
"Structured publishing is not yet implemented.")
log.info(_('Publication: {publication} created').format(publication=publication.pk))