def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = FilePublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(pk=repository_version_pk)
log.info(_('Publishing: repository={repo}, version={ver}, publisher={pub}').format(
repo=repository_version.repository.name,
ver=repository_version.number,
pub=publisher.name,
))
with WorkingDirectory():
with Publication.create(repository_version, publisher, pass_through=True) as publication:
manifest = Manifest(publisher.manifest)
manifest.write(populate(publication))
metadata = PublishedMetadata(
relative_path=os.path.basename(manifest.relative_path),
publication=publication,
file=File(open(manifest.relative_path, 'rb')))
metadata.save()
log.info(_('Publication: {publication} created').format(publication=publication.pk))
def publish_verbatim(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a verbatim Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = DebVerbatimPublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_('Publishing (verbatim): repository={repo}, version={ver}, publisher={pub}'
).format(
repo=repository_version.repository.name,
ver=repository_version.number,
pub=publisher.name,
))
with WorkingDirectory():
with Publication.create(repository_version,
publisher,
pass_through=True) as publication:
pass
log.info(
_('Publication (verbatim): {publication} created').format(
publication=publication.pk))
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = FilePublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(pk=repository_version_pk)
log.info(
_('Publishing: repository=%(repository)s, version=%(version)d, publisher=%(publisher)s'),
{
'repository': repository_version.repository.name,
'version': repository_version.number,
'publisher': publisher.name,
})
with WorkingDirectory():
with Publication.create(repository_version, publisher) as publication:
manifest = Manifest('PULP_MANIFEST')
manifest.write(populate(publication))
metadata = PublishedMetadata(
relative_path=os.path.basename(manifest.relative_path),
publication=publication,
file=File(open(manifest.relative_path, 'rb')))
metadata.save()
log.info(
_('Publication: %(publication)s created'),
{
'publication': publication.pk
})
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = AnsiblePublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(pk=repository_version_pk)
log.info(
_('Publishing: repository=%(repository)s, version=%(version)d, publisher=%(publisher)s'),
{
'repository': repository_version.repository.name,
'version': repository_version.number,
'publisher': publisher.name,
})
with WorkingDirectory():
with Publication.create(repository_version, publisher) as publication:
populate(publication)
log.info(
_('Publication: %(publication)s created'),
{
'publication': publication.pk
})
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = GemPublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_('Publishing: repository=%(repository)s, version=%(version)d, publisher=%(publisher)s'
), {
'repository': repository_version.repository.name,
'version': repository_version.number,
'publisher': publisher.name,
})
with WorkingDirectory():
with Publication.create(repository_version, publisher) as publication:
specs = []
latest_versions = {}
prerelease_specs = []
for content in GemContent.objects.filter(
pk__in=publication.repository_version.content).order_by(
'-created'):
for content_artifact in content.contentartifact_set.all():
published_artifact = PublishedArtifact(
relative_path=content_artifact.relative_path,
publication=publication,
content_artifact=content_artifact)
published_artifact.save()
if re.fullmatch(r"[0-9.]*", content.version):
specs.append(Key(content.name, content.version))
old_ver = latest_versions.get(content.name)
if old_ver is None or version.parse(
old_ver) < version.parse(content.version):
latest_versions[content.name] = content.version
else:
prerelease_specs.append(Key(content.name, content.version))
latest_specs = [
Key(name, version)
for name, version in latest_versions.items()
]
_publish_specs(specs, 'specs.4.8', publication)
_publish_specs(latest_specs, 'latest_specs.4.8', publication)
_publish_specs(prerelease_specs, 'prerelease_specs.4.8',
publication)
log.info(_('Publication: %(publication)s created'),
{'publication': publication.pk})
def publish(repository_version_pk):
"""
Create a Publication based on a RepositoryVersion.
Args:
repository_version_pk (str): Create a publication from this repository version.
"""
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
with Publication.create(repository_version,
pass_through=True) as publication:
pass
log.info(
_('Publication: {publication} created').format(
publication=publication.pk))
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = PuppetPublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_('Publishing: repository=%(repository)s, version=%(version)d, publisher=%(publisher)s'
), {
'repository': repository_version.repository.name,
'version': repository_version.number,
'publisher': publisher.name,
})
with WorkingDirectory():
with Publication.create(repository_version, publisher) as publication:
# Write any Artifacts (files) to the file system and the database.
# each.
# artifact = YourArtifactWriter.write(relative_path)
# published_artifact = PublishedArtifact(
# relative_path=artifact.relative_path,
# publication=publication,
# content_artifact=artifact)
# published_artifact.save()
# Write any metadata files to the file system, and the database.
# metadata = YourMetadataWriter.write(relative_path)
# metadata = PublishedMetadata(
# relative_path=os.path.basename(manifest.relative_path),
# publication=publication,
# file=File(open(manifest.relative_path, 'rb')))
# metadata.save()
pass
log.info(_('Publication: %(publication)s created'),
{'publication': publication.pk})
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = PluginTemplatePublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_('Publishing: repository={repo}, version={ver}, publisher={pub}').
format(repo=repository_version.repository.name,
ver=repository_version.number,
pub=publisher.name))
with WorkingDirectory():
with Publication.create(repository_version, publisher) as publication:
# Write any Artifacts (files) to the file system, and the database.
#
# artifact = YourArtifactWriter.write(relative_path)
# published_artifact = PublishedArtifact(
# relative_path=artifact.relative_path,
# publication=publication,
# content_artifact=artifact)
# published_artifact.save()
# Write any metadata files to the file system, and the database.
#
# metadata = YourMetadataWriter.write(relative_path)
# metadata = PublishedMetadata(
# relative_path=os.path.basename(manifest.relative_path),
# publication=publication,
# file=File(open(manifest.relative_path, 'rb')))
# metadata.save()
pass
log.info(
_('Publication: {publication} created').format(
publication=publication.pk))
def publish_verbatim(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a verbatim Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = DebVerbatimPublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(_('Publishing (verbatim): repository={repo}, version={ver}, publisher={pub}').format(
repo=repository_version.repository.name,
ver=repository_version.number,
pub=publisher.name,
))
with WorkingDirectory():
with Publication.create(repository_version, publisher, pass_through=True) as publication:
pass
log.info(_('Publication (verbatim): {publication} created').format(publication=publication.pk))
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = CookbookPublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_("Publishing: repository=%(repository)s, version=%(version)d, publisher=%(publisher)s"
),
{
"repository": repository_version.repository.name,
"version": repository_version.number,
"publisher": publisher.name,
},
)
with WorkingDirectory():
with Publication.create(repository_version, publisher) as publication:
check_repo_version_constraint(publication)
universe = Universe("__universe__")
universe.write(populate(publication))
metadata = PublishedMetadata(
relative_path=os.path.basename(universe.relative_path),
publication=publication,
file=File(open(universe.relative_path, "rb")),
)
metadata.save()
log.info(_("Publication: %(publication)s created"),
{"publication": publication.pk})
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = RpmPublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(pk=repository_version_pk)
log.info(_('Publishing: repository={repo}, version={version}, publisher={publisher}').format(
repo=repository_version.repository.name,
version=repository_version.number,
publisher=publisher.name,
))
with WorkingDirectory():
with Publication.create(repository_version, publisher) as publication:
populate(publication)
# Prepare metadata files
repomd_path = os.path.join(os.getcwd(), "repomd.xml")
pri_xml_path = os.path.join(os.getcwd(), "primary.xml.gz")
fil_xml_path = os.path.join(os.getcwd(), "filelists.xml.gz")
oth_xml_path = os.path.join(os.getcwd(), "other.xml.gz")
pri_db_path = os.path.join(os.getcwd(), "primary.sqlite")
fil_db_path = os.path.join(os.getcwd(), "filelists.sqlite")
oth_db_path = os.path.join(os.getcwd(), "other.sqlite")
upd_xml_path = os.path.join(os.getcwd(), "updateinfo.xml.gz")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
artifacts = publication.published_artifact.all()
pri_xml.set_num_of_pkgs(len(artifacts))
fil_xml.set_num_of_pkgs(len(artifacts))
oth_xml.set_num_of_pkgs(len(artifacts))
# Process all packages
for artifact in artifacts:
# TODO: pass attributes from db rather than use the filesystem
pkg = cr.package_from_rpm(artifact.content_artifact.artifact.file.path)
pkg.location_href = artifact.content_artifact.relative_path
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=publication.repository_version.content):
upd_xml.add_chunk(update_record_xml(update_record))
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
repomdrecords = (("primary", pri_xml_path, pri_db),
("filelists", fil_xml_path, fil_db),
("other", oth_xml_path, oth_db),
("primary_db", pri_db_path, None),
("filelists_db", fil_db_path, None),
("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None))
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
record.fill(cr.SHA256)
if (db_to_update):
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
repomd.set_record(record)
metadata = PublishedMetadata(
relative_path=os.path.join(REPODATA_PATH, os.path.basename(path)),
publication=publication,
file=File(open(os.path.basename(path), 'rb'))
)
metadata.save()
open(repomd_path, "w").write(repomd.xml_dump())
metadata = PublishedMetadata(
relative_path=os.path.join(REPODATA_PATH, os.path.basename(repomd_path)),
publication=publication,
file=File(open(os.path.basename(repomd_path), 'rb'))
)
metadata.save()
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = DebPublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_('Publishing: repository={repo}, version={ver}, publisher={pub}').
format(repo=repository_version.repository.name,
ver=repository_version.number,
pub=publisher.name))
with WorkingDirectory():
with Publication.create(repository_version,
publisher,
pass_through=False) as publication:
if publisher.simple:
repository = repository_version.repository
release = deb822.Release()
# TODO: release['Label']
release['Codename'] = 'default'
release['Components'] = 'all'
release['Architectures'] = ''
if repository.description:
release['Description'] = repository.description
release['MD5sum'] = []
release['SHA1'] = []
release['SHA256'] = []
release['SHA512'] = []
package_index_files = {}
for package in Package.objects.filter(
pk__in=repository_version.content.order_by(
'-_created')):
published_artifact = PublishedArtifact(
relative_path=package.filename(),
publication=publication,
content_artifact=package.contentartifact_set.get(),
)
published_artifact.save()
if package.architecture not in package_index_files:
package_index_path = os.path.join(
'dists',
'default',
'all',
'binary-{}'.format(package.architecture),
'Packages',
)
os.makedirs(os.path.dirname(package_index_path),
exist_ok=True)
package_index_files[package.architecture] = (open(
package_index_path, 'wb'), package_index_path)
package.to822('all').dump(
package_index_files[package.architecture][0])
package_index_files[package.architecture][0].write(b'\n')
for package_index_file, package_index_path in package_index_files.values(
):
package_index_file.close()
gz_package_index_path = _zip_file(package_index_path)
_add_to_release(release, package_index_path)
_add_to_release(release, gz_package_index_path)
package_index = PublishedMetadata(
relative_path=package_index_path,
publication=publication,
file=File(open(package_index_path, 'rb')),
)
package_index.save()
gz_package_index = PublishedMetadata(
relative_path=gz_package_index_path,
publication=publication,
file=File(open(gz_package_index_path, 'rb')),
)
gz_package_index.save()
release['Architectures'] = ', '.join(
package_index_files.keys())
release_path = os.path.join('dists', 'default', 'Release')
os.makedirs(os.path.dirname(release_path), exist_ok=True)
with open(release_path, 'wb') as release_file:
release.dump(release_file)
release_metadata = PublishedMetadata(
relative_path=release_path,
publication=publication,
file=File(open(release_path, 'rb')),
)
release_metadata.save()
if publisher.structured:
raise NotImplementedError(
"Structured publishing is not yet implemented.")
log.info(
_('Publication: {publication} created').format(
publication=publication.pk))
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = DebPublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(pk=repository_version_pk)
log.info(_('Publishing: repository={repo}, version={ver}, publisher={pub}').format(
repo=repository_version.repository.name,
ver=repository_version.number,
pub=publisher.name
))
with WorkingDirectory():
with Publication.create(repository_version, publisher, pass_through=False) as publication:
if publisher.simple:
repository = repository_version.repository
release = deb822.Release()
# TODO: release['Label']
release['Codename'] = 'default'
release['Components'] = 'all'
release['Architectures'] = ''
if repository.description:
release['Description'] = repository.description
release['MD5sum'] = []
release['SHA1'] = []
release['SHA256'] = []
release['SHA512'] = []
package_index_files = {}
for package in Package.objects.filter(
pk__in=repository_version.content.order_by('-_created')
):
published_artifact = PublishedArtifact(
relative_path=package.filename(),
publication=publication,
content_artifact=package.contentartifact_set.get(),
)
published_artifact.save()
if package.architecture not in package_index_files:
package_index_path = os.path.join(
'dists',
'default',
'all',
'binary-{}'.format(package.architecture),
'Packages',
)
os.makedirs(os.path.dirname(
package_index_path), exist_ok=True)
package_index_files[package.architecture] = (
open(package_index_path, 'wb'), package_index_path)
package.to822('all').dump(
package_index_files[package.architecture][0])
package_index_files[package.architecture][0].write(b'\n')
for package_index_file, package_index_path in package_index_files.values():
package_index_file.close()
gz_package_index_path = _zip_file(package_index_path)
_add_to_release(release, package_index_path)
_add_to_release(release, gz_package_index_path)
package_index = PublishedMetadata(
relative_path=package_index_path,
publication=publication,
file=File(open(package_index_path, 'rb')),
)
package_index.save()
gz_package_index = PublishedMetadata(
relative_path=gz_package_index_path,
publication=publication,
file=File(open(gz_package_index_path, 'rb')),
)
gz_package_index.save()
release['Architectures'] = ', '.join(package_index_files.keys())
release_path = os.path.join('dists', 'default', 'Release')
os.makedirs(os.path.dirname(release_path), exist_ok=True)
with open(release_path, 'wb') as release_file:
release.dump(release_file)
release_metadata = PublishedMetadata(
relative_path=release_path,
publication=publication,
file=File(open(release_path, 'rb')),
)
release_metadata.save()
if publisher.structured:
raise NotImplementedError(
"Structured publishing is not yet implemented.")
log.info(_('Publication: {publication} created').format(publication=publication.pk))
def publish(publisher_pk, repository_version_pk):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Args:
publisher_pk (str): Use the publish settings provided by this publisher.
repository_version_pk (str): Create a publication from this repository version.
"""
publisher = RpmPublisher.objects.get(pk=publisher_pk)
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_('Publishing: repository={repo}, version={version}, publisher={publisher}'
).format(
repo=repository_version.repository.name,
version=repository_version.number,
publisher=publisher.name,
))
with WorkingDirectory():
with Publication.create(repository_version, publisher) as publication:
populate(publication)
# Prepare metadata files
repomd_path = os.path.join(os.getcwd(), "repomd.xml")
pri_xml_path = os.path.join(os.getcwd(), "primary.xml.gz")
fil_xml_path = os.path.join(os.getcwd(), "filelists.xml.gz")
oth_xml_path = os.path.join(os.getcwd(), "other.xml.gz")
pri_db_path = os.path.join(os.getcwd(), "primary.sqlite")
fil_db_path = os.path.join(os.getcwd(), "filelists.sqlite")
oth_db_path = os.path.join(os.getcwd(), "other.sqlite")
upd_xml_path = os.path.join(os.getcwd(), "updateinfo.xml.gz")
pri_xml = cr.PrimaryXmlFile(pri_xml_path)
fil_xml = cr.FilelistsXmlFile(fil_xml_path)
oth_xml = cr.OtherXmlFile(oth_xml_path)
pri_db = cr.PrimarySqlite(pri_db_path)
fil_db = cr.FilelistsSqlite(fil_db_path)
oth_db = cr.OtherSqlite(oth_db_path)
upd_xml = cr.UpdateInfoXmlFile(upd_xml_path)
artifacts = publication.published_artifact.all()
pri_xml.set_num_of_pkgs(len(artifacts))
fil_xml.set_num_of_pkgs(len(artifacts))
oth_xml.set_num_of_pkgs(len(artifacts))
# Process all packages
for artifact in artifacts:
# TODO: pass attributes from db rather than use the filesystem
pkg = cr.package_from_rpm(
artifact.content_artifact.artifact.file.path)
pkg.location_href = artifact.content_artifact.relative_path
pri_xml.add_pkg(pkg)
fil_xml.add_pkg(pkg)
oth_xml.add_pkg(pkg)
pri_db.add_pkg(pkg)
fil_db.add_pkg(pkg)
oth_db.add_pkg(pkg)
# Process update records
for update_record in UpdateRecord.objects.filter(
pk__in=publication.repository_version.content):
upd_xml.add_chunk(update_record_xml(update_record))
pri_xml.close()
fil_xml.close()
oth_xml.close()
upd_xml.close()
repomd = cr.Repomd()
repomdrecords = (("primary", pri_xml_path,
pri_db), ("filelists", fil_xml_path, fil_db),
("other", oth_xml_path,
oth_db), ("primary_db", pri_db_path, None),
("filelists_db", fil_db_path,
None), ("other_db", oth_db_path, None),
("updateinfo", upd_xml_path, None))
for name, path, db_to_update in repomdrecords:
record = cr.RepomdRecord(name, path)
record.fill(cr.SHA256)
if (db_to_update):
db_to_update.dbinfo_update(record.checksum)
db_to_update.close()
repomd.set_record(record)
metadata = PublishedMetadata(
relative_path=os.path.join(REPODATA_PATH,
os.path.basename(path)),
publication=publication,
file=File(open(os.path.basename(path), 'rb')))
metadata.save()
open(repomd_path, "w").write(repomd.xml_dump())
metadata = PublishedMetadata(
relative_path=os.path.join(REPODATA_PATH,
os.path.basename(repomd_path)),
publication=publication,
file=File(open(os.path.basename(repomd_path), 'rb')))
metadata.save()