def get_buildroot(self, worker_metadatas):
"""
Build the buildroot entry of the metadata.
:return: list, containing dicts of partial metadata
"""
buildroots = []
buildroot = koji_get_buildroot(build_id=self.build_id, tasker=self.tasker,
osbs=self.osbs, rpms=False)
buildroot['id'] = '{}-{}'.format(buildroot['container']['arch'], buildroot['id'])
registry = self.workflow.push_conf.docker_registries[0]
build_name = get_unique_images(self.workflow)[0].to_str()
manifest_digest = registry.digests[build_name]
digest_version = get_manifest_media_version(manifest_digest)
media_type = get_manifest_media_type(digest_version)
buildroot['extra']['osbs']['koji'] = {
'build_name': build_name,
'builder_image_id': {media_type: manifest_digest.default}
}
buildroots.append(buildroot)
return buildroots
def get_output(self, worker_metadatas, buildroot_id):
"""
Build the output entry of the metadata.
:return: list, containing dicts of partial metadata
"""
outputs = []
output_file = None
if self.source_build:
registry = self.workflow.push_conf.docker_registries[0]
build_name = get_unique_images(self.workflow)[0]
pullspec = copy.deepcopy(build_name)
pullspec.registry = registry.uri
outputs, output_file = get_output(workflow=self.workflow, buildroot_id=buildroot_id,
pullspec=pullspec, platform=os.uname()[4],
source_build=True, logs=None)
else:
for platform in worker_metadatas:
for instance in worker_metadatas[platform]['output']:
instance['buildroot_id'] = '{}-{}'.format(platform, instance['buildroot_id'])
outputs.append(instance)
return outputs, output_file
def add_floating_tags(self, session, manifest_list_data, floating_images):
list_type = manifest_list_data.get("media_type")
manifest = manifest_list_data.get("manifest")
manifest_digest = manifest_list_data.get("manifest_digest")
for image in floating_images:
target_repo = image.to_str(registry=False, tag=False)
# We have to call store_manifest_in_repository directly for each
# referenced manifest, since each one should be a new tag that requires uploading
# the manifest again
self.log.debug("storing %s as %s", target_repo, image.tag)
self.manifest_util.store_manifest_in_repository(session,
manifest,
list_type,
target_repo,
target_repo,
ref=image.tag)
# And store the manifest list in the push_conf
push_conf_registry = self.workflow.push_conf.add_docker_registry(
session.registry, insecure=session.insecure)
for image in floating_images:
push_conf_registry.digests[image.tag] = manifest_digest
registry_image = get_unique_images(self.workflow)[0]
return registry_image.get_repo(
explicit_namespace=False), manifest_digest
def get_output(self, buildroot_id):
pullspec = get_unique_images(self.workflow)[0]
return koji_get_output(workflow=self.workflow,
buildroot_id=buildroot_id,
pullspec=pullspec,
platform=os.uname()[4],
source_build=True)
def get_output(self, worker_metadatas, buildroot_id):
registry = self.workflow.push_conf.docker_registries[0]
build_name = get_unique_images(self.workflow)[0]
pullspec = copy.deepcopy(build_name)
pullspec.registry = registry.uri
return koji_get_output(workflow=self.workflow, buildroot_id=buildroot_id,
pullspec=pullspec, platform=os.uname()[4],
source_build=True, logs=None)
def get_output(self, buildroot_id: str) -> List[Dict[str, Any]]:
outputs = super().get_output(buildroot_id)
pullspec = get_unique_images(self.workflow)[0]
metadatas, output_file = koji_get_output(
workflow=self.workflow,
buildroot_id=buildroot_id,
pullspec=pullspec,
platform=os.uname()[4],
source_build=True,
)
self.workflow.data.koji_upload_files.append({
"local_filename":
output_file.filename,
"dest_filename":
output_file.metadata['filename'],
})
outputs.extend(metadatas)
return outputs
def run(self):
primary_images = get_primary_images(self.workflow)
unique_images = get_unique_images(self.workflow)
self.non_floating_images = primary_images + unique_images
for registry, source in self.sort_annotations().items():
session = self.manifest_util.get_registry_session(registry)
if self.group:
return self.group_manifests_and_tag(session, source)
else:
if len(source) != 1:
raise RuntimeError('Without grouping only one source is expected')
# source.values() isn't a list and can't be indexed, so this clumsy workaround
_, orig_digest = source.popitem()
source_digest = orig_digest['digest']
source_repo = orig_digest['repository']
return self.tag_manifest_into_registry(session, source_digest, source_repo,
self.non_floating_images)
def run(self):
primary_images = get_primary_images(self.workflow)
unique_images = get_unique_images(self.workflow)
self.non_floating_images = primary_images + unique_images
session = self.manifest_util.get_registry_session()
built_images = self.get_built_images(session)
if self.group:
return self.group_manifests_and_tag(session, built_images)
else:
if len(built_images) != 1:
raise RuntimeError(
'Without grouping only one built image is expected')
built_image = built_images[0]
source_digest = built_image.manifest_digest
source_repo = built_image.repository
return self.tag_manifest_into_registry(session, source_digest,
source_repo,
self.non_floating_images)
def group_manifests_and_tag(self, session, worker_digests):
"""
Creates a manifest list or OCI image index that groups the different manifests
in worker_digests, then tags the result with with all the configured tags found
in workflow.tag_conf.
"""
self.log.info("%s: Creating manifest list", session.registry)
# Extract information about the manifests that we will group - we get the
# size and content type of the manifest by querying the registry
manifests = []
for platform, worker_image in worker_digests.items():
repository = worker_image['repository']
digest = worker_image['digest']
media_type = get_manifest_media_type(worker_image['version'])
if media_type not in self.manifest_util.manifest_media_types:
continue
content, _, media_type, size = self.manifest_util.get_manifest(
session, repository, digest)
manifests.append({
'content':
content,
'repository':
repository,
'digest':
digest,
'size':
size,
'media_type':
media_type,
'architecture':
self.goarch.get(platform, platform),
})
list_type, list_json = self.manifest_util.build_list(manifests)
self.log.info("%s: Created manifest, Content-Type=%s\n%s",
session.registry, list_type, list_json)
# Now push the manifest list to the registry once per each tag
self.log.info("%s: Tagging manifest list", session.registry)
for image in self.non_floating_images:
target_repo = image.to_str(registry=False, tag=False)
# We have to call store_manifest_in_repository directly for each
# referenced manifest, since they potentially come from different repos
for manifest in manifests:
self.manifest_util.store_manifest_in_repository(
session,
manifest['content'],
manifest['media_type'],
manifest['repository'],
target_repo,
ref=manifest['digest'])
self.manifest_util.store_manifest_in_repository(session,
list_json,
list_type,
target_repo,
target_repo,
ref=image.tag)
# Get the digest of the manifest list using one of the tags
registry_image = get_unique_images(self.workflow)[0]
_, digest_str, _, _ = self.manifest_util.get_manifest(
session, registry_image.to_str(registry=False, tag=False),
registry_image.tag)
if list_type == MEDIA_TYPE_OCI_V1_INDEX:
digest = ManifestDigest(oci_index=digest_str)
else:
digest = ManifestDigest(v2_list=digest_str)
# And store the manifest list in the push_conf
push_conf_registry = self.workflow.push_conf.add_docker_registry(
session.registry, insecure=session.insecure)
tags = []
for image in self.non_floating_images:
push_conf_registry.digests[image.tag] = digest
tags.append(image.tag)
self.log.info("%s: Manifest list digest is %s", session.registry,
digest_str)
self.log.debug("tags: %s digest: %s", tags, digest)
return {
'manifest': list_json,
'media_type': list_type,
'manifest_digest': digest
}
def set_group_manifest_info(self, extra, worker_metadatas):
version_release = None
primary_images = get_primary_images(self.workflow)
floating_images = get_floating_images(self.workflow)
unique_images = get_unique_images(self.workflow)
if primary_images:
version_release = primary_images[0].tag
if is_scratch_build(self.workflow):
tags = [image.tag for image in self.workflow.tag_conf.images]
version_release = tags[0]
else:
assert version_release is not None, 'Unable to find version-release image'
tags = [image.tag for image in primary_images]
floating_tags = [image.tag for image in floating_images]
unique_tags = [image.tag for image in unique_images]
manifest_data = self.workflow.postbuild_results.get(PLUGIN_GROUP_MANIFESTS_KEY, {})
if manifest_data and is_manifest_list(manifest_data.get("media_type")):
manifest_digest = manifest_data.get("manifest_digest")
index = {}
index['tags'] = tags
index['floating_tags'] = floating_tags
index['unique_tags'] = unique_tags
build_image = get_unique_images(self.workflow)[0]
repo = ImageName.parse(build_image).to_str(registry=False, tag=False)
# group_manifests added the registry, so this should be valid
registries = self.workflow.push_conf.all_registries
digest_version = get_manifest_media_version(manifest_digest)
digest = manifest_digest.default
for registry in registries:
pullspec = "{0}/{1}@{2}".format(registry.uri, repo, digest)
index['pull'] = [pullspec]
pullspec = "{0}/{1}:{2}".format(registry.uri, repo,
version_release)
index['pull'].append(pullspec)
# Store each digest with according media type
index['digests'] = {}
media_type = get_manifest_media_type(digest_version)
index['digests'][media_type] = digest
break
extra['image']['index'] = index
# group_manifests returns None if didn't run, {} if group=False
else:
for platform in worker_metadatas:
if platform == "x86_64":
for instance in worker_metadatas[platform]['output']:
if instance['type'] == 'docker-image':
# koji_upload, running in the worker, doesn't have the full tags
# so set them here
instance['extra']['docker']['tags'] = tags
instance['extra']['docker']['floating_tags'] = floating_tags
instance['extra']['docker']['unique_tags'] = unique_tags
repositories = []
for pullspec in instance['extra']['docker']['repositories']:
if '@' not in pullspec:
image = ImageName.parse(pullspec)
image.tag = version_release
pullspec = image.to_str()
repositories.append(pullspec)
instance['extra']['docker']['repositories'] = repositories
self.log.debug("reset tags to so that docker is %s",
instance['extra']['docker'])
annotations = get_worker_build_info(self.workflow, platform).\
build.get_annotations()
digests = {}
if 'digests' in annotations:
digests = get_digests_map_from_annotations(annotations['digests'])
instance['extra']['docker']['digests'] = digests
def set_group_manifest_info(self, extra):
version_release = None
primary_images = get_primary_images(self.workflow)
if primary_images:
version_release = primary_images[0].tag
if is_scratch_build(self.workflow):
tags = [image.tag for image in self.workflow.data.tag_conf.images]
version_release = tags[0]
else:
assert version_release is not None, 'Unable to find version-release image'
tags = [image.tag for image in primary_images]
floating_tags = [
image.tag for image in get_floating_images(self.workflow)
]
unique_images = get_unique_images(self.workflow)
unique_tags = [image.tag for image in unique_images]
manifest_data = self.workflow.data.postbuild_results.get(
PLUGIN_GROUP_MANIFESTS_KEY, {})
if manifest_data and is_manifest_list(manifest_data.get("media_type")):
manifest_digest = manifest_data["manifest_digest"]
digest = manifest_digest.default
build_image = unique_images[0]
repo = ImageName.parse(build_image).to_str(registry=False,
tag=False)
# group_manifests added the registry, so this should be valid
registry_uri = self.workflow.conf.registry['uri']
digest_version = get_manifest_media_version(manifest_digest)
media_type = get_manifest_media_type(digest_version)
extra['image']['index'] = {
'tags':
tags,
'floating_tags':
floating_tags,
'unique_tags':
unique_tags,
'pull': [
f'{registry_uri}/{repo}@{digest}',
f'{registry_uri}/{repo}:{version_release}',
],
'digests': {
media_type: digest
},
}
# group_manifests returns None if didn't run, {} if group=False
else:
platform = "x86_64"
_, instance = next(
self._iter_build_metadata_outputs(platform,
{"type": "docker-image"}),
(None, None),
)
if instance:
# koji_upload, running in the worker, doesn't have the full tags
# so set them here
instance['extra']['docker']['tags'] = tags
instance['extra']['docker']['floating_tags'] = floating_tags
instance['extra']['docker']['unique_tags'] = unique_tags
repositories = []
for pullspec in instance['extra']['docker']['repositories']:
if '@' not in pullspec:
image = ImageName.parse(pullspec)
image.tag = version_release
pullspec = image.to_str()
repositories.append(pullspec)
instance['extra']['docker']['repositories'] = repositories
self.log.debug("reset tags to so that docker is %s",
instance['extra']['docker'])
