class GroupManifestsPlugin(PostBuildPlugin):
is_allowed_to_fail = False
key = PLUGIN_GROUP_MANIFESTS_KEY
def __init__(self,
tasker,
workflow,
registries=None,
group=True,
goarch=None):
"""
constructor
:param tasker: ContainerTasker instance
:param workflow: DockerBuildWorkflow instance
:param registries: dict, keys are docker registries, values are dicts containing
per-registry parameters.
Params:
* "secret" optional string - path to the secret, which stores
login and password for remote registry
:param group: bool, if true, create a manifest list; otherwise only add tags to
amd64 image manifest
:param goarch: dict, keys are platform, values are go language platform names
"""
# call parent constructor
super(GroupManifestsPlugin, self).__init__(tasker, workflow)
self.group = get_group_manifests(self.workflow, group)
plat_des_fallback = []
for platform, architecture in (goarch or {}).items():
plat_dic = {'platform': platform, 'architecture': architecture}
plat_des_fallback.append(plat_dic)
platform_descriptors = get_platform_descriptors(
self.workflow, plat_des_fallback)
goarch_from_pd = {}
for platform in platform_descriptors:
goarch_from_pd[platform['platform']] = platform['architecture']
self.goarch = goarch_from_pd
self.manifest_util = ManifestUtil(self.workflow, registries, self.log)
self.non_floating_images = None
def group_manifests_and_tag(self, session, worker_digests):
"""
Creates a manifest list or OCI image index that groups the different manifests
in worker_digests, then tags the result with with all the configured tags found
in workflow.tag_conf.
"""
self.log.info("%s: Creating manifest list", session.registry)
# Extract information about the manifests that we will group - we get the
# size and content type of the manifest by querying the registry
manifests = []
for platform, worker_image in worker_digests.items():
repository = worker_image['repository']
digest = worker_image['digest']
media_type = get_manifest_media_type(worker_image['version'])
if media_type not in self.manifest_util.manifest_media_types:
continue
content, _, media_type, size = self.manifest_util.get_manifest(
session, repository, digest)
manifests.append({
'content':
content,
'repository':
repository,
'digest':
digest,
'size':
size,
'media_type':
media_type,
'architecture':
self.goarch.get(platform, platform),
})
list_type, list_json = self.manifest_util.build_list(manifests)
self.log.info("%s: Created manifest, Content-Type=%s\n%s",
session.registry, list_type, list_json)
# Now push the manifest list to the registry once per each tag
self.log.info("%s: Tagging manifest list", session.registry)
for image in self.non_floating_images:
target_repo = image.to_str(registry=False, tag=False)
# We have to call store_manifest_in_repository directly for each
# referenced manifest, since they potentially come from different repos
for manifest in manifests:
self.manifest_util.store_manifest_in_repository(
session,
manifest['content'],
manifest['media_type'],
manifest['repository'],
target_repo,
ref=manifest['digest'])
self.manifest_util.store_manifest_in_repository(session,
list_json,
list_type,
target_repo,
target_repo,
ref=image.tag)
# Get the digest of the manifest list using one of the tags
registry_image = get_unique_images(self.workflow)[0]
_, digest_str, _, _ = self.manifest_util.get_manifest(
session, registry_image.to_str(registry=False, tag=False),
registry_image.tag)
if list_type == MEDIA_TYPE_OCI_V1_INDEX:
digest = ManifestDigest(oci_index=digest_str)
else:
digest = ManifestDigest(v2_list=digest_str)
# And store the manifest list in the push_conf
push_conf_registry = self.workflow.push_conf.add_docker_registry(
session.registry, insecure=session.insecure)
tags = []
for image in self.non_floating_images:
push_conf_registry.digests[image.tag] = digest
tags.append(image.tag)
self.log.info("%s: Manifest list digest is %s", session.registry,
digest_str)
self.log.debug("tags: %s digest: %s", tags, digest)
return {
'manifest': list_json,
'media_type': list_type,
'manifest_digest': digest
}
def sort_annotations(self):
"""
Return a map of maps to look up a single "worker digest" that has information
about where to find an image manifest for each registry/architecture combination:
worker_digest = <result>[registry][architecture]
"""
all_annotations = self.workflow.build_result.annotations[
'worker-builds']
all_platforms = set(all_annotations)
if len(all_platforms) == 0:
raise RuntimeError("No worker builds found, cannot group them")
return self.manifest_util.sort_annotations(all_annotations)
def tag_manifest_into_registry(self, session, source_digest, source_repo,
images):
manifest, media, digest = self.manifest_util.tag_manifest_into_registry(
session, source_digest, source_repo, images)
return {
'manifest': manifest,
'media_type': media,
'manifest_digest': digest
}
def run(self):
primary_images = get_primary_images(self.workflow)
unique_images = get_unique_images(self.workflow)
self.non_floating_images = primary_images + unique_images
for registry, source in self.sort_annotations().items():
session = self.manifest_util.get_registry_session(registry)
if self.group:
return self.group_manifests_and_tag(session, source)
else:
if len(source) != 1:
raise RuntimeError(
'Without grouping only one source is expected')
# source.values() isn't a list and can't be indexed, so this clumsy workaround
_, orig_digest = source.popitem()
source_digest = orig_digest['digest']
source_repo = orig_digest['repository']
return self.tag_manifest_into_registry(
session, source_digest, source_repo,
self.non_floating_images)
class GroupManifestsPlugin(PostBuildPlugin):
is_allowed_to_fail = False
key = PLUGIN_GROUP_MANIFESTS_KEY
def __init__(self, workflow):
"""
constructor
:param workflow: DockerBuildWorkflow instance
"""
# call parent constructor
super(GroupManifestsPlugin, self).__init__(workflow)
self.group = self.workflow.conf.group_manifests
self.goarch = self.workflow.conf.platform_to_goarch_mapping
self.manifest_util = ManifestUtil(self.workflow, self.log)
self.non_floating_images = None
def get_built_images(self, session: RegistrySession) -> List[BuiltImage]:
"""Get information about all the per-arch images that were built by the build tasks."""
tag_conf = self.workflow.data.tag_conf
client = RegistryClient(session)
built_images = []
for platform in get_platforms(self.workflow.data):
# At this point, only the unique image has been built and pushed. Primary tags will
# be pushed by this plugin, floating tags by the push_floating_tags plugin.
image = tag_conf.get_unique_images_with_platform(platform)[0]
manifest_digests = client.get_manifest_digests(image,
versions=("v2",
"oci"))
if len(manifest_digests) != 1:
raise RuntimeError(
f"Expected to find a single manifest digest for {image}, "
f"but found multiple: {manifest_digests}")
manifest_version, manifest_digest = manifest_digests.popitem()
built_images.append(
BuiltImage(image, platform, manifest_digest, manifest_version))
return built_images
def group_manifests_and_tag(
self, session: RegistrySession, built_images: List[BuiltImage]
) -> Dict[str, Union[str, ManifestDigest]]:
"""
Creates a manifest list or OCI image index that groups the different manifests
in built_images, then tags the result with all the configured tags found
in workflow.data.tag_conf.
"""
self.log.info("%s: Creating manifest list", session.registry)
# Extract information about the manifests that we will group - we get the
# size and content type of the manifest by querying the registry
manifests = []
for built_image in built_images:
repository = built_image.repository
manifest_digest = built_image.manifest_digest
media_type = get_manifest_media_type(built_image.manifest_version)
if media_type not in self.manifest_util.manifest_media_types:
continue
content, _, media_type, size = self.manifest_util.get_manifest(
session, repository, manifest_digest)
manifests.append({
'content': content,
'repository': repository,
'digest': manifest_digest,
'size': size,
'media_type': media_type,
'architecture': self.goarch[built_image.platform],
})
list_type, list_json = self.manifest_util.build_list(manifests)
self.log.info("%s: Created manifest, Content-Type=%s\n%s",
session.registry, list_type, list_json)
# Now push the manifest list to the registry once per each tag
self.log.info("%s: Tagging manifest list", session.registry)
for image in self.non_floating_images:
target_repo = image.to_str(registry=False, tag=False)
# We have to call store_manifest_in_repository directly for each
# referenced manifest, since they potentially come from different repos
for manifest in manifests:
self.manifest_util.store_manifest_in_repository(
session,
manifest['content'],
manifest['media_type'],
manifest['repository'],
target_repo,
ref=manifest['digest'])
self.manifest_util.store_manifest_in_repository(session,
list_json,
list_type,
target_repo,
target_repo,
ref=image.tag)
# Get the digest of the manifest list using one of the tags
registry_image = get_unique_images(self.workflow)[0]
_, digest_str, _, _ = self.manifest_util.get_manifest(
session, registry_image.to_str(registry=False, tag=False),
registry_image.tag)
if list_type == MEDIA_TYPE_OCI_V1_INDEX:
digest = ManifestDigest(oci_index=digest_str)
else:
digest = ManifestDigest(v2_list=digest_str)
tags = []
for image in self.non_floating_images:
tags.append(image.tag)
self.log.info("%s: Manifest list digest is %s", session.registry,
digest_str)
self.log.debug("tags: %s digest: %s", tags, digest)
return {
'manifest': list_json,
'media_type': list_type,
'manifest_digest': digest
}
def tag_manifest_into_registry(self, session, source_digest: str,
source_repo, images):
manifest, media, digest = self.manifest_util.tag_manifest_into_registry(
session, source_digest, source_repo, images)
return {
'manifest': manifest.decode('utf-8'),
'media_type': media,
'manifest_digest': digest,
}
def run(self):
primary_images = get_primary_images(self.workflow)
unique_images = get_unique_images(self.workflow)
self.non_floating_images = primary_images + unique_images
session = self.manifest_util.get_registry_session()
built_images = self.get_built_images(session)
if self.group:
return self.group_manifests_and_tag(session, built_images)
else:
if len(built_images) != 1:
raise RuntimeError(
'Without grouping only one built image is expected')
built_image = built_images[0]
source_digest = built_image.manifest_digest
source_repo = built_image.repository
return self.tag_manifest_into_registry(session, source_digest,
source_repo,
self.non_floating_images)