def get_buildroot(self, worker_metadatas): """ Build the buildroot entry of the metadata. :return: list, containing dicts of partial metadata """ buildroots = [] buildroot = koji_get_buildroot(build_id=self.build_id, tasker=self.tasker, osbs=self.osbs, rpms=False) buildroot['id'] = '{}-{}'.format(buildroot['container']['arch'], buildroot['id']) registry = self.workflow.push_conf.docker_registries[0] build_name = get_unique_images(self.workflow)[0].to_str() manifest_digest = registry.digests[build_name] digest_version = get_manifest_media_version(manifest_digest) media_type = get_manifest_media_type(digest_version) buildroot['extra']['osbs']['koji'] = { 'build_name': build_name, 'builder_image_id': {media_type: manifest_digest.default} } buildroots.append(buildroot) return buildroots
def get_output(self, worker_metadatas, buildroot_id): """ Build the output entry of the metadata. :return: list, containing dicts of partial metadata """ outputs = [] output_file = None if self.source_build: registry = self.workflow.push_conf.docker_registries[0] build_name = get_unique_images(self.workflow)[0] pullspec = copy.deepcopy(build_name) pullspec.registry = registry.uri outputs, output_file = get_output(workflow=self.workflow, buildroot_id=buildroot_id, pullspec=pullspec, platform=os.uname()[4], source_build=True, logs=None) else: for platform in worker_metadatas: for instance in worker_metadatas[platform]['output']: instance['buildroot_id'] = '{}-{}'.format(platform, instance['buildroot_id']) outputs.append(instance) return outputs, output_file
def add_floating_tags(self, session, manifest_list_data, floating_images): list_type = manifest_list_data.get("media_type") manifest = manifest_list_data.get("manifest") manifest_digest = manifest_list_data.get("manifest_digest") for image in floating_images: target_repo = image.to_str(registry=False, tag=False) # We have to call store_manifest_in_repository directly for each # referenced manifest, since each one should be a new tag that requires uploading # the manifest again self.log.debug("storing %s as %s", target_repo, image.tag) self.manifest_util.store_manifest_in_repository(session, manifest, list_type, target_repo, target_repo, ref=image.tag) # And store the manifest list in the push_conf push_conf_registry = self.workflow.push_conf.add_docker_registry( session.registry, insecure=session.insecure) for image in floating_images: push_conf_registry.digests[image.tag] = manifest_digest registry_image = get_unique_images(self.workflow)[0] return registry_image.get_repo( explicit_namespace=False), manifest_digest
def get_output(self, buildroot_id): pullspec = get_unique_images(self.workflow)[0] return koji_get_output(workflow=self.workflow, buildroot_id=buildroot_id, pullspec=pullspec, platform=os.uname()[4], source_build=True)
def get_output(self, worker_metadatas, buildroot_id): registry = self.workflow.push_conf.docker_registries[0] build_name = get_unique_images(self.workflow)[0] pullspec = copy.deepcopy(build_name) pullspec.registry = registry.uri return koji_get_output(workflow=self.workflow, buildroot_id=buildroot_id, pullspec=pullspec, platform=os.uname()[4], source_build=True, logs=None)
def get_output(self, buildroot_id: str) -> List[Dict[str, Any]]: outputs = super().get_output(buildroot_id) pullspec = get_unique_images(self.workflow)[0] metadatas, output_file = koji_get_output( workflow=self.workflow, buildroot_id=buildroot_id, pullspec=pullspec, platform=os.uname()[4], source_build=True, ) self.workflow.data.koji_upload_files.append({ "local_filename": output_file.filename, "dest_filename": output_file.metadata['filename'], }) outputs.extend(metadatas) return outputs
def run(self): primary_images = get_primary_images(self.workflow) unique_images = get_unique_images(self.workflow) self.non_floating_images = primary_images + unique_images for registry, source in self.sort_annotations().items(): session = self.manifest_util.get_registry_session(registry) if self.group: return self.group_manifests_and_tag(session, source) else: if len(source) != 1: raise RuntimeError('Without grouping only one source is expected') # source.values() isn't a list and can't be indexed, so this clumsy workaround _, orig_digest = source.popitem() source_digest = orig_digest['digest'] source_repo = orig_digest['repository'] return self.tag_manifest_into_registry(session, source_digest, source_repo, self.non_floating_images)
def run(self): primary_images = get_primary_images(self.workflow) unique_images = get_unique_images(self.workflow) self.non_floating_images = primary_images + unique_images session = self.manifest_util.get_registry_session() built_images = self.get_built_images(session) if self.group: return self.group_manifests_and_tag(session, built_images) else: if len(built_images) != 1: raise RuntimeError( 'Without grouping only one built image is expected') built_image = built_images[0] source_digest = built_image.manifest_digest source_repo = built_image.repository return self.tag_manifest_into_registry(session, source_digest, source_repo, self.non_floating_images)
def group_manifests_and_tag(self, session, worker_digests): """ Creates a manifest list or OCI image index that groups the different manifests in worker_digests, then tags the result with with all the configured tags found in workflow.tag_conf. """ self.log.info("%s: Creating manifest list", session.registry) # Extract information about the manifests that we will group - we get the # size and content type of the manifest by querying the registry manifests = [] for platform, worker_image in worker_digests.items(): repository = worker_image['repository'] digest = worker_image['digest'] media_type = get_manifest_media_type(worker_image['version']) if media_type not in self.manifest_util.manifest_media_types: continue content, _, media_type, size = self.manifest_util.get_manifest( session, repository, digest) manifests.append({ 'content': content, 'repository': repository, 'digest': digest, 'size': size, 'media_type': media_type, 'architecture': self.goarch.get(platform, platform), }) list_type, list_json = self.manifest_util.build_list(manifests) self.log.info("%s: Created manifest, Content-Type=%s\n%s", session.registry, list_type, list_json) # Now push the manifest list to the registry once per each tag self.log.info("%s: Tagging manifest list", session.registry) for image in self.non_floating_images: target_repo = image.to_str(registry=False, tag=False) # We have to call store_manifest_in_repository directly for each # referenced manifest, since they potentially come from different repos for manifest in manifests: self.manifest_util.store_manifest_in_repository( session, manifest['content'], manifest['media_type'], manifest['repository'], target_repo, ref=manifest['digest']) self.manifest_util.store_manifest_in_repository(session, list_json, list_type, target_repo, target_repo, ref=image.tag) # Get the digest of the manifest list using one of the tags registry_image = get_unique_images(self.workflow)[0] _, digest_str, _, _ = self.manifest_util.get_manifest( session, registry_image.to_str(registry=False, tag=False), registry_image.tag) if list_type == MEDIA_TYPE_OCI_V1_INDEX: digest = ManifestDigest(oci_index=digest_str) else: digest = ManifestDigest(v2_list=digest_str) # And store the manifest list in the push_conf push_conf_registry = self.workflow.push_conf.add_docker_registry( session.registry, insecure=session.insecure) tags = [] for image in self.non_floating_images: push_conf_registry.digests[image.tag] = digest tags.append(image.tag) self.log.info("%s: Manifest list digest is %s", session.registry, digest_str) self.log.debug("tags: %s digest: %s", tags, digest) return { 'manifest': list_json, 'media_type': list_type, 'manifest_digest': digest }
def set_group_manifest_info(self, extra, worker_metadatas): version_release = None primary_images = get_primary_images(self.workflow) floating_images = get_floating_images(self.workflow) unique_images = get_unique_images(self.workflow) if primary_images: version_release = primary_images[0].tag if is_scratch_build(self.workflow): tags = [image.tag for image in self.workflow.tag_conf.images] version_release = tags[0] else: assert version_release is not None, 'Unable to find version-release image' tags = [image.tag for image in primary_images] floating_tags = [image.tag for image in floating_images] unique_tags = [image.tag for image in unique_images] manifest_data = self.workflow.postbuild_results.get(PLUGIN_GROUP_MANIFESTS_KEY, {}) if manifest_data and is_manifest_list(manifest_data.get("media_type")): manifest_digest = manifest_data.get("manifest_digest") index = {} index['tags'] = tags index['floating_tags'] = floating_tags index['unique_tags'] = unique_tags build_image = get_unique_images(self.workflow)[0] repo = ImageName.parse(build_image).to_str(registry=False, tag=False) # group_manifests added the registry, so this should be valid registries = self.workflow.push_conf.all_registries digest_version = get_manifest_media_version(manifest_digest) digest = manifest_digest.default for registry in registries: pullspec = "{0}/{1}@{2}".format(registry.uri, repo, digest) index['pull'] = [pullspec] pullspec = "{0}/{1}:{2}".format(registry.uri, repo, version_release) index['pull'].append(pullspec) # Store each digest with according media type index['digests'] = {} media_type = get_manifest_media_type(digest_version) index['digests'][media_type] = digest break extra['image']['index'] = index # group_manifests returns None if didn't run, {} if group=False else: for platform in worker_metadatas: if platform == "x86_64": for instance in worker_metadatas[platform]['output']: if instance['type'] == 'docker-image': # koji_upload, running in the worker, doesn't have the full tags # so set them here instance['extra']['docker']['tags'] = tags instance['extra']['docker']['floating_tags'] = floating_tags instance['extra']['docker']['unique_tags'] = unique_tags repositories = [] for pullspec in instance['extra']['docker']['repositories']: if '@' not in pullspec: image = ImageName.parse(pullspec) image.tag = version_release pullspec = image.to_str() repositories.append(pullspec) instance['extra']['docker']['repositories'] = repositories self.log.debug("reset tags to so that docker is %s", instance['extra']['docker']) annotations = get_worker_build_info(self.workflow, platform).\ build.get_annotations() digests = {} if 'digests' in annotations: digests = get_digests_map_from_annotations(annotations['digests']) instance['extra']['docker']['digests'] = digests
def set_group_manifest_info(self, extra): version_release = None primary_images = get_primary_images(self.workflow) if primary_images: version_release = primary_images[0].tag if is_scratch_build(self.workflow): tags = [image.tag for image in self.workflow.data.tag_conf.images] version_release = tags[0] else: assert version_release is not None, 'Unable to find version-release image' tags = [image.tag for image in primary_images] floating_tags = [ image.tag for image in get_floating_images(self.workflow) ] unique_images = get_unique_images(self.workflow) unique_tags = [image.tag for image in unique_images] manifest_data = self.workflow.data.postbuild_results.get( PLUGIN_GROUP_MANIFESTS_KEY, {}) if manifest_data and is_manifest_list(manifest_data.get("media_type")): manifest_digest = manifest_data["manifest_digest"] digest = manifest_digest.default build_image = unique_images[0] repo = ImageName.parse(build_image).to_str(registry=False, tag=False) # group_manifests added the registry, so this should be valid registry_uri = self.workflow.conf.registry['uri'] digest_version = get_manifest_media_version(manifest_digest) media_type = get_manifest_media_type(digest_version) extra['image']['index'] = { 'tags': tags, 'floating_tags': floating_tags, 'unique_tags': unique_tags, 'pull': [ f'{registry_uri}/{repo}@{digest}', f'{registry_uri}/{repo}:{version_release}', ], 'digests': { media_type: digest }, } # group_manifests returns None if didn't run, {} if group=False else: platform = "x86_64" _, instance = next( self._iter_build_metadata_outputs(platform, {"type": "docker-image"}), (None, None), ) if instance: # koji_upload, running in the worker, doesn't have the full tags # so set them here instance['extra']['docker']['tags'] = tags instance['extra']['docker']['floating_tags'] = floating_tags instance['extra']['docker']['unique_tags'] = unique_tags repositories = [] for pullspec in instance['extra']['docker']['repositories']: if '@' not in pullspec: image = ImageName.parse(pullspec) image.tag = version_release pullspec = image.to_str() repositories.append(pullspec) instance['extra']['docker']['repositories'] = repositories self.log.debug("reset tags to so that docker is %s", instance['extra']['docker'])