def _store_manifest_digest(self, image: ImageName, use_original_tag: bool) -> None: """Store media type and digest for manifest list or v2 schema 2 manifest digest""" image_str = image.to_str() manifest_list = self._get_manifest_list(image) reg_client = self._get_registry_client(image.registry) if manifest_list: digest_dict = get_checksums(BytesIO(manifest_list.content), ['sha256']) media_type = get_manifest_media_type('v2_list') else: digests_dict = reg_client.get_all_manifests(image, versions=('v2',)) media_type = get_manifest_media_type('v2') try: manifest_digest_response = digests_dict['v2'] except KeyError as exc: raise RuntimeError( 'Unable to fetch manifest list or ' 'v2 schema 2 digest for {} (Does image exist?)'.format(image_str) ) from exc digest_dict = get_checksums(BytesIO(manifest_digest_response.content), ['sha256']) manifest_digest = 'sha256:{}'.format(digest_dict['sha256sum']) parent_digests = {media_type: manifest_digest} if use_original_tag: # image tag may have been replaced with a ref for autorebuild; use original tag # to simplify fetching parent_images_digests data in other plugins image = image.copy() base_image_key: ImageName = self.workflow.data.dockerfile_images.base_image_key image.tag = base_image_key.tag image_str = image.to_str() self.workflow.data.parent_images_digests[image_str] = parent_digests
def _fetch_manifest_digest(self, image: ImageName) -> Dict[str, str]: """Fetch media type and digest for manifest list or v2 schema 2 manifest digest""" image_str = image.to_str() manifest_list = self._get_manifest_list(image) reg_client = self._get_registry_client(image.registry) if manifest_list: digest_dict = get_checksums(BytesIO(manifest_list.content), ['sha256']) media_type = get_manifest_media_type('v2_list') else: digests_dict = reg_client.get_all_manifests(image, versions=('v2', )) media_type = get_manifest_media_type('v2') try: manifest_digest_response = digests_dict['v2'] except KeyError as exc: raise RuntimeError( 'Unable to fetch manifest list or ' 'v2 schema 2 digest for {} (Does image exist?)'.format( image_str)) from exc digest_dict = get_checksums( BytesIO(manifest_digest_response.content), ['sha256']) manifest_digest = 'sha256:{}'.format(digest_dict['sha256sum']) parent_digest = {media_type: manifest_digest} return parent_digest
def push_with_skopeo(self, image: Dict[str, str], registry_image: ImageName, insecure: bool, docker_push_secret: str) -> None: cmd = ['skopeo', 'copy'] if docker_push_secret is not None: dockercfg = Dockercfg(docker_push_secret) cmd.append('--authfile=' + dockercfg.json_secret_path) if insecure: cmd.append('--dest-tls-verify=false') if image['type'] == IMAGE_TYPE_OCI: # ref_name is added by 'flatpak_create_oci' # we have to be careful when changing the source container image type # since assumption here is that source container image will always be 'docker-archive' source_img = 'oci:{path}:{ref_name}'.format(**image) cmd.append('--format=v2s2') elif image['type'] == IMAGE_TYPE_DOCKER_ARCHIVE: source_img = 'docker-archive://{path}'.format(**image) else: raise RuntimeError("Attempt to push unsupported image type %s with skopeo" % image['type']) dest_img = 'docker://' + registry_image.to_str() cmd += [source_img, dest_img] try: retries.run_cmd(cmd) except subprocess.CalledProcessError as e: self.log.error("push failed with output:\n%s", e.output) raise
def _get_image_with_digest(self, image: ImageName) -> Optional[ImageName]: image_str = image.to_str() try: image_metadata = self.workflow.data.parent_images_digests[image_str] except KeyError: return None v2_list_type = get_manifest_media_type('v2_list') v2_type = get_manifest_media_type('v2') raw_digest = image_metadata.get(v2_list_type) or image_metadata.get(v2_type) if not raw_digest: return None digest = raw_digest.split(':', 1)[1] image_name = image.to_str(tag=False) new_image = '{}@sha256:{}'.format(image_name, digest) return ImageName.parse(new_image)
def _pin_to_digest(self, image: ImageName, digests: Dict[str, str]) -> ImageName: v2_list_type = get_manifest_media_type('v2_list') v2_type = get_manifest_media_type('v2') # one of v2_list, v2 *must* be present in the dict raw_digest = digests.get(v2_list_type) or digests[v2_type] digest = raw_digest.split(':', 1)[1] image_name = image.to_str(tag=False) new_image = '{}@sha256:{}'.format(image_name, digest) return ImageName.parse(new_image)
def check_manifest_digest(self, image: ImageName, build_info: Dict[str, Any]) -> None: """Check if the manifest list digest is correct. Compares the manifest list digest with the value in koji metadata. Raises a ValueError if the manifest list does not refer to the koji build. :param image: ImageName, image to inspect :param build_info: dict, koji build metadata """ image_str = image.to_str() v2_list_type = get_manifest_media_type('v2_list') v2_type = get_manifest_media_type('v2') image_digest_data = self.workflow.data.parent_images_digests[image_str] if v2_list_type in image_digest_data: media_type = v2_list_type elif v2_type in image_digest_data: media_type = v2_type else: # This should not happen - raise just to be safe: raise RuntimeError('Unexpected parent image digest data for {}. ' 'v2 or v2_list expected, got {}'.format( image, image_digest_data)) digest = image_digest_data[media_type] try: koji_digest = build_info['extra']['image']['index']['digests'][ media_type] except KeyError as exc: err_msg = ( "Koji build ({}) for parent image '{}' does not have manifest digest data " "for the expected media type '{}'. This parent image MUST be rebuilt" .format(build_info['id'], image_str, media_type)) self.log.error(err_msg) raise ValueError(err_msg) from exc expected_digest = koji_digest self.log.info( 'Verifying manifest digest (%s) for parent %s against its ' 'koji reference (%s)', digest, image_str, expected_digest) if digest != expected_digest: rebuild_msg = 'This parent image MUST be rebuilt' mismatch_msg = ( 'Manifest digest (%s) for parent image %s does not match value in its ' 'koji reference (%s). %s') if not self._deep_manifest_list_inspection: self.log.error(mismatch_msg, digest, image_str, expected_digest, rebuild_msg) raise ValueError( mismatch_msg % (digest, image_str, expected_digest, rebuild_msg)) deep_inspection_msg = 'Checking manifest list contents...' self.log.warning(mismatch_msg, digest, image_str, expected_digest, deep_inspection_msg) if not self.manifest_list_entries_match(image, build_info['id']): err_msg = ( 'Manifest list for parent image %s differs from the manifest list for ' 'its koji reference. %s') self.log.error(err_msg, image_str, rebuild_msg) raise ValueError(err_msg % (image_str, rebuild_msg))
def get_output(workflow: DockerBuildWorkflow, buildroot_id: str, pullspec: ImageName, platform: str, source_build: bool = False): """ Build the 'output' section of the metadata. :param buildroot_id: str, buildroot_id :param pullspec: ImageName :param platform: str, output platform :param source_build: bool, is source_build ? :param logs: list, of Output logs :return: tuple, list of Output instances, and extra Output file """ def add_buildroot_id(output: Output) -> Output: output.metadata.update({'buildroot_id': buildroot_id}) return output extra_output_file = None output_files: List[Output] = [] image_id: str if source_build: manifest = workflow.data.koji_source_manifest image_id = manifest['config']['digest'] # we are using digest from manifest, because we can't get diff_ids # unless we pull image, which would fail due because there are so many layers layer_sizes = [{ 'digest': layer['digest'], 'size': layer['size'] } for layer in manifest['layers']] platform = os.uname()[4] else: imageutil = workflow.imageutil image_id = imageutil.get_inspect_for_image(pullspec, platform=platform)['Id'] parent_id = None if not workflow.data.dockerfile_images.base_from_scratch: parent_id = imageutil.base_image_inspect(platform)['Id'] image_archive = str( workflow.build_dir.platform_dir(platform).exported_squashed_image) layer_sizes = imageutil.get_uncompressed_image_layer_sizes( image_archive) digests = get_manifest_digests(pullspec, workflow.conf.registry['uri'], workflow.conf.registry['insecure'], workflow.conf.registry.get('secret', None)) if digests.v2: config_manifest_digest = digests.v2 config_manifest_type = 'v2' else: config_manifest_digest = digests.oci config_manifest_type = 'oci' config = get_config_from_registry( pullspec, workflow.conf.registry['uri'], config_manifest_digest, workflow.conf.registry['insecure'], workflow.conf.registry.get('secret', None), config_manifest_type) # We don't need container_config section if config and 'container_config' in config: del config['container_config'] digest_pullspec = f"{pullspec.to_str(tag=False)}@{select_digest(digests)}" repositories = [pullspec.to_str(), digest_pullspec] typed_digests = { get_manifest_media_type(version): digest for version, digest in digests.items() if version != "v1" } tag_conf = workflow.data.tag_conf if source_build: image_type = IMAGE_TYPE_DOCKER_ARCHIVE tags = sorted(set(image.tag for image in tag_conf.images)) else: image_metadatas = workflow.data.postbuild_results[ FetchDockerArchivePlugin.key] image_type = image_metadatas[platform]["type"] tags = sorted( image.tag for image in tag_conf.get_unique_images_with_platform(platform)) metadata, output = get_image_output(image_type, image_id, platform, pullspec) metadata.update({ 'arch': platform, 'type': 'docker-image', 'components': [], 'extra': { 'image': { 'arch': platform, }, 'docker': { 'id': image_id, 'repositories': repositories, 'layer_sizes': layer_sizes, 'tags': tags, 'config': config, 'digests': typed_digests, }, }, }) if not config: del metadata['extra']['docker']['config'] if not source_build: metadata['components'] = get_image_components(workflow, platform) if not workflow.data.dockerfile_images.base_from_scratch: metadata['extra']['docker']['parent_id'] = parent_id # Add the 'docker save' image to the output image = add_buildroot_id(output) # when doing regular build, worker already uploads image, # so orchestrator needs only metadata, # but source contaiener build didn't upload that image yet, # so we want metadata, and the image to upload if source_build: output_files.append(metadata) extra_output_file = output else: output_files.append(image) if not source_build: # add operator manifests to output operator_manifests_path = (workflow.data.postbuild_results.get( PLUGIN_EXPORT_OPERATOR_MANIFESTS_KEY)) if operator_manifests_path: manifests_metadata = get_output_metadata( operator_manifests_path, OPERATOR_MANIFESTS_ARCHIVE) operator_manifests_output = Output( filename=operator_manifests_path, metadata=manifests_metadata) add_custom_type(operator_manifests_output, KOJI_BTYPE_OPERATOR_MANIFESTS) operator_manifests = add_buildroot_id(operator_manifests_output) output_files.append(operator_manifests) return output_files, extra_output_file
def get_output(workflow: DockerBuildWorkflow, buildroot_id: str, pullspec: ImageName, platform: str, source_build: bool = False): """ Build the 'output' section of the metadata. :param buildroot_id: str, buildroot_id :param pullspec: ImageName :param platform: str, output platform :param source_build: bool, is source_build ? :param logs: list, of Output logs :return: tuple, list of Output instances, and extra Output file """ def add_buildroot_id(output: Output) -> Output: output.metadata.update({'buildroot_id': buildroot_id}) return output extra_output_file = None output_files: List[Output] = [] image_id: str if source_build: manifest = workflow.data.koji_source_manifest image_id = manifest['config']['digest'] # we are using digest from manifest, because we can't get diff_ids # unless we pull image, which would fail due because there are so many layers layer_sizes = [{ 'digest': layer['digest'], 'size': layer['size'] } for layer in manifest['layers']] platform = os.uname()[4] else: imageutil = workflow.imageutil image_id = imageutil.get_inspect_for_image(pullspec, platform=platform)['Id'] inspect = imageutil.base_image_inspect(platform) parent_id = inspect['Id'] if inspect else None image_archive = str( workflow.build_dir.platform_dir(platform).exported_squashed_image) layer_sizes = imageutil.get_uncompressed_image_layer_sizes( image_archive) digests = get_manifest_digests(pullspec, workflow.conf.registry['uri'], workflow.conf.registry['insecure'], workflow.conf.registry.get('secret', None)) if digests.v2: config_manifest_digest = digests.v2 config_manifest_type = 'v2' else: config_manifest_digest = digests.oci config_manifest_type = 'oci' config = get_config_from_registry( pullspec, workflow.conf.registry['uri'], config_manifest_digest, workflow.conf.registry['insecure'], workflow.conf.registry.get('secret', None), config_manifest_type) # We don't need container_config section if config and 'container_config' in config: del config['container_config'] digest_pullspec = f"{pullspec.to_str(tag=False)}@{select_digest(digests)}" repositories = [pullspec.to_str(), digest_pullspec] typed_digests = { get_manifest_media_type(version): digest for version, digest in digests.items() if version != "v1" } tag_conf = workflow.data.tag_conf if source_build: tags = sorted(set(image.tag for image in tag_conf.images)) else: tags = sorted( image.tag for image in tag_conf.get_unique_images_with_platform(platform)) # since we are storing oci image as v2s2 all images now have 'docker-archive' type metadata, output = get_image_output(IMAGE_TYPE_DOCKER_ARCHIVE, image_id, platform, pullspec) metadata.update({ 'arch': platform, 'type': 'docker-image', 'components': [], 'extra': { 'image': { 'arch': platform, }, 'docker': { 'id': image_id, 'repositories': repositories, 'layer_sizes': layer_sizes, 'tags': tags, 'config': config, 'digests': typed_digests, }, }, }) if not config: del metadata['extra']['docker']['config'] if not source_build: metadata['components'] = get_image_components(workflow.data, platform) if parent_id is not None: metadata['extra']['docker']['parent_id'] = parent_id # Add the 'docker save' image to the output image = add_buildroot_id(output) if source_build: output_files.append(metadata) extra_output_file = output else: output_files.append(image) return output_files, extra_output_file