def _get_manifest_list(self, image): """try to figure out manifest list""" if image in self.manifest_list_cache: return self.manifest_list_cache[image] manifest_list = get_manifest_list( image, image.registry, insecure=self.parent_registry_insecure) if '@sha256:' in str(image) and not manifest_list: # we want to adjust the tag only for manifest list fetching image = image.copy() try: config_blob = get_config_from_registry( image, image.registry, image.tag, insecure=self.parent_registry_insecure) except (HTTPError, RetryError, Timeout) as ex: self.log.warning('Unable to fetch config for %s, got error %s', image, ex.response.status_code) raise RuntimeError('Unable to fetch config for base image') release = config_blob['config']['Labels']['release'] version = config_blob['config']['Labels']['version'] docker_tag = "%s-%s" % (version, release) image.tag = docker_tag manifest_list = get_manifest_list( image, image.registry, insecure=self.parent_registry_insecure) self.manifest_list_cache[image] = manifest_list return self.manifest_list_cache[image]
def check_existing_vr_tag(self): """ Checks if version-release tag (primary not floating tag) exists already, and fails plugin if it does. """ primary_images = get_primary_images(self.workflow) if not primary_images: return vr_image = primary_images[0] should_fail = False for registry_name, registry in self.registries.items(): pullspec = vr_image.copy() pullspec.registry = registry_name insecure = registry.get('insecure', False) secret = registry.get('secret', None) manifest_list = get_manifest_list(pullspec, registry_name, insecure, secret) if manifest_list: self.log.error("Primary tag already exists in registry: %s", pullspec) should_fail = True if should_fail: raise RuntimeError("Primary tag already exists in registry")
def check_manifest_list(self, build_image, orchestrator_platform, platforms, current_buildimage): registry_name, image = build_image.split('/', 1) repo, tag = image.rsplit(':', 1) registry = ImageName(registry=registry_name, repo=repo, tag=tag) manifest_list = get_manifest_list(registry, registry_name, insecure=True) # we don't have manifest list, but we want to build on different platforms if not manifest_list: raise RuntimeError("Buildroot image isn't manifest list," " which is needed for specified arch") arch_digests = {} image_name = build_image.rsplit(':', 1)[0] manifest_list_dict = manifest_list.json() for manifest in manifest_list_dict['manifests']: arch = manifest['platform']['architecture'] arch_digests[arch] = image_name + '@' + manifest['digest'] arch_to_platform = get_goarch_to_platform_mapping(self.workflow, self.plat_des_fallback) for arch in arch_digests: self.build_image_digests[arch_to_platform[arch]] = arch_digests[arch] # orchestrator platform is in manifest list if orchestrator_platform not in self.build_image_digests: raise RuntimeError("Platform for orchestrator '%s' isn't in manifest list" % orchestrator_platform) if ('@sha256:' in current_buildimage and self.build_image_digests[orchestrator_platform] != current_buildimage): raise RuntimeError("Orchestrator is using image digest '%s' which isn't" " in manifest list" % current_buildimage)
def check_manifest_list(self, build_image, orchestrator_platform, platforms, current_buildimage): registry_name, image = build_image.split('/', 1) repo, tag = image.rsplit(':', 1) registry = ImageName(registry=registry_name, repo=repo, tag=tag) manifest_list = get_manifest_list(registry, registry_name, insecure=True) # we don't have manifest list, but we want to build on different platforms if not manifest_list: raise RuntimeError("Buildroot image isn't manifest list," " which is needed for specified arch") arch_digests = {} image_name = build_image.rsplit(':', 1)[0] manifest_list_dict = manifest_list.json() for manifest in manifest_list_dict['manifests']: arch = manifest['platform']['architecture'] arch_digests[arch] = image_name + '@' + manifest['digest'] arch_to_platform = get_goarch_to_platform_mapping(self.workflow, self.plat_des_fallback) for arch in arch_digests: self.build_image_digests[arch_to_platform[arch]] = arch_digests[arch] # orchestrator platform is in manifest list if orchestrator_platform not in self.build_image_digests: raise RuntimeError("Platform for orchestrator '%s' isn't in manifest list" % orchestrator_platform) if ('@sha256:' in current_buildimage and self.build_image_digests[orchestrator_platform] != current_buildimage): raise RuntimeError("Orchestrator is using image digest '%s' which isn't" " in manifest list" % current_buildimage)
def validate_platforms_in_base_image(self, base_image): expected_platforms = self.get_expected_platforms() if not expected_platforms: self.log.info('Skipping validation of available platforms ' 'because expected platforms are unknown') return if len(expected_platforms) == 1: self.log.info('Skipping validation of available platforms for base image ' 'because this is a single platform build') return if not base_image.registry: self.log.info('Cannot validate available platforms for base image ' 'because base image registry is not defined') return try: platform_to_arch = get_platform_to_goarch_mapping(self.workflow) except KeyError: self.log.info('Cannot validate available platforms for base image ' 'because platform descriptors are not defined') return if '@sha256:' in str(base_image): # we want to adjust the tag only for manifest list fetching base_image = base_image.copy() try: config_blob = get_config_from_registry(base_image, base_image.registry, base_image.tag, insecure=self.parent_registry_insecure) except (HTTPError, RetryError, Timeout) as ex: self.log.warning('Unable to fetch config for %s, got error %s', base_image, ex.response.status_code) raise RuntimeError('Unable to fetch config for base image') release = config_blob['config']['Labels']['release'] version = config_blob['config']['Labels']['version'] docker_tag = "%s-%s" % (version, release) base_image.tag = docker_tag manifest_list = get_manifest_list(base_image, base_image.registry, insecure=self.parent_registry_insecure) if not manifest_list: raise RuntimeError('Unable to fetch manifest list for base image') all_manifests = manifest_list.json()['manifests'] manifest_list_arches = set( manifest['platform']['architecture'] for manifest in all_manifests) expected_arches = set( platform_to_arch[platform] for platform in expected_platforms) self.log.info('Manifest list arches: %s, expected arches: %s', manifest_list_arches, expected_arches) assert manifest_list_arches >= expected_arches, \ 'Missing arches in manifest list for base image' self.log.info('Base image is a manifest list for all required platforms')
def manifest_list_entries_match(self, image, build_id): """Check whether manifest list entries are in koji. Compares the digest in each manifest list entry with the koji build archive for the entry's architecture. Returns True if they all match. :param image: ImageName, image to inspect :param build_id: int, koji build ID for the image :return: bool, True if the manifest list content refers to the koji build archives """ if not image.registry: self.log.warning( 'Could not fetch manifest list for %s: missing registry ref', image) return False v2_type = get_manifest_media_type('v2') insecure = self._source_registry.get('insecure', False) dockercfg_path = self._source_registry.get('secret') manifest_list_response = get_manifest_list( image, image.registry, insecure=insecure, dockercfg_path=dockercfg_path) if not manifest_list_response: self.log.warning('Could not fetch manifest list for %s', image) return False manifest_list_data = {} manifest_list = json.loads(manifest_list_response.content) for manifest in manifest_list['manifests']: if manifest['mediaType'] != v2_type: self.log.warning('Unexpected media type in manifest list: %s', manifest) return False arch = manifest['platform']['architecture'] v2_digest = manifest['digest'] manifest_list_data[arch] = v2_digest archives = self.koji_session.listArchives(build_id) koji_archives_data = {} for archive in (a for a in archives if a['btype'] == KOJI_BTYPE_IMAGE): arch = archive['extra']['docker']['config']['architecture'] v2_digest = archive['extra']['docker']['digests'][v2_type] koji_archives_data[arch] = v2_digest if koji_archives_data == manifest_list_data: self.log.info( 'Deeper manifest list check verified v2 manifest references match' ) return True self.log.warning( 'Manifest list refs "%s" do not match koji archive refs "%s"', manifest_list_data, koji_archives_data) return False
def test_get_manifest_list(tmpdir, image, registry, insecure, creds, path): kwargs = {} image = ImageName.parse(image) kwargs['image'] = image if creds: temp_dir = mkdtemp(dir=str(tmpdir)) with open(os.path.join(temp_dir, '.dockercfg'), 'w+') as dockerconfig: dockerconfig.write( json.dumps( {registry: { 'username': creds[0], 'password': creds[1] }})) kwargs['dockercfg_path'] = temp_dir kwargs['registry'] = registry if insecure is not None: kwargs['insecure'] = insecure def request_callback(request, all_headers=True): if creds and creds[0] and creds[1]: assert request.headers['Authorization'] media_type = request.headers['Accept'] if media_type.endswith('list.v2+json'): digest = 'v2_list-digest' elif media_type.endswith('v2+json'): digest = 'v2-digest' elif media_type.endswith('v1+json'): digest = 'v1-digest' else: raise ValueError('Unexpected media type {}'.format(media_type)) media_type_prefix = media_type.split('+')[0] if all_headers: headers = { 'Content-Type': '{}+jsonish'.format(media_type_prefix), } if not media_type.endswith('list.v2+json'): headers['Docker-Content-Digest'] = digest else: headers = {} return (200, headers, '') if registry.startswith('http'): url = registry + path else: # In the insecure case, we should try the https URL, and when that produces # an error, fall back to http if insecure: https_url = 'https://' + registry + path responses.add(responses.GET, https_url, body=ConnectionError()) url = 'http://' + registry + path else: url = 'https://' + registry + path responses.add_callback(responses.GET, url, callback=request_callback) manifest_list = get_manifest_list(**kwargs) assert manifest_list