def set_manifest_list_expectations(self):
# Decide whether we expect v2schema2list based on whether
# group_manifests grouped any manifests
if self.workflow.postbuild_results.get(PLUGIN_GROUP_MANIFESTS_KEY):
self.expect_v2schema2list = True
platforms = get_platforms(self.workflow)
if not platforms:
self.log.debug('Cannot check if only manifest list digest should be checked '
'because we have no platforms list')
return
try:
platform_to_goarch = get_platform_to_goarch_mapping(self.workflow)
except KeyError:
self.log.debug('Cannot check if only manifest list digest should be checked '
'because there are no platform descriptors')
return
for plat in platforms:
if platform_to_goarch[plat] == 'amd64':
break
else:
self.log.debug('amd64 was not built, only manifest list digest is available')
self.expect_v2schema2list_only = True
self.expect_v2schema2 = False
def get_manifest_list_only_expectation(self):
"""
Get expectation for manifest list only
:return: bool, expect manifest list only?
"""
if not self.workflow.postbuild_results.get(PLUGIN_GROUP_MANIFESTS_KEY):
self.log.debug('Cannot check if only manifest list digest should be returned '
'because group manifests plugin did not run')
return False
platforms = get_platforms(self.workflow)
if not platforms:
self.log.debug('Cannot check if only manifest list digest should be returned '
'because we have no platforms list')
return False
try:
platform_to_goarch = get_platform_to_goarch_mapping(self.workflow)
except KeyError:
self.log.debug('Cannot check if only manifest list digest should be returned '
'because there are no platform descriptors')
return False
for plat in platforms:
if platform_to_goarch[plat] == 'amd64':
self.log.debug('amd64 was built, all media types available')
return False
self.log.debug('amd64 was not built, only manifest list digest is available')
return True
def set_manifest_list_expectations(self):
# Decide whether we expect v2schema2list based on whether
# group_manifests grouped any manifests
if self.workflow.postbuild_results.get(PLUGIN_GROUP_MANIFESTS_KEY):
self.expect_v2schema2list = True
platforms = get_platforms(self.workflow)
if not platforms:
self.log.debug(
'Cannot check if only manifest list digest should be checked '
'because we have no platforms list')
return
try:
platform_to_goarch = get_platform_to_goarch_mapping(
self.workflow)
except KeyError:
self.log.debug(
'Cannot check if only manifest list digest should be checked '
'because there are no platform descriptors')
return
for plat in platforms:
if platform_to_goarch[plat] == 'amd64':
break
else:
self.log.debug(
'amd64 was not built, only manifest list digest is available'
)
self.expect_v2schema2list_only = True
self.expect_v2schema2 = False
def set_manifest_list_expectations(self, expected_media_types):
if not self.workflow.postbuild_results.get(PLUGIN_GROUP_MANIFESTS_KEY):
self.log.debug('Cannot check if only manifest list digest should be returned '
'because group manifests plugin did not run')
return expected_media_types
platforms = get_platforms(self.workflow)
if not platforms:
self.log.debug('Cannot check if only manifest list digest should be returned '
'because we have no platforms list')
return expected_media_types
try:
platform_to_goarch = get_platform_to_goarch_mapping(self.workflow)
except KeyError:
self.log.debug('Cannot check if only manifest list digest should be returned '
'because there are no platform descriptors')
return expected_media_types
for plat in platforms:
if platform_to_goarch[plat] == 'amd64':
self.log.debug('amd64 was built, all media types available')
return expected_media_types
self.log.debug('amd64 was not built, only manifest list digest is available')
return [MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST]
def validate_platforms_in_base_image(self, base_image):
expected_platforms = self.get_expected_platforms()
if not expected_platforms:
self.log.info('Skipping validation of available platforms '
'because expected platforms are unknown')
return
if len(expected_platforms) == 1:
self.log.info('Skipping validation of available platforms for base image '
'because this is a single platform build')
return
if not base_image.registry:
self.log.info('Cannot validate available platforms for base image '
'because base image registry is not defined')
return
try:
platform_to_arch = get_platform_to_goarch_mapping(self.workflow)
except KeyError:
self.log.info('Cannot validate available platforms for base image '
'because platform descriptors are not defined')
return
if '@sha256:' in str(base_image):
# we want to adjust the tag only for manifest list fetching
base_image = base_image.copy()
try:
config_blob = get_config_from_registry(base_image, base_image.registry,
base_image.tag,
insecure=self.parent_registry_insecure)
except (HTTPError, RetryError, Timeout) as ex:
self.log.warning('Unable to fetch config for %s, got error %s',
base_image, ex.response.status_code)
raise RuntimeError('Unable to fetch config for base image')
release = config_blob['config']['Labels']['release']
version = config_blob['config']['Labels']['version']
docker_tag = "%s-%s" % (version, release)
base_image.tag = docker_tag
manifest_list = get_manifest_list(base_image, base_image.registry,
insecure=self.parent_registry_insecure)
if not manifest_list:
raise RuntimeError('Unable to fetch manifest list for base image')
all_manifests = manifest_list.json()['manifests']
manifest_list_arches = set(
manifest['platform']['architecture'] for manifest in all_manifests)
expected_arches = set(
platform_to_arch[platform] for platform in expected_platforms)
self.log.info('Manifest list arches: %s, expected arches: %s',
manifest_list_arches, expected_arches)
assert manifest_list_arches >= expected_arches, \
'Missing arches in manifest list for base image'
self.log.info('Base image is a manifest list for all required platforms')
def _validate_platforms_in_image(self, image):
"""Ensure that the image provides all platforms expected for the build."""
expected_platforms = get_platforms(self.workflow)
if not expected_platforms:
self.log.info('Skipping validation of available platforms '
'because expected platforms are unknown')
return
if not image.registry:
self.log.info('Cannot validate available platforms for base image '
'because base image registry is not defined')
return
try:
platform_to_arch = get_platform_to_goarch_mapping(self.workflow)
except KeyError:
self.log.info('Cannot validate available platforms for base image '
'because platform descriptors are not defined')
return
manifest_list = self._get_manifest_list(image)
if not manifest_list:
if len(expected_platforms) == 1:
self.log.warning(
'Skipping validation of available platforms for base image: '
'this is a single platform build and base image has no manifest '
'list')
return
else:
raise RuntimeError(
'Unable to fetch manifest list for base image {}'.format(
image))
all_manifests = manifest_list.json()['manifests']
manifest_list_arches = set(manifest['platform']['architecture']
for manifest in all_manifests)
expected_arches = set(platform_to_arch[platform]
for platform in expected_platforms)
self.log.info('Manifest list arches: %s, expected arches: %s',
manifest_list_arches, expected_arches)
missing_arches = expected_arches - manifest_list_arches
if missing_arches:
arches_str = ', '.join(sorted(missing_arches))
raise RuntimeError(
'Base image {} not available for arches: {}'.format(
image, arches_str))
self.log.info(
'Base image is a manifest list for all required platforms')
def test_get_platform_to_goarch_mapping(self, fallback, config, expect):
tasker, workflow = self.prepare()
workflow.plugin_workspace[ReactorConfigPlugin.key] = {}
config_json = read_yaml(config, 'schemas/config.json')
workspace = workflow.plugin_workspace[ReactorConfigPlugin.key]
workspace[WORKSPACE_CONF_KEY] = ReactorConfig(config_json)
kwargs = {}
if fallback:
kwargs['descriptors_fallback'] = {'x86_64': 'amd64'}
platform_to_goarch = get_platform_to_goarch_mapping(workflow, **kwargs)
goarch_to_platform = get_goarch_to_platform_mapping(workflow, **kwargs)
for plat, goarch in expect.items():
assert platform_to_goarch[plat] == goarch
assert goarch_to_platform[goarch] == plat
def test_get_platform_to_goarch_mapping(self, fallback, config, expect):
tasker, workflow = self.prepare()
workflow.plugin_workspace[ReactorConfigPlugin.key] = {}
config_json = read_yaml(config, 'schemas/config.json')
workspace = workflow.plugin_workspace[ReactorConfigPlugin.key]
workspace[WORKSPACE_CONF_KEY] = ReactorConfig(config_json)
kwargs = {}
if fallback:
kwargs['descriptors_fallback'] = {'x86_64': 'amd64'}
platform_to_goarch = get_platform_to_goarch_mapping(workflow, **kwargs)
goarch_to_platform = get_goarch_to_platform_mapping(workflow, **kwargs)
for plat, goarch in expect.items():
assert platform_to_goarch[plat] == goarch
assert goarch_to_platform[goarch] == plat
def _validate_platforms_in_image(self, image):
"""Ensure that the image provides all platforms expected for the build."""
expected_platforms = self._get_expected_platforms()
if not expected_platforms:
self.log.info('Skipping validation of available platforms '
'because expected platforms are unknown')
return
if len(expected_platforms) == 1:
self.log.info(
'Skipping validation of available platforms for base image '
'because this is a single platform build')
return
if not image.registry:
self.log.info('Cannot validate available platforms for base image '
'because base image registry is not defined')
return
try:
platform_to_arch = get_platform_to_goarch_mapping(self.workflow)
except KeyError:
self.log.info('Cannot validate available platforms for base image '
'because platform descriptors are not defined')
return
manifest_list = self._get_manifest_list(image)
if not manifest_list:
raise RuntimeError('Unable to fetch manifest list for base image')
all_manifests = manifest_list.json()['manifests']
manifest_list_arches = set(manifest['platform']['architecture']
for manifest in all_manifests)
expected_arches = set(platform_to_arch[platform]
for platform in expected_platforms)
self.log.info('Manifest list arches: %s, expected arches: %s',
manifest_list_arches, expected_arches)
assert manifest_list_arches >= expected_arches, \
'Missing arches in manifest list for base image'
self.log.info(
'Base image is a manifest list for all required platforms')
def get_manifest_list_only_expectation(self):
"""
Get expectation for manifest list only
:return: bool, expect manifest list only?
"""
manifest_results = self.workflow.postbuild_results.get(
PLUGIN_GROUP_MANIFESTS_KEY)
if not manifest_results or not is_manifest_list(
manifest_results.get("media_type")):
self.log.debug(
'Cannot check if only manifest list digest should be returned '
'because group manifests plugin did not run')
return False
platforms = get_platforms(self.workflow)
if not platforms:
self.log.debug(
'Cannot check if only manifest list digest should be returned '
'because we have no platforms list')
return False
try:
platform_to_goarch = get_platform_to_goarch_mapping(self.workflow)
except KeyError:
self.log.debug(
'Cannot check if only manifest list digest should be returned '
'because there are no platform descriptors')
return False
for plat in platforms:
if platform_to_goarch[plat] == 'amd64':
self.log.debug('amd64 was built, all media types available')
return False
self.log.debug(
'amd64 was not built, only manifest list digest is available')
return True
def manifest_list_entries_match(self, image, build_id):
"""Check whether manifest list entries are in koji.
Compares the digest in each manifest list entry with the koji build
archive for the entry's architecture. Returns True if they all match.
:param image: ImageName, image to inspect
:param build_id: int, koji build ID for the image
:return: bool, True if the manifest list content refers to the koji build archives
"""
if not image.registry:
self.log.warning(
'Could not fetch manifest list for %s: missing registry ref',
image)
return False
v2_type = get_manifest_media_type('v2')
reg_client = self._get_registry_client(image.registry)
manifest_list_response = reg_client.get_manifest_list(image)
if not manifest_list_response:
self.log.warning('Could not fetch manifest list for %s', image)
return False
manifest_list_data = {}
manifest_list = json.loads(manifest_list_response.content)
for manifest in manifest_list['manifests']:
if manifest['mediaType'] != v2_type:
self.log.warning('Unexpected media type in manifest list: %s',
manifest)
return False
arch = manifest['platform']['architecture']
v2_digest = manifest['digest']
manifest_list_data[arch] = v2_digest
archives = self.koji_session.listArchives(build_id)
koji_archives_data = {}
for archive in (a for a in archives if a['btype'] == KOJI_BTYPE_IMAGE):
arch = archive['extra']['docker']['config']['architecture']
v2_digest = archive['extra']['docker']['digests'][v2_type]
koji_archives_data[arch] = v2_digest
platform_to_arch_dict = get_platform_to_goarch_mapping(self.workflow)
architectures = [
platform_to_arch_dict[platform] for platform in self.platforms
]
missing_arches = [
a for a in architectures if a not in koji_archives_data
]
if missing_arches:
self.log.warning(
'Architectures "%s" are missing in Koji archives "%s"',
missing_arches, koji_archives_data)
return False
# manifest lists can be manually pushed to the registry to make sure a specific tag
# (e.g., latest) is available for all platforms.
# In such cases these manifest lists may include images from different koji builds.
# We only want to check the digests for the images built in the current parent koji build
err_msg = 'Manifest list digest %s differs from Koji archive digest %s for platform %s'
unmatched_digests = False
for arch in architectures:
if manifest_list_data[arch] != koji_archives_data[arch]:
unmatched_digests = True
self.log.warning(err_msg, manifest_list_data[arch],
koji_archives_data[arch], arch)
if unmatched_digests:
return False
self.log.info(
'Deeper manifest list check verified v2 manifest references match')
return True
