def _get_manifest_list(self, image):
"""try to figure out manifest list"""
if image in self.manifest_list_cache:
return self.manifest_list_cache[image]
manifest_list = get_manifest_list(
image, image.registry, insecure=self.parent_registry_insecure)
if '@sha256:' in str(image) and not manifest_list:
# we want to adjust the tag only for manifest list fetching
image = image.copy()
try:
config_blob = get_config_from_registry(
image,
image.registry,
image.tag,
insecure=self.parent_registry_insecure)
except (HTTPError, RetryError, Timeout) as ex:
self.log.warning('Unable to fetch config for %s, got error %s',
image, ex.response.status_code)
raise RuntimeError('Unable to fetch config for base image')
release = config_blob['config']['Labels']['release']
version = config_blob['config']['Labels']['version']
docker_tag = "%s-%s" % (version, release)
image.tag = docker_tag
manifest_list = get_manifest_list(
image, image.registry, insecure=self.parent_registry_insecure)
self.manifest_list_cache[image] = manifest_list
return self.manifest_list_cache[image]
def check_existing_vr_tag(self):
"""
Checks if version-release tag (primary not floating tag) exists already,
and fails plugin if it does.
"""
primary_images = get_primary_images(self.workflow)
if not primary_images:
return
vr_image = primary_images[0]
should_fail = False
for registry_name, registry in self.registries.items():
pullspec = vr_image.copy()
pullspec.registry = registry_name
insecure = registry.get('insecure', False)
secret = registry.get('secret', None)
manifest_list = get_manifest_list(pullspec, registry_name, insecure, secret)
if manifest_list:
self.log.error("Primary tag already exists in registry: %s", pullspec)
should_fail = True
if should_fail:
raise RuntimeError("Primary tag already exists in registry")
def check_manifest_list(self, build_image, orchestrator_platform, platforms,
current_buildimage):
registry_name, image = build_image.split('/', 1)
repo, tag = image.rsplit(':', 1)
registry = ImageName(registry=registry_name, repo=repo, tag=tag)
manifest_list = get_manifest_list(registry, registry_name, insecure=True)
# we don't have manifest list, but we want to build on different platforms
if not manifest_list:
raise RuntimeError("Buildroot image isn't manifest list,"
" which is needed for specified arch")
arch_digests = {}
image_name = build_image.rsplit(':', 1)[0]
manifest_list_dict = manifest_list.json()
for manifest in manifest_list_dict['manifests']:
arch = manifest['platform']['architecture']
arch_digests[arch] = image_name + '@' + manifest['digest']
arch_to_platform = get_goarch_to_platform_mapping(self.workflow, self.plat_des_fallback)
for arch in arch_digests:
self.build_image_digests[arch_to_platform[arch]] = arch_digests[arch]
# orchestrator platform is in manifest list
if orchestrator_platform not in self.build_image_digests:
raise RuntimeError("Platform for orchestrator '%s' isn't in manifest list"
% orchestrator_platform)
if ('@sha256:' in current_buildimage and
self.build_image_digests[orchestrator_platform] != current_buildimage):
raise RuntimeError("Orchestrator is using image digest '%s' which isn't"
" in manifest list" % current_buildimage)
def check_manifest_list(self, build_image, orchestrator_platform, platforms,
current_buildimage):
registry_name, image = build_image.split('/', 1)
repo, tag = image.rsplit(':', 1)
registry = ImageName(registry=registry_name, repo=repo, tag=tag)
manifest_list = get_manifest_list(registry, registry_name, insecure=True)
# we don't have manifest list, but we want to build on different platforms
if not manifest_list:
raise RuntimeError("Buildroot image isn't manifest list,"
" which is needed for specified arch")
arch_digests = {}
image_name = build_image.rsplit(':', 1)[0]
manifest_list_dict = manifest_list.json()
for manifest in manifest_list_dict['manifests']:
arch = manifest['platform']['architecture']
arch_digests[arch] = image_name + '@' + manifest['digest']
arch_to_platform = get_goarch_to_platform_mapping(self.workflow, self.plat_des_fallback)
for arch in arch_digests:
self.build_image_digests[arch_to_platform[arch]] = arch_digests[arch]
# orchestrator platform is in manifest list
if orchestrator_platform not in self.build_image_digests:
raise RuntimeError("Platform for orchestrator '%s' isn't in manifest list"
% orchestrator_platform)
if ('@sha256:' in current_buildimage and
self.build_image_digests[orchestrator_platform] != current_buildimage):
raise RuntimeError("Orchestrator is using image digest '%s' which isn't"
" in manifest list" % current_buildimage)
def validate_platforms_in_base_image(self, base_image):
expected_platforms = self.get_expected_platforms()
if not expected_platforms:
self.log.info('Skipping validation of available platforms '
'because expected platforms are unknown')
return
if len(expected_platforms) == 1:
self.log.info('Skipping validation of available platforms for base image '
'because this is a single platform build')
return
if not base_image.registry:
self.log.info('Cannot validate available platforms for base image '
'because base image registry is not defined')
return
try:
platform_to_arch = get_platform_to_goarch_mapping(self.workflow)
except KeyError:
self.log.info('Cannot validate available platforms for base image '
'because platform descriptors are not defined')
return
if '@sha256:' in str(base_image):
# we want to adjust the tag only for manifest list fetching
base_image = base_image.copy()
try:
config_blob = get_config_from_registry(base_image, base_image.registry,
base_image.tag,
insecure=self.parent_registry_insecure)
except (HTTPError, RetryError, Timeout) as ex:
self.log.warning('Unable to fetch config for %s, got error %s',
base_image, ex.response.status_code)
raise RuntimeError('Unable to fetch config for base image')
release = config_blob['config']['Labels']['release']
version = config_blob['config']['Labels']['version']
docker_tag = "%s-%s" % (version, release)
base_image.tag = docker_tag
manifest_list = get_manifest_list(base_image, base_image.registry,
insecure=self.parent_registry_insecure)
if not manifest_list:
raise RuntimeError('Unable to fetch manifest list for base image')
all_manifests = manifest_list.json()['manifests']
manifest_list_arches = set(
manifest['platform']['architecture'] for manifest in all_manifests)
expected_arches = set(
platform_to_arch[platform] for platform in expected_platforms)
self.log.info('Manifest list arches: %s, expected arches: %s',
manifest_list_arches, expected_arches)
assert manifest_list_arches >= expected_arches, \
'Missing arches in manifest list for base image'
self.log.info('Base image is a manifest list for all required platforms')
def manifest_list_entries_match(self, image, build_id):
"""Check whether manifest list entries are in koji.
Compares the digest in each manifest list entry with the koji build
archive for the entry's architecture. Returns True if they all match.
:param image: ImageName, image to inspect
:param build_id: int, koji build ID for the image
:return: bool, True if the manifest list content refers to the koji build archives
"""
if not image.registry:
self.log.warning(
'Could not fetch manifest list for %s: missing registry ref',
image)
return False
v2_type = get_manifest_media_type('v2')
insecure = self._source_registry.get('insecure', False)
dockercfg_path = self._source_registry.get('secret')
manifest_list_response = get_manifest_list(
image,
image.registry,
insecure=insecure,
dockercfg_path=dockercfg_path)
if not manifest_list_response:
self.log.warning('Could not fetch manifest list for %s', image)
return False
manifest_list_data = {}
manifest_list = json.loads(manifest_list_response.content)
for manifest in manifest_list['manifests']:
if manifest['mediaType'] != v2_type:
self.log.warning('Unexpected media type in manifest list: %s',
manifest)
return False
arch = manifest['platform']['architecture']
v2_digest = manifest['digest']
manifest_list_data[arch] = v2_digest
archives = self.koji_session.listArchives(build_id)
koji_archives_data = {}
for archive in (a for a in archives if a['btype'] == KOJI_BTYPE_IMAGE):
arch = archive['extra']['docker']['config']['architecture']
v2_digest = archive['extra']['docker']['digests'][v2_type]
koji_archives_data[arch] = v2_digest
if koji_archives_data == manifest_list_data:
self.log.info(
'Deeper manifest list check verified v2 manifest references match'
)
return True
self.log.warning(
'Manifest list refs "%s" do not match koji archive refs "%s"',
manifest_list_data, koji_archives_data)
return False
def test_get_manifest_list(tmpdir, image, registry, insecure, creds, path):
kwargs = {}
image = ImageName.parse(image)
kwargs['image'] = image
if creds:
temp_dir = mkdtemp(dir=str(tmpdir))
with open(os.path.join(temp_dir, '.dockercfg'), 'w+') as dockerconfig:
dockerconfig.write(
json.dumps(
{registry: {
'username': creds[0],
'password': creds[1]
}}))
kwargs['dockercfg_path'] = temp_dir
kwargs['registry'] = registry
if insecure is not None:
kwargs['insecure'] = insecure
def request_callback(request, all_headers=True):
if creds and creds[0] and creds[1]:
assert request.headers['Authorization']
media_type = request.headers['Accept']
if media_type.endswith('list.v2+json'):
digest = 'v2_list-digest'
elif media_type.endswith('v2+json'):
digest = 'v2-digest'
elif media_type.endswith('v1+json'):
digest = 'v1-digest'
else:
raise ValueError('Unexpected media type {}'.format(media_type))
media_type_prefix = media_type.split('+')[0]
if all_headers:
headers = {
'Content-Type': '{}+jsonish'.format(media_type_prefix),
}
if not media_type.endswith('list.v2+json'):
headers['Docker-Content-Digest'] = digest
else:
headers = {}
return (200, headers, '')
if registry.startswith('http'):
url = registry + path
else:
# In the insecure case, we should try the https URL, and when that produces
# an error, fall back to http
if insecure:
https_url = 'https://' + registry + path
responses.add(responses.GET, https_url, body=ConnectionError())
url = 'http://' + registry + path
else:
url = 'https://' + registry + path
responses.add_callback(responses.GET, url, callback=request_callback)
manifest_list = get_manifest_list(**kwargs)
assert manifest_list