def _update_index_image_pull_spec( output_pull_spec, request_id, arches, from_index=None, overwrite_from_index=False, overwrite_from_index_token=None, resolved_prebuild_from_index=None, add_or_rm=False, ): """ Update the request with the modified index image. This function was created so that code didn't need to be duplicated for the ``add`` and ``rm`` request types. :param str output_pull_spec: pull spec of the index image generated by IIB :param int request_id: the ID of the IIB build request :param set arches: the set of arches that were built as part of this request :param str from_index: the pull specification of the container image containing the index that the index image build was based from. :param bool overwrite_from_index: if True, overwrite the input ``from_index`` with the built index image. :param str overwrite_from_index_token: the token used for overwriting the input ``from_index`` image. :param str resolved_prebuild_from_index: resolved index image before starting the build. :param bool add_or_rm: true if the request is an ``Add`` or ``Rm`` request. defaults to false :raises IIBError: if the manifest list couldn't be created and pushed """ conf = get_worker_config() if from_index and overwrite_from_index: _overwrite_from_index( request_id, output_pull_spec, from_index, resolved_prebuild_from_index, overwrite_from_index_token, ) index_image = from_index elif conf['iib_index_image_output_registry']: index_image = output_pull_spec.replace( conf['iib_registry'], conf['iib_index_image_output_registry'], 1) log.info( 'Changed the index_image pull specification from %s to %s', output_pull_spec, index_image, ) else: index_image = output_pull_spec payload = {'arches': list(arches), 'index_image': index_image} if add_or_rm: with set_registry_token(overwrite_from_index_token, index_image): index_image_resolved = get_resolved_image(index_image) payload['index_image_resolved'] = index_image_resolved update_request(request_id, payload, exc_msg='Failed setting the index image on the request')
def _resolve_image_pull_specs(bundle_metadata, labels, pinned_by_iib): """ Resolve image pull specifications to container image digests. :param dict bundle_metadata: the dictionary of CSV's and relatedImages pull specifications :param dict labels: the dictionary of labels to be set on the bundle image :param bool pinned_by_iib: whether or not the bundle image has already been processed by IIB to perform image pinning of related images. """ # Resolve pull specs to container image digests replacement_pullspecs = {} for pullspec in bundle_metadata['found_pullspecs']: new_pullspec = ImageName.parse(pullspec.to_str()) if not pinned_by_iib: # Resolve the image only if it has not already been processed by IIB. This # helps making sure the pullspec is valid resolved_image = ImageName.parse(get_resolved_image(pullspec.to_str())) # If the tag is in the format "<algorithm>:<checksum>", the image is already pinned. # Otherwise, always pin it to a digest. if ':' not in ImageName.parse(pullspec).tag: log.debug('%s will be pinned to %s', pullspec, resolved_image.to_str()) new_pullspec = resolved_image labels['com.redhat.iib.pinned'] = 'true' replacement_pullspecs[pullspec] = new_pullspec if replacement_pullspecs: _replace_csv_pullspecs(bundle_metadata, replacement_pullspecs)
def _resolve_image_pull_specs(bundle_metadata, labels, pinned_by_iib, registry_replacements): """ Resolve image pull specifications to container image digests. :param dict bundle_metadata: the dictionary of CSV's and relatedImages pull specifications :param dict labels: the dictionary of labels to be set on the bundle image :param str registry_replacements: the customization dictionary which specifies replacement of registry in the pull specifications. :param bool pinned_by_iib: whether or not the bundle image has already been processed by IIB to perform image pinning of related images. """ # Resolve pull specs to container image digests replacement_pullspecs = {} for pullspec in bundle_metadata['found_pullspecs']: replacement_needed = False new_pullspec = ImageName.parse(pullspec.to_str()) if not pinned_by_iib: # Resolve the image only if it has not already been processed by IIB. This # helps making sure the pullspec is valid resolved_image = ImageName.parse( get_resolved_image(pullspec.to_str())) # If the tag is in the format "<algorithm>:<checksum>", the image is already pinned. # Otherwise, always pin it to a digest. if ':' not in ImageName.parse(pullspec).tag: log.debug('%s will be pinned to %s', pullspec, resolved_image.to_str()) new_pullspec = resolved_image replacement_needed = True labels['com.redhat.iib.pinned'] = 'true' # Apply registry modifications new_registry = registry_replacements.get(new_pullspec.registry) if new_registry: replacement_needed = True new_pullspec.registry = new_registry if replacement_needed: log.debug('%s will be replaced with %s', pullspec, new_pullspec.to_str()) replacement_pullspecs[pullspec] = new_pullspec # Apply modifications to the operator bundle image metadata for operator_csv in bundle_metadata['operator_csvs']: csv_file_name = os.path.basename(operator_csv.path) log.info('Replacing the pull specifications on %s', csv_file_name) operator_csv.replace_pullspecs_everywhere(replacement_pullspecs) log.info('Setting spec.relatedImages on %s', csv_file_name) operator_csv.set_related_images() operator_csv.dump()
def test_get_resolved_image_schema_1(mock_si): image_manifest_schema_1 = textwrap.dedent("""\ { "schemaVersion": 1, "name": "repository/name", "tag": "1.0.0", "architecture": "amd64", "fsLayers": [], "history": [], "signatures": [ { "header": {}, "signature": "text-that-changes-per-request", "protected": "spam" } ] } """) skopeo_output = { "Name": "registry.example.com/repository/name", "Tag": "1.0.0", "Digest": "sha256:aa6680b35f45cf0fd6fb5f417159257ba410a47b8fa20d37b4c7fcd4a564b3fb", "RepoTags": ["1.0.0", "latest"], "Created": "2019-12-04T06:41:46.3149046Z", "DockerVersion": "19.03.2", "Labels": {}, "Architecture": "amd64", "Os": "linux", "Layers": [], "Env": [], } mock_si.side_effect = [image_manifest_schema_1, skopeo_output] rv = utils.get_resolved_image('registry.example.com/repository/name:1.0.0') assert rv == ( 'registry.example.com/repository/name@sha256:aa6680b35f45cf0fd6fb5f417159257ba410a47b8fa2' '0d37b4c7fcd4a564b3fb') mock_si.assert_has_calls([ mock.call('docker://registry.example.com/repository/name:1.0.0', '--raw', return_json=False), mock.call('docker://registry.example.com/repository/name:1.0.0'), ])
def test_get_resolved_image_manifest_list(mock_si): mock_si.return_value = ( r'{"manifests":[{"digest":"sha256:9e0c275e0bcb495773b10a18e499985d782810e47b4fce076422acb4b' r'c3da3dd","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":' r'{"architecture":"amd64","os":"linux"},"size":529},{"digest":"sha256:85313b812ad747dd19cf1' r'8078795b576cc4ae9cd2ca2ccccd7b5c12722b2effd","mediaType":"application\/vnd.docker.distrib' r'ution.manifest.v2+json","platform":{"architecture":"arm64","os":"linux","variant":"v8"},"' r'size":529},{"digest":"sha256:567785922b920b35aee6a217f70433fd437b335ad45054743c960d1aaa14' r'3dcd","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"ar' r'chitecture":"ppc64le","os":"linux"},"size":529}],"mediaType":"application\/vnd.docker.dis' r'tribution.manifest.list.v2+json","schemaVersion":2}') rv = utils.get_resolved_image('docker.io/library/centos:8') assert rv == ( 'docker.io/library/centos@sha256:fe8d824220415eed5477b63addf40fb06c3b049404242b31982106ac' '204f6700') mock_si.assert_called_once_with('docker://docker.io/library/centos:8', '--raw', return_json=False)
def test_get_resolved_image(mock_si, pull_spec, expected): mock_si.return_value = textwrap.dedent(''' { "schemaVersion": 2, "mediaType": "application/vnd.docker.distribution.manifest.v2+json", "config": { "mediaType": "application/vnd.docker.container.image.v1+json", "size": 5545, "digest": "sha256:720713e1a4410985aacd7008719efd13d8a32e76d08d34fca202a60ff43e516d" }, "layers": [ { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", "size": 76275160, "digest": "sha256:a3ac36470b00df382448e79f7a749aa6833e4ac9cc90e3391f778820db9fa407" }, { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", "size": 1598, "digest": "sha256:82a8f4ea76cb6f833c5f179b3e6eda9f2267ed8ac7d1bf652f88ac3e9cc453d1" }, { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", "size": 3500790, "digest": "sha256:e1a6856f83e7ab214d6a8200d5fd22f2311e794c91c59eae3fd49699cbc4a14e" }, { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", "size": 8236572, "digest": "sha256:c82b363416dcd84a2f1c292c3a85b21cbf01f5f2ee7f8b88f4dcfffe53ce549d" }, { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", "size": 92298818, "digest": "sha256:8befc59eb9f1a3f40d3de0eccca8762c95800322c3a83fe40bbc0273df394ac1" } ] } ''' # noqa: E501 ).strip('\n') rv = utils.get_resolved_image(pull_spec) assert rv == expected
def _verify_index_image(resolved_prebuild_from_index, unresolved_from_index, overwrite_from_index_token=None): """ Verify if the index image has changed since the IIB build request started. :param str resolved_prebuild_from_index: resolved index image before starting the build :param str unresolved_from_index: unresolved index image provided as API input :param str overwrite_from_index_token: the token used for overwriting the input ``from_index`` image. This is required to use ``overwrite_from_index``. The format of the token must be in the format "user:password". :raises IIBError: if the index image has changed since IIB build started. """ with set_registry_token(overwrite_from_index_token, unresolved_from_index): resolved_post_build_from_index = get_resolved_image( unresolved_from_index) if resolved_post_build_from_index != resolved_prebuild_from_index: raise IIBError( 'The supplied from_index image changed during the IIB request.' ' Please resubmit the request.')
def handle_regenerate_bundle_request( from_bundle_image, organization, request_id, registry_auths=None ): """ Coordinate the work needed to regenerate the operator bundle image. :param str from_bundle_image: the pull specification of the bundle image to be regenerated. :param str organization: the name of the organization the bundle should be regenerated for. :param int request_id: the ID of the IIB build request :param dict registry_auths: Provide the dockerconfig.json for authentication to private registries, defaults to ``None``. :raises IIBError: if the regenerate bundle image build fails. """ _cleanup() set_request_state(request_id, 'in_progress', 'Resolving from_bundle_image') with set_registry_auths(registry_auths): from_bundle_image_resolved = get_resolved_image(from_bundle_image) arches = get_image_arches(from_bundle_image_resolved) if not arches: raise IIBError( 'No arches were found in the resolved from_bundle_image ' f'{from_bundle_image_resolved}' ) pinned_by_iib = yaml.load( get_image_label(from_bundle_image_resolved, 'com.redhat.iib.pinned') or 'false' ) arches_str = ', '.join(sorted(arches)) log.debug('Set to regenerate the bundle image for the following arches: %s', arches_str) payload = { 'from_bundle_image_resolved': from_bundle_image_resolved, 'state': 'in_progress', 'state_reason': f'Regenerating the bundle image for the following arches: {arches_str}', } exc_msg = 'Failed setting the resolved "from_bundle_image" on the request' update_request(request_id, payload, exc_msg=exc_msg) # Pull the from_bundle_image to ensure steps later on don't fail due to registry timeouts podman_pull(from_bundle_image_resolved) with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir: manifests_path = os.path.join(temp_dir, 'manifests') _copy_files_from_image(from_bundle_image_resolved, '/manifests', manifests_path) metadata_path = os.path.join(temp_dir, 'metadata') _copy_files_from_image(from_bundle_image_resolved, '/metadata', metadata_path) new_labels = _adjust_operator_bundle( manifests_path, metadata_path, request_id, organization, pinned_by_iib ) with open(os.path.join(temp_dir, 'Dockerfile'), 'w') as dockerfile: dockerfile.write( textwrap.dedent( f"""\ FROM {from_bundle_image_resolved} COPY ./manifests /manifests COPY ./metadata /metadata """ ) ) for name, value in new_labels.items(): dockerfile.write(f'LABEL {name}={value}\n') for arch in sorted(arches): _build_image(temp_dir, 'Dockerfile', request_id, arch) _push_image(request_id, arch) set_request_state(request_id, 'in_progress', 'Creating the manifest list') output_pull_spec = _create_and_push_manifest_list(request_id, arches, []) conf = get_worker_config() if conf['iib_index_image_output_registry']: old_output_pull_spec = output_pull_spec output_pull_spec = output_pull_spec.replace( conf['iib_registry'], conf['iib_index_image_output_registry'], 1 ) log.info( 'Changed the bundle_image pull specification from %s to %s', old_output_pull_spec, output_pull_spec, ) payload = { 'arches': list(arches), 'bundle_image': output_pull_spec, 'state': 'complete', 'state_reason': 'The request completed successfully', } update_request(request_id, payload, exc_msg='Failed setting the bundle image on the request')