def test_create_and_push_manifest_list(mock_run_cmd, mock_td, tmp_path):
mock_td.return_value.__enter__.return_value = tmp_path
build._create_and_push_manifest_list(3, {'amd64', 's390x'})
expected_manifest = textwrap.dedent(
'''\
image: registry:8443/iib-build:3
manifests:
- image: registry:8443/iib-build:3-amd64
platform:
architecture: amd64
os: linux
- image: registry:8443/iib-build:3-s390x
platform:
architecture: s390x
os: linux
'''
)
manifest = os.path.join(tmp_path, 'manifest.yaml')
with open(manifest, 'r') as manifest_f:
assert manifest_f.read() == expected_manifest
mock_run_cmd.assert_called_once()
manifest_tool_args = mock_run_cmd.call_args[0][0]
assert manifest_tool_args[0] == 'manifest-tool'
assert manifest in manifest_tool_args
def test_create_and_push_manifest_list_failure_to_rm_manifest_list(
mock_run_cmd, mock_td, tmp_path):
mock_td.return_value.__enter__.return_value = tmp_path
mock_run_cmd.side_effect = IIBError('Different error never seen before!')
error_msg = 'Error removing local manifest list: Different error never seen before!'
with pytest.raises(IIBError, match=error_msg):
build._create_and_push_manifest_list(3, {'amd64', 's390x'}, [])
def handle_create_empty_index_request(
from_index,
request_id,
output_fbc=False,
binary_image=None,
labels=None,
binary_image_config=None,
):
"""Coordinate the the work needed to create the index image with labels.
:param str from_index: the pull specification of the container image containing the index that
the index image build will be based from.
:param int request_id: the ID of the IIB build request
:param bool output_fbc: specifies whether a File-based Catalog index image should be created
:param str binary_image: the pull specification of the container image where the opm binary
gets copied from.
:param dict labels: the dict of labels required to be added to a new index image
:param dict binary_image_config: the dict of config required to identify the appropriate
``binary_image`` to use.
"""
_cleanup()
prebuild_info = prepare_request_for_build(
request_id,
RequestConfigCreateIndexImage(
_binary_image=binary_image,
from_index=from_index,
binary_image_config=binary_image_config,
),
)
from_index_resolved = prebuild_info['from_index_resolved']
prebuild_info['labels'] = labels
if not output_fbc and is_image_fbc(from_index_resolved):
log.debug('%s is FBC index image', from_index_resolved)
err_msg = 'Cannot create SQLite index image from File-Based Catalog index image'
log.error(err_msg)
raise IIBError(err_msg)
_update_index_image_build_state(request_id, prebuild_info)
with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir:
set_request_state(request_id, 'in_progress', 'Checking operators present in index image')
operators = _get_present_operators(from_index_resolved, temp_dir)
# if output_fbc parameter is true, create an empty FBC index image
# else create empty SQLite index image
if output_fbc:
log.debug('Creating empty FBC index image from %s', from_index)
opm_create_empty_fbc(
request_id=request_id,
temp_dir=temp_dir,
from_index_resolved=from_index_resolved,
from_index=from_index,
binary_image=prebuild_info['binary_image'],
operators=operators,
)
else:
set_request_state(request_id, 'in_progress', 'Removing operators from index image')
_opm_index_rm(temp_dir, operators, prebuild_info['binary_image'], from_index_resolved)
set_request_state(
request_id, 'in_progress', 'Getting and updating labels for new index image'
)
iib_labels = {
'com.redhat.index.delivery.version': prebuild_info['ocp_version'],
'com.redhat.index.delivery.distribution_scope': prebuild_info['distribution_scope'],
}
if labels:
iib_labels.update(labels)
for index_label, value in iib_labels.items():
_add_label_to_index(index_label, value, temp_dir, 'index.Dockerfile')
arches = prebuild_info['arches']
for arch in sorted(arches):
_build_image(temp_dir, 'index.Dockerfile', request_id, arch)
_push_image(request_id, arch)
set_request_state(request_id, 'in_progress', 'Creating the manifest list')
output_pull_spec = _create_and_push_manifest_list(request_id, arches, [])
_update_index_image_pull_spec(
output_pull_spec=output_pull_spec,
request_id=request_id,
arches=arches,
from_index=from_index,
resolved_prebuild_from_index=from_index_resolved,
)
set_request_state(request_id, 'complete', 'The empty index image was successfully created')
def _add_bundles_missing_in_source(
source_index_bundles,
target_index_bundles,
base_dir,
binary_image,
source_from_index,
request_id,
arch,
ocp_version,
overwrite_target_index_token=None,
distribution_scope=None,
):
"""
Rebuild index image with bundles missing from source image but present in target image.
If no bundles are missing in the source index image, the index image is still rebuilt
using the new binary image.
:param list source_index_bundles: bundles present in the source index image.
:param list target_index_bundles: bundles present in the target index image.
:param str base_dir: base directory where operation files will be located.
:param str binary_image: binary image to be used by the new index image.
:param str source_from_index: index image, whose data will be contained in the new index image.
:param int request_id: the ID of the IIB build request.
:param str arch: the architecture to build this image for.
:param str ocp_version: ocp version which will be added as a label to the image.
:param str overwrite_target_index_token: the token used for overwriting the input
``source_from_index`` image. This is required for non-privileged users to use
``overwrite_target_index``. The format of the token must be in the format "user:password".
:return: bundles which were added to the index image.
:rtype: list
"""
set_request_state(request_id, 'in_progress',
'Adding bundles missing in source index image')
log.info(
'Adding bundles from target index image which are missing from source index image'
)
missing_bundles = []
source_bundle_digests = []
source_bundle_csv_names = []
target_bundle_digests = []
for bundle in source_index_bundles:
if '@sha256:' in bundle['bundlePath']:
source_bundle_digests.append(
bundle['bundlePath'].split('@sha256:')[-1])
source_bundle_csv_names.append(bundle['csvName'])
else:
raise IIBError(
f'Bundle {bundle["bundlePath"]} in the source index image is not defined via digest'
)
for bundle in target_index_bundles:
if '@sha256:' in bundle['bundlePath']:
target_bundle_digests.append(
(bundle['bundlePath'].split('@sha256:')[-1], bundle))
else:
raise IIBError(
f'Bundle {bundle["bundlePath"]} in the target index image is not defined via digest'
)
for target_bundle_digest, bundle in target_bundle_digests:
if (target_bundle_digest not in source_bundle_digests
and bundle['csvName'] not in source_bundle_csv_names):
missing_bundles.append(bundle)
missing_bundle_paths = [bundle['bundlePath'] for bundle in missing_bundles]
if missing_bundle_paths:
log.info('%s bundles are missing in the source index image.',
len(missing_bundle_paths))
else:
log.info(
'No bundles are missing in the source index image. However, the index image is '
'still being rebuilt with the new binary image.')
_opm_index_add(
base_dir,
missing_bundle_paths,
binary_image,
source_from_index,
overwrite_target_index_token,
)
_add_label_to_index('com.redhat.index.delivery.version', ocp_version,
base_dir, 'index.Dockerfile')
_add_label_to_index(
'com.redhat.index.delivery.distribution_scope',
distribution_scope,
base_dir,
'index.Dockerfile',
)
_build_image(base_dir, 'index.Dockerfile', request_id, arch)
_push_image(request_id, arch)
_create_and_push_manifest_list(request_id, [arch])
log.info('New index image created')
return missing_bundles
def handle_merge_request(
source_from_index,
deprecation_list,
request_id,
binary_image=None,
target_index=None,
overwrite_target_index=False,
overwrite_target_index_token=None,
distribution_scope=None,
binary_image_config=None,
):
"""
Coordinate the work needed to merge old (N) index image with new (N+1) index image.
:param str source_from_index: pull specification to be used as the base for building the new
index image.
:param str target_index: pull specification of content stage index image for the
corresponding target index image.
:param list deprecation_list: list of deprecated bundles for the target index image.
:param int request_id: the ID of the IIB build request.
:param str binary_image: the pull specification of the container image where the opm binary
gets copied from.
:param bool overwrite_target_index: if True, overwrite the input ``target_index`` with
the built index image.
:param str overwrite_target_index_token: the token used for overwriting the input
``target_index`` image. This is required for non-privileged users to use
``overwrite_target_index``. The format of the token must be in the format "user:password".
:param str distribution_scope: the scope for distribution of the index image, defaults to
``None``.
:raises IIBError: if the index image merge fails.
"""
_cleanup()
prebuild_info = _prepare_request_for_build(
request_id,
binary_image,
overwrite_from_index_token=overwrite_target_index_token,
source_from_index=source_from_index,
target_index=target_index,
distribution_scope=distribution_scope,
binary_image_config=binary_image_config,
)
_update_index_image_build_state(request_id, prebuild_info)
with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir:
set_request_state(request_id, 'in_progress',
'Getting bundles present in the index images')
log.info('Getting bundles present in the source index image')
source_index_bundles = _get_present_bundles(source_from_index,
temp_dir)
target_index_bundles = []
if target_index:
log.info('Getting bundles present in the target index image')
target_index_bundles = _get_present_bundles(target_index, temp_dir)
arches = list(prebuild_info['arches'])
arch = 'amd64' if 'amd64' in arches else arches[0]
missing_bundles = _add_bundles_missing_in_source(
source_index_bundles,
target_index_bundles,
temp_dir,
prebuild_info['binary_image'],
source_from_index,
request_id,
arch,
prebuild_info['target_ocp_version'],
overwrite_target_index_token,
distribution_scope=prebuild_info['distribution_scope'],
)
set_request_state(request_id, 'in_progress',
'Deprecating bundles in the deprecation list')
log.info('Deprecating bundles in the deprecation list')
intermediate_bundles = source_index_bundles + missing_bundles
deprecate_bundles = _get_bundles_from_deprecation_list(
intermediate_bundles, deprecation_list)
intermediate_image_name = _get_external_arch_pull_spec(
request_id, arch, include_transport=False)
if deprecate_bundles:
_deprecate_bundles(
deprecate_bundles,
temp_dir,
prebuild_info['binary_image'],
intermediate_image_name,
overwrite_target_index_token,
)
for arch in sorted(prebuild_info['arches']):
_build_image(temp_dir, 'index.Dockerfile', request_id, arch)
_push_image(request_id, arch)
output_pull_spec = _create_and_push_manifest_list(request_id,
prebuild_info['arches'])
_update_index_image_pull_spec(
output_pull_spec,
request_id,
prebuild_info['arches'],
target_index,
overwrite_target_index,
overwrite_target_index_token,
prebuild_info['target_index_resolved'],
)
set_request_state(request_id, 'complete',
'The index image was successfully cleaned and updated.')
def _add_bundles_missing_in_source(
source_index_bundles,
target_index_bundles,
base_dir,
binary_image,
source_from_index,
request_id,
arch,
ocp_version,
overwrite_target_index_token=None,
distribution_scope=None,
):
"""
Rebuild index image with bundles missing from source image but present in target image.
If no bundles are missing in the source index image, the index image is still rebuilt
using the new binary image.
:param list source_index_bundles: bundles present in the source index image.
:param list target_index_bundles: bundles present in the target index image.
:param str base_dir: base directory where operation files will be located.
:param str binary_image: binary image to be used by the new index image.
:param str source_from_index: index image, whose data will be contained in the new index image.
:param int request_id: the ID of the IIB build request.
:param str arch: the architecture to build this image for.
:param str ocp_version: ocp version which will be added as a label to the image.
:param str overwrite_target_index_token: the token used for overwriting the input
``source_from_index`` image. This is required to use ``overwrite_target_index``.
The format of the token must be in the format "user:password".
:return: tuple where the first value is a list of bundles which were added to the index image
and the second value is a list of bundles in the new index whose ocp_version range does not
satisfy the ocp_version value of the target index.
:rtype: tuple
"""
set_request_state(request_id, 'in_progress',
'Adding bundles missing in source index image')
log.info(
'Adding bundles from target index image which are missing from source index image'
)
missing_bundles = []
missing_bundle_paths = []
# This list stores the bundles whose ocp_version range does not satisfy the ocp_version
# of the target index
invalid_bundles = []
source_bundle_digests = []
source_bundle_csv_names = []
target_bundle_digests = []
for bundle in source_index_bundles:
if '@sha256:' in bundle['bundlePath']:
source_bundle_digests.append(
bundle['bundlePath'].split('@sha256:')[-1])
source_bundle_csv_names.append(bundle['csvName'])
else:
raise IIBError(
f'Bundle {bundle["bundlePath"]} in the source index image is not defined via digest'
)
for bundle in target_index_bundles:
if '@sha256:' in bundle['bundlePath']:
target_bundle_digests.append(
(bundle['bundlePath'].split('@sha256:')[-1], bundle))
else:
raise IIBError(
f'Bundle {bundle["bundlePath"]} in the target index image is not defined via digest'
)
for target_bundle_digest, bundle in target_bundle_digests:
if (target_bundle_digest not in source_bundle_digests
and bundle['csvName'] not in source_bundle_csv_names):
missing_bundles.append(bundle)
missing_bundle_paths.append(bundle['bundlePath'])
for bundle in itertools.chain(missing_bundles, source_index_bundles):
if not is_bundle_version_valid(bundle['bundlePath'], ocp_version):
invalid_bundles.append(bundle)
if invalid_bundles:
log.info(
'%s bundles have invalid version label and will be deprecated.',
len(invalid_bundles))
_opm_index_add(
base_dir,
missing_bundle_paths,
binary_image,
from_index=source_from_index,
overwrite_from_index_token=overwrite_target_index_token,
# Use podman until opm's default mechanism is more resilient:
# https://bugzilla.redhat.com/show_bug.cgi?id=1937097
container_tool='podman',
)
_add_label_to_index('com.redhat.index.delivery.version', ocp_version,
base_dir, 'index.Dockerfile')
_add_label_to_index(
'com.redhat.index.delivery.distribution_scope',
distribution_scope,
base_dir,
'index.Dockerfile',
)
_build_image(base_dir, 'index.Dockerfile', request_id, arch)
_push_image(request_id, arch)
_create_and_push_manifest_list(request_id, [arch])
log.info('New index image created')
return (missing_bundles, invalid_bundles)
def handle_merge_request(
source_from_index,
deprecation_list,
request_id,
binary_image=None,
target_index=None,
overwrite_target_index=False,
overwrite_target_index_token=None,
distribution_scope=None,
binary_image_config=None,
):
"""
Coordinate the work needed to merge old (N) index image with new (N+1) index image.
:param str source_from_index: pull specification to be used as the base for building the new
index image.
:param str target_index: pull specification of content stage index image for the
corresponding target index image.
:param list deprecation_list: list of deprecated bundles for the target index image.
:param int request_id: the ID of the IIB build request.
:param str binary_image: the pull specification of the container image where the opm binary
gets copied from.
:param bool overwrite_target_index: if True, overwrite the input ``target_index`` with
the built index image.
:param str overwrite_target_index_token: the token used for overwriting the input
``target_index`` image. This is required to use ``overwrite_target_index``.
The format of the token must be in the format "user:password".
:param str distribution_scope: the scope for distribution of the index image, defaults to
``None``.
:raises IIBError: if the index image merge fails.
"""
_cleanup()
prebuild_info = prepare_request_for_build(
request_id,
RequestConfigMerge(
_binary_image=binary_image,
overwrite_target_index_token=overwrite_target_index_token,
source_from_index=source_from_index,
target_index=target_index,
distribution_scope=distribution_scope,
binary_image_config=binary_image_config,
),
)
_update_index_image_build_state(request_id, prebuild_info)
source_from_index_resolved = prebuild_info['source_from_index_resolved']
target_index_resolved = prebuild_info['target_index_resolved']
with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir:
set_request_state(request_id, 'in_progress',
'Getting bundles present in the index images')
log.info('Getting bundles present in the source index image')
with set_registry_token(overwrite_target_index_token,
source_from_index):
source_index_bundles, source_index_bundles_pull_spec = _get_present_bundles(
source_from_index_resolved, temp_dir)
target_index_bundles = []
if target_index:
log.info('Getting bundles present in the target index image')
target_index_bundles, _ = _get_present_bundles(
target_index_resolved, temp_dir)
arches = list(prebuild_info['arches'])
arch = 'amd64' if 'amd64' in arches else arches[0]
missing_bundles, invalid_version_bundles = _add_bundles_missing_in_source(
source_index_bundles,
target_index_bundles,
temp_dir,
prebuild_info['binary_image'],
source_from_index_resolved,
request_id,
arch,
prebuild_info['target_ocp_version'],
overwrite_target_index_token,
distribution_scope=prebuild_info['distribution_scope'],
)
set_request_state(request_id, 'in_progress',
'Deprecating bundles in the deprecation list')
log.info('Deprecating bundles in the deprecation list')
intermediate_bundles = [
bundle['bundlePath'] for bundle in missing_bundles
] + source_index_bundles_pull_spec
deprecation_bundles = get_bundles_from_deprecation_list(
intermediate_bundles, deprecation_list)
# We do not need to pass the invalid_version_bundles through the
# get_bundles_from_deprecation_list function because we already know
# they are present in the newly created index.
deprecation_bundles = deprecation_bundles + [
bundle['bundlePath'] for bundle in invalid_version_bundles
]
intermediate_image_name = _get_external_arch_pull_spec(
request_id, arch, include_transport=False)
if deprecation_bundles:
deprecate_bundles(
deprecation_bundles,
temp_dir,
prebuild_info['binary_image'],
intermediate_image_name,
overwrite_target_index_token,
)
for arch in sorted(prebuild_info['arches']):
_build_image(temp_dir, 'index.Dockerfile', request_id, arch)
_push_image(request_id, arch)
# If the container-tool podman is used in the opm commands above, opm will create temporary
# files and directories without the write permission. This will cause the context manager
# to fail to delete these files. Adjust the file modes to avoid this error.
chmod_recursively(
temp_dir,
dir_mode=(stat.S_IRWXU | stat.S_IRWXG),
file_mode=(stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP
| stat.S_IWGRP),
)
output_pull_spec = _create_and_push_manifest_list(request_id,
prebuild_info['arches'])
_update_index_image_pull_spec(
output_pull_spec,
request_id,
prebuild_info['arches'],
target_index,
overwrite_target_index,
overwrite_target_index_token,
target_index_resolved,
)
set_request_state(request_id, 'complete',
'The index image was successfully cleaned and updated.')
def handle_regenerate_bundle_request(
from_bundle_image, organization, request_id, registry_auths=None
):
"""
Coordinate the work needed to regenerate the operator bundle image.
:param str from_bundle_image: the pull specification of the bundle image to be regenerated.
:param str organization: the name of the organization the bundle should be regenerated for.
:param int request_id: the ID of the IIB build request
:param dict registry_auths: Provide the dockerconfig.json for authentication to private
registries, defaults to ``None``.
:raises IIBError: if the regenerate bundle image build fails.
"""
_cleanup()
set_request_state(request_id, 'in_progress', 'Resolving from_bundle_image')
with set_registry_auths(registry_auths):
from_bundle_image_resolved = get_resolved_image(from_bundle_image)
arches = get_image_arches(from_bundle_image_resolved)
if not arches:
raise IIBError(
'No arches were found in the resolved from_bundle_image '
f'{from_bundle_image_resolved}'
)
pinned_by_iib = yaml.load(
get_image_label(from_bundle_image_resolved, 'com.redhat.iib.pinned') or 'false'
)
arches_str = ', '.join(sorted(arches))
log.debug('Set to regenerate the bundle image for the following arches: %s', arches_str)
payload = {
'from_bundle_image_resolved': from_bundle_image_resolved,
'state': 'in_progress',
'state_reason': f'Regenerating the bundle image for the following arches: {arches_str}',
}
exc_msg = 'Failed setting the resolved "from_bundle_image" on the request'
update_request(request_id, payload, exc_msg=exc_msg)
# Pull the from_bundle_image to ensure steps later on don't fail due to registry timeouts
podman_pull(from_bundle_image_resolved)
with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir:
manifests_path = os.path.join(temp_dir, 'manifests')
_copy_files_from_image(from_bundle_image_resolved, '/manifests', manifests_path)
metadata_path = os.path.join(temp_dir, 'metadata')
_copy_files_from_image(from_bundle_image_resolved, '/metadata', metadata_path)
new_labels = _adjust_operator_bundle(
manifests_path, metadata_path, request_id, organization, pinned_by_iib
)
with open(os.path.join(temp_dir, 'Dockerfile'), 'w') as dockerfile:
dockerfile.write(
textwrap.dedent(
f"""\
FROM {from_bundle_image_resolved}
COPY ./manifests /manifests
COPY ./metadata /metadata
"""
)
)
for name, value in new_labels.items():
dockerfile.write(f'LABEL {name}={value}\n')
for arch in sorted(arches):
_build_image(temp_dir, 'Dockerfile', request_id, arch)
_push_image(request_id, arch)
set_request_state(request_id, 'in_progress', 'Creating the manifest list')
output_pull_spec = _create_and_push_manifest_list(request_id, arches, [])
conf = get_worker_config()
if conf['iib_index_image_output_registry']:
old_output_pull_spec = output_pull_spec
output_pull_spec = output_pull_spec.replace(
conf['iib_registry'], conf['iib_index_image_output_registry'], 1
)
log.info(
'Changed the bundle_image pull specification from %s to %s',
old_output_pull_spec,
output_pull_spec,
)
payload = {
'arches': list(arches),
'bundle_image': output_pull_spec,
'state': 'complete',
'state_reason': 'The request completed successfully',
}
update_request(request_id, payload, exc_msg='Failed setting the bundle image on the request')
def handle_merge_request(
source_from_index,
deprecation_list,
request_id,
binary_image=None,
target_index=None,
overwrite_target_index=False,
overwrite_target_index_token=None,
distribution_scope=None,
binary_image_config=None,
build_tags=None,
):
"""
Coordinate the work needed to merge old (N) index image with new (N+1) index image.
:param str source_from_index: pull specification to be used as the base for building the new
index image.
:param str target_index: pull specification of content stage index image for the
corresponding target index image.
:param list deprecation_list: list of deprecated bundles for the target index image.
:param int request_id: the ID of the IIB build request.
:param str binary_image: the pull specification of the container image where the opm binary
gets copied from.
:param bool overwrite_target_index: if True, overwrite the input ``target_index`` with
the built index image.
:param str overwrite_target_index_token: the token used for overwriting the input
``target_index`` image. This is required to use ``overwrite_target_index``.
The format of the token must be in the format "user:password".
:param str distribution_scope: the scope for distribution of the index image, defaults to
``None``.
:param build_tags: list of extra tag to use for intermetdiate index image
:raises IIBError: if the index image merge fails.
"""
_cleanup()
prebuild_info = prepare_request_for_build(
request_id,
RequestConfigMerge(
_binary_image=binary_image,
overwrite_target_index_token=overwrite_target_index_token,
source_from_index=source_from_index,
target_index=target_index,
distribution_scope=distribution_scope,
binary_image_config=binary_image_config,
),
)
_update_index_image_build_state(request_id, prebuild_info)
source_from_index_resolved = prebuild_info['source_from_index_resolved']
target_index_resolved = prebuild_info['target_index_resolved']
dockerfile_name = 'index.Dockerfile'
with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir:
with set_registry_token(overwrite_target_index_token, source_from_index):
source_fbc = is_image_fbc(source_from_index_resolved)
target_fbc = is_image_fbc(target_index_resolved)
# do not remove - logging requested by stakeholders
if source_fbc:
log.info("Processing source index image as File-Based Catalog image")
if target_fbc:
log.info("Processing target index image as File-Based Catalog image")
if source_fbc and not target_fbc:
err_msg = (
'Cannot merge source File-Based Catalog index image into target SQLite index image.'
)
log.error(err_msg)
raise IIBError(err_msg)
set_request_state(request_id, 'in_progress', 'Getting bundles present in the index images')
log.info('Getting bundles present in the source index image')
with set_registry_token(overwrite_target_index_token, source_from_index):
source_index_bundles, source_index_bundles_pull_spec = _get_present_bundles(
source_from_index_resolved, temp_dir
)
target_index_bundles = []
if target_index:
log.info('Getting bundles present in the target index image')
target_index_bundles, _ = _get_present_bundles(target_index_resolved, temp_dir)
arches = list(prebuild_info['arches'])
arch = sorted(arches)[0]
missing_bundles, invalid_version_bundles = _add_bundles_missing_in_source(
source_index_bundles,
target_index_bundles,
temp_dir,
prebuild_info['binary_image'],
source_from_index_resolved,
request_id,
arch,
prebuild_info['target_ocp_version'],
overwrite_target_index_token,
distribution_scope=prebuild_info['distribution_scope'],
)
missing_bundle_paths = [bundle['bundlePath'] for bundle in missing_bundles]
if missing_bundle_paths:
add_max_ocp_version_property(missing_bundle_paths, temp_dir)
set_request_state(request_id, 'in_progress', 'Deprecating bundles in the deprecation list')
log.info('Deprecating bundles in the deprecation list')
intermediate_bundles = missing_bundle_paths + source_index_bundles_pull_spec
deprecation_bundles = get_bundles_from_deprecation_list(
intermediate_bundles, deprecation_list
)
# We do not need to pass the invalid_version_bundles through the
# get_bundles_from_deprecation_list function because we already know
# they are present in the newly created index.
deprecation_bundles = deprecation_bundles + [
bundle['bundlePath'] for bundle in invalid_version_bundles
]
if deprecation_bundles:
intermediate_image_name = _get_external_arch_pull_spec(
request_id, arch, include_transport=False
)
# we can check if source index is FBC or not because intermediate_image
# will be always the same type because it is built
# from source index image in _add_bundles_missing_in_source()
if source_fbc:
deprecate_bundles_fbc(
bundles=deprecation_bundles,
base_dir=temp_dir,
binary_image=prebuild_info['binary_image'],
from_index=intermediate_image_name,
)
else:
# opm can only deprecate a bundle image on an existing index image. Build and
# push a temporary index image to satisfy this requirement. Any arch will do.
# NOTE: we cannot use local builds because opm commands fails,
# index image has to be pushed to registry
_build_image(temp_dir, 'index.Dockerfile', request_id, arch)
_push_image(request_id, arch)
deprecate_bundles(
bundles=deprecation_bundles,
base_dir=temp_dir,
binary_image=prebuild_info['binary_image'],
from_index=intermediate_image_name,
overwrite_target_index_token=overwrite_target_index_token,
)
if target_fbc:
index_db_file = os.path.join(temp_dir, get_worker_config()['temp_index_db_path'])
# make sure FBC is generated right before build
fbc_dir = opm_migrate(index_db=index_db_file, base_dir=temp_dir)
if not source_fbc:
# when source image is not FBC, but final image should be an FBC image
# we have to generate Dockerfile for FBC (with hidden index.db)
dockerfile_path = os.path.join(temp_dir, dockerfile_name)
if os.path.isfile(dockerfile_path):
log.info('Removing previously generated dockerfile.')
os.remove(dockerfile_path)
opm_generate_dockerfile(
fbc_dir=fbc_dir,
base_dir=temp_dir,
index_db=index_db_file,
binary_image=prebuild_info['binary_image'],
dockerfile_name=dockerfile_name,
)
_add_label_to_index(
'com.redhat.index.delivery.version',
prebuild_info['target_ocp_version'],
temp_dir,
dockerfile_name,
)
_add_label_to_index(
'com.redhat.index.delivery.distribution_scope',
prebuild_info['distribution_scope'],
temp_dir,
dockerfile_name,
)
for arch in sorted(prebuild_info['arches']):
_build_image(temp_dir, dockerfile_name, request_id, arch)
_push_image(request_id, arch)
# If the container-tool podman is used in the opm commands above, opm will create temporary
# files and directories without the write permission. This will cause the context manager
# to fail to delete these files. Adjust the file modes to avoid this error.
chmod_recursively(
temp_dir,
dir_mode=(stat.S_IRWXU | stat.S_IRWXG),
file_mode=(stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP),
)
output_pull_spec = _create_and_push_manifest_list(
request_id, prebuild_info['arches'], build_tags
)
_update_index_image_pull_spec(
output_pull_spec,
request_id,
prebuild_info['arches'],
target_index,
overwrite_target_index,
overwrite_target_index_token,
target_index_resolved,
)
set_request_state(
request_id, 'complete', 'The index image was successfully cleaned and updated.'
)
def handle_create_empty_index_request(from_index,
request_id,
binary_image=None,
labels=None,
binary_image_config=None):
"""Coordinate the the work needed to create the index image with labels.
:param str from_index: the pull specification of the container image containing the index that
the index image build will be based from.
:param int request_id: the ID of the IIB build request
:param str binary_image: the pull specification of the container image where the opm binary
gets copied from.
:param dict labels: the dict of labels required to be added to a new index image
:param dict binary_image_config: the dict of config required to identify the appropriate
``binary_image`` to use.
"""
_cleanup()
prebuild_info = prepare_request_for_build(
request_id,
RequestConfigCreateIndexImage(
_binary_image=binary_image,
from_index=from_index,
binary_image_config=binary_image_config,
),
)
from_index_resolved = prebuild_info['from_index_resolved']
prebuild_info['labels'] = labels
_update_index_image_build_state(request_id, prebuild_info)
with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir:
set_request_state(request_id, 'in_progress',
'Checking operators present in index image')
operators = _get_present_operators(from_index_resolved, temp_dir)
set_request_state(request_id, 'in_progress',
'Removing operators from index image')
_opm_index_rm(temp_dir, operators, prebuild_info['binary_image'],
from_index_resolved)
set_request_state(request_id, 'in_progress',
'Getting and updating labels for new index image')
iib_labels = {
'com.redhat.index.delivery.version':
prebuild_info['ocp_version'],
'com.redhat.index.delivery.distribution_scope':
prebuild_info['distribution_scope'],
}
iib_labels.update(labels)
for index_label, value in iib_labels.items():
_add_label_to_index(index_label, value, temp_dir,
'index.Dockerfile')
arches = prebuild_info['arches']
for arch in sorted(arches):
_build_image(temp_dir, 'index.Dockerfile', request_id, arch)
_push_image(request_id, arch)
set_request_state(request_id, 'in_progress', 'Creating the manifest list')
output_pull_spec = _create_and_push_manifest_list(request_id, arches)
_update_index_image_pull_spec(
output_pull_spec=output_pull_spec,
request_id=request_id,
arches=arches,
from_index=from_index,
resolved_prebuild_from_index=from_index_resolved,
)
set_request_state(request_id, 'complete',
'The empty index image was successfully created')
def test_create_and_push_manifest_list(mock_open, mock_run_cmd, mock_td,
tmp_path):
mock_td.return_value.__enter__.return_value = tmp_path
mock_run_cmd.side_effect = [
IIBError('Manifest list not found locally.'),
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
]
output = []
mock_open().__enter__().write.side_effect = lambda x: output.append(x)
build._create_and_push_manifest_list(3, {'amd64', 's390x'},
['extra_build_tag1'])
expected_calls = [
mock.call(
['buildah', 'manifest', 'rm', 'registry:8443/iib-build:3'],
exc_msg=
('Failed to remove local manifest list. registry:8443/iib-build:3 does not exist'
),
),
mock.call(
['buildah', 'manifest', 'create', 'registry:8443/iib-build:3'],
exc_msg=
'Failed to create the manifest list locally: registry:8443/iib-build:3',
),
mock.call(
[
'buildah',
'manifest',
'add',
'registry:8443/iib-build:3',
'docker://registry:8443/iib-build:3-amd64',
],
exc_msg=('Failed to add docker://registry:8443/iib-build:3-amd64'
' to the local manifest list: registry:8443/iib-build:3'),
),
mock.call(
[
'buildah',
'manifest',
'add',
'registry:8443/iib-build:3',
'docker://registry:8443/iib-build:3-s390x',
],
exc_msg=('Failed to add docker://registry:8443/iib-build:3-s390x'
' to the local manifest list: registry:8443/iib-build:3'),
),
mock.call(
[
'buildah',
'manifest',
'push',
'--all',
'--format',
'v2s2',
'registry:8443/iib-build:3',
'docker://registry:8443/iib-build:3',
],
exc_msg=
'Failed to push the manifest list to registry:8443/iib-build:3',
),
mock.call(
[
'buildah', 'manifest', 'rm',
'registry:8443/iib-build:extra_build_tag1'
],
exc_msg='Failed to remove local manifest list. '
'registry:8443/iib-build:extra_build_tag1 does not exist',
),
mock.call(
[
'buildah', 'manifest', 'create',
'registry:8443/iib-build:extra_build_tag1'
],
exc_msg='Failed to create the manifest list locally: '
'registry:8443/iib-build:extra_build_tag1',
),
mock.call(
[
'buildah',
'manifest',
'add',
'registry:8443/iib-build:extra_build_tag1',
'docker://registry:8443/iib-build:3-amd64',
],
exc_msg=
('Failed to add docker://registry:8443/iib-build:3-amd64'
' to the local manifest list: registry:8443/iib-build:extra_build_tag1'
),
),
mock.call(
[
'buildah',
'manifest',
'add',
'registry:8443/iib-build:extra_build_tag1',
'docker://registry:8443/iib-build:3-s390x',
],
exc_msg=
('Failed to add docker://registry:8443/iib-build:3-s390x'
' to the local manifest list: registry:8443/iib-build:extra_build_tag1'
),
),
mock.call(
[
'buildah',
'manifest',
'push',
'--all',
'--format',
'v2s2',
'registry:8443/iib-build:extra_build_tag1',
'docker://registry:8443/iib-build:extra_build_tag1',
],
exc_msg=
'Failed to push the manifest list to registry:8443/iib-build:extra_build_tag1',
),
]
assert mock_run_cmd.call_args_list == expected_calls
