def test_cleanup(mock_rdc, mock_run_cmd):
build._cleanup()
mock_run_cmd.assert_called_once()
rmi_args = mock_run_cmd.call_args[0][0]
assert rmi_args[0:2] == ['podman', 'rmi']
mock_rdc.assert_called_once_with()
def handle_create_empty_index_request(
from_index,
request_id,
output_fbc=False,
binary_image=None,
labels=None,
binary_image_config=None,
):
"""Coordinate the the work needed to create the index image with labels.
:param str from_index: the pull specification of the container image containing the index that
the index image build will be based from.
:param int request_id: the ID of the IIB build request
:param bool output_fbc: specifies whether a File-based Catalog index image should be created
:param str binary_image: the pull specification of the container image where the opm binary
gets copied from.
:param dict labels: the dict of labels required to be added to a new index image
:param dict binary_image_config: the dict of config required to identify the appropriate
``binary_image`` to use.
"""
_cleanup()
prebuild_info = prepare_request_for_build(
request_id,
RequestConfigCreateIndexImage(
_binary_image=binary_image,
from_index=from_index,
binary_image_config=binary_image_config,
),
)
from_index_resolved = prebuild_info['from_index_resolved']
prebuild_info['labels'] = labels
if not output_fbc and is_image_fbc(from_index_resolved):
log.debug('%s is FBC index image', from_index_resolved)
err_msg = 'Cannot create SQLite index image from File-Based Catalog index image'
log.error(err_msg)
raise IIBError(err_msg)
_update_index_image_build_state(request_id, prebuild_info)
with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir:
set_request_state(request_id, 'in_progress', 'Checking operators present in index image')
operators = _get_present_operators(from_index_resolved, temp_dir)
# if output_fbc parameter is true, create an empty FBC index image
# else create empty SQLite index image
if output_fbc:
log.debug('Creating empty FBC index image from %s', from_index)
opm_create_empty_fbc(
request_id=request_id,
temp_dir=temp_dir,
from_index_resolved=from_index_resolved,
from_index=from_index,
binary_image=prebuild_info['binary_image'],
operators=operators,
)
else:
set_request_state(request_id, 'in_progress', 'Removing operators from index image')
_opm_index_rm(temp_dir, operators, prebuild_info['binary_image'], from_index_resolved)
set_request_state(
request_id, 'in_progress', 'Getting and updating labels for new index image'
)
iib_labels = {
'com.redhat.index.delivery.version': prebuild_info['ocp_version'],
'com.redhat.index.delivery.distribution_scope': prebuild_info['distribution_scope'],
}
if labels:
iib_labels.update(labels)
for index_label, value in iib_labels.items():
_add_label_to_index(index_label, value, temp_dir, 'index.Dockerfile')
arches = prebuild_info['arches']
for arch in sorted(arches):
_build_image(temp_dir, 'index.Dockerfile', request_id, arch)
_push_image(request_id, arch)
set_request_state(request_id, 'in_progress', 'Creating the manifest list')
output_pull_spec = _create_and_push_manifest_list(request_id, arches, [])
_update_index_image_pull_spec(
output_pull_spec=output_pull_spec,
request_id=request_id,
arches=arches,
from_index=from_index,
resolved_prebuild_from_index=from_index_resolved,
)
set_request_state(request_id, 'complete', 'The empty index image was successfully created')
def handle_merge_request(
source_from_index,
deprecation_list,
request_id,
binary_image=None,
target_index=None,
overwrite_target_index=False,
overwrite_target_index_token=None,
distribution_scope=None,
binary_image_config=None,
):
"""
Coordinate the work needed to merge old (N) index image with new (N+1) index image.
:param str source_from_index: pull specification to be used as the base for building the new
index image.
:param str target_index: pull specification of content stage index image for the
corresponding target index image.
:param list deprecation_list: list of deprecated bundles for the target index image.
:param int request_id: the ID of the IIB build request.
:param str binary_image: the pull specification of the container image where the opm binary
gets copied from.
:param bool overwrite_target_index: if True, overwrite the input ``target_index`` with
the built index image.
:param str overwrite_target_index_token: the token used for overwriting the input
``target_index`` image. This is required for non-privileged users to use
``overwrite_target_index``. The format of the token must be in the format "user:password".
:param str distribution_scope: the scope for distribution of the index image, defaults to
``None``.
:raises IIBError: if the index image merge fails.
"""
_cleanup()
prebuild_info = _prepare_request_for_build(
request_id,
binary_image,
overwrite_from_index_token=overwrite_target_index_token,
source_from_index=source_from_index,
target_index=target_index,
distribution_scope=distribution_scope,
binary_image_config=binary_image_config,
)
_update_index_image_build_state(request_id, prebuild_info)
with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir:
set_request_state(request_id, 'in_progress',
'Getting bundles present in the index images')
log.info('Getting bundles present in the source index image')
source_index_bundles = _get_present_bundles(source_from_index,
temp_dir)
target_index_bundles = []
if target_index:
log.info('Getting bundles present in the target index image')
target_index_bundles = _get_present_bundles(target_index, temp_dir)
arches = list(prebuild_info['arches'])
arch = 'amd64' if 'amd64' in arches else arches[0]
missing_bundles = _add_bundles_missing_in_source(
source_index_bundles,
target_index_bundles,
temp_dir,
prebuild_info['binary_image'],
source_from_index,
request_id,
arch,
prebuild_info['target_ocp_version'],
overwrite_target_index_token,
distribution_scope=prebuild_info['distribution_scope'],
)
set_request_state(request_id, 'in_progress',
'Deprecating bundles in the deprecation list')
log.info('Deprecating bundles in the deprecation list')
intermediate_bundles = source_index_bundles + missing_bundles
deprecate_bundles = _get_bundles_from_deprecation_list(
intermediate_bundles, deprecation_list)
intermediate_image_name = _get_external_arch_pull_spec(
request_id, arch, include_transport=False)
if deprecate_bundles:
_deprecate_bundles(
deprecate_bundles,
temp_dir,
prebuild_info['binary_image'],
intermediate_image_name,
overwrite_target_index_token,
)
for arch in sorted(prebuild_info['arches']):
_build_image(temp_dir, 'index.Dockerfile', request_id, arch)
_push_image(request_id, arch)
output_pull_spec = _create_and_push_manifest_list(request_id,
prebuild_info['arches'])
_update_index_image_pull_spec(
output_pull_spec,
request_id,
prebuild_info['arches'],
target_index,
overwrite_target_index,
overwrite_target_index_token,
prebuild_info['target_index_resolved'],
)
set_request_state(request_id, 'complete',
'The index image was successfully cleaned and updated.')
def handle_merge_request(
source_from_index,
deprecation_list,
request_id,
binary_image=None,
target_index=None,
overwrite_target_index=False,
overwrite_target_index_token=None,
distribution_scope=None,
binary_image_config=None,
):
"""
Coordinate the work needed to merge old (N) index image with new (N+1) index image.
:param str source_from_index: pull specification to be used as the base for building the new
index image.
:param str target_index: pull specification of content stage index image for the
corresponding target index image.
:param list deprecation_list: list of deprecated bundles for the target index image.
:param int request_id: the ID of the IIB build request.
:param str binary_image: the pull specification of the container image where the opm binary
gets copied from.
:param bool overwrite_target_index: if True, overwrite the input ``target_index`` with
the built index image.
:param str overwrite_target_index_token: the token used for overwriting the input
``target_index`` image. This is required to use ``overwrite_target_index``.
The format of the token must be in the format "user:password".
:param str distribution_scope: the scope for distribution of the index image, defaults to
``None``.
:raises IIBError: if the index image merge fails.
"""
_cleanup()
prebuild_info = prepare_request_for_build(
request_id,
RequestConfigMerge(
_binary_image=binary_image,
overwrite_target_index_token=overwrite_target_index_token,
source_from_index=source_from_index,
target_index=target_index,
distribution_scope=distribution_scope,
binary_image_config=binary_image_config,
),
)
_update_index_image_build_state(request_id, prebuild_info)
source_from_index_resolved = prebuild_info['source_from_index_resolved']
target_index_resolved = prebuild_info['target_index_resolved']
with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir:
set_request_state(request_id, 'in_progress',
'Getting bundles present in the index images')
log.info('Getting bundles present in the source index image')
with set_registry_token(overwrite_target_index_token,
source_from_index):
source_index_bundles, source_index_bundles_pull_spec = _get_present_bundles(
source_from_index_resolved, temp_dir)
target_index_bundles = []
if target_index:
log.info('Getting bundles present in the target index image')
target_index_bundles, _ = _get_present_bundles(
target_index_resolved, temp_dir)
arches = list(prebuild_info['arches'])
arch = 'amd64' if 'amd64' in arches else arches[0]
missing_bundles, invalid_version_bundles = _add_bundles_missing_in_source(
source_index_bundles,
target_index_bundles,
temp_dir,
prebuild_info['binary_image'],
source_from_index_resolved,
request_id,
arch,
prebuild_info['target_ocp_version'],
overwrite_target_index_token,
distribution_scope=prebuild_info['distribution_scope'],
)
set_request_state(request_id, 'in_progress',
'Deprecating bundles in the deprecation list')
log.info('Deprecating bundles in the deprecation list')
intermediate_bundles = [
bundle['bundlePath'] for bundle in missing_bundles
] + source_index_bundles_pull_spec
deprecation_bundles = get_bundles_from_deprecation_list(
intermediate_bundles, deprecation_list)
# We do not need to pass the invalid_version_bundles through the
# get_bundles_from_deprecation_list function because we already know
# they are present in the newly created index.
deprecation_bundles = deprecation_bundles + [
bundle['bundlePath'] for bundle in invalid_version_bundles
]
intermediate_image_name = _get_external_arch_pull_spec(
request_id, arch, include_transport=False)
if deprecation_bundles:
deprecate_bundles(
deprecation_bundles,
temp_dir,
prebuild_info['binary_image'],
intermediate_image_name,
overwrite_target_index_token,
)
for arch in sorted(prebuild_info['arches']):
_build_image(temp_dir, 'index.Dockerfile', request_id, arch)
_push_image(request_id, arch)
# If the container-tool podman is used in the opm commands above, opm will create temporary
# files and directories without the write permission. This will cause the context manager
# to fail to delete these files. Adjust the file modes to avoid this error.
chmod_recursively(
temp_dir,
dir_mode=(stat.S_IRWXU | stat.S_IRWXG),
file_mode=(stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP
| stat.S_IWGRP),
)
output_pull_spec = _create_and_push_manifest_list(request_id,
prebuild_info['arches'])
_update_index_image_pull_spec(
output_pull_spec,
request_id,
prebuild_info['arches'],
target_index,
overwrite_target_index,
overwrite_target_index_token,
target_index_resolved,
)
set_request_state(request_id, 'complete',
'The index image was successfully cleaned and updated.')
def handle_regenerate_bundle_request(
from_bundle_image, organization, request_id, registry_auths=None
):
"""
Coordinate the work needed to regenerate the operator bundle image.
:param str from_bundle_image: the pull specification of the bundle image to be regenerated.
:param str organization: the name of the organization the bundle should be regenerated for.
:param int request_id: the ID of the IIB build request
:param dict registry_auths: Provide the dockerconfig.json for authentication to private
registries, defaults to ``None``.
:raises IIBError: if the regenerate bundle image build fails.
"""
_cleanup()
set_request_state(request_id, 'in_progress', 'Resolving from_bundle_image')
with set_registry_auths(registry_auths):
from_bundle_image_resolved = get_resolved_image(from_bundle_image)
arches = get_image_arches(from_bundle_image_resolved)
if not arches:
raise IIBError(
'No arches were found in the resolved from_bundle_image '
f'{from_bundle_image_resolved}'
)
pinned_by_iib = yaml.load(
get_image_label(from_bundle_image_resolved, 'com.redhat.iib.pinned') or 'false'
)
arches_str = ', '.join(sorted(arches))
log.debug('Set to regenerate the bundle image for the following arches: %s', arches_str)
payload = {
'from_bundle_image_resolved': from_bundle_image_resolved,
'state': 'in_progress',
'state_reason': f'Regenerating the bundle image for the following arches: {arches_str}',
}
exc_msg = 'Failed setting the resolved "from_bundle_image" on the request'
update_request(request_id, payload, exc_msg=exc_msg)
# Pull the from_bundle_image to ensure steps later on don't fail due to registry timeouts
podman_pull(from_bundle_image_resolved)
with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir:
manifests_path = os.path.join(temp_dir, 'manifests')
_copy_files_from_image(from_bundle_image_resolved, '/manifests', manifests_path)
metadata_path = os.path.join(temp_dir, 'metadata')
_copy_files_from_image(from_bundle_image_resolved, '/metadata', metadata_path)
new_labels = _adjust_operator_bundle(
manifests_path, metadata_path, request_id, organization, pinned_by_iib
)
with open(os.path.join(temp_dir, 'Dockerfile'), 'w') as dockerfile:
dockerfile.write(
textwrap.dedent(
f"""\
FROM {from_bundle_image_resolved}
COPY ./manifests /manifests
COPY ./metadata /metadata
"""
)
)
for name, value in new_labels.items():
dockerfile.write(f'LABEL {name}={value}\n')
for arch in sorted(arches):
_build_image(temp_dir, 'Dockerfile', request_id, arch)
_push_image(request_id, arch)
set_request_state(request_id, 'in_progress', 'Creating the manifest list')
output_pull_spec = _create_and_push_manifest_list(request_id, arches, [])
conf = get_worker_config()
if conf['iib_index_image_output_registry']:
old_output_pull_spec = output_pull_spec
output_pull_spec = output_pull_spec.replace(
conf['iib_registry'], conf['iib_index_image_output_registry'], 1
)
log.info(
'Changed the bundle_image pull specification from %s to %s',
old_output_pull_spec,
output_pull_spec,
)
payload = {
'arches': list(arches),
'bundle_image': output_pull_spec,
'state': 'complete',
'state_reason': 'The request completed successfully',
}
update_request(request_id, payload, exc_msg='Failed setting the bundle image on the request')
def handle_merge_request(
source_from_index,
deprecation_list,
request_id,
binary_image=None,
target_index=None,
overwrite_target_index=False,
overwrite_target_index_token=None,
distribution_scope=None,
binary_image_config=None,
build_tags=None,
):
"""
Coordinate the work needed to merge old (N) index image with new (N+1) index image.
:param str source_from_index: pull specification to be used as the base for building the new
index image.
:param str target_index: pull specification of content stage index image for the
corresponding target index image.
:param list deprecation_list: list of deprecated bundles for the target index image.
:param int request_id: the ID of the IIB build request.
:param str binary_image: the pull specification of the container image where the opm binary
gets copied from.
:param bool overwrite_target_index: if True, overwrite the input ``target_index`` with
the built index image.
:param str overwrite_target_index_token: the token used for overwriting the input
``target_index`` image. This is required to use ``overwrite_target_index``.
The format of the token must be in the format "user:password".
:param str distribution_scope: the scope for distribution of the index image, defaults to
``None``.
:param build_tags: list of extra tag to use for intermetdiate index image
:raises IIBError: if the index image merge fails.
"""
_cleanup()
prebuild_info = prepare_request_for_build(
request_id,
RequestConfigMerge(
_binary_image=binary_image,
overwrite_target_index_token=overwrite_target_index_token,
source_from_index=source_from_index,
target_index=target_index,
distribution_scope=distribution_scope,
binary_image_config=binary_image_config,
),
)
_update_index_image_build_state(request_id, prebuild_info)
source_from_index_resolved = prebuild_info['source_from_index_resolved']
target_index_resolved = prebuild_info['target_index_resolved']
dockerfile_name = 'index.Dockerfile'
with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir:
with set_registry_token(overwrite_target_index_token, source_from_index):
source_fbc = is_image_fbc(source_from_index_resolved)
target_fbc = is_image_fbc(target_index_resolved)
# do not remove - logging requested by stakeholders
if source_fbc:
log.info("Processing source index image as File-Based Catalog image")
if target_fbc:
log.info("Processing target index image as File-Based Catalog image")
if source_fbc and not target_fbc:
err_msg = (
'Cannot merge source File-Based Catalog index image into target SQLite index image.'
)
log.error(err_msg)
raise IIBError(err_msg)
set_request_state(request_id, 'in_progress', 'Getting bundles present in the index images')
log.info('Getting bundles present in the source index image')
with set_registry_token(overwrite_target_index_token, source_from_index):
source_index_bundles, source_index_bundles_pull_spec = _get_present_bundles(
source_from_index_resolved, temp_dir
)
target_index_bundles = []
if target_index:
log.info('Getting bundles present in the target index image')
target_index_bundles, _ = _get_present_bundles(target_index_resolved, temp_dir)
arches = list(prebuild_info['arches'])
arch = sorted(arches)[0]
missing_bundles, invalid_version_bundles = _add_bundles_missing_in_source(
source_index_bundles,
target_index_bundles,
temp_dir,
prebuild_info['binary_image'],
source_from_index_resolved,
request_id,
arch,
prebuild_info['target_ocp_version'],
overwrite_target_index_token,
distribution_scope=prebuild_info['distribution_scope'],
)
missing_bundle_paths = [bundle['bundlePath'] for bundle in missing_bundles]
if missing_bundle_paths:
add_max_ocp_version_property(missing_bundle_paths, temp_dir)
set_request_state(request_id, 'in_progress', 'Deprecating bundles in the deprecation list')
log.info('Deprecating bundles in the deprecation list')
intermediate_bundles = missing_bundle_paths + source_index_bundles_pull_spec
deprecation_bundles = get_bundles_from_deprecation_list(
intermediate_bundles, deprecation_list
)
# We do not need to pass the invalid_version_bundles through the
# get_bundles_from_deprecation_list function because we already know
# they are present in the newly created index.
deprecation_bundles = deprecation_bundles + [
bundle['bundlePath'] for bundle in invalid_version_bundles
]
if deprecation_bundles:
intermediate_image_name = _get_external_arch_pull_spec(
request_id, arch, include_transport=False
)
# we can check if source index is FBC or not because intermediate_image
# will be always the same type because it is built
# from source index image in _add_bundles_missing_in_source()
if source_fbc:
deprecate_bundles_fbc(
bundles=deprecation_bundles,
base_dir=temp_dir,
binary_image=prebuild_info['binary_image'],
from_index=intermediate_image_name,
)
else:
# opm can only deprecate a bundle image on an existing index image. Build and
# push a temporary index image to satisfy this requirement. Any arch will do.
# NOTE: we cannot use local builds because opm commands fails,
# index image has to be pushed to registry
_build_image(temp_dir, 'index.Dockerfile', request_id, arch)
_push_image(request_id, arch)
deprecate_bundles(
bundles=deprecation_bundles,
base_dir=temp_dir,
binary_image=prebuild_info['binary_image'],
from_index=intermediate_image_name,
overwrite_target_index_token=overwrite_target_index_token,
)
if target_fbc:
index_db_file = os.path.join(temp_dir, get_worker_config()['temp_index_db_path'])
# make sure FBC is generated right before build
fbc_dir = opm_migrate(index_db=index_db_file, base_dir=temp_dir)
if not source_fbc:
# when source image is not FBC, but final image should be an FBC image
# we have to generate Dockerfile for FBC (with hidden index.db)
dockerfile_path = os.path.join(temp_dir, dockerfile_name)
if os.path.isfile(dockerfile_path):
log.info('Removing previously generated dockerfile.')
os.remove(dockerfile_path)
opm_generate_dockerfile(
fbc_dir=fbc_dir,
base_dir=temp_dir,
index_db=index_db_file,
binary_image=prebuild_info['binary_image'],
dockerfile_name=dockerfile_name,
)
_add_label_to_index(
'com.redhat.index.delivery.version',
prebuild_info['target_ocp_version'],
temp_dir,
dockerfile_name,
)
_add_label_to_index(
'com.redhat.index.delivery.distribution_scope',
prebuild_info['distribution_scope'],
temp_dir,
dockerfile_name,
)
for arch in sorted(prebuild_info['arches']):
_build_image(temp_dir, dockerfile_name, request_id, arch)
_push_image(request_id, arch)
# If the container-tool podman is used in the opm commands above, opm will create temporary
# files and directories without the write permission. This will cause the context manager
# to fail to delete these files. Adjust the file modes to avoid this error.
chmod_recursively(
temp_dir,
dir_mode=(stat.S_IRWXU | stat.S_IRWXG),
file_mode=(stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP),
)
output_pull_spec = _create_and_push_manifest_list(
request_id, prebuild_info['arches'], build_tags
)
_update_index_image_pull_spec(
output_pull_spec,
request_id,
prebuild_info['arches'],
target_index,
overwrite_target_index,
overwrite_target_index_token,
target_index_resolved,
)
set_request_state(
request_id, 'complete', 'The index image was successfully cleaned and updated.'
)
def handle_create_empty_index_request(from_index,
request_id,
binary_image=None,
labels=None,
binary_image_config=None):
"""Coordinate the the work needed to create the index image with labels.
:param str from_index: the pull specification of the container image containing the index that
the index image build will be based from.
:param int request_id: the ID of the IIB build request
:param str binary_image: the pull specification of the container image where the opm binary
gets copied from.
:param dict labels: the dict of labels required to be added to a new index image
:param dict binary_image_config: the dict of config required to identify the appropriate
``binary_image`` to use.
"""
_cleanup()
prebuild_info = prepare_request_for_build(
request_id,
RequestConfigCreateIndexImage(
_binary_image=binary_image,
from_index=from_index,
binary_image_config=binary_image_config,
),
)
from_index_resolved = prebuild_info['from_index_resolved']
prebuild_info['labels'] = labels
_update_index_image_build_state(request_id, prebuild_info)
with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir:
set_request_state(request_id, 'in_progress',
'Checking operators present in index image')
operators = _get_present_operators(from_index_resolved, temp_dir)
set_request_state(request_id, 'in_progress',
'Removing operators from index image')
_opm_index_rm(temp_dir, operators, prebuild_info['binary_image'],
from_index_resolved)
set_request_state(request_id, 'in_progress',
'Getting and updating labels for new index image')
iib_labels = {
'com.redhat.index.delivery.version':
prebuild_info['ocp_version'],
'com.redhat.index.delivery.distribution_scope':
prebuild_info['distribution_scope'],
}
iib_labels.update(labels)
for index_label, value in iib_labels.items():
_add_label_to_index(index_label, value, temp_dir,
'index.Dockerfile')
arches = prebuild_info['arches']
for arch in sorted(arches):
_build_image(temp_dir, 'index.Dockerfile', request_id, arch)
_push_image(request_id, arch)
set_request_state(request_id, 'in_progress', 'Creating the manifest list')
output_pull_spec = _create_and_push_manifest_list(request_id, arches)
_update_index_image_pull_spec(
output_pull_spec=output_pull_spec,
request_id=request_id,
arches=arches,
from_index=from_index,
resolved_prebuild_from_index=from_index_resolved,
)
set_request_state(request_id, 'complete',
'The empty index image was successfully created')
