def test_handle_rm_request(
mock_iifbc,
mock_alti,
mock_uiips,
mock_capml,
mock_srs,
mock_srs2,
mock_vii,
mock_pi,
mock_bi,
mock_uiibs,
mock_oir,
mock_prfb,
mock_cleanup,
binary_image,
):
arches = {'amd64', 's390x'}
mock_iifbc.return_value = False
mock_prfb.return_value = {
'arches': arches,
'binary_image': binary_image,
'binary_image_resolved': 'binary-image@sha256:abcdef',
'from_index_resolved': 'from-index@sha256:bcdefg',
'ocp_version': 'v4.6',
'distribution_scope': 'PROD',
}
binary_image_config = {'prod': {'v4.6': 'some_image'}}
build.handle_rm_request(
['some-operator'],
3,
'from-index:latest',
binary_image,
binary_image_config=binary_image_config,
)
mock_cleanup.assert_called_once()
mock_prfb.assert_called_once_with(
3,
RequestConfigAddRm(
_binary_image=binary_image,
from_index='from-index:latest',
overwrite_from_index_token=None,
add_arches=None,
binary_image_config=binary_image_config,
distribution_scope=None,
),
)
mock_oir.assert_called_once()
assert mock_alti.call_count == 2
assert mock_bi.call_count == len(arches)
assert mock_pi.call_count == len(arches)
mock_vii.assert_not_called()
assert mock_srs.call_count == 2
mock_capml.assert_called_once_with(3, {'s390x', 'amd64'}, None)
mock_uiips.assert_called_once()
assert mock_srs.call_args[0][1] == 'complete'
def test_handle_add_request_check_index_label_behavior(
mock_srt,
mock_gpb,
mock_alti,
mock_grb,
mock_gb,
mock_capml,
mock_srs,
mock_uiips,
mock_vii,
mock_pi,
mock_bi,
mock_oia,
mock_uiibs,
mock_prfb,
mock_vl,
mock_cleanup,
mock_ugrb,
mock_dep_b,
mock_sir,
mock_run_cmd,
):
arches = {'amd64', 's390x'}
binary_image_config = {'prod': {'v4.5': 'some_image'}}
mock_prfb.return_value = {
'arches': arches,
'binary_image': 'binary-image:latest',
'binary_image_resolved': 'binary-image@sha256:abcdef',
'from_index_resolved': 'from-index@sha256:bcdefg',
'ocp_version': 'v4.5',
'distribution_scope': 'stage',
}
mock_grb.return_value = ['some-bundle@sha', 'some-deprecation-bundle@sha']
output_pull_spec = 'quay.io/namespace/some-image:3'
mock_capml.return_value = output_pull_spec
mock_gpb.return_value = [{
'bundlePath': 'random_bundle@sha'
}], ['random_bundle@sha']
bundles = ['some-bundle:2.3-1', 'some-deprecation-bundle:1.1-1']
cnr_token = 'token'
organization = 'org'
greenwave_config = {'some_key': 'other_value'}
mock_ugrb.return_value = [
'random_bundle@sha', 'some-deprecation-bundle@sha'
]
deprecation_list = ['random_bundle@sha', 'some-deprecation-bundle@sha']
# Assume default labels are set on the index
label_state = {'LABEL_SET': 'default_labels_set'}
def _add_label_to_index(*args):
# Set the labels in the index again making sure they were wiped out
if label_state['LABEL_SET'] == 'wiping_out_labels':
label_state['LABEL_SET'] = 'setting_label_in_add_label_to_index'
mock_alti.side_effect = _add_label_to_index
def deprecate_bundles_mock(*args, **kwargs):
# Wipe out the labels on the index
label_state['LABEL_SET'] = 'wiping_out_labels'
mock_dep_b.side_effect = deprecate_bundles_mock
port = 0
my_mock = mock.MagicMock()
mock_sir.return_value = (port, my_mock)
mock_run_cmd.return_value = '{"packageName": "package1", "version": "v1.0", \
"bundlePath": "bundle1"\n}'
build.handle_add_request(
bundles,
3,
'binary-image:latest',
'from-index:latest',
['s390x'],
cnr_token,
organization,
True,
False,
None,
None,
greenwave_config,
binary_image_config=binary_image_config,
deprecation_list=deprecation_list,
)
mock_sir.assert_called_once()
mock_run_cmd.assert_called_once()
mock_run_cmd.assert_has_calls([
mock.call(
[
'grpcurl', '-plaintext', f'localhost:{port}',
'api.Registry/ListBundles'
],
exc_msg=mock.ANY,
),
])
mock_cleanup.assert_called_once()
mock_vl.assert_called_once()
mock_prfb.assert_called_once_with(
3,
RequestConfigAddRm(
_binary_image='binary-image:latest',
from_index='from-index:latest',
overwrite_from_index_token=None,
add_arches=['s390x'],
bundles=['some-bundle:2.3-1', 'some-deprecation-bundle:1.1-1'],
distribution_scope=None,
binary_image_config=binary_image_config,
),
)
mock_dep_b.assert_called_once_with(
['random_bundle@sha', 'some-deprecation-bundle@sha'],
mock.ANY,
'binary-image:latest',
'containers-storage:localhost/iib-build:3-amd64',
None,
container_tool='podman',
)
# Assert the labels are set again once they were wiped out
assert label_state['LABEL_SET'] == 'setting_label_in_add_label_to_index'
assert mock_alti.call_count == 2
def test_handle_add_request(
mock_srt,
mock_gpb,
mock_alti,
mock_grb,
mock_gb,
mock_capml,
mock_srs,
mock_srs2,
mock_uiips,
mock_vii,
mock_pi,
mock_bi,
mock_oia,
mock_uiibs,
mock_prfb,
mock_vl,
mock_cleanup,
mock_ugrb,
mock_dep_b,
mock_sir,
mock_run_cmd,
force_backport,
binary_image,
distribution_scope,
deprecate_bundles,
):
arches = {'amd64', 's390x'}
binary_image_config = {'prod': {'v4.5': 'some_image'}}
mock_prfb.return_value = {
'arches': arches,
'binary_image': binary_image or 'some_image',
'binary_image_resolved': 'binary-image@sha256:abcdef',
'from_index_resolved': 'from-index@sha256:bcdefg',
'ocp_version': 'v4.5',
'distribution_scope': distribution_scope,
}
mock_grb.return_value = ['some-bundle@sha', 'some-deprecation-bundle@sha']
output_pull_spec = 'quay.io/namespace/some-image:3'
mock_capml.return_value = output_pull_spec
mock_gpb.return_value = [{
'bundlePath': 'random_bundle@sha'
}], ['random_bundle@sha']
bundles = ['some-bundle:2.3-1', 'some-deprecation-bundle:1.1-1']
cnr_token = 'token'
organization = 'org'
greenwave_config = {'some_key': 'other_value'}
deprecation_list = []
if deprecate_bundles:
mock_ugrb.return_value = [
'random_bundle@sha', 'some-deprecation-bundle@sha'
]
deprecation_list = ['random_bundle@sha', 'some-deprecation-bundle@sha']
# Simulate opm's behavior of creating files that cannot be deleted
def side_effect(_, temp_dir, *args, **kwargs):
read_only_dir = os.path.join(temp_dir, 'read-only-dir')
os.mkdir(read_only_dir)
with open(os.path.join(read_only_dir, 'read-only-file'), 'w') as f:
os.chmod(f.fileno(), stat.S_IRUSR | stat.S_IRGRP)
# Make the dir read-only *after* populating it
os.chmod(read_only_dir, mode=stat.S_IRUSR | stat.S_IRGRP)
mock_dep_b.side_effect = side_effect
port = 0
my_mock = mock.MagicMock()
mock_sir.return_value = (port, my_mock)
mock_run_cmd.return_value = '{"packageName": "package1", "version": "v1.0", \
"bundlePath": "bundle1"\n}'
build.handle_add_request(
bundles,
3,
binary_image,
'from-index:latest',
['s390x'],
cnr_token,
organization,
force_backport,
False,
None,
None,
greenwave_config,
binary_image_config=binary_image_config,
deprecation_list=deprecation_list,
)
mock_sir.assert_called_once()
mock_run_cmd.assert_called_once()
mock_run_cmd.assert_has_calls([
mock.call(
[
'grpcurl', '-plaintext', f'localhost:{port}',
'api.Registry/ListBundles'
],
exc_msg=mock.ANY,
),
])
mock_cleanup.assert_called_once()
mock_vl.assert_called_once()
mock_prfb.assert_called_once_with(
3,
RequestConfigAddRm(
_binary_image=binary_image,
from_index='from-index:latest',
overwrite_from_index_token=None,
add_arches=['s390x'],
bundles=['some-bundle:2.3-1', 'some-deprecation-bundle:1.1-1'],
distribution_scope=None,
binary_image_config=binary_image_config,
),
)
mock_gb.assert_called_once()
assert 2 == mock_alti.call_count
mock_oia.assert_called_once()
if distribution_scope in ['dev', 'stage']:
assert mock_oia.call_args[0][5]
else:
assert not mock_oia.call_args[0][5]
mock_srt.assert_called_once()
if deprecate_bundles:
# Take into account the temporarily created index image
assert mock_bi.call_count == len(arches) + 1
else:
assert mock_bi.call_count == len(arches)
assert mock_pi.call_count == len(arches)
mock_uiips.assert_called_once()
mock_vii.assert_not_called()
mock_capml.assert_called_once()
assert mock_srs.call_count == 4
if deprecate_bundles:
mock_dep_b.assert_called_once_with(
['random_bundle@sha', 'some-deprecation-bundle@sha'],
mock.ANY,
binary_image or 'some_image',
'containers-storage:localhost/iib-build:3-amd64',
None,
container_tool='podman',
)
else:
mock_dep_b.assert_not_called()
def handle_rm_request(
operators,
request_id,
from_index,
binary_image=None,
add_arches=None,
overwrite_from_index=False,
overwrite_from_index_token=None,
distribution_scope=None,
binary_image_config=None,
):
"""
Coordinate the work needed to remove the input operators and rebuild the index image.
:param list operators: a list of strings representing the name of the operators to
remove from the index image.
:param int request_id: the ID of the IIB build request
:param str from_index: the pull specification of the container image containing the index that
the index image build will be based from.
:param str binary_image: the pull specification of the container image where the opm binary
gets copied from.
:param list add_arches: the list of arches to build in addition to the arches ``from_index`` is
currently built for.
:param bool overwrite_from_index: if True, overwrite the input ``from_index`` with the built
index image.
:param str overwrite_from_index_token: the token used for overwriting the input
``from_index`` image. This is required to use ``overwrite_from_index``.
The format of the token must be in the format "user:password".
:param str distribution_scope: the scope for distribution of the index image, defaults to
``None``.
:param dict binary_image_config: the dict of config required to identify the appropriate
``binary_image`` to use.
:raises IIBError: if the index image build fails.
"""
_cleanup()
prebuild_info = prepare_request_for_build(
request_id,
RequestConfigAddRm(
_binary_image=binary_image,
from_index=from_index,
overwrite_from_index_token=overwrite_from_index_token,
add_arches=add_arches,
distribution_scope=distribution_scope,
binary_image_config=binary_image_config,
),
)
_update_index_image_build_state(request_id, prebuild_info)
from_index_resolved = prebuild_info['from_index_resolved']
with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir:
_opm_index_rm(
temp_dir,
operators,
prebuild_info['binary_image'],
from_index_resolved,
overwrite_from_index_token,
)
_add_label_to_index(
'com.redhat.index.delivery.version',
prebuild_info['ocp_version'],
temp_dir,
'index.Dockerfile',
)
_add_label_to_index(
'com.redhat.index.delivery.distribution_scope',
prebuild_info['distribution_scope'],
temp_dir,
'index.Dockerfile',
)
arches = prebuild_info['arches']
for arch in sorted(arches):
_build_image(temp_dir, 'index.Dockerfile', request_id, arch)
_push_image(request_id, arch)
set_request_state(request_id, 'in_progress', 'Creating the manifest list')
output_pull_spec = _create_and_push_manifest_list(request_id, arches)
_update_index_image_pull_spec(
output_pull_spec,
request_id,
arches,
from_index,
overwrite_from_index,
overwrite_from_index_token,
from_index_resolved,
add_or_rm=True,
)
set_request_state(
request_id, 'complete',
'The operator(s) were successfully removed from the index image')
def handle_add_request(
bundles,
request_id,
binary_image=None,
from_index=None,
add_arches=None,
cnr_token=None,
organization=None,
force_backport=False,
overwrite_from_index=False,
overwrite_from_index_token=None,
distribution_scope=None,
greenwave_config=None,
binary_image_config=None,
deprecation_list=None,
):
"""
Coordinate the the work needed to build the index image with the input bundles.
:param list bundles: a list of strings representing the pull specifications of the bundles to
add to the index image being built.
:param int request_id: the ID of the IIB build request
:param str binary_image: the pull specification of the container image where the opm binary
gets copied from.
:param str from_index: the pull specification of the container image containing the index that
the index image build will be based from.
:param list add_arches: the list of arches to build in addition to the arches ``from_index`` is
currently built for; if ``from_index`` is ``None``, then this is used as the list of arches
to build the index image for
:param str cnr_token: the token required to push backported packages to the legacy
app registry via OMPS.
:param str organization: organization name in the legacy app registry to which the backported
packages should be pushed to.
:param bool force_backport: if True, always export packages to the legacy app registry via OMPS.
:param bool overwrite_from_index: if True, overwrite the input ``from_index`` with the built
index image.
:param str overwrite_from_index_token: the token used for overwriting the input
``from_index`` image. This is required to use ``overwrite_from_index``.
The format of the token must be in the format "user:password".
:param str distribution_scope: the scope for distribution of the index image, defaults to
``None``.
:param dict greenwave_config: the dict of config required to query Greenwave to gate bundles.
:param dict binary_image_config: the dict of config required to identify the appropriate
``binary_image`` to use.
:param list deprecation_list: list of deprecated bundles for the target index image. Defaults
to ``None``.
:raises IIBError: if the index image build fails or legacy support is required and one of
``cnr_token`` or ``organization`` is not specified.
"""
_cleanup()
# Resolve bundles to their digests
set_request_state(request_id, 'in_progress', 'Resolving the bundles')
resolved_bundles = get_resolved_bundles(bundles)
verify_labels(resolved_bundles)
# Check if Gating passes for all the bundles
if greenwave_config:
gate_bundles(resolved_bundles, greenwave_config)
prebuild_info = prepare_request_for_build(
request_id,
RequestConfigAddRm(
_binary_image=binary_image,
from_index=from_index,
overwrite_from_index_token=overwrite_from_index_token,
add_arches=add_arches,
bundles=bundles,
distribution_scope=distribution_scope,
binary_image_config=binary_image_config,
),
)
from_index_resolved = prebuild_info['from_index_resolved']
log.info(
'Checking if interacting with the legacy app registry is required')
legacy_support_packages = get_legacy_support_packages(
resolved_bundles,
request_id,
prebuild_info['ocp_version'],
force_backport=force_backport)
if legacy_support_packages:
validate_legacy_params_and_config(legacy_support_packages,
resolved_bundles, cnr_token,
organization)
_update_index_image_build_state(request_id, prebuild_info)
present_bundles = []
present_bundles_pull_spec = []
with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir:
if from_index:
msg = 'Checking if bundles are already present in index image'
log.info(msg)
set_request_state(request_id, 'in_progress', msg)
with set_registry_token(overwrite_from_index_token,
from_index_resolved):
present_bundles, present_bundles_pull_spec = _get_present_bundles(
from_index_resolved, temp_dir)
filtered_bundles = _get_missing_bundles(present_bundles,
resolved_bundles)
excluded_bundles = [
bundle for bundle in resolved_bundles
if bundle not in filtered_bundles
]
resolved_bundles = filtered_bundles
if excluded_bundles:
log.info(
'Following bundles are already present in the index image: %s',
' '.join(excluded_bundles),
)
_opm_index_add(
temp_dir,
resolved_bundles,
prebuild_info['binary_image_resolved'],
from_index_resolved,
overwrite_from_index_token,
(prebuild_info['distribution_scope'] in ['dev', 'stage']),
)
deprecation_bundles = get_bundles_from_deprecation_list(
present_bundles_pull_spec + resolved_bundles, deprecation_list
or [])
arches = prebuild_info['arches']
if deprecation_bundles:
# opm can only deprecate a bundle image on an existing index image. Build and
# push a temporary index image to satisfy this requirement. Any arch will do.
arch = sorted(arches)[0]
log.info(
'Building a temporary index image to satisfy the deprecation requirement'
)
_build_image(temp_dir, 'index.Dockerfile', request_id, arch)
intermediate_image_name = _get_local_pull_spec(
request_id, arch, include_transport=True)
deprecate_bundles(
deprecation_bundles,
temp_dir,
prebuild_info['binary_image'],
intermediate_image_name,
overwrite_from_index_token,
# Use podman so opm can find the image locally
container_tool='podman',
)
_add_label_to_index(
'com.redhat.index.delivery.version',
prebuild_info['ocp_version'],
temp_dir,
'index.Dockerfile',
)
_add_label_to_index(
'com.redhat.index.delivery.distribution_scope',
prebuild_info['distribution_scope'],
temp_dir,
'index.Dockerfile',
)
for arch in sorted(arches):
_build_image(temp_dir, 'index.Dockerfile', request_id, arch)
_push_image(request_id, arch)
# If the container-tool podman is used in the opm commands above, opm will create temporary
# files and directories without the write permission. This will cause the context manager
# to fail to delete these files. Adjust the file modes to avoid this error.
chmod_recursively(
temp_dir,
dir_mode=(stat.S_IRWXU | stat.S_IRWXG),
file_mode=(stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP
| stat.S_IWGRP),
)
set_request_state(request_id, 'in_progress', 'Creating the manifest list')
output_pull_spec = _create_and_push_manifest_list(request_id, arches)
if legacy_support_packages:
export_legacy_packages(legacy_support_packages, request_id,
output_pull_spec, cnr_token, organization)
_update_index_image_pull_spec(
output_pull_spec,
request_id,
arches,
from_index,
overwrite_from_index,
overwrite_from_index_token,
from_index_resolved,
add_or_rm=True,
)
set_request_state(
request_id, 'complete',
'The operator bundle(s) were successfully added to the index image')
def handle_add_request(
bundles,
request_id,
binary_image=None,
from_index=None,
add_arches=None,
cnr_token=None,
organization=None,
force_backport=False,
overwrite_from_index=False,
overwrite_from_index_token=None,
distribution_scope=None,
greenwave_config=None,
binary_image_config=None,
deprecation_list=None,
):
"""
Coordinate the the work needed to build the index image with the input bundles.
:param list bundles: a list of strings representing the pull specifications of the bundles to
add to the index image being built.
:param int request_id: the ID of the IIB build request
:param str binary_image: the pull specification of the container image where the opm binary
gets copied from.
:param str from_index: the pull specification of the container image containing the index that
the index image build will be based from.
:param list add_arches: the list of arches to build in addition to the arches ``from_index`` is
currently built for; if ``from_index`` is ``None``, then this is used as the list of arches
to build the index image for
:param str cnr_token: (deprecated) legacy support was disabled.
the token required to push backported packages to the legacy app registry via OMPS.
:param str organization: (deprecated) legacy support was disabled.
organization name in the legacy app registry to which the backported packages
should be pushed to.
:param bool force_backport: (deprecated) legacy support was disabled.
if True, always export packages to the legacy app registry via OMPS.
:param bool overwrite_from_index: if True, overwrite the input ``from_index`` with the built
index image.
:param str overwrite_from_index_token: the token used for overwriting the input
``from_index`` image. This is required to use ``overwrite_from_index``.
The format of the token must be in the format "user:password".
:param str distribution_scope: the scope for distribution of the index image, defaults to
``None``.
:param dict greenwave_config: the dict of config required to query Greenwave to gate bundles.
:param dict binary_image_config: the dict of config required to identify the appropriate
``binary_image`` to use.
:param list deprecation_list: list of deprecated bundles for the target index image. Defaults
to ``None``.
:raises IIBError: if the index image build fails.
"""
_cleanup()
# Resolve bundles to their digests
set_request_state(request_id, 'in_progress', 'Resolving the bundles')
resolved_bundles = get_resolved_bundles(bundles)
verify_labels(resolved_bundles)
# Check if Gating passes for all the bundles
if greenwave_config:
gate_bundles(resolved_bundles, greenwave_config)
prebuild_info = prepare_request_for_build(
request_id,
RequestConfigAddRm(
_binary_image=binary_image,
from_index=from_index,
overwrite_from_index_token=overwrite_from_index_token,
add_arches=add_arches,
bundles=bundles,
distribution_scope=distribution_scope,
binary_image_config=binary_image_config,
),
)
from_index_resolved = prebuild_info['from_index_resolved']
if (cnr_token and organization) or force_backport:
log.warning(
"Legacy support is deprecated in IIB. "
"cnr_token, organization and force_backport parameters will be ignored."
)
_update_index_image_build_state(request_id, prebuild_info)
present_bundles = []
present_bundles_pull_spec = []
with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir:
if from_index:
msg = 'Checking if bundles are already present in index image'
log.info(msg)
set_request_state(request_id, 'in_progress', msg)
with set_registry_token(overwrite_from_index_token,
from_index_resolved):
present_bundles, present_bundles_pull_spec = _get_present_bundles(
from_index_resolved, temp_dir)
filtered_bundles = _get_missing_bundles(present_bundles,
resolved_bundles)
excluded_bundles = [
bundle for bundle in resolved_bundles
if bundle not in filtered_bundles
]
resolved_bundles = filtered_bundles
if excluded_bundles:
log.info(
'Following bundles are already present in the index image: %s',
' '.join(excluded_bundles),
)
_opm_index_add(
temp_dir,
resolved_bundles,
prebuild_info['binary_image_resolved'],
from_index_resolved,
overwrite_from_index_token,
(prebuild_info['distribution_scope'] in ['dev', 'stage']),
)
# Add the max ocp version property
# We need to ensure that any bundle which has deprecated/removed API(s) in 1.22/ocp 4.9
# will have this property to prevent users from upgrading clusters to 4.9 before upgrading
# the operator installed to a version that is compatible with 4.9
# Get the CSV name and version (not just the bundle path)
db_path = temp_dir + "/database/index.db"
port, rpc_proc = _serve_index_registry(db_path)
raw_bundles = run_cmd(
[
'grpcurl', '-plaintext', f'localhost:{port}',
'api.Registry/ListBundles'
],
exc_msg='Failed to get bundle data from index image',
)
rpc_proc.kill()
# Get bundle json for bundles in the request
updated_bundles = list(
filter(lambda b: b['bundlePath'] in resolved_bundles,
_get_bundle_json(raw_bundles)))
for bundle in updated_bundles:
if _requires_max_ocp_version(bundle['bundlePath']):
log.info('adding property for %s', bundle['bundlePath'])
max_openshift_version_property = {
'type': 'olm.maxOpenShiftVersion',
'value': '4.8',
'operatorbundle_name': bundle['csvName'],
'operatorbundle_version': bundle['version'],
'operatorbundle_path': bundle['bundlePath'],
}
_add_property_to_index(db_path, max_openshift_version_property)
log.info('property added for %s', bundle['bundlePath'])
deprecation_bundles = get_bundles_from_deprecation_list(
present_bundles_pull_spec + resolved_bundles, deprecation_list
or [])
arches = prebuild_info['arches']
if deprecation_bundles:
# opm can only deprecate a bundle image on an existing index image. Build and
# push a temporary index image to satisfy this requirement. Any arch will do.
arch = sorted(arches)[0]
log.info(
'Building a temporary index image to satisfy the deprecation requirement'
)
_build_image(temp_dir, 'index.Dockerfile', request_id, arch)
intermediate_image_name = _get_local_pull_spec(
request_id, arch, include_transport=True)
deprecate_bundles(
deprecation_bundles,
temp_dir,
prebuild_info['binary_image'],
intermediate_image_name,
overwrite_from_index_token,
# Use podman so opm can find the image locally
container_tool='podman',
)
_add_label_to_index(
'com.redhat.index.delivery.version',
prebuild_info['ocp_version'],
temp_dir,
'index.Dockerfile',
)
_add_label_to_index(
'com.redhat.index.delivery.distribution_scope',
prebuild_info['distribution_scope'],
temp_dir,
'index.Dockerfile',
)
for arch in sorted(arches):
_build_image(temp_dir, 'index.Dockerfile', request_id, arch)
_push_image(request_id, arch)
# If the container-tool podman is used in the opm commands above, opm will create temporary
# files and directories without the write permission. This will cause the context manager
# to fail to delete these files. Adjust the file modes to avoid this error.
chmod_recursively(
temp_dir,
dir_mode=(stat.S_IRWXU | stat.S_IRWXG),
file_mode=(stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP
| stat.S_IWGRP),
)
set_request_state(request_id, 'in_progress', 'Creating the manifest list')
output_pull_spec = _create_and_push_manifest_list(request_id, arches)
_update_index_image_pull_spec(
output_pull_spec,
request_id,
arches,
from_index,
overwrite_from_index,
overwrite_from_index_token,
from_index_resolved,
add_or_rm=True,
)
set_request_state(
request_id, 'complete',
'The operator bundle(s) were successfully added to the index image')
def handle_add_request(
bundles,
request_id,
binary_image=None,
from_index=None,
add_arches=None,
cnr_token=None,
organization=None,
force_backport=False,
overwrite_from_index=False,
overwrite_from_index_token=None,
distribution_scope=None,
greenwave_config=None,
binary_image_config=None,
deprecation_list=None,
build_tags=None,
):
"""
Coordinate the the work needed to build the index image with the input bundles.
:param list bundles: a list of strings representing the pull specifications of the bundles to
add to the index image being built.
:param int request_id: the ID of the IIB build request
:param str binary_image: the pull specification of the container image where the opm binary
gets copied from.
:param str from_index: the pull specification of the container image containing the index that
the index image build will be based from.
:param list add_arches: the list of arches to build in addition to the arches ``from_index`` is
currently built for; if ``from_index`` is ``None``, then this is used as the list of arches
to build the index image for
:param str cnr_token: (deprecated) legacy support was disabled.
the token required to push backported packages to the legacy app registry via OMPS.
:param str organization: (deprecated) legacy support was disabled.
organization name in the legacy app registry to which the backported packages
should be pushed to.
:param bool force_backport: (deprecated) legacy support was disabled.
if True, always export packages to the legacy app registry via OMPS.
:param bool overwrite_from_index: if True, overwrite the input ``from_index`` with the built
index image.
:param str overwrite_from_index_token: the token used for overwriting the input
``from_index`` image. This is required to use ``overwrite_from_index``.
The format of the token must be in the format "user:password".
:param str distribution_scope: the scope for distribution of the index image, defaults to
``None``.
:param dict greenwave_config: the dict of config required to query Greenwave to gate bundles.
:param dict binary_image_config: the dict of config required to identify the appropriate
``binary_image`` to use.
:param list deprecation_list: list of deprecated bundles for the target index image. Defaults
to ``None``.
:param list build_tags: List of tags which will be applied to intermediate index images.
:raises IIBError: if the index image build fails.
"""
_cleanup()
# Resolve bundles to their digests
set_request_state(request_id, 'in_progress', 'Resolving the bundles')
resolved_bundles = get_resolved_bundles(bundles)
verify_labels(resolved_bundles)
# Check if Gating passes for all the bundles
if greenwave_config:
gate_bundles(resolved_bundles, greenwave_config)
prebuild_info = prepare_request_for_build(
request_id,
RequestConfigAddRm(
_binary_image=binary_image,
from_index=from_index,
overwrite_from_index_token=overwrite_from_index_token,
add_arches=add_arches,
bundles=bundles,
distribution_scope=distribution_scope,
binary_image_config=binary_image_config,
),
)
from_index_resolved = prebuild_info['from_index_resolved']
with set_registry_token(overwrite_from_index_token, from_index_resolved):
is_fbc = is_image_fbc(from_index_resolved) if from_index else False
if is_fbc:
# logging requested by stakeholders do not delete
log.info("Processing File-Based Catalog image")
if (cnr_token and organization) or force_backport:
log.warning(
"Legacy support is deprecated in IIB. "
"cnr_token, organization and force_backport parameters will be ignored."
)
_update_index_image_build_state(request_id, prebuild_info)
present_bundles = []
present_bundles_pull_spec = []
with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir:
if from_index:
msg = 'Checking if bundles are already present in index image'
log.info(msg)
set_request_state(request_id, 'in_progress', msg)
with set_registry_token(overwrite_from_index_token, from_index_resolved):
present_bundles, present_bundles_pull_spec = _get_present_bundles(
from_index_resolved, temp_dir
)
filtered_bundles = _get_missing_bundles(present_bundles, resolved_bundles)
excluded_bundles = [
bundle for bundle in resolved_bundles if bundle not in filtered_bundles
]
resolved_bundles = filtered_bundles
if excluded_bundles:
log.info(
'Following bundles are already present in the index image: %s',
' '.join(excluded_bundles),
)
if is_fbc:
opm_registry_add_fbc(
base_dir=temp_dir,
bundles=resolved_bundles,
binary_image=prebuild_info['binary_image_resolved'],
from_index=from_index_resolved,
overwrite_from_index_token=overwrite_from_index_token,
overwrite_csv=(prebuild_info['distribution_scope'] in ['dev', 'stage']),
)
else:
_opm_index_add(
base_dir=temp_dir,
bundles=resolved_bundles,
binary_image=prebuild_info['binary_image_resolved'],
from_index=from_index_resolved,
overwrite_from_index_token=overwrite_from_index_token,
overwrite_csv=(prebuild_info['distribution_scope'] in ['dev', 'stage']),
)
# Add the max ocp version property
# We need to ensure that any bundle which has deprecated/removed API(s) in 1.22/ocp 4.9
# will have this property to prevent users from upgrading clusters to 4.9 before upgrading
# the operator installed to a version that is compatible with 4.9
if resolved_bundles:
add_max_ocp_version_property(resolved_bundles, temp_dir)
deprecation_bundles = get_bundles_from_deprecation_list(
present_bundles_pull_spec + resolved_bundles, deprecation_list or []
)
arches = prebuild_info['arches']
if deprecation_bundles:
if is_fbc:
deprecate_bundles_fbc(
bundles=deprecation_bundles,
base_dir=temp_dir,
binary_image=prebuild_info['binary_image'],
from_index=from_index_resolved,
)
else:
# opm can only deprecate a bundle image on an existing index image. Build and
# push a temporary index image to satisfy this requirement. Any arch will do.
arch = sorted(arches)[0]
log.info('Building a temporary index image to satisfy the deprecation requirement')
_build_image(temp_dir, 'index.Dockerfile', request_id, arch)
_push_image(request_id, arch)
intermediate_image_name = _get_external_arch_pull_spec(
request_id, arch, include_transport=False
)
deprecate_bundles(
bundles=deprecation_bundles,
base_dir=temp_dir,
binary_image=prebuild_info['binary_image'],
from_index=intermediate_image_name,
overwrite_target_index_token=overwrite_from_index_token,
# Use podman so opm can find the image locally
container_tool='podman',
)
_add_label_to_index(
'com.redhat.index.delivery.version',
prebuild_info['ocp_version'],
temp_dir,
'index.Dockerfile',
)
_add_label_to_index(
'com.redhat.index.delivery.distribution_scope',
prebuild_info['distribution_scope'],
temp_dir,
'index.Dockerfile',
)
if is_fbc:
index_db_file = os.path.join(temp_dir, get_worker_config()['temp_index_db_path'])
# make sure FBC is generated right before build
opm_migrate(index_db=index_db_file, base_dir=temp_dir)
for arch in sorted(arches):
_build_image(temp_dir, 'index.Dockerfile', request_id, arch)
_push_image(request_id, arch)
# If the container-tool podman is used in the opm commands above, opm will create temporary
# files and directories without the write permission. This will cause the context manager
# to fail to delete these files. Adjust the file modes to avoid this error.
chmod_recursively(
temp_dir,
dir_mode=(stat.S_IRWXU | stat.S_IRWXG),
file_mode=(stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP),
)
set_request_state(request_id, 'in_progress', 'Creating the manifest list')
output_pull_spec = _create_and_push_manifest_list(request_id, arches, build_tags)
_update_index_image_pull_spec(
output_pull_spec,
request_id,
arches,
from_index,
overwrite_from_index,
overwrite_from_index_token,
from_index_resolved,
add_or_rm=True,
)
set_request_state(
request_id, 'complete', 'The operator bundle(s) were successfully added to the index image'
)
def test_handle_add_request_check_index_label_behavior(
mock_iifbc,
mock_srt,
mock_gpb,
mock_alti,
mock_grb,
mock_gb,
mock_capml,
mock_srs,
mock_uiips,
mock_vii,
mock_pi,
mock_bi,
mock_oia,
mock_uiibs,
mock_prfb,
mock_vl,
mock_cleanup,
mock_ugrb,
mock_dep_b,
mock_ors,
mock_run_cmd,
mock_sqlite,
mock_gwc,
):
arches = {'amd64', 's390x'}
binary_image_config = {'prod': {'v4.5': 'some_image'}}
mock_iifbc.return_value = False
mock_prfb.return_value = {
'arches': arches,
'binary_image': 'binary-image:latest',
'binary_image_resolved': 'binary-image@sha256:abcdef',
'from_index_resolved': 'from-index@sha256:bcdefg',
'ocp_version': 'v4.5',
'distribution_scope': 'stage',
}
mock_grb.return_value = ['some-bundle@sha', 'some-deprecation-bundle@sha']
output_pull_spec = 'quay.io/namespace/some-image:3'
mock_capml.return_value = output_pull_spec
mock_gpb.return_value = [{
'bundlePath': 'random_bundle@sha'
}], ['random_bundle@sha']
bundles = ['some-bundle:2.3-1', 'some-deprecation-bundle:1.1-1']
cnr_token = 'token'
organization = 'org'
greenwave_config = {'some_key': 'other_value'}
mock_ugrb.return_value = [
'random_bundle@sha', 'some-deprecation-bundle@sha'
]
deprecation_list = ['random_bundle@sha', 'some-deprecation-bundle@sha']
# Assume default labels are set on the index
label_state = {'LABEL_SET': 'default_labels_set'}
mock_gwc.return_value = {
'iib_registry': 'quay.io',
'iib_image_push_template': '{registry}/iib-build:{request_id}',
}
def _add_label_to_index(*args):
# Set the labels in the index again making sure they were wiped out
if label_state['LABEL_SET'] == 'wiping_out_labels':
label_state['LABEL_SET'] = 'setting_label_in_add_label_to_index'
mock_alti.side_effect = _add_label_to_index
def deprecate_bundles_mock(*args, **kwargs):
# Wipe out the labels on the index
label_state['LABEL_SET'] = 'wiping_out_labels'
mock_dep_b.side_effect = deprecate_bundles_mock
port = 0
my_mock = mock.MagicMock()
mock_ors.return_value = (port, my_mock)
mock_run_cmd.side_effect = [
'{"packageName": "package1", "version": "v1.0", "csvName": "random-csv", \
"bundlePath": "some-bundle@sha"\n}',
'{"passed":false, "outputs": [{"message": "olm.maxOpenShiftVersion not present"}]}',
]
mock_sqlite.execute.return_value = 200
build.handle_add_request(
bundles,
3,
'binary-image:latest',
'from-index:latest',
['s390x'],
cnr_token,
organization,
True,
False,
None,
None,
greenwave_config,
binary_image_config=binary_image_config,
deprecation_list=deprecation_list,
)
mock_ors.assert_called_once()
assert mock_run_cmd.call_count == 2
mock_run_cmd.assert_has_calls([
mock.call(
[
'grpcurl', '-plaintext', f'localhost:{port}',
'api.Registry/ListBundles'
],
exc_msg=mock.ANY,
),
mock.call(
[
'operator-sdk',
'bundle',
'validate',
'some-bundle@sha',
'--select-optional',
'name=community',
'--output=json-alpha1',
'--image-builder',
'none',
],
strict=False,
),
])
mock_cleanup.assert_called_once()
mock_vl.assert_called_once()
mock_prfb.assert_called_once_with(
3,
RequestConfigAddRm(
_binary_image='binary-image:latest',
from_index='from-index:latest',
overwrite_from_index_token=None,
add_arches=['s390x'],
bundles=['some-bundle:2.3-1', 'some-deprecation-bundle:1.1-1'],
distribution_scope=None,
binary_image_config=binary_image_config,
),
)
mock_dep_b.assert_called_once_with(
bundles=['random_bundle@sha', 'some-deprecation-bundle@sha'],
base_dir=mock.ANY,
binary_image='binary-image:latest',
from_index='quay.io/iib-build:3-amd64',
overwrite_target_index_token=None,
container_tool='podman',
)
# Assert the labels are set again once they were wiped out
assert label_state['LABEL_SET'] == 'setting_label_in_add_label_to_index'
assert mock_alti.call_count == 2
def test_handle_rm_request_fbc(
mock_ogd,
mock_om,
mock_ghid,
mock_iifbc,
mock_alti,
mock_uiips,
mock_capml,
mock_srs,
mock_srs2,
mock_vii,
mock_pi,
mock_bi,
mock_runcmd,
mock_orrf,
mock_uiibs,
mock_prfb,
mock_c,
):
mock_iifbc.return_value = True
mock_prfb.return_value = {
'arches': {'amd64', 's390x'},
'binary_image': 'binary-image:latest',
'binary_image_resolved': 'binary-image@sha256:abcdef',
'from_index_resolved': 'from-index@sha256:bcdefg',
'ocp_version': 'v4.6',
'distribution_scope': 'PROD',
}
mock_ghid.return_value = "/tmp/xyz/database/index.db"
mock_ogd.return_value = "/tmp/xyz/index.Dockerfile"
mock_om.return_value = "/tmp/xyz/catalog"
mock_orrf.return_value = "/tmp/xyz/index.Dockerfile"
build.handle_rm_request(
operators=['some-operator'],
request_id=5,
from_index='from-index:latest',
binary_image='binary-image:latest',
binary_image_config={'prod': {
'v4.6': 'some_image'
}},
)
mock_prfb.assert_called_once_with(
5,
RequestConfigAddRm(
_binary_image='binary-image:latest',
from_index='from-index:latest',
overwrite_from_index_token=None,
add_arches=None,
binary_image_config={'prod': {
'v4.6': 'some_image'
}},
distribution_scope=None,
),
)
mock_orrf.assert_called_once()
assert mock_alti.call_count == 2
assert mock_bi.call_count == 2
assert mock_pi.call_count == 2
assert mock_srs.call_count == 2
mock_capml.assert_called_once_with(5, {'s390x', 'amd64'}, None)
mock_uiips.assert_called_once()
assert mock_srs.call_args[0][1] == 'complete'