def test_gate_bundles_success(mock_requests, mock_gkbn): mock_gkbn.return_value = 'n-v-r' mock_requests.return_value.ok = True mock_requests.return_value.json.return_value = {"policies_satisfied": True} greenwave_config = { 'subject_type': 'koji_build', 'decision_context': 'iib_cvp_redhat_operator', 'product_version': 'cvp', } greenwave.gate_bundles(['some-bundle'], greenwave_config) mock_gkbn.assert_called_once_with('some-bundle') mock_requests.assert_called_once()
def test_gate_bundles_invalid_json(mock_requests, mock_gkbn): mock_gkbn.return_value = 'n-v-r' mock_requests.return_value.ok = True mock_requests.return_value.json.side_effect = json.JSONDecodeError("error", "\n\n", 1) greenwave_config = { 'subject_type': 'koji_build', 'decision_context': 'iib_cvp_redhat_operator', 'product_version': 'cvp', } error_msg = 'Key "policies_satisfied" missing in Greenwave response for some-bundle' with pytest.raises(IIBError, match=error_msg): greenwave.gate_bundles(['some-bundle'], greenwave_config) mock_gkbn.assert_called_once_with('some-bundle') mock_requests.assert_called_once()
def test_gate_bundles_failure( mock_requests, mock_gkbn, greenwave_request_success, greenwave_json_rv, error_msg ): mock_gkbn.return_value = 'n-v-r' mock_requests.return_value.ok = greenwave_request_success mock_requests.return_value.json.return_value = greenwave_json_rv greenwave_config = { 'subject_type': 'koji_build', 'decision_context': 'iib_cvp_redhat_operator', 'product_version': 'cvp', } with pytest.raises(IIBError, match=error_msg): greenwave.gate_bundles(['some-bundle'], greenwave_config) mock_gkbn.assert_called_once_with('some-bundle') mock_requests.assert_called_once()
def handle_add_request( bundles, binary_image, request_id, from_index=None, add_arches=None, cnr_token=None, organization=None, force_backport=False, overwrite_from_index=False, overwrite_from_index_token=None, greenwave_config=None, ): """ Coordinate the the work needed to build the index image with the input bundles. :param list bundles: a list of strings representing the pull specifications of the bundles to add to the index image being built. :param str binary_image: the pull specification of the container image where the opm binary gets copied from. :param int request_id: the ID of the IIB build request :param str from_index: the pull specification of the container image containing the index that the index image build will be based from. :param list add_arches: the list of arches to build in addition to the arches ``from_index`` is currently built for; if ``from_index`` is ``None``, then this is used as the list of arches to build the index image for :param str cnr_token: the token required to push backported packages to the legacy app registry via OMPS. :param str organization: organization name in the legacy app registry to which the backported packages should be pushed to. :param bool force_backport: if True, always export packages to the legacy app registry via OMPS. :param bool overwrite_from_index: if True, overwrite the input ``from_index`` with the built index image. :param str overwrite_from_index_token: the token used for overwriting the input ``from_index`` image. This is required for non-privileged users to use ``overwrite_from_index``. The format of the token must be in the format "user:password". :param dict greenwave_config: the dict of config required to query Greenwave to gate bundles. :raises IIBError: if the index image build fails or legacy support is required and one of ``cnr_token`` or ``organization`` is not specified. """ _cleanup() # Resolve bundles to their digests set_request_state(request_id, 'in_progress', 'Resolving the bundles') resolved_bundles = _get_resolved_bundles(bundles) _verify_labels(resolved_bundles) # Check if Gating passes for all the bundles if greenwave_config: gate_bundles(resolved_bundles, greenwave_config) prebuild_info = _prepare_request_for_build(binary_image, request_id, from_index, overwrite_from_index_token, add_arches, bundles) log.info( 'Checking if interacting with the legacy app registry is required') legacy_support_packages = get_legacy_support_packages( resolved_bundles, request_id, prebuild_info['ocp_version'], force_backport=force_backport) if legacy_support_packages: validate_legacy_params_and_config(legacy_support_packages, resolved_bundles, cnr_token, organization) _update_index_image_build_state(request_id, prebuild_info) with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir: _opm_index_add( temp_dir, resolved_bundles, prebuild_info['binary_image_resolved'], from_index, overwrite_from_index_token, ) _add_ocp_label_to_index( prebuild_info['ocp_version'], temp_dir, 'index.Dockerfile', ) arches = prebuild_info['arches'] for arch in sorted(arches): _build_image(temp_dir, 'index.Dockerfile', request_id, arch) _push_image(request_id, arch) if from_index: _verify_index_image(prebuild_info['from_index_resolved'], from_index, overwrite_from_index_token) set_request_state(request_id, 'in_progress', 'Creating the manifest list') output_pull_spec = _create_and_push_manifest_list(request_id, arches) if legacy_support_packages: export_legacy_packages(legacy_support_packages, request_id, output_pull_spec, cnr_token, organization) _update_index_image_pull_spec( output_pull_spec, request_id, arches, from_index, overwrite_from_index, overwrite_from_index_token, ) set_request_state( request_id, 'complete', 'The operator bundle(s) were successfully added to the index image', )
def handle_add_request( bundles, request_id, binary_image=None, from_index=None, add_arches=None, cnr_token=None, organization=None, force_backport=False, overwrite_from_index=False, overwrite_from_index_token=None, distribution_scope=None, greenwave_config=None, binary_image_config=None, deprecation_list=None, ): """ Coordinate the the work needed to build the index image with the input bundles. :param list bundles: a list of strings representing the pull specifications of the bundles to add to the index image being built. :param int request_id: the ID of the IIB build request :param str binary_image: the pull specification of the container image where the opm binary gets copied from. :param str from_index: the pull specification of the container image containing the index that the index image build will be based from. :param list add_arches: the list of arches to build in addition to the arches ``from_index`` is currently built for; if ``from_index`` is ``None``, then this is used as the list of arches to build the index image for :param str cnr_token: the token required to push backported packages to the legacy app registry via OMPS. :param str organization: organization name in the legacy app registry to which the backported packages should be pushed to. :param bool force_backport: if True, always export packages to the legacy app registry via OMPS. :param bool overwrite_from_index: if True, overwrite the input ``from_index`` with the built index image. :param str overwrite_from_index_token: the token used for overwriting the input ``from_index`` image. This is required to use ``overwrite_from_index``. The format of the token must be in the format "user:password". :param str distribution_scope: the scope for distribution of the index image, defaults to ``None``. :param dict greenwave_config: the dict of config required to query Greenwave to gate bundles. :param dict binary_image_config: the dict of config required to identify the appropriate ``binary_image`` to use. :param list deprecation_list: list of deprecated bundles for the target index image. Defaults to ``None``. :raises IIBError: if the index image build fails or legacy support is required and one of ``cnr_token`` or ``organization`` is not specified. """ _cleanup() # Resolve bundles to their digests set_request_state(request_id, 'in_progress', 'Resolving the bundles') resolved_bundles = get_resolved_bundles(bundles) verify_labels(resolved_bundles) # Check if Gating passes for all the bundles if greenwave_config: gate_bundles(resolved_bundles, greenwave_config) prebuild_info = prepare_request_for_build( request_id, RequestConfigAddRm( _binary_image=binary_image, from_index=from_index, overwrite_from_index_token=overwrite_from_index_token, add_arches=add_arches, bundles=bundles, distribution_scope=distribution_scope, binary_image_config=binary_image_config, ), ) from_index_resolved = prebuild_info['from_index_resolved'] log.info( 'Checking if interacting with the legacy app registry is required') legacy_support_packages = get_legacy_support_packages( resolved_bundles, request_id, prebuild_info['ocp_version'], force_backport=force_backport) if legacy_support_packages: validate_legacy_params_and_config(legacy_support_packages, resolved_bundles, cnr_token, organization) _update_index_image_build_state(request_id, prebuild_info) present_bundles = [] present_bundles_pull_spec = [] with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir: if from_index: msg = 'Checking if bundles are already present in index image' log.info(msg) set_request_state(request_id, 'in_progress', msg) with set_registry_token(overwrite_from_index_token, from_index_resolved): present_bundles, present_bundles_pull_spec = _get_present_bundles( from_index_resolved, temp_dir) filtered_bundles = _get_missing_bundles(present_bundles, resolved_bundles) excluded_bundles = [ bundle for bundle in resolved_bundles if bundle not in filtered_bundles ] resolved_bundles = filtered_bundles if excluded_bundles: log.info( 'Following bundles are already present in the index image: %s', ' '.join(excluded_bundles), ) _opm_index_add( temp_dir, resolved_bundles, prebuild_info['binary_image_resolved'], from_index_resolved, overwrite_from_index_token, (prebuild_info['distribution_scope'] in ['dev', 'stage']), ) deprecation_bundles = get_bundles_from_deprecation_list( present_bundles_pull_spec + resolved_bundles, deprecation_list or []) arches = prebuild_info['arches'] if deprecation_bundles: # opm can only deprecate a bundle image on an existing index image. Build and # push a temporary index image to satisfy this requirement. Any arch will do. arch = sorted(arches)[0] log.info( 'Building a temporary index image to satisfy the deprecation requirement' ) _build_image(temp_dir, 'index.Dockerfile', request_id, arch) intermediate_image_name = _get_local_pull_spec( request_id, arch, include_transport=True) deprecate_bundles( deprecation_bundles, temp_dir, prebuild_info['binary_image'], intermediate_image_name, overwrite_from_index_token, # Use podman so opm can find the image locally container_tool='podman', ) _add_label_to_index( 'com.redhat.index.delivery.version', prebuild_info['ocp_version'], temp_dir, 'index.Dockerfile', ) _add_label_to_index( 'com.redhat.index.delivery.distribution_scope', prebuild_info['distribution_scope'], temp_dir, 'index.Dockerfile', ) for arch in sorted(arches): _build_image(temp_dir, 'index.Dockerfile', request_id, arch) _push_image(request_id, arch) # If the container-tool podman is used in the opm commands above, opm will create temporary # files and directories without the write permission. This will cause the context manager # to fail to delete these files. Adjust the file modes to avoid this error. chmod_recursively( temp_dir, dir_mode=(stat.S_IRWXU | stat.S_IRWXG), file_mode=(stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP), ) set_request_state(request_id, 'in_progress', 'Creating the manifest list') output_pull_spec = _create_and_push_manifest_list(request_id, arches) if legacy_support_packages: export_legacy_packages(legacy_support_packages, request_id, output_pull_spec, cnr_token, organization) _update_index_image_pull_spec( output_pull_spec, request_id, arches, from_index, overwrite_from_index, overwrite_from_index_token, from_index_resolved, add_or_rm=True, ) set_request_state( request_id, 'complete', 'The operator bundle(s) were successfully added to the index image')
def handle_add_request( bundles, request_id, binary_image=None, from_index=None, add_arches=None, cnr_token=None, organization=None, force_backport=False, overwrite_from_index=False, overwrite_from_index_token=None, distribution_scope=None, greenwave_config=None, binary_image_config=None, deprecation_list=None, ): """ Coordinate the the work needed to build the index image with the input bundles. :param list bundles: a list of strings representing the pull specifications of the bundles to add to the index image being built. :param int request_id: the ID of the IIB build request :param str binary_image: the pull specification of the container image where the opm binary gets copied from. :param str from_index: the pull specification of the container image containing the index that the index image build will be based from. :param list add_arches: the list of arches to build in addition to the arches ``from_index`` is currently built for; if ``from_index`` is ``None``, then this is used as the list of arches to build the index image for :param str cnr_token: (deprecated) legacy support was disabled. the token required to push backported packages to the legacy app registry via OMPS. :param str organization: (deprecated) legacy support was disabled. organization name in the legacy app registry to which the backported packages should be pushed to. :param bool force_backport: (deprecated) legacy support was disabled. if True, always export packages to the legacy app registry via OMPS. :param bool overwrite_from_index: if True, overwrite the input ``from_index`` with the built index image. :param str overwrite_from_index_token: the token used for overwriting the input ``from_index`` image. This is required to use ``overwrite_from_index``. The format of the token must be in the format "user:password". :param str distribution_scope: the scope for distribution of the index image, defaults to ``None``. :param dict greenwave_config: the dict of config required to query Greenwave to gate bundles. :param dict binary_image_config: the dict of config required to identify the appropriate ``binary_image`` to use. :param list deprecation_list: list of deprecated bundles for the target index image. Defaults to ``None``. :raises IIBError: if the index image build fails. """ _cleanup() # Resolve bundles to their digests set_request_state(request_id, 'in_progress', 'Resolving the bundles') resolved_bundles = get_resolved_bundles(bundles) verify_labels(resolved_bundles) # Check if Gating passes for all the bundles if greenwave_config: gate_bundles(resolved_bundles, greenwave_config) prebuild_info = prepare_request_for_build( request_id, RequestConfigAddRm( _binary_image=binary_image, from_index=from_index, overwrite_from_index_token=overwrite_from_index_token, add_arches=add_arches, bundles=bundles, distribution_scope=distribution_scope, binary_image_config=binary_image_config, ), ) from_index_resolved = prebuild_info['from_index_resolved'] if (cnr_token and organization) or force_backport: log.warning( "Legacy support is deprecated in IIB. " "cnr_token, organization and force_backport parameters will be ignored." ) _update_index_image_build_state(request_id, prebuild_info) present_bundles = [] present_bundles_pull_spec = [] with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir: if from_index: msg = 'Checking if bundles are already present in index image' log.info(msg) set_request_state(request_id, 'in_progress', msg) with set_registry_token(overwrite_from_index_token, from_index_resolved): present_bundles, present_bundles_pull_spec = _get_present_bundles( from_index_resolved, temp_dir) filtered_bundles = _get_missing_bundles(present_bundles, resolved_bundles) excluded_bundles = [ bundle for bundle in resolved_bundles if bundle not in filtered_bundles ] resolved_bundles = filtered_bundles if excluded_bundles: log.info( 'Following bundles are already present in the index image: %s', ' '.join(excluded_bundles), ) _opm_index_add( temp_dir, resolved_bundles, prebuild_info['binary_image_resolved'], from_index_resolved, overwrite_from_index_token, (prebuild_info['distribution_scope'] in ['dev', 'stage']), ) # Add the max ocp version property # We need to ensure that any bundle which has deprecated/removed API(s) in 1.22/ocp 4.9 # will have this property to prevent users from upgrading clusters to 4.9 before upgrading # the operator installed to a version that is compatible with 4.9 # Get the CSV name and version (not just the bundle path) db_path = temp_dir + "/database/index.db" port, rpc_proc = _serve_index_registry(db_path) raw_bundles = run_cmd( [ 'grpcurl', '-plaintext', f'localhost:{port}', 'api.Registry/ListBundles' ], exc_msg='Failed to get bundle data from index image', ) rpc_proc.kill() # Get bundle json for bundles in the request updated_bundles = list( filter(lambda b: b['bundlePath'] in resolved_bundles, _get_bundle_json(raw_bundles))) for bundle in updated_bundles: if _requires_max_ocp_version(bundle['bundlePath']): log.info('adding property for %s', bundle['bundlePath']) max_openshift_version_property = { 'type': 'olm.maxOpenShiftVersion', 'value': '4.8', 'operatorbundle_name': bundle['csvName'], 'operatorbundle_version': bundle['version'], 'operatorbundle_path': bundle['bundlePath'], } _add_property_to_index(db_path, max_openshift_version_property) log.info('property added for %s', bundle['bundlePath']) deprecation_bundles = get_bundles_from_deprecation_list( present_bundles_pull_spec + resolved_bundles, deprecation_list or []) arches = prebuild_info['arches'] if deprecation_bundles: # opm can only deprecate a bundle image on an existing index image. Build and # push a temporary index image to satisfy this requirement. Any arch will do. arch = sorted(arches)[0] log.info( 'Building a temporary index image to satisfy the deprecation requirement' ) _build_image(temp_dir, 'index.Dockerfile', request_id, arch) intermediate_image_name = _get_local_pull_spec( request_id, arch, include_transport=True) deprecate_bundles( deprecation_bundles, temp_dir, prebuild_info['binary_image'], intermediate_image_name, overwrite_from_index_token, # Use podman so opm can find the image locally container_tool='podman', ) _add_label_to_index( 'com.redhat.index.delivery.version', prebuild_info['ocp_version'], temp_dir, 'index.Dockerfile', ) _add_label_to_index( 'com.redhat.index.delivery.distribution_scope', prebuild_info['distribution_scope'], temp_dir, 'index.Dockerfile', ) for arch in sorted(arches): _build_image(temp_dir, 'index.Dockerfile', request_id, arch) _push_image(request_id, arch) # If the container-tool podman is used in the opm commands above, opm will create temporary # files and directories without the write permission. This will cause the context manager # to fail to delete these files. Adjust the file modes to avoid this error. chmod_recursively( temp_dir, dir_mode=(stat.S_IRWXU | stat.S_IRWXG), file_mode=(stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP), ) set_request_state(request_id, 'in_progress', 'Creating the manifest list') output_pull_spec = _create_and_push_manifest_list(request_id, arches) _update_index_image_pull_spec( output_pull_spec, request_id, arches, from_index, overwrite_from_index, overwrite_from_index_token, from_index_resolved, add_or_rm=True, ) set_request_state( request_id, 'complete', 'The operator bundle(s) were successfully added to the index image')
def handle_add_request( bundles, request_id, binary_image=None, from_index=None, add_arches=None, cnr_token=None, organization=None, force_backport=False, overwrite_from_index=False, overwrite_from_index_token=None, distribution_scope=None, greenwave_config=None, binary_image_config=None, deprecation_list=None, build_tags=None, ): """ Coordinate the the work needed to build the index image with the input bundles. :param list bundles: a list of strings representing the pull specifications of the bundles to add to the index image being built. :param int request_id: the ID of the IIB build request :param str binary_image: the pull specification of the container image where the opm binary gets copied from. :param str from_index: the pull specification of the container image containing the index that the index image build will be based from. :param list add_arches: the list of arches to build in addition to the arches ``from_index`` is currently built for; if ``from_index`` is ``None``, then this is used as the list of arches to build the index image for :param str cnr_token: (deprecated) legacy support was disabled. the token required to push backported packages to the legacy app registry via OMPS. :param str organization: (deprecated) legacy support was disabled. organization name in the legacy app registry to which the backported packages should be pushed to. :param bool force_backport: (deprecated) legacy support was disabled. if True, always export packages to the legacy app registry via OMPS. :param bool overwrite_from_index: if True, overwrite the input ``from_index`` with the built index image. :param str overwrite_from_index_token: the token used for overwriting the input ``from_index`` image. This is required to use ``overwrite_from_index``. The format of the token must be in the format "user:password". :param str distribution_scope: the scope for distribution of the index image, defaults to ``None``. :param dict greenwave_config: the dict of config required to query Greenwave to gate bundles. :param dict binary_image_config: the dict of config required to identify the appropriate ``binary_image`` to use. :param list deprecation_list: list of deprecated bundles for the target index image. Defaults to ``None``. :param list build_tags: List of tags which will be applied to intermediate index images. :raises IIBError: if the index image build fails. """ _cleanup() # Resolve bundles to their digests set_request_state(request_id, 'in_progress', 'Resolving the bundles') resolved_bundles = get_resolved_bundles(bundles) verify_labels(resolved_bundles) # Check if Gating passes for all the bundles if greenwave_config: gate_bundles(resolved_bundles, greenwave_config) prebuild_info = prepare_request_for_build( request_id, RequestConfigAddRm( _binary_image=binary_image, from_index=from_index, overwrite_from_index_token=overwrite_from_index_token, add_arches=add_arches, bundles=bundles, distribution_scope=distribution_scope, binary_image_config=binary_image_config, ), ) from_index_resolved = prebuild_info['from_index_resolved'] with set_registry_token(overwrite_from_index_token, from_index_resolved): is_fbc = is_image_fbc(from_index_resolved) if from_index else False if is_fbc: # logging requested by stakeholders do not delete log.info("Processing File-Based Catalog image") if (cnr_token and organization) or force_backport: log.warning( "Legacy support is deprecated in IIB. " "cnr_token, organization and force_backport parameters will be ignored." ) _update_index_image_build_state(request_id, prebuild_info) present_bundles = [] present_bundles_pull_spec = [] with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir: if from_index: msg = 'Checking if bundles are already present in index image' log.info(msg) set_request_state(request_id, 'in_progress', msg) with set_registry_token(overwrite_from_index_token, from_index_resolved): present_bundles, present_bundles_pull_spec = _get_present_bundles( from_index_resolved, temp_dir ) filtered_bundles = _get_missing_bundles(present_bundles, resolved_bundles) excluded_bundles = [ bundle for bundle in resolved_bundles if bundle not in filtered_bundles ] resolved_bundles = filtered_bundles if excluded_bundles: log.info( 'Following bundles are already present in the index image: %s', ' '.join(excluded_bundles), ) if is_fbc: opm_registry_add_fbc( base_dir=temp_dir, bundles=resolved_bundles, binary_image=prebuild_info['binary_image_resolved'], from_index=from_index_resolved, overwrite_from_index_token=overwrite_from_index_token, overwrite_csv=(prebuild_info['distribution_scope'] in ['dev', 'stage']), ) else: _opm_index_add( base_dir=temp_dir, bundles=resolved_bundles, binary_image=prebuild_info['binary_image_resolved'], from_index=from_index_resolved, overwrite_from_index_token=overwrite_from_index_token, overwrite_csv=(prebuild_info['distribution_scope'] in ['dev', 'stage']), ) # Add the max ocp version property # We need to ensure that any bundle which has deprecated/removed API(s) in 1.22/ocp 4.9 # will have this property to prevent users from upgrading clusters to 4.9 before upgrading # the operator installed to a version that is compatible with 4.9 if resolved_bundles: add_max_ocp_version_property(resolved_bundles, temp_dir) deprecation_bundles = get_bundles_from_deprecation_list( present_bundles_pull_spec + resolved_bundles, deprecation_list or [] ) arches = prebuild_info['arches'] if deprecation_bundles: if is_fbc: deprecate_bundles_fbc( bundles=deprecation_bundles, base_dir=temp_dir, binary_image=prebuild_info['binary_image'], from_index=from_index_resolved, ) else: # opm can only deprecate a bundle image on an existing index image. Build and # push a temporary index image to satisfy this requirement. Any arch will do. arch = sorted(arches)[0] log.info('Building a temporary index image to satisfy the deprecation requirement') _build_image(temp_dir, 'index.Dockerfile', request_id, arch) _push_image(request_id, arch) intermediate_image_name = _get_external_arch_pull_spec( request_id, arch, include_transport=False ) deprecate_bundles( bundles=deprecation_bundles, base_dir=temp_dir, binary_image=prebuild_info['binary_image'], from_index=intermediate_image_name, overwrite_target_index_token=overwrite_from_index_token, # Use podman so opm can find the image locally container_tool='podman', ) _add_label_to_index( 'com.redhat.index.delivery.version', prebuild_info['ocp_version'], temp_dir, 'index.Dockerfile', ) _add_label_to_index( 'com.redhat.index.delivery.distribution_scope', prebuild_info['distribution_scope'], temp_dir, 'index.Dockerfile', ) if is_fbc: index_db_file = os.path.join(temp_dir, get_worker_config()['temp_index_db_path']) # make sure FBC is generated right before build opm_migrate(index_db=index_db_file, base_dir=temp_dir) for arch in sorted(arches): _build_image(temp_dir, 'index.Dockerfile', request_id, arch) _push_image(request_id, arch) # If the container-tool podman is used in the opm commands above, opm will create temporary # files and directories without the write permission. This will cause the context manager # to fail to delete these files. Adjust the file modes to avoid this error. chmod_recursively( temp_dir, dir_mode=(stat.S_IRWXU | stat.S_IRWXG), file_mode=(stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP), ) set_request_state(request_id, 'in_progress', 'Creating the manifest list') output_pull_spec = _create_and_push_manifest_list(request_id, arches, build_tags) _update_index_image_pull_spec( output_pull_spec, request_id, arches, from_index, overwrite_from_index, overwrite_from_index_token, from_index_resolved, add_or_rm=True, ) set_request_state( request_id, 'complete', 'The operator bundle(s) were successfully added to the index image' )