def test_update_index_image_pull_spec(mock_ur, mock_ofi, mock_gwc, iib_index_image_output_registry, from_index, overwrite, expected): default_no_registry = 'namespace/some-image:3' default = f'quay.io/{default_no_registry}' expected_pull_spec = expected.format( default=default, default_no_registry=default_no_registry) request_id = 2 arches = {'amd64'} overwrite_token = 'username:password' mock_gwc.return_value = { 'iib_index_image_output_registry': iib_index_image_output_registry, 'iib_registry': 'quay.io', } build._update_index_image_pull_spec(default, request_id, arches, from_index, overwrite, overwrite_token) mock_ur.assert_called_once() update_request_payload = mock_ur.call_args[0][1] assert update_request_payload.keys() == {'arches', 'index_image'} assert update_request_payload['index_image'] == expected_pull_spec if overwrite: mock_ofi.assert_called_once_with(request_id, default, from_index, overwrite_token) else: mock_ofi.assert_not_called()
def handle_create_empty_index_request( from_index, request_id, output_fbc=False, binary_image=None, labels=None, binary_image_config=None, ): """Coordinate the the work needed to create the index image with labels. :param str from_index: the pull specification of the container image containing the index that the index image build will be based from. :param int request_id: the ID of the IIB build request :param bool output_fbc: specifies whether a File-based Catalog index image should be created :param str binary_image: the pull specification of the container image where the opm binary gets copied from. :param dict labels: the dict of labels required to be added to a new index image :param dict binary_image_config: the dict of config required to identify the appropriate ``binary_image`` to use. """ _cleanup() prebuild_info = prepare_request_for_build( request_id, RequestConfigCreateIndexImage( _binary_image=binary_image, from_index=from_index, binary_image_config=binary_image_config, ), ) from_index_resolved = prebuild_info['from_index_resolved'] prebuild_info['labels'] = labels if not output_fbc and is_image_fbc(from_index_resolved): log.debug('%s is FBC index image', from_index_resolved) err_msg = 'Cannot create SQLite index image from File-Based Catalog index image' log.error(err_msg) raise IIBError(err_msg) _update_index_image_build_state(request_id, prebuild_info) with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir: set_request_state(request_id, 'in_progress', 'Checking operators present in index image') operators = _get_present_operators(from_index_resolved, temp_dir) # if output_fbc parameter is true, create an empty FBC index image # else create empty SQLite index image if output_fbc: log.debug('Creating empty FBC index image from %s', from_index) opm_create_empty_fbc( request_id=request_id, temp_dir=temp_dir, from_index_resolved=from_index_resolved, from_index=from_index, binary_image=prebuild_info['binary_image'], operators=operators, ) else: set_request_state(request_id, 'in_progress', 'Removing operators from index image') _opm_index_rm(temp_dir, operators, prebuild_info['binary_image'], from_index_resolved) set_request_state( request_id, 'in_progress', 'Getting and updating labels for new index image' ) iib_labels = { 'com.redhat.index.delivery.version': prebuild_info['ocp_version'], 'com.redhat.index.delivery.distribution_scope': prebuild_info['distribution_scope'], } if labels: iib_labels.update(labels) for index_label, value in iib_labels.items(): _add_label_to_index(index_label, value, temp_dir, 'index.Dockerfile') arches = prebuild_info['arches'] for arch in sorted(arches): _build_image(temp_dir, 'index.Dockerfile', request_id, arch) _push_image(request_id, arch) set_request_state(request_id, 'in_progress', 'Creating the manifest list') output_pull_spec = _create_and_push_manifest_list(request_id, arches, []) _update_index_image_pull_spec( output_pull_spec=output_pull_spec, request_id=request_id, arches=arches, from_index=from_index, resolved_prebuild_from_index=from_index_resolved, ) set_request_state(request_id, 'complete', 'The empty index image was successfully created')
def handle_merge_request( source_from_index, deprecation_list, request_id, binary_image=None, target_index=None, overwrite_target_index=False, overwrite_target_index_token=None, distribution_scope=None, binary_image_config=None, ): """ Coordinate the work needed to merge old (N) index image with new (N+1) index image. :param str source_from_index: pull specification to be used as the base for building the new index image. :param str target_index: pull specification of content stage index image for the corresponding target index image. :param list deprecation_list: list of deprecated bundles for the target index image. :param int request_id: the ID of the IIB build request. :param str binary_image: the pull specification of the container image where the opm binary gets copied from. :param bool overwrite_target_index: if True, overwrite the input ``target_index`` with the built index image. :param str overwrite_target_index_token: the token used for overwriting the input ``target_index`` image. This is required for non-privileged users to use ``overwrite_target_index``. The format of the token must be in the format "user:password". :param str distribution_scope: the scope for distribution of the index image, defaults to ``None``. :raises IIBError: if the index image merge fails. """ _cleanup() prebuild_info = _prepare_request_for_build( request_id, binary_image, overwrite_from_index_token=overwrite_target_index_token, source_from_index=source_from_index, target_index=target_index, distribution_scope=distribution_scope, binary_image_config=binary_image_config, ) _update_index_image_build_state(request_id, prebuild_info) with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir: set_request_state(request_id, 'in_progress', 'Getting bundles present in the index images') log.info('Getting bundles present in the source index image') source_index_bundles = _get_present_bundles(source_from_index, temp_dir) target_index_bundles = [] if target_index: log.info('Getting bundles present in the target index image') target_index_bundles = _get_present_bundles(target_index, temp_dir) arches = list(prebuild_info['arches']) arch = 'amd64' if 'amd64' in arches else arches[0] missing_bundles = _add_bundles_missing_in_source( source_index_bundles, target_index_bundles, temp_dir, prebuild_info['binary_image'], source_from_index, request_id, arch, prebuild_info['target_ocp_version'], overwrite_target_index_token, distribution_scope=prebuild_info['distribution_scope'], ) set_request_state(request_id, 'in_progress', 'Deprecating bundles in the deprecation list') log.info('Deprecating bundles in the deprecation list') intermediate_bundles = source_index_bundles + missing_bundles deprecate_bundles = _get_bundles_from_deprecation_list( intermediate_bundles, deprecation_list) intermediate_image_name = _get_external_arch_pull_spec( request_id, arch, include_transport=False) if deprecate_bundles: _deprecate_bundles( deprecate_bundles, temp_dir, prebuild_info['binary_image'], intermediate_image_name, overwrite_target_index_token, ) for arch in sorted(prebuild_info['arches']): _build_image(temp_dir, 'index.Dockerfile', request_id, arch) _push_image(request_id, arch) output_pull_spec = _create_and_push_manifest_list(request_id, prebuild_info['arches']) _update_index_image_pull_spec( output_pull_spec, request_id, prebuild_info['arches'], target_index, overwrite_target_index, overwrite_target_index_token, prebuild_info['target_index_resolved'], ) set_request_state(request_id, 'complete', 'The index image was successfully cleaned and updated.')
def test_update_index_image_pull_spec( mock_st_rgstr_tknm, mock_get_rslv_img, mock_ur, mock_ofi, mock_gwc, iib_index_image_output_registry, from_index, overwrite, expected, resolved_from_index, add_or_rm, ): default_no_registry = 'namespace/some-image:3' default = f'quay.io/{default_no_registry}' expected_pull_spec = expected.format( default=default, default_no_registry=default_no_registry) request_id = 2 arches = {'amd64'} overwrite_token = 'username:password' mock_get_rslv_img.return_value = "quay.io/ns/iib@sha256:abcdef1234" mock_gwc.return_value = { 'iib_index_image_output_registry': iib_index_image_output_registry, 'iib_registry': 'quay.io', } if add_or_rm: build._update_index_image_pull_spec( default, request_id, arches, from_index, overwrite, overwrite_token, resolved_prebuild_from_index=resolved_from_index, add_or_rm=add_or_rm, ) else: build._update_index_image_pull_spec( default, request_id, arches, from_index, overwrite, overwrite_token, resolved_prebuild_from_index=resolved_from_index, ) mock_ur.assert_called_once() update_request_payload = mock_ur.call_args[0][1] if add_or_rm: assert update_request_payload.keys() == { 'arches', 'index_image', 'index_image_resolved' } else: assert update_request_payload.keys() == {'arches', 'index_image'} assert update_request_payload['index_image'] == expected_pull_spec if overwrite: mock_ofi.assert_called_once_with(request_id, default, from_index, resolved_from_index, overwrite_token) else: mock_ofi.assert_not_called()
def handle_merge_request( source_from_index, deprecation_list, request_id, binary_image=None, target_index=None, overwrite_target_index=False, overwrite_target_index_token=None, distribution_scope=None, binary_image_config=None, ): """ Coordinate the work needed to merge old (N) index image with new (N+1) index image. :param str source_from_index: pull specification to be used as the base for building the new index image. :param str target_index: pull specification of content stage index image for the corresponding target index image. :param list deprecation_list: list of deprecated bundles for the target index image. :param int request_id: the ID of the IIB build request. :param str binary_image: the pull specification of the container image where the opm binary gets copied from. :param bool overwrite_target_index: if True, overwrite the input ``target_index`` with the built index image. :param str overwrite_target_index_token: the token used for overwriting the input ``target_index`` image. This is required to use ``overwrite_target_index``. The format of the token must be in the format "user:password". :param str distribution_scope: the scope for distribution of the index image, defaults to ``None``. :raises IIBError: if the index image merge fails. """ _cleanup() prebuild_info = prepare_request_for_build( request_id, RequestConfigMerge( _binary_image=binary_image, overwrite_target_index_token=overwrite_target_index_token, source_from_index=source_from_index, target_index=target_index, distribution_scope=distribution_scope, binary_image_config=binary_image_config, ), ) _update_index_image_build_state(request_id, prebuild_info) source_from_index_resolved = prebuild_info['source_from_index_resolved'] target_index_resolved = prebuild_info['target_index_resolved'] with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir: set_request_state(request_id, 'in_progress', 'Getting bundles present in the index images') log.info('Getting bundles present in the source index image') with set_registry_token(overwrite_target_index_token, source_from_index): source_index_bundles, source_index_bundles_pull_spec = _get_present_bundles( source_from_index_resolved, temp_dir) target_index_bundles = [] if target_index: log.info('Getting bundles present in the target index image') target_index_bundles, _ = _get_present_bundles( target_index_resolved, temp_dir) arches = list(prebuild_info['arches']) arch = 'amd64' if 'amd64' in arches else arches[0] missing_bundles, invalid_version_bundles = _add_bundles_missing_in_source( source_index_bundles, target_index_bundles, temp_dir, prebuild_info['binary_image'], source_from_index_resolved, request_id, arch, prebuild_info['target_ocp_version'], overwrite_target_index_token, distribution_scope=prebuild_info['distribution_scope'], ) set_request_state(request_id, 'in_progress', 'Deprecating bundles in the deprecation list') log.info('Deprecating bundles in the deprecation list') intermediate_bundles = [ bundle['bundlePath'] for bundle in missing_bundles ] + source_index_bundles_pull_spec deprecation_bundles = get_bundles_from_deprecation_list( intermediate_bundles, deprecation_list) # We do not need to pass the invalid_version_bundles through the # get_bundles_from_deprecation_list function because we already know # they are present in the newly created index. deprecation_bundles = deprecation_bundles + [ bundle['bundlePath'] for bundle in invalid_version_bundles ] intermediate_image_name = _get_external_arch_pull_spec( request_id, arch, include_transport=False) if deprecation_bundles: deprecate_bundles( deprecation_bundles, temp_dir, prebuild_info['binary_image'], intermediate_image_name, overwrite_target_index_token, ) for arch in sorted(prebuild_info['arches']): _build_image(temp_dir, 'index.Dockerfile', request_id, arch) _push_image(request_id, arch) # If the container-tool podman is used in the opm commands above, opm will create temporary # files and directories without the write permission. This will cause the context manager # to fail to delete these files. Adjust the file modes to avoid this error. chmod_recursively( temp_dir, dir_mode=(stat.S_IRWXU | stat.S_IRWXG), file_mode=(stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP), ) output_pull_spec = _create_and_push_manifest_list(request_id, prebuild_info['arches']) _update_index_image_pull_spec( output_pull_spec, request_id, prebuild_info['arches'], target_index, overwrite_target_index, overwrite_target_index_token, target_index_resolved, ) set_request_state(request_id, 'complete', 'The index image was successfully cleaned and updated.')
def handle_merge_request( source_from_index, deprecation_list, request_id, binary_image=None, target_index=None, overwrite_target_index=False, overwrite_target_index_token=None, distribution_scope=None, binary_image_config=None, build_tags=None, ): """ Coordinate the work needed to merge old (N) index image with new (N+1) index image. :param str source_from_index: pull specification to be used as the base for building the new index image. :param str target_index: pull specification of content stage index image for the corresponding target index image. :param list deprecation_list: list of deprecated bundles for the target index image. :param int request_id: the ID of the IIB build request. :param str binary_image: the pull specification of the container image where the opm binary gets copied from. :param bool overwrite_target_index: if True, overwrite the input ``target_index`` with the built index image. :param str overwrite_target_index_token: the token used for overwriting the input ``target_index`` image. This is required to use ``overwrite_target_index``. The format of the token must be in the format "user:password". :param str distribution_scope: the scope for distribution of the index image, defaults to ``None``. :param build_tags: list of extra tag to use for intermetdiate index image :raises IIBError: if the index image merge fails. """ _cleanup() prebuild_info = prepare_request_for_build( request_id, RequestConfigMerge( _binary_image=binary_image, overwrite_target_index_token=overwrite_target_index_token, source_from_index=source_from_index, target_index=target_index, distribution_scope=distribution_scope, binary_image_config=binary_image_config, ), ) _update_index_image_build_state(request_id, prebuild_info) source_from_index_resolved = prebuild_info['source_from_index_resolved'] target_index_resolved = prebuild_info['target_index_resolved'] dockerfile_name = 'index.Dockerfile' with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir: with set_registry_token(overwrite_target_index_token, source_from_index): source_fbc = is_image_fbc(source_from_index_resolved) target_fbc = is_image_fbc(target_index_resolved) # do not remove - logging requested by stakeholders if source_fbc: log.info("Processing source index image as File-Based Catalog image") if target_fbc: log.info("Processing target index image as File-Based Catalog image") if source_fbc and not target_fbc: err_msg = ( 'Cannot merge source File-Based Catalog index image into target SQLite index image.' ) log.error(err_msg) raise IIBError(err_msg) set_request_state(request_id, 'in_progress', 'Getting bundles present in the index images') log.info('Getting bundles present in the source index image') with set_registry_token(overwrite_target_index_token, source_from_index): source_index_bundles, source_index_bundles_pull_spec = _get_present_bundles( source_from_index_resolved, temp_dir ) target_index_bundles = [] if target_index: log.info('Getting bundles present in the target index image') target_index_bundles, _ = _get_present_bundles(target_index_resolved, temp_dir) arches = list(prebuild_info['arches']) arch = sorted(arches)[0] missing_bundles, invalid_version_bundles = _add_bundles_missing_in_source( source_index_bundles, target_index_bundles, temp_dir, prebuild_info['binary_image'], source_from_index_resolved, request_id, arch, prebuild_info['target_ocp_version'], overwrite_target_index_token, distribution_scope=prebuild_info['distribution_scope'], ) missing_bundle_paths = [bundle['bundlePath'] for bundle in missing_bundles] if missing_bundle_paths: add_max_ocp_version_property(missing_bundle_paths, temp_dir) set_request_state(request_id, 'in_progress', 'Deprecating bundles in the deprecation list') log.info('Deprecating bundles in the deprecation list') intermediate_bundles = missing_bundle_paths + source_index_bundles_pull_spec deprecation_bundles = get_bundles_from_deprecation_list( intermediate_bundles, deprecation_list ) # We do not need to pass the invalid_version_bundles through the # get_bundles_from_deprecation_list function because we already know # they are present in the newly created index. deprecation_bundles = deprecation_bundles + [ bundle['bundlePath'] for bundle in invalid_version_bundles ] if deprecation_bundles: intermediate_image_name = _get_external_arch_pull_spec( request_id, arch, include_transport=False ) # we can check if source index is FBC or not because intermediate_image # will be always the same type because it is built # from source index image in _add_bundles_missing_in_source() if source_fbc: deprecate_bundles_fbc( bundles=deprecation_bundles, base_dir=temp_dir, binary_image=prebuild_info['binary_image'], from_index=intermediate_image_name, ) else: # opm can only deprecate a bundle image on an existing index image. Build and # push a temporary index image to satisfy this requirement. Any arch will do. # NOTE: we cannot use local builds because opm commands fails, # index image has to be pushed to registry _build_image(temp_dir, 'index.Dockerfile', request_id, arch) _push_image(request_id, arch) deprecate_bundles( bundles=deprecation_bundles, base_dir=temp_dir, binary_image=prebuild_info['binary_image'], from_index=intermediate_image_name, overwrite_target_index_token=overwrite_target_index_token, ) if target_fbc: index_db_file = os.path.join(temp_dir, get_worker_config()['temp_index_db_path']) # make sure FBC is generated right before build fbc_dir = opm_migrate(index_db=index_db_file, base_dir=temp_dir) if not source_fbc: # when source image is not FBC, but final image should be an FBC image # we have to generate Dockerfile for FBC (with hidden index.db) dockerfile_path = os.path.join(temp_dir, dockerfile_name) if os.path.isfile(dockerfile_path): log.info('Removing previously generated dockerfile.') os.remove(dockerfile_path) opm_generate_dockerfile( fbc_dir=fbc_dir, base_dir=temp_dir, index_db=index_db_file, binary_image=prebuild_info['binary_image'], dockerfile_name=dockerfile_name, ) _add_label_to_index( 'com.redhat.index.delivery.version', prebuild_info['target_ocp_version'], temp_dir, dockerfile_name, ) _add_label_to_index( 'com.redhat.index.delivery.distribution_scope', prebuild_info['distribution_scope'], temp_dir, dockerfile_name, ) for arch in sorted(prebuild_info['arches']): _build_image(temp_dir, dockerfile_name, request_id, arch) _push_image(request_id, arch) # If the container-tool podman is used in the opm commands above, opm will create temporary # files and directories without the write permission. This will cause the context manager # to fail to delete these files. Adjust the file modes to avoid this error. chmod_recursively( temp_dir, dir_mode=(stat.S_IRWXU | stat.S_IRWXG), file_mode=(stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP), ) output_pull_spec = _create_and_push_manifest_list( request_id, prebuild_info['arches'], build_tags ) _update_index_image_pull_spec( output_pull_spec, request_id, prebuild_info['arches'], target_index, overwrite_target_index, overwrite_target_index_token, target_index_resolved, ) set_request_state( request_id, 'complete', 'The index image was successfully cleaned and updated.' )
def handle_create_empty_index_request(from_index, request_id, binary_image=None, labels=None, binary_image_config=None): """Coordinate the the work needed to create the index image with labels. :param str from_index: the pull specification of the container image containing the index that the index image build will be based from. :param int request_id: the ID of the IIB build request :param str binary_image: the pull specification of the container image where the opm binary gets copied from. :param dict labels: the dict of labels required to be added to a new index image :param dict binary_image_config: the dict of config required to identify the appropriate ``binary_image`` to use. """ _cleanup() prebuild_info = prepare_request_for_build( request_id, RequestConfigCreateIndexImage( _binary_image=binary_image, from_index=from_index, binary_image_config=binary_image_config, ), ) from_index_resolved = prebuild_info['from_index_resolved'] prebuild_info['labels'] = labels _update_index_image_build_state(request_id, prebuild_info) with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir: set_request_state(request_id, 'in_progress', 'Checking operators present in index image') operators = _get_present_operators(from_index_resolved, temp_dir) set_request_state(request_id, 'in_progress', 'Removing operators from index image') _opm_index_rm(temp_dir, operators, prebuild_info['binary_image'], from_index_resolved) set_request_state(request_id, 'in_progress', 'Getting and updating labels for new index image') iib_labels = { 'com.redhat.index.delivery.version': prebuild_info['ocp_version'], 'com.redhat.index.delivery.distribution_scope': prebuild_info['distribution_scope'], } iib_labels.update(labels) for index_label, value in iib_labels.items(): _add_label_to_index(index_label, value, temp_dir, 'index.Dockerfile') arches = prebuild_info['arches'] for arch in sorted(arches): _build_image(temp_dir, 'index.Dockerfile', request_id, arch) _push_image(request_id, arch) set_request_state(request_id, 'in_progress', 'Creating the manifest list') output_pull_spec = _create_and_push_manifest_list(request_id, arches) _update_index_image_pull_spec( output_pull_spec=output_pull_spec, request_id=request_id, arches=arches, from_index=from_index, resolved_prebuild_from_index=from_index_resolved, ) set_request_state(request_id, 'complete', 'The empty index image was successfully created')