def test_cleanup(mock_rdc, mock_run_cmd): build._cleanup() mock_run_cmd.assert_called_once() rmi_args = mock_run_cmd.call_args[0][0] assert rmi_args[0:2] == ['podman', 'rmi'] mock_rdc.assert_called_once_with()
def handle_create_empty_index_request( from_index, request_id, output_fbc=False, binary_image=None, labels=None, binary_image_config=None, ): """Coordinate the the work needed to create the index image with labels. :param str from_index: the pull specification of the container image containing the index that the index image build will be based from. :param int request_id: the ID of the IIB build request :param bool output_fbc: specifies whether a File-based Catalog index image should be created :param str binary_image: the pull specification of the container image where the opm binary gets copied from. :param dict labels: the dict of labels required to be added to a new index image :param dict binary_image_config: the dict of config required to identify the appropriate ``binary_image`` to use. """ _cleanup() prebuild_info = prepare_request_for_build( request_id, RequestConfigCreateIndexImage( _binary_image=binary_image, from_index=from_index, binary_image_config=binary_image_config, ), ) from_index_resolved = prebuild_info['from_index_resolved'] prebuild_info['labels'] = labels if not output_fbc and is_image_fbc(from_index_resolved): log.debug('%s is FBC index image', from_index_resolved) err_msg = 'Cannot create SQLite index image from File-Based Catalog index image' log.error(err_msg) raise IIBError(err_msg) _update_index_image_build_state(request_id, prebuild_info) with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir: set_request_state(request_id, 'in_progress', 'Checking operators present in index image') operators = _get_present_operators(from_index_resolved, temp_dir) # if output_fbc parameter is true, create an empty FBC index image # else create empty SQLite index image if output_fbc: log.debug('Creating empty FBC index image from %s', from_index) opm_create_empty_fbc( request_id=request_id, temp_dir=temp_dir, from_index_resolved=from_index_resolved, from_index=from_index, binary_image=prebuild_info['binary_image'], operators=operators, ) else: set_request_state(request_id, 'in_progress', 'Removing operators from index image') _opm_index_rm(temp_dir, operators, prebuild_info['binary_image'], from_index_resolved) set_request_state( request_id, 'in_progress', 'Getting and updating labels for new index image' ) iib_labels = { 'com.redhat.index.delivery.version': prebuild_info['ocp_version'], 'com.redhat.index.delivery.distribution_scope': prebuild_info['distribution_scope'], } if labels: iib_labels.update(labels) for index_label, value in iib_labels.items(): _add_label_to_index(index_label, value, temp_dir, 'index.Dockerfile') arches = prebuild_info['arches'] for arch in sorted(arches): _build_image(temp_dir, 'index.Dockerfile', request_id, arch) _push_image(request_id, arch) set_request_state(request_id, 'in_progress', 'Creating the manifest list') output_pull_spec = _create_and_push_manifest_list(request_id, arches, []) _update_index_image_pull_spec( output_pull_spec=output_pull_spec, request_id=request_id, arches=arches, from_index=from_index, resolved_prebuild_from_index=from_index_resolved, ) set_request_state(request_id, 'complete', 'The empty index image was successfully created')
def handle_merge_request( source_from_index, deprecation_list, request_id, binary_image=None, target_index=None, overwrite_target_index=False, overwrite_target_index_token=None, distribution_scope=None, binary_image_config=None, ): """ Coordinate the work needed to merge old (N) index image with new (N+1) index image. :param str source_from_index: pull specification to be used as the base for building the new index image. :param str target_index: pull specification of content stage index image for the corresponding target index image. :param list deprecation_list: list of deprecated bundles for the target index image. :param int request_id: the ID of the IIB build request. :param str binary_image: the pull specification of the container image where the opm binary gets copied from. :param bool overwrite_target_index: if True, overwrite the input ``target_index`` with the built index image. :param str overwrite_target_index_token: the token used for overwriting the input ``target_index`` image. This is required for non-privileged users to use ``overwrite_target_index``. The format of the token must be in the format "user:password". :param str distribution_scope: the scope for distribution of the index image, defaults to ``None``. :raises IIBError: if the index image merge fails. """ _cleanup() prebuild_info = _prepare_request_for_build( request_id, binary_image, overwrite_from_index_token=overwrite_target_index_token, source_from_index=source_from_index, target_index=target_index, distribution_scope=distribution_scope, binary_image_config=binary_image_config, ) _update_index_image_build_state(request_id, prebuild_info) with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir: set_request_state(request_id, 'in_progress', 'Getting bundles present in the index images') log.info('Getting bundles present in the source index image') source_index_bundles = _get_present_bundles(source_from_index, temp_dir) target_index_bundles = [] if target_index: log.info('Getting bundles present in the target index image') target_index_bundles = _get_present_bundles(target_index, temp_dir) arches = list(prebuild_info['arches']) arch = 'amd64' if 'amd64' in arches else arches[0] missing_bundles = _add_bundles_missing_in_source( source_index_bundles, target_index_bundles, temp_dir, prebuild_info['binary_image'], source_from_index, request_id, arch, prebuild_info['target_ocp_version'], overwrite_target_index_token, distribution_scope=prebuild_info['distribution_scope'], ) set_request_state(request_id, 'in_progress', 'Deprecating bundles in the deprecation list') log.info('Deprecating bundles in the deprecation list') intermediate_bundles = source_index_bundles + missing_bundles deprecate_bundles = _get_bundles_from_deprecation_list( intermediate_bundles, deprecation_list) intermediate_image_name = _get_external_arch_pull_spec( request_id, arch, include_transport=False) if deprecate_bundles: _deprecate_bundles( deprecate_bundles, temp_dir, prebuild_info['binary_image'], intermediate_image_name, overwrite_target_index_token, ) for arch in sorted(prebuild_info['arches']): _build_image(temp_dir, 'index.Dockerfile', request_id, arch) _push_image(request_id, arch) output_pull_spec = _create_and_push_manifest_list(request_id, prebuild_info['arches']) _update_index_image_pull_spec( output_pull_spec, request_id, prebuild_info['arches'], target_index, overwrite_target_index, overwrite_target_index_token, prebuild_info['target_index_resolved'], ) set_request_state(request_id, 'complete', 'The index image was successfully cleaned and updated.')
def handle_merge_request( source_from_index, deprecation_list, request_id, binary_image=None, target_index=None, overwrite_target_index=False, overwrite_target_index_token=None, distribution_scope=None, binary_image_config=None, ): """ Coordinate the work needed to merge old (N) index image with new (N+1) index image. :param str source_from_index: pull specification to be used as the base for building the new index image. :param str target_index: pull specification of content stage index image for the corresponding target index image. :param list deprecation_list: list of deprecated bundles for the target index image. :param int request_id: the ID of the IIB build request. :param str binary_image: the pull specification of the container image where the opm binary gets copied from. :param bool overwrite_target_index: if True, overwrite the input ``target_index`` with the built index image. :param str overwrite_target_index_token: the token used for overwriting the input ``target_index`` image. This is required to use ``overwrite_target_index``. The format of the token must be in the format "user:password". :param str distribution_scope: the scope for distribution of the index image, defaults to ``None``. :raises IIBError: if the index image merge fails. """ _cleanup() prebuild_info = prepare_request_for_build( request_id, RequestConfigMerge( _binary_image=binary_image, overwrite_target_index_token=overwrite_target_index_token, source_from_index=source_from_index, target_index=target_index, distribution_scope=distribution_scope, binary_image_config=binary_image_config, ), ) _update_index_image_build_state(request_id, prebuild_info) source_from_index_resolved = prebuild_info['source_from_index_resolved'] target_index_resolved = prebuild_info['target_index_resolved'] with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir: set_request_state(request_id, 'in_progress', 'Getting bundles present in the index images') log.info('Getting bundles present in the source index image') with set_registry_token(overwrite_target_index_token, source_from_index): source_index_bundles, source_index_bundles_pull_spec = _get_present_bundles( source_from_index_resolved, temp_dir) target_index_bundles = [] if target_index: log.info('Getting bundles present in the target index image') target_index_bundles, _ = _get_present_bundles( target_index_resolved, temp_dir) arches = list(prebuild_info['arches']) arch = 'amd64' if 'amd64' in arches else arches[0] missing_bundles, invalid_version_bundles = _add_bundles_missing_in_source( source_index_bundles, target_index_bundles, temp_dir, prebuild_info['binary_image'], source_from_index_resolved, request_id, arch, prebuild_info['target_ocp_version'], overwrite_target_index_token, distribution_scope=prebuild_info['distribution_scope'], ) set_request_state(request_id, 'in_progress', 'Deprecating bundles in the deprecation list') log.info('Deprecating bundles in the deprecation list') intermediate_bundles = [ bundle['bundlePath'] for bundle in missing_bundles ] + source_index_bundles_pull_spec deprecation_bundles = get_bundles_from_deprecation_list( intermediate_bundles, deprecation_list) # We do not need to pass the invalid_version_bundles through the # get_bundles_from_deprecation_list function because we already know # they are present in the newly created index. deprecation_bundles = deprecation_bundles + [ bundle['bundlePath'] for bundle in invalid_version_bundles ] intermediate_image_name = _get_external_arch_pull_spec( request_id, arch, include_transport=False) if deprecation_bundles: deprecate_bundles( deprecation_bundles, temp_dir, prebuild_info['binary_image'], intermediate_image_name, overwrite_target_index_token, ) for arch in sorted(prebuild_info['arches']): _build_image(temp_dir, 'index.Dockerfile', request_id, arch) _push_image(request_id, arch) # If the container-tool podman is used in the opm commands above, opm will create temporary # files and directories without the write permission. This will cause the context manager # to fail to delete these files. Adjust the file modes to avoid this error. chmod_recursively( temp_dir, dir_mode=(stat.S_IRWXU | stat.S_IRWXG), file_mode=(stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP), ) output_pull_spec = _create_and_push_manifest_list(request_id, prebuild_info['arches']) _update_index_image_pull_spec( output_pull_spec, request_id, prebuild_info['arches'], target_index, overwrite_target_index, overwrite_target_index_token, target_index_resolved, ) set_request_state(request_id, 'complete', 'The index image was successfully cleaned and updated.')
def handle_regenerate_bundle_request( from_bundle_image, organization, request_id, registry_auths=None ): """ Coordinate the work needed to regenerate the operator bundle image. :param str from_bundle_image: the pull specification of the bundle image to be regenerated. :param str organization: the name of the organization the bundle should be regenerated for. :param int request_id: the ID of the IIB build request :param dict registry_auths: Provide the dockerconfig.json for authentication to private registries, defaults to ``None``. :raises IIBError: if the regenerate bundle image build fails. """ _cleanup() set_request_state(request_id, 'in_progress', 'Resolving from_bundle_image') with set_registry_auths(registry_auths): from_bundle_image_resolved = get_resolved_image(from_bundle_image) arches = get_image_arches(from_bundle_image_resolved) if not arches: raise IIBError( 'No arches were found in the resolved from_bundle_image ' f'{from_bundle_image_resolved}' ) pinned_by_iib = yaml.load( get_image_label(from_bundle_image_resolved, 'com.redhat.iib.pinned') or 'false' ) arches_str = ', '.join(sorted(arches)) log.debug('Set to regenerate the bundle image for the following arches: %s', arches_str) payload = { 'from_bundle_image_resolved': from_bundle_image_resolved, 'state': 'in_progress', 'state_reason': f'Regenerating the bundle image for the following arches: {arches_str}', } exc_msg = 'Failed setting the resolved "from_bundle_image" on the request' update_request(request_id, payload, exc_msg=exc_msg) # Pull the from_bundle_image to ensure steps later on don't fail due to registry timeouts podman_pull(from_bundle_image_resolved) with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir: manifests_path = os.path.join(temp_dir, 'manifests') _copy_files_from_image(from_bundle_image_resolved, '/manifests', manifests_path) metadata_path = os.path.join(temp_dir, 'metadata') _copy_files_from_image(from_bundle_image_resolved, '/metadata', metadata_path) new_labels = _adjust_operator_bundle( manifests_path, metadata_path, request_id, organization, pinned_by_iib ) with open(os.path.join(temp_dir, 'Dockerfile'), 'w') as dockerfile: dockerfile.write( textwrap.dedent( f"""\ FROM {from_bundle_image_resolved} COPY ./manifests /manifests COPY ./metadata /metadata """ ) ) for name, value in new_labels.items(): dockerfile.write(f'LABEL {name}={value}\n') for arch in sorted(arches): _build_image(temp_dir, 'Dockerfile', request_id, arch) _push_image(request_id, arch) set_request_state(request_id, 'in_progress', 'Creating the manifest list') output_pull_spec = _create_and_push_manifest_list(request_id, arches, []) conf = get_worker_config() if conf['iib_index_image_output_registry']: old_output_pull_spec = output_pull_spec output_pull_spec = output_pull_spec.replace( conf['iib_registry'], conf['iib_index_image_output_registry'], 1 ) log.info( 'Changed the bundle_image pull specification from %s to %s', old_output_pull_spec, output_pull_spec, ) payload = { 'arches': list(arches), 'bundle_image': output_pull_spec, 'state': 'complete', 'state_reason': 'The request completed successfully', } update_request(request_id, payload, exc_msg='Failed setting the bundle image on the request')
def handle_merge_request( source_from_index, deprecation_list, request_id, binary_image=None, target_index=None, overwrite_target_index=False, overwrite_target_index_token=None, distribution_scope=None, binary_image_config=None, build_tags=None, ): """ Coordinate the work needed to merge old (N) index image with new (N+1) index image. :param str source_from_index: pull specification to be used as the base for building the new index image. :param str target_index: pull specification of content stage index image for the corresponding target index image. :param list deprecation_list: list of deprecated bundles for the target index image. :param int request_id: the ID of the IIB build request. :param str binary_image: the pull specification of the container image where the opm binary gets copied from. :param bool overwrite_target_index: if True, overwrite the input ``target_index`` with the built index image. :param str overwrite_target_index_token: the token used for overwriting the input ``target_index`` image. This is required to use ``overwrite_target_index``. The format of the token must be in the format "user:password". :param str distribution_scope: the scope for distribution of the index image, defaults to ``None``. :param build_tags: list of extra tag to use for intermetdiate index image :raises IIBError: if the index image merge fails. """ _cleanup() prebuild_info = prepare_request_for_build( request_id, RequestConfigMerge( _binary_image=binary_image, overwrite_target_index_token=overwrite_target_index_token, source_from_index=source_from_index, target_index=target_index, distribution_scope=distribution_scope, binary_image_config=binary_image_config, ), ) _update_index_image_build_state(request_id, prebuild_info) source_from_index_resolved = prebuild_info['source_from_index_resolved'] target_index_resolved = prebuild_info['target_index_resolved'] dockerfile_name = 'index.Dockerfile' with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir: with set_registry_token(overwrite_target_index_token, source_from_index): source_fbc = is_image_fbc(source_from_index_resolved) target_fbc = is_image_fbc(target_index_resolved) # do not remove - logging requested by stakeholders if source_fbc: log.info("Processing source index image as File-Based Catalog image") if target_fbc: log.info("Processing target index image as File-Based Catalog image") if source_fbc and not target_fbc: err_msg = ( 'Cannot merge source File-Based Catalog index image into target SQLite index image.' ) log.error(err_msg) raise IIBError(err_msg) set_request_state(request_id, 'in_progress', 'Getting bundles present in the index images') log.info('Getting bundles present in the source index image') with set_registry_token(overwrite_target_index_token, source_from_index): source_index_bundles, source_index_bundles_pull_spec = _get_present_bundles( source_from_index_resolved, temp_dir ) target_index_bundles = [] if target_index: log.info('Getting bundles present in the target index image') target_index_bundles, _ = _get_present_bundles(target_index_resolved, temp_dir) arches = list(prebuild_info['arches']) arch = sorted(arches)[0] missing_bundles, invalid_version_bundles = _add_bundles_missing_in_source( source_index_bundles, target_index_bundles, temp_dir, prebuild_info['binary_image'], source_from_index_resolved, request_id, arch, prebuild_info['target_ocp_version'], overwrite_target_index_token, distribution_scope=prebuild_info['distribution_scope'], ) missing_bundle_paths = [bundle['bundlePath'] for bundle in missing_bundles] if missing_bundle_paths: add_max_ocp_version_property(missing_bundle_paths, temp_dir) set_request_state(request_id, 'in_progress', 'Deprecating bundles in the deprecation list') log.info('Deprecating bundles in the deprecation list') intermediate_bundles = missing_bundle_paths + source_index_bundles_pull_spec deprecation_bundles = get_bundles_from_deprecation_list( intermediate_bundles, deprecation_list ) # We do not need to pass the invalid_version_bundles through the # get_bundles_from_deprecation_list function because we already know # they are present in the newly created index. deprecation_bundles = deprecation_bundles + [ bundle['bundlePath'] for bundle in invalid_version_bundles ] if deprecation_bundles: intermediate_image_name = _get_external_arch_pull_spec( request_id, arch, include_transport=False ) # we can check if source index is FBC or not because intermediate_image # will be always the same type because it is built # from source index image in _add_bundles_missing_in_source() if source_fbc: deprecate_bundles_fbc( bundles=deprecation_bundles, base_dir=temp_dir, binary_image=prebuild_info['binary_image'], from_index=intermediate_image_name, ) else: # opm can only deprecate a bundle image on an existing index image. Build and # push a temporary index image to satisfy this requirement. Any arch will do. # NOTE: we cannot use local builds because opm commands fails, # index image has to be pushed to registry _build_image(temp_dir, 'index.Dockerfile', request_id, arch) _push_image(request_id, arch) deprecate_bundles( bundles=deprecation_bundles, base_dir=temp_dir, binary_image=prebuild_info['binary_image'], from_index=intermediate_image_name, overwrite_target_index_token=overwrite_target_index_token, ) if target_fbc: index_db_file = os.path.join(temp_dir, get_worker_config()['temp_index_db_path']) # make sure FBC is generated right before build fbc_dir = opm_migrate(index_db=index_db_file, base_dir=temp_dir) if not source_fbc: # when source image is not FBC, but final image should be an FBC image # we have to generate Dockerfile for FBC (with hidden index.db) dockerfile_path = os.path.join(temp_dir, dockerfile_name) if os.path.isfile(dockerfile_path): log.info('Removing previously generated dockerfile.') os.remove(dockerfile_path) opm_generate_dockerfile( fbc_dir=fbc_dir, base_dir=temp_dir, index_db=index_db_file, binary_image=prebuild_info['binary_image'], dockerfile_name=dockerfile_name, ) _add_label_to_index( 'com.redhat.index.delivery.version', prebuild_info['target_ocp_version'], temp_dir, dockerfile_name, ) _add_label_to_index( 'com.redhat.index.delivery.distribution_scope', prebuild_info['distribution_scope'], temp_dir, dockerfile_name, ) for arch in sorted(prebuild_info['arches']): _build_image(temp_dir, dockerfile_name, request_id, arch) _push_image(request_id, arch) # If the container-tool podman is used in the opm commands above, opm will create temporary # files and directories without the write permission. This will cause the context manager # to fail to delete these files. Adjust the file modes to avoid this error. chmod_recursively( temp_dir, dir_mode=(stat.S_IRWXU | stat.S_IRWXG), file_mode=(stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP), ) output_pull_spec = _create_and_push_manifest_list( request_id, prebuild_info['arches'], build_tags ) _update_index_image_pull_spec( output_pull_spec, request_id, prebuild_info['arches'], target_index, overwrite_target_index, overwrite_target_index_token, target_index_resolved, ) set_request_state( request_id, 'complete', 'The index image was successfully cleaned and updated.' )
def handle_create_empty_index_request(from_index, request_id, binary_image=None, labels=None, binary_image_config=None): """Coordinate the the work needed to create the index image with labels. :param str from_index: the pull specification of the container image containing the index that the index image build will be based from. :param int request_id: the ID of the IIB build request :param str binary_image: the pull specification of the container image where the opm binary gets copied from. :param dict labels: the dict of labels required to be added to a new index image :param dict binary_image_config: the dict of config required to identify the appropriate ``binary_image`` to use. """ _cleanup() prebuild_info = prepare_request_for_build( request_id, RequestConfigCreateIndexImage( _binary_image=binary_image, from_index=from_index, binary_image_config=binary_image_config, ), ) from_index_resolved = prebuild_info['from_index_resolved'] prebuild_info['labels'] = labels _update_index_image_build_state(request_id, prebuild_info) with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir: set_request_state(request_id, 'in_progress', 'Checking operators present in index image') operators = _get_present_operators(from_index_resolved, temp_dir) set_request_state(request_id, 'in_progress', 'Removing operators from index image') _opm_index_rm(temp_dir, operators, prebuild_info['binary_image'], from_index_resolved) set_request_state(request_id, 'in_progress', 'Getting and updating labels for new index image') iib_labels = { 'com.redhat.index.delivery.version': prebuild_info['ocp_version'], 'com.redhat.index.delivery.distribution_scope': prebuild_info['distribution_scope'], } iib_labels.update(labels) for index_label, value in iib_labels.items(): _add_label_to_index(index_label, value, temp_dir, 'index.Dockerfile') arches = prebuild_info['arches'] for arch in sorted(arches): _build_image(temp_dir, 'index.Dockerfile', request_id, arch) _push_image(request_id, arch) set_request_state(request_id, 'in_progress', 'Creating the manifest list') output_pull_spec = _create_and_push_manifest_list(request_id, arches) _update_index_image_pull_spec( output_pull_spec=output_pull_spec, request_id=request_id, arches=arches, from_index=from_index, resolved_prebuild_from_index=from_index_resolved, ) set_request_state(request_id, 'complete', 'The empty index image was successfully created')