def replace_image_references(manifest_dir, replacements_file, dry_run=False):
"""
Use replacements_file to modify the image references in the CSVs found in the manifest_dir.
:param str manifest_dir: the path to the directory where the manifest files are stored
:param file replacements_file: the file-like object to the replacements file. The format of
this file is a simple JSON object where each attribute is a string representing the
original image reference and the value is a string representing the new value for the
image reference
:param bool dry_run: whether or not to apply the replacements
:raises ValueError: if more than one CSV in manifest_dir
:raises ValueError: if validation fails
"""
abs_manifest_dir = _normalize_dir_path(manifest_dir)
logger.info('Replacing image references in CSV')
operator_manifest = OperatorManifest.from_directory(abs_manifest_dir)
if not _should_apply_replacements(manifest_dir):
logger.warning('Skipping replacements')
return
replacements = {}
for k, v in json.load(replacements_file).items():
replacements[ImageName.parse(k)] = ImageName.parse(v)
logger.info('%s -> %s', k, v)
operator_manifest.csv.replace_pullspecs_everywhere(replacements)
logger.info('Setting related images section')
operator_manifest.csv.set_related_images()
if not dry_run:
operator_manifest.csv.dump()
logger.info('Image references replaced')
def test_image_name_parse_image_name(caplog):
warning = 'Attempting to parse ImageName test:latest as an ImageName'
test = ImageName.parse("test")
assert warning not in caplog.text
image_test = ImageName.parse(test)
assert warning in caplog.text
assert test is image_test
def _resolve_image_pull_specs(bundle_metadata, labels, pinned_by_iib):
"""
Resolve image pull specifications to container image digests.
:param dict bundle_metadata: the dictionary of CSV's and relatedImages pull specifications
:param dict labels: the dictionary of labels to be set on the bundle image
:param bool pinned_by_iib: whether or not the bundle image has already been processed by
IIB to perform image pinning of related images.
"""
# Resolve pull specs to container image digests
replacement_pullspecs = {}
for pullspec in bundle_metadata['found_pullspecs']:
new_pullspec = ImageName.parse(pullspec.to_str())
if not pinned_by_iib:
# Resolve the image only if it has not already been processed by IIB. This
# helps making sure the pullspec is valid
resolved_image = ImageName.parse(get_resolved_image(pullspec.to_str()))
# If the tag is in the format "<algorithm>:<checksum>", the image is already pinned.
# Otherwise, always pin it to a digest.
if ':' not in ImageName.parse(pullspec).tag:
log.debug('%s will be pinned to %s', pullspec, resolved_image.to_str())
new_pullspec = resolved_image
labels['com.redhat.iib.pinned'] = 'true'
replacement_pullspecs[pullspec] = new_pullspec
if replacement_pullspecs:
_replace_csv_pullspecs(bundle_metadata, replacement_pullspecs)
def _overwrite_from_index(request_id,
output_pull_spec,
from_index,
overwrite_from_index_token=None):
"""
Overwrite the ``from_index`` image.
:param int request_id: the ID of the request this index image is for.
:param str output_pull_spec: the pull specification of the manifest list for the index image
that IIB built.
:param str from_index: the pull specification of the image to overwrite.
:param str overwrite_from_index_token: the user supplied token to use when overwriting the
``from_index`` image. If this is not set, IIB's configured credentials will be used.
:raises IIBError: if one of the skopeo commands fails.
"""
state_reason = f'Overwriting the index image {from_index} with {output_pull_spec}'
log.info(state_reason)
set_request_state(request_id, 'in_progress', state_reason)
new_index_src = f'docker://{output_pull_spec}'
temp_dir = None
try:
if overwrite_from_index_token:
output_pull_spec_registry = ImageName.parse(
output_pull_spec).registry
from_index_registry = ImageName.parse(from_index).registry
# If the registries are the same and `overwrite_from_index_token` was supplied, that
# means that IIB's token will likely not have access to read the `from_index` image.
# This means IIB must first export the manifest list and all the manifests locally and
# then overwrite the `from_index` image with the exported version using the user
# supplied token.
#
# When a newer version of buildah is available in RHEL 8, then that can be used instead
# of the manifest-tool to create the manifest list locally which means this workaround
# can be removed.
if output_pull_spec_registry == from_index_registry:
temp_dir = tempfile.TemporaryDirectory(prefix='iib-')
new_index_src = f'oci:{temp_dir.name}'
log.info(
'The registry used by IIB (%s) is also the registry where from_index (%s) will '
'be overwritten using the user supplied token. Will perform a workaround which '
'will cause the manifest digests to change but the content is the same.',
output_pull_spec_registry,
from_index_registry,
)
exc_msg = f'Failed to export {output_pull_spec} to the OCI format'
_skopeo_copy(f'docker://{output_pull_spec}',
new_index_src,
copy_all=True,
exc_msg=exc_msg)
exc_msg = f'Failed to overwrite the input from_index container image of {from_index}'
with set_registry_token(overwrite_from_index_token, from_index):
_skopeo_copy(new_index_src,
f'docker://{from_index}',
copy_all=True,
exc_msg=exc_msg)
finally:
if temp_dir:
temp_dir.cleanup()
def test_image_name_comparison():
# make sure that both "==" and "!=" are implemented right on both Python major releases
i1 = ImageName(registry='foo.com', namespace='spam', repo='bar', tag='1')
i2 = ImageName(registry='foo.com', namespace='spam', repo='bar', tag='1')
assert i1 == i2
assert not i1 != i2
i2 = ImageName(registry='foo.com', namespace='spam', repo='bar', tag='2')
assert not i1 == i2
assert i1 != i2
def _resolve_image_pull_specs(bundle_metadata, labels, pinned_by_iib,
registry_replacements):
"""
Resolve image pull specifications to container image digests.
:param dict bundle_metadata: the dictionary of CSV's and relatedImages pull specifications
:param dict labels: the dictionary of labels to be set on the bundle image
:param str registry_replacements: the customization dictionary which specifies replacement of
registry in the pull specifications.
:param bool pinned_by_iib: whether or not the bundle image has already been processed by
IIB to perform image pinning of related images.
"""
# Resolve pull specs to container image digests
replacement_pullspecs = {}
for pullspec in bundle_metadata['found_pullspecs']:
replacement_needed = False
new_pullspec = ImageName.parse(pullspec.to_str())
if not pinned_by_iib:
# Resolve the image only if it has not already been processed by IIB. This
# helps making sure the pullspec is valid
resolved_image = ImageName.parse(
get_resolved_image(pullspec.to_str()))
# If the tag is in the format "<algorithm>:<checksum>", the image is already pinned.
# Otherwise, always pin it to a digest.
if ':' not in ImageName.parse(pullspec).tag:
log.debug('%s will be pinned to %s', pullspec,
resolved_image.to_str())
new_pullspec = resolved_image
replacement_needed = True
labels['com.redhat.iib.pinned'] = 'true'
# Apply registry modifications
new_registry = registry_replacements.get(new_pullspec.registry)
if new_registry:
replacement_needed = True
new_pullspec.registry = new_registry
if replacement_needed:
log.debug('%s will be replaced with %s', pullspec,
new_pullspec.to_str())
replacement_pullspecs[pullspec] = new_pullspec
# Apply modifications to the operator bundle image metadata
for operator_csv in bundle_metadata['operator_csvs']:
csv_file_name = os.path.basename(operator_csv.path)
log.info('Replacing the pull specifications on %s', csv_file_name)
operator_csv.replace_pullspecs_everywhere(replacement_pullspecs)
log.info('Setting spec.relatedImages on %s', csv_file_name)
operator_csv.set_related_images()
operator_csv.dump()
def set_registry_token(token, container_image):
"""
Configure authentication to the registry that ``container_image`` is from.
This context manager will reset the authentication to the way it was after it exits. If
``token`` is falsy, this context manager will do nothing.
:param str token: the token in the format of ``username:password``
:param str container_image: the pull specification of the container image to parse to determine
the registry this token is for.
:return: None
:rtype: None
"""
if not token:
log.debug(
'Not changing the Docker configuration since no overwrite_from_index_token was provided'
)
yield
return
if not container_image:
log.debug('Not changing the Docker configuration since no from_index was provided')
yield
return
registry = ImageName.parse(container_image).registry
encoded_token = base64.b64encode(token.encode('utf-8')).decode('utf-8')
registry_auths = {'auths': {registry: {'auth': encoded_token}}}
with set_registry_auths(registry_auths):
yield
def set_registry_token(token, container_image):
"""
Configure authentication to the registry that ``container_image`` is from.
This context manager will reset the authentication to the way it was after it exits. If
``token`` is falsy, this context manager will do nothing.
:param str token: the token in the format of ``username:password``
:param str container_image: the pull specification of the container image to parse to determine
the registry this token is for.
:return: None
:rtype: None
"""
if not token:
log.debug(
'Not changing the Docker configuration since no overwrite_from_index_token was provided'
)
yield
return
if not container_image:
log.debug(
'Not changing the Docker configuration since no from_index was provided'
)
yield
return
docker_config_path = os.path.join(os.path.expanduser('~'), '.docker',
'config.json')
try:
log.debug('Removing the Docker config symlink at %s',
docker_config_path)
try:
os.remove(docker_config_path)
except FileNotFoundError:
log.debug('The Docker config symlink at %s does not exist',
docker_config_path)
conf = get_worker_config()
if os.path.exists(conf.iib_docker_config_template):
with open(conf.iib_docker_config_template, 'r') as f:
docker_config = json.load(f)
else:
docker_config = {}
registry = ImageName.parse(container_image).registry
log.debug(
'Setting the override token for the registry %s in the Docker config',
registry)
docker_config.setdefault('auths', {})
encoded_token = base64.b64encode(token.encode('utf-8')).decode('utf-8')
docker_config['auths'][registry] = {'auth': encoded_token}
with open(docker_config_path, 'w') as f:
json.dump(docker_config, f)
yield
finally:
reset_docker_config()
def _replace_image_name_from_labels(bundle_metadata, replacement_template):
"""
Replace repo/image-name in the CSV pull specs with values from their labels.
:param dict bundle_metadata: the dictionary of CSV's and relatedImages pull specifications
:param str replacement_template: the template specifying which label values to use for
replacement
"""
replacement_pullspecs = {}
for pullspec in bundle_metadata['found_pullspecs']:
new_pullspec = ImageName.parse(pullspec.to_str())
pullspec_labels = get_image_labels(pullspec.to_str())
try:
modified_namespace_repo = replacement_template.format(**pullspec_labels)
except KeyError:
raise IIBError(
f'Pull spec {pullspec.to_str()} is missing one or more label(s)'
f' required in the image_name_from_labels {replacement_template}.'
f' Available labels: {", ".join(list(pullspec_labels.keys()))}'
)
namespace_repo_list = modified_namespace_repo.split('/', 1)
if len(namespace_repo_list) == 1:
namespace_repo_list.insert(0, None)
new_pullspec.namespace, new_pullspec.repo = namespace_repo_list
replacement_pullspecs[pullspec] = new_pullspec
# Related images have already been set when resolving pull_specs.
_replace_csv_pullspecs(bundle_metadata, replacement_pullspecs)
def test_write_related_bundles_file(mock_gwc, mock_gil, tmpdir):
related_bundles_dir = tmpdir.mkdir('related_bundles')
mock_gwc.return_value = {
'iib_request_related_bundles_dir': related_bundles_dir
}
mock_gil.side_effect = ['true', 'false', None]
related_bundles = related_bundles_dir.join('1_related_bundles.json')
bundle_metadata = {
'found_pullspecs': [
ImageName.parse('bundle1:not_latest'),
ImageName.parse('not_a_bundle:latest'),
ImageName.parse('simple_container_image:latest'),
]
}
build_regenerate_bundle._write_related_bundles_file(bundle_metadata, 1)
assert json.load(related_bundles) == ['bundle1:not_latest']
assert mock_gil.call_count == 3
def test_write_related_bundles_file_already_present(mock_gwc, mock_gil,
tmpdir):
related_bundles_dir = tmpdir.mkdir('related_bundles')
mock_gwc.return_value = {
'iib_request_related_bundles_dir': related_bundles_dir
}
related_bundles = related_bundles_dir.join('1_related_bundles.json')
related_bundles.write('random text')
bundle_metadata = {
'found_pullspecs': [
ImageName.parse('bundle1:latest'),
ImageName.parse('not_a_bundle:latest'),
ImageName.parse('simple_container_image:latest'),
]
}
mock_gil.side_effect = ['true', 'true', None]
build_regenerate_bundle._write_related_bundles_file(bundle_metadata, 1)
mock_gil.call_count == 3
def test_image_name_enclose(repo, organization, enclosed_repo, registry, tag):
reference = repo
if tag:
reference = '{}:{}'.format(repo, tag)
if registry:
reference = '{}/{}'.format(registry, reference)
image_name = ImageName.parse(reference)
assert image_name.get_repo() == repo
assert image_name.registry == registry
assert image_name.tag == (tag or 'latest')
image_name.enclose(organization)
assert image_name.get_repo() == enclosed_repo
# Verify that registry and tag are unaffected
assert image_name.registry == registry
assert image_name.tag == (tag or 'latest')
def _apply_registry_replacements(bundle_metadata, registry_replacements):
"""
Apply registry replacements from the config customizations.
:param dict bundle_metadata: the dictionary of CSV's and relatedImages pull specifications
:param str registry_replacements: the customization dictionary which specifies replacement of
registry in the pull specifications.
"""
replacement_pullspecs = {}
for pullspec in bundle_metadata['found_pullspecs']:
new_pullspec = ImageName.parse(pullspec.to_str())
# Apply registry modifications
new_registry = registry_replacements.get(new_pullspec.registry)
if new_registry:
new_pullspec.registry = new_registry
replacement_pullspecs[pullspec] = new_pullspec
if replacement_pullspecs:
_replace_csv_pullspecs(bundle_metadata, replacement_pullspecs)
def _apply_repo_enclosure(bundle_metadata, org_enclose_repo_namespace, org_enclose_repo_glue):
"""
Apply repo_enclosure customization to the bundle image.
:param dict bundle_metadata: the dictionary of CSV's and relatedImages pull specifications
:param str org_enclose_repo_namespace: the string sprecifying the namespace of the modified
pull specs in the CSV files
:param str org_enclose_repo_glue: the string specifying the enclosure glue to be applied
to modify the pull specs in the CSV files
"""
replacement_pullspecs = {}
for pullspec in bundle_metadata['found_pullspecs']:
new_pullspec = ImageName.parse(pullspec.to_str())
repo_parts = new_pullspec.repo.split('/')
if new_pullspec.namespace and org_enclose_repo_namespace != new_pullspec.namespace:
repo_parts.insert(0, new_pullspec.namespace)
new_pullspec.namespace = org_enclose_repo_namespace
new_pullspec.repo = org_enclose_repo_glue.join(repo_parts)
replacement_pullspecs[pullspec] = new_pullspec
_replace_csv_pullspecs(bundle_metadata, replacement_pullspecs)
def _adjust_operator_bundle(manifests_path, metadata_path, organization=None):
"""
Apply modifications to the operator manifests at the given location.
For any container image pull spec found in the Operator CSV files, replace floating
tags with pinned digests, e.g. `image:latest` becomes `image@sha256:...`.
If spec.relatedImages is not set, it will be set with the pinned digests. If it is set but
there are also RELATED_IMAGE_* environment variables set, an exception will be raised.
This method relies on the OperatorManifest class to properly identify and apply the
modifications as needed.
:param str manifests_path: the full path to the directory containing the operator manifests.
:param str metadata_path: the full path to the directory containing the bundle metadata files.
:param str organization: the organization this bundle is for. If no organization is provided,
no custom behavior will be applied.
:raises IIBError: if the operator manifest has invalid entries
:return: a dictionary of labels to set on the bundle
:rtype: dict
"""
package_name, labels = _apply_package_name_suffix(metadata_path,
organization)
operator_manifest = OperatorManifest.from_directory(manifests_path)
found_pullspecs = set()
operator_csvs = []
for operator_csv in operator_manifest.files:
if operator_csv.has_related_images():
csv_file_name = os.path.basename(operator_csv.path)
if operator_csv.has_related_image_envs():
raise IIBError(
f'The ClusterServiceVersion file {csv_file_name} has entries in '
'spec.relatedImages and one or more containers have RELATED_IMAGE_* '
'environment variables set. This is not allowed for bundles regenerated with '
'IIB.')
log.debug(
'Skipping pinning since the ClusterServiceVersion file %s has entries in '
'spec.relatedImages',
csv_file_name,
)
continue
operator_csvs.append(operator_csv)
for pullspec in operator_csv.get_pullspecs():
found_pullspecs.add(pullspec)
conf = get_worker_config()
registry_replacements = (conf['iib_organization_customizations'].get(
organization, {}).get('registry_replacements', {}))
# Resolve pull specs to container image digests
replacement_pullspecs = {}
for pullspec in found_pullspecs:
replacement_needed = False
if ':' not in ImageName.parse(pullspec).tag:
replacement_needed = True
# Always resolve the image to make sure it's valid
resolved_image = ImageName.parse(_get_resolved_image(
pullspec.to_str()))
if registry_replacements.get(resolved_image.registry):
replacement_needed = True
resolved_image.registry = registry_replacements[
resolved_image.registry]
if replacement_needed:
log.debug(
'%s will be replaced with %s',
pullspec,
resolved_image.to_str(),
)
replacement_pullspecs[pullspec] = resolved_image
# Apply modifications to the operator bundle image metadata
for operator_csv in operator_csvs:
csv_file_name = os.path.basename(operator_csv.path)
log.info('Replacing the pull specifications on %s', csv_file_name)
operator_csv.replace_pullspecs_everywhere(replacement_pullspecs)
log.info('Setting spec.relatedImages on %s', csv_file_name)
operator_csv.set_related_images()
operator_csv.dump()
if organization:
_adjust_csv_annotations(operator_manifest.files, package_name,
organization)
return labels
assert str(exc_info.value) == msg
regular_file = tmpdir.join("some_file")
regular_file.write("hello")
with pytest.raises(RuntimeError) as exc_info:
OperatorManifest.from_directory(str(regular_file))
msg = "Path does not exist or is not a directory: {}".format(
regular_file)
assert str(exc_info.value) == msg
TEST_DATA = {
"repository.com/image-name:latest":
ImageName(registry="repository.com", repo="image-name"),
"repository.com/prefix/image-name:1":
ImageName(registry="repository.com",
namespace="prefix",
repo="image-name",
tag="1"),
"repository.com/prefix/image-name@sha256:12345":
ImageName(registry="repository.com",
namespace="prefix",
repo="image-name",
tag="sha256:12345"),
"repository.com/prefix/image-name:latest":
ImageName(registry="repository.com", namespace="prefix",
repo="image-name"),
"image-name:latest":
ImageName(repo="image-name"),
def __init__(self, name, value, replace, path):
self._name = name
self._value = ImageName.parse(value)
self._replace = ImageName.parse(replace)
self._path = path
def find_in_data(self, data):
return ImageName.parse(chain_get(data, self.path))
def test_image_name_parse():
for inp, parsed in TEST_DATA.items():
assert ImageName.parse(inp) == parsed