def test_adjust_csv_annotations(tmpdir):
manifests_dir = tmpdir.mkdir('manifests')
manifests_dir.join('backup.crd.yaml').write(
'apiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition'
)
csv = manifests_dir.join('mig-operator.v1.1.1.clusterserviceversion.yaml')
csv.write(
'apiVersion: operators.coreos.com/v1alpha1\nkind: ClusterServiceVersion'
)
operator_manifest = OperatorManifest.from_directory(str(manifests_dir))
build._adjust_csv_annotations(operator_manifest.files, 'amqp-streams',
'company-marketplace')
with open(csv, 'r') as f:
csv_content = yaml.load(f)
assert csv_content == {
'apiVersion': 'operators.coreos.com/v1alpha1',
'kind': 'ClusterServiceVersion',
'metadata': {
'annotations': {
'marketplace.company.io/remote-workflow':
('https://marketplace.company.com/en-us/operators/amqp-streams/pricing'
),
'marketplace.company.io/support-workflow':
('https://marketplace.company.com/en-us/operators/amqp-streams/support'
),
}
},
}
def replace_image_references(manifest_dir, replacements_file, dry_run=False):
"""
Use replacements_file to modify the image references in the CSVs found in the manifest_dir.
:param str manifest_dir: the path to the directory where the manifest files are stored
:param file replacements_file: the file-like object to the replacements file. The format of
this file is a simple JSON object where each attribute is a string representing the
original image reference and the value is a string representing the new value for the
image reference
:param bool dry_run: whether or not to apply the replacements
:raises ValueError: if more than one CSV in manifest_dir
:raises ValueError: if validation fails
"""
abs_manifest_dir = _normalize_dir_path(manifest_dir)
logger.info('Replacing image references in CSV')
operator_manifest = OperatorManifest.from_directory(abs_manifest_dir)
if not _should_apply_replacements(manifest_dir):
logger.warning('Skipping replacements')
return
replacements = {}
for k, v in json.load(replacements_file).items():
replacements[ImageName.parse(k)] = ImageName.parse(v)
logger.info('%s -> %s', k, v)
operator_manifest.csv.replace_pullspecs_everywhere(replacements)
logger.info('Setting related images section')
operator_manifest.csv.set_related_images()
if not dry_run:
operator_manifest.csv.dump()
logger.info('Image references replaced')
def test_directory_does_not_exist(self, tmpdir):
nonexistent = tmpdir.join("nonexistent")
with pytest.raises(RuntimeError) as exc_info:
OperatorManifest.from_directory(str(nonexistent))
msg = "Path does not exist or is not a directory: {}".format(
nonexistent)
assert str(exc_info.value) == msg
regular_file = tmpdir.join("some_file")
regular_file.write("hello")
with pytest.raises(RuntimeError) as exc_info:
OperatorManifest.from_directory(str(regular_file))
msg = "Path does not exist or is not a directory: {}".format(
regular_file)
assert str(exc_info.value) == msg
def _should_apply_replacements(manifest_dir):
abs_manifest_dir = _normalize_dir_path(manifest_dir)
operator_manifest = OperatorManifest.from_directory(abs_manifest_dir)
if operator_manifest.csv.has_related_images():
csv_file_name = os.path.basename(operator_manifest.csv.path)
if operator_manifest.csv.has_related_image_envs():
raise ValueError(
f'The ClusterServiceVersion file {csv_file_name} has entries in '
'spec.relatedImages and one or more containers have RELATED_IMAGE_* '
'environment variables set. This is not allowed.')
return False
return True
def test_adjust_csv_annotations_no_customizations(mock_yaml_dump, tmpdir):
manifests_dir = tmpdir.mkdir('manifests')
manifests_dir.join('backup.crd.yaml').write(
'apiVersion: apiextensions.k8s.io/v1beta1\nkind: CustomResourceDefinition'
)
csv = manifests_dir.join('mig-operator.v1.1.1.clusterserviceversion.yaml')
csv.write(
'apiVersion: operators.coreos.com/v1alpha1\nkind: ClusterServiceVersion'
)
operator_manifest = OperatorManifest.from_directory(str(manifests_dir))
build._adjust_csv_annotations(operator_manifest.files, 'amqp-streams',
'mos-eisley')
mock_yaml_dump.assert_not_called()
def test_from_directory_no_csvs(self, tmpdir):
subdir = tmpdir.mkdir("nested")
original = tmpdir.join("original.yaml")
replaced = subdir.join("replaced.yaml")
original_data = ORIGINAL.data
original_data["kind"] = "IDK"
with open(str(original), "w") as f:
yaml.dump(original_data, f)
replaced_data = REPLACED.data
del replaced_data["kind"]
with open(str(replaced), "w") as f:
yaml.dump(replaced_data, f)
manifest = OperatorManifest.from_directory(str(tmpdir))
assert manifest.files == []
def test_from_directory(self, tmpdir):
subdir = tmpdir.mkdir("nested")
original = tmpdir.join("original.yaml")
original.write(ORIGINAL.content)
replaced = subdir.join("replaced.yaml")
replaced.write(REPLACED.content)
manifest = OperatorManifest.from_directory(str(tmpdir))
original_csv = manifest.files[0]
replaced_csv = manifest.files[1]
assert original_csv.path == str(original)
assert replaced_csv.path == str(replaced)
assert original_csv.data == ORIGINAL.data
assert replaced_csv.data == REPLACED.data
def test_replace_image_name_from_labels(mock_gil, name_label, tmpdir):
manifests_dir = tmpdir.mkdir('manifests')
csv1 = manifests_dir.join('1.clusterserviceversion.yaml')
csv_template = textwrap.dedent("""\
apiVersion: operators.example.com/v1
kind: ClusterServiceVersion
metadata:
name: amqstreams.v1.0.0
namespace: placeholder
annotations:
containerImage: {registry}/{operator}{image}{ref}
""")
image_digest = '654321'
mock_gil.return_value = {'name': name_label, 'version': 'rhel-8'}
csv_related_images_template = csv_template + textwrap.dedent("""\
spec:
relatedImages:
- name: {related_name}
image: {registry}/{operator}{image}{related_ref}
""")
csv1.write(
csv_related_images_template.format(
registry='quay.io',
operator='operator',
image='/image',
ref=':v1',
related_name=f'image-{image_digest}-annotation',
related_ref='@sha256:749327',
))
operator_manifest = OperatorManifest.from_directory(str(manifests_dir))
bundle_metadata = build_regenerate_bundle._get_bundle_metadata(
operator_manifest, False, perform_sanity_checks=False)
build_regenerate_bundle._replace_image_name_from_labels(
bundle_metadata, '{name}-original-{version}')
assert csv1.read_text('utf-8') == csv_related_images_template.format(
registry='quay.io',
ref=':v1',
related_name='image-654321-annotation',
related_ref='@sha256:749327',
operator=name_label,
image='-original-rhel-8',
)
assert mock_gil.call_count == 2
def test_apply_repo_enclosure(original_image, eclosure_namespace,
expected_image, tmpdir):
manifests_dir = tmpdir.mkdir('manifests')
csv1 = manifests_dir.join('1.clusterserviceversion.yaml')
csv_template = textwrap.dedent("""\
apiVersion: operators.example.com/v1
kind: ClusterServiceVersion
metadata:
name: amqstreams.v1.0.0
namespace: placeholder
annotations:
containerImage: {registry}/{operator}{image}{ref}
""")
image_digest = '654321'
csv_related_images_template = csv_template + textwrap.dedent("""\
spec:
relatedImages:
- name: {related_name}
image: {registry}/{operator}{image}{related_ref}
""")
csv1.write(
csv_related_images_template.format(
registry='quay.io',
operator='operator',
image=original_image,
ref=':v1',
related_name=f'image-{image_digest}-annotation',
related_ref='@sha256:749327',
))
operator_manifest = OperatorManifest.from_directory(str(manifests_dir))
bundle_metadata = build_regenerate_bundle._get_bundle_metadata(
operator_manifest, False, perform_sanity_checks=False)
build_regenerate_bundle._apply_repo_enclosure(bundle_metadata,
eclosure_namespace, '----')
assert csv1.read_text('utf-8') == csv_related_images_template.format(
registry='quay.io',
ref=':v1',
related_name='image-654321-annotation',
related_ref='@sha256:749327',
operator=f'{eclosure_namespace}/',
image=expected_image,
)
def extract_image_references(manifest_dir, output):
"""
Identify all the image references from the CSVs found in manifest_dir.
:param str manifest_dir: the path to the directory where the manifest files are stored
:param file output: the file-like object to store the extracted image references
:return: the list of image references extracted from the CSVs
:rtype: list<str>
:raises ValueError: if more than one CSV in manifest_dir
"""
abs_manifest_dir = _normalize_dir_path(manifest_dir)
logger.info('Extracting image references from %s', abs_manifest_dir)
operator_manifest = OperatorManifest.from_directory(abs_manifest_dir)
image_references = [
str(pullspec) for pullspec in operator_manifest.csv.get_pullspecs()
]
json.dump(image_references, output)
return image_references
def test_replace_image_name_from_labels_invalid_labels(mock_gil, tmpdir):
manifests_dir = tmpdir.mkdir('manifests')
csv1 = manifests_dir.join('1.clusterserviceversion.yaml')
csv_template = textwrap.dedent("""\
apiVersion: operators.example.com/v1
kind: ClusterServiceVersion
metadata:
name: amqstreams.v1.0.0
namespace: placeholder
annotations:
containerImage: {registry}/{operator}{image}{ref}
""")
image_digest = '654321'
mock_gil.return_value = {'name': 'namespace/reponame', 'version': 'rhel-8'}
csv_related_images_template = csv_template + textwrap.dedent("""\
spec:
relatedImages:
- name: {related_name}
image: {registry}/{operator}{image}{related_ref}
""")
csv1.write(
csv_related_images_template.format(
registry='quay.io',
operator='operator',
image='/image',
ref=':v1',
related_name=f'image-{image_digest}-annotation',
related_ref='@sha256:749327',
))
operator_manifest = OperatorManifest.from_directory(str(manifests_dir))
bundle_metadata = build_regenerate_bundle._get_bundle_metadata(
operator_manifest, False, perform_sanity_checks=False)
expected = (
r' is missing one or more label\(s\) required in the '
r'image_name_from_labels {name}-original-{unknown_label}. Available labels: name, version'
)
with pytest.raises(IIBError, match=expected):
build_regenerate_bundle._replace_image_name_from_labels(
bundle_metadata, '{name}-original-{unknown_label}')
def _adjust_operator_bundle(manifests_path, metadata_path, organization=None):
"""
Apply modifications to the operator manifests at the given location.
For any container image pull spec found in the Operator CSV files, replace floating
tags with pinned digests, e.g. `image:latest` becomes `image@sha256:...`.
If spec.relatedImages is not set, it will be set with the pinned digests. If it is set but
there are also RELATED_IMAGE_* environment variables set, an exception will be raised.
This method relies on the OperatorManifest class to properly identify and apply the
modifications as needed.
:param str manifests_path: the full path to the directory containing the operator manifests.
:param str metadata_path: the full path to the directory containing the bundle metadata files.
:param str organization: the organization this bundle is for. If no organization is provided,
no custom behavior will be applied.
:raises IIBError: if the operator manifest has invalid entries
:return: a dictionary of labels to set on the bundle
:rtype: dict
"""
package_name, labels = _apply_package_name_suffix(metadata_path,
organization)
operator_manifest = OperatorManifest.from_directory(manifests_path)
found_pullspecs = set()
operator_csvs = []
for operator_csv in operator_manifest.files:
if operator_csv.has_related_images():
csv_file_name = os.path.basename(operator_csv.path)
if operator_csv.has_related_image_envs():
raise IIBError(
f'The ClusterServiceVersion file {csv_file_name} has entries in '
'spec.relatedImages and one or more containers have RELATED_IMAGE_* '
'environment variables set. This is not allowed for bundles regenerated with '
'IIB.')
log.debug(
'Skipping pinning since the ClusterServiceVersion file %s has entries in '
'spec.relatedImages',
csv_file_name,
)
continue
operator_csvs.append(operator_csv)
for pullspec in operator_csv.get_pullspecs():
found_pullspecs.add(pullspec)
conf = get_worker_config()
registry_replacements = (conf['iib_organization_customizations'].get(
organization, {}).get('registry_replacements', {}))
# Resolve pull specs to container image digests
replacement_pullspecs = {}
for pullspec in found_pullspecs:
replacement_needed = False
if ':' not in ImageName.parse(pullspec).tag:
replacement_needed = True
# Always resolve the image to make sure it's valid
resolved_image = ImageName.parse(_get_resolved_image(
pullspec.to_str()))
if registry_replacements.get(resolved_image.registry):
replacement_needed = True
resolved_image.registry = registry_replacements[
resolved_image.registry]
if replacement_needed:
log.debug(
'%s will be replaced with %s',
pullspec,
resolved_image.to_str(),
)
replacement_pullspecs[pullspec] = resolved_image
# Apply modifications to the operator bundle image metadata
for operator_csv in operator_csvs:
csv_file_name = os.path.basename(operator_csv.path)
log.info('Replacing the pull specifications on %s', csv_file_name)
operator_csv.replace_pullspecs_everywhere(replacement_pullspecs)
log.info('Setting spec.relatedImages on %s', csv_file_name)
operator_csv.set_related_images()
operator_csv.dump()
if organization:
_adjust_csv_annotations(operator_manifest.files, package_name,
organization)
return labels
def _adjust_operator_bundle(
manifests_path, metadata_path, request_id, organization=None, pinned_by_iib=False
):
"""
Apply modifications to the operator manifests at the given location.
For any container image pull spec found in the Operator CSV files, replace floating
tags with pinned digests, e.g. `image:latest` becomes `image@sha256:...`.
If spec.relatedImages is not set, it will be set with the pinned digests. If it is set but
there are also RELATED_IMAGE_* environment variables set, the relatedImages will be regenerated
and the digests will be pinned again.
This method relies on the OperatorManifest class to properly identify and apply the
modifications as needed.
:param str manifests_path: the full path to the directory containing the operator manifests.
:param str metadata_path: the full path to the directory containing the bundle metadata files.
:param int request_id: the ID of the IIB build request.
:param str organization: the organization this bundle is for. If no organization is provided,
no custom behavior will be applied.
:param bool pinned_by_iib: whether or not the bundle image has already been processed by
IIB to perform image pinning of related images.
:raises IIBError: if the operator manifest has invalid entries
:return: a dictionary of labels to set on the bundle
:rtype: dict
"""
try:
operator_manifest = OperatorManifest.from_directory(manifests_path)
except (ruamel.yaml.YAMLError, ruamel.yaml.constructor.DuplicateKeyError) as e:
error = f'The Operator Manifest is not in a valid YAML format: {e}'
log.exception(error)
raise IIBError(error)
conf = get_worker_config()
organization_customizations = conf['iib_organization_customizations'].get(organization, [])
if not organization_customizations:
organization_customizations = [
{'type': 'resolve_image_pullspecs'},
{'type': 'related_bundles'},
{'type': 'package_name_suffix'},
{'type': 'registry_replacements'},
{'type': 'image_name_from_labels'},
{'type': 'csv_annotations'},
{'type': 'enclose_repo'},
]
annotations_yaml = _get_package_annotations(metadata_path)
package_name = annotations_yaml['annotations'][
'operators.operatorframework.io.bundle.package.v1'
]
labels = {}
# Perform the customizations in order
for customization in organization_customizations:
customization_type = customization['type']
if customization_type == 'package_name_suffix':
package_name_suffix = customization.get('suffix')
if package_name_suffix:
log.info('Applying package_name_suffix : %s', package_name_suffix)
package_name, package_labels = _apply_package_name_suffix(
metadata_path, package_name_suffix
)
labels = {**labels, **package_labels}
elif customization_type == 'registry_replacements':
registry_replacements = customization.get('replacements', {})
if registry_replacements:
log.info('Applying registry replacements')
bundle_metadata = _get_bundle_metadata(operator_manifest, pinned_by_iib)
_apply_registry_replacements(bundle_metadata, registry_replacements)
elif customization_type == 'csv_annotations' and organization:
org_csv_annotations = customization.get('annotations')
if org_csv_annotations:
log.info('Applying csv annotations for organization %s', organization)
_adjust_csv_annotations(operator_manifest.files, package_name, org_csv_annotations)
elif customization_type == 'image_name_from_labels':
org_image_name_template = customization.get('template', '')
if org_image_name_template:
bundle_metadata = _get_bundle_metadata(operator_manifest, pinned_by_iib)
_replace_image_name_from_labels(bundle_metadata, org_image_name_template)
elif customization_type == 'enclose_repo':
org_enclose_repo_namespace = customization.get('namespace')
org_enclose_repo_glue = customization.get('enclosure_glue')
if org_enclose_repo_namespace and org_enclose_repo_glue:
log.info(
'Applying enclose_repo customization with namespace %s and enclosure_glue %s'
' for organizaton %s',
org_enclose_repo_namespace,
org_enclose_repo_glue,
organization,
)
bundle_metadata = _get_bundle_metadata(operator_manifest, pinned_by_iib)
_apply_repo_enclosure(
bundle_metadata, org_enclose_repo_namespace, org_enclose_repo_glue
)
elif customization_type == 'related_bundles':
log.info('Applying related_bundles customization')
bundle_metadata = _get_bundle_metadata(operator_manifest, pinned_by_iib)
_write_related_bundles_file(bundle_metadata, request_id)
elif customization_type == 'resolve_image_pullspecs':
log.info('Resolving image pull specs')
bundle_metadata = _get_bundle_metadata(operator_manifest, pinned_by_iib)
_resolve_image_pull_specs(bundle_metadata, labels, pinned_by_iib)
return labels