def _push_image(request_id, arch):
"""
Push the single arch container image to the configured registry.
:param int request_id: the ID of the IIB build request
:param str arch: the architecture of the container image to push
:raises IIBError: if the push fails
"""
source = _get_local_pull_spec(request_id, arch)
destination = _get_external_arch_pull_spec(request_id,
arch,
include_transport=True)
log.info('Pushing the container image %s to %s', source, destination)
run_cmd(
['podman', 'push', '-q', source, destination],
exc_msg=
f'Failed to push the container image to {destination} for the arch {arch}',
)
log.debug(
f'Verifying that {destination} was pushed as a v2 manifest due to RHBZ#1810768'
)
skopeo_raw = skopeo_inspect(destination, '--raw')
if skopeo_raw['schemaVersion'] != 2:
log.warning(
'The manifest for %s ended up using schema version 1 due to RHBZ#1810768. Manually '
'fixing it with skopeo.',
destination,
)
exc_msg = f'Failed to fix the manifest schema version on {destination}'
_skopeo_copy(destination, destination, exc_msg=exc_msg)
def test_run_cmd_failed_opm(mock_sub_run):
mock_rv = mock.Mock()
mock_rv.returncode = 1
mock_rv.stderr = (
'time="2020-03-09T08:58:21-04:00" level=info msg="loading bundle file" '
'dir=bundle_tmp922306995/manifests file=volumesnapshotlocation.crd.yaml load=bundle\n'
'time="2020-03-09T08:58:21-04:00" level=fatal msg="permissive mode disabled" '
'bundles="[quay.io/ns/some_bundle:v1.0]" error="error loading bundle from image: Error '
'adding package error loading bundle into db: cam-operator.v1.0.1 specifies replacement '
'that couldn\'t be found"')
mock_sub_run.return_value = mock_rv
expected_exc = (
'Failed to add the bundles to the index image: error loading bundle from image: Error '
'adding package error loading bundle into db: cam-operator.v1.0.1 specifies replacement '
'that couldn\'t be found')
with pytest.raises(IIBError, match=expected_exc):
utils.run_cmd(
[
'opm', 'index', 'add', '--generate', '--bundles',
'quay.io/ns/some_bundle:v1.0'
],
exc_msg='Failed to add the bundles to the index image',
)
mock_sub_run.assert_called_once()
def _skopeo_copy(source, destination, copy_all=False, exc_msg=None):
"""
Wrap the ``skopeo copy`` command.
:param str source: the source to copy
:param str destination: the destination to copy the source to
:param bool copy_all: if True, it passes ``--all`` to the command
:param str exc_msg: a custom exception message to provide
:raises IIBError: if the copy fails
"""
skopeo_timeout = get_worker_config()['iib_skopeo_timeout']
log.debug('Copying the container image %s to %s', source, destination)
args = [
'skopeo',
'--command-timeout',
skopeo_timeout,
'copy',
'--format',
'v2s2',
]
if copy_all:
args.append('--all')
args.extend([source, destination])
run_cmd(
args,
exc_msg=exc_msg or f'Failed to copy {source} to {destination}',
)
def _build_image(dockerfile_dir, dockerfile_name, request_id, arch):
"""
Build the index image for the specified architecture.
:param str dockerfile_dir: the path to the directory containing the data used for
building the container image
:param str dockerfile_name: the name of the Dockerfile in the dockerfile_dir to
be used when building the container image
:param int request_id: the ID of the IIB build request
:param str arch: the architecture to build this image for
:raises IIBError: if the build fails
"""
destination = _get_local_pull_spec(request_id, arch)
log.info(
'Building the container image with the %s dockerfile for arch %s and tagging it as %s',
dockerfile_name,
arch,
destination,
)
dockerfile_path = os.path.join(dockerfile_dir, dockerfile_name)
run_cmd(
[
'buildah',
'bud',
'--no-cache',
'--override-arch',
arch,
'-t',
destination,
'-f',
dockerfile_path,
],
{'cwd': dockerfile_dir},
exc_msg=f'Failed to build the container image on the arch {arch}',
)
def _deprecate_bundles(bundles,
base_dir,
binary_image,
from_index,
overwrite_target_index_token=None):
"""
Deprecate the specified bundles from the index image.
Only Dockerfile is created, no build is performed.
:param list bundles: pull specifications of bundles to deprecate.
:param str base_dir: base directory where operation files will be located.
:param str binary_image: binary image to be used by the new index image.
:param str from_index: index image, from which the bundles will be deprecated.
:param str overwrite_target_index_token: the token used for overwriting the input
``from_index`` image. This is required for non-privileged users to use
``overwrite_target_index``. The format of the token must be in the format "user:password".
"""
cmd = [
'opm',
'index',
'deprecatetruncate',
'--generate',
'--binary-image',
binary_image,
'--from-index',
from_index,
'--bundles',
','.join(bundles),
]
with set_registry_token(overwrite_target_index_token, from_index):
run_cmd(cmd, {'cwd': base_dir},
exc_msg='Failed to deprecate the bundles')
def opm_migrate(index_db, base_dir):
"""
Migrate SQLite database to File-Based catalog using opm command.
:param str index_db: path to SQLite index.db which should migrated to FBC.
:param str base_dir: base directory where catalog should be created.
:return: Returns path to directory containing file-based catalog.
:rtype: str
"""
from iib.workers.tasks.utils import run_cmd
fbc_dir_path = os.path.join(base_dir, 'catalog')
# It may happen that we need to regenerate file-based catalog
# based on updated index.db therefore we have to remove the outdated catalog
# to be able to generate new one
if os.path.exists(fbc_dir_path):
shutil.rmtree(fbc_dir_path)
cmd = ['opm', 'migrate', index_db, fbc_dir_path]
run_cmd(cmd, {'cwd': base_dir},
exc_msg='Failed to migrate index.db to file-based catalog')
log.info("Migration to file-based catalog was completed.")
return fbc_dir_path
def _opm_index_export(rebuilt_index_image, package, temp_dir):
"""
Export the package that needs to be backported.
:param str rebuilt_index_image: the pull specification of the index image rebuilt by IIB.
:param set package: a string representing the name of the package to be exported.
:param str temp_dir: path to the temporary directory where the package will be exported to.
:raises IIBError: if the export of packages fails.
"""
cmd = [
'opm',
'index',
'export',
'--index',
rebuilt_index_image,
'--package',
package,
'--download-folder',
package,
]
log.info('Generating the backported operator for package: %s', package)
run_cmd(
cmd,
{'cwd': temp_dir},
exc_msg=f'Failed to push {package} to the legacy application registry',
)
def opm_registry_deprecatetruncate(base_dir, index_db, bundles):
"""
Deprecate bundles from index.db.
:param str base_dir: base directory where operation files will be located.
:param str index_db: path to index.db used with opm registry deprecatetruncate.
:param list bundles: pull specifications of bundles to deprecate.
"""
from iib.workers.tasks.utils import run_cmd
log.debug(
'Run opm registry deprecatetruncate on database %s and bundles %s',
index_db,
' '.join(bundles),
)
cmd = [
'opm',
'registry',
'deprecatetruncate',
'--database',
index_db,
'--bundle-images',
','.join(bundles),
'--allow-package-removal',
]
run_cmd(cmd, {'cwd': base_dir},
exc_msg=f'Failed to deprecate the bundles on {index_db}')
def test_run_cmd(mock_sub_run):
mock_rv = mock.Mock()
mock_rv.returncode = 0
mock_sub_run.return_value = mock_rv
utils.run_cmd(['echo', 'hello world'], {'cwd': '/some/path'})
mock_sub_run.assert_called_once()
def test_run_cmd_failed(mock_sub_run, exc_msg):
mock_rv = mock.Mock()
mock_rv.returncode = 1
mock_rv.stderr = 'some failure'
mock_sub_run.return_value = mock_rv
expected_exc = exc_msg or 'An unexpected error occurred'
with pytest.raises(IIBError, match=expected_exc):
utils.run_cmd(['echo', 'hello'], exc_msg=exc_msg)
mock_sub_run.assert_called_once()
def test_run_cmd_failed_opm(mock_sub_run):
mock_rv = mock.Mock()
mock_rv.returncode = 1
mock_rv.stderr = textwrap.dedent('''
time="2020-05-12T15:42:19Z" level=info msg="loading bundle file" dir=bundle_tmp775962984/manifests file=serverstatusrequest.crd.yaml load=bundle
time="2020-05-12T15:42:19Z" level=info msg="loading bundle file" dir=bundle_tmp775962984/manifests file=volumesnapshotlocation.crd.yaml load=bundle
time="2020-05-12T15:42:19Z" level=error msg="permissive mode disabled" bundles="[registry/namespace/bundle:v1.0-14]" error="error loading bundle from image: Error adding package error loading bundle into db: cam-operator.v1.0.1 specifies replacement that couldn't be found"
Error: error loading bundle from image: Error adding package error loading bundle into db: cam-operator.v1.0.1 specifies replacement that couldn't be found
Usage:
opm index add [flags]
Examples:
# Create an index image from scratch with a single bundle image
opm index add --bundles quay.io/operator-framework/operator-bundle-prometheus@sha256:a3ee653ffa8a0d2bbb2fabb150a94da6e878b6e9eb07defd40dc884effde11a0 --tag quay.io/operator-framework/monitoring:1.0.0
# Add a single bundle image to an index image
opm index add --bundles quay.io/operator-framework/operator-bundle-prometheus:0.15.0 --from-index quay.io/operator-framework/monitoring:1.0.0 --tag quay.io/operator-framework/monitoring:1.0.1
# Add multiple bundles to an index and generate a Dockerfile instead of an image
opm index add --bundles quay.io/operator-framework/operator-bundle-prometheus:0.15.0,quay.io/operator-framework/operator-bundle-prometheus:0.22.2 --generate
Flags:
-i, --binary-image opm container image for on-image opm command
-b, --bundles strings comma separated list of bundles to add
-c, --container-tool string tool to interact with container images (save, build, etc.). One of: [docker, podman] (default "podman")
-f, --from-index string previous index to add to
--generate if enabled, just creates the dockerfile and saves it to local disk
-h, --help help for add
--mode string graph update mode that defines how channel graphs are updated. One of: [replaces, semver, semver-skippatch] (default "replaces")
-d, --out-dockerfile string if generating the dockerfile, this flag is used to (optionally) specify a dockerfile name
--permissive allow registry load errors
--skip-tls skip TLS certificate verification for container image registries while pulling bundles
-t, --tag string custom tag for container image being built
'''
# noqa: E501
)
mock_sub_run.return_value = mock_rv
expected_exc = (
'Failed to add the bundles to the index image: error loading bundle from image: Error '
'adding package error loading bundle into db: cam-operator.v1.0.1 specifies replacement '
'that couldn\'t be found')
with pytest.raises(IIBError, match=expected_exc):
utils.run_cmd(
[
'opm', 'index', 'add', '--generate', '--bundles',
'quay.io/ns/some_bundle:v1.0'
],
exc_msg='Failed to add the bundles to the index image',
)
mock_sub_run.assert_called_once()
def _opm_registry_add(
base_dir,
index_db,
bundles,
overwrite_csv=False,
container_tool=None,
):
"""
Add the input bundles to an operator index database.
This only runs operations on index database.
:param str base_dir: the base directory to generate the database and index.Dockerfile in.
:param str index_db: relative path to SQLite index.db database file
:param list bundles: a list of strings representing the pull specifications of the bundles to
add to the index image being built.
:param bool overwrite_csv: a boolean determining if a bundle will be replaced if the CSV
already exists.
:param str container_tool: the container tool to be used to operate on the index image
"""
from iib.workers.tasks.utils import run_cmd
# The bundles are not resolved since these are stable tags, and references
# to a bundle image using a digest fails when using the opm command.
bundle_str = ','.join(bundles) or '""'
cmd = [
'opm',
'registry',
'add',
'--database',
index_db,
# This enables substitutes-for functionality for rebuilds. See
# https://github.com/operator-framework/enhancements/blob/master/enhancements/substitutes-for.md
'--enable-alpha',
'--bundle-images',
bundle_str,
]
if container_tool:
cmd.append('--container-tool')
cmd.append(container_tool)
log.info('Generating the database file with the following bundle(s): %s',
', '.join(bundles))
if overwrite_csv:
log.info('Using force to add bundle(s) to index')
cmd.extend(['--overwrite-latest'])
run_cmd(cmd, {'cwd': base_dir},
exc_msg='Failed to add the bundles to the index image')
def _cleanup():
"""
Remove all existing container images on the host.
This will ensure that the host will not run out of disk space due to stale data, and that
all images referenced using floating tags will be up to date on the host.
:raises IIBError: if the command to remove the container images fails
"""
log.info('Removing all existing container images')
run_cmd(
['podman', 'rmi', '--all', '--force'],
exc_msg='Failed to remove the existing container images',
)
def test_run_cmd_failed_buildah(mock_sub_run):
mock_rv = mock.Mock()
mock_rv.returncode = 1
mock_rv.stderr = textwrap.dedent('''
2021-12-14 08:52:39,144 iib.workers.tasks.utils DEBUG utils.run_cmd Running the command "buildah bud --no-cache --override-arch s390x --arch s390x -t iib-build:56056-s390x -f /tmp/iib-ozo81z6o/index.Dockerfile"
2021-12-14 08:55:10,212 iib.workers.tasks.utils ERROR utils.run_cmd The command "buildah bud --no-cache --override-arch s390x --arch s390x -t iib-build:56056-s390x -f /tmp/iib-ozo81z6o/index.Dockerfile" failed with: Trying to pull registry.redhat.io/openshift4/ose-operator-registry@sha256:72498731bbea4307178f9d0d237bf2a8439bfa8f580f87c35e5a73cb1c854bd6...
Getting image source signatures
Checking if image destination supports signatures
Copying blob sha256:b381d16488eb8afbbaed78ff48e8b4702b04c236400524dfd2ae759127422edf
Copying blob sha256:27cb39a08c6eb46426e92622c4edea9b9b8495b2401d02c773e239dd40d99a22
Copying blob sha256:3eabe22a2aec9181c0849b1a23a6104a81bcf00bea55a52a45dba613f0afd896
Copying blob sha256:3224b0f72681ebcfaec3c51b3d7efe187a5cab0355b4bbe6cffadde0d17d2292
Copying blob sha256:45ac5acd44f7a277e412330b36e908278d979fa0de30ca0628ef0729f61d825e
Copying blob sha256:45ac5acd44f7a277e412330b36e908278d979fa0de30ca0628ef0729f61d825e
Copying blob sha256:3eabe22a2aec9181c0849b1a23a6104a81bcf00bea55a52a45dba613f0afd896
Copying blob sha256:27cb39a08c6eb46426e92622c4edea9b9b8495b2401d02c773e239dd40d99a22
error creating build container: reading blob sha256:3224b0f72681ebcfaec3c51b3d7efe187a5cab0355b4bbe6cffadde0d17d2292: Error fetching blob: invalid status code from registry 503 (Service Unavailable)
time="2021-12-14T08:55:10-05:00" level=error msg="exit status 125"
'''
# noqa: E501
)
mock_sub_run.return_value = mock_rv
expected_exc = (
r'Failed build the index image: error creating build container: 503 \(Service Unavailable\)'
)
with pytest.raises(ExternalServiceError, match=expected_exc):
utils.run_cmd(
[
'buildah',
'bud',
'--no-cache',
'--format',
'docker',
'--override-arch',
's390x',
'--arch',
's390x',
'-t',
'iib-build:56056-s390x',
'-f',
'/tmp/iib-ozo81z6o/index.Dockerfile',
],
exc_msg='Failed build the index image',
)
mock_sub_run.assert_called_once()
def _opm_index_add(base_dir,
bundles,
binary_image,
from_index=None,
overwrite_from_index_token=None):
"""
Add the input bundles to an operator index.
This only produces the index.Dockerfile file and does not build the container image.
:param str base_dir: the base directory to generate the database and index.Dockerfile in.
:param list bundles: a list of strings representing the pull specifications of the bundles to
add to the index image being built.
:param str binary_image: the pull specification of the container image where the opm binary
gets copied from. This should point to a digest or stable tag.
:param str from_index: the pull specification of the container image containing the index that
the index image build will be based from.
:param str overwrite_from_index_token: the token used for overwriting the input
``from_index`` image. This is required for non-privileged users to use
``overwrite_from_index``. The format of the token must be in the format "user:password".
:raises IIBError: if the ``opm index add`` command fails.
"""
# The bundles are not resolved since these are stable tags, and references
# to a bundle image using a digest fails when using the opm command.
cmd = [
'opm',
'index',
'add',
'--generate',
'--bundles',
','.join(bundles),
'--binary-image',
binary_image,
]
log.info('Generating the database file with the following bundle(s): %s',
', '.join(bundles))
if from_index:
log.info('Using the existing database from %s', from_index)
# from_index is not resolved because podman does not support digest references
# https://github.com/containers/libpod/issues/5234 is filed for it
cmd.extend(['--from-index', from_index])
with set_registry_token(overwrite_from_index_token, from_index):
run_cmd(
cmd,
{'cwd': base_dir},
exc_msg='Failed to add the bundles to the index image',
)
def _create_and_push_manifest_list(request_id, arches):
"""
Create and push the manifest list to the configured registry.
:param int request_id: the ID of the IIB build request
:param iter arches: an iterable of arches to create the manifest list for
:return: the pull specification of the manifest list
:rtype: str
:raises IIBError: if creating or pushing the manifest list fails
"""
output_pull_spec = get_rebuilt_image_pull_spec(request_id)
log.info('Creating the manifest list %s', output_pull_spec)
with tempfile.TemporaryDirectory(prefix='iib-') as temp_dir:
manifest_yaml = os.path.abspath(os.path.join(temp_dir,
'manifest.yaml'))
with open(manifest_yaml, 'w+') as manifest_yaml_f:
manifest_yaml_f.write(
textwrap.dedent(f'''\
image: {output_pull_spec}
manifests:
'''))
for arch in sorted(arches):
arch_pull_spec = _get_external_arch_pull_spec(request_id, arch)
log.debug(
'Adding the manifest %s to the manifest list %s',
arch_pull_spec,
output_pull_spec,
)
manifest_yaml_f.write(
textwrap.dedent(f'''\
- image: {arch_pull_spec}
platform:
architecture: {arch}
os: linux
'''))
# Return back to the beginning of the file to output it to the logs
manifest_yaml_f.seek(0)
log.debug(
'Created the manifest configuration with the following content:\n%s',
manifest_yaml_f.read(),
)
run_cmd(
['manifest-tool', 'push', 'from-spec', manifest_yaml],
exc_msg=f'Failed to push the manifest list to {output_pull_spec}',
)
return output_pull_spec
def _opm_index_rm(base_dir,
operators,
binary_image,
from_index,
overwrite_from_index_token=None):
"""
Remove the input operators from the operator index.
This only produces the index.Dockerfile file and does not build the container image.
:param str base_dir: the base directory to generate the database and index.Dockerfile in.
:param list operators: a list of strings representing the names of the operators to
remove from the index image.
:param str binary_image: the pull specification of the container image where the opm binary
gets copied from.
:param str from_index: the pull specification of the container image containing the index that
the index image build will be based from.
:param str overwrite_from_index_token: the token used for overwriting the input
``from_index`` image. This is required for non-privileged users to use
``overwrite_from_index``. The format of the token must be in the format "user:password".
:raises IIBError: if the ``opm index rm`` command fails.
"""
cmd = [
'opm',
'index',
'rm',
'--generate',
'--binary-image',
binary_image,
'--from-index',
from_index,
'--operators',
','.join(operators),
]
log.info(
'Generating the database file from an existing database %s and excluding'
' the following operator(s): %s',
from_index,
', '.join(operators),
)
with set_registry_token(overwrite_from_index_token, from_index):
run_cmd(
cmd,
{'cwd': base_dir},
exc_msg='Failed to remove operators from the index image',
)
def _requires_max_ocp_version(bundle):
"""
Check if the bundle requires the olm.maxOpenShiftVersion property.
This property is required for bundles using deprecated APIs that don't already
have the olm.maxOpenShiftVersion property set.
:param str bundle: a string representing the bundle pull specification
:returns bool:
"""
cmd = [
'operator-sdk',
'bundle',
'validate',
bundle,
'--select-optional',
'name=community',
'--output=json-alpha1',
'--image-builder',
'none',
]
result = run_cmd(cmd, strict=False)
if result:
output = json.loads(result)
# check if the bundle validation failed
if not output['passed']:
# check if the failure is due to the presence of deprecated APIs
# and absence of the 'olm.maxOpenShiftVersion' property
# Note: there is no other error in the sdk that mentions this field
for msg in output['outputs']:
if 'olm.maxOpenShiftVersion' in msg['message']:
return True
return False
def _serve_cmd_at_port(serve_cmd, cwd, port, max_tries, wait_time):
"""
Start an opm service at a specified port.
:param list serve_cmd: opm command to be run (serve FBC or index.db).
:param str cwd: path to folder which should be set as current working directory.
:param str int port: port to start the service on.
:param max_tries: how many times to try to start the service before giving up.
:param wait_time: time to wait before checking if the service is initialized.
:return: object of the running Popen process.
:rtype: subprocess.Popen
:raises IIBError: if the process has failed to initialize too many times, or an unexpected
error occurred.
:raises AddressAlreadyInUse: if the specified port is already being used by another service.
"""
from iib.workers.tasks.utils import run_cmd
log.debug('Run command %s with up to %d retries', ' '.join(serve_cmd),
max_tries)
for _ in range(max_tries):
rpc_proc = subprocess.Popen(
serve_cmd,
cwd=cwd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
)
start_time = time.time()
while time.time() - start_time < wait_time:
time.sleep(1)
ret = rpc_proc.poll()
# process has terminated
if ret is not None:
stderr = rpc_proc.stderr.read()
if 'address already in use' in stderr:
raise AddressAlreadyInUse(
f'Port {port} is already used by a different service')
raise IIBError(
f'Command "{" ".join(serve_cmd)}" has failed with error "{stderr}"'
)
# query the service to see if it has started
try:
output = run_cmd([
'grpcurl', '-plaintext', f'localhost:{port}', 'list',
'api.Registry'
])
except IIBError:
output = ''
if 'api.Registry.ListBundles' in output or 'api.Registry.ListPackages' in output:
log.debug('Started the command "%s"', ' '.join(serve_cmd))
log.info('Index registry service has been initialized.')
return rpc_proc
rpc_proc.terminate()
raise IIBError(
f'Index registry has not been initialized after {max_tries} tries')
def test_run_cmd_with_cmd_repr(mock_sub_run, caplog):
# Setting the logging level via caplog.set_level is not sufficient. The flask
# related settings from previous tests interfere with this.
utils_logger = logging.getLogger('iib.workers.tasks.utils')
utils_logger.disabled = False
utils_logger.setLevel(logging.DEBUG)
mock_sub_run.return_value = mock.Mock(returncode=0)
secret = 'top-secret'
secret_redacted = '*****'
utils.run_cmd(['echo', secret], cmd_repr=['echo', secret_redacted])
mock_sub_run.assert_called_once()
assert secret not in caplog.text
assert secret_redacted in caplog.text
def _serve_index_registry_at_port(db_path, port, max_tries, wait_time):
"""
Start an image registry service at a specified port.
:param str db_path: path to index database containing the registry data.
:param str int port: port to start the service on.
:param max_tries: how many times to try to start the service before giving up.
:param wait_time: time to wait before checking if the service is initialized.
:return: object of the running Popen process.
:rtype: Popen
:raises IIBError: if the process has failed to initialize too many times, or an unexpected
error occured.
:raises AddressAlreadyInUse: if the specified port is already being used by another service.
"""
cmd = [
'opm', 'registry', 'serve', '-p',
str(port), '-d', db_path, '-t', '/dev/null'
]
for attempt in range(max_tries):
rpc_proc = subprocess.Popen(
cmd,
cwd=os.path.dirname(db_path),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
)
start_time = time.time()
while time.time() - start_time < wait_time:
time.sleep(1)
ret = rpc_proc.poll()
# process has terminated
if ret is not None:
stderr = rpc_proc.stderr.read()
if 'address already in use' in stderr:
raise AddressAlreadyInUse(
f'Port {port} is already used by a different service')
raise IIBError(
f'Command "{" ".join(cmd)}" has failed with error "{stderr}"'
)
# query the service to see if it has started
try:
output = run_cmd([
'grpcurl', '-plaintext', f'localhost:{port}', 'list',
'api.Registry'
])
except IIBError:
output = ''
if 'api.Registry.ListBundles' in output:
log.debug('Started the command "%s"', ' '.join(cmd))
log.info('Index registry service has been initialized.')
return rpc_proc
rpc_proc.kill()
raise IIBError(
f'Index registry has not been initialized after {max_tries} tries')
def _cleanup():
"""
Remove all existing container images on the host.
This will ensure that the host will not run out of disk space due to stale data, and that
all images referenced using floating tags will be up to date on the host.
Additionally, this function will reset the Docker ``config.json`` to
``iib_docker_config_template``.
:raises IIBError: if the command to remove the container images fails
"""
log.info('Removing all existing container images')
run_cmd(
['podman', 'rmi', '--all', '--force'],
exc_msg='Failed to remove the existing container images',
)
reset_docker_config()
def _build_image(dockerfile_dir: str, dockerfile_name: str, request_id: int, arch: str) -> None:
"""
Build the index image for the specified architecture.
:param str dockerfile_dir: the path to the directory containing the data used for
building the container image
:param str dockerfile_name: the name of the Dockerfile in the dockerfile_dir to
be used when building the container image
:param int request_id: the ID of the IIB build request
:param str arch: the architecture to build this image for
:raises IIBError: if the build fails
"""
destination: str = _get_local_pull_spec(request_id, arch)
log.info(
'Building the container image with the %s dockerfile for arch %s and tagging it as %s',
dockerfile_name,
arch,
destination,
)
dockerfile_path: str = os.path.join(dockerfile_dir, dockerfile_name)
# NOTE: It's important to provide both --override-arch and --arch to ensure the metadata
# on the image, **and** on its config blob are set correctly.
#
# NOTE: The argument "--format docker" ensures buildah will not generate an index image with
# default OCI v1 manifest but always use Docker v2 format.
run_cmd(
[
'buildah',
'bud',
'--no-cache',
'--format',
'docker',
'--override-arch',
arch,
'--arch',
arch,
'-t',
destination,
'-f',
dockerfile_path,
],
{'cwd': dockerfile_dir},
exc_msg=f'Failed to build the container image on the arch {arch}',
)
def _opm_registry_rm(index_db_path, operators, base_dir):
"""
Generate and run the opm command to remove operator package from index db provided.
:param str index_db_path: path where the input index image is temporarily copied
:param list operators: list of operator packages to be removed
:param base_dir: the base directory to generate the database and index.Dockerfile in.
"""
from iib.workers.tasks.utils import run_cmd
cmd = [
'opm',
'registry',
'rm',
'--database',
index_db_path,
'--packages',
','.join(operators),
]
run_cmd(cmd, {'cwd': base_dir},
exc_msg='Failed to remove operators from the index image')
def test_run_cmd_failed_buildah_manifest_rm(mock_sub_run):
mock_rv = mock.Mock()
mock_rv.returncode = 1
mock_rv.stderr = textwrap.dedent('''
1 error occurred:
* something: image not known
'''
# noqa: E501
)
mock_sub_run.return_value = mock_rv
expected_exc = 'Manifest list not found locally.'
with pytest.raises(IIBError, match=expected_exc):
utils.run_cmd(
['buildah', 'manifest', 'rm', 'something'],
exc_msg=
'Failed to remove local manifest list. something does not exist',
)
mock_sub_run.assert_called_once()
def _opm_index_rm(base_dir, operators, binary_image, from_index):
"""
Remove the input operators from the operator index.
This only produces the index.Dockerfile file and does not build the container image.
:param str base_dir: the base directory to generate the database and index.Dockerfile in.
:param list operators: a list of strings representing the names of the operators to
remove from the index image.
:param str binary_image: the pull specification of the container image where the opm binary
gets copied from.
:param str from_index: the pull specification of the container image containing the index that
the index image build will be based from.
:raises IIBError: if the ``opm index rm`` command fails.
"""
cmd = [
'opm',
'index',
'rm',
'--generate',
'--binary-image',
binary_image,
'--from-index',
from_index,
'--operators',
','.join(operators),
]
log.info(
'Generating the database file from an existing database %s and excluding'
' the following operator(s): %s',
from_index,
', '.join(operators),
)
run_cmd(
cmd,
{'cwd': base_dir},
exc_msg='Failed to remove operators from the index image',
)
def _copy_files_from_image(image, src_path, dest_path):
"""
Copy a file from the container image into the given destination path.
The file may be a directory.
:param str image: the pull specification of the container image.
:param str src_path: the full path within the container image to copy from.
:param str dest_path: the full path on the local host to copy into.
"""
# Check that image is pullable
podman_pull(image)
# One way to copy a file from the image is to create a container from its filesystem
# so the contents can be read. To create a container, podman always requires that a
# command for the container is set. In this method, however, the command is not needed
# because the container is never started, only created. Use a dummy command to satisfy
# podman.
container_command = 'unused'
container_id = run_cmd(
['podman', 'create', image, container_command],
exc_msg=f'Failed to create a container for {image}',
).strip()
try:
run_cmd(
['podman', 'cp', f'{container_id}:{src_path}', dest_path],
exc_msg=
f'Failed to copy the contents of {container_id}:{src_path} into {dest_path}',
)
finally:
try:
run_cmd(
['podman', 'rm', container_id],
exc_msg=
f'Failed to remove the container {container_id} for image {image}',
)
except IIBError as e:
# Failure to remove the temporary container shouldn't cause the IIB request to fail.
log.exception(e)
def _skopeo_copy(source,
destination,
copy_all=False,
dest_token=None,
exc_msg=None):
"""
Wrap the ``skopeo copy`` command.
:param str source: the source to copy
:param str destination: the destination to copy the source to
:param bool copy_all: if True, it passes ``--all`` to the command
:param str dest_token: the token to pass to the ``--dest-token` parameter of the command.
If not provided, ``--dest-token`` parameter is also not provided.
:param str exc_msg: a custom exception message to provide
:raises IIBError: if the copy fails
"""
skopeo_timeout = get_worker_config()['iib_skopeo_timeout']
log.debug('Copying the container image %s to %s', source, destination)
cmd = [
'skopeo',
'--command-timeout',
skopeo_timeout,
'copy',
'--format',
'v2s2',
]
if copy_all:
cmd.append('--all')
if dest_token:
log.debug('Using user-provided token to copy the container image')
cmd.append('--dest-creds')
cmd.append(dest_token)
cmd.extend([source, destination])
cmd_repr = ['*****' if part == dest_token else part for part in cmd]
run_cmd(cmd,
exc_msg=exc_msg or f'Failed to copy {source} to {destination}',
cmd_repr=cmd_repr)
def test_run_cmd_failed_buildah_registry_unavailable(
mock_sub_run: mock.MagicMock) -> None:
mock_rv: mock.Mock = mock.Mock()
mock_rv.returncode = 1
mock_rv.stderr = textwrap.dedent('''
error creating build container:
Error determining manifest MIME type for docker://registry.redhat.io/openshift4/ose-operator-registry@sha256:8f3d471eccaad18e61fe6326c544cfcfaff35c012c6d5c4da01cbe887e03b904:
Error reading manifest sha256:db6fd9f033865da55ab2e4647ae283a51556cd11ef4241361ac04cb05b5ef795 in registry.redhat.io/openshift4/ose-operator-registry:
received unexpected HTTP status: 503 Service Unavailable
'''
# noqa: E501 W291
).replace('\n', '')
mock_sub_run.return_value = mock_rv
expected_exc: str = 'error creating build container: 503 Service Unavailable'
with pytest.raises(ExternalServiceError, match=expected_exc):
utils.run_cmd(
[
'buildah',
'bud',
'--no-cache',
'--format',
'docker',
'--override-arch',
'amd64',
'--arch',
'amd64',
'-t',
'foo',
'-f',
'bar',
],
exc_msg=f'Failed to build the container image on the arch amd64',
)
mock_sub_run.assert_called_once()
def _get_present_bundles(from_index, base_dir):
"""
Get a list of bundles already present in the index image.
:param str from_index: index image to inspect.
:param str base_dir: base directory to create temporary files in.
:return: list of unique present bundles as provided by the grpc query and a list of unique
bundle pull_specs
:rtype: list, list
:raises IIBError: if any of the commands fail.
"""
db_path = _get_index_database(from_index, base_dir)
port, rpc_proc = _serve_index_registry(db_path)
bundles = run_cmd(
[
'grpcurl', '-plaintext', f'localhost:{port}',
'api.Registry/ListBundles'
],
exc_msg='Failed to get bundle data from index image',
)
rpc_proc.kill()
# If no data is returned there are not bundles present
if not bundles:
return [], []
# Transform returned data to parsable json
unique_present_bundles = []
unique_present_bundles_pull_spec = []
present_bundles = [
json.loads(bundle) for bundle in re.split(r'(?<=})\n(?={)', bundles)
]
for bundle in present_bundles:
bundle_path = bundle['bundlePath']
if bundle_path in unique_present_bundles_pull_spec:
continue
unique_present_bundles.append(bundle)
unique_present_bundles_pull_spec.append(bundle_path)
return unique_present_bundles, unique_present_bundles_pull_spec
