def set_registry_token(token, container_image):
"""
Configure authentication to the registry that ``container_image`` is from.
This context manager will reset the authentication to the way it was after it exits. If
``token`` is falsy, this context manager will do nothing.
:param str token: the token in the format of ``username:password``
:param str container_image: the pull specification of the container image to parse to determine
the registry this token is for.
:return: None
:rtype: None
"""
if not token:
log.debug(
'Not changing the Docker configuration since no overwrite_from_index_token was provided'
)
yield
return
if not container_image:
log.debug(
'Not changing the Docker configuration since no from_index was provided'
)
yield
return
docker_config_path = os.path.join(os.path.expanduser('~'), '.docker',
'config.json')
try:
log.debug('Removing the Docker config symlink at %s',
docker_config_path)
try:
os.remove(docker_config_path)
except FileNotFoundError:
log.debug('The Docker config symlink at %s does not exist',
docker_config_path)
conf = get_worker_config()
if os.path.exists(conf.iib_docker_config_template):
with open(conf.iib_docker_config_template, 'r') as f:
docker_config = json.load(f)
else:
docker_config = {}
registry = ImageName.parse(container_image).registry
log.debug(
'Setting the override token for the registry %s in the Docker config',
registry)
docker_config.setdefault('auths', {})
encoded_token = base64.b64encode(token.encode('utf-8')).decode('utf-8')
docker_config['auths'][registry] = {'auth': encoded_token}
with open(docker_config_path, 'w') as f:
json.dump(docker_config, f)
yield
finally:
reset_docker_config()
def _skopeo_copy(source, destination, copy_all=False, exc_msg=None):
"""
Wrap the ``skopeo copy`` command.
:param str source: the source to copy
:param str destination: the destination to copy the source to
:param bool copy_all: if True, it passes ``--all`` to the command
:param str exc_msg: a custom exception message to provide
:raises IIBError: if the copy fails
"""
skopeo_timeout = get_worker_config()['iib_skopeo_timeout']
log.debug('Copying the container image %s to %s', source, destination)
cmd = [
'skopeo',
'--command-timeout',
skopeo_timeout,
'copy',
'--format',
'v2s2',
]
if copy_all:
cmd.append('--all')
cmd.extend([source, destination])
run_cmd(cmd,
exc_msg=exc_msg or f'Failed to copy {source} to {destination}')
def skopeo_inspect(*args, return_json=True, require_media_type=False):
"""
Wrap the ``skopeo inspect`` command.
:param args: any arguments to pass to ``skopeo inspect``
:param bool return_json: if ``True``, the output will be parsed as JSON and returned
:param bool require_media_type: if ``True``, ``mediaType`` will be checked in the output
and it will be ignored when ``return_json`` is ``False``
:return: a dictionary of the JSON output from the skopeo inspect command
:rtype: dict
:raises IIBError: if the command fails and if ``mediaType`` is not found in the output while
``require_media_type`` is ``True``
"""
exc_msg = None
for arg in args:
if arg.startswith('docker://'):
exc_msg = f'Failed to inspect {arg}. Make sure it exists and is accessible to IIB.'
break
skopeo_timeout = get_worker_config().iib_skopeo_timeout
cmd = ['skopeo', '--command-timeout', skopeo_timeout, 'inspect'] + list(args)
output = run_cmd(cmd, exc_msg=exc_msg)
if not return_json:
return output
json_output = json.loads(output)
if require_media_type and not json_output.get('mediaType'):
raise IIBError('mediaType not found')
return json_output
def upload_file_to_s3_bucket(file_path, s3_key_prefix, s3_file_name):
"""
Upload artifact file to AWS S3 bucket.
:param str file_path: the path of the file locally where the artifact file is stored
:param str s3_key_prefix: the logical location of the file in the S3 bucket
:param str s3_file_name: the name of the file in S3 bucket
:raises IIBError: when unable to upload file to the S3 bucket
"""
conf = get_worker_config()
log.info(
'Uploading file %s/%s to S3 bucket: %s',
s3_key_prefix,
s3_file_name,
conf['iib_aws_s3_bucket_name'],
)
s3 = boto3.resource(service_name='s3')
try:
s3.meta.client.upload_file(
Filename=file_path,
Bucket=conf['iib_aws_s3_bucket_name'],
Key=f'{s3_key_prefix}/{s3_file_name}',
)
except ClientError as error:
log.exception(error)
raise IIBError(
f'Unable to upload file {file_path} to bucket {conf["iib_aws_s3_bucket_name"]}: {error}'
)
def test_request_logger(tmpdir):
# Setting the logging level via caplog.set_level is not sufficient. The flask
# related settings from previous tests interfere with this.
utils_logger = logging.getLogger('iib.workers.tasks.utils')
utils_logger.disabled = False
utils_logger.setLevel(logging.DEBUG)
logs_dir = tmpdir.join('logs')
logs_dir.mkdir()
get_worker_config().iib_request_logs_dir = str(logs_dir)
original_handlers_count = len(logging.getLogger().handlers)
@utils.request_logger
def mock_handler(spam, eggs, request_id, bacon):
logging.getLogger('iib.workers.tasks.utils').info('this is a test')
expected_message = ' iib.workers.tasks.utils INFO test_utils.mock_handler this is a test\n'
mock_handler('spam', 'eggs', 123, 'bacon')
assert logs_dir.join('123.log').read().endswith(expected_message)
assert original_handlers_count == len(logging.getLogger().handlers)
mock_handler('spam', 'eggs', bacon='bacon', request_id=321)
assert logs_dir.join('321.log').read().endswith(expected_message)
assert original_handlers_count == len(logging.getLogger().handlers)
def _push_package_manifest(package_dir, cnr_token, organization):
"""
Push ``manifests.zip`` file created for an exported package to OMPS.
:param str package_dir: path to the exported package directory.
:param str cnr_token: the token required to push backported packages to the legacy
app registry via OMPS.
:param str organization: the organization name in the legacy app registry to which
the backported packages should be pushed to.
:raises IIBError: if the push is unsucessful
"""
conf = get_worker_config()
base_dir, _ = _get_base_dir_and_pkg_name(package_dir)
with open(f'{base_dir}/manifests.zip', 'rb') as fobj:
files = {'file': (fobj.name, fobj)}
log.info('Files are %s', files)
resp = requests.post(
f'{conf["iib_omps_url"]}{organization}/zipfile',
headers={'Authorization': cnr_token},
files=files,
)
if not resp.ok:
log.error('Request to OMPS failed: %s', resp.text)
try:
error_msg = resp.json().get('message',
'An unknown error occured')
except json.JSONDecodeError:
error_msg = resp.text
raise IIBError(
f'Push to {organization} in the legacy app registry was unsucessful: {error_msg}'
)
def gate_bundles(bundles, greenwave_config):
"""
Check if all bundle images have passed gating tests in the CVP pipeline.
This function queries Greenwave to check if the policies are satisfied for each bundle image.
:param list bundles: a list of strings representing the pull specifications of the bundles to
be gated.
:param dict greenwave_config: the dict of config required to query Greenwave to gate bundles.
:raises IIBError: if any of the bundles fail the gating checks or IIB fails to get a
response from Greenwave.
"""
conf = get_worker_config()
_validate_greenwave_params_and_config(conf, greenwave_config)
log.info('Gating on bundles: %s', ', '.join(bundles))
gating_unsatisfied_bundles = []
for bundle in bundles:
koji_build_nvr = _get_koji_build_nvr(bundle)
log.debug('Querying Greenwave for decision on %s', koji_build_nvr)
payload = deepcopy(greenwave_config)
payload['subject_identifier'] = koji_build_nvr
log.debug(
'Querying Greenwave with decision_context: %s, product_version: %s, '
'subject_identifier: %s and subject_type: %s',
payload["decision_context"],
payload["product_version"],
payload["subject_identifier"],
payload["subject_type"],
)
request_url = f'{conf["iib_greenwave_url"].rstrip("/")}/decision'
resp = requests.post(request_url, json=payload)
try:
data = resp.json()
except json.JSONDecodeError:
log.error('Error encountered in decoding JSON %s', resp.text)
data = {}
if not resp.ok:
error_msg = data.get('message') or resp.text
log.error('Request to Greenwave failed: %s', error_msg)
raise IIBError(f'Gating check failed for {bundle}: {error_msg}')
try:
if not data['policies_satisfied']:
log.info('Gating decision for %s: %s', bundle, data)
gating_unsatisfied_bundles.append(bundle)
except KeyError:
log.error('Missing key "policies_satisfied" for %s: %s', bundle, data)
raise IIBError(f'Key "policies_satisfied" missing in Greenwave response for {bundle}')
if gating_unsatisfied_bundles:
error_msg = (
f'Unsatisfied Greenwave policy for {", ".join(gating_unsatisfied_bundles)} '
f'with decision_context: {greenwave_config["decision_context"]}, '
f'product_version: {greenwave_config["product_version"]}, '
f'and subject_type: {greenwave_config["subject_type"]}'
)
raise IIBError(error_msg)
def _write_related_bundles_file(bundle_metadata, request_id):
"""
Get bundle images in the CSV files of the bundle being regenerated and store them in a file.
:param dict bundle_metadata: the dictionary of CSV's and relatedImages pull specifications
:param int request_id: the ID of the IIB build request
"""
worker_config = get_worker_config()
related_bundles_dir = worker_config['iib_request_related_bundles_dir']
related_bundles_file = os.path.join(related_bundles_dir, f'{request_id}_related_bundles.json')
related_bundle_images = []
for related_pullspec_obj in bundle_metadata['found_pullspecs']:
related_pullspec = related_pullspec_obj.to_str()
if yaml.load(
get_image_label(related_pullspec, 'com.redhat.delivery.operator.bundle') or 'false'
):
related_bundle_images.append(related_pullspec)
log.debug('Writing related bundle images to %s', related_bundles_file)
with open(related_bundles_file, 'w') as output_file:
json.dump(related_bundle_images, output_file)
if worker_config['iib_aws_s3_bucket_name']:
upload_file_to_s3_bucket(
related_bundles_file, 'related_bundles', f'{request_id}_related_bundles.json'
)
def get_request(request_id):
"""
Get the IIB build request from the REST API.
:param int request_id: the ID of the IIB request
:return: the request
:rtype: dict
:raises IIBError: if the HTTP request fails
"""
# Prevent a circular import
from iib.workers.config import get_worker_config
config = get_worker_config()
request_url = f'{config.iib_api_url.rstrip("/")}/builds/{request_id}'
log.info('Getting the request %d', request_id)
try:
rv = requests_session.get(request_url, timeout=config.iib_api_timeout)
except requests.RequestException:
msg = f'The connection failed when getting the request {request_id}'
log.exception(msg)
raise IIBError(msg)
if not rv.ok:
log.error(
'The worker failed to get the request %d. The status was %d. The text was:\n%s',
request_id,
rv.status_code,
rv.text,
)
raise IIBError(f'The worker failed to get the request {request_id}')
return rv.json()
def _adjust_csv_annotations(operator_csvs, package_name, organization):
"""
Annotate ClusterServiceVersion objects based on an organization configuration.
:param list operator_csvs: the list of ``OperatorCSV`` objects to examine.
:param str package_name: the operator package name.
:param str organization: the organization this bundle is for. This determines what annotations
to make.
"""
conf = get_worker_config()
org_csv_annotations = (conf['iib_organization_customizations'].get(
organization, {}).get('csv_annotations'))
if not org_csv_annotations:
log.debug(
'The organization %s does not have CSV annotations configured',
organization)
return
for operator_csv in operator_csvs:
log.debug('Processing the ClusterServiceVersion file %s',
os.path.basename(operator_csv.path))
csv_annotations = operator_csv.data.setdefault('metadata',
{}).setdefault(
'annotations', {})
for annotation, value_template in org_csv_annotations.items():
value = value_template.format(package_name=package_name)
csv_annotations[annotation] = value
operator_csv.dump()
def _serve_index_registry(db_path):
"""
Locally start OPM registry service, which can be communicated with using gRPC queries.
Due to IIB's paralellism, the service can run multiple times, which could lead to port
binding conflicts. Resolution of port conflicts is handled in this function as well.
:param str db_path: path to index database containing the registry data.
:return: tuple containing port number of the running service and the running Popen object.
:rtype: (int, Popen)
:raises IIBError: if all tried ports are in use, or the command failed for another reason.
"""
conf = get_worker_config()
port_start = conf['iib_grpc_start_port']
port_end = port_start + conf['iib_grpc_max_port_tries']
for port in range(port_start, port_end):
try:
return (
port,
_serve_index_registry_at_port(db_path, port,
conf['iib_grpc_max_tries'],
conf['iib_grpc_init_wait_time']),
)
except AddressAlreadyInUse:
log.info('Port %d is in use, trying another.', port)
err_msg = f'No free port has been found after {conf.get("iib_grpc_max_port_tries")} attempts.'
log.error(err_msg)
raise IIBError(err_msg)
def _update_index_image_pull_spec(
output_pull_spec,
request_id,
arches,
from_index=None,
overwrite_from_index=False,
overwrite_from_index_token=None,
resolved_prebuild_from_index=None,
add_or_rm=False,
):
"""
Update the request with the modified index image.
This function was created so that code didn't need to be duplicated for the ``add`` and ``rm``
request types.
:param str output_pull_spec: pull spec of the index image generated by IIB
:param int request_id: the ID of the IIB build request
:param set arches: the set of arches that were built as part of this request
:param str from_index: the pull specification of the container image containing the index that
the index image build was based from.
:param bool overwrite_from_index: if True, overwrite the input ``from_index`` with the built
index image.
:param str overwrite_from_index_token: the token used for overwriting the input
``from_index`` image.
:param str resolved_prebuild_from_index: resolved index image before starting the build.
:param bool add_or_rm: true if the request is an ``Add`` or ``Rm`` request. defaults to false
:raises IIBError: if the manifest list couldn't be created and pushed
"""
conf = get_worker_config()
if from_index and overwrite_from_index:
_overwrite_from_index(
request_id,
output_pull_spec,
from_index,
resolved_prebuild_from_index,
overwrite_from_index_token,
)
index_image = from_index
elif conf['iib_index_image_output_registry']:
index_image = output_pull_spec.replace(
conf['iib_registry'], conf['iib_index_image_output_registry'], 1)
log.info(
'Changed the index_image pull specification from %s to %s',
output_pull_spec,
index_image,
)
else:
index_image = output_pull_spec
payload = {'arches': list(arches), 'index_image': index_image}
if add_or_rm:
with set_registry_token(overwrite_from_index_token, index_image):
index_image_resolved = get_resolved_image(index_image)
payload['index_image_resolved'] = index_image_resolved
update_request(request_id,
payload,
exc_msg='Failed setting the index image on the request')
def create_index_db(*args, **kwargs):
db_file = os.path.join(tmpdir,
get_worker_config()['temp_index_db_path'])
os.makedirs(os.path.dirname(db_file), exist_ok=True)
with open(db_file, 'w'):
pass
return db_file
def retry(attempts=get_worker_config().iib_total_attempts, wait_on=Exception, logger=None):
"""
Retry a section of code until success or max attempts are reached.
:param int attempts: the total number of attempts to make before erroring out
:param Exception wait_on: the exception on encountering which the function will be retried
:param logging logger: the logger to log the messages on
:raises IIBError: if the maximum attempts are reached
"""
def wrapper(function):
@functools.wraps(function)
def inner(*args, **kwargs):
remaining_attempts = attempts
while True:
try:
return function(*args, **kwargs)
except wait_on as e:
remaining_attempts -= 1
if remaining_attempts <= 0:
if logger is not None:
logger.exception(
'The maximum number of attempts (%s) have failed', attempts
)
raise
if logger is not None:
logger.warning(
'Exception %r raised from %r. Retrying now',
e,
f'{function.__module__}.{function.__name__}',
)
return inner
return wrapper
def _get_free_port_for_grpc():
"""Return free port for gRPC service from range set in IIB config."""
log.debug('Finding free port for gRPC')
conf = get_worker_config()
port_start = conf['iib_grpc_start_port']
port_end = port_start + conf['iib_grpc_max_port_tries']
return _get_free_port(port_start, port_end)
def create_dogpile_region():
"""Create and configure a dogpile region."""
conf = get_worker_config()
return make_region().configure(
conf.iib_dogpile_backend,
expiration_time=conf.iib_dogpile_expiration_time,
arguments=conf.iib_dogpile_arguments,
)
def _finish_request_post_build(
output_pull_spec,
request_id,
arches,
from_index=None,
overwrite_from_index=False,
overwrite_from_index_token=None,
):
"""
Finish the request after the manifest list has been pushed.
This function was created so that code didn't need to be duplicated for the ``add`` and ``rm``
request types.
:param str output_pull_spec: pull spec of the index image generated by IIB
:param int request_id: the ID of the IIB build request
:param set arches: the set of arches that were built as part of this request
:param str from_index: the pull specification of the container image containing the index that
the index image build was based from.
:param bool overwrite_from_index: if True, overwrite the input ``from_index`` with the built
index image.
:param str overwrite_from_index_token: the token used for overwriting the input
``from_index`` image.
:raises IIBError: if the manifest list couldn't be created and pushed
"""
conf = get_worker_config()
if from_index and overwrite_from_index:
log.info(
f'Ovewriting the index image {from_index} with {output_pull_spec}')
index_image = from_index
exc_msg = f'Failed to overwrite the input from_index container image of {index_image}'
args = [f'docker://{output_pull_spec}', f'docker://{index_image}']
_skopeo_copy(*args,
copy_all=True,
dest_token=overwrite_from_index_token,
exc_msg=exc_msg)
elif conf['iib_index_image_output_registry']:
index_image = output_pull_spec.replace(
conf['iib_registry'], conf['iib_index_image_output_registry'], 1)
log.info(
'Changed the index_image pull specification from %s to %s',
output_pull_spec,
index_image,
)
else:
index_image = output_pull_spec
payload = {
'arches': list(arches),
'index_image': index_image,
'state': 'complete',
'state_reason': 'The request completed successfully',
}
update_request(request_id,
payload,
exc_msg='Failed setting the index image on the request')
def get_rebuilt_image_pull_spec(request_id):
"""
Generate the pull specification of the container image rebuilt by IIB.
:param int request_id: the ID of the IIB build request
:return: pull specification of the rebuilt container image
:rtype: str
"""
conf = get_worker_config()
return conf['iib_image_push_template'].format(
registry=conf['iib_registry'], request_id=request_id)
def _serve_cmd_at_port_defaults(serve_cmd, cwd, port):
"""
Call `_serve_cmd_at_port()` with default values from IIB config.
:param list serve_cmd: opm command to be run (serve FBC or index.db).
:param str cwd: path to folder which should be set as current working directory.
:param str int port: port to start the service on.
"""
log.debug('Run _serve_cmd_at_port with default loaded from IIB config.')
conf = get_worker_config()
return _serve_cmd_at_port(serve_cmd, cwd, port, conf['iib_grpc_max_tries'],
conf['iib_grpc_init_wait_time'])
def test_request_logger_no_request_id(tmpdir):
logs_dir = tmpdir.join('logs')
logs_dir.mkdir()
get_worker_config().iib_request_logs_dir = str(logs_dir)
@utils.request_logger
def mock_handler(spam, eggs, request_id, bacon):
raise ValueError('Handler executed unexpectedly')
with pytest.raises(IIBError, match='Unable to get "request_id" from'):
mock_handler('spam', 'eggs', None, 'bacon')
assert not logs_dir.listdir()
def set_registry_auths(registry_auths):
"""
Configure authentication to the registry with provided dockerconfig.json.
This context manager will reset the authentication to the way it was after it exits. If
``registry_auths`` is falsy, this context manager will do nothing.
:param dict registry_auths: dockerconfig.json auth only information to private registries
:return: None
:rtype: None
"""
if not registry_auths:
log.debug(
'Not changing the Docker configuration since no registry_auths were provided'
)
yield
return
docker_config_path = os.path.join(os.path.expanduser('~'), '.docker',
'config.json')
try:
log.debug('Removing the Docker config symlink at %s',
docker_config_path)
try:
os.remove(docker_config_path)
except FileNotFoundError:
log.debug('The Docker config symlink at %s does not exist',
docker_config_path)
conf = get_worker_config()
if os.path.exists(conf.iib_docker_config_template):
with open(conf.iib_docker_config_template, 'r') as f:
docker_config = json.load(f)
else:
docker_config = {}
registries = list(registry_auths.get('auths', {}).keys())
log.debug(
'Setting the override token for the registries %s in the Docker config',
registries)
docker_config.setdefault('auths', {})
docker_config['auths'].update(registry_auths.get('auths', {}))
with open(docker_config_path, 'w') as f:
json.dump(docker_config, f)
yield
finally:
reset_docker_config()
def request_logger(func):
"""
Log messages relevant to the current request to a dedicated file.
If ``iib_request_logs_dir`` is set, a temporary log handler is added before the decorated
function is invoked. It's then removed once the decorated function completes execution.
If ``iib_request_logs_dir`` is not set, the temporary log handler will not be added.
:param function func: the function to be decorated. The function must take the ``request_id``
parameter.
:return: the decorated function
:rtype: function
"""
worker_config = get_worker_config()
log_dir = worker_config.iib_request_logs_dir
log_level = worker_config.iib_request_logs_level
log_format = worker_config.iib_request_logs_format
@functools.wraps(func)
def wrapper(*args, **kwargs):
request_log_handler = None
if log_dir:
request_id = _get_function_arg_value('request_id', func, args, kwargs)
if not request_id:
raise IIBError(f'Unable to get "request_id" from {func.__name__}')
# for better filtering of all logs for one build in SPLUNK
log_formatter = TaskFormatter(
log_format.format(request_id=f'request-{request_id}'), use_color=False
)
log_file_path = os.path.join(log_dir, f'{request_id}.log')
request_log_handler = logging.FileHandler(log_file_path)
request_log_handler.setLevel(log_level)
request_log_handler.setFormatter(log_formatter)
# Bandit complaining on too permissive logs
os.chmod(log_file_path, 0o664)
logger = logging.getLogger()
logger.addHandler(request_log_handler)
worker_info = f'Host: {socket.getfqdn()}; User: {getpass.getuser()}'
logger.info(worker_info)
try:
return func(*args, **kwargs)
finally:
if request_log_handler:
logger.removeHandler(request_log_handler)
request_log_handler.flush()
if worker_config['iib_aws_s3_bucket_name']:
upload_file_to_s3_bucket(log_file_path, 'request_logs', f'{request_id}.log')
return wrapper
def add_max_ocp_version_property(resolved_bundles, temp_dir):
"""
Add the max ocp version property to bundles.
We need to ensure that any bundle which has deprecated/removed API(s) in 1.22/ocp 4.9
will have this property to prevent users from upgrading clusters to 4.9 before upgrading
the operator installed to a version that is compatible with 4.9
:param list resolved_bundles: list of resolved bundles to which the max ocp version property
will be added if missing
:param str temp_dir: directory location of the index image
"""
# Get the CSV name and version (not just the bundle path)
temp_index_db_path = get_worker_config()['temp_index_db_path']
db_path = os.path.join(temp_dir, temp_index_db_path)
port, rpc_proc = opm_registry_serve(db_path=db_path)
raw_bundles = run_cmd(
['grpcurl', '-plaintext', f'localhost:{port}', 'api.Registry/ListBundles'],
exc_msg='Failed to get bundle data from index image',
)
rpc_proc.terminate()
# This branch is hit when `bundles` attribute is empty and the index image is empty.
# Ideally the code should not reach here if the bundles attribute is empty but adding
# this here as a failsafe if it's called from some other place. Also, if the bundles
# attribute is not empty, the index image cannot be empty here because we add the
# bundle to the index before adding the maxOpenShiftVersion property
if not raw_bundles:
log.info('No bundles found in the index image')
return
# Filter index image bundles to get pull spec for bundles in the request
updated_bundles = list(
filter(lambda b: b['bundlePath'] in resolved_bundles, get_bundle_json(raw_bundles))
)
for bundle in updated_bundles:
if _requires_max_ocp_version(bundle['bundlePath']):
log.info('adding property for %s', bundle['bundlePath'])
max_openshift_version_property = {
'type': 'olm.maxOpenShiftVersion',
'value': '4.8',
'operatorbundle_name': bundle['csvName'],
'operatorbundle_version': bundle['version'],
'operatorbundle_path': bundle['bundlePath'],
}
_add_property_to_index(db_path, max_openshift_version_property)
log.info('property added for %s', bundle['bundlePath'])
def reset_docker_config():
"""Create a symlink from ``iib_docker_config_template`` to ``~/.docker/config.json``."""
conf = get_worker_config()
docker_config_path = os.path.join(os.path.expanduser('~'), '.docker',
'config.json')
try:
log.debug('Removing the Docker config at %s', docker_config_path)
os.remove(docker_config_path)
except FileNotFoundError:
pass
if os.path.exists(conf.iib_docker_config_template):
log.debug('Creating a symlink from %s to %s',
conf.iib_docker_config_template, docker_config_path)
os.symlink(conf.iib_docker_config_template, docker_config_path)
def get_hidden_index_database(from_index, base_dir):
"""
Get hidden database file from the specified index image and save it locally.
:param str from_index: index image to get database file from.
:param str base_dir: base directory to which the database file should be saved.
:return: path to the copied database file.
:rtype: str
"""
from iib.workers.tasks.build import _copy_files_from_image
log.info("Store hidden index.db from %s", from_index)
conf = get_worker_config()
base_db_file = os.path.join(base_dir, conf['temp_index_db_path'])
os.makedirs(os.path.dirname(base_db_file), exist_ok=True)
_copy_files_from_image(from_index, conf['hidden_index_db_path'], base_db_file)
return base_db_file
def verify_labels(bundles):
"""
Verify that the required labels are set on the input bundles.
:param list bundles: a list of strings representing the pull specifications of the bundles to
add to the index image being built.
:raises IIBError: if one of the bundles does not have the correct label value.
"""
conf = get_worker_config()
if not conf['iib_required_labels']:
return
for bundle in bundles:
labels = get_image_labels(bundle)
for label, value in conf['iib_required_labels'].items():
if labels.get(label) != value:
raise IIBError(f'The bundle {bundle} does not have the label {label}={value}')
def request_logger(func):
"""
Log messages relevant to the current request to a dedicated file.
If ``iib_request_logs_dir`` is set, a temporary log handler is added before the decorated
function is invoked. It's then removed once the decorated function completes execution.
If ``iib_request_logs_dir`` is not set, the temporary log handler will not be added.
:param function func: the function to be decorated. The function must take the ``request_id``
parameter.
:return: the decorated function
:rtype: function
"""
worker_config = get_worker_config()
log_dir = worker_config.iib_request_logs_dir
log_level = worker_config.iib_request_logs_level
log_format = worker_config.iib_request_logs_format
@functools.wraps(func)
def wrapper(*args, **kwargs):
request_log_handler = None
if log_dir:
log_formatter = logging.Formatter(log_format)
request_id = _get_function_arg_value('request_id', func, args,
kwargs)
if not request_id:
raise IIBError(
f'Unable to get "request_id" from {func.__name__}')
log_file_path = os.path.join(log_dir, f'{request_id}.log')
request_log_handler = logging.FileHandler(log_file_path)
request_log_handler.setLevel(log_level)
request_log_handler.setFormatter(log_formatter)
os.chmod(log_file_path, 0o775)
logger = logging.getLogger()
logger.addHandler(request_log_handler)
try:
return func(*args, **kwargs)
finally:
if request_log_handler:
logger.removeHandler(request_log_handler)
return wrapper
def _get_or_create_temp_index_db_file(base_dir,
from_index=None,
overwrite_from_index_token=None,
ignore_existing=False):
"""
Get path to temp index.db used for opm registry commands.
If index.db does not exist it will be created, ether by copying from from_index
or as creating empty index.db file if from_index is not set.
:param str base_dir: base directory where index.db file will be located.
:param str from_index: index image, from which we should copy index.
:param bool ignore_existing: if set it forces to copy index.db from `from_index`.
`from_index` must be set
:return: Returns path to index.db located in base_dir.
:rtype: str
"""
from iib.workers.tasks.build import _get_index_database
from iib.workers.tasks.utils import set_registry_token
index_db_file = os.path.join(base_dir,
get_worker_config()['temp_index_db_path'])
if not ignore_existing and os.path.exists(index_db_file):
log.debug('Temp index.db already exist for %s', from_index)
return index_db_file
log.info('Temp index.db does not exist yet for %s', from_index)
if from_index:
log.info('Using the existing database from %s', from_index)
with set_registry_token(overwrite_from_index_token, from_index):
if is_image_fbc(from_index):
return get_hidden_index_database(from_index, base_dir)
return _get_index_database(from_index, base_dir)
log.info('Creating empty database file %s', index_db_file)
index_db_dir = os.path.dirname(index_db_file)
if not os.path.exists(index_db_dir):
os.makedirs(index_db_dir, exist_ok=True)
with open(index_db_file, 'w'):
pass
return index_db_file
def skopeo_inspect(*args):
"""
Wrap the ``skopeo inspect`` command.
:param args: any arguments to pass to ``skopeo inspect``
:return: a dictionary of the JSON output from the skopeo inspect command
:rtype: dict
:raises IIBError: if the command fails
"""
exc_msg = None
for arg in args:
if arg.startswith('docker://'):
exc_msg = f'Failed to inspect {arg}. Make sure it exists and is accessible to IIB.'
break
skopeo_timeout = get_worker_config().iib_skopeo_timeout
cmd = ['skopeo', '--command-timeout', skopeo_timeout, 'inspect'
] + list(args)
return json.loads(run_cmd(cmd, exc_msg=exc_msg))
def update_request(request_id, payload, exc_msg=None):
"""
Update the IIB build request.
:param int request_id: the ID of the IIB request
:param dict payload: the payload to send to the PATCH API endpoint
:param str exc_msg: an optional custom exception that can be a template
:return: the updated request
:rtype: dict
:raises IIBError: if the request to the IIB API fails
"""
# Prevent a circular import
from iib.workers.config import get_worker_config
config = get_worker_config()
request_url = f'{config.iib_api_url.rstrip("/")}/builds/{request_id}'
log.info('Patching the request %d with %r', request_id, payload)
try:
rv = requests_auth_session.patch(request_url,
json=payload,
timeout=config.iib_api_timeout)
except requests.RequestException:
msg = f'The connection failed when updating the request {request_id}'
log.exception(msg)
raise IIBError(msg)
if not rv.ok:
log.error(
'The worker failed to update the request %d. The status was %d. The text was:\n%s',
request_id,
rv.status_code,
rv.text,
)
if exc_msg:
_exc_msg = exc_msg.format(**payload)
else:
_exc_msg = f'The worker failed to update the request {request_id}'
raise IIBError(_exc_msg)
return rv.json()
