class KojiTagBuildPlugin(Plugin):
"""
Tag build in koji
Authentication is with Kerberos unless the koji_ssl_certs
configuration parameter is given, in which case it should be a
path at which 'cert', 'ca', and 'serverca' are the certificates
for SSL authentication.
If Kerberos is used for authentication, the default principal will
be used (from the kernel keyring) unless both koji_keytab and
koji_principal are specified. The koji_keytab parameter is a
keytab name like 'type:name', and so can be used to specify a key
in a Kubernetes secret by specifying 'FILE:/path/to/key'.
"""
key = PLUGIN_KOJI_TAG_BUILD_KEY
is_allowed_to_fail = False
args_from_user_params = map_to_user_params("target:koji_target")
def __init__(self, workflow, target=None, poll_interval=5):
"""
constructor
:param workflow: DockerBuildWorkflow instance
:param target: str, koji target
:param poll_interval: int, seconds between Koji task status requests
"""
super(KojiTagBuildPlugin, self).__init__(workflow)
self.target = target
self.poll_interval = poll_interval
def run(self):
"""
Run the plugin.
"""
if is_scratch_build(self.workflow):
self.log.info('scratch build, skipping plugin')
return
if not self.target:
self.log.info('no koji target provided, skipping plugin')
return
build_id = self.workflow.data.plugins_results.get(KojiImportPlugin.key)
if not build_id:
self.log.info('No koji build from %s', KojiImportPlugin.key)
return
session = get_koji_session(self.workflow.conf)
build_tag = tag_koji_build(session,
build_id,
self.target,
poll_interval=self.poll_interval)
return build_tag
def args_from_user_params(user_params: dict) -> dict:
args = {}
build_kwargs_from_user_params = map_to_user_params(
"component",
"git_branch",
"git_ref",
"git_uri",
"koji_task_id",
"filesystem_koji_task_id",
"scratch",
"target:koji_target",
"user",
"yum_repourls",
"koji_parent_build",
"isolated",
"reactor_config_map",
"reactor_config_override",
"git_commit_depth",
"flatpak",
"operator_csv_modifications_url",
)
if build_kwargs := build_kwargs_from_user_params(user_params):
args["build_kwargs"] = build_kwargs
class AddFilesystemPlugin(Plugin):
"""
Creates a base image by using a filesystem generated through Koji
Submits an image build task to Koji based on image build
configuration file to create the filesystem to be used in
creating the base image:
https://docs.pagure.org/koji/image_build/
Once image build task is complete the tarball is downloaded.
The existing FROM instruction value is replaced with a
FROM scratch and ADD <filesystem> <to_image> for Dockerfile
of each platform.
The "FROM" instruction should be in the following format:
FROM koji/image-build[:image-build-conf]
Where image-build-conf is the file name of the image build
configuration to be used. If omitted, image-build.conf is used.
This file is expected to be in the same folder as the Dockerfile.
Runs as a pre build plugin in order to properly adjust base image.
"""
key = PLUGIN_ADD_FILESYSTEM_KEY
is_allowed_to_fail = False
DEFAULT_IMAGE_BUILD_CONF = dedent('''\
[image-build]
name = default-name
arches = x86_64
format = docker
disk_size = 10
target = {target}
install_tree = {install_tree}
repo = {repo}
ksurl = {ksurl}
kickstart = kickstart.ks
[factory-parameters]
create_docker_metadata = False
''')
args_from_user_params = map_to_user_params(
"repos:yum_repourls",
"koji_target",
)
def __init__(self,
workflow,
poll_interval=5,
blocksize=DEFAULT_DOWNLOAD_BLOCK_SIZE,
repos=None,
koji_target=None):
"""
:param workflow: DockerBuildWorkflow instance
:param poll_interval: int, seconds between polling Koji while waiting
for task completion
:param blocksize: int, chunk size for downloading files from koji
:param repos: list<str>: list of yum repo URLs to be used during
base filesystem creation. First value will also
be used as install_tree. Only baseurl value is used
from each repo file.
:param koji_target: str, koji target name
"""
# call parent constructor
super(AddFilesystemPlugin, self).__init__(workflow)
self.poll_interval = poll_interval
self.blocksize = blocksize
self.repos = repos or []
self.architectures = get_platforms(self.workflow.data)
self.scratch = util.is_scratch_build(self.workflow)
self.koji_target = koji_target
self.session = None
def is_image_build_type(self, base_image):
return base_image.strip().lower() == 'koji/image-build'
def extract_base_url(self, repo_url):
yum_repo = YumRepo(repo_url)
yum_repo.fetch()
if not yum_repo.is_valid():
return []
repo = yum_repo.config
return [
repo.get(section, 'baseurl') for section in repo.sections()
if repo.has_option(section, 'baseurl')
]
def get_default_image_build_conf(self):
"""Create a default image build config
:rtype: ConfigParser
:return: Initialized config with defaults
"""
target = self.koji_target
vcs_info = self.workflow.source.get_vcs_info()
ksurl = '{}#{}'.format(vcs_info.vcs_url, vcs_info.vcs_ref)
base_urls = []
for repo in self.repos:
for url in self.extract_base_url(repo):
# Imagefactory only supports $arch variable.
url = url.replace('$basearch', '$arch')
base_urls.append(url)
install_tree = base_urls[0] if base_urls else ''
repo = ','.join(base_urls)
kwargs = {
'target': target,
'ksurl': ksurl,
'install_tree': install_tree,
'repo': repo,
}
config_fp = StringIO(self.DEFAULT_IMAGE_BUILD_CONF.format(**kwargs))
config = ConfigParser()
config.read_file(config_fp)
self.update_config_from_dockerfile(config)
return config
def update_config_from_dockerfile(self, config):
"""Updates build config with values from the Dockerfile
Updates:
* set "name" from LABEL com.redhat.component (if exists)
* set "version" from LABEL version (if exists)
:param config: ConfigParser object
"""
labels = Labels(self.workflow.build_dir.any_platform.dockerfile.labels)
for config_key, label in (
('name', Labels.LABEL_TYPE_COMPONENT),
('version', Labels.LABEL_TYPE_VERSION),
):
try:
_, value = labels.get_name_and_value(label)
except KeyError:
pass
else:
config.set('image-build', config_key, value)
def parse_image_build_config(self, config_file_name):
# Logic taken from koji.cli.koji.handle_image_build.
# Unable to re-use koji's code because "cli" is not
# a package of koji and this logic is intermingled
# with CLI specific instructions.
args = []
opts = {}
config = self.get_default_image_build_conf()
config.read(config_file_name)
if self.architectures:
config.set('image-build', 'arches', ','.join(self.architectures))
# else just use what was provided by the user in image-build.conf
config_str = StringIO()
config.write(config_str)
self.log.debug('Image Build Config: \n%s', config_str.getvalue())
image_name = None
section = 'image-build'
for option in ('name', 'version', 'arches', 'target', 'install_tree'):
value = config.get(section, option)
if not value:
raise ValueError('{} cannot be empty'.format(option))
if option == 'arches':
# pylint: disable=no-member
value = [arch for arch in value.split(',') if arch]
elif option == 'name':
image_name = value
args.append(value)
config.remove_option(section, option)
for option, value in config.items(section):
if option in ('repo', 'format'):
value = [v for v in value.split(',') if v]
elif option in ('disk_size', ):
value = int(value)
opts[option] = value
section = 'ova-options'
if config.has_section(section):
ova = []
for k, v in config.items(section):
ova.append('{}={}'.format(k, v))
opts['ova_option'] = ova
section = 'factory-parameters'
if config.has_section(section):
factory = []
for option, value in config.items(section):
factory.append((option, value))
opts['factory_parameter'] = factory
return image_name, args, {'opts': opts}
def build_filesystem(self, image_build_conf):
# Image build conf file should be in the same folder as Dockerfile
image_build_conf = self.workflow.build_dir.any_platform.path / image_build_conf
if not os.path.exists(image_build_conf):
raise RuntimeError(
'Image build configuration file not found: {}'.format(
image_build_conf))
image_name, args, kwargs = self.parse_image_build_config(
image_build_conf)
if self.scratch:
kwargs['opts']['scratch'] = True
task_id = self.session.buildImageOz(*args, **kwargs)
return task_id, image_name
def find_filesystem(self, task_id, filesystem_regex):
for f in self.session.listTaskOutput(task_id):
f = f.strip()
match = filesystem_regex.match(f)
if match:
return task_id, match.group(0)
# Not found in this task, search sub tasks
for sub_task in self.session.getTaskChildren(task_id):
found = self.find_filesystem(sub_task['id'], filesystem_regex)
if found:
return found
return None
def download_filesystem(self, task_id, filesystem_regex,
build_dir: BuildDir):
found = self.find_filesystem(task_id, filesystem_regex)
if found is None:
raise RuntimeError(
'Filesystem not found as task output: {}'.format(
filesystem_regex.pattern))
task_id, file_name = found
self.log.info('Downloading filesystem: %s from task ID: %s', file_name,
task_id)
file_path = build_dir.path / file_name
if file_path.exists():
raise RuntimeError(
f'Filesystem {file_name} already exists at {file_path}')
with open(file_path, 'w') as f:
for chunk in stream_task_output(self.session, task_id, file_name,
self.blocksize):
f.write(chunk)
return file_name
def run_image_task(self, image_build_conf):
task_id, image_name = self.build_filesystem(image_build_conf)
task = TaskWatcher(self.session, task_id, self.poll_interval)
try:
task.wait()
except BuildCanceledException:
self.log.info("Build was canceled, canceling task %s", task_id)
try:
self.session.cancelTask(task_id)
self.log.info('task %s canceled', task_id)
except Exception as exc:
self.log.info("Exception while canceling a task (ignored): %s",
util.exception_message(exc))
if task.failed():
try:
# Koji may re-raise the error that caused task to fail
task_result = self.session.getTaskResult(task_id)
except Exception as exc:
task_result = util.exception_message(exc)
raise RuntimeError('image task, {}, failed: {}'.format(
task_id, task_result))
return task_id, image_name
def get_image_build_conf(self):
image_build_conf = None
for parent in self.workflow.data.dockerfile_images:
if base_image_is_custom(parent.to_str()):
image_build_conf = parent.tag
break
if not image_build_conf or image_build_conf == 'latest':
image_build_conf = 'image-build.conf'
return image_build_conf
def update_repos_from_composes(self):
resolve_comp_result = self.workflow.data.plugins_results.get(
PLUGIN_RESOLVE_COMPOSES_KEY)
if not resolve_comp_result:
return
for compose_info in resolve_comp_result['composes']:
self.log.info('adding repo file from compose: %s',
compose_info['result_repofile'])
self.repos.append(compose_info['result_repofile'])
def _add_filesystem_to_dockerfile(self, file_name, build_dir: BuildDir):
"""
Put an ADD instruction into the Dockerfile (to include the filesystem
into the container image to be built)
"""
content = 'ADD {0} /\n'.format(file_name)
lines = build_dir.dockerfile.lines
# as we insert elements we have to keep track of the increment for inserting
offset = 1
for item in build_dir.dockerfile.structure:
if item['instruction'] == 'FROM' and base_image_is_custom(
item['value'].split()[0]):
lines.insert(item['endline'] + offset, content)
offset += 1
build_dir.dockerfile.lines = lines
new_parents = []
for image in build_dir.dockerfile.parent_images:
if base_image_is_custom(image):
new_parents.append('scratch')
else:
new_parents.append(image)
build_dir.dockerfile.parent_images = new_parents
self.log.info('added "%s" as image filesystem', file_name)
def inject_filesystem(self, task_id, image_name, build_dir: BuildDir):
prefix = '{}.*{}'.format(image_name, build_dir.platform)
pattern = (
r'{}.*(\.tar|\.tar\.gz|\.tar\.bz2|\.tar\.xz)$'.format(prefix))
filesystem_regex = re.compile(pattern, re.IGNORECASE)
file_name = self.download_filesystem(task_id, filesystem_regex,
build_dir)
self._add_filesystem_to_dockerfile(file_name, build_dir)
def run(self):
if not self.workflow.data.dockerfile_images.custom_parent_image:
self.log.info('Nothing to do for non-custom base images')
return
self.update_repos_from_composes()
image_build_conf = self.get_image_build_conf()
self.session = get_koji_session(self.workflow.conf)
task_id, image_name = self.run_image_task(image_build_conf)
inject_filesystem_call = functools.partial(self.inject_filesystem,
task_id, image_name)
self.workflow.build_dir.for_each_platform(inject_filesystem_call)
return {
'filesystem-koji-task-id': task_id,
}
class ResolveRemoteSourcePlugin(Plugin):
"""Initiate a new Cachito request for sources
This plugin will read the remote_sources configuration from
container.yaml in the git repository, use it to make a request
to Cachito, and wait for the request to complete.
"""
key = PLUGIN_RESOLVE_REMOTE_SOURCE
is_allowed_to_fail = False
REMOTE_SOURCE = "unpacked_remote_sources"
args_from_user_params = map_to_user_params("dependency_replacements")
def __init__(self, workflow, dependency_replacements=None):
"""
:param workflow: DockerBuildWorkflow instance
:param dependency_replacements: list<str>, dependencies for the cachito fetched artifact to
be replaced. Must be of the form pkg_manager:name:version[:new_name]
"""
super(ResolveRemoteSourcePlugin, self).__init__(workflow)
self._cachito_session = None
self._osbs = None
self._dependency_replacements = self.parse_dependency_replacements(dependency_replacements)
self.single_remote_source_params = self.workflow.source.config.remote_source
self.multiple_remote_sources_params = self.workflow.source.config.remote_sources
def parse_dependency_replacements(self, replacement_strings):
"""Parse dependency_replacements param and return cachito-reaady dependency replacement dict
param replacement_strings: list<str>, pkg_manager:name:version[:new_name]
return: list<dict>, cachito formated dependency replacements param
"""
if not replacement_strings:
return
dependency_replacements = []
for dr_str in replacement_strings:
pkg_manager, name, version, new_name = (dr_str.split(':', 3) + [None] * 4)[:4]
if None in [pkg_manager, name, version]:
raise ValueError('Cachito dependency replacements must be '
'"pkg_manager:name:version[:new_name]". got {}'.format(dr_str))
dr = {'type': pkg_manager, 'name': name, 'version': version}
if new_name:
dr['new_name'] = new_name
dependency_replacements.append(dr)
return dependency_replacements
def run(self) -> Optional[List[Dict[str, Any]]]:
if (not self.workflow.conf.allow_multiple_remote_sources
and self.multiple_remote_sources_params):
raise ValueError('Multiple remote sources are not enabled, '
'use single remote source in container.yaml')
if not (self.single_remote_source_params or self.multiple_remote_sources_params):
self.log.info('Aborting plugin execution: missing remote source configuration')
return None
if not self.workflow.conf.cachito:
raise RuntimeError('No Cachito configuration defined')
if self._dependency_replacements and not is_scratch_build(self.workflow):
raise ValueError('Cachito dependency replacements are only allowed for scratch builds')
if self._dependency_replacements and self.multiple_remote_sources_params:
raise ValueError('Cachito dependency replacements are not allowed '
'for multiple remote sources')
processed_remote_sources = self.process_remote_sources()
self.inject_remote_sources(processed_remote_sources)
return [
self.remote_source_to_output(remote_source)
for remote_source in processed_remote_sources
]
def process_remote_sources(self) -> List[RemoteSource]:
"""Process remote source requests and return information about the processed sources."""
user = self.get_koji_user()
self.log.info('Using user "%s" for cachito request', user)
processed_remote_sources = []
if self.multiple_remote_sources_params:
self.verify_multiple_remote_sources_names_are_unique()
open_requests = {
remote_source["name"]: self.cachito_session.request_sources(
user=user,
dependency_replacements=self._dependency_replacements,
**remote_source["remote_source"]
)
for remote_source in self.multiple_remote_sources_params
}
completed_requests = {
name: self.cachito_session.wait_for_request(request)
for name, request in open_requests.items()
}
for name, request in completed_requests.items():
processed_remote_sources.append(self.process_request(request, name))
else:
open_request = self.cachito_session.request_sources(
user=user,
dependency_replacements=self._dependency_replacements,
**self.single_remote_source_params
)
completed_request = self.cachito_session.wait_for_request(open_request)
processed_remote_sources.append(self.process_request(completed_request, None))
return processed_remote_sources
def inject_remote_sources(self, remote_sources: List[RemoteSource]) -> None:
"""Inject processed remote sources into build dirs and add build args to workflow."""
inject_sources = functools.partial(self.inject_into_build_dir, remote_sources)
self.workflow.build_dir.for_all_platforms_copy(inject_sources)
# For single remote_source workflow, inject all build args directly
if self.single_remote_source_params:
self.workflow.data.buildargs.update(remote_sources[0].build_args)
self.add_general_buildargs()
def inject_into_build_dir(
self, remote_sources: List[RemoteSource], build_dir: BuildDir,
) -> List[Path]:
"""Inject processed remote sources into a build directory.
For each remote source, create a dedicated directory, unpack the downloaded tarball
into it and inject the configuration files and an environment file.
Return a list of the newly created directories.
"""
created_dirs = []
for remote_source in remote_sources:
dest_dir = build_dir.path.joinpath(self.REMOTE_SOURCE, remote_source.name or "")
if dest_dir.exists():
raise RuntimeError(
f"Conflicting path {dest_dir.relative_to(build_dir.path)} already exists "
"in the dist-git repository"
)
dest_dir.mkdir(parents=True)
created_dirs.append(dest_dir)
with tarfile.open(remote_source.tarball_path) as tf:
tf.extractall(dest_dir)
config_files = self.cachito_session.get_request_config_files(remote_source.id)
self.generate_cachito_config_files(dest_dir, config_files)
# Create cachito.env file with environment variables received from cachito request
self.generate_cachito_env_file(dest_dir, remote_source.build_args)
return created_dirs
def get_buildargs(self, request_id: int, remote_source_name: Optional[str]) -> Dict[str, str]:
build_args = {}
env_vars = self.cachito_session.get_request_env_vars(request_id)
for env_var, value_info in env_vars.items():
build_arg_value = value_info['value']
kind = value_info['kind']
if kind == 'path':
name = remote_source_name or ''
build_arg_value = os.path.join(REMOTE_SOURCE_DIR, name, value_info['value'])
self.log.debug(
'Setting the Cachito environment variable "%s" to the absolute path "%s"',
env_var,
build_arg_value,
)
build_args[env_var] = build_arg_value
elif kind == 'literal':
self.log.debug(
'Setting the Cachito environment variable "%s" to a literal value "%s"',
env_var,
build_arg_value,
)
build_args[env_var] = build_arg_value
else:
raise RuntimeError(f'Unknown kind {kind} got from Cachito.')
return build_args
def source_request_to_json(self, source_request):
"""Create a relevant representation of the source request"""
required = ('packages', 'ref', 'repo')
optional = ('dependencies', 'flags', 'pkg_managers', 'environment_variables',
'configuration_files', 'content_manifest')
data = {}
try:
data.update({k: source_request[k] for k in required})
except KeyError as exc:
msg = 'Received invalid source request from Cachito: {}'.format(source_request)
self.log.exception(msg)
raise ValueError(msg) from exc
data.update({k: source_request[k] for k in optional if k in source_request})
return data
def get_koji_user(self):
unknown_user = self.workflow.conf.cachito.get('unknown_user', 'unknown_user')
try:
koji_task_id = int(self.workflow.user_params.get('koji_task_id'))
except (ValueError, TypeError, AttributeError):
msg = 'Unable to get koji user: Invalid Koji task ID'
self.log.warning(msg)
return unknown_user
koji_session = get_koji_session(self.workflow.conf)
return get_koji_task_owner(koji_session, koji_task_id).get('name', unknown_user)
@property
def cachito_session(self):
if not self._cachito_session:
self._cachito_session = get_cachito_session(self.workflow.conf)
return self._cachito_session
def verify_multiple_remote_sources_names_are_unique(self):
names = [remote_source['name'] for remote_source in self.multiple_remote_sources_params]
duplicate_names = [name for name, count in Counter(names).items() if count > 1]
if duplicate_names:
raise ValueError(f'Provided remote sources parameters contain '
f'non unique names: {duplicate_names}')
def process_request(self, source_request: dict, name: Optional[str]) -> RemoteSource:
"""Download the tarball for a request and return info about the processed remote source."""
tarball_filename = RemoteSource.tarball_filename(name)
dest_dir = str(self.workflow.build_dir.any_platform.path)
tarball_dest_path = self.cachito_session.download_sources(
source_request,
dest_dir=dest_dir,
dest_filename=tarball_filename,
)
build_args = self.get_buildargs(source_request["id"], name)
remote_source = RemoteSource(
id=source_request["id"],
name=name,
json_data=self.source_request_to_json(source_request),
build_args=build_args,
tarball_path=Path(tarball_dest_path),
)
return remote_source
def remote_source_to_output(self, remote_source: RemoteSource) -> Dict[str, Any]:
"""Convert a processed remote source to a dict to be used as output of this plugin."""
download_url = self.cachito_session.assemble_download_url(remote_source.id)
json_filename = RemoteSource.json_filename(remote_source.name)
return {
"id": remote_source.id,
"name": remote_source.name,
"url": download_url,
"remote_source_json": {
"json": remote_source.json_data,
"filename": json_filename,
},
"remote_source_tarball": {
"filename": remote_source.tarball_path.name,
"path": str(remote_source.tarball_path),
},
}
def generate_cachito_config_files(self, dest_dir: Path, config_files: List[dict]) -> None:
"""Inject cachito provided configuration files
:param dest_dir: destination directory for config files
:param config_files: configuration files from cachito
"""
for config in config_files:
config_path = dest_dir / config['path']
if config['type'] == CFG_TYPE_B64:
data = base64.b64decode(config['content'])
config_path.write_bytes(data)
else:
err_msg = "Unknown cachito configuration file data type '{}'".format(config['type'])
raise ValueError(err_msg)
config_path.chmod(0o444)
def generate_cachito_env_file(self, dest_dir: Path, build_args: Dict[str, str]) -> None:
"""
Generate cachito.env file with exported environment variables received from
cachito request.
:param dest_dir: destination directory for env file
:param build_args: build arguments to set
"""
self.log.info('Creating %s file with environment variables '
'received from cachito request', CACHITO_ENV_FILENAME)
# Use dedicated dir in container build workdir for cachito.env
abs_path = dest_dir / CACHITO_ENV_FILENAME
with open(abs_path, 'w') as f:
f.write('#!/bin/bash\n')
for env_var, value in build_args.items():
f.write('export {}={}\n'.format(env_var, shlex.quote(value)))
def add_general_buildargs(self) -> None:
"""Adds general build arguments
To copy the sources into the build image, Dockerfile should contain
COPY $REMOTE_SOURCE $REMOTE_SOURCE_DIR
or COPY $REMOTE_SOURCES $REMOTE_SOURCES_DIR
"""
if self.multiple_remote_sources_params:
args_for_dockerfile_to_add = {
'REMOTE_SOURCES': self.REMOTE_SOURCE,
'REMOTE_SOURCES_DIR': REMOTE_SOURCE_DIR,
}
else:
args_for_dockerfile_to_add = {
'REMOTE_SOURCE': self.REMOTE_SOURCE,
'REMOTE_SOURCE_DIR': REMOTE_SOURCE_DIR,
CACHITO_ENV_ARG_ALIAS: os.path.join(REMOTE_SOURCE_DIR, CACHITO_ENV_FILENAME),
}
self.workflow.data.buildargs.update(args_for_dockerfile_to_add)
class PinOperatorDigestsPlugin(PreBuildPlugin):
"""
Plugin runs for operator manifest bundle builds.
- finds container pullspecs in operator ClusterServiceVersion files
- computes replacement pullspecs:
- replaces tags with manifest list digests
- replaces repos (and namespaces) based on operator_manifests.repo_replacements
configuration in container.yaml and r-c-m*
- replaces registries based on operator_manifests.registry_post_replace in r-c-m*
- replaces pullspecs in ClusterServiceVersion files based on replacement pullspec mapping
- creates relatedImages sections in ClusterServiceVersion files
Files that already have a relatedImages section are excluded.
* reactor-config-map
"""
key = PLUGIN_PIN_OPERATOR_DIGESTS_KEY
is_allowed_to_fail = False
args_from_user_params = map_to_user_params(
"operator_csv_modifications_url",
)
def __init__(self, workflow, operator_csv_modifications_url=None):
"""
Initialize pin_operator_digests plugin
:param workflow: DockerBuildWorkflow instance
:param operator_csv_modifications_url: str, URL to JSON file with operator
CSV modifications
"""
super(PinOperatorDigestsPlugin, self).__init__(workflow)
self.user_config = workflow.source.config.operator_manifests
self.operator_csv_modifications_url = operator_csv_modifications_url
site_config = self.workflow.conf.operator_manifests
self.operator_csv_modification_allowed_attributes = set(
tuple(key_path)
for key_path in (
site_config
.get('csv_modifications', {})
.get('allowed_attributes', [])
)
)
def _validate_operator_csv_modifications_schema(self, modifications):
"""Validate if provided operator CSV modification are valid according schema"""
schema = load_schema(
'atomic_reactor',
'schemas/operator_csv_modifications.json'
)
validate_with_schema(modifications, schema)
def _validate_operator_csv_modifications_duplicated_images(self, modifications):
"""Validate if provided operator CSV modifications doesn't provide duplicated entries"""
original_pullspecs = set()
duplicated = set()
for repl in modifications.get('pullspec_replacements', ()):
pullspec = ImageName.parse(repl['original'])
if pullspec in original_pullspecs:
duplicated.add(pullspec)
self.log.error(
"Operator CSV modifications contains duplicated "
"original replacement pullspec %s", pullspec)
original_pullspecs.add(pullspec)
if duplicated:
raise RuntimeError(
f"Provided CSV modifications contain duplicated "
f"original entries in pullspec_replacement: "
f"{', '.join(sorted(str(dup) for dup in duplicated))}"
)
def _validate_operator_csv_modifications_allowed_keys(self, modifications):
"""Validate if used attributes in update/append are allowed to be modified"""
allowed_attrs = self.operator_csv_modification_allowed_attributes
def to_str(path):
return '.'.join(part.replace('.', r'\.') for part in path)
def validate(mods):
not_allowed = []
for key_path in terminal_key_paths(mods):
if key_path not in allowed_attrs:
self.log.error(
"Operator CSV attribute %s is not allowed to be modified",
key_path
)
not_allowed.append(key_path)
if not_allowed:
raise RuntimeError(
f"Operator CSV attributes: {', '.join(to_str(attr) for attr in not_allowed)}; "
f"are not allowed to be modified "
f"by service configuration. Attributes allowed for modification "
f"are: {', '.join(to_str(attr) for attr in allowed_attrs) or 'N/A'}"
)
validate(modifications.get('update', {}))
validate(modifications.get('append', {}))
def _validate_operator_csv_modifications(self, modifications):
"""Validate if provided operator CSV modification correct"""
self._validate_operator_csv_modifications_schema(modifications)
self._validate_operator_csv_modifications_duplicated_images(modifications)
self._validate_operator_csv_modifications_allowed_keys(modifications)
def _fetch_operator_csv_modifications(self):
"""Fetch operator CSV modifications"""
if not self.operator_csv_modifications_url:
return None
session = get_retrying_requests_session()
self.log.info(
"Fetching operator CSV modifications data from %s",
self.operator_csv_modifications_url
)
resp = session.get(self.operator_csv_modifications_url)
try:
resp.raise_for_status()
except Exception as exc:
raise RuntimeError(
f"Failed to fetch the operator CSV modification JSON "
f"from {self.operator_csv_modifications_url}: {exc}"
) from exc
try:
csv_modifications = resp.json()
except Exception as exc:
# catching raw Exception because requests uses various json decoders
# in different versions
raise RuntimeError(
f"Failed to parse operator CSV modification JSON "
f"from {self.operator_csv_modifications_url}: {exc}"
) from exc
self.log.info("Operator CSV modifications: %s", csv_modifications)
self._validate_operator_csv_modifications(csv_modifications)
return csv_modifications
def run(self):
"""
Run pin_operator_digest plugin
"""
if not self.should_run():
return
if self.operator_csv_modifications_url:
self.log.info(
"Operator CSV modification URL specified: %s",
self.operator_csv_modifications_url
)
operator_manifest = self._get_operator_manifest()
replacement_pullspecs = {}
related_images_metadata = {
'pullspecs': [],
'created_by_osbs': True,
}
operator_manifests_metadata = {
'related_images': related_images_metadata,
'custom_csv_modifications_applied': bool(self.operator_csv_modifications_url)
}
should_skip = self._skip_all()
if should_skip:
self.log.warning("skip_all defined for operator manifests")
pullspecs = self._get_pullspecs(operator_manifest.csv, should_skip)
if operator_manifest.csv.has_related_images() or should_skip:
if self.operator_csv_modifications_url:
raise RuntimeError(
"OSBS cannot modify operator CSV file because this operator bundle "
"is managed by owner (digest pinning explicitly disabled or "
"RelatedImages section in CSV exists)"
)
# related images already exists
related_images_metadata['created_by_osbs'] = False
related_images_metadata['pullspecs'] = [{
'original': item,
'new': item,
'pinned': False,
'replaced': False,
} for item in pullspecs]
else:
if pullspecs:
replacement_pullspecs = self._get_replacement_pullspecs(pullspecs)
related_images_metadata['pullspecs'] = replacement_pullspecs
else:
# no pullspecs don't create relatedImages section
related_images_metadata['created_by_osbs'] = False
if should_skip:
return operator_manifests_metadata
replacement_pullspecs = {
repl['original']: repl['new']
for repl in replacement_pullspecs
if repl['replaced']
}
operator_csv = operator_manifest.csv
self.log.info("Updating operator CSV file")
if not operator_csv.has_related_images():
self.log.info("Replacing pullspecs in %s", operator_csv.path)
# Replace pullspecs everywhere, not just in locations in which they
# are expected to be found - OCP 4.4 workaround
operator_csv.replace_pullspecs_everywhere(replacement_pullspecs)
self.log.info("Creating relatedImages section in %s", operator_csv.path)
operator_csv.set_related_images()
operator_csv_modifications = self._fetch_operator_csv_modifications()
if operator_csv_modifications:
operator_csv.modifications_append(
operator_csv_modifications.get('append', {}))
operator_csv.modifications_update(
operator_csv_modifications.get('update', {}))
self.workflow.build_dir.for_each_platform(operator_csv.dump)
else:
self.log.warning("%s has a relatedImages section, skipping", operator_csv.path)
return operator_manifests_metadata
def should_run(self):
"""
Determine if this is an operator manifest bundle build
:return: bool, should plugin run?
"""
if not has_operator_bundle_manifest(self.workflow):
self.log.info("Not an operator manifest bundle build, skipping plugin")
return False
if not self.workflow.conf.operator_manifests:
msg = "operator_manifests configuration missing in reactor config map, aborting"
self.log.warning(msg)
return False
return True
def _skip_all(self):
skip_all = self.user_config.get("skip_all", False)
if not skip_all:
return False
site_config = self.workflow.conf.operator_manifests
allowed_packages = site_config.get("skip_all_allow_list", [])
# any_platform: the component label should be equal for all platforms
parser = self.workflow.build_dir.any_platform.dockerfile_with_parent_env(
self.workflow.imageutil.base_image_inspect()
)
dockerfile_labels = parser.labels
labels = Labels(dockerfile_labels)
component_label = labels.get_name(Labels.LABEL_TYPE_COMPONENT)
component = dockerfile_labels[component_label]
if component in allowed_packages:
return True
else:
raise RuntimeError("Koji package: {} isn't allowed to use skip_all for operator "
"bundles".format(component))
def _get_operator_manifest(self):
if self.workflow.source.config.operator_manifests is None:
raise RuntimeError("operator_manifests configuration missing in container.yaml")
self.log.info("Looking for operator CSV files in %s", self.workflow.build_dir.path)
manifests_dir = self.workflow.source.config.operator_manifests["manifests_dir"]
kwargs = {'repo_dir': self.workflow.build_dir.any_platform.path}
operator_manifest = OperatorManifest.from_directory(
os.path.join(self.workflow.build_dir.any_platform.path, manifests_dir), **kwargs)
self.log.info("Found operator CSV file: %s", operator_manifest.csv.path)
return operator_manifest
def _get_pullspecs(self, operator_csv, skip_all):
"""Get pullspecs from CSV file
:param OperatorCSV operator_csv: a cluster service version (CSV) file
from where to find out pullspecs.
:return: a list of pullspecs sorted by each one's string representation.
If CSV does not have spec.relatedImages, all pullspecs will be
found out from all possible locations. If CSV has spec.relatedImages,
return the pullspecs contained.
:rtype: list[ImageName]
:raises RuntimeError: if the CSV has both spec.relatedImages and
pullspecs referenced by environment variables prefixed with
RELATED_IMAGE_.
"""
self.log.info("Looking for pullspecs in operator CSV file")
pullspec_set = set()
if not operator_csv.has_related_images():
if skip_all and operator_csv.get_pullspecs():
raise RuntimeError("skip_all defined but relatedImages section doesn't exist")
self.log.info("Getting pullspecs from %s", operator_csv.path)
pullspec_set.update(operator_csv.get_pullspecs())
elif operator_csv.has_related_image_envs():
msg = ("Both relatedImages and RELATED_IMAGE_* env vars present in {}. "
"Please remove the relatedImages section, it will be reconstructed "
"automatically.".format(operator_csv.path))
if not skip_all:
raise RuntimeError(msg)
else:
pullspec_set.update(operator_csv.get_related_image_pullspecs())
# Make sure pullspecs are handled in a deterministic order
# ImageName does not implement ordering, use str() as key for sorting
pullspecs = sorted(pullspec_set, key=str)
if pullspecs:
pullspec_lines = "\n".join(image.to_str() for image in pullspecs)
self.log.info("Found pullspecs:\n%s", pullspec_lines)
else:
self.log.info("No pullspecs found")
return pullspecs
def _get_replacement_pullspecs(self, pullspecs):
"""Replace components of pullspecs
:param pullspecs: a list of pullspecs.
:type pullspecs: list[ImageName]
:return: a list of replacement result. Each of the replacement result
is a mapping containing key/value pairs:
* ``original``: ImageName, the original pullspec.
* ``new``: ImageName, the replaced/non-replaced pullspec.
* ``pinned``: bool, indicate whether the tag is replaced with a
specific digest.
* ``replaced``: bool, indicate whether the new pullspec has change
of repository or registry.
:rtype: list[dict[str, ImageName or bool]]
:raises RuntimeError: if pullspecs cannot be properly replaced
"""
if self.operator_csv_modifications_url:
replacements = self._get_replacement_pullspecs_from_csv_modifications(pullspecs)
else:
replacements = self._get_replacement_pullspecs_OSBS_resolution(pullspecs)
replacement_lines = "\n".join(
"{original} -> {new}".format(**r) if r['replaced']
else "{original} - no change".format(**r)
for r in replacements
)
self.log.info("To be replaced:\n%s", replacement_lines)
return replacements
def _get_replacement_pullspecs_from_csv_modifications(self, pullspecs):
"""Replace components of pullspecs based on externally provided CSV modifications
:param pullspecs: a list of pullspecs.
:type pullspecs: list[ImageName]
:return: a list of replacement result. Each of the replacement result
is a mapping containing key/value pairs:
* ``original``: ImageName, the original pullspec.
* ``new``: ImageName, the replaced/non-replaced pullspec.
* ``pinned``: bool, indicate whether the tag is replaced with a
specific digest.
* ``replaced``: bool, indicate whether the new pullspec has change
of repository or registry.
:rtype: list[dict[str, ImageName or bool]]
:raises RuntimeError: if provided CSV modification doesn't contain all
required pullspecs or contain different ones
"""
operator_csv_modifications = self._fetch_operator_csv_modifications()
mod_pullspec_repl = operator_csv_modifications.get('pullspec_replacements', [])
# check if modification data contains all required pullspecs
pullspecs_set = set(pullspecs)
mod_pullspecs_set = set((ImageName.parse(p['original']) for p in mod_pullspec_repl))
missing = pullspecs_set - mod_pullspecs_set
if missing:
raise RuntimeError(
f"Provided operator CSV modifications misses following pullspecs: "
f"{', '.join(sorted(str(p) for p in missing))}"
)
extra = mod_pullspecs_set - pullspecs_set
if extra:
raise RuntimeError(
f"Provided operator CSV modifications defines extra pullspecs: "
f"{','.join(sorted(str(p) for p in extra))}"
)
# Copy replacements from provided CSV modifications file, fill missing 'replaced' filed
replacements = [
{
'original': ImageName.parse(repl['original']),
'new': ImageName.parse(repl['new']),
'pinned': repl['pinned'],
'replaced': repl['original'] != repl['new']
}
for repl in mod_pullspec_repl
]
return replacements
def _get_replacement_pullspecs_OSBS_resolution(self, pullspecs):
"""
Replace components of pullspecs according to operator manifest
replacement config
:param pullspecs: a list of pullspecs.
:type pullspecs: list[ImageName]
:return: a list of replacement result. Each of the replacement result
is a mapping containing key/value pairs:
* ``original``: ImageName, the original pullspec.
* ``new``: ImageName, the replaced/non-replaced pullspec.
* ``pinned``: bool, indicate whether the tag is replaced with a
specific digest.
* ``replaced``: bool, indicate whether the new pullspec has change
of repository or registry.
:rtype: list[dict[str, ImageName or bool]]
:raises RuntimeError: if the registry of a pullspec is not allowed.
Refer to the ``operator_manifest.allowed_registries`` in atomic
reactor config.
"""
self.log.info("Computing replacement pullspecs")
replacements = []
pin_digest, replace_repo, replace_registry = self._are_features_enabled()
if not any([pin_digest, replace_repo, replace_registry]):
self.log.warning("All replacement features disabled")
replacer = PullspecReplacer(user_config=self.user_config, workflow=self.workflow)
for p in pullspecs:
if not replacer.registry_is_allowed(p):
raise RuntimeError("Registry not allowed: {} (in {})".format(p.registry, p))
for original in pullspecs:
self.log.info("Computing replacement for %s", original)
replaced = original
pinned = False
if pin_digest:
self.log.debug("Making sure tag is manifest list digest")
replaced = replacer.pin_digest(original)
if replaced != original:
pinned = True
if replace_repo:
self.log.debug("Replacing namespace/repo")
replaced = replacer.replace_repo(replaced)
if replace_registry:
self.log.debug("Replacing registry")
replaced = replacer.replace_registry(replaced)
self.log.info("Final pullspec: %s", replaced)
replacements.append({
'original': original,
'new': replaced,
'pinned': pinned,
'replaced': replaced != original
})
return replacements
def _are_features_enabled(self):
pin_digest = self.user_config.get("enable_digest_pinning", True)
replace_repo = self.user_config.get("enable_repo_replacements", True)
replace_registry = self.user_config.get("enable_registry_replacements", True)
if not pin_digest:
self.log.warning("User disabled digest pinning")
if not replace_repo:
self.log.warning("User disabled repo replacements")
if not replace_registry:
self.log.warning("User disabled registry replacements")
return pin_digest, replace_repo, replace_registry
class InjectParentImage(Plugin):
"""
Modifies parent image to be used based on given Koji build.
It first attempts to find the list of available repositories
from '.extra.image.index.pull' in Koji build information. If
not found, the first archive in Koji build that defines a non-empty
'.extra.docker.repositories' list is used.
This list provides the pull reference for the container image
associated with Koji build. If it contains multiple item, the
manifest digest, @sha256, is preferred. Otherwise, the first
repository in list is used.
The namespace and repository for the new parent image must match
the namespace and repository for the parent image defined in
Dockerfile.
This plugin returns the identifier of the Koji build used.
"""
key = PLUGIN_INJECT_PARENT_IMAGE_KEY
is_allowed_to_fail = False
args_from_user_params = map_to_user_params("koji_parent_build")
def __init__(self, workflow, koji_parent_build=None):
"""
:param workflow: DockerBuildWorkflow instance
:param koji_parent_build: str, either Koji build ID or Koji build NVR
"""
super(InjectParentImage, self).__init__(workflow)
self.koji_session = get_koji_session(self.workflow.conf)
try:
self.koji_parent_build = int(koji_parent_build)
except (ValueError, TypeError):
self.koji_parent_build = koji_parent_build
self._koji_parent_build_info = None
self._repositories = None
self._new_parent_image = None
def run(self):
if not self.koji_parent_build:
self.log.info('no koji parent build, skipping plugin')
return
if self.workflow.data.dockerfile_images.base_from_scratch:
self.log.info("from scratch can't inject parent image")
return
if self.workflow.data.dockerfile_images.custom_base_image:
self.log.info("custom base image builds can't inject parent image")
return
self.find_repositories()
self.select_new_parent_image()
self.adjust_new_parent_image()
self.set_new_parent_image()
return self._koji_parent_build_info['id']
def find_repositories(self):
self._repositories = (self.find_repositories_from_build() or
self.find_repositories_from_archive())
if not self._repositories:
raise RuntimeError('A suitable archive for Koji build {} was not found'
.format(self._koji_parent_build_info['nvr']))
def find_repositories_from_build(self):
self._koji_parent_build_info = self.koji_session.getBuild(self.koji_parent_build)
if not self._koji_parent_build_info:
raise RuntimeError('Koji build, {}, not found'.format(self.koji_parent_build))
repositories = graceful_chain_get(self._koji_parent_build_info,
'extra', 'image', 'index', 'pull')
if repositories:
self.log.info('Using repositories from build info')
return repositories
def find_repositories_from_archive(self):
for archive in self.koji_session.listArchives(self._koji_parent_build_info['id']):
repositories = graceful_chain_get(archive, 'extra', 'docker', 'repositories')
if repositories:
self.log.info('Using repositories from archive %d', archive['id'])
return repositories
return None
def select_new_parent_image(self):
for repository in self._repositories:
if '@' in repository:
self._new_parent_image = repository
break
# v2 manifest digest, not found, just pick the first one.
if not self._new_parent_image:
self._new_parent_image = self._repositories[0]
self.log.info('New parent image is %s', self._new_parent_image)
def adjust_new_parent_image(self):
new_parent_image = ImageName.parse(self._new_parent_image)
organization = self.workflow.conf.registries_organization
source_registry_docker_uri = self.workflow.conf.source_registry['uri'].docker_uri
if new_parent_image.registry != source_registry_docker_uri:
new_parent_image.registry = source_registry_docker_uri
if organization:
new_parent_image.enclose(organization)
self._new_parent_image = new_parent_image.to_str()
def set_new_parent_image(self):
df_images = self.workflow.data.dockerfile_images
base_image_key = df_images.base_image_key
df_images[base_image_key] = self._new_parent_image
class CheckUserSettingsPlugin(PreBuildPlugin):
"""
Pre plugin will check user settings on early phase to fail early and save resources.
Aim of this plugin to checks:
* Dockerfile
* container.yaml
* git repo
for incorrect options or mutually exclusive options
"""
key = PLUGIN_CHECK_USER_SETTINGS
is_allowed_to_fail = False
args_from_user_params = map_to_user_params("flatpak")
def __init__(self, workflow, flatpak=False):
"""
:param workflow: DockerBuildWorkflow instance
:param flatpak: bool, if build is for flatpak
"""
super(CheckUserSettingsPlugin, self).__init__(workflow)
self.flatpak = flatpak
def dockerfile_checks(self):
"""Checks for Dockerfile"""
if self.flatpak:
self.log.info(
"Skipping Dockerfile checks because this is flatpak build "
"without user Dockerfile")
return
self.label_version_check()
self.appregistry_bundle_label_mutually_exclusive()
self.operator_bundle_from_scratch()
def label_version_check(self):
"""Check that Dockerfile version has correct name."""
msg = "Dockerfile version label can't contain '/' character"
self.log.debug("Running check: %s", msg)
# any_platform: the version label should be equal for all platforms
parser = self.workflow.build_dir.any_platform.dockerfile_with_parent_env(
self.workflow.imageutil.base_image_inspect())
dockerfile_labels = parser.labels
labels = Labels(parser.labels)
component_label = labels.get_name(Labels.LABEL_TYPE_VERSION)
label_version = dockerfile_labels[component_label]
if '/' in label_version:
raise ValueError(msg)
def appregistry_bundle_label_mutually_exclusive(self):
"""Labels com.redhat.com.delivery.appregistry and
com.redhat.delivery.operator.bundle
are mutually exclusive. Fail when both are specified.
"""
msg = ("only one of labels com.redhat.com.delivery.appregistry "
"and com.redhat.delivery.operator.bundle is allowed")
self.log.debug("Running check: %s", msg)
if (has_operator_appregistry_manifest(self.workflow)
and has_operator_bundle_manifest(self.workflow)):
raise ValueError(msg)
def operator_bundle_from_scratch(self):
"""Only from scratch image can be used for operator bundle build"""
msg = "Operator bundle build can be only 'FROM scratch' build (single stage)"
self.log.debug("Running check: %s", msg)
if not has_operator_bundle_manifest(self.workflow):
return
df_images = self.workflow.data.dockerfile_images
if not df_images.base_from_scratch or len(
df_images.original_parents) > 1:
raise ValueError(msg)
def validate_user_config_files(self):
"""Validate some user config files"""
read_fetch_artifacts_koji(self.workflow)
read_fetch_artifacts_pnc(self.workflow)
read_fetch_artifacts_url(self.workflow)
read_content_sets(self.workflow)
def isolated_from_scratch_build(self):
"""Isolated builds for FROM scratch builds are prohibited
except operator bundle images"""
if (self.workflow.data.dockerfile_images.base_from_scratch
and is_isolated_build(self.workflow)
and not has_operator_bundle_manifest(self.workflow)):
raise RuntimeError('"FROM scratch" image build cannot be isolated '
'(except operator bundle images)')
def isolated_builds_checks(self):
"""Validate if isolated build was used correctly"""
self.isolated_from_scratch_build()
def run(self):
"""
run the plugin
"""
self.dockerfile_checks()
self.validate_user_config_files()
self.isolated_builds_checks()
class FetchSourcesPlugin(Plugin):
"""Download sources that may be used in further steps to compose Source Containers"""
key = PLUGIN_FETCH_SOURCES_KEY
is_allowed_to_fail = False
SRPMS_DOWNLOAD_DIR = 'image_sources'
REMOTE_SOURCES_DOWNLOAD_DIR = 'remote_sources'
MAVEN_SOURCES_DOWNLOAD_DIR = 'maven_sources'
args_from_user_params = map_to_user_params(
"koji_build_id:sources_for_koji_build_id",
"koji_build_nvr:sources_for_koji_build_nvr",
"signing_intent",
)
def __init__(
self,
workflow,
koji_build_id=None,
koji_build_nvr=None,
signing_intent=None,
):
"""
:param workflow: DockerBuildWorkflow instance
:param koji_build_id: int, container image koji build id
:param koji_build_nvr: str, container image koji build NVR
:param signing_intent: str, ODCS signing intent name
"""
if not koji_build_id and not koji_build_nvr:
err_msg = (
'{} expects either koji_build_id or koji_build_nvr to be defined'
.format(self.__class__.__name__))
raise TypeError(err_msg)
type_errors = []
if koji_build_id is not None and not isinstance(koji_build_id, int):
type_errors.append('koji_build_id must be an int. Got {}'.format(
type(koji_build_id)))
if koji_build_nvr is not None and not isinstance(koji_build_nvr, str):
type_errors.append('koji_build_nvr must be a str. Got {}'.format(
type(koji_build_nvr)))
if type_errors:
raise TypeError(type_errors)
super(FetchSourcesPlugin, self).__init__(workflow)
self.koji_build = None
self.koji_build_id = koji_build_id
self.koji_build_nvr = koji_build_nvr
self.signing_intent = signing_intent
self.session = get_koji_session(self.workflow.conf)
self.pathinfo = self.workflow.conf.koji_path_info
self._pnc_util = None
@property
def pnc_util(self):
if not self._pnc_util:
pnc_map = self.workflow.conf.pnc
if not pnc_map:
raise RuntimeError(
'No PNC configuration found in reactor config map')
self._pnc_util = PNCUtil(pnc_map)
return self._pnc_util
def run(self):
"""
:return: dict, binary image koji build id and nvr, and path to directory with
downloaded sources
"""
self.set_koji_image_build_data()
self.check_lookaside_cache_usage()
signing_intent = self.get_signing_intent()
koji_config = self.workflow.conf.koji
insecure = koji_config.get('insecure_download', False)
urls = self.get_srpm_urls(signing_intent['keys'], insecure=insecure)
urls_remote, remote_sources_map = self.get_remote_urls()
urls_maven = (self.get_kojifile_source_urls() +
self.get_remote_file_urls() + self.get_pnc_source_urls())
if not any([urls, urls_remote, urls_maven]):
msg = "No srpms or remote sources or maven sources found for source" \
" container, would produce empty source container image"
self.log.error(msg)
raise RuntimeError(msg)
sources_dir = None
remote_sources_dir = None
maven_sources_dir = None
if urls:
sources_dir = self.download_sources(urls, insecure=insecure)
if urls_remote:
remote_sources_dir = self.download_sources(
urls_remote,
insecure=insecure,
download_dir=self.REMOTE_SOURCES_DOWNLOAD_DIR)
self.exclude_files_from_remote_sources(remote_sources_map,
remote_sources_dir)
if urls_maven:
maven_sources_dir = self.download_sources(
urls_maven,
insecure=insecure,
download_dir=self.MAVEN_SOURCES_DOWNLOAD_DIR)
return {
'sources_for_koji_build_id': self.koji_build_id,
'sources_for_nvr': self.koji_build_nvr,
'image_sources_dir': sources_dir,
'remote_sources_dir': remote_sources_dir,
'maven_sources_dir': maven_sources_dir,
'signing_intent': self.signing_intent,
}
def download_sources(self,
sources,
insecure=False,
download_dir=SRPMS_DOWNLOAD_DIR):
"""Download sources content
Download content in the given URLs into a new temporary directory and
return a list with each downloaded artifact's path.
:param sources: list, dicts with URLs to download
:param insecure: bool, whether to perform TLS checks of urls
:param download_dir: str, directory where to download content
:return: str, paths to directory with downloaded sources
"""
dest_dir: Path = self.workflow.build_dir.source_container_sources_dir / download_dir
dest_dir.mkdir(parents=True, exist_ok=True)
req_session = get_retrying_requests_session()
for source in sources:
subdir: Path = dest_dir / source.get('subdir', '')
subdir.mkdir(parents=True, exist_ok=True)
checksums = source.get('checksums', {})
download_url(source['url'],
subdir,
insecure=insecure,
session=req_session,
dest_filename=source.get('dest'),
expected_checksums=checksums)
return str(dest_dir)
def set_koji_image_build_data(self):
build_identifier = self.koji_build_nvr or self.koji_build_id
# strict means this raises a koji.GenericError informing no matching build was found in
# case the build does not exist
self.koji_build = self.session.getBuild(build_identifier, strict=True)
if self.koji_build_id and (self.koji_build_id !=
self.koji_build['build_id']):
err_msg = (
'koji_build_id {} does not match koji_build_nvr {} with id {}. '
'When specifying both an id and an nvr, they should point to the same image build'
.format(self.koji_build_id, self.koji_build_nvr,
self.koji_build['build_id']))
raise ValueError(err_msg)
build_extras = self.koji_build['extra']
if 'image' not in build_extras:
err_msg = (
'koji build {} is not image build which source container requires'
.format(self.koji_build['nvr']))
raise ValueError(err_msg)
elif 'sources_for_nvr' in self.koji_build['extra']['image']:
err_msg = (
'koji build {} is source container build, source container can not '
'use source container build image'.format(
self.koji_build['nvr']))
raise ValueError(err_msg)
if not self.koji_build_id:
self.koji_build_id = self.koji_build['build_id']
if not self.koji_build_nvr:
self.koji_build_nvr = self.koji_build['nvr']
def _get_cache_allowlist(self) -> List[Dict[str, Any]]:
src_config = self.workflow.conf.source_container
allowlist_cache_url = src_config.get('lookaside_cache_allowlist')
if not allowlist_cache_url:
self.log.debug('no "lookaside_cache_allowlist" defined, '
'not allowing any lookaside cache usage')
return []
self.log.debug(
'"lookaside_cache_allowlist" defined, might allow lookaside cache usage'
)
request_session = get_retrying_requests_session()
response = request_session.get(allowlist_cache_url)
response.raise_for_status()
allowlist_cache_yaml = yaml.safe_load(response.text)
return allowlist_cache_yaml
def check_lookaside_cache_usage(self):
"""Check usage of lookaside cache, and fail if used"""
git_uri, git_commit = self.koji_build['source'].split('#')
source = GitSource('git',
git_uri,
provider_params={'git_commit': git_commit})
source_path = source.get()
sources_cache_file = os.path.join(source_path, 'sources')
uses_cache = False
if os.path.exists(sources_cache_file):
if os.path.getsize(sources_cache_file) > 0:
uses_cache = True
source.remove_workdir()
if not uses_cache:
return
allowlist_cache_yaml = self._get_cache_allowlist()
should_raise = True
if allowlist_cache_yaml:
build_component = self.koji_build['package_name']
build_task = self.koji_build['extra']['container_koji_task_id']
task_info = self.session.getTaskInfo(build_task, request=True)
build_target = task_info['request'][1]
for allowed in allowlist_cache_yaml:
if allowed['component'] != build_component:
continue
for target in allowed['targets']:
if re.match(target, build_target):
self.log.debug(
'target "%s" for component "%s" has allowed using '
'lookaside cache', build_target, build_component)
should_raise = False
break
if not should_raise:
break
if should_raise:
raise RuntimeError(
'Repository is using lookaside cache, which is not allowed '
'for source container builds')
def assemble_srpm_url(self, base_url, srpm_filename, sign_key=None):
"""Assemble the URL used to fetch an SRPM file
:param base_url: str, Koji root base URL with the given build artifacts
:param srpm_filename: str, name of the SRPM file
:param sign_key: str, key used to sign the SRPM, as listed in the signing intent
:return: list, strings with URLs pointing to SRPM files
"""
srpm_info = koji.parse_NVRA(srpm_filename)
if sign_key:
srpm_path = self.pathinfo.signed(srpm_info, sign_key)
else:
srpm_path = self.pathinfo.rpm(srpm_info)
return '/'.join([base_url, srpm_path])
def _process_remote_source(self, koji_build, archives,
remote_sources_path):
self.log.debug('remote_source_url defined')
remote_sources_urls = []
remote_json_map = {}
remote_source = {}
remote_source['url'] = os.path.join(remote_sources_path,
REMOTE_SOURCE_TARBALL_FILENAME)
remote_source['dest'] = '-'.join(
[koji_build['nvr'], REMOTE_SOURCE_TARBALL_FILENAME])
remote_sources_urls.append(remote_source)
cachito_json_url = os.path.join(remote_sources_path,
REMOTE_SOURCE_JSON_FILENAME)
remote_json_map[remote_source['dest']] = cachito_json_url
archive_found = False
json_found = False
all_archives = []
for archive in archives:
if archive['filename'] == REMOTE_SOURCE_TARBALL_FILENAME:
archive_found = True
elif archive['filename'] == REMOTE_SOURCE_JSON_FILENAME:
json_found = True
all_archives.append(archive['filename'])
if not (archive_found and json_found):
message = ', '.join(
part
for t, part in ((archive_found,
"remote source archive missing"),
(json_found, "remote source json missing"))
if not t)
raise RuntimeError(message)
elif len(archives) > 2:
raise RuntimeError(
'There can be just one remote sources archive and one '
'remote sources json, got: {}'.format(all_archives))
return remote_sources_urls, remote_json_map
def _process_multiple_remote_sources(self, koji_build, archives,
remote_sources_path):
self.log.debug('remote_sources defined')
remote_sources_urls = []
remote_json_map = {}
remote_sources = koji_build['extra']['typeinfo']['remote-sources']
wrong_archives = False
all_archives = []
for remote_s in remote_sources:
remote_archive = None
remote_json = None
if len(remote_s['archives']) != 2:
self.log.error(
'remote source "%s" does not contain 2 archives, but "%s"',
remote_s['name'], remote_s['archives'])
wrong_archives = True
else:
for archive in remote_s['archives']:
if archive.endswith('.json'):
remote_json = archive
else:
remote_archive = archive
if not remote_json:
self.log.error(
'remote source json, for remote source "%s" not found '
'in archives "%s"', remote_s['name'],
remote_s['archives'])
wrong_archives = True
else:
remote_source = {}
remote_source['url'] = os.path.join(
remote_sources_path, remote_archive)
remote_source['dest'] = '-'.join(
[koji_build['nvr'], remote_archive])
remote_sources_urls.append(remote_source)
cachito_json_url = os.path.join(remote_sources_path,
remote_json)
remote_json_map[remote_source['dest']] = cachito_json_url
all_archives.append(remote_archive)
all_archives.append(remote_json)
if wrong_archives:
raise RuntimeError(
'Problems with archives in remote sources: {}'.format(
remote_sources))
extra_archives = []
for archive in archives:
if archive['filename'] in all_archives:
all_archives.remove(archive['filename'])
else:
extra_archives.append(archive['filename'])
if all_archives:
raise RuntimeError(
'Remote source files from metadata missing in koji '
'archives: {}'.format(all_archives))
if extra_archives:
raise RuntimeError('Remote source archives in koji missing from '
'metadata: {}'.format(extra_archives))
return remote_sources_urls, remote_json_map
def _get_remote_urls_helper(self, koji_build):
"""Fetch remote source urls from specific build
:param koji_build: dict, koji build
:return: str, URL pointing to remote sources
"""
self.log.debug('get remote_urls: %s', koji_build['build_id'])
archives = self.session.listArchives(koji_build['build_id'],
type=KOJI_BTYPE_REMOTE_SOURCES)
self.log.debug('archives: %s', archives)
remote_sources_path = self.pathinfo.typedir(
koji_build, btype=KOJI_BTYPE_REMOTE_SOURCES)
remote_sources_urls = []
remote_json_map = {}
if 'remote_source_url' in koji_build['extra']['image']:
remote_sources_urls, remote_json_map = \
self._process_remote_source(koji_build, archives, remote_sources_path)
elif 'remote_sources' in koji_build['extra']['image']:
remote_sources_urls, remote_json_map = \
self._process_multiple_remote_sources(koji_build, archives, remote_sources_path)
return remote_sources_urls, remote_json_map
def _get_kojifile_source_urls_helper(self, koji_build):
"""Fetch kojifile source urls from specific build
:param koji_build: dict, koji build
:return: list, dicts with URL pointing to kojifile sources
"""
self.log.debug('get kojifile_source_urls: %s', koji_build['build_id'])
images = self.session.listArchives(koji_build['build_id'],
type='image')
self.log.debug('images: %s', images)
sources = []
kojifile_build_ids = {
kojifile['build_id']
for image in images
for kojifile in self.session.listArchives(imageID=image['id'],
type='maven')
}
for build_id in kojifile_build_ids:
source_build = self.session.getBuild(build_id, strict=True)
if source_build['owner_name'] == PNC_SYSTEM_USER:
pnc_build_id = source_build['extra']['external_build_id']
url, dest_filename = self.pnc_util.get_scm_archive_from_build_id(
build_id=pnc_build_id)
source = {
'url': url,
'subdir': source_build['nvr'],
'dest': '__'.join([source_build['nvr'], dest_filename])
}
else:
source_archive = None
maven_build_path = self.pathinfo.mavenbuild(source_build)
for archive in self.session.listArchives(
buildID=source_build['build_id'], type='maven'):
if archive['filename'].endswith('-project-sources.tar.gz'):
source_archive = archive
break
if not source_archive:
raise RuntimeError(
f"No sources found for {source_build['nvr']}")
maven_file_path = self.pathinfo.mavenfile(source_archive)
url = maven_build_path + '/' + maven_file_path
source = {
'url':
url,
'subdir':
source_build['nvr'],
'dest':
'__'.join(
[source_build['nvr'], source_archive['filename']]),
'checksums': {
koji.CHECKSUM_TYPES[source_archive['checksum_type']]:
source_archive['checksum']
}
}
sources.append(source)
return sources
def _get_pnc_source_urls_helper(self, koji_build):
"""Fetch PNC source urls from specific build
:param koji_build: dict, koji build
:return: list, dicts with URL pointing to PNC sources
"""
sources = []
if 'pnc' not in koji_build['extra']['image']:
self.log.info("No PNC build ids found")
return sources
build_ids = set()
for build in koji_build['extra']['image']['pnc']['builds']:
build_ids.add(build['id'])
self.log.debug('PNC build ids: %s', build_ids)
for build_id in build_ids:
url, dest_filename = self.pnc_util.get_scm_archive_from_build_id(
build_id=build_id)
source = {
'url': url,
'subdir': str(build_id),
'dest': '__'.join([str(build_id), dest_filename])
}
sources.append(source)
return sources
def _get_remote_file_urls_helper(self, koji_build):
"""Fetch remote source file urls from specific build
:param koji_build: dict, koji build
:return: str, URL pointing to remote source files
"""
self.log.debug('get remote_file_urls: %s', koji_build['build_id'])
archives = self.session.listArchives(koji_build['build_id'],
type='remote-source-file')
self.log.debug('archives: %s', archives)
remote_source_files_path = self.pathinfo.typedir(
koji_build, btype='remote-source-file')
sources = []
for archive in archives:
if archive['type_name'] == 'tar':
# download each remote-source-file archive into it's own subdirectory
# with the same name as the archive
source = {
'url':
os.path.join(remote_source_files_path,
archive['filename']),
'subdir':
archive['filename'].rsplit('.', 2)[0],
'dest':
archive['filename'],
'checksums': {
koji.CHECKSUM_TYPES[archive['checksum_type']]:
archive['checksum']
}
}
sources.append(source)
return sources
def get_remote_urls(self):
"""Fetch remote source urls from all builds
:return: list, dicts with URL pointing to remote sources
"""
remote_sources_urls = []
remote_sources_map = {}
remote_source, remote_json = self._get_remote_urls_helper(
self.koji_build)
remote_sources_urls.extend(remote_source)
remote_sources_map.update(remote_json)
koji_build = self.koji_build
while 'parent_build_id' in koji_build['extra']['image']:
koji_build = self.session.getBuild(
koji_build['extra']['image']['parent_build_id'], strict=True)
remote_source, remote_json = self._get_remote_urls_helper(
koji_build)
remote_sources_urls.extend(remote_source)
remote_sources_map.update(remote_json)
return remote_sources_urls, remote_sources_map
def get_kojifile_source_urls(self):
"""Fetch kojifile source urls from all builds
:return: list, dicts with URL pointing to kojifile source files
"""
kojifile_sources = []
kojifile_source = self._get_kojifile_source_urls_helper(
self.koji_build)
kojifile_sources.extend(kojifile_source)
koji_build = self.koji_build
while 'parent_build_id' in koji_build['extra']['image']:
koji_build = self.session.getBuild(
koji_build['extra']['image']['parent_build_id'], strict=True)
kojifile_source = self._get_kojifile_source_urls_helper(koji_build)
kojifile_sources.extend(kojifile_source)
return kojifile_sources
def get_pnc_source_urls(self):
"""Fetch PNC build source urls from all builds
:return: list, dicts with URL pointing to PNC build source archives
"""
sources = []
source = self._get_pnc_source_urls_helper(self.koji_build)
sources.extend(source)
koji_build = self.koji_build
while 'parent_build_id' in koji_build['extra']['image']:
koji_build = self.session.getBuild(
koji_build['extra']['image']['parent_build_id'], strict=True)
source = self._get_pnc_source_urls_helper(koji_build)
sources.extend(source)
return sources
def get_remote_file_urls(self):
"""Fetch remote source file urls from all builds
:return: list, dicts with URL pointing to remote source files
"""
sources = []
source = self._get_remote_file_urls_helper(self.koji_build)
sources.extend(source)
koji_build = self.koji_build
while 'parent_build_id' in koji_build['extra']['image']:
koji_build = self.session.getBuild(
koji_build['extra']['image']['parent_build_id'], strict=True)
source = self._get_remote_file_urls_helper(koji_build)
sources.extend(source)
return sources
def get_denylisted_srpms(self):
src_config = self.workflow.conf.source_container
denylist_srpms = src_config.get('denylist_srpms')
if not denylist_srpms:
self.log.debug(
'denylist_srpms is not defined in reactor_config_map')
return []
denylist_url = denylist_srpms['denylist_url']
denylist_key = denylist_srpms['denylist_key']
req_session = get_retrying_requests_session()
response = req_session.get(denylist_url)
response.raise_for_status()
response_json = response.json()
if denylist_key not in response_json:
self.log.debug('deny list json : %s', response_json)
raise RuntimeError(
'Denylist key: {} missing in denylist json from : {}'.format(
denylist_key, denylist_url))
deny_list = response_json[denylist_key]
if not isinstance(deny_list, list):
self.log.error('Wrong denylist: %s', repr(deny_list))
raise RuntimeError(
'Denylist value in key: {} is not list: {}'.format(
denylist_key, type(deny_list)))
wrong_types = [pkg for pkg in deny_list if not isinstance(pkg, str)]
if wrong_types:
self.log.error('Wrong types in denylist, should be str: %s',
repr(deny_list))
raise RuntimeError('Values in denylist has to be all strings')
self.log.debug('denylisted srpms: %s', deny_list)
return deny_list
def get_srpm_urls(self, sigkeys=None, insecure=False):
"""Fetch SRPM download URLs for each image generated by a build
Build each possible SRPM URL and check if the URL is available,
respecting the signing intent preference order.
:param sigkeys: list, strings for keys which signed the srpms to be fetched
:return: list, strings with URLs pointing to SRPM files
"""
if not sigkeys:
sigkeys = ['']
self.log.debug('get srpm_urls: %s', self.koji_build_id)
archives = self.session.listArchives(self.koji_build_id, type='image')
self.log.debug('archives: %s', archives)
rpms = [
rpm for archive in archives
for rpm in self.session.listRPMs(imageID=archive['id'])
]
denylist_srpms = self.get_denylisted_srpms()
srpm_build_paths = {}
for rpm in rpms:
rpm_id = rpm['id']
self.log.debug('Resolving SRPM for RPM ID: %s', rpm_id)
if rpm['external_repo_name'] != 'INTERNAL':
msg = ('RPM comes from an external repo (RPM ID: {}). '
'External RPMs are currently not supported.'
).format(rpm_id)
raise RuntimeError(msg)
rpm_hdr = self.session.getRPMHeaders(rpm_id, headers=['SOURCERPM'])
if 'SOURCERPM' not in rpm_hdr:
raise RuntimeError(
'Missing SOURCERPM header (RPM ID: {})'.format(rpm_id))
srpm_name = rpm_hdr['SOURCERPM'].rsplit('-', 2)[0]
if any(denied == srpm_name for denied in denylist_srpms):
self.log.debug('skipping denylisted srpm %s',
rpm_hdr['SOURCERPM'])
continue
srpm_filename = rpm_hdr['SOURCERPM']
if srpm_filename in srpm_build_paths:
continue
rpm_build = self.session.getBuild(rpm['build_id'], strict=True)
base_url = self.pathinfo.build(rpm_build)
srpm_build_paths[srpm_filename] = base_url
srpm_urls = []
missing_srpms = []
req_session = get_retrying_requests_session()
for srpm_filename, base_url in srpm_build_paths.items():
for sigkey in sigkeys:
# koji uses lowercase for paths. We make sure the sigkey is in lower case
url_candidate = self.assemble_srpm_url(base_url, srpm_filename,
sigkey.lower())
# allow redirects, head call doesn't do it by default
request = req_session.head(url_candidate,
verify=not insecure,
allow_redirects=True)
if request.ok:
srpm_urls.append({'url': url_candidate})
self.log.debug('%s is available for signing key "%s"',
srpm_filename, sigkey)
break
else:
self.log.error(
'%s not found for the given signing intent: %s"',
srpm_filename, self.signing_intent)
missing_srpms.append(srpm_filename)
if missing_srpms:
raise RuntimeError(
'Could not find files signed by any of {} for these SRPMS: {}'.
format(sigkeys, missing_srpms))
return srpm_urls
def get_signing_intent(self):
"""Get the signing intent to be used to fetch files from Koji
:return: dict, signing intent object as per atomic_reactor/schemas/config.json
"""
odcs_config = self.workflow.conf.odcs_config
if odcs_config is None:
self.log.warning(
'No ODCS configuration available. Allowing unsigned SRPMs')
return {'keys': None}
if not self.signing_intent:
try:
self.signing_intent = self.koji_build['extra']['image'][
'odcs']['signing_intent']
except (KeyError, TypeError):
self.log.debug(
'Image koji build, %s(%s), does not define signing_intent.',
self.koji_build_nvr, self.koji_build_id)
self.signing_intent = odcs_config.default_signing_intent
signing_intent = odcs_config.get_signing_intent_by_name(
self.signing_intent)
return signing_intent
def _get_denylist_sources(self, request_session, denylist_sources_url):
response = request_session.get(denylist_sources_url)
response.raise_for_status()
denylist_sources_yaml = yaml.safe_load(response.text)
# prepend os.sep for 2 reasons:
# - so fnmatch will match exact dir/file when using * + exclude
# glob.glob doesn't need it
# - so endswith for package will match also full name
return [
os.path.join(os.sep, k, item)
for k, v in denylist_sources_yaml.items() for item in v
]
def _create_full_remote_sources_map(self, request_session,
remote_sources_map,
remote_sources_dir):
full_remote_sources_map = {}
for remote_source, remote_source_json in remote_sources_map.items():
remote_source_archive = os.path.join(remote_sources_dir,
remote_source)
response = request_session.get(remote_source_json)
response.raise_for_status()
response_json = response.json()
full_remote_sources_map[remote_source_archive] = response_json
return full_remote_sources_map
def _check_if_package_excluded(self, packages, denylist_sources,
remote_archive):
# check if any package in cachito json matches excluded entry
# strip leading os.sep as package names can include git path with '/' before package name
# or just package name, or package name with leading '@' depending on package type
denylist_packages = {k.lstrip(os.sep) for k in denylist_sources}
for package in packages:
for exclude_path in denylist_packages:
if package.get('name').endswith(exclude_path):
self.log.debug('Package excluded: "%s" from "%s"',
package.get('name'), remote_archive)
return True
return False
def _delete_app_directory(self, remote_source_dir, unpack_dir,
remote_archive):
vendor_dir = os.path.join(unpack_dir, 'app', 'vendor')
if os.path.exists(vendor_dir):
shutil.move(vendor_dir, remote_source_dir)
self.log.debug('Removing app from "%s"', remote_archive)
shutil.rmtree(os.path.join(unpack_dir, 'app'))
# shutil.move will create missing parent directory
shutil.move(os.path.join(remote_source_dir, 'vendor'), vendor_dir)
self.log.debug('Keeping vendor in app from "%s"', remote_archive)
else:
self.log.debug('Removing app from "%s"', remote_archive)
shutil.rmtree(os.path.join(unpack_dir, 'app'))
def _get_excluded_matches(self, unpack_dir, denylist_sources):
matches = []
# py2 glob.glob doesn't support recursive, hence os.walk & fnmatch
# py3 can use: glob.glob(os.path.join(unpack_dir, '**', exclude), recursive=True)
for root, dirnames, filenames in os.walk(unpack_dir):
for entry in dirnames + filenames:
full_path = os.path.join(root, entry)
for exclude in denylist_sources:
if full_path.endswith(exclude):
matches.append(full_path)
break
return matches
def _remove_excluded_matches(self, matches):
for entry in matches:
if os.path.exists(entry):
if os.path.isdir(entry):
self.log.debug("Removing excluded directory %s", entry)
shutil.rmtree(entry)
else:
self.log.debug("Removing excluded file %s", entry)
os.unlink(entry)
def exclude_files_from_remote_sources(self, remote_sources_map,
remote_sources_dir):
"""
:param remote_sources_map: dict, keys are filenames of sources from cachito,
values are url with json from cachito
:param remote_sources_dir: str, dir with downloaded sources from cachito
"""
src_config = self.workflow.conf.source_container
denylist_sources_url = src_config.get('denylist_sources')
if not denylist_sources_url:
self.log.debug('no "denylist_sources" defined, not excluding any '
'files from remote sources')
return
request_session = get_retrying_requests_session()
denylist_sources = self._get_denylist_sources(request_session,
denylist_sources_url)
# key: full path to source archive, value: cachito json
full_remote_sources_map = self._create_full_remote_sources_map(
request_session, remote_sources_map, remote_sources_dir)
for remote_archive, remote_json in full_remote_sources_map.items():
unpack_dir = remote_archive + '_unpacked'
with tarfile.open(remote_archive) as tf:
tf.extractall(unpack_dir)
delete_app = self._check_if_package_excluded(
remote_json['packages'], denylist_sources, remote_archive)
# if any package in cachito json matched excluded entry,
# remove 'app' from sources, except 'app/vendor' when exists
if delete_app and os.path.exists(os.path.join(unpack_dir, 'app')):
self._delete_app_directory(remote_sources_dir, unpack_dir,
remote_archive)
# search for excluded matches
matches = self._get_excluded_matches(unpack_dir, denylist_sources)
self._remove_excluded_matches(matches)
# delete former archive
os.unlink(remote_archive)
# re-create new archive without excluded content
with tarfile.open(remote_archive, "w:gz") as tar:
for add_file in os.listdir(unpack_dir):
tar.add(os.path.join(unpack_dir, add_file),
arcname=add_file)
# cleanup unpacked dir
shutil.rmtree(unpack_dir)
class FlatpakUpdateDockerfilePlugin(PreBuildPlugin):
key = "flatpak_update_dockerfile"
is_allowed_to_fail = False
args_from_user_params = map_to_user_params("compose_ids")
def __init__(self, workflow):
"""
constructor
:param workflow: DockerBuildWorkflow instance
"""
# call parent constructor
super(FlatpakUpdateDockerfilePlugin, self).__init__(workflow)
def update_dockerfile(self, builder, compose_info, build_dir: BuildDir):
# Update the dockerfile
# We need to enable all the modules other than the platform pseudo-module
enable_modules_str = ' '.join(builder.get_enable_modules())
install_packages_str = ' '.join(builder.get_install_packages())
replacements = {
'@ENABLE_MODULES@': enable_modules_str,
'@INSTALL_PACKAGES@': install_packages_str,
'@RELEASE@': compose_info.main_module.version,
}
dockerfile = build_dir.dockerfile
content = dockerfile.content
# Perform the substitutions; simple approach - should be efficient enough
for old, new in replacements.items():
content = content.replace(old, new)
dockerfile.content = content
def create_includepkgs_file_and_cleanupscript(self, builder,
build_dir: BuildDir):
# Create a file describing which packages from the base yum repositories are included
includepkgs = builder.get_includepkgs()
includepkgs_path = build_dir.path / FLATPAK_INCLUDEPKGS_FILENAME
with open(includepkgs_path, 'w') as f:
f.write('includepkgs = ' + ','.join(includepkgs) + '\n')
# Create the cleanup script
cleanupscript = build_dir.path / FLATPAK_CLEANUPSCRIPT_FILENAME
with open(cleanupscript, 'w') as f:
f.write(builder.get_cleanup_script())
os.chmod(cleanupscript, 0o0500)
return [includepkgs_path, cleanupscript]
def run(self):
"""
run the plugin
"""
if not is_flatpak_build(self.workflow):
self.log.info('not flatpak build, skipping plugin')
return
resolve_comp_result = self.workflow.data.prebuild_results.get(
PLUGIN_RESOLVE_COMPOSES_KEY)
flatpak_util = FlatpakUtil(workflow_config=self.workflow.conf,
source_config=self.workflow.source.config,
composes=resolve_comp_result['composes'])
compose_info = flatpak_util.get_flatpak_compose_info()
source = flatpak_util.get_flatpak_source_info()
builder = FlatpakBuilder(source, None, None)
builder.precheck()
flatpak_update = functools.partial(self.update_dockerfile, builder,
compose_info)
self.workflow.build_dir.for_each_platform(flatpak_update)
create_files = functools.partial(
self.create_includepkgs_file_and_cleanupscript, builder)
self.workflow.build_dir.for_all_platforms_copy(create_files)
class AddImageContentManifestPlugin(Plugin):
"""
Add the ICM JSON file to the IMAGE_BUILD_INFO_DIR/content_manifests
directory, for the current platform. Filename will be '{IMAGE_NVR}.json'
ICM examples:
WITHOUT content_sets specified:
{
"metadata": {
"icm_version": 1,
"icm_spec": "https://link.to.icm.specification",
"image_layer_index": 3
},
"content_sets" : [],
"image_contents": [
{
"purl": "pkg:golang/github.com%2Frelease-engineering%2Fretrodep%[email protected]",
"dependencies": [{"purl": "pkg:golang/github.com%2Fop%[email protected]"}],
"sources": [{"purl": "pkg:golang/github.com%2FMasterminds%[email protected]"}]
}
]
}
WITH content_sets specified:
{
"metadata": {
"icm_version": 1,
"icm_spec": "https://link.to.icm.specification",
"image_layer_index": 2
},
"content_sets": [
"rhel-8-for-x86_64-baseos-rpms",
"rhel-8-for-x86_64-appstream-rpms"
],
"image_contents": [
{
"purl": "pkg:golang/github.com%2Frelease-engineering%2Fretrodep%[email protected]",
"dependencies": [{"purl": "pkg:golang/github.com%2Fop%[email protected]"}],
"sources": [{"purl": "pkg:golang/github.com%2FMasterminds%[email protected]"}]
}
]
}
"""
key = PLUGIN_ADD_IMAGE_CONTENT_MANIFEST
is_allowed_to_fail = False
minimal_icm: Dict[str, Any] = {
'metadata': {
'icm_version':
1,
'icm_spec':
('https://raw.githubusercontent.com/containerbuildsystem/atomic-reactor/'
'master/atomic_reactor/schemas/content_manifest.json'),
'image_layer_index':
1
},
'content_sets': [],
'image_contents': [],
}
args_from_user_params = map_to_user_params("remote_sources")
def __init__(self, workflow, destdir=IMAGE_BUILD_INFO_DIR):
"""
:param workflow: DockerBuildWorkflow instance
:param destdir: image path to carry content_manifests data dir
"""
super(AddImageContentManifestPlugin, self).__init__(workflow)
self.content_manifests_dir = os.path.join(destdir, 'content_manifests')
wf_data = self.workflow.data
remote_source_results = wf_data.plugins_results.get(
PLUGIN_RESOLVE_REMOTE_SOURCE) or []
self.remote_source_ids = [
remote_source['id'] for remote_source in remote_source_results
]
fetch_maven_results = wf_data.plugins_results.get(
PLUGIN_FETCH_MAVEN_KEY) or {}
self.pnc_artifact_ids = fetch_maven_results.get(
'pnc_artifact_ids') or []
@functools.cached_property
def icm_file_name(self):
"""Determine the name for the ICM file (name-version-release.json)."""
# parse Dockerfile for any platform, the N-V-R labels should be equal for all platforms
dockerfile = self.workflow.build_dir.any_platform.dockerfile_with_parent_env(
self.workflow.imageutil.base_image_inspect())
labels = Labels(dockerfile.labels)
_, name = labels.get_name_and_value(Labels.LABEL_TYPE_COMPONENT)
_, version = labels.get_name_and_value(Labels.LABEL_TYPE_VERSION)
_, release = labels.get_name_and_value(Labels.LABEL_TYPE_RELEASE)
return f"{name}-{version}-{release}.json"
@property
def layer_index(self) -> int:
# inspect any platform, we expect the number of layers to be equal for all platforms
inspect = self.workflow.imageutil.base_image_inspect()
if not inspect:
# Base images ('FROM koji/image-build') and 'FROM scratch' images do not have any
# base image. When building with `podman build --squash`, such images get squashed
# to only 1 layer => the layer index in this case is 0 (the first and only layer).
# This is only true for build tasks that behave like `podman build --squash`
return 0
return len(inspect[INSPECT_ROOTFS][INSPECT_ROOTFS_LAYERS])
@functools.cached_property
def _icm_base(self) -> dict:
"""Create the platform-independent skeleton of the ICM document.
:return: dict, the ICM as a Python dict
"""
icm = deepcopy(self.minimal_icm)
if self.remote_source_ids:
icm = self.cachito_session.get_image_content_manifest(
self.remote_source_ids)
if self.pnc_artifact_ids:
purl_specs = self.pnc_util.get_artifact_purl_specs(
self.pnc_artifact_ids)
for purl_spec in purl_specs:
icm['image_contents'].append({'purl': purl_spec})
icm['metadata']['image_layer_index'] = self.layer_index
return icm
def make_icm(self, platform: str) -> dict:
"""Create the complete ICM document for the specified platform."""
icm = deepcopy(self._icm_base)
content_sets = read_content_sets(self.workflow) or {}
icm['content_sets'] = content_sets.get(platform, [])
self.log.debug('Output ICM content_sets: %s', icm['content_sets'])
self.log.debug('Output ICM metadata: %s', icm['metadata'])
# Validate; `json.dumps()` converts `icm` to str. Confusingly, `read_yaml`
# *will* validate JSON
read_yaml(json.dumps(icm), 'schemas/content_manifest.json')
return icm
def _write_json_file(self, icm: dict, build_dir: BuildDir) -> None:
out_file_path = build_dir.path / self.icm_file_name
if out_file_path.exists():
raise RuntimeError(f'File {out_file_path} already exists in repo')
with open(out_file_path, 'w') as outfile:
json.dump(icm, outfile, indent=4)
self.log.debug('ICM JSON saved to: %s', out_file_path)
def _add_to_dockerfile(self, build_dir: BuildDir) -> None:
"""
Put an ADD instruction into the Dockerfile (to include the ICM file
into the container image to be built)
"""
dest_file_path = os.path.join(self.content_manifests_dir,
self.icm_file_name)
content = 'ADD {0} {1}'.format(self.icm_file_name, dest_file_path)
lines = build_dir.dockerfile.lines
# Put it before last instruction
lines.insert(-1, content + '\n')
build_dir.dockerfile.lines = lines
def inject_icm(self, build_dir: BuildDir) -> None:
"""Inject the ICM document to a build directory."""
self.log.debug(
"Injecting ICM to the build directory for the %s platform",
build_dir.platform)
icm = self.make_icm(build_dir.platform)
self._write_json_file(icm, build_dir)
self._add_to_dockerfile(build_dir)
self.log.info('added "%s" to "%s"', self.icm_file_name,
self.content_manifests_dir)
def run(self):
"""Run the plugin."""
self.workflow.build_dir.for_each_platform(self.inject_icm)
@property
def cachito_session(self):
if not self.workflow.conf.cachito:
raise RuntimeError('No Cachito configuration defined')
return get_cachito_session(self.workflow.conf)
@property
def pnc_util(self):
pnc_map = self.workflow.conf.pnc
if not pnc_map:
raise RuntimeError(
'No PNC configuration found in reactor config map')
return PNCUtil(pnc_map)
class CheckAndSetPlatformsPlugin(PreBuildPlugin):
key = PLUGIN_CHECK_AND_SET_PLATFORMS_KEY
is_allowed_to_fail = False
args_from_user_params = map_to_user_params("koji_target")
def __init__(self, workflow, koji_target=None):
"""
constructor
:param workflow: DockerBuildWorkflow instance
:param koji_target: str, Koji build target name
"""
# call parent constructor
super(CheckAndSetPlatformsPlugin, self).__init__(workflow)
self.koji_target = koji_target
def _limit_platforms(self, platforms: List[str]) -> List[str]:
"""Limit platforms in a specific range by platforms config.
:param platforms: a list of platforms to be filtered.
:type platforms: list[str]
:return: the limited platforms.
:rtype: list[str]
"""
final_platforms = set(platforms)
source_config = self.workflow.source.config
only_platforms = set(source_config.only_platforms)
excluded_platforms = set(source_config.excluded_platforms)
if only_platforms:
if only_platforms == excluded_platforms:
self.log.warning('only and not platforms are the same: %r', only_platforms)
final_platforms &= only_platforms
return list(final_platforms - excluded_platforms)
def run(self) -> Optional[List[str]]:
"""
run the plugin
"""
if self.koji_target:
koji_session = get_koji_session(self.workflow.conf)
self.log.info("Checking koji target for platforms")
event_id = koji_session.getLastEvent()['id']
target_info = koji_session.getBuildTarget(self.koji_target, event=event_id)
build_tag = target_info['build_tag']
koji_build_conf = koji_session.getBuildConfig(build_tag, event=event_id)
koji_platforms = koji_build_conf['arches']
if not koji_platforms:
self.log.info("No platforms found in koji target")
return None
platforms = koji_platforms.split()
self.log.info("Koji platforms are %s", sorted(platforms))
if is_scratch_build(self.workflow) or is_isolated_build(self.workflow):
override_platforms = set(get_orchestrator_platforms(self.workflow) or [])
if override_platforms and override_platforms != set(platforms):
sorted_platforms = sorted(override_platforms)
self.log.info("Received user specified platforms %s", sorted_platforms)
self.log.info("Using them instead of koji platforms")
# platforms from user params do not match platforms from koji target
# that almost certainly means they were overridden and should be used
return sorted_platforms
else:
platforms = get_orchestrator_platforms(self.workflow)
user_platforms = sorted(platforms) if platforms else None
self.log.info("No koji platforms. User specified platforms are %s", user_platforms)
if not platforms:
raise RuntimeError("Cannot determine platforms; no koji target or platform list")
# Filter platforms based on configured remote hosts
remote_host_pools = self.workflow.conf.remote_hosts.get("pools", {})
enabled_platforms = []
defined_but_disabled = []
def has_enabled_hosts(platform: str) -> bool:
platform_hosts = remote_host_pools.get(platform, {})
return any(host_info["enabled"] for host_info in platform_hosts.values())
for p in platforms:
if has_enabled_hosts(p):
enabled_platforms.append(p)
elif p in remote_host_pools:
defined_but_disabled.append(p)
else:
self.log.warning("No remote hosts found for platform '%s' in "
"reactor config map, skipping", p)
if defined_but_disabled:
msg = 'Platforms specified in config map, but have all remote hosts disabled' \
' {}'.format(defined_but_disabled)
raise RuntimeError(msg)
final_platforms = self._limit_platforms(enabled_platforms)
self.log.info("platforms in limits : %s", final_platforms)
if not final_platforms:
self.log.error("platforms in limits are empty")
raise RuntimeError("No platforms to build for")
self.workflow.build_dir.init_build_dirs(final_platforms, self.workflow.source)
return final_platforms
class TagAndPushPlugin(PostBuildPlugin):
"""
Use tags from workflow.data.tag_conf and push the images to workflow.conf.registry
"""
key = "tag_and_push"
is_allowed_to_fail = False
args_from_user_params = map_to_user_params("koji_target")
def __init__(self, workflow, koji_target=None):
"""
constructor
:param workflow: DockerBuildWorkflow instance
:param koji_target: str, used only for sourcecontainers
"""
# call parent constructor
super(TagAndPushPlugin, self).__init__(workflow)
self.registry = self.workflow.conf.registry
self.koji_target = koji_target
def push_with_skopeo(self, image: Dict[str, str], registry_image: ImageName, insecure: bool,
docker_push_secret: str) -> None:
cmd = ['skopeo', 'copy']
if docker_push_secret is not None:
dockercfg = Dockercfg(docker_push_secret)
cmd.append('--authfile=' + dockercfg.json_secret_path)
if insecure:
cmd.append('--dest-tls-verify=false')
if image['type'] == IMAGE_TYPE_OCI:
# ref_name is added by 'flatpak_create_oci'
# we have to be careful when changing the source container image type
# since assumption here is that source container image will always be 'docker-archive'
source_img = 'oci:{path}:{ref_name}'.format(**image)
cmd.append('--format=v2s2')
elif image['type'] == IMAGE_TYPE_DOCKER_ARCHIVE:
source_img = 'docker-archive://{path}'.format(**image)
else:
raise RuntimeError("Attempt to push unsupported image type %s with skopeo" %
image['type'])
dest_img = 'docker://' + registry_image.to_str()
cmd += [source_img, dest_img]
try:
retries.run_cmd(cmd)
except subprocess.CalledProcessError as e:
self.log.error("push failed with output:\n%s", e.output)
raise
def source_get_unique_image(self) -> ImageName:
source_result = self.workflow.data.prebuild_results[PLUGIN_FETCH_SOURCES_KEY]
koji_build_id = source_result['sources_for_koji_build_id']
kojisession = get_koji_session(self.workflow.conf)
timestamp = osbs.utils.utcnow().strftime('%Y%m%d%H%M%S')
random.seed()
current_platform = platform.processor() or 'x86_64'
tag_segments = [
self.koji_target or 'none',
str(random.randrange(10**(RAND_DIGITS - 1), 10**RAND_DIGITS)),
timestamp,
current_platform
]
tag = '-'.join(tag_segments)
get_build_meta = kojisession.getBuild(koji_build_id)
pull_specs = get_build_meta['extra']['image']['index']['pull']
source_image_spec = ImageName.parse(pull_specs[0])
source_image_spec.tag = tag
organization = self.workflow.conf.registries_organization
if organization:
source_image_spec.enclose(organization)
source_image_spec.registry = self.workflow.conf.registry['uri']
return source_image_spec
def get_repositories(self) -> Dict[str, List[str]]:
# usually repositories formed from NVR labels
# these should be used for pulling and layering
primary_repositories = []
for image in self.workflow.data.tag_conf.primary_images:
primary_repositories.append(image.to_str())
# unique unpredictable repositories
unique_repositories = []
for image in self.workflow.data.tag_conf.unique_images:
unique_repositories.append(image.to_str())
# floating repositories
# these should be used for pulling and layering
floating_repositories = []
for image in self.workflow.data.tag_conf.floating_images:
floating_repositories.append(image.to_str())
return {
"primary": primary_repositories,
"unique": unique_repositories,
"floating": floating_repositories,
}
def run(self) -> Dict[str, Union[List, Dict[str, List[str]]]]:
is_source_build = PLUGIN_FETCH_SOURCES_KEY in self.workflow.data.prebuild_results
if not is_source_build and not is_flatpak_build(self.workflow):
self.log.info('not a flatpak or source build, skipping plugin')
return {'pushed_images': [],
'repositories': self.get_repositories()}
pushed_images = []
wf_data = self.workflow.data
tag_conf = wf_data.tag_conf
images = []
if is_source_build:
source_image = self.source_get_unique_image()
plugin_results = wf_data.buildstep_result[PLUGIN_SOURCE_CONTAINER_KEY]
image = plugin_results['image_metadata']
tag_conf.add_unique_image(source_image)
images.append((image, source_image))
else:
for image_platform in get_platforms(self.workflow.data):
plugin_results = wf_data.postbuild_results[PLUGIN_FLATPAK_CREATE_OCI]
image = plugin_results[image_platform]['metadata']
registry_image = tag_conf.get_unique_images_with_platform(image_platform)[0]
images.append((image, registry_image))
insecure = self.registry.get('insecure', False)
docker_push_secret = self.registry.get('secret', None)
self.log.info("Registry %s secret %s", self.registry['uri'], docker_push_secret)
for image, registry_image in images:
max_retries = DOCKER_PUSH_MAX_RETRIES
for retry in range(max_retries + 1):
self.push_with_skopeo(image, registry_image, insecure, docker_push_secret)
if is_source_build:
manifests_dict = get_all_manifests(registry_image, self.registry['uri'],
insecure,
docker_push_secret, versions=('v2',))
try:
koji_source_manifest_response = manifests_dict['v2']
except KeyError as exc:
raise RuntimeError(
f'Unable to fetch v2 schema 2 digest for {registry_image.to_str()}'
) from exc
wf_data.koji_source_manifest = koji_source_manifest_response.json()
digests = get_manifest_digests(registry_image, self.registry['uri'],
insecure, docker_push_secret)
if not (digests.v2 or digests.oci) and (retry < max_retries):
sleep_time = DOCKER_PUSH_BACKOFF_FACTOR * (2 ** retry)
self.log.info("Retrying push because V2 schema 2 or "
"OCI manifest not found in %is", sleep_time)
time.sleep(sleep_time)
else:
break
pushed_images.append(registry_image)
self.log.info("All images were tagged and pushed")
return {'pushed_images': pushed_images,
'repositories': self.get_repositories()}
class KojiImportBase(PostBuildPlugin):
"""
Import this build to Koji
Submits a successful build to Koji using the Content Generator API,
https://docs.pagure.org/koji/content_generators
Authentication is with Kerberos unless the koji_ssl_certs
configuration parameter is given, in which case it should be a
path at which 'cert', 'ca', and 'serverca' are the certificates
for SSL authentication.
If Kerberos is used for authentication, the default principal will
be used (from the kernel keyring) unless both koji_keytab and
koji_principal are specified. The koji_keytab parameter is a
keytab name like 'type:name', and so can be used to specify a key
in a Kubernetes secret by specifying 'FILE:/path/to/key'.
Runs as an exit plugin in order to capture logs from all other
plugins.
"""
is_allowed_to_fail = False
args_from_user_params = map_to_user_params("userdata")
def __init__(self,
workflow,
blocksize=None,
poll_interval=5,
userdata=None):
"""
constructor
:param workflow: DockerBuildWorkflow instance
:param blocksize: int, blocksize to use for uploading files
:param poll_interval: int, seconds between Koji task status requests
:param userdata: dict, custom user data
"""
super(KojiImportBase, self).__init__(workflow)
self.blocksize = blocksize
self.poll_interval = poll_interval
self.osbs = get_openshift_session(self.workflow.conf,
self.workflow.namespace)
self.build_id = None
self.session = None
self.userdata = userdata
self.koji_task_id = None
koji_task_id = self.workflow.user_params.get('koji_task_id')
if koji_task_id is not None:
try:
self.koji_task_id = int(koji_task_id)
except ValueError:
# Why pass 1 to exc_info originally?
self.log.error("invalid task ID %r", koji_task_id, exc_info=1)
@cached_property
def _builds_metadatas(self) -> Dict[str, Any]:
"""Get builds metadata returned from gather_builds_metadata plugin.
:return: a mapping from platform to metadata mapping. e.g. {"x86_64": {...}}
"""
metadatas = self.workflow.data.postbuild_results.get(
PLUGIN_GATHER_BUILDS_METADATA_KEY, {})
if not metadatas:
self.log.warning(
"No build metadata is found. Check if %s plugin ran already.",
PLUGIN_GATHER_BUILDS_METADATA_KEY,
)
return metadatas
def _iter_build_metadata_outputs(
self,
platform: Optional[str] = None,
_filter: Optional[Dict[str, Any]] = None,
) -> Iterator[Tuple[str, Dict[str, Any]]]:
"""Iterate outputs from build metadata.
:param platform: iterate outputs for a specific platform. If omitted,
no platform is limited.
:type platform: str or None
:param _filter: key/value pairs to filter outputs. If omitted, no
output is filtered out.
:type _filter: dict[str, any] or None
:return: an iterator that yields a tuple in form (platform, output).
"""
for build_platform, metadata in self._builds_metadatas.items():
if platform is not None and build_platform != platform:
continue
for output in metadata["output"]:
if _filter:
if all(
output.get(key) == value
for key, value in _filter.items()):
yield build_platform, output
else:
yield build_platform, output
def get_output(self, *args):
# Must be implemented by subclasses
raise NotImplementedError
def get_buildroot(self, *args):
# Must be implemented by subclasses
raise NotImplementedError
def set_help(self, extra: Dict[str, Any]) -> None:
"""Set extra.image.help"""
result = self.workflow.data.prebuild_results.get(AddHelpPlugin.key)
if not result:
return
extra['image']['help'] = result['help_file']
def set_media_types(self, extra):
media_types = []
# Append media_types from verify images
media_results = self.workflow.data.postbuild_results.get(
PLUGIN_VERIFY_MEDIA_KEY)
if media_results:
media_types += media_results
if media_types:
extra['image']['media_types'] = sorted(set(media_types))
def set_go_metadata(self, extra):
go = self.workflow.source.config.go
if go:
self.log.user_warning(
f"Using 'go' key in {REPO_CONTAINER_CONFIG} is deprecated in favor of using "
f"Cachito integration")
self.log.debug("Setting Go metadata: %s", go)
extra['image']['go'] = go
def set_operators_metadata(self, extra):
wf_data = self.workflow.data
# upload metadata from bundle (part of image)
op_bundle_metadata = wf_data.prebuild_results.get(
PLUGIN_PIN_OPERATOR_DIGESTS_KEY)
if op_bundle_metadata:
op_related_images = op_bundle_metadata['related_images']
pullspecs = [{
'original': str(p['original']),
'new': str(p['new']),
'pinned': p['pinned'],
} for p in op_related_images['pullspecs']]
koji_operator_manifests = {
'custom_csv_modifications_applied':
op_bundle_metadata['custom_csv_modifications_applied'],
'related_images': {
'pullspecs': pullspecs,
'created_by_osbs': op_related_images['created_by_osbs'],
}
}
extra['image']['operator_manifests'] = koji_operator_manifests
# update push plugin and uploaded manifests file independently as push plugin may fail
op_push_res = wf_data.postbuild_results.get(
PLUGIN_PUSH_OPERATOR_MANIFESTS_KEY)
if op_push_res:
extra.update({"operator_manifests": {"appregistry": op_push_res}})
outputs = self._iter_build_metadata_outputs(
_filter={'filename': OPERATOR_MANIFESTS_ARCHIVE})
for _, _ in outputs:
extra['operator_manifests_archive'] = OPERATOR_MANIFESTS_ARCHIVE
operators_typeinfo = {
KOJI_BTYPE_OPERATOR_MANIFESTS: {
'archive': OPERATOR_MANIFESTS_ARCHIVE,
},
}
extra.setdefault('typeinfo', {}).update(operators_typeinfo)
return # only one worker can process operator manifests
def set_pnc_build_metadata(self, extra):
plugin_results = self.workflow.data.prebuild_results.get(
PLUGIN_FETCH_MAVEN_KEY) or {}
pnc_build_metadata = plugin_results.get('pnc_build_metadata')
if pnc_build_metadata:
extra['image']['pnc'] = pnc_build_metadata
def set_remote_sources_metadata(self, extra):
remote_source_result = self.workflow.data.prebuild_results.get(
PLUGIN_RESOLVE_REMOTE_SOURCE)
if remote_source_result:
if self.workflow.conf.allow_multiple_remote_sources:
remote_sources_image_metadata = [{
"name":
remote_source["name"],
"url":
remote_source["url"].rstrip('/download')
} for remote_source in remote_source_result]
extra["image"][
"remote_sources"] = remote_sources_image_metadata
remote_sources_typeinfo_metadata = [{
"name":
remote_source["name"],
"url":
remote_source["url"].rstrip('/download'),
"archives": [
remote_source["remote_source_json"]["filename"],
remote_source["remote_source_tarball"]["filename"],
],
} for remote_source in remote_source_result]
else:
extra["image"]["remote_source_url"] = remote_source_result[0][
"url"]
remote_sources_typeinfo_metadata = {
"remote_source_url": remote_source_result[0]["url"]
}
remote_source_typeinfo = {
KOJI_BTYPE_REMOTE_SOURCES: remote_sources_typeinfo_metadata,
}
extra.setdefault("typeinfo", {}).update(remote_source_typeinfo)
def set_remote_source_file_metadata(self, extra):
maven_url_sources_metadata_results = self.workflow.data.postbuild_results.get(
PLUGIN_MAVEN_URL_SOURCES_METADATA_KEY) or {}
fetch_maven_results = self.workflow.data.prebuild_results.get(
PLUGIN_FETCH_MAVEN_KEY) or {}
remote_source_files = maven_url_sources_metadata_results.get(
'remote_source_files')
no_source_artifacts = fetch_maven_results.get('no_source')
if remote_source_files or no_source_artifacts:
r_s_f_typeinfo = {
KOJI_BTYPE_REMOTE_SOURCE_FILE: {},
}
if remote_source_files:
r_s_f_typeinfo[KOJI_BTYPE_REMOTE_SOURCE_FILE][
'remote_source_files'] = []
for remote_source_file in remote_source_files:
r_s_f_extra = remote_source_file['metadata']['extra']
r_s_f_typeinfo[KOJI_BTYPE_REMOTE_SOURCE_FILE][
'remote_source_files'].append({
r_s_f_extra['source-url']:
r_s_f_extra['artifacts']
})
if no_source_artifacts:
r_s_f_typeinfo[KOJI_BTYPE_REMOTE_SOURCE_FILE][
'no_source'] = no_source_artifacts
extra.setdefault('typeinfo', {}).update(r_s_f_typeinfo)
def set_group_manifest_info(self, extra):
version_release = None
primary_images = get_primary_images(self.workflow)
if primary_images:
version_release = primary_images[0].tag
if is_scratch_build(self.workflow):
tags = [image.tag for image in self.workflow.data.tag_conf.images]
version_release = tags[0]
else:
assert version_release is not None, 'Unable to find version-release image'
tags = [image.tag for image in primary_images]
floating_tags = [
image.tag for image in get_floating_images(self.workflow)
]
unique_images = get_unique_images(self.workflow)
unique_tags = [image.tag for image in unique_images]
manifest_data = self.workflow.data.postbuild_results.get(
PLUGIN_GROUP_MANIFESTS_KEY, {})
if manifest_data and is_manifest_list(manifest_data.get("media_type")):
manifest_digest = manifest_data["manifest_digest"]
digest = manifest_digest.default
build_image = unique_images[0]
repo = ImageName.parse(build_image).to_str(registry=False,
tag=False)
# group_manifests added the registry, so this should be valid
registry_uri = self.workflow.conf.registry['uri']
digest_version = get_manifest_media_version(manifest_digest)
media_type = get_manifest_media_type(digest_version)
extra['image']['index'] = {
'tags':
tags,
'floating_tags':
floating_tags,
'unique_tags':
unique_tags,
'pull': [
f'{registry_uri}/{repo}@{digest}',
f'{registry_uri}/{repo}:{version_release}',
],
'digests': {
media_type: digest
},
}
# group_manifests returns None if didn't run, {} if group=False
else:
platform = "x86_64"
_, instance = next(
self._iter_build_metadata_outputs(platform,
{"type": "docker-image"}),
(None, None),
)
if instance:
# koji_upload, running in the worker, doesn't have the full tags
# so set them here
instance['extra']['docker']['tags'] = tags
instance['extra']['docker']['floating_tags'] = floating_tags
instance['extra']['docker']['unique_tags'] = unique_tags
repositories = []
for pullspec in instance['extra']['docker']['repositories']:
if '@' not in pullspec:
image = ImageName.parse(pullspec)
image.tag = version_release
pullspec = image.to_str()
repositories.append(pullspec)
instance['extra']['docker']['repositories'] = repositories
self.log.debug("reset tags to so that docker is %s",
instance['extra']['docker'])
def _update_extra(self, extra):
# Must be implemented by subclasses
"""
:param extra: A dictionary, representing koji's 'build.extra' metadata
"""
raise NotImplementedError
def _update_build(self, build):
# Must be implemented by subclasses
raise NotImplementedError
def _get_build_extra(self) -> Dict[str, Any]:
extra = {
'image': {},
'osbs_build': {
'subtypes': []
},
'submitter': self.session.getLoggedInUser()['name'],
}
if self.koji_task_id is not None:
extra['container_koji_task_id'] = self.koji_task_id
self.log.info("build configuration created by Koji Task ID %s",
self.koji_task_id)
self._update_extra(extra)
self.set_media_types(extra)
return extra
def get_build(self):
start_time = int(atomic_reactor_start_time)
koji_task_owner = get_koji_task_owner(self.session,
self.koji_task_id).get('name')
build = {
'start_time': start_time,
'end_time': int(time.time()),
'extra': self._get_build_extra(),
'owner': koji_task_owner,
}
self._update_build(build)
return build
def combine_metadata_fragments(self) -> Dict[str, Any]:
"""Construct the CG metadata and collect the output files for upload later."""
def add_buildroot_id(output: Output, buildroot_id: str) -> Output:
output.metadata.update({'buildroot_id': buildroot_id})
return Output(filename=output.filename, metadata=output.metadata)
def add_log_type(output: Output) -> Output:
output.metadata.update({'type': 'log', 'arch': 'noarch'})
return Output(filename=output.filename, metadata=output.metadata)
build = self.get_build()
buildroot = self.get_buildroot()
buildroot_id = buildroot[0]['id']
# Collect the output files, which will be uploaded later.
koji_upload_files = self.workflow.data.koji_upload_files
output: List[Dict[str, Any]] # List of metadatas
# The corresponding output file, only has one for source build
output_file: Optional[Output]
output, output_file = self.get_output(buildroot_id)
if output_file:
koji_upload_files.append({
"local_filename": output_file.filename,
"dest_filename": output[0]["filename"],
})
# Collect log files
osbs_logs = OSBSLogs(self.log, get_platforms(self.workflow.data))
log_files_output = [
add_log_type(add_buildroot_id(md, buildroot_id)) for md in
osbs_logs.get_log_files(self.osbs, self.workflow.pipeline_run_name)
]
for log_file_output in log_files_output:
output.append(log_file_output.metadata)
koji_upload_files.append({
"local_filename":
log_file_output.filename,
"dest_filename":
log_file_output.metadata["filename"],
})
remote_source_file_outputs, kojifile_components = get_maven_metadata(
self.workflow.data)
# add maven components alongside RPM components
for metadata in output:
if metadata['type'] == 'docker-image':
metadata['components'] += kojifile_components
# add remote sources tarballs and remote sources json files to output
for remote_source_output in [
*get_source_tarballs_output(self.workflow),
*get_remote_sources_json_output(self.workflow)
]:
add_custom_type(remote_source_output, KOJI_BTYPE_REMOTE_SOURCES)
remote_source = add_buildroot_id(remote_source_output,
buildroot_id)
output.append(remote_source.metadata)
koji_upload_files.append({
"local_filename":
remote_source.filename,
"dest_filename":
remote_source.metadata["filename"],
})
for remote_source_file_output in remote_source_file_outputs:
remote_source_file = add_buildroot_id(remote_source_file_output,
buildroot_id)
output.append(remote_source_file.metadata)
koji_upload_files.append({
"local_filename":
remote_source_file_output.filename,
"dest_filename":
remote_source_file_output.metadata["filename"],
})
koji_metadata = {
'metadata_version': 0,
'build': build,
'buildroots': buildroot,
'output': output,
}
return koji_metadata
def upload_file(self, local_filename: str, dest_filename: str,
serverdir: str) -> str:
"""
Upload a file to koji
:return: str, pathname on server
"""
self.log.debug("uploading %r to %r as %r", local_filename, serverdir,
dest_filename)
kwargs = {}
if self.blocksize is not None:
kwargs['blocksize'] = self.blocksize
self.log.debug("using blocksize %d", self.blocksize)
callback = KojiUploadLogger(self.log).callback
self.session.uploadWrapper(local_filename,
serverdir,
name=dest_filename,
callback=callback,
**kwargs)
# In case dest_filename includes path. uploadWrapper can handle this by itself.
path = os.path.join(serverdir, os.path.basename(dest_filename))
self.log.debug("uploaded %r", path)
return path
def upload_scratch_metadata(self, koji_metadata, koji_upload_dir):
metadata_file = NamedTemporaryFile(prefix="metadata",
suffix=".json",
mode='wb',
delete=False)
metadata_file.write(
json.dumps(koji_metadata, indent=2).encode('utf-8'))
metadata_file.close()
local_filename = metadata_file.name
try:
uploaded_filename = self.upload_file(local_filename,
"metadata.json",
koji_upload_dir)
log = logging.LoggerAdapter(self.log, {'arch': METADATA_TAG})
log.info(uploaded_filename)
finally:
os.unlink(local_filename)
def get_server_dir(self):
return koji_cli.lib.unique_path('koji-upload')
def _upload_output_files(self, server_dir: str) -> None:
"""Helper method to upload collected output files."""
for upload_info in self.workflow.data.koji_upload_files:
self.upload_file(upload_info["local_filename"],
upload_info["dest_filename"], server_dir)
def run(self):
"""
Run the plugin.
"""
# get the session and token information in case we need to refund a failed build
self.session = get_koji_session(self.workflow.conf)
server_dir = self.get_server_dir()
koji_metadata = self.combine_metadata_fragments()
if is_scratch_build(self.workflow):
self.upload_scratch_metadata(koji_metadata, server_dir)
return
# for all builds which have koji task
if self.koji_task_id:
task_info = self.session.getTaskInfo(self.koji_task_id)
task_state = koji.TASK_STATES[task_info['state']]
if task_state != 'OPEN':
self.log.error(
"Koji task is not in Open state, but in %s, not importing build",
task_state)
return
self._upload_output_files(server_dir)
build_token = self.workflow.data.reserved_token
build_id = self.workflow.data.reserved_build_id
if build_id is not None and build_token is not None:
koji_metadata['build']['build_id'] = build_id
try:
if build_token:
build_info = self.session.CGImport(koji_metadata,
server_dir,
token=build_token)
else:
build_info = self.session.CGImport(koji_metadata, server_dir)
except Exception:
self.log.debug("metadata: %r", koji_metadata)
raise
# Older versions of CGImport do not return a value.
build_id = build_info.get("id") if build_info else None
self.log.debug("Build information: %s",
json.dumps(build_info, sort_keys=True, indent=4))
return build_id
class InjectYumReposPlugin(Plugin):
key = "inject_yum_repos"
is_allowed_to_fail = False
args_from_user_params = map_to_user_params("target:koji_target", )
def __init__(self, workflow, target=None, inject_proxy=None):
"""
constructor
:param workflow: DockerBuildWorkflow instance
:param target: string, koji target to use as a source
:param inject_proxy: set proxy server for this repo
"""
super().__init__(workflow)
self.target = target
self.repourls = {}
self.inject_proxy = inject_proxy
self.yum_repos = defaultdict(list)
self.allowed_domains = self.workflow.conf.yum_repo_allowed_domains
self.include_koji_repo = False
self._builder_ca_bundle = None
self._ca_bundle_pem = None
self.platforms = get_platforms(workflow.data)
resolve_comp_result = self.workflow.data.plugins_results.get(
PLUGIN_RESOLVE_COMPOSES_KEY)
self.include_koji_repo = resolve_comp_result['include_koji_repo']
self.repourls = resolve_comp_result['yum_repourls']
def validate_yum_repo_files_url(self):
if not self.allowed_domains:
return
errors = []
checked = set()
for platform in self.platforms:
for repourl in self.repourls.get(platform, []):
if repourl in checked:
continue
repo_domain = urlparse(repourl).netloc
checked.add(repourl)
if repo_domain not in self.allowed_domains:
errors.append(
'Yum repo URL {} is not in list of allowed domains: {}'
.format(repourl, self.allowed_domains))
if errors:
raise ValueError(
'Errors found while checking yum repo urls: \n{}'.format(
'\n'.join(errors)))
def _final_user_line(self):
user = self._find_final_user()
if user:
return user
if not self.workflow.data.dockerfile_images.base_from_scratch:
# Inspect any platform: the User should be equal for all platforms
inspect = self.workflow.imageutil.base_image_inspect()
user = inspect.get(INSPECT_CONFIG, {}).get('User')
if user:
return f'USER {user}'
return ''
def _find_final_user(self):
"""Find the user in USER instruction in the last build stage"""
dockerfile = self.workflow.build_dir.any_platform.dockerfile_with_parent_env(
self.workflow.imageutil.base_image_inspect())
for insndesc in reversed(dockerfile.structure):
if insndesc['instruction'] == 'USER':
return insndesc['content'] # we will reuse the line verbatim
if insndesc['instruction'] == 'FROM':
break # no USER specified in final stage
def _cleanup_lines(self, platform):
lines = [
"RUN rm -f " + " ".join((f"'{repo.dst_filename}'"
for repo in self.yum_repos[platform]))
]
if self._builder_ca_bundle:
lines.append(f'RUN rm -f /tmp/{self._ca_bundle_pem}')
final_user_line = self._final_user_line()
if final_user_line:
lines.insert(0, "USER root")
lines.append(final_user_line)
return lines
def add_koji_repo(self):
xmlrpc = get_koji_session(self.workflow.conf)
pathinfo = self.workflow.conf.koji_path_info
proxy = self.workflow.conf.yum_proxy
if not self.target:
self.log.info('no target provided, not adding koji repo')
return
target_info = xmlrpc.getBuildTarget(self.target)
if target_info is None:
self.log.error("provided target '%s' doesn't exist", self.target)
raise RuntimeError("Provided target '%s' doesn't exist!" %
self.target)
tag_info = xmlrpc.getTag(target_info['build_tag_name'])
if not tag_info or 'name' not in tag_info:
self.log.warning("No tag info was retrieved")
return
repo_info = xmlrpc.getRepo(tag_info['id'])
if not repo_info or 'id' not in repo_info:
self.log.warning("No repo info was retrieved")
return
# to use urljoin, we would have to append '/', so let's append everything
baseurl = pathinfo.repo(repo_info['id'],
tag_info['name']) + "/$basearch"
self.log.info("baseurl = '%s'", baseurl)
repo = {
'name': 'atomic-reactor-koji-plugin-%s' % self.target,
'baseurl': baseurl,
'enabled': 1,
'gpgcheck': 0,
}
# yum doesn't accept a certificate path in sslcacert - it requires a db with added cert
# dnf ignores that option completely
# we have to fall back to sslverify=0 everytime we get https repo from brew so we'll surely
# be able to pull from it
if baseurl.startswith("https://"):
self.log.info("Ignoring certificates in the repo")
repo['sslverify'] = 0
if proxy:
self.log.info("Setting yum proxy to %s", proxy)
repo['proxy'] = proxy
yum_repo = YumRepo(os.path.join(YUM_REPOS_DIR, self.target))
path = yum_repo.dst_filename
self.log.info("yum repo of koji target: '%s'", path)
yum_repo.content = render_yum_repo(repo, escape_dollars=False)
for platform in self.platforms:
self.yum_repos[platform].append(yum_repo)
def _inject_into_repo_files(self, build_dir: BuildDir):
"""Inject repo files into a relative directory inside the build context"""
host_repos_path = build_dir.path / RELATIVE_REPOS_PATH
self.log.info("creating directory for yum repos: %s", host_repos_path)
os.mkdir(host_repos_path)
allow_repo_dir_in_dockerignore(build_dir.path)
for repo in self.yum_repos[build_dir.platform]:
# Update every repo accordingly in a repofile
# input_buf ---- updated ----> updated_buf
with StringIO(repo.content.decode()) as input_buf, StringIO(
) as updated_buf:
for line in input_buf:
updated_buf.write(line)
# Apply sslcacert to every repo in a repofile
if line.lstrip().startswith(
'[') and self._builder_ca_bundle:
updated_buf.write(
f'sslcacert=/tmp/{self._ca_bundle_pem}\n')
yum_repo = YumRepo(repourl=repo.dst_filename,
content=updated_buf.getvalue(),
dst_repos_dir=host_repos_path,
add_hash=False)
yum_repo.write_content()
def _inject_into_dockerfile(self, build_dir: BuildDir):
build_dir.dockerfile.add_lines("ADD %s* %s" %
(RELATIVE_REPOS_PATH, YUM_REPOS_DIR),
all_stages=True,
at_start=True,
skip_scratch=True)
if self._builder_ca_bundle:
shutil.copyfile(self._builder_ca_bundle,
build_dir.path / self._ca_bundle_pem)
build_dir.dockerfile.add_lines(
f'ADD {self._ca_bundle_pem} /tmp/{self._ca_bundle_pem}',
all_stages=True,
at_start=True,
skip_scratch=True)
if not self.workflow.data.dockerfile_images.base_from_scratch:
build_dir.dockerfile.add_lines(
*self._cleanup_lines(build_dir.platform))
def run(self):
"""
run the plugin
"""
if not self.workflow.data.dockerfile_images:
self.log.info(
"Skipping plugin, from scratch stage(s) can't add repos")
return
if self.include_koji_repo:
self.add_koji_repo()
else:
self.log.info(
"'include_koji_repo parameter is set to '%s', not including koji repo",
self.include_koji_repo)
if self.repourls and not is_scratch_build(self.workflow):
self.validate_yum_repo_files_url()
fetched_yum_repos = {}
for platform in self.platforms:
for repourl in self.repourls.get(platform, []):
if repourl in fetched_yum_repos:
yum_repo = fetched_yum_repos[repourl]
self.yum_repos[platform].append(yum_repo)
continue
yum_repo = YumRepo(repourl)
self.log.info("fetching yum repo from '%s'", yum_repo.repourl)
try:
yum_repo.fetch()
except Exception as e:
msg = "Failed to fetch yum repo {repo}: {exc}".format(
repo=yum_repo.repourl, exc=e)
raise RuntimeError(msg) from e
else:
self.log.info("fetched yum repo from '%s'",
yum_repo.repourl)
if self.inject_proxy:
if yum_repo.is_valid():
yum_repo.set_proxy_for_all_repos(self.inject_proxy)
self.log.debug("saving yum repo '%s', length %d",
yum_repo.dst_filename, len(yum_repo.content))
self.yum_repos[platform].append(yum_repo)
fetched_yum_repos[repourl] = yum_repo
if not self.yum_repos:
return
self._builder_ca_bundle = self.workflow.conf.builder_ca_bundle
if self._builder_ca_bundle:
self._ca_bundle_pem = os.path.basename(self._builder_ca_bundle)
self.workflow.build_dir.for_each_platform(self._inject_into_repo_files)
self.workflow.build_dir.for_each_platform(self._inject_into_dockerfile)
for platform in self.platforms:
for repo in self.yum_repos[platform]:
self.log.info("injected yum repo: %s for '%s' platform",
repo.dst_filename, platform)
class ResolveComposesPlugin(PreBuildPlugin):
"""Request a new, or use existing, ODCS compose
This plugin will read the configuration in git repository
and request ODCS to create a corresponding yum repository.
"""
key = PLUGIN_RESOLVE_COMPOSES_KEY
is_allowed_to_fail = False
args_from_user_params = util.map_to_user_params(
"koji_target",
"signing_intent",
"compose_ids",
"repourls:yum_repourls",
)
def __init__(self, workflow, koji_target=None, signing_intent=None, compose_ids=tuple(),
repourls=None, minimum_time_to_expire=MINIMUM_TIME_TO_EXPIRE):
"""
:param workflow: DockerBuildWorkflow instance
:param koji_target: str, koji target contains build tag to be used
when requesting compose from "tag"
:param signing_intent: override the signing intent from git repo configuration
:param compose_ids: use the given compose_ids instead of requesting a new one
:param repourls: list of str, URLs to the repo files
:param minimum_time_to_expire: int, used in deciding when to extend compose's time
to expire in seconds
"""
super(ResolveComposesPlugin, self).__init__(workflow)
if signing_intent and compose_ids:
raise ValueError('signing_intent and compose_ids cannot be used at the same time')
self.signing_intent = signing_intent
self.compose_ids = compose_ids
self.koji_target = koji_target
self.minimum_time_to_expire = minimum_time_to_expire
self._koji_session = None
self._odcs_client = None
self.odcs_config = None
self.compose_config = None
self.composes_info = None
self._parent_signing_intent = None
self.repourls = repourls or []
self.has_complete_repos = len(self.repourls) > 0
self.plugin_result = self.workflow.data.prebuild_results.get(PLUGIN_KOJI_PARENT_KEY)
self.all_compose_ids = list(self.compose_ids)
self.new_compose_ids = []
self.parent_compose_ids = []
self.include_koji_repo = False
self.yum_repourls = defaultdict(list)
self.architectures = get_platforms(self.workflow.data)
def run(self):
if self.allow_inheritance():
self.adjust_for_inherit()
self.workflow.data.all_yum_repourls = self.repourls
try:
self.read_configs()
except SkipResolveComposesPlugin as abort_exc:
self.log.info('Aborting plugin execution: %s', abort_exc)
for arch in self.architectures:
self.yum_repourls[arch].extend(self.repourls)
return self.make_result()
self.adjust_compose_config()
self.request_compose_if_needed()
try:
self.wait_for_composes()
except WaitComposeToFinishTimeout as e:
self.log.info(str(e))
for compose_id in self.new_compose_ids:
if self.odcs_client.get_compose_status(compose_id) in ['wait', 'generating']:
self.log.info('Canceling the compose %s', compose_id)
self.odcs_client.cancel_compose(compose_id)
else:
self.log.info('The compose %s is not in progress, skip canceling', compose_id)
raise
self.resolve_signing_intent()
self.forward_composes()
return self.make_result()
def allow_inheritance(self):
"""Returns boolean if composes can be inherited"""
if not self.workflow.source.config.inherit:
return False
self.log.info("Inheritance requested in container.yaml file")
if is_scratch_build(self.workflow) or is_isolated_build(self.workflow):
msg = ("'inherit: true' in the compose section of container.yaml "
"is not allowed for scratch or isolated builds. "
"Skipping inheritance.")
self.log.warning(msg)
self.log.user_warning(message=msg)
return False
return True
def adjust_for_inherit(self):
if self.workflow.data.dockerfile_images.base_from_scratch:
self.log.debug('This is a base image based on scratch. '
'Skipping adjusting composes for inheritance.')
return
if not self.plugin_result:
return
build_info = self.plugin_result[BASE_IMAGE_KOJI_BUILD]
parent_repourls = []
try:
self.parent_compose_ids = build_info['extra']['image']['odcs']['compose_ids']
except (KeyError, TypeError):
self.log.debug('Parent koji build, %s(%s), does not define compose_ids.'
'Cannot add compose_ids for inheritance from parent.',
build_info['nvr'], build_info['id'])
try:
parent_repourls = build_info['extra']['image']['yum_repourls']
except (KeyError, TypeError):
self.log.debug('Parent koji build, %s(%s), does not define yum_repourls. '
'Cannot add yum_repourls for inheritance from parent.',
build_info['nvr'], build_info['id'])
all_compose_ids = set(self.compose_ids)
original_compose_ids = deepcopy(all_compose_ids)
all_compose_ids.update(self.parent_compose_ids)
self.all_compose_ids = list(all_compose_ids)
for compose_id in all_compose_ids:
if compose_id not in original_compose_ids:
self.log.info('Inheriting compose id %s', compose_id)
all_yum_repos = set(self.repourls)
original_yum_repos = deepcopy(all_yum_repos)
all_yum_repos.update(parent_repourls)
self.repourls = list(all_yum_repos)
for repo in all_yum_repos:
if repo not in original_yum_repos:
self.log.info('Inheriting yum repo %s', repo)
if len(parent_repourls) > 0:
self.has_complete_repos = True
def read_configs(self):
self.odcs_config = self.workflow.conf.odcs_config
if not self.odcs_config:
raise SkipResolveComposesPlugin('ODCS config not found')
data = self.workflow.source.config.compose
if not data and not self.all_compose_ids:
raise SkipResolveComposesPlugin('"compose" config not set and compose_ids not given')
pulp_data = util.read_content_sets(self.workflow) or {}
platforms = get_platforms(self.workflow.data)
if platforms:
platforms = sorted(platforms) # sorted to keep predictable for tests
koji_tag = None
if self.koji_target:
target_info = self.koji_session.getBuildTarget(self.koji_target, strict=True)
koji_tag = target_info['build_tag_name']
self.compose_config = ComposeConfig(data, pulp_data, self.odcs_config, koji_tag=koji_tag,
arches=platforms)
def adjust_compose_config(self):
if self.signing_intent:
self.compose_config.set_signing_intent(self.signing_intent)
self.adjust_signing_intent_from_parent()
def adjust_signing_intent_from_parent(self):
if self.workflow.data.dockerfile_images.base_from_scratch:
self.log.debug('This is a base image based on scratch. '
'Signing intent will not be adjusted for it.')
return
if not self.plugin_result:
self.log.debug("%s plugin didn't run, signing intent will not be adjusted",
PLUGIN_KOJI_PARENT_KEY)
return
build_info = self.plugin_result[BASE_IMAGE_KOJI_BUILD]
try:
parent_signing_intent_name = build_info['extra']['image']['odcs']['signing_intent']
except (KeyError, TypeError):
self.log.debug('Parent koji build, %s(%s), does not define signing_intent. '
'Cannot adjust for current build.',
build_info['nvr'], build_info['id'])
return
self._parent_signing_intent = (self.odcs_config
.get_signing_intent_by_name(parent_signing_intent_name))
current_signing_intent = self.compose_config.signing_intent
# Calculate the least restrictive signing intent
new_signing_intent = min(self._parent_signing_intent, current_signing_intent,
key=lambda x: x['restrictiveness'])
if new_signing_intent != current_signing_intent:
self.log.info('Signing intent downgraded to "%s" to match Koji parent build',
new_signing_intent['name'])
self.compose_config.set_signing_intent(new_signing_intent['name'])
def request_compose_if_needed(self):
if self.compose_ids:
self.log.debug('ODCS compose not requested, using given compose IDs')
return
if not self.workflow.source.config.compose:
self.log.debug('ODCS compose not provided, using parents compose IDs')
return
self.compose_config.validate_for_request()
for compose_request in self.compose_config.render_requests():
compose_info = self.odcs_client.start_compose(**compose_request)
self.new_compose_ids.append(compose_info['id'])
self.all_compose_ids.extend(self.new_compose_ids)
def wait_for_composes(self):
self.log.debug('Waiting for ODCS composes to be available: %s', self.all_compose_ids)
self.composes_info = []
for compose_id in self.all_compose_ids:
compose_info = self.odcs_client.wait_for_compose(compose_id)
if self._needs_renewal(compose_info):
sigkeys = compose_info.get('sigkeys', '').split()
updated_signing_intent = self.odcs_config.get_signing_intent_by_keys(sigkeys)
if set(sigkeys) != set(updated_signing_intent['keys']):
self.log.info('Updating signing keys in "%s" from "%s", to "%s" in compose '
'"%s" due to sigkeys deprecation',
updated_signing_intent['name'],
sigkeys,
updated_signing_intent['keys'],
compose_info['id']
)
sigkeys = updated_signing_intent['keys']
compose_info = self.odcs_client.renew_compose(compose_id, sigkeys)
compose_id = compose_info['id']
self.new_compose_ids.append(compose_id)
compose_info = self.odcs_client.wait_for_compose(compose_id)
self.composes_info.append(compose_info)
# A module compose is not standalone - it depends on packages from the
# virtual platform module - if no extra repourls or other composes are
# provided, we'll need packages from the target build tag using the
# 'koji' plugin.
# We assume other types of composes might provide all the packages needed -
# though we don't really know that for sure - a compose with packages
# listed might list all the packages that are needed, or might also require
# packages from some other source.
if compose_info['source_type'] != 2: # PungiSourceType.MODULE
self.has_complete_repos = True
self.all_compose_ids = [item['id'] for item in self.composes_info]
def _needs_renewal(self, compose_info):
if compose_info['state_name'] == 'removed':
return True
time_to_expire = datetime.strptime(compose_info['time_to_expire'],
ODCS_DATETIME_FORMAT)
now = datetime.utcnow()
seconds_left = (time_to_expire - now).total_seconds()
return seconds_left <= self.minimum_time_to_expire
def resolve_signing_intent(self):
"""Determine the correct signing intent
Regardless of what was requested, or provided as signing_intent plugin parameter,
consult sigkeys of the actual composes used to guarantee information accuracy.
"""
all_signing_intents = [
self.odcs_config.get_signing_intent_by_keys(compose_info.get('sigkeys', []))
for compose_info in self.composes_info
]
# Because composes_info may contain composes that were passed as
# plugin parameters, add the parent signing intent to avoid the
# overall signing intent from surpassing parent's.
if self._parent_signing_intent:
all_signing_intents.append(self._parent_signing_intent)
# Calculate the least restrictive signing intent
signing_intent = min(all_signing_intents, key=lambda x: x['restrictiveness'])
self.log.info('Signing intent for build is %s', signing_intent['name'])
self.compose_config.set_signing_intent(signing_intent['name'])
def forward_composes(self):
for compose_info in self.composes_info:
result_repofile = compose_info['result_repofile']
try:
arches = compose_info['arches']
except KeyError:
self.yum_repourls['noarch'].append(result_repofile)
else:
for arch in arches.split():
self.yum_repourls[arch].append(result_repofile)
# we should almost never have a None entry from composes,
# but we can have yum_repos added, so if we do, we need to merge
# it with all other repos.
self.yum_repourls['noarch'].extend(self.repourls)
if 'noarch' in self.yum_repourls:
noarch_repos = self.yum_repourls.pop('noarch')
for arch in self.yum_repourls:
self.yum_repourls[arch].extend(noarch_repos)
# If we don't think the set of packages available from the user-supplied repourls,
# inherited repourls, and composed repositories is complete, set the 'include_koji_repo'
# kwarg so that the so that the 'yum_repourls' kwarg that we just set doesn't
# result in the 'koji' plugin being omitted.
if not self.has_complete_repos:
self.include_koji_repo = True
def make_result(self):
signing_intent = None
signing_intent_overridden = False
if self.compose_config:
signing_intent = self.compose_config.signing_intent['name']
signing_intent_overridden = self.compose_config.has_signing_intent_changed()
result = {
'composes': self.composes_info,
'yum_repourls': self.yum_repourls,
'include_koji_repo': self.include_koji_repo,
'signing_intent': signing_intent,
'signing_intent_overridden': signing_intent_overridden,
}
self.log.debug('plugin result: %s', result)
return result
@property
def odcs_client(self):
if not self._odcs_client:
self._odcs_client = get_odcs_session(self.workflow.conf)
return self._odcs_client
@property
def koji_session(self):
if not self._koji_session:
self._koji_session = get_koji_session(self.workflow.conf)
return self._koji_session
