class KojiTagBuildPlugin(Plugin): """ Tag build in koji Authentication is with Kerberos unless the koji_ssl_certs configuration parameter is given, in which case it should be a path at which 'cert', 'ca', and 'serverca' are the certificates for SSL authentication. If Kerberos is used for authentication, the default principal will be used (from the kernel keyring) unless both koji_keytab and koji_principal are specified. The koji_keytab parameter is a keytab name like 'type:name', and so can be used to specify a key in a Kubernetes secret by specifying 'FILE:/path/to/key'. """ key = PLUGIN_KOJI_TAG_BUILD_KEY is_allowed_to_fail = False args_from_user_params = map_to_user_params("target:koji_target") def __init__(self, workflow, target=None, poll_interval=5): """ constructor :param workflow: DockerBuildWorkflow instance :param target: str, koji target :param poll_interval: int, seconds between Koji task status requests """ super(KojiTagBuildPlugin, self).__init__(workflow) self.target = target self.poll_interval = poll_interval def run(self): """ Run the plugin. """ if is_scratch_build(self.workflow): self.log.info('scratch build, skipping plugin') return if not self.target: self.log.info('no koji target provided, skipping plugin') return build_id = self.workflow.data.plugins_results.get(KojiImportPlugin.key) if not build_id: self.log.info('No koji build from %s', KojiImportPlugin.key) return session = get_koji_session(self.workflow.conf) build_tag = tag_koji_build(session, build_id, self.target, poll_interval=self.poll_interval) return build_tag
def args_from_user_params(user_params: dict) -> dict: args = {} build_kwargs_from_user_params = map_to_user_params( "component", "git_branch", "git_ref", "git_uri", "koji_task_id", "filesystem_koji_task_id", "scratch", "target:koji_target", "user", "yum_repourls", "koji_parent_build", "isolated", "reactor_config_map", "reactor_config_override", "git_commit_depth", "flatpak", "operator_csv_modifications_url", ) if build_kwargs := build_kwargs_from_user_params(user_params): args["build_kwargs"] = build_kwargs
class AddFilesystemPlugin(Plugin): """ Creates a base image by using a filesystem generated through Koji Submits an image build task to Koji based on image build configuration file to create the filesystem to be used in creating the base image: https://docs.pagure.org/koji/image_build/ Once image build task is complete the tarball is downloaded. The existing FROM instruction value is replaced with a FROM scratch and ADD <filesystem> <to_image> for Dockerfile of each platform. The "FROM" instruction should be in the following format: FROM koji/image-build[:image-build-conf] Where image-build-conf is the file name of the image build configuration to be used. If omitted, image-build.conf is used. This file is expected to be in the same folder as the Dockerfile. Runs as a pre build plugin in order to properly adjust base image. """ key = PLUGIN_ADD_FILESYSTEM_KEY is_allowed_to_fail = False DEFAULT_IMAGE_BUILD_CONF = dedent('''\ [image-build] name = default-name arches = x86_64 format = docker disk_size = 10 target = {target} install_tree = {install_tree} repo = {repo} ksurl = {ksurl} kickstart = kickstart.ks [factory-parameters] create_docker_metadata = False ''') args_from_user_params = map_to_user_params( "repos:yum_repourls", "koji_target", ) def __init__(self, workflow, poll_interval=5, blocksize=DEFAULT_DOWNLOAD_BLOCK_SIZE, repos=None, koji_target=None): """ :param workflow: DockerBuildWorkflow instance :param poll_interval: int, seconds between polling Koji while waiting for task completion :param blocksize: int, chunk size for downloading files from koji :param repos: list<str>: list of yum repo URLs to be used during base filesystem creation. First value will also be used as install_tree. Only baseurl value is used from each repo file. :param koji_target: str, koji target name """ # call parent constructor super(AddFilesystemPlugin, self).__init__(workflow) self.poll_interval = poll_interval self.blocksize = blocksize self.repos = repos or [] self.architectures = get_platforms(self.workflow.data) self.scratch = util.is_scratch_build(self.workflow) self.koji_target = koji_target self.session = None def is_image_build_type(self, base_image): return base_image.strip().lower() == 'koji/image-build' def extract_base_url(self, repo_url): yum_repo = YumRepo(repo_url) yum_repo.fetch() if not yum_repo.is_valid(): return [] repo = yum_repo.config return [ repo.get(section, 'baseurl') for section in repo.sections() if repo.has_option(section, 'baseurl') ] def get_default_image_build_conf(self): """Create a default image build config :rtype: ConfigParser :return: Initialized config with defaults """ target = self.koji_target vcs_info = self.workflow.source.get_vcs_info() ksurl = '{}#{}'.format(vcs_info.vcs_url, vcs_info.vcs_ref) base_urls = [] for repo in self.repos: for url in self.extract_base_url(repo): # Imagefactory only supports $arch variable. url = url.replace('$basearch', '$arch') base_urls.append(url) install_tree = base_urls[0] if base_urls else '' repo = ','.join(base_urls) kwargs = { 'target': target, 'ksurl': ksurl, 'install_tree': install_tree, 'repo': repo, } config_fp = StringIO(self.DEFAULT_IMAGE_BUILD_CONF.format(**kwargs)) config = ConfigParser() config.read_file(config_fp) self.update_config_from_dockerfile(config) return config def update_config_from_dockerfile(self, config): """Updates build config with values from the Dockerfile Updates: * set "name" from LABEL com.redhat.component (if exists) * set "version" from LABEL version (if exists) :param config: ConfigParser object """ labels = Labels(self.workflow.build_dir.any_platform.dockerfile.labels) for config_key, label in ( ('name', Labels.LABEL_TYPE_COMPONENT), ('version', Labels.LABEL_TYPE_VERSION), ): try: _, value = labels.get_name_and_value(label) except KeyError: pass else: config.set('image-build', config_key, value) def parse_image_build_config(self, config_file_name): # Logic taken from koji.cli.koji.handle_image_build. # Unable to re-use koji's code because "cli" is not # a package of koji and this logic is intermingled # with CLI specific instructions. args = [] opts = {} config = self.get_default_image_build_conf() config.read(config_file_name) if self.architectures: config.set('image-build', 'arches', ','.join(self.architectures)) # else just use what was provided by the user in image-build.conf config_str = StringIO() config.write(config_str) self.log.debug('Image Build Config: \n%s', config_str.getvalue()) image_name = None section = 'image-build' for option in ('name', 'version', 'arches', 'target', 'install_tree'): value = config.get(section, option) if not value: raise ValueError('{} cannot be empty'.format(option)) if option == 'arches': # pylint: disable=no-member value = [arch for arch in value.split(',') if arch] elif option == 'name': image_name = value args.append(value) config.remove_option(section, option) for option, value in config.items(section): if option in ('repo', 'format'): value = [v for v in value.split(',') if v] elif option in ('disk_size', ): value = int(value) opts[option] = value section = 'ova-options' if config.has_section(section): ova = [] for k, v in config.items(section): ova.append('{}={}'.format(k, v)) opts['ova_option'] = ova section = 'factory-parameters' if config.has_section(section): factory = [] for option, value in config.items(section): factory.append((option, value)) opts['factory_parameter'] = factory return image_name, args, {'opts': opts} def build_filesystem(self, image_build_conf): # Image build conf file should be in the same folder as Dockerfile image_build_conf = self.workflow.build_dir.any_platform.path / image_build_conf if not os.path.exists(image_build_conf): raise RuntimeError( 'Image build configuration file not found: {}'.format( image_build_conf)) image_name, args, kwargs = self.parse_image_build_config( image_build_conf) if self.scratch: kwargs['opts']['scratch'] = True task_id = self.session.buildImageOz(*args, **kwargs) return task_id, image_name def find_filesystem(self, task_id, filesystem_regex): for f in self.session.listTaskOutput(task_id): f = f.strip() match = filesystem_regex.match(f) if match: return task_id, match.group(0) # Not found in this task, search sub tasks for sub_task in self.session.getTaskChildren(task_id): found = self.find_filesystem(sub_task['id'], filesystem_regex) if found: return found return None def download_filesystem(self, task_id, filesystem_regex, build_dir: BuildDir): found = self.find_filesystem(task_id, filesystem_regex) if found is None: raise RuntimeError( 'Filesystem not found as task output: {}'.format( filesystem_regex.pattern)) task_id, file_name = found self.log.info('Downloading filesystem: %s from task ID: %s', file_name, task_id) file_path = build_dir.path / file_name if file_path.exists(): raise RuntimeError( f'Filesystem {file_name} already exists at {file_path}') with open(file_path, 'w') as f: for chunk in stream_task_output(self.session, task_id, file_name, self.blocksize): f.write(chunk) return file_name def run_image_task(self, image_build_conf): task_id, image_name = self.build_filesystem(image_build_conf) task = TaskWatcher(self.session, task_id, self.poll_interval) try: task.wait() except BuildCanceledException: self.log.info("Build was canceled, canceling task %s", task_id) try: self.session.cancelTask(task_id) self.log.info('task %s canceled', task_id) except Exception as exc: self.log.info("Exception while canceling a task (ignored): %s", util.exception_message(exc)) if task.failed(): try: # Koji may re-raise the error that caused task to fail task_result = self.session.getTaskResult(task_id) except Exception as exc: task_result = util.exception_message(exc) raise RuntimeError('image task, {}, failed: {}'.format( task_id, task_result)) return task_id, image_name def get_image_build_conf(self): image_build_conf = None for parent in self.workflow.data.dockerfile_images: if base_image_is_custom(parent.to_str()): image_build_conf = parent.tag break if not image_build_conf or image_build_conf == 'latest': image_build_conf = 'image-build.conf' return image_build_conf def update_repos_from_composes(self): resolve_comp_result = self.workflow.data.plugins_results.get( PLUGIN_RESOLVE_COMPOSES_KEY) if not resolve_comp_result: return for compose_info in resolve_comp_result['composes']: self.log.info('adding repo file from compose: %s', compose_info['result_repofile']) self.repos.append(compose_info['result_repofile']) def _add_filesystem_to_dockerfile(self, file_name, build_dir: BuildDir): """ Put an ADD instruction into the Dockerfile (to include the filesystem into the container image to be built) """ content = 'ADD {0} /\n'.format(file_name) lines = build_dir.dockerfile.lines # as we insert elements we have to keep track of the increment for inserting offset = 1 for item in build_dir.dockerfile.structure: if item['instruction'] == 'FROM' and base_image_is_custom( item['value'].split()[0]): lines.insert(item['endline'] + offset, content) offset += 1 build_dir.dockerfile.lines = lines new_parents = [] for image in build_dir.dockerfile.parent_images: if base_image_is_custom(image): new_parents.append('scratch') else: new_parents.append(image) build_dir.dockerfile.parent_images = new_parents self.log.info('added "%s" as image filesystem', file_name) def inject_filesystem(self, task_id, image_name, build_dir: BuildDir): prefix = '{}.*{}'.format(image_name, build_dir.platform) pattern = ( r'{}.*(\.tar|\.tar\.gz|\.tar\.bz2|\.tar\.xz)$'.format(prefix)) filesystem_regex = re.compile(pattern, re.IGNORECASE) file_name = self.download_filesystem(task_id, filesystem_regex, build_dir) self._add_filesystem_to_dockerfile(file_name, build_dir) def run(self): if not self.workflow.data.dockerfile_images.custom_parent_image: self.log.info('Nothing to do for non-custom base images') return self.update_repos_from_composes() image_build_conf = self.get_image_build_conf() self.session = get_koji_session(self.workflow.conf) task_id, image_name = self.run_image_task(image_build_conf) inject_filesystem_call = functools.partial(self.inject_filesystem, task_id, image_name) self.workflow.build_dir.for_each_platform(inject_filesystem_call) return { 'filesystem-koji-task-id': task_id, }
class ResolveRemoteSourcePlugin(Plugin): """Initiate a new Cachito request for sources This plugin will read the remote_sources configuration from container.yaml in the git repository, use it to make a request to Cachito, and wait for the request to complete. """ key = PLUGIN_RESOLVE_REMOTE_SOURCE is_allowed_to_fail = False REMOTE_SOURCE = "unpacked_remote_sources" args_from_user_params = map_to_user_params("dependency_replacements") def __init__(self, workflow, dependency_replacements=None): """ :param workflow: DockerBuildWorkflow instance :param dependency_replacements: list<str>, dependencies for the cachito fetched artifact to be replaced. Must be of the form pkg_manager:name:version[:new_name] """ super(ResolveRemoteSourcePlugin, self).__init__(workflow) self._cachito_session = None self._osbs = None self._dependency_replacements = self.parse_dependency_replacements(dependency_replacements) self.single_remote_source_params = self.workflow.source.config.remote_source self.multiple_remote_sources_params = self.workflow.source.config.remote_sources def parse_dependency_replacements(self, replacement_strings): """Parse dependency_replacements param and return cachito-reaady dependency replacement dict param replacement_strings: list<str>, pkg_manager:name:version[:new_name] return: list<dict>, cachito formated dependency replacements param """ if not replacement_strings: return dependency_replacements = [] for dr_str in replacement_strings: pkg_manager, name, version, new_name = (dr_str.split(':', 3) + [None] * 4)[:4] if None in [pkg_manager, name, version]: raise ValueError('Cachito dependency replacements must be ' '"pkg_manager:name:version[:new_name]". got {}'.format(dr_str)) dr = {'type': pkg_manager, 'name': name, 'version': version} if new_name: dr['new_name'] = new_name dependency_replacements.append(dr) return dependency_replacements def run(self) -> Optional[List[Dict[str, Any]]]: if (not self.workflow.conf.allow_multiple_remote_sources and self.multiple_remote_sources_params): raise ValueError('Multiple remote sources are not enabled, ' 'use single remote source in container.yaml') if not (self.single_remote_source_params or self.multiple_remote_sources_params): self.log.info('Aborting plugin execution: missing remote source configuration') return None if not self.workflow.conf.cachito: raise RuntimeError('No Cachito configuration defined') if self._dependency_replacements and not is_scratch_build(self.workflow): raise ValueError('Cachito dependency replacements are only allowed for scratch builds') if self._dependency_replacements and self.multiple_remote_sources_params: raise ValueError('Cachito dependency replacements are not allowed ' 'for multiple remote sources') processed_remote_sources = self.process_remote_sources() self.inject_remote_sources(processed_remote_sources) return [ self.remote_source_to_output(remote_source) for remote_source in processed_remote_sources ] def process_remote_sources(self) -> List[RemoteSource]: """Process remote source requests and return information about the processed sources.""" user = self.get_koji_user() self.log.info('Using user "%s" for cachito request', user) processed_remote_sources = [] if self.multiple_remote_sources_params: self.verify_multiple_remote_sources_names_are_unique() open_requests = { remote_source["name"]: self.cachito_session.request_sources( user=user, dependency_replacements=self._dependency_replacements, **remote_source["remote_source"] ) for remote_source in self.multiple_remote_sources_params } completed_requests = { name: self.cachito_session.wait_for_request(request) for name, request in open_requests.items() } for name, request in completed_requests.items(): processed_remote_sources.append(self.process_request(request, name)) else: open_request = self.cachito_session.request_sources( user=user, dependency_replacements=self._dependency_replacements, **self.single_remote_source_params ) completed_request = self.cachito_session.wait_for_request(open_request) processed_remote_sources.append(self.process_request(completed_request, None)) return processed_remote_sources def inject_remote_sources(self, remote_sources: List[RemoteSource]) -> None: """Inject processed remote sources into build dirs and add build args to workflow.""" inject_sources = functools.partial(self.inject_into_build_dir, remote_sources) self.workflow.build_dir.for_all_platforms_copy(inject_sources) # For single remote_source workflow, inject all build args directly if self.single_remote_source_params: self.workflow.data.buildargs.update(remote_sources[0].build_args) self.add_general_buildargs() def inject_into_build_dir( self, remote_sources: List[RemoteSource], build_dir: BuildDir, ) -> List[Path]: """Inject processed remote sources into a build directory. For each remote source, create a dedicated directory, unpack the downloaded tarball into it and inject the configuration files and an environment file. Return a list of the newly created directories. """ created_dirs = [] for remote_source in remote_sources: dest_dir = build_dir.path.joinpath(self.REMOTE_SOURCE, remote_source.name or "") if dest_dir.exists(): raise RuntimeError( f"Conflicting path {dest_dir.relative_to(build_dir.path)} already exists " "in the dist-git repository" ) dest_dir.mkdir(parents=True) created_dirs.append(dest_dir) with tarfile.open(remote_source.tarball_path) as tf: tf.extractall(dest_dir) config_files = self.cachito_session.get_request_config_files(remote_source.id) self.generate_cachito_config_files(dest_dir, config_files) # Create cachito.env file with environment variables received from cachito request self.generate_cachito_env_file(dest_dir, remote_source.build_args) return created_dirs def get_buildargs(self, request_id: int, remote_source_name: Optional[str]) -> Dict[str, str]: build_args = {} env_vars = self.cachito_session.get_request_env_vars(request_id) for env_var, value_info in env_vars.items(): build_arg_value = value_info['value'] kind = value_info['kind'] if kind == 'path': name = remote_source_name or '' build_arg_value = os.path.join(REMOTE_SOURCE_DIR, name, value_info['value']) self.log.debug( 'Setting the Cachito environment variable "%s" to the absolute path "%s"', env_var, build_arg_value, ) build_args[env_var] = build_arg_value elif kind == 'literal': self.log.debug( 'Setting the Cachito environment variable "%s" to a literal value "%s"', env_var, build_arg_value, ) build_args[env_var] = build_arg_value else: raise RuntimeError(f'Unknown kind {kind} got from Cachito.') return build_args def source_request_to_json(self, source_request): """Create a relevant representation of the source request""" required = ('packages', 'ref', 'repo') optional = ('dependencies', 'flags', 'pkg_managers', 'environment_variables', 'configuration_files', 'content_manifest') data = {} try: data.update({k: source_request[k] for k in required}) except KeyError as exc: msg = 'Received invalid source request from Cachito: {}'.format(source_request) self.log.exception(msg) raise ValueError(msg) from exc data.update({k: source_request[k] for k in optional if k in source_request}) return data def get_koji_user(self): unknown_user = self.workflow.conf.cachito.get('unknown_user', 'unknown_user') try: koji_task_id = int(self.workflow.user_params.get('koji_task_id')) except (ValueError, TypeError, AttributeError): msg = 'Unable to get koji user: Invalid Koji task ID' self.log.warning(msg) return unknown_user koji_session = get_koji_session(self.workflow.conf) return get_koji_task_owner(koji_session, koji_task_id).get('name', unknown_user) @property def cachito_session(self): if not self._cachito_session: self._cachito_session = get_cachito_session(self.workflow.conf) return self._cachito_session def verify_multiple_remote_sources_names_are_unique(self): names = [remote_source['name'] for remote_source in self.multiple_remote_sources_params] duplicate_names = [name for name, count in Counter(names).items() if count > 1] if duplicate_names: raise ValueError(f'Provided remote sources parameters contain ' f'non unique names: {duplicate_names}') def process_request(self, source_request: dict, name: Optional[str]) -> RemoteSource: """Download the tarball for a request and return info about the processed remote source.""" tarball_filename = RemoteSource.tarball_filename(name) dest_dir = str(self.workflow.build_dir.any_platform.path) tarball_dest_path = self.cachito_session.download_sources( source_request, dest_dir=dest_dir, dest_filename=tarball_filename, ) build_args = self.get_buildargs(source_request["id"], name) remote_source = RemoteSource( id=source_request["id"], name=name, json_data=self.source_request_to_json(source_request), build_args=build_args, tarball_path=Path(tarball_dest_path), ) return remote_source def remote_source_to_output(self, remote_source: RemoteSource) -> Dict[str, Any]: """Convert a processed remote source to a dict to be used as output of this plugin.""" download_url = self.cachito_session.assemble_download_url(remote_source.id) json_filename = RemoteSource.json_filename(remote_source.name) return { "id": remote_source.id, "name": remote_source.name, "url": download_url, "remote_source_json": { "json": remote_source.json_data, "filename": json_filename, }, "remote_source_tarball": { "filename": remote_source.tarball_path.name, "path": str(remote_source.tarball_path), }, } def generate_cachito_config_files(self, dest_dir: Path, config_files: List[dict]) -> None: """Inject cachito provided configuration files :param dest_dir: destination directory for config files :param config_files: configuration files from cachito """ for config in config_files: config_path = dest_dir / config['path'] if config['type'] == CFG_TYPE_B64: data = base64.b64decode(config['content']) config_path.write_bytes(data) else: err_msg = "Unknown cachito configuration file data type '{}'".format(config['type']) raise ValueError(err_msg) config_path.chmod(0o444) def generate_cachito_env_file(self, dest_dir: Path, build_args: Dict[str, str]) -> None: """ Generate cachito.env file with exported environment variables received from cachito request. :param dest_dir: destination directory for env file :param build_args: build arguments to set """ self.log.info('Creating %s file with environment variables ' 'received from cachito request', CACHITO_ENV_FILENAME) # Use dedicated dir in container build workdir for cachito.env abs_path = dest_dir / CACHITO_ENV_FILENAME with open(abs_path, 'w') as f: f.write('#!/bin/bash\n') for env_var, value in build_args.items(): f.write('export {}={}\n'.format(env_var, shlex.quote(value))) def add_general_buildargs(self) -> None: """Adds general build arguments To copy the sources into the build image, Dockerfile should contain COPY $REMOTE_SOURCE $REMOTE_SOURCE_DIR or COPY $REMOTE_SOURCES $REMOTE_SOURCES_DIR """ if self.multiple_remote_sources_params: args_for_dockerfile_to_add = { 'REMOTE_SOURCES': self.REMOTE_SOURCE, 'REMOTE_SOURCES_DIR': REMOTE_SOURCE_DIR, } else: args_for_dockerfile_to_add = { 'REMOTE_SOURCE': self.REMOTE_SOURCE, 'REMOTE_SOURCE_DIR': REMOTE_SOURCE_DIR, CACHITO_ENV_ARG_ALIAS: os.path.join(REMOTE_SOURCE_DIR, CACHITO_ENV_FILENAME), } self.workflow.data.buildargs.update(args_for_dockerfile_to_add)
class PinOperatorDigestsPlugin(PreBuildPlugin): """ Plugin runs for operator manifest bundle builds. - finds container pullspecs in operator ClusterServiceVersion files - computes replacement pullspecs: - replaces tags with manifest list digests - replaces repos (and namespaces) based on operator_manifests.repo_replacements configuration in container.yaml and r-c-m* - replaces registries based on operator_manifests.registry_post_replace in r-c-m* - replaces pullspecs in ClusterServiceVersion files based on replacement pullspec mapping - creates relatedImages sections in ClusterServiceVersion files Files that already have a relatedImages section are excluded. * reactor-config-map """ key = PLUGIN_PIN_OPERATOR_DIGESTS_KEY is_allowed_to_fail = False args_from_user_params = map_to_user_params( "operator_csv_modifications_url", ) def __init__(self, workflow, operator_csv_modifications_url=None): """ Initialize pin_operator_digests plugin :param workflow: DockerBuildWorkflow instance :param operator_csv_modifications_url: str, URL to JSON file with operator CSV modifications """ super(PinOperatorDigestsPlugin, self).__init__(workflow) self.user_config = workflow.source.config.operator_manifests self.operator_csv_modifications_url = operator_csv_modifications_url site_config = self.workflow.conf.operator_manifests self.operator_csv_modification_allowed_attributes = set( tuple(key_path) for key_path in ( site_config .get('csv_modifications', {}) .get('allowed_attributes', []) ) ) def _validate_operator_csv_modifications_schema(self, modifications): """Validate if provided operator CSV modification are valid according schema""" schema = load_schema( 'atomic_reactor', 'schemas/operator_csv_modifications.json' ) validate_with_schema(modifications, schema) def _validate_operator_csv_modifications_duplicated_images(self, modifications): """Validate if provided operator CSV modifications doesn't provide duplicated entries""" original_pullspecs = set() duplicated = set() for repl in modifications.get('pullspec_replacements', ()): pullspec = ImageName.parse(repl['original']) if pullspec in original_pullspecs: duplicated.add(pullspec) self.log.error( "Operator CSV modifications contains duplicated " "original replacement pullspec %s", pullspec) original_pullspecs.add(pullspec) if duplicated: raise RuntimeError( f"Provided CSV modifications contain duplicated " f"original entries in pullspec_replacement: " f"{', '.join(sorted(str(dup) for dup in duplicated))}" ) def _validate_operator_csv_modifications_allowed_keys(self, modifications): """Validate if used attributes in update/append are allowed to be modified""" allowed_attrs = self.operator_csv_modification_allowed_attributes def to_str(path): return '.'.join(part.replace('.', r'\.') for part in path) def validate(mods): not_allowed = [] for key_path in terminal_key_paths(mods): if key_path not in allowed_attrs: self.log.error( "Operator CSV attribute %s is not allowed to be modified", key_path ) not_allowed.append(key_path) if not_allowed: raise RuntimeError( f"Operator CSV attributes: {', '.join(to_str(attr) for attr in not_allowed)}; " f"are not allowed to be modified " f"by service configuration. Attributes allowed for modification " f"are: {', '.join(to_str(attr) for attr in allowed_attrs) or 'N/A'}" ) validate(modifications.get('update', {})) validate(modifications.get('append', {})) def _validate_operator_csv_modifications(self, modifications): """Validate if provided operator CSV modification correct""" self._validate_operator_csv_modifications_schema(modifications) self._validate_operator_csv_modifications_duplicated_images(modifications) self._validate_operator_csv_modifications_allowed_keys(modifications) def _fetch_operator_csv_modifications(self): """Fetch operator CSV modifications""" if not self.operator_csv_modifications_url: return None session = get_retrying_requests_session() self.log.info( "Fetching operator CSV modifications data from %s", self.operator_csv_modifications_url ) resp = session.get(self.operator_csv_modifications_url) try: resp.raise_for_status() except Exception as exc: raise RuntimeError( f"Failed to fetch the operator CSV modification JSON " f"from {self.operator_csv_modifications_url}: {exc}" ) from exc try: csv_modifications = resp.json() except Exception as exc: # catching raw Exception because requests uses various json decoders # in different versions raise RuntimeError( f"Failed to parse operator CSV modification JSON " f"from {self.operator_csv_modifications_url}: {exc}" ) from exc self.log.info("Operator CSV modifications: %s", csv_modifications) self._validate_operator_csv_modifications(csv_modifications) return csv_modifications def run(self): """ Run pin_operator_digest plugin """ if not self.should_run(): return if self.operator_csv_modifications_url: self.log.info( "Operator CSV modification URL specified: %s", self.operator_csv_modifications_url ) operator_manifest = self._get_operator_manifest() replacement_pullspecs = {} related_images_metadata = { 'pullspecs': [], 'created_by_osbs': True, } operator_manifests_metadata = { 'related_images': related_images_metadata, 'custom_csv_modifications_applied': bool(self.operator_csv_modifications_url) } should_skip = self._skip_all() if should_skip: self.log.warning("skip_all defined for operator manifests") pullspecs = self._get_pullspecs(operator_manifest.csv, should_skip) if operator_manifest.csv.has_related_images() or should_skip: if self.operator_csv_modifications_url: raise RuntimeError( "OSBS cannot modify operator CSV file because this operator bundle " "is managed by owner (digest pinning explicitly disabled or " "RelatedImages section in CSV exists)" ) # related images already exists related_images_metadata['created_by_osbs'] = False related_images_metadata['pullspecs'] = [{ 'original': item, 'new': item, 'pinned': False, 'replaced': False, } for item in pullspecs] else: if pullspecs: replacement_pullspecs = self._get_replacement_pullspecs(pullspecs) related_images_metadata['pullspecs'] = replacement_pullspecs else: # no pullspecs don't create relatedImages section related_images_metadata['created_by_osbs'] = False if should_skip: return operator_manifests_metadata replacement_pullspecs = { repl['original']: repl['new'] for repl in replacement_pullspecs if repl['replaced'] } operator_csv = operator_manifest.csv self.log.info("Updating operator CSV file") if not operator_csv.has_related_images(): self.log.info("Replacing pullspecs in %s", operator_csv.path) # Replace pullspecs everywhere, not just in locations in which they # are expected to be found - OCP 4.4 workaround operator_csv.replace_pullspecs_everywhere(replacement_pullspecs) self.log.info("Creating relatedImages section in %s", operator_csv.path) operator_csv.set_related_images() operator_csv_modifications = self._fetch_operator_csv_modifications() if operator_csv_modifications: operator_csv.modifications_append( operator_csv_modifications.get('append', {})) operator_csv.modifications_update( operator_csv_modifications.get('update', {})) self.workflow.build_dir.for_each_platform(operator_csv.dump) else: self.log.warning("%s has a relatedImages section, skipping", operator_csv.path) return operator_manifests_metadata def should_run(self): """ Determine if this is an operator manifest bundle build :return: bool, should plugin run? """ if not has_operator_bundle_manifest(self.workflow): self.log.info("Not an operator manifest bundle build, skipping plugin") return False if not self.workflow.conf.operator_manifests: msg = "operator_manifests configuration missing in reactor config map, aborting" self.log.warning(msg) return False return True def _skip_all(self): skip_all = self.user_config.get("skip_all", False) if not skip_all: return False site_config = self.workflow.conf.operator_manifests allowed_packages = site_config.get("skip_all_allow_list", []) # any_platform: the component label should be equal for all platforms parser = self.workflow.build_dir.any_platform.dockerfile_with_parent_env( self.workflow.imageutil.base_image_inspect() ) dockerfile_labels = parser.labels labels = Labels(dockerfile_labels) component_label = labels.get_name(Labels.LABEL_TYPE_COMPONENT) component = dockerfile_labels[component_label] if component in allowed_packages: return True else: raise RuntimeError("Koji package: {} isn't allowed to use skip_all for operator " "bundles".format(component)) def _get_operator_manifest(self): if self.workflow.source.config.operator_manifests is None: raise RuntimeError("operator_manifests configuration missing in container.yaml") self.log.info("Looking for operator CSV files in %s", self.workflow.build_dir.path) manifests_dir = self.workflow.source.config.operator_manifests["manifests_dir"] kwargs = {'repo_dir': self.workflow.build_dir.any_platform.path} operator_manifest = OperatorManifest.from_directory( os.path.join(self.workflow.build_dir.any_platform.path, manifests_dir), **kwargs) self.log.info("Found operator CSV file: %s", operator_manifest.csv.path) return operator_manifest def _get_pullspecs(self, operator_csv, skip_all): """Get pullspecs from CSV file :param OperatorCSV operator_csv: a cluster service version (CSV) file from where to find out pullspecs. :return: a list of pullspecs sorted by each one's string representation. If CSV does not have spec.relatedImages, all pullspecs will be found out from all possible locations. If CSV has spec.relatedImages, return the pullspecs contained. :rtype: list[ImageName] :raises RuntimeError: if the CSV has both spec.relatedImages and pullspecs referenced by environment variables prefixed with RELATED_IMAGE_. """ self.log.info("Looking for pullspecs in operator CSV file") pullspec_set = set() if not operator_csv.has_related_images(): if skip_all and operator_csv.get_pullspecs(): raise RuntimeError("skip_all defined but relatedImages section doesn't exist") self.log.info("Getting pullspecs from %s", operator_csv.path) pullspec_set.update(operator_csv.get_pullspecs()) elif operator_csv.has_related_image_envs(): msg = ("Both relatedImages and RELATED_IMAGE_* env vars present in {}. " "Please remove the relatedImages section, it will be reconstructed " "automatically.".format(operator_csv.path)) if not skip_all: raise RuntimeError(msg) else: pullspec_set.update(operator_csv.get_related_image_pullspecs()) # Make sure pullspecs are handled in a deterministic order # ImageName does not implement ordering, use str() as key for sorting pullspecs = sorted(pullspec_set, key=str) if pullspecs: pullspec_lines = "\n".join(image.to_str() for image in pullspecs) self.log.info("Found pullspecs:\n%s", pullspec_lines) else: self.log.info("No pullspecs found") return pullspecs def _get_replacement_pullspecs(self, pullspecs): """Replace components of pullspecs :param pullspecs: a list of pullspecs. :type pullspecs: list[ImageName] :return: a list of replacement result. Each of the replacement result is a mapping containing key/value pairs: * ``original``: ImageName, the original pullspec. * ``new``: ImageName, the replaced/non-replaced pullspec. * ``pinned``: bool, indicate whether the tag is replaced with a specific digest. * ``replaced``: bool, indicate whether the new pullspec has change of repository or registry. :rtype: list[dict[str, ImageName or bool]] :raises RuntimeError: if pullspecs cannot be properly replaced """ if self.operator_csv_modifications_url: replacements = self._get_replacement_pullspecs_from_csv_modifications(pullspecs) else: replacements = self._get_replacement_pullspecs_OSBS_resolution(pullspecs) replacement_lines = "\n".join( "{original} -> {new}".format(**r) if r['replaced'] else "{original} - no change".format(**r) for r in replacements ) self.log.info("To be replaced:\n%s", replacement_lines) return replacements def _get_replacement_pullspecs_from_csv_modifications(self, pullspecs): """Replace components of pullspecs based on externally provided CSV modifications :param pullspecs: a list of pullspecs. :type pullspecs: list[ImageName] :return: a list of replacement result. Each of the replacement result is a mapping containing key/value pairs: * ``original``: ImageName, the original pullspec. * ``new``: ImageName, the replaced/non-replaced pullspec. * ``pinned``: bool, indicate whether the tag is replaced with a specific digest. * ``replaced``: bool, indicate whether the new pullspec has change of repository or registry. :rtype: list[dict[str, ImageName or bool]] :raises RuntimeError: if provided CSV modification doesn't contain all required pullspecs or contain different ones """ operator_csv_modifications = self._fetch_operator_csv_modifications() mod_pullspec_repl = operator_csv_modifications.get('pullspec_replacements', []) # check if modification data contains all required pullspecs pullspecs_set = set(pullspecs) mod_pullspecs_set = set((ImageName.parse(p['original']) for p in mod_pullspec_repl)) missing = pullspecs_set - mod_pullspecs_set if missing: raise RuntimeError( f"Provided operator CSV modifications misses following pullspecs: " f"{', '.join(sorted(str(p) for p in missing))}" ) extra = mod_pullspecs_set - pullspecs_set if extra: raise RuntimeError( f"Provided operator CSV modifications defines extra pullspecs: " f"{','.join(sorted(str(p) for p in extra))}" ) # Copy replacements from provided CSV modifications file, fill missing 'replaced' filed replacements = [ { 'original': ImageName.parse(repl['original']), 'new': ImageName.parse(repl['new']), 'pinned': repl['pinned'], 'replaced': repl['original'] != repl['new'] } for repl in mod_pullspec_repl ] return replacements def _get_replacement_pullspecs_OSBS_resolution(self, pullspecs): """ Replace components of pullspecs according to operator manifest replacement config :param pullspecs: a list of pullspecs. :type pullspecs: list[ImageName] :return: a list of replacement result. Each of the replacement result is a mapping containing key/value pairs: * ``original``: ImageName, the original pullspec. * ``new``: ImageName, the replaced/non-replaced pullspec. * ``pinned``: bool, indicate whether the tag is replaced with a specific digest. * ``replaced``: bool, indicate whether the new pullspec has change of repository or registry. :rtype: list[dict[str, ImageName or bool]] :raises RuntimeError: if the registry of a pullspec is not allowed. Refer to the ``operator_manifest.allowed_registries`` in atomic reactor config. """ self.log.info("Computing replacement pullspecs") replacements = [] pin_digest, replace_repo, replace_registry = self._are_features_enabled() if not any([pin_digest, replace_repo, replace_registry]): self.log.warning("All replacement features disabled") replacer = PullspecReplacer(user_config=self.user_config, workflow=self.workflow) for p in pullspecs: if not replacer.registry_is_allowed(p): raise RuntimeError("Registry not allowed: {} (in {})".format(p.registry, p)) for original in pullspecs: self.log.info("Computing replacement for %s", original) replaced = original pinned = False if pin_digest: self.log.debug("Making sure tag is manifest list digest") replaced = replacer.pin_digest(original) if replaced != original: pinned = True if replace_repo: self.log.debug("Replacing namespace/repo") replaced = replacer.replace_repo(replaced) if replace_registry: self.log.debug("Replacing registry") replaced = replacer.replace_registry(replaced) self.log.info("Final pullspec: %s", replaced) replacements.append({ 'original': original, 'new': replaced, 'pinned': pinned, 'replaced': replaced != original }) return replacements def _are_features_enabled(self): pin_digest = self.user_config.get("enable_digest_pinning", True) replace_repo = self.user_config.get("enable_repo_replacements", True) replace_registry = self.user_config.get("enable_registry_replacements", True) if not pin_digest: self.log.warning("User disabled digest pinning") if not replace_repo: self.log.warning("User disabled repo replacements") if not replace_registry: self.log.warning("User disabled registry replacements") return pin_digest, replace_repo, replace_registry
class InjectParentImage(Plugin): """ Modifies parent image to be used based on given Koji build. It first attempts to find the list of available repositories from '.extra.image.index.pull' in Koji build information. If not found, the first archive in Koji build that defines a non-empty '.extra.docker.repositories' list is used. This list provides the pull reference for the container image associated with Koji build. If it contains multiple item, the manifest digest, @sha256, is preferred. Otherwise, the first repository in list is used. The namespace and repository for the new parent image must match the namespace and repository for the parent image defined in Dockerfile. This plugin returns the identifier of the Koji build used. """ key = PLUGIN_INJECT_PARENT_IMAGE_KEY is_allowed_to_fail = False args_from_user_params = map_to_user_params("koji_parent_build") def __init__(self, workflow, koji_parent_build=None): """ :param workflow: DockerBuildWorkflow instance :param koji_parent_build: str, either Koji build ID or Koji build NVR """ super(InjectParentImage, self).__init__(workflow) self.koji_session = get_koji_session(self.workflow.conf) try: self.koji_parent_build = int(koji_parent_build) except (ValueError, TypeError): self.koji_parent_build = koji_parent_build self._koji_parent_build_info = None self._repositories = None self._new_parent_image = None def run(self): if not self.koji_parent_build: self.log.info('no koji parent build, skipping plugin') return if self.workflow.data.dockerfile_images.base_from_scratch: self.log.info("from scratch can't inject parent image") return if self.workflow.data.dockerfile_images.custom_base_image: self.log.info("custom base image builds can't inject parent image") return self.find_repositories() self.select_new_parent_image() self.adjust_new_parent_image() self.set_new_parent_image() return self._koji_parent_build_info['id'] def find_repositories(self): self._repositories = (self.find_repositories_from_build() or self.find_repositories_from_archive()) if not self._repositories: raise RuntimeError('A suitable archive for Koji build {} was not found' .format(self._koji_parent_build_info['nvr'])) def find_repositories_from_build(self): self._koji_parent_build_info = self.koji_session.getBuild(self.koji_parent_build) if not self._koji_parent_build_info: raise RuntimeError('Koji build, {}, not found'.format(self.koji_parent_build)) repositories = graceful_chain_get(self._koji_parent_build_info, 'extra', 'image', 'index', 'pull') if repositories: self.log.info('Using repositories from build info') return repositories def find_repositories_from_archive(self): for archive in self.koji_session.listArchives(self._koji_parent_build_info['id']): repositories = graceful_chain_get(archive, 'extra', 'docker', 'repositories') if repositories: self.log.info('Using repositories from archive %d', archive['id']) return repositories return None def select_new_parent_image(self): for repository in self._repositories: if '@' in repository: self._new_parent_image = repository break # v2 manifest digest, not found, just pick the first one. if not self._new_parent_image: self._new_parent_image = self._repositories[0] self.log.info('New parent image is %s', self._new_parent_image) def adjust_new_parent_image(self): new_parent_image = ImageName.parse(self._new_parent_image) organization = self.workflow.conf.registries_organization source_registry_docker_uri = self.workflow.conf.source_registry['uri'].docker_uri if new_parent_image.registry != source_registry_docker_uri: new_parent_image.registry = source_registry_docker_uri if organization: new_parent_image.enclose(organization) self._new_parent_image = new_parent_image.to_str() def set_new_parent_image(self): df_images = self.workflow.data.dockerfile_images base_image_key = df_images.base_image_key df_images[base_image_key] = self._new_parent_image
class CheckUserSettingsPlugin(PreBuildPlugin): """ Pre plugin will check user settings on early phase to fail early and save resources. Aim of this plugin to checks: * Dockerfile * container.yaml * git repo for incorrect options or mutually exclusive options """ key = PLUGIN_CHECK_USER_SETTINGS is_allowed_to_fail = False args_from_user_params = map_to_user_params("flatpak") def __init__(self, workflow, flatpak=False): """ :param workflow: DockerBuildWorkflow instance :param flatpak: bool, if build is for flatpak """ super(CheckUserSettingsPlugin, self).__init__(workflow) self.flatpak = flatpak def dockerfile_checks(self): """Checks for Dockerfile""" if self.flatpak: self.log.info( "Skipping Dockerfile checks because this is flatpak build " "without user Dockerfile") return self.label_version_check() self.appregistry_bundle_label_mutually_exclusive() self.operator_bundle_from_scratch() def label_version_check(self): """Check that Dockerfile version has correct name.""" msg = "Dockerfile version label can't contain '/' character" self.log.debug("Running check: %s", msg) # any_platform: the version label should be equal for all platforms parser = self.workflow.build_dir.any_platform.dockerfile_with_parent_env( self.workflow.imageutil.base_image_inspect()) dockerfile_labels = parser.labels labels = Labels(parser.labels) component_label = labels.get_name(Labels.LABEL_TYPE_VERSION) label_version = dockerfile_labels[component_label] if '/' in label_version: raise ValueError(msg) def appregistry_bundle_label_mutually_exclusive(self): """Labels com.redhat.com.delivery.appregistry and com.redhat.delivery.operator.bundle are mutually exclusive. Fail when both are specified. """ msg = ("only one of labels com.redhat.com.delivery.appregistry " "and com.redhat.delivery.operator.bundle is allowed") self.log.debug("Running check: %s", msg) if (has_operator_appregistry_manifest(self.workflow) and has_operator_bundle_manifest(self.workflow)): raise ValueError(msg) def operator_bundle_from_scratch(self): """Only from scratch image can be used for operator bundle build""" msg = "Operator bundle build can be only 'FROM scratch' build (single stage)" self.log.debug("Running check: %s", msg) if not has_operator_bundle_manifest(self.workflow): return df_images = self.workflow.data.dockerfile_images if not df_images.base_from_scratch or len( df_images.original_parents) > 1: raise ValueError(msg) def validate_user_config_files(self): """Validate some user config files""" read_fetch_artifacts_koji(self.workflow) read_fetch_artifacts_pnc(self.workflow) read_fetch_artifacts_url(self.workflow) read_content_sets(self.workflow) def isolated_from_scratch_build(self): """Isolated builds for FROM scratch builds are prohibited except operator bundle images""" if (self.workflow.data.dockerfile_images.base_from_scratch and is_isolated_build(self.workflow) and not has_operator_bundle_manifest(self.workflow)): raise RuntimeError('"FROM scratch" image build cannot be isolated ' '(except operator bundle images)') def isolated_builds_checks(self): """Validate if isolated build was used correctly""" self.isolated_from_scratch_build() def run(self): """ run the plugin """ self.dockerfile_checks() self.validate_user_config_files() self.isolated_builds_checks()
class FetchSourcesPlugin(Plugin): """Download sources that may be used in further steps to compose Source Containers""" key = PLUGIN_FETCH_SOURCES_KEY is_allowed_to_fail = False SRPMS_DOWNLOAD_DIR = 'image_sources' REMOTE_SOURCES_DOWNLOAD_DIR = 'remote_sources' MAVEN_SOURCES_DOWNLOAD_DIR = 'maven_sources' args_from_user_params = map_to_user_params( "koji_build_id:sources_for_koji_build_id", "koji_build_nvr:sources_for_koji_build_nvr", "signing_intent", ) def __init__( self, workflow, koji_build_id=None, koji_build_nvr=None, signing_intent=None, ): """ :param workflow: DockerBuildWorkflow instance :param koji_build_id: int, container image koji build id :param koji_build_nvr: str, container image koji build NVR :param signing_intent: str, ODCS signing intent name """ if not koji_build_id and not koji_build_nvr: err_msg = ( '{} expects either koji_build_id or koji_build_nvr to be defined' .format(self.__class__.__name__)) raise TypeError(err_msg) type_errors = [] if koji_build_id is not None and not isinstance(koji_build_id, int): type_errors.append('koji_build_id must be an int. Got {}'.format( type(koji_build_id))) if koji_build_nvr is not None and not isinstance(koji_build_nvr, str): type_errors.append('koji_build_nvr must be a str. Got {}'.format( type(koji_build_nvr))) if type_errors: raise TypeError(type_errors) super(FetchSourcesPlugin, self).__init__(workflow) self.koji_build = None self.koji_build_id = koji_build_id self.koji_build_nvr = koji_build_nvr self.signing_intent = signing_intent self.session = get_koji_session(self.workflow.conf) self.pathinfo = self.workflow.conf.koji_path_info self._pnc_util = None @property def pnc_util(self): if not self._pnc_util: pnc_map = self.workflow.conf.pnc if not pnc_map: raise RuntimeError( 'No PNC configuration found in reactor config map') self._pnc_util = PNCUtil(pnc_map) return self._pnc_util def run(self): """ :return: dict, binary image koji build id and nvr, and path to directory with downloaded sources """ self.set_koji_image_build_data() self.check_lookaside_cache_usage() signing_intent = self.get_signing_intent() koji_config = self.workflow.conf.koji insecure = koji_config.get('insecure_download', False) urls = self.get_srpm_urls(signing_intent['keys'], insecure=insecure) urls_remote, remote_sources_map = self.get_remote_urls() urls_maven = (self.get_kojifile_source_urls() + self.get_remote_file_urls() + self.get_pnc_source_urls()) if not any([urls, urls_remote, urls_maven]): msg = "No srpms or remote sources or maven sources found for source" \ " container, would produce empty source container image" self.log.error(msg) raise RuntimeError(msg) sources_dir = None remote_sources_dir = None maven_sources_dir = None if urls: sources_dir = self.download_sources(urls, insecure=insecure) if urls_remote: remote_sources_dir = self.download_sources( urls_remote, insecure=insecure, download_dir=self.REMOTE_SOURCES_DOWNLOAD_DIR) self.exclude_files_from_remote_sources(remote_sources_map, remote_sources_dir) if urls_maven: maven_sources_dir = self.download_sources( urls_maven, insecure=insecure, download_dir=self.MAVEN_SOURCES_DOWNLOAD_DIR) return { 'sources_for_koji_build_id': self.koji_build_id, 'sources_for_nvr': self.koji_build_nvr, 'image_sources_dir': sources_dir, 'remote_sources_dir': remote_sources_dir, 'maven_sources_dir': maven_sources_dir, 'signing_intent': self.signing_intent, } def download_sources(self, sources, insecure=False, download_dir=SRPMS_DOWNLOAD_DIR): """Download sources content Download content in the given URLs into a new temporary directory and return a list with each downloaded artifact's path. :param sources: list, dicts with URLs to download :param insecure: bool, whether to perform TLS checks of urls :param download_dir: str, directory where to download content :return: str, paths to directory with downloaded sources """ dest_dir: Path = self.workflow.build_dir.source_container_sources_dir / download_dir dest_dir.mkdir(parents=True, exist_ok=True) req_session = get_retrying_requests_session() for source in sources: subdir: Path = dest_dir / source.get('subdir', '') subdir.mkdir(parents=True, exist_ok=True) checksums = source.get('checksums', {}) download_url(source['url'], subdir, insecure=insecure, session=req_session, dest_filename=source.get('dest'), expected_checksums=checksums) return str(dest_dir) def set_koji_image_build_data(self): build_identifier = self.koji_build_nvr or self.koji_build_id # strict means this raises a koji.GenericError informing no matching build was found in # case the build does not exist self.koji_build = self.session.getBuild(build_identifier, strict=True) if self.koji_build_id and (self.koji_build_id != self.koji_build['build_id']): err_msg = ( 'koji_build_id {} does not match koji_build_nvr {} with id {}. ' 'When specifying both an id and an nvr, they should point to the same image build' .format(self.koji_build_id, self.koji_build_nvr, self.koji_build['build_id'])) raise ValueError(err_msg) build_extras = self.koji_build['extra'] if 'image' not in build_extras: err_msg = ( 'koji build {} is not image build which source container requires' .format(self.koji_build['nvr'])) raise ValueError(err_msg) elif 'sources_for_nvr' in self.koji_build['extra']['image']: err_msg = ( 'koji build {} is source container build, source container can not ' 'use source container build image'.format( self.koji_build['nvr'])) raise ValueError(err_msg) if not self.koji_build_id: self.koji_build_id = self.koji_build['build_id'] if not self.koji_build_nvr: self.koji_build_nvr = self.koji_build['nvr'] def _get_cache_allowlist(self) -> List[Dict[str, Any]]: src_config = self.workflow.conf.source_container allowlist_cache_url = src_config.get('lookaside_cache_allowlist') if not allowlist_cache_url: self.log.debug('no "lookaside_cache_allowlist" defined, ' 'not allowing any lookaside cache usage') return [] self.log.debug( '"lookaside_cache_allowlist" defined, might allow lookaside cache usage' ) request_session = get_retrying_requests_session() response = request_session.get(allowlist_cache_url) response.raise_for_status() allowlist_cache_yaml = yaml.safe_load(response.text) return allowlist_cache_yaml def check_lookaside_cache_usage(self): """Check usage of lookaside cache, and fail if used""" git_uri, git_commit = self.koji_build['source'].split('#') source = GitSource('git', git_uri, provider_params={'git_commit': git_commit}) source_path = source.get() sources_cache_file = os.path.join(source_path, 'sources') uses_cache = False if os.path.exists(sources_cache_file): if os.path.getsize(sources_cache_file) > 0: uses_cache = True source.remove_workdir() if not uses_cache: return allowlist_cache_yaml = self._get_cache_allowlist() should_raise = True if allowlist_cache_yaml: build_component = self.koji_build['package_name'] build_task = self.koji_build['extra']['container_koji_task_id'] task_info = self.session.getTaskInfo(build_task, request=True) build_target = task_info['request'][1] for allowed in allowlist_cache_yaml: if allowed['component'] != build_component: continue for target in allowed['targets']: if re.match(target, build_target): self.log.debug( 'target "%s" for component "%s" has allowed using ' 'lookaside cache', build_target, build_component) should_raise = False break if not should_raise: break if should_raise: raise RuntimeError( 'Repository is using lookaside cache, which is not allowed ' 'for source container builds') def assemble_srpm_url(self, base_url, srpm_filename, sign_key=None): """Assemble the URL used to fetch an SRPM file :param base_url: str, Koji root base URL with the given build artifacts :param srpm_filename: str, name of the SRPM file :param sign_key: str, key used to sign the SRPM, as listed in the signing intent :return: list, strings with URLs pointing to SRPM files """ srpm_info = koji.parse_NVRA(srpm_filename) if sign_key: srpm_path = self.pathinfo.signed(srpm_info, sign_key) else: srpm_path = self.pathinfo.rpm(srpm_info) return '/'.join([base_url, srpm_path]) def _process_remote_source(self, koji_build, archives, remote_sources_path): self.log.debug('remote_source_url defined') remote_sources_urls = [] remote_json_map = {} remote_source = {} remote_source['url'] = os.path.join(remote_sources_path, REMOTE_SOURCE_TARBALL_FILENAME) remote_source['dest'] = '-'.join( [koji_build['nvr'], REMOTE_SOURCE_TARBALL_FILENAME]) remote_sources_urls.append(remote_source) cachito_json_url = os.path.join(remote_sources_path, REMOTE_SOURCE_JSON_FILENAME) remote_json_map[remote_source['dest']] = cachito_json_url archive_found = False json_found = False all_archives = [] for archive in archives: if archive['filename'] == REMOTE_SOURCE_TARBALL_FILENAME: archive_found = True elif archive['filename'] == REMOTE_SOURCE_JSON_FILENAME: json_found = True all_archives.append(archive['filename']) if not (archive_found and json_found): message = ', '.join( part for t, part in ((archive_found, "remote source archive missing"), (json_found, "remote source json missing")) if not t) raise RuntimeError(message) elif len(archives) > 2: raise RuntimeError( 'There can be just one remote sources archive and one ' 'remote sources json, got: {}'.format(all_archives)) return remote_sources_urls, remote_json_map def _process_multiple_remote_sources(self, koji_build, archives, remote_sources_path): self.log.debug('remote_sources defined') remote_sources_urls = [] remote_json_map = {} remote_sources = koji_build['extra']['typeinfo']['remote-sources'] wrong_archives = False all_archives = [] for remote_s in remote_sources: remote_archive = None remote_json = None if len(remote_s['archives']) != 2: self.log.error( 'remote source "%s" does not contain 2 archives, but "%s"', remote_s['name'], remote_s['archives']) wrong_archives = True else: for archive in remote_s['archives']: if archive.endswith('.json'): remote_json = archive else: remote_archive = archive if not remote_json: self.log.error( 'remote source json, for remote source "%s" not found ' 'in archives "%s"', remote_s['name'], remote_s['archives']) wrong_archives = True else: remote_source = {} remote_source['url'] = os.path.join( remote_sources_path, remote_archive) remote_source['dest'] = '-'.join( [koji_build['nvr'], remote_archive]) remote_sources_urls.append(remote_source) cachito_json_url = os.path.join(remote_sources_path, remote_json) remote_json_map[remote_source['dest']] = cachito_json_url all_archives.append(remote_archive) all_archives.append(remote_json) if wrong_archives: raise RuntimeError( 'Problems with archives in remote sources: {}'.format( remote_sources)) extra_archives = [] for archive in archives: if archive['filename'] in all_archives: all_archives.remove(archive['filename']) else: extra_archives.append(archive['filename']) if all_archives: raise RuntimeError( 'Remote source files from metadata missing in koji ' 'archives: {}'.format(all_archives)) if extra_archives: raise RuntimeError('Remote source archives in koji missing from ' 'metadata: {}'.format(extra_archives)) return remote_sources_urls, remote_json_map def _get_remote_urls_helper(self, koji_build): """Fetch remote source urls from specific build :param koji_build: dict, koji build :return: str, URL pointing to remote sources """ self.log.debug('get remote_urls: %s', koji_build['build_id']) archives = self.session.listArchives(koji_build['build_id'], type=KOJI_BTYPE_REMOTE_SOURCES) self.log.debug('archives: %s', archives) remote_sources_path = self.pathinfo.typedir( koji_build, btype=KOJI_BTYPE_REMOTE_SOURCES) remote_sources_urls = [] remote_json_map = {} if 'remote_source_url' in koji_build['extra']['image']: remote_sources_urls, remote_json_map = \ self._process_remote_source(koji_build, archives, remote_sources_path) elif 'remote_sources' in koji_build['extra']['image']: remote_sources_urls, remote_json_map = \ self._process_multiple_remote_sources(koji_build, archives, remote_sources_path) return remote_sources_urls, remote_json_map def _get_kojifile_source_urls_helper(self, koji_build): """Fetch kojifile source urls from specific build :param koji_build: dict, koji build :return: list, dicts with URL pointing to kojifile sources """ self.log.debug('get kojifile_source_urls: %s', koji_build['build_id']) images = self.session.listArchives(koji_build['build_id'], type='image') self.log.debug('images: %s', images) sources = [] kojifile_build_ids = { kojifile['build_id'] for image in images for kojifile in self.session.listArchives(imageID=image['id'], type='maven') } for build_id in kojifile_build_ids: source_build = self.session.getBuild(build_id, strict=True) if source_build['owner_name'] == PNC_SYSTEM_USER: pnc_build_id = source_build['extra']['external_build_id'] url, dest_filename = self.pnc_util.get_scm_archive_from_build_id( build_id=pnc_build_id) source = { 'url': url, 'subdir': source_build['nvr'], 'dest': '__'.join([source_build['nvr'], dest_filename]) } else: source_archive = None maven_build_path = self.pathinfo.mavenbuild(source_build) for archive in self.session.listArchives( buildID=source_build['build_id'], type='maven'): if archive['filename'].endswith('-project-sources.tar.gz'): source_archive = archive break if not source_archive: raise RuntimeError( f"No sources found for {source_build['nvr']}") maven_file_path = self.pathinfo.mavenfile(source_archive) url = maven_build_path + '/' + maven_file_path source = { 'url': url, 'subdir': source_build['nvr'], 'dest': '__'.join( [source_build['nvr'], source_archive['filename']]), 'checksums': { koji.CHECKSUM_TYPES[source_archive['checksum_type']]: source_archive['checksum'] } } sources.append(source) return sources def _get_pnc_source_urls_helper(self, koji_build): """Fetch PNC source urls from specific build :param koji_build: dict, koji build :return: list, dicts with URL pointing to PNC sources """ sources = [] if 'pnc' not in koji_build['extra']['image']: self.log.info("No PNC build ids found") return sources build_ids = set() for build in koji_build['extra']['image']['pnc']['builds']: build_ids.add(build['id']) self.log.debug('PNC build ids: %s', build_ids) for build_id in build_ids: url, dest_filename = self.pnc_util.get_scm_archive_from_build_id( build_id=build_id) source = { 'url': url, 'subdir': str(build_id), 'dest': '__'.join([str(build_id), dest_filename]) } sources.append(source) return sources def _get_remote_file_urls_helper(self, koji_build): """Fetch remote source file urls from specific build :param koji_build: dict, koji build :return: str, URL pointing to remote source files """ self.log.debug('get remote_file_urls: %s', koji_build['build_id']) archives = self.session.listArchives(koji_build['build_id'], type='remote-source-file') self.log.debug('archives: %s', archives) remote_source_files_path = self.pathinfo.typedir( koji_build, btype='remote-source-file') sources = [] for archive in archives: if archive['type_name'] == 'tar': # download each remote-source-file archive into it's own subdirectory # with the same name as the archive source = { 'url': os.path.join(remote_source_files_path, archive['filename']), 'subdir': archive['filename'].rsplit('.', 2)[0], 'dest': archive['filename'], 'checksums': { koji.CHECKSUM_TYPES[archive['checksum_type']]: archive['checksum'] } } sources.append(source) return sources def get_remote_urls(self): """Fetch remote source urls from all builds :return: list, dicts with URL pointing to remote sources """ remote_sources_urls = [] remote_sources_map = {} remote_source, remote_json = self._get_remote_urls_helper( self.koji_build) remote_sources_urls.extend(remote_source) remote_sources_map.update(remote_json) koji_build = self.koji_build while 'parent_build_id' in koji_build['extra']['image']: koji_build = self.session.getBuild( koji_build['extra']['image']['parent_build_id'], strict=True) remote_source, remote_json = self._get_remote_urls_helper( koji_build) remote_sources_urls.extend(remote_source) remote_sources_map.update(remote_json) return remote_sources_urls, remote_sources_map def get_kojifile_source_urls(self): """Fetch kojifile source urls from all builds :return: list, dicts with URL pointing to kojifile source files """ kojifile_sources = [] kojifile_source = self._get_kojifile_source_urls_helper( self.koji_build) kojifile_sources.extend(kojifile_source) koji_build = self.koji_build while 'parent_build_id' in koji_build['extra']['image']: koji_build = self.session.getBuild( koji_build['extra']['image']['parent_build_id'], strict=True) kojifile_source = self._get_kojifile_source_urls_helper(koji_build) kojifile_sources.extend(kojifile_source) return kojifile_sources def get_pnc_source_urls(self): """Fetch PNC build source urls from all builds :return: list, dicts with URL pointing to PNC build source archives """ sources = [] source = self._get_pnc_source_urls_helper(self.koji_build) sources.extend(source) koji_build = self.koji_build while 'parent_build_id' in koji_build['extra']['image']: koji_build = self.session.getBuild( koji_build['extra']['image']['parent_build_id'], strict=True) source = self._get_pnc_source_urls_helper(koji_build) sources.extend(source) return sources def get_remote_file_urls(self): """Fetch remote source file urls from all builds :return: list, dicts with URL pointing to remote source files """ sources = [] source = self._get_remote_file_urls_helper(self.koji_build) sources.extend(source) koji_build = self.koji_build while 'parent_build_id' in koji_build['extra']['image']: koji_build = self.session.getBuild( koji_build['extra']['image']['parent_build_id'], strict=True) source = self._get_remote_file_urls_helper(koji_build) sources.extend(source) return sources def get_denylisted_srpms(self): src_config = self.workflow.conf.source_container denylist_srpms = src_config.get('denylist_srpms') if not denylist_srpms: self.log.debug( 'denylist_srpms is not defined in reactor_config_map') return [] denylist_url = denylist_srpms['denylist_url'] denylist_key = denylist_srpms['denylist_key'] req_session = get_retrying_requests_session() response = req_session.get(denylist_url) response.raise_for_status() response_json = response.json() if denylist_key not in response_json: self.log.debug('deny list json : %s', response_json) raise RuntimeError( 'Denylist key: {} missing in denylist json from : {}'.format( denylist_key, denylist_url)) deny_list = response_json[denylist_key] if not isinstance(deny_list, list): self.log.error('Wrong denylist: %s', repr(deny_list)) raise RuntimeError( 'Denylist value in key: {} is not list: {}'.format( denylist_key, type(deny_list))) wrong_types = [pkg for pkg in deny_list if not isinstance(pkg, str)] if wrong_types: self.log.error('Wrong types in denylist, should be str: %s', repr(deny_list)) raise RuntimeError('Values in denylist has to be all strings') self.log.debug('denylisted srpms: %s', deny_list) return deny_list def get_srpm_urls(self, sigkeys=None, insecure=False): """Fetch SRPM download URLs for each image generated by a build Build each possible SRPM URL and check if the URL is available, respecting the signing intent preference order. :param sigkeys: list, strings for keys which signed the srpms to be fetched :return: list, strings with URLs pointing to SRPM files """ if not sigkeys: sigkeys = [''] self.log.debug('get srpm_urls: %s', self.koji_build_id) archives = self.session.listArchives(self.koji_build_id, type='image') self.log.debug('archives: %s', archives) rpms = [ rpm for archive in archives for rpm in self.session.listRPMs(imageID=archive['id']) ] denylist_srpms = self.get_denylisted_srpms() srpm_build_paths = {} for rpm in rpms: rpm_id = rpm['id'] self.log.debug('Resolving SRPM for RPM ID: %s', rpm_id) if rpm['external_repo_name'] != 'INTERNAL': msg = ('RPM comes from an external repo (RPM ID: {}). ' 'External RPMs are currently not supported.' ).format(rpm_id) raise RuntimeError(msg) rpm_hdr = self.session.getRPMHeaders(rpm_id, headers=['SOURCERPM']) if 'SOURCERPM' not in rpm_hdr: raise RuntimeError( 'Missing SOURCERPM header (RPM ID: {})'.format(rpm_id)) srpm_name = rpm_hdr['SOURCERPM'].rsplit('-', 2)[0] if any(denied == srpm_name for denied in denylist_srpms): self.log.debug('skipping denylisted srpm %s', rpm_hdr['SOURCERPM']) continue srpm_filename = rpm_hdr['SOURCERPM'] if srpm_filename in srpm_build_paths: continue rpm_build = self.session.getBuild(rpm['build_id'], strict=True) base_url = self.pathinfo.build(rpm_build) srpm_build_paths[srpm_filename] = base_url srpm_urls = [] missing_srpms = [] req_session = get_retrying_requests_session() for srpm_filename, base_url in srpm_build_paths.items(): for sigkey in sigkeys: # koji uses lowercase for paths. We make sure the sigkey is in lower case url_candidate = self.assemble_srpm_url(base_url, srpm_filename, sigkey.lower()) # allow redirects, head call doesn't do it by default request = req_session.head(url_candidate, verify=not insecure, allow_redirects=True) if request.ok: srpm_urls.append({'url': url_candidate}) self.log.debug('%s is available for signing key "%s"', srpm_filename, sigkey) break else: self.log.error( '%s not found for the given signing intent: %s"', srpm_filename, self.signing_intent) missing_srpms.append(srpm_filename) if missing_srpms: raise RuntimeError( 'Could not find files signed by any of {} for these SRPMS: {}'. format(sigkeys, missing_srpms)) return srpm_urls def get_signing_intent(self): """Get the signing intent to be used to fetch files from Koji :return: dict, signing intent object as per atomic_reactor/schemas/config.json """ odcs_config = self.workflow.conf.odcs_config if odcs_config is None: self.log.warning( 'No ODCS configuration available. Allowing unsigned SRPMs') return {'keys': None} if not self.signing_intent: try: self.signing_intent = self.koji_build['extra']['image'][ 'odcs']['signing_intent'] except (KeyError, TypeError): self.log.debug( 'Image koji build, %s(%s), does not define signing_intent.', self.koji_build_nvr, self.koji_build_id) self.signing_intent = odcs_config.default_signing_intent signing_intent = odcs_config.get_signing_intent_by_name( self.signing_intent) return signing_intent def _get_denylist_sources(self, request_session, denylist_sources_url): response = request_session.get(denylist_sources_url) response.raise_for_status() denylist_sources_yaml = yaml.safe_load(response.text) # prepend os.sep for 2 reasons: # - so fnmatch will match exact dir/file when using * + exclude # glob.glob doesn't need it # - so endswith for package will match also full name return [ os.path.join(os.sep, k, item) for k, v in denylist_sources_yaml.items() for item in v ] def _create_full_remote_sources_map(self, request_session, remote_sources_map, remote_sources_dir): full_remote_sources_map = {} for remote_source, remote_source_json in remote_sources_map.items(): remote_source_archive = os.path.join(remote_sources_dir, remote_source) response = request_session.get(remote_source_json) response.raise_for_status() response_json = response.json() full_remote_sources_map[remote_source_archive] = response_json return full_remote_sources_map def _check_if_package_excluded(self, packages, denylist_sources, remote_archive): # check if any package in cachito json matches excluded entry # strip leading os.sep as package names can include git path with '/' before package name # or just package name, or package name with leading '@' depending on package type denylist_packages = {k.lstrip(os.sep) for k in denylist_sources} for package in packages: for exclude_path in denylist_packages: if package.get('name').endswith(exclude_path): self.log.debug('Package excluded: "%s" from "%s"', package.get('name'), remote_archive) return True return False def _delete_app_directory(self, remote_source_dir, unpack_dir, remote_archive): vendor_dir = os.path.join(unpack_dir, 'app', 'vendor') if os.path.exists(vendor_dir): shutil.move(vendor_dir, remote_source_dir) self.log.debug('Removing app from "%s"', remote_archive) shutil.rmtree(os.path.join(unpack_dir, 'app')) # shutil.move will create missing parent directory shutil.move(os.path.join(remote_source_dir, 'vendor'), vendor_dir) self.log.debug('Keeping vendor in app from "%s"', remote_archive) else: self.log.debug('Removing app from "%s"', remote_archive) shutil.rmtree(os.path.join(unpack_dir, 'app')) def _get_excluded_matches(self, unpack_dir, denylist_sources): matches = [] # py2 glob.glob doesn't support recursive, hence os.walk & fnmatch # py3 can use: glob.glob(os.path.join(unpack_dir, '**', exclude), recursive=True) for root, dirnames, filenames in os.walk(unpack_dir): for entry in dirnames + filenames: full_path = os.path.join(root, entry) for exclude in denylist_sources: if full_path.endswith(exclude): matches.append(full_path) break return matches def _remove_excluded_matches(self, matches): for entry in matches: if os.path.exists(entry): if os.path.isdir(entry): self.log.debug("Removing excluded directory %s", entry) shutil.rmtree(entry) else: self.log.debug("Removing excluded file %s", entry) os.unlink(entry) def exclude_files_from_remote_sources(self, remote_sources_map, remote_sources_dir): """ :param remote_sources_map: dict, keys are filenames of sources from cachito, values are url with json from cachito :param remote_sources_dir: str, dir with downloaded sources from cachito """ src_config = self.workflow.conf.source_container denylist_sources_url = src_config.get('denylist_sources') if not denylist_sources_url: self.log.debug('no "denylist_sources" defined, not excluding any ' 'files from remote sources') return request_session = get_retrying_requests_session() denylist_sources = self._get_denylist_sources(request_session, denylist_sources_url) # key: full path to source archive, value: cachito json full_remote_sources_map = self._create_full_remote_sources_map( request_session, remote_sources_map, remote_sources_dir) for remote_archive, remote_json in full_remote_sources_map.items(): unpack_dir = remote_archive + '_unpacked' with tarfile.open(remote_archive) as tf: tf.extractall(unpack_dir) delete_app = self._check_if_package_excluded( remote_json['packages'], denylist_sources, remote_archive) # if any package in cachito json matched excluded entry, # remove 'app' from sources, except 'app/vendor' when exists if delete_app and os.path.exists(os.path.join(unpack_dir, 'app')): self._delete_app_directory(remote_sources_dir, unpack_dir, remote_archive) # search for excluded matches matches = self._get_excluded_matches(unpack_dir, denylist_sources) self._remove_excluded_matches(matches) # delete former archive os.unlink(remote_archive) # re-create new archive without excluded content with tarfile.open(remote_archive, "w:gz") as tar: for add_file in os.listdir(unpack_dir): tar.add(os.path.join(unpack_dir, add_file), arcname=add_file) # cleanup unpacked dir shutil.rmtree(unpack_dir)
class FlatpakUpdateDockerfilePlugin(PreBuildPlugin): key = "flatpak_update_dockerfile" is_allowed_to_fail = False args_from_user_params = map_to_user_params("compose_ids") def __init__(self, workflow): """ constructor :param workflow: DockerBuildWorkflow instance """ # call parent constructor super(FlatpakUpdateDockerfilePlugin, self).__init__(workflow) def update_dockerfile(self, builder, compose_info, build_dir: BuildDir): # Update the dockerfile # We need to enable all the modules other than the platform pseudo-module enable_modules_str = ' '.join(builder.get_enable_modules()) install_packages_str = ' '.join(builder.get_install_packages()) replacements = { '@ENABLE_MODULES@': enable_modules_str, '@INSTALL_PACKAGES@': install_packages_str, '@RELEASE@': compose_info.main_module.version, } dockerfile = build_dir.dockerfile content = dockerfile.content # Perform the substitutions; simple approach - should be efficient enough for old, new in replacements.items(): content = content.replace(old, new) dockerfile.content = content def create_includepkgs_file_and_cleanupscript(self, builder, build_dir: BuildDir): # Create a file describing which packages from the base yum repositories are included includepkgs = builder.get_includepkgs() includepkgs_path = build_dir.path / FLATPAK_INCLUDEPKGS_FILENAME with open(includepkgs_path, 'w') as f: f.write('includepkgs = ' + ','.join(includepkgs) + '\n') # Create the cleanup script cleanupscript = build_dir.path / FLATPAK_CLEANUPSCRIPT_FILENAME with open(cleanupscript, 'w') as f: f.write(builder.get_cleanup_script()) os.chmod(cleanupscript, 0o0500) return [includepkgs_path, cleanupscript] def run(self): """ run the plugin """ if not is_flatpak_build(self.workflow): self.log.info('not flatpak build, skipping plugin') return resolve_comp_result = self.workflow.data.prebuild_results.get( PLUGIN_RESOLVE_COMPOSES_KEY) flatpak_util = FlatpakUtil(workflow_config=self.workflow.conf, source_config=self.workflow.source.config, composes=resolve_comp_result['composes']) compose_info = flatpak_util.get_flatpak_compose_info() source = flatpak_util.get_flatpak_source_info() builder = FlatpakBuilder(source, None, None) builder.precheck() flatpak_update = functools.partial(self.update_dockerfile, builder, compose_info) self.workflow.build_dir.for_each_platform(flatpak_update) create_files = functools.partial( self.create_includepkgs_file_and_cleanupscript, builder) self.workflow.build_dir.for_all_platforms_copy(create_files)
class AddImageContentManifestPlugin(Plugin): """ Add the ICM JSON file to the IMAGE_BUILD_INFO_DIR/content_manifests directory, for the current platform. Filename will be '{IMAGE_NVR}.json' ICM examples: WITHOUT content_sets specified: { "metadata": { "icm_version": 1, "icm_spec": "https://link.to.icm.specification", "image_layer_index": 3 }, "content_sets" : [], "image_contents": [ { "purl": "pkg:golang/github.com%2Frelease-engineering%2Fretrodep%[email protected]", "dependencies": [{"purl": "pkg:golang/github.com%2Fop%[email protected]"}], "sources": [{"purl": "pkg:golang/github.com%2FMasterminds%[email protected]"}] } ] } WITH content_sets specified: { "metadata": { "icm_version": 1, "icm_spec": "https://link.to.icm.specification", "image_layer_index": 2 }, "content_sets": [ "rhel-8-for-x86_64-baseos-rpms", "rhel-8-for-x86_64-appstream-rpms" ], "image_contents": [ { "purl": "pkg:golang/github.com%2Frelease-engineering%2Fretrodep%[email protected]", "dependencies": [{"purl": "pkg:golang/github.com%2Fop%[email protected]"}], "sources": [{"purl": "pkg:golang/github.com%2FMasterminds%[email protected]"}] } ] } """ key = PLUGIN_ADD_IMAGE_CONTENT_MANIFEST is_allowed_to_fail = False minimal_icm: Dict[str, Any] = { 'metadata': { 'icm_version': 1, 'icm_spec': ('https://raw.githubusercontent.com/containerbuildsystem/atomic-reactor/' 'master/atomic_reactor/schemas/content_manifest.json'), 'image_layer_index': 1 }, 'content_sets': [], 'image_contents': [], } args_from_user_params = map_to_user_params("remote_sources") def __init__(self, workflow, destdir=IMAGE_BUILD_INFO_DIR): """ :param workflow: DockerBuildWorkflow instance :param destdir: image path to carry content_manifests data dir """ super(AddImageContentManifestPlugin, self).__init__(workflow) self.content_manifests_dir = os.path.join(destdir, 'content_manifests') wf_data = self.workflow.data remote_source_results = wf_data.plugins_results.get( PLUGIN_RESOLVE_REMOTE_SOURCE) or [] self.remote_source_ids = [ remote_source['id'] for remote_source in remote_source_results ] fetch_maven_results = wf_data.plugins_results.get( PLUGIN_FETCH_MAVEN_KEY) or {} self.pnc_artifact_ids = fetch_maven_results.get( 'pnc_artifact_ids') or [] @functools.cached_property def icm_file_name(self): """Determine the name for the ICM file (name-version-release.json).""" # parse Dockerfile for any platform, the N-V-R labels should be equal for all platforms dockerfile = self.workflow.build_dir.any_platform.dockerfile_with_parent_env( self.workflow.imageutil.base_image_inspect()) labels = Labels(dockerfile.labels) _, name = labels.get_name_and_value(Labels.LABEL_TYPE_COMPONENT) _, version = labels.get_name_and_value(Labels.LABEL_TYPE_VERSION) _, release = labels.get_name_and_value(Labels.LABEL_TYPE_RELEASE) return f"{name}-{version}-{release}.json" @property def layer_index(self) -> int: # inspect any platform, we expect the number of layers to be equal for all platforms inspect = self.workflow.imageutil.base_image_inspect() if not inspect: # Base images ('FROM koji/image-build') and 'FROM scratch' images do not have any # base image. When building with `podman build --squash`, such images get squashed # to only 1 layer => the layer index in this case is 0 (the first and only layer). # This is only true for build tasks that behave like `podman build --squash` return 0 return len(inspect[INSPECT_ROOTFS][INSPECT_ROOTFS_LAYERS]) @functools.cached_property def _icm_base(self) -> dict: """Create the platform-independent skeleton of the ICM document. :return: dict, the ICM as a Python dict """ icm = deepcopy(self.minimal_icm) if self.remote_source_ids: icm = self.cachito_session.get_image_content_manifest( self.remote_source_ids) if self.pnc_artifact_ids: purl_specs = self.pnc_util.get_artifact_purl_specs( self.pnc_artifact_ids) for purl_spec in purl_specs: icm['image_contents'].append({'purl': purl_spec}) icm['metadata']['image_layer_index'] = self.layer_index return icm def make_icm(self, platform: str) -> dict: """Create the complete ICM document for the specified platform.""" icm = deepcopy(self._icm_base) content_sets = read_content_sets(self.workflow) or {} icm['content_sets'] = content_sets.get(platform, []) self.log.debug('Output ICM content_sets: %s', icm['content_sets']) self.log.debug('Output ICM metadata: %s', icm['metadata']) # Validate; `json.dumps()` converts `icm` to str. Confusingly, `read_yaml` # *will* validate JSON read_yaml(json.dumps(icm), 'schemas/content_manifest.json') return icm def _write_json_file(self, icm: dict, build_dir: BuildDir) -> None: out_file_path = build_dir.path / self.icm_file_name if out_file_path.exists(): raise RuntimeError(f'File {out_file_path} already exists in repo') with open(out_file_path, 'w') as outfile: json.dump(icm, outfile, indent=4) self.log.debug('ICM JSON saved to: %s', out_file_path) def _add_to_dockerfile(self, build_dir: BuildDir) -> None: """ Put an ADD instruction into the Dockerfile (to include the ICM file into the container image to be built) """ dest_file_path = os.path.join(self.content_manifests_dir, self.icm_file_name) content = 'ADD {0} {1}'.format(self.icm_file_name, dest_file_path) lines = build_dir.dockerfile.lines # Put it before last instruction lines.insert(-1, content + '\n') build_dir.dockerfile.lines = lines def inject_icm(self, build_dir: BuildDir) -> None: """Inject the ICM document to a build directory.""" self.log.debug( "Injecting ICM to the build directory for the %s platform", build_dir.platform) icm = self.make_icm(build_dir.platform) self._write_json_file(icm, build_dir) self._add_to_dockerfile(build_dir) self.log.info('added "%s" to "%s"', self.icm_file_name, self.content_manifests_dir) def run(self): """Run the plugin.""" self.workflow.build_dir.for_each_platform(self.inject_icm) @property def cachito_session(self): if not self.workflow.conf.cachito: raise RuntimeError('No Cachito configuration defined') return get_cachito_session(self.workflow.conf) @property def pnc_util(self): pnc_map = self.workflow.conf.pnc if not pnc_map: raise RuntimeError( 'No PNC configuration found in reactor config map') return PNCUtil(pnc_map)
class CheckAndSetPlatformsPlugin(PreBuildPlugin): key = PLUGIN_CHECK_AND_SET_PLATFORMS_KEY is_allowed_to_fail = False args_from_user_params = map_to_user_params("koji_target") def __init__(self, workflow, koji_target=None): """ constructor :param workflow: DockerBuildWorkflow instance :param koji_target: str, Koji build target name """ # call parent constructor super(CheckAndSetPlatformsPlugin, self).__init__(workflow) self.koji_target = koji_target def _limit_platforms(self, platforms: List[str]) -> List[str]: """Limit platforms in a specific range by platforms config. :param platforms: a list of platforms to be filtered. :type platforms: list[str] :return: the limited platforms. :rtype: list[str] """ final_platforms = set(platforms) source_config = self.workflow.source.config only_platforms = set(source_config.only_platforms) excluded_platforms = set(source_config.excluded_platforms) if only_platforms: if only_platforms == excluded_platforms: self.log.warning('only and not platforms are the same: %r', only_platforms) final_platforms &= only_platforms return list(final_platforms - excluded_platforms) def run(self) -> Optional[List[str]]: """ run the plugin """ if self.koji_target: koji_session = get_koji_session(self.workflow.conf) self.log.info("Checking koji target for platforms") event_id = koji_session.getLastEvent()['id'] target_info = koji_session.getBuildTarget(self.koji_target, event=event_id) build_tag = target_info['build_tag'] koji_build_conf = koji_session.getBuildConfig(build_tag, event=event_id) koji_platforms = koji_build_conf['arches'] if not koji_platforms: self.log.info("No platforms found in koji target") return None platforms = koji_platforms.split() self.log.info("Koji platforms are %s", sorted(platforms)) if is_scratch_build(self.workflow) or is_isolated_build(self.workflow): override_platforms = set(get_orchestrator_platforms(self.workflow) or []) if override_platforms and override_platforms != set(platforms): sorted_platforms = sorted(override_platforms) self.log.info("Received user specified platforms %s", sorted_platforms) self.log.info("Using them instead of koji platforms") # platforms from user params do not match platforms from koji target # that almost certainly means they were overridden and should be used return sorted_platforms else: platforms = get_orchestrator_platforms(self.workflow) user_platforms = sorted(platforms) if platforms else None self.log.info("No koji platforms. User specified platforms are %s", user_platforms) if not platforms: raise RuntimeError("Cannot determine platforms; no koji target or platform list") # Filter platforms based on configured remote hosts remote_host_pools = self.workflow.conf.remote_hosts.get("pools", {}) enabled_platforms = [] defined_but_disabled = [] def has_enabled_hosts(platform: str) -> bool: platform_hosts = remote_host_pools.get(platform, {}) return any(host_info["enabled"] for host_info in platform_hosts.values()) for p in platforms: if has_enabled_hosts(p): enabled_platforms.append(p) elif p in remote_host_pools: defined_but_disabled.append(p) else: self.log.warning("No remote hosts found for platform '%s' in " "reactor config map, skipping", p) if defined_but_disabled: msg = 'Platforms specified in config map, but have all remote hosts disabled' \ ' {}'.format(defined_but_disabled) raise RuntimeError(msg) final_platforms = self._limit_platforms(enabled_platforms) self.log.info("platforms in limits : %s", final_platforms) if not final_platforms: self.log.error("platforms in limits are empty") raise RuntimeError("No platforms to build for") self.workflow.build_dir.init_build_dirs(final_platforms, self.workflow.source) return final_platforms
class TagAndPushPlugin(PostBuildPlugin): """ Use tags from workflow.data.tag_conf and push the images to workflow.conf.registry """ key = "tag_and_push" is_allowed_to_fail = False args_from_user_params = map_to_user_params("koji_target") def __init__(self, workflow, koji_target=None): """ constructor :param workflow: DockerBuildWorkflow instance :param koji_target: str, used only for sourcecontainers """ # call parent constructor super(TagAndPushPlugin, self).__init__(workflow) self.registry = self.workflow.conf.registry self.koji_target = koji_target def push_with_skopeo(self, image: Dict[str, str], registry_image: ImageName, insecure: bool, docker_push_secret: str) -> None: cmd = ['skopeo', 'copy'] if docker_push_secret is not None: dockercfg = Dockercfg(docker_push_secret) cmd.append('--authfile=' + dockercfg.json_secret_path) if insecure: cmd.append('--dest-tls-verify=false') if image['type'] == IMAGE_TYPE_OCI: # ref_name is added by 'flatpak_create_oci' # we have to be careful when changing the source container image type # since assumption here is that source container image will always be 'docker-archive' source_img = 'oci:{path}:{ref_name}'.format(**image) cmd.append('--format=v2s2') elif image['type'] == IMAGE_TYPE_DOCKER_ARCHIVE: source_img = 'docker-archive://{path}'.format(**image) else: raise RuntimeError("Attempt to push unsupported image type %s with skopeo" % image['type']) dest_img = 'docker://' + registry_image.to_str() cmd += [source_img, dest_img] try: retries.run_cmd(cmd) except subprocess.CalledProcessError as e: self.log.error("push failed with output:\n%s", e.output) raise def source_get_unique_image(self) -> ImageName: source_result = self.workflow.data.prebuild_results[PLUGIN_FETCH_SOURCES_KEY] koji_build_id = source_result['sources_for_koji_build_id'] kojisession = get_koji_session(self.workflow.conf) timestamp = osbs.utils.utcnow().strftime('%Y%m%d%H%M%S') random.seed() current_platform = platform.processor() or 'x86_64' tag_segments = [ self.koji_target or 'none', str(random.randrange(10**(RAND_DIGITS - 1), 10**RAND_DIGITS)), timestamp, current_platform ] tag = '-'.join(tag_segments) get_build_meta = kojisession.getBuild(koji_build_id) pull_specs = get_build_meta['extra']['image']['index']['pull'] source_image_spec = ImageName.parse(pull_specs[0]) source_image_spec.tag = tag organization = self.workflow.conf.registries_organization if organization: source_image_spec.enclose(organization) source_image_spec.registry = self.workflow.conf.registry['uri'] return source_image_spec def get_repositories(self) -> Dict[str, List[str]]: # usually repositories formed from NVR labels # these should be used for pulling and layering primary_repositories = [] for image in self.workflow.data.tag_conf.primary_images: primary_repositories.append(image.to_str()) # unique unpredictable repositories unique_repositories = [] for image in self.workflow.data.tag_conf.unique_images: unique_repositories.append(image.to_str()) # floating repositories # these should be used for pulling and layering floating_repositories = [] for image in self.workflow.data.tag_conf.floating_images: floating_repositories.append(image.to_str()) return { "primary": primary_repositories, "unique": unique_repositories, "floating": floating_repositories, } def run(self) -> Dict[str, Union[List, Dict[str, List[str]]]]: is_source_build = PLUGIN_FETCH_SOURCES_KEY in self.workflow.data.prebuild_results if not is_source_build and not is_flatpak_build(self.workflow): self.log.info('not a flatpak or source build, skipping plugin') return {'pushed_images': [], 'repositories': self.get_repositories()} pushed_images = [] wf_data = self.workflow.data tag_conf = wf_data.tag_conf images = [] if is_source_build: source_image = self.source_get_unique_image() plugin_results = wf_data.buildstep_result[PLUGIN_SOURCE_CONTAINER_KEY] image = plugin_results['image_metadata'] tag_conf.add_unique_image(source_image) images.append((image, source_image)) else: for image_platform in get_platforms(self.workflow.data): plugin_results = wf_data.postbuild_results[PLUGIN_FLATPAK_CREATE_OCI] image = plugin_results[image_platform]['metadata'] registry_image = tag_conf.get_unique_images_with_platform(image_platform)[0] images.append((image, registry_image)) insecure = self.registry.get('insecure', False) docker_push_secret = self.registry.get('secret', None) self.log.info("Registry %s secret %s", self.registry['uri'], docker_push_secret) for image, registry_image in images: max_retries = DOCKER_PUSH_MAX_RETRIES for retry in range(max_retries + 1): self.push_with_skopeo(image, registry_image, insecure, docker_push_secret) if is_source_build: manifests_dict = get_all_manifests(registry_image, self.registry['uri'], insecure, docker_push_secret, versions=('v2',)) try: koji_source_manifest_response = manifests_dict['v2'] except KeyError as exc: raise RuntimeError( f'Unable to fetch v2 schema 2 digest for {registry_image.to_str()}' ) from exc wf_data.koji_source_manifest = koji_source_manifest_response.json() digests = get_manifest_digests(registry_image, self.registry['uri'], insecure, docker_push_secret) if not (digests.v2 or digests.oci) and (retry < max_retries): sleep_time = DOCKER_PUSH_BACKOFF_FACTOR * (2 ** retry) self.log.info("Retrying push because V2 schema 2 or " "OCI manifest not found in %is", sleep_time) time.sleep(sleep_time) else: break pushed_images.append(registry_image) self.log.info("All images were tagged and pushed") return {'pushed_images': pushed_images, 'repositories': self.get_repositories()}
class KojiImportBase(PostBuildPlugin): """ Import this build to Koji Submits a successful build to Koji using the Content Generator API, https://docs.pagure.org/koji/content_generators Authentication is with Kerberos unless the koji_ssl_certs configuration parameter is given, in which case it should be a path at which 'cert', 'ca', and 'serverca' are the certificates for SSL authentication. If Kerberos is used for authentication, the default principal will be used (from the kernel keyring) unless both koji_keytab and koji_principal are specified. The koji_keytab parameter is a keytab name like 'type:name', and so can be used to specify a key in a Kubernetes secret by specifying 'FILE:/path/to/key'. Runs as an exit plugin in order to capture logs from all other plugins. """ is_allowed_to_fail = False args_from_user_params = map_to_user_params("userdata") def __init__(self, workflow, blocksize=None, poll_interval=5, userdata=None): """ constructor :param workflow: DockerBuildWorkflow instance :param blocksize: int, blocksize to use for uploading files :param poll_interval: int, seconds between Koji task status requests :param userdata: dict, custom user data """ super(KojiImportBase, self).__init__(workflow) self.blocksize = blocksize self.poll_interval = poll_interval self.osbs = get_openshift_session(self.workflow.conf, self.workflow.namespace) self.build_id = None self.session = None self.userdata = userdata self.koji_task_id = None koji_task_id = self.workflow.user_params.get('koji_task_id') if koji_task_id is not None: try: self.koji_task_id = int(koji_task_id) except ValueError: # Why pass 1 to exc_info originally? self.log.error("invalid task ID %r", koji_task_id, exc_info=1) @cached_property def _builds_metadatas(self) -> Dict[str, Any]: """Get builds metadata returned from gather_builds_metadata plugin. :return: a mapping from platform to metadata mapping. e.g. {"x86_64": {...}} """ metadatas = self.workflow.data.postbuild_results.get( PLUGIN_GATHER_BUILDS_METADATA_KEY, {}) if not metadatas: self.log.warning( "No build metadata is found. Check if %s plugin ran already.", PLUGIN_GATHER_BUILDS_METADATA_KEY, ) return metadatas def _iter_build_metadata_outputs( self, platform: Optional[str] = None, _filter: Optional[Dict[str, Any]] = None, ) -> Iterator[Tuple[str, Dict[str, Any]]]: """Iterate outputs from build metadata. :param platform: iterate outputs for a specific platform. If omitted, no platform is limited. :type platform: str or None :param _filter: key/value pairs to filter outputs. If omitted, no output is filtered out. :type _filter: dict[str, any] or None :return: an iterator that yields a tuple in form (platform, output). """ for build_platform, metadata in self._builds_metadatas.items(): if platform is not None and build_platform != platform: continue for output in metadata["output"]: if _filter: if all( output.get(key) == value for key, value in _filter.items()): yield build_platform, output else: yield build_platform, output def get_output(self, *args): # Must be implemented by subclasses raise NotImplementedError def get_buildroot(self, *args): # Must be implemented by subclasses raise NotImplementedError def set_help(self, extra: Dict[str, Any]) -> None: """Set extra.image.help""" result = self.workflow.data.prebuild_results.get(AddHelpPlugin.key) if not result: return extra['image']['help'] = result['help_file'] def set_media_types(self, extra): media_types = [] # Append media_types from verify images media_results = self.workflow.data.postbuild_results.get( PLUGIN_VERIFY_MEDIA_KEY) if media_results: media_types += media_results if media_types: extra['image']['media_types'] = sorted(set(media_types)) def set_go_metadata(self, extra): go = self.workflow.source.config.go if go: self.log.user_warning( f"Using 'go' key in {REPO_CONTAINER_CONFIG} is deprecated in favor of using " f"Cachito integration") self.log.debug("Setting Go metadata: %s", go) extra['image']['go'] = go def set_operators_metadata(self, extra): wf_data = self.workflow.data # upload metadata from bundle (part of image) op_bundle_metadata = wf_data.prebuild_results.get( PLUGIN_PIN_OPERATOR_DIGESTS_KEY) if op_bundle_metadata: op_related_images = op_bundle_metadata['related_images'] pullspecs = [{ 'original': str(p['original']), 'new': str(p['new']), 'pinned': p['pinned'], } for p in op_related_images['pullspecs']] koji_operator_manifests = { 'custom_csv_modifications_applied': op_bundle_metadata['custom_csv_modifications_applied'], 'related_images': { 'pullspecs': pullspecs, 'created_by_osbs': op_related_images['created_by_osbs'], } } extra['image']['operator_manifests'] = koji_operator_manifests # update push plugin and uploaded manifests file independently as push plugin may fail op_push_res = wf_data.postbuild_results.get( PLUGIN_PUSH_OPERATOR_MANIFESTS_KEY) if op_push_res: extra.update({"operator_manifests": {"appregistry": op_push_res}}) outputs = self._iter_build_metadata_outputs( _filter={'filename': OPERATOR_MANIFESTS_ARCHIVE}) for _, _ in outputs: extra['operator_manifests_archive'] = OPERATOR_MANIFESTS_ARCHIVE operators_typeinfo = { KOJI_BTYPE_OPERATOR_MANIFESTS: { 'archive': OPERATOR_MANIFESTS_ARCHIVE, }, } extra.setdefault('typeinfo', {}).update(operators_typeinfo) return # only one worker can process operator manifests def set_pnc_build_metadata(self, extra): plugin_results = self.workflow.data.prebuild_results.get( PLUGIN_FETCH_MAVEN_KEY) or {} pnc_build_metadata = plugin_results.get('pnc_build_metadata') if pnc_build_metadata: extra['image']['pnc'] = pnc_build_metadata def set_remote_sources_metadata(self, extra): remote_source_result = self.workflow.data.prebuild_results.get( PLUGIN_RESOLVE_REMOTE_SOURCE) if remote_source_result: if self.workflow.conf.allow_multiple_remote_sources: remote_sources_image_metadata = [{ "name": remote_source["name"], "url": remote_source["url"].rstrip('/download') } for remote_source in remote_source_result] extra["image"][ "remote_sources"] = remote_sources_image_metadata remote_sources_typeinfo_metadata = [{ "name": remote_source["name"], "url": remote_source["url"].rstrip('/download'), "archives": [ remote_source["remote_source_json"]["filename"], remote_source["remote_source_tarball"]["filename"], ], } for remote_source in remote_source_result] else: extra["image"]["remote_source_url"] = remote_source_result[0][ "url"] remote_sources_typeinfo_metadata = { "remote_source_url": remote_source_result[0]["url"] } remote_source_typeinfo = { KOJI_BTYPE_REMOTE_SOURCES: remote_sources_typeinfo_metadata, } extra.setdefault("typeinfo", {}).update(remote_source_typeinfo) def set_remote_source_file_metadata(self, extra): maven_url_sources_metadata_results = self.workflow.data.postbuild_results.get( PLUGIN_MAVEN_URL_SOURCES_METADATA_KEY) or {} fetch_maven_results = self.workflow.data.prebuild_results.get( PLUGIN_FETCH_MAVEN_KEY) or {} remote_source_files = maven_url_sources_metadata_results.get( 'remote_source_files') no_source_artifacts = fetch_maven_results.get('no_source') if remote_source_files or no_source_artifacts: r_s_f_typeinfo = { KOJI_BTYPE_REMOTE_SOURCE_FILE: {}, } if remote_source_files: r_s_f_typeinfo[KOJI_BTYPE_REMOTE_SOURCE_FILE][ 'remote_source_files'] = [] for remote_source_file in remote_source_files: r_s_f_extra = remote_source_file['metadata']['extra'] r_s_f_typeinfo[KOJI_BTYPE_REMOTE_SOURCE_FILE][ 'remote_source_files'].append({ r_s_f_extra['source-url']: r_s_f_extra['artifacts'] }) if no_source_artifacts: r_s_f_typeinfo[KOJI_BTYPE_REMOTE_SOURCE_FILE][ 'no_source'] = no_source_artifacts extra.setdefault('typeinfo', {}).update(r_s_f_typeinfo) def set_group_manifest_info(self, extra): version_release = None primary_images = get_primary_images(self.workflow) if primary_images: version_release = primary_images[0].tag if is_scratch_build(self.workflow): tags = [image.tag for image in self.workflow.data.tag_conf.images] version_release = tags[0] else: assert version_release is not None, 'Unable to find version-release image' tags = [image.tag for image in primary_images] floating_tags = [ image.tag for image in get_floating_images(self.workflow) ] unique_images = get_unique_images(self.workflow) unique_tags = [image.tag for image in unique_images] manifest_data = self.workflow.data.postbuild_results.get( PLUGIN_GROUP_MANIFESTS_KEY, {}) if manifest_data and is_manifest_list(manifest_data.get("media_type")): manifest_digest = manifest_data["manifest_digest"] digest = manifest_digest.default build_image = unique_images[0] repo = ImageName.parse(build_image).to_str(registry=False, tag=False) # group_manifests added the registry, so this should be valid registry_uri = self.workflow.conf.registry['uri'] digest_version = get_manifest_media_version(manifest_digest) media_type = get_manifest_media_type(digest_version) extra['image']['index'] = { 'tags': tags, 'floating_tags': floating_tags, 'unique_tags': unique_tags, 'pull': [ f'{registry_uri}/{repo}@{digest}', f'{registry_uri}/{repo}:{version_release}', ], 'digests': { media_type: digest }, } # group_manifests returns None if didn't run, {} if group=False else: platform = "x86_64" _, instance = next( self._iter_build_metadata_outputs(platform, {"type": "docker-image"}), (None, None), ) if instance: # koji_upload, running in the worker, doesn't have the full tags # so set them here instance['extra']['docker']['tags'] = tags instance['extra']['docker']['floating_tags'] = floating_tags instance['extra']['docker']['unique_tags'] = unique_tags repositories = [] for pullspec in instance['extra']['docker']['repositories']: if '@' not in pullspec: image = ImageName.parse(pullspec) image.tag = version_release pullspec = image.to_str() repositories.append(pullspec) instance['extra']['docker']['repositories'] = repositories self.log.debug("reset tags to so that docker is %s", instance['extra']['docker']) def _update_extra(self, extra): # Must be implemented by subclasses """ :param extra: A dictionary, representing koji's 'build.extra' metadata """ raise NotImplementedError def _update_build(self, build): # Must be implemented by subclasses raise NotImplementedError def _get_build_extra(self) -> Dict[str, Any]: extra = { 'image': {}, 'osbs_build': { 'subtypes': [] }, 'submitter': self.session.getLoggedInUser()['name'], } if self.koji_task_id is not None: extra['container_koji_task_id'] = self.koji_task_id self.log.info("build configuration created by Koji Task ID %s", self.koji_task_id) self._update_extra(extra) self.set_media_types(extra) return extra def get_build(self): start_time = int(atomic_reactor_start_time) koji_task_owner = get_koji_task_owner(self.session, self.koji_task_id).get('name') build = { 'start_time': start_time, 'end_time': int(time.time()), 'extra': self._get_build_extra(), 'owner': koji_task_owner, } self._update_build(build) return build def combine_metadata_fragments(self) -> Dict[str, Any]: """Construct the CG metadata and collect the output files for upload later.""" def add_buildroot_id(output: Output, buildroot_id: str) -> Output: output.metadata.update({'buildroot_id': buildroot_id}) return Output(filename=output.filename, metadata=output.metadata) def add_log_type(output: Output) -> Output: output.metadata.update({'type': 'log', 'arch': 'noarch'}) return Output(filename=output.filename, metadata=output.metadata) build = self.get_build() buildroot = self.get_buildroot() buildroot_id = buildroot[0]['id'] # Collect the output files, which will be uploaded later. koji_upload_files = self.workflow.data.koji_upload_files output: List[Dict[str, Any]] # List of metadatas # The corresponding output file, only has one for source build output_file: Optional[Output] output, output_file = self.get_output(buildroot_id) if output_file: koji_upload_files.append({ "local_filename": output_file.filename, "dest_filename": output[0]["filename"], }) # Collect log files osbs_logs = OSBSLogs(self.log, get_platforms(self.workflow.data)) log_files_output = [ add_log_type(add_buildroot_id(md, buildroot_id)) for md in osbs_logs.get_log_files(self.osbs, self.workflow.pipeline_run_name) ] for log_file_output in log_files_output: output.append(log_file_output.metadata) koji_upload_files.append({ "local_filename": log_file_output.filename, "dest_filename": log_file_output.metadata["filename"], }) remote_source_file_outputs, kojifile_components = get_maven_metadata( self.workflow.data) # add maven components alongside RPM components for metadata in output: if metadata['type'] == 'docker-image': metadata['components'] += kojifile_components # add remote sources tarballs and remote sources json files to output for remote_source_output in [ *get_source_tarballs_output(self.workflow), *get_remote_sources_json_output(self.workflow) ]: add_custom_type(remote_source_output, KOJI_BTYPE_REMOTE_SOURCES) remote_source = add_buildroot_id(remote_source_output, buildroot_id) output.append(remote_source.metadata) koji_upload_files.append({ "local_filename": remote_source.filename, "dest_filename": remote_source.metadata["filename"], }) for remote_source_file_output in remote_source_file_outputs: remote_source_file = add_buildroot_id(remote_source_file_output, buildroot_id) output.append(remote_source_file.metadata) koji_upload_files.append({ "local_filename": remote_source_file_output.filename, "dest_filename": remote_source_file_output.metadata["filename"], }) koji_metadata = { 'metadata_version': 0, 'build': build, 'buildroots': buildroot, 'output': output, } return koji_metadata def upload_file(self, local_filename: str, dest_filename: str, serverdir: str) -> str: """ Upload a file to koji :return: str, pathname on server """ self.log.debug("uploading %r to %r as %r", local_filename, serverdir, dest_filename) kwargs = {} if self.blocksize is not None: kwargs['blocksize'] = self.blocksize self.log.debug("using blocksize %d", self.blocksize) callback = KojiUploadLogger(self.log).callback self.session.uploadWrapper(local_filename, serverdir, name=dest_filename, callback=callback, **kwargs) # In case dest_filename includes path. uploadWrapper can handle this by itself. path = os.path.join(serverdir, os.path.basename(dest_filename)) self.log.debug("uploaded %r", path) return path def upload_scratch_metadata(self, koji_metadata, koji_upload_dir): metadata_file = NamedTemporaryFile(prefix="metadata", suffix=".json", mode='wb', delete=False) metadata_file.write( json.dumps(koji_metadata, indent=2).encode('utf-8')) metadata_file.close() local_filename = metadata_file.name try: uploaded_filename = self.upload_file(local_filename, "metadata.json", koji_upload_dir) log = logging.LoggerAdapter(self.log, {'arch': METADATA_TAG}) log.info(uploaded_filename) finally: os.unlink(local_filename) def get_server_dir(self): return koji_cli.lib.unique_path('koji-upload') def _upload_output_files(self, server_dir: str) -> None: """Helper method to upload collected output files.""" for upload_info in self.workflow.data.koji_upload_files: self.upload_file(upload_info["local_filename"], upload_info["dest_filename"], server_dir) def run(self): """ Run the plugin. """ # get the session and token information in case we need to refund a failed build self.session = get_koji_session(self.workflow.conf) server_dir = self.get_server_dir() koji_metadata = self.combine_metadata_fragments() if is_scratch_build(self.workflow): self.upload_scratch_metadata(koji_metadata, server_dir) return # for all builds which have koji task if self.koji_task_id: task_info = self.session.getTaskInfo(self.koji_task_id) task_state = koji.TASK_STATES[task_info['state']] if task_state != 'OPEN': self.log.error( "Koji task is not in Open state, but in %s, not importing build", task_state) return self._upload_output_files(server_dir) build_token = self.workflow.data.reserved_token build_id = self.workflow.data.reserved_build_id if build_id is not None and build_token is not None: koji_metadata['build']['build_id'] = build_id try: if build_token: build_info = self.session.CGImport(koji_metadata, server_dir, token=build_token) else: build_info = self.session.CGImport(koji_metadata, server_dir) except Exception: self.log.debug("metadata: %r", koji_metadata) raise # Older versions of CGImport do not return a value. build_id = build_info.get("id") if build_info else None self.log.debug("Build information: %s", json.dumps(build_info, sort_keys=True, indent=4)) return build_id
class InjectYumReposPlugin(Plugin): key = "inject_yum_repos" is_allowed_to_fail = False args_from_user_params = map_to_user_params("target:koji_target", ) def __init__(self, workflow, target=None, inject_proxy=None): """ constructor :param workflow: DockerBuildWorkflow instance :param target: string, koji target to use as a source :param inject_proxy: set proxy server for this repo """ super().__init__(workflow) self.target = target self.repourls = {} self.inject_proxy = inject_proxy self.yum_repos = defaultdict(list) self.allowed_domains = self.workflow.conf.yum_repo_allowed_domains self.include_koji_repo = False self._builder_ca_bundle = None self._ca_bundle_pem = None self.platforms = get_platforms(workflow.data) resolve_comp_result = self.workflow.data.plugins_results.get( PLUGIN_RESOLVE_COMPOSES_KEY) self.include_koji_repo = resolve_comp_result['include_koji_repo'] self.repourls = resolve_comp_result['yum_repourls'] def validate_yum_repo_files_url(self): if not self.allowed_domains: return errors = [] checked = set() for platform in self.platforms: for repourl in self.repourls.get(platform, []): if repourl in checked: continue repo_domain = urlparse(repourl).netloc checked.add(repourl) if repo_domain not in self.allowed_domains: errors.append( 'Yum repo URL {} is not in list of allowed domains: {}' .format(repourl, self.allowed_domains)) if errors: raise ValueError( 'Errors found while checking yum repo urls: \n{}'.format( '\n'.join(errors))) def _final_user_line(self): user = self._find_final_user() if user: return user if not self.workflow.data.dockerfile_images.base_from_scratch: # Inspect any platform: the User should be equal for all platforms inspect = self.workflow.imageutil.base_image_inspect() user = inspect.get(INSPECT_CONFIG, {}).get('User') if user: return f'USER {user}' return '' def _find_final_user(self): """Find the user in USER instruction in the last build stage""" dockerfile = self.workflow.build_dir.any_platform.dockerfile_with_parent_env( self.workflow.imageutil.base_image_inspect()) for insndesc in reversed(dockerfile.structure): if insndesc['instruction'] == 'USER': return insndesc['content'] # we will reuse the line verbatim if insndesc['instruction'] == 'FROM': break # no USER specified in final stage def _cleanup_lines(self, platform): lines = [ "RUN rm -f " + " ".join((f"'{repo.dst_filename}'" for repo in self.yum_repos[platform])) ] if self._builder_ca_bundle: lines.append(f'RUN rm -f /tmp/{self._ca_bundle_pem}') final_user_line = self._final_user_line() if final_user_line: lines.insert(0, "USER root") lines.append(final_user_line) return lines def add_koji_repo(self): xmlrpc = get_koji_session(self.workflow.conf) pathinfo = self.workflow.conf.koji_path_info proxy = self.workflow.conf.yum_proxy if not self.target: self.log.info('no target provided, not adding koji repo') return target_info = xmlrpc.getBuildTarget(self.target) if target_info is None: self.log.error("provided target '%s' doesn't exist", self.target) raise RuntimeError("Provided target '%s' doesn't exist!" % self.target) tag_info = xmlrpc.getTag(target_info['build_tag_name']) if not tag_info or 'name' not in tag_info: self.log.warning("No tag info was retrieved") return repo_info = xmlrpc.getRepo(tag_info['id']) if not repo_info or 'id' not in repo_info: self.log.warning("No repo info was retrieved") return # to use urljoin, we would have to append '/', so let's append everything baseurl = pathinfo.repo(repo_info['id'], tag_info['name']) + "/$basearch" self.log.info("baseurl = '%s'", baseurl) repo = { 'name': 'atomic-reactor-koji-plugin-%s' % self.target, 'baseurl': baseurl, 'enabled': 1, 'gpgcheck': 0, } # yum doesn't accept a certificate path in sslcacert - it requires a db with added cert # dnf ignores that option completely # we have to fall back to sslverify=0 everytime we get https repo from brew so we'll surely # be able to pull from it if baseurl.startswith("https://"): self.log.info("Ignoring certificates in the repo") repo['sslverify'] = 0 if proxy: self.log.info("Setting yum proxy to %s", proxy) repo['proxy'] = proxy yum_repo = YumRepo(os.path.join(YUM_REPOS_DIR, self.target)) path = yum_repo.dst_filename self.log.info("yum repo of koji target: '%s'", path) yum_repo.content = render_yum_repo(repo, escape_dollars=False) for platform in self.platforms: self.yum_repos[platform].append(yum_repo) def _inject_into_repo_files(self, build_dir: BuildDir): """Inject repo files into a relative directory inside the build context""" host_repos_path = build_dir.path / RELATIVE_REPOS_PATH self.log.info("creating directory for yum repos: %s", host_repos_path) os.mkdir(host_repos_path) allow_repo_dir_in_dockerignore(build_dir.path) for repo in self.yum_repos[build_dir.platform]: # Update every repo accordingly in a repofile # input_buf ---- updated ----> updated_buf with StringIO(repo.content.decode()) as input_buf, StringIO( ) as updated_buf: for line in input_buf: updated_buf.write(line) # Apply sslcacert to every repo in a repofile if line.lstrip().startswith( '[') and self._builder_ca_bundle: updated_buf.write( f'sslcacert=/tmp/{self._ca_bundle_pem}\n') yum_repo = YumRepo(repourl=repo.dst_filename, content=updated_buf.getvalue(), dst_repos_dir=host_repos_path, add_hash=False) yum_repo.write_content() def _inject_into_dockerfile(self, build_dir: BuildDir): build_dir.dockerfile.add_lines("ADD %s* %s" % (RELATIVE_REPOS_PATH, YUM_REPOS_DIR), all_stages=True, at_start=True, skip_scratch=True) if self._builder_ca_bundle: shutil.copyfile(self._builder_ca_bundle, build_dir.path / self._ca_bundle_pem) build_dir.dockerfile.add_lines( f'ADD {self._ca_bundle_pem} /tmp/{self._ca_bundle_pem}', all_stages=True, at_start=True, skip_scratch=True) if not self.workflow.data.dockerfile_images.base_from_scratch: build_dir.dockerfile.add_lines( *self._cleanup_lines(build_dir.platform)) def run(self): """ run the plugin """ if not self.workflow.data.dockerfile_images: self.log.info( "Skipping plugin, from scratch stage(s) can't add repos") return if self.include_koji_repo: self.add_koji_repo() else: self.log.info( "'include_koji_repo parameter is set to '%s', not including koji repo", self.include_koji_repo) if self.repourls and not is_scratch_build(self.workflow): self.validate_yum_repo_files_url() fetched_yum_repos = {} for platform in self.platforms: for repourl in self.repourls.get(platform, []): if repourl in fetched_yum_repos: yum_repo = fetched_yum_repos[repourl] self.yum_repos[platform].append(yum_repo) continue yum_repo = YumRepo(repourl) self.log.info("fetching yum repo from '%s'", yum_repo.repourl) try: yum_repo.fetch() except Exception as e: msg = "Failed to fetch yum repo {repo}: {exc}".format( repo=yum_repo.repourl, exc=e) raise RuntimeError(msg) from e else: self.log.info("fetched yum repo from '%s'", yum_repo.repourl) if self.inject_proxy: if yum_repo.is_valid(): yum_repo.set_proxy_for_all_repos(self.inject_proxy) self.log.debug("saving yum repo '%s', length %d", yum_repo.dst_filename, len(yum_repo.content)) self.yum_repos[platform].append(yum_repo) fetched_yum_repos[repourl] = yum_repo if not self.yum_repos: return self._builder_ca_bundle = self.workflow.conf.builder_ca_bundle if self._builder_ca_bundle: self._ca_bundle_pem = os.path.basename(self._builder_ca_bundle) self.workflow.build_dir.for_each_platform(self._inject_into_repo_files) self.workflow.build_dir.for_each_platform(self._inject_into_dockerfile) for platform in self.platforms: for repo in self.yum_repos[platform]: self.log.info("injected yum repo: %s for '%s' platform", repo.dst_filename, platform)
class ResolveComposesPlugin(PreBuildPlugin): """Request a new, or use existing, ODCS compose This plugin will read the configuration in git repository and request ODCS to create a corresponding yum repository. """ key = PLUGIN_RESOLVE_COMPOSES_KEY is_allowed_to_fail = False args_from_user_params = util.map_to_user_params( "koji_target", "signing_intent", "compose_ids", "repourls:yum_repourls", ) def __init__(self, workflow, koji_target=None, signing_intent=None, compose_ids=tuple(), repourls=None, minimum_time_to_expire=MINIMUM_TIME_TO_EXPIRE): """ :param workflow: DockerBuildWorkflow instance :param koji_target: str, koji target contains build tag to be used when requesting compose from "tag" :param signing_intent: override the signing intent from git repo configuration :param compose_ids: use the given compose_ids instead of requesting a new one :param repourls: list of str, URLs to the repo files :param minimum_time_to_expire: int, used in deciding when to extend compose's time to expire in seconds """ super(ResolveComposesPlugin, self).__init__(workflow) if signing_intent and compose_ids: raise ValueError('signing_intent and compose_ids cannot be used at the same time') self.signing_intent = signing_intent self.compose_ids = compose_ids self.koji_target = koji_target self.minimum_time_to_expire = minimum_time_to_expire self._koji_session = None self._odcs_client = None self.odcs_config = None self.compose_config = None self.composes_info = None self._parent_signing_intent = None self.repourls = repourls or [] self.has_complete_repos = len(self.repourls) > 0 self.plugin_result = self.workflow.data.prebuild_results.get(PLUGIN_KOJI_PARENT_KEY) self.all_compose_ids = list(self.compose_ids) self.new_compose_ids = [] self.parent_compose_ids = [] self.include_koji_repo = False self.yum_repourls = defaultdict(list) self.architectures = get_platforms(self.workflow.data) def run(self): if self.allow_inheritance(): self.adjust_for_inherit() self.workflow.data.all_yum_repourls = self.repourls try: self.read_configs() except SkipResolveComposesPlugin as abort_exc: self.log.info('Aborting plugin execution: %s', abort_exc) for arch in self.architectures: self.yum_repourls[arch].extend(self.repourls) return self.make_result() self.adjust_compose_config() self.request_compose_if_needed() try: self.wait_for_composes() except WaitComposeToFinishTimeout as e: self.log.info(str(e)) for compose_id in self.new_compose_ids: if self.odcs_client.get_compose_status(compose_id) in ['wait', 'generating']: self.log.info('Canceling the compose %s', compose_id) self.odcs_client.cancel_compose(compose_id) else: self.log.info('The compose %s is not in progress, skip canceling', compose_id) raise self.resolve_signing_intent() self.forward_composes() return self.make_result() def allow_inheritance(self): """Returns boolean if composes can be inherited""" if not self.workflow.source.config.inherit: return False self.log.info("Inheritance requested in container.yaml file") if is_scratch_build(self.workflow) or is_isolated_build(self.workflow): msg = ("'inherit: true' in the compose section of container.yaml " "is not allowed for scratch or isolated builds. " "Skipping inheritance.") self.log.warning(msg) self.log.user_warning(message=msg) return False return True def adjust_for_inherit(self): if self.workflow.data.dockerfile_images.base_from_scratch: self.log.debug('This is a base image based on scratch. ' 'Skipping adjusting composes for inheritance.') return if not self.plugin_result: return build_info = self.plugin_result[BASE_IMAGE_KOJI_BUILD] parent_repourls = [] try: self.parent_compose_ids = build_info['extra']['image']['odcs']['compose_ids'] except (KeyError, TypeError): self.log.debug('Parent koji build, %s(%s), does not define compose_ids.' 'Cannot add compose_ids for inheritance from parent.', build_info['nvr'], build_info['id']) try: parent_repourls = build_info['extra']['image']['yum_repourls'] except (KeyError, TypeError): self.log.debug('Parent koji build, %s(%s), does not define yum_repourls. ' 'Cannot add yum_repourls for inheritance from parent.', build_info['nvr'], build_info['id']) all_compose_ids = set(self.compose_ids) original_compose_ids = deepcopy(all_compose_ids) all_compose_ids.update(self.parent_compose_ids) self.all_compose_ids = list(all_compose_ids) for compose_id in all_compose_ids: if compose_id not in original_compose_ids: self.log.info('Inheriting compose id %s', compose_id) all_yum_repos = set(self.repourls) original_yum_repos = deepcopy(all_yum_repos) all_yum_repos.update(parent_repourls) self.repourls = list(all_yum_repos) for repo in all_yum_repos: if repo not in original_yum_repos: self.log.info('Inheriting yum repo %s', repo) if len(parent_repourls) > 0: self.has_complete_repos = True def read_configs(self): self.odcs_config = self.workflow.conf.odcs_config if not self.odcs_config: raise SkipResolveComposesPlugin('ODCS config not found') data = self.workflow.source.config.compose if not data and not self.all_compose_ids: raise SkipResolveComposesPlugin('"compose" config not set and compose_ids not given') pulp_data = util.read_content_sets(self.workflow) or {} platforms = get_platforms(self.workflow.data) if platforms: platforms = sorted(platforms) # sorted to keep predictable for tests koji_tag = None if self.koji_target: target_info = self.koji_session.getBuildTarget(self.koji_target, strict=True) koji_tag = target_info['build_tag_name'] self.compose_config = ComposeConfig(data, pulp_data, self.odcs_config, koji_tag=koji_tag, arches=platforms) def adjust_compose_config(self): if self.signing_intent: self.compose_config.set_signing_intent(self.signing_intent) self.adjust_signing_intent_from_parent() def adjust_signing_intent_from_parent(self): if self.workflow.data.dockerfile_images.base_from_scratch: self.log.debug('This is a base image based on scratch. ' 'Signing intent will not be adjusted for it.') return if not self.plugin_result: self.log.debug("%s plugin didn't run, signing intent will not be adjusted", PLUGIN_KOJI_PARENT_KEY) return build_info = self.plugin_result[BASE_IMAGE_KOJI_BUILD] try: parent_signing_intent_name = build_info['extra']['image']['odcs']['signing_intent'] except (KeyError, TypeError): self.log.debug('Parent koji build, %s(%s), does not define signing_intent. ' 'Cannot adjust for current build.', build_info['nvr'], build_info['id']) return self._parent_signing_intent = (self.odcs_config .get_signing_intent_by_name(parent_signing_intent_name)) current_signing_intent = self.compose_config.signing_intent # Calculate the least restrictive signing intent new_signing_intent = min(self._parent_signing_intent, current_signing_intent, key=lambda x: x['restrictiveness']) if new_signing_intent != current_signing_intent: self.log.info('Signing intent downgraded to "%s" to match Koji parent build', new_signing_intent['name']) self.compose_config.set_signing_intent(new_signing_intent['name']) def request_compose_if_needed(self): if self.compose_ids: self.log.debug('ODCS compose not requested, using given compose IDs') return if not self.workflow.source.config.compose: self.log.debug('ODCS compose not provided, using parents compose IDs') return self.compose_config.validate_for_request() for compose_request in self.compose_config.render_requests(): compose_info = self.odcs_client.start_compose(**compose_request) self.new_compose_ids.append(compose_info['id']) self.all_compose_ids.extend(self.new_compose_ids) def wait_for_composes(self): self.log.debug('Waiting for ODCS composes to be available: %s', self.all_compose_ids) self.composes_info = [] for compose_id in self.all_compose_ids: compose_info = self.odcs_client.wait_for_compose(compose_id) if self._needs_renewal(compose_info): sigkeys = compose_info.get('sigkeys', '').split() updated_signing_intent = self.odcs_config.get_signing_intent_by_keys(sigkeys) if set(sigkeys) != set(updated_signing_intent['keys']): self.log.info('Updating signing keys in "%s" from "%s", to "%s" in compose ' '"%s" due to sigkeys deprecation', updated_signing_intent['name'], sigkeys, updated_signing_intent['keys'], compose_info['id'] ) sigkeys = updated_signing_intent['keys'] compose_info = self.odcs_client.renew_compose(compose_id, sigkeys) compose_id = compose_info['id'] self.new_compose_ids.append(compose_id) compose_info = self.odcs_client.wait_for_compose(compose_id) self.composes_info.append(compose_info) # A module compose is not standalone - it depends on packages from the # virtual platform module - if no extra repourls or other composes are # provided, we'll need packages from the target build tag using the # 'koji' plugin. # We assume other types of composes might provide all the packages needed - # though we don't really know that for sure - a compose with packages # listed might list all the packages that are needed, or might also require # packages from some other source. if compose_info['source_type'] != 2: # PungiSourceType.MODULE self.has_complete_repos = True self.all_compose_ids = [item['id'] for item in self.composes_info] def _needs_renewal(self, compose_info): if compose_info['state_name'] == 'removed': return True time_to_expire = datetime.strptime(compose_info['time_to_expire'], ODCS_DATETIME_FORMAT) now = datetime.utcnow() seconds_left = (time_to_expire - now).total_seconds() return seconds_left <= self.minimum_time_to_expire def resolve_signing_intent(self): """Determine the correct signing intent Regardless of what was requested, or provided as signing_intent plugin parameter, consult sigkeys of the actual composes used to guarantee information accuracy. """ all_signing_intents = [ self.odcs_config.get_signing_intent_by_keys(compose_info.get('sigkeys', [])) for compose_info in self.composes_info ] # Because composes_info may contain composes that were passed as # plugin parameters, add the parent signing intent to avoid the # overall signing intent from surpassing parent's. if self._parent_signing_intent: all_signing_intents.append(self._parent_signing_intent) # Calculate the least restrictive signing intent signing_intent = min(all_signing_intents, key=lambda x: x['restrictiveness']) self.log.info('Signing intent for build is %s', signing_intent['name']) self.compose_config.set_signing_intent(signing_intent['name']) def forward_composes(self): for compose_info in self.composes_info: result_repofile = compose_info['result_repofile'] try: arches = compose_info['arches'] except KeyError: self.yum_repourls['noarch'].append(result_repofile) else: for arch in arches.split(): self.yum_repourls[arch].append(result_repofile) # we should almost never have a None entry from composes, # but we can have yum_repos added, so if we do, we need to merge # it with all other repos. self.yum_repourls['noarch'].extend(self.repourls) if 'noarch' in self.yum_repourls: noarch_repos = self.yum_repourls.pop('noarch') for arch in self.yum_repourls: self.yum_repourls[arch].extend(noarch_repos) # If we don't think the set of packages available from the user-supplied repourls, # inherited repourls, and composed repositories is complete, set the 'include_koji_repo' # kwarg so that the so that the 'yum_repourls' kwarg that we just set doesn't # result in the 'koji' plugin being omitted. if not self.has_complete_repos: self.include_koji_repo = True def make_result(self): signing_intent = None signing_intent_overridden = False if self.compose_config: signing_intent = self.compose_config.signing_intent['name'] signing_intent_overridden = self.compose_config.has_signing_intent_changed() result = { 'composes': self.composes_info, 'yum_repourls': self.yum_repourls, 'include_koji_repo': self.include_koji_repo, 'signing_intent': signing_intent, 'signing_intent_overridden': signing_intent_overridden, } self.log.debug('plugin result: %s', result) return result @property def odcs_client(self): if not self._odcs_client: self._odcs_client = get_odcs_session(self.workflow.conf) return self._odcs_client @property def koji_session(self): if not self._koji_session: self._koji_session = get_koji_session(self.workflow.conf) return self._koji_session