def _prepare_file_directory(self): """ Creates a temporary directory that will contain all the files mentioned in the outputs section Returns path to the temporary directory """ prepdir = tempfile.mkdtemp(prefix="koji-cg-import") mmd_path = os.path.join(prepdir, "modulemd.txt") log.info("Writing generic modulemd.yaml to %r" % mmd_path) with open(mmd_path, "w", encoding="utf-8") as mmd_f: mmd_f.write(self._get_fixed_mmd()) mmd_path = os.path.join(prepdir, "modulemd.src.txt") self._download_source_modulemd(self.module.mmd(), mmd_path) for arch in self.arches: mmd_path = os.path.join(prepdir, "modulemd.%s.txt" % arch) log.info("Writing %s modulemd.yaml to %r" % (arch, mmd_path)) mmd = self._finalize_mmd(arch) with open(mmd_path, "w", encoding="utf-8") as mmd_f: mmd_f.write(mmd) log_path = os.path.join(prepdir, "build.log") try: source = build_logs.path(db_session, self.module) log.info("Moving logs from %r to %r" % (source, log_path)) shutil.copy(source, log_path) except IOError as e: log.exception(e) return prepdir
def query_policies(self, return_all=False): """ Query policies to greenwave :param return_all: Return all policies, if False select by subject_type and decision_context :type return_all: bool :return: response :rtype: dict """ response = self._greenwave_query('policies') if return_all: return response try: selective_resp = { "policies": [ pol for pol in response["policies"] if pol["decision_context"] == self.decision_context and pol["subject_type"] == self.subject_type ] } except KeyError: log.exception( "Incorrect greenwave response (Mandatory key is missing)") raise GreenwaveError( "Incorrect greenwave response (Mandatory key is missing)") return selective_resp
def is_released_as_per_schedule(pp_release): """ Check if the specified scheduled task date has been reached. Returns True if it has. """ if not conf.product_pages_schedule_task_name: log.debug(config_msg, "product_pages_schedule_task_name") return False schedule_url = "{}/api/v7/releases/{}/schedule-tasks/?fields=name,date_finish".format( conf.product_pages_url.rstrip("/"), pp_release) try: pp_rv = requests.get(schedule_url, timeout=15) pp_json = pp_rv.json() # Catch requests failures and JSON parsing errors except (requests.exceptions.RequestException, ValueError): log.exception( "The query to the Product Pages at %s failed. Assuming it is not available.", schedule_url, ) return False name = conf.product_pages_schedule_task_name.lower().strip() for task in pp_json: if task['name'].lower().strip() == name: task_date = task['date_finish'] if datetime.strptime(task_date, "%Y-%m-%d").date() >= datetime.utcnow().date(): log.debug( "The task date %s hasn't been reached yet. Not adding a stream suffix.", task_date ) return False return True # Schedule task not available; rely on GA date return False
def start_build_component(db_session, builder, c): """ Submits single component build to builder. Called in thread by QueueBasedThreadPool in continue_batch_build. This function runs inside separate threads that share one SQLAlchemy session object to update a module build state once there is something wrong when one of its components is submitted to Koji to build. """ import koji try: c.task_id, c.state, c.state_reason, c.nvr = builder.build( artifact_name=c.package, source=c.scmurl) except Exception as e: c.state = koji.BUILD_STATES["FAILED"] c.state_reason = "Failed to build artifact %s: %s" % (c.package, str(e)) log.exception(e) with BUILD_COMPONENT_DB_SESSION_LOCK: c.module_build.transition(conf, models.BUILD_STATES["failed"], failure_type="infra") db_session.commit() return if not c.task_id and c.is_building: c.state = koji.BUILD_STATES["FAILED"] c.state_reason = "Failed to build artifact %s: Builder did not return task ID" % (c.package) with BUILD_COMPONENT_DB_SESSION_LOCK: c.module_build.transition(conf, models.BUILD_STATES["failed"], failure_type="infra") db_session.commit() return
def is_released(pp_release, url): """ Check if the stream has been released. Return True if it has. """ try: pp_rv = requests.get(url, timeout=15) pp_json = pp_rv.json() # Catch requests failures and JSON parsing errors except (requests.exceptions.RequestException, ValueError): log.exception( "The query to the Product Pages at %s failed. Assuming it is not yet released.", url, ) return False ga_date = pp_json.get("ga_date") if not ga_date: log.debug("A release date for the release %s could not be determined", pp_release) return False if datetime.strptime(ga_date, "%Y-%m-%d").date() >= datetime.utcnow().date(): log.debug( "The release %s hasn't been released yet. Not adding a stream suffix.", ga_date ) return False return True
def _get_default_modules(stream, default_modules_scm_url): """ Get the base module's default modules. :param str stream: the stream of the base module :param str default_modules_scm_url: the SCM URL to the default modules :return: a dictionary where the keys are default module names and the values are default module streams :rtype: dict :raise RuntimeError: if no default modules can be retrieved for that stream """ scm_obj = scm.SCM(default_modules_scm_url) temp_dir = tempfile.mkdtemp() try: log.debug("Cloning the default modules repo at %s", default_modules_scm_url) scm_obj.clone(temp_dir) log.debug("Checking out the branch %s", stream) try: scm_obj.checkout_ref(stream) except UnprocessableEntity: # If the checkout fails, try seeing if this is a rawhide build. In this case, the branch # should actually be conf.rawhide_branch. The check to see if this is a rawhide build # is done after the first checkout failure for performance reasons, since it avoids an # unnecessary connection and query to Koji. if conf.uses_rawhide: log.debug( "Checking out the branch %s from the default modules repo failed. Trying to " "determine if this stream represents rawhide.", stream, ) if _get_rawhide_version() == stream: log.debug( "The stream represents rawhide, will try checking out %s", conf.rawhide_branch, ) # There's no try/except here because we want the outer except block to # catch this in the event the rawhide branch doesn't exist scm_obj.checkout_ref(conf.rawhide_branch) else: # If it's not a rawhide build, then the branch should have existed raise else: # If it's not a rawhide build, then the branch should have existed raise idx = Modulemd.ModuleIndex.new() idx.update_from_defaults_directory( path=scm_obj.sourcedir, overrides_path=os.path.join(scm_obj.sourcedir, "overrides"), strict=True, ) return idx.get_default_streams() except: # noqa: E722 msg = "Failed to retrieve the default modules" log.exception(msg) raise RuntimeError(msg) finally: shutil.rmtree(temp_dir)
def patch(self, api_version, id): username, groups = module_build_service.web.auth.get_user(request) try: r = json.loads(request.get_data().decode("utf-8")) except Exception: log.exception("Invalid JSON submitted") raise ValidationError("Invalid JSON submitted") if "owner" in r: if conf.no_auth is not True: raise ValidationError( "The request contains 'owner' parameter, however NO_AUTH is not allowed" ) elif username == "anonymous": username = r["owner"] self.check_groups(username, groups) module = models.ModuleBuild.query.filter_by(id=id).first() if not module: raise NotFound("No such module found.") if module.owner != username and not (conf.admin_groups & groups): raise Forbidden( "You are not owner of this build and therefore cannot modify it." ) if not r.get("state"): log.error("Invalid JSON submitted") raise ValidationError("Invalid JSON submitted") state = r["state"] valid_input_states = ("failed", str(models.BUILD_STATES["failed"])) if state not in valid_input_states: raise ValidationError( "An invalid state was submitted. Valid states values are: {}". format(", ".join(valid_input_states))) valid_states_to_cancel = ("build", "init", "wait") module_state_name = models.INVERSE_BUILD_STATES[module.state] if module_state_name not in valid_states_to_cancel: log.error( "The user %s attempted to cancel a build in the %s state", username, module_state_name, ) raise ValidationError( "To cancel a module build, it must be in one of the following states: {}" .format(", ".join(valid_states_to_cancel))) module.transition(db.session, conf, models.BUILD_STATES["failed"], "Canceled by %s." % username) db.session.add(module) db.session.commit() return jsonify(module.extended_json(db.session, True, api_version)), 200
def _dict_from_request(request): if "multipart/form-data" in request.headers.get("Content-Type", ""): data = request.form.to_dict() else: try: data = json.loads(request.get_data().decode("utf-8")) except Exception: log.exception("Invalid JSON submitted") raise ValidationError("Invalid JSON submitted") return data
def process_message(self, event_info): # Choose a handler for this message handler, build = self._map_message(db_session, event_info) if handler is None: log.debug("No event handler associated with msg %s", event_info["msg_id"]) return idx = "%s: %s, %s" % (handler.__name__, event_info["event"], event_info["msg_id"]) if handler is no_op_handler: log.debug("Handler is NO_OP: %s", idx) return if not build: log.debug("No module associated with msg %s", event_info["msg_id"]) return MBSConsumer.current_module_build_id = build.id log.info("Calling %s", idx) kwargs = event_info.copy() kwargs.pop("event") try: if conf.celery_broker_url: # handlers are also Celery tasks, when celery_broker_url is configured, # call "delay" method to run the handlers as Celery async tasks func = getattr(handler, "delay") func(**kwargs) else: handler(**kwargs) except Exception as e: log.exception("Could not process message handler.") db_session.rollback() db_session.refresh(build) build.transition( db_session, conf, state=models.BUILD_STATES["failed"], state_reason=str(e), failure_type="infra", ) db_session.commit() # Allow caller to do something when error is occurred. raise finally: MBSConsumer.current_module_build_id = None log.debug("Done with %s", idx)
def _greenwave_query(self, query_type, payload=None): """ Make a query to greenwave :param query_type: will be part of url :type query_type: str :param payload: request payload used in 'decision' query :type payload: str :return: response :rtype: dict """ query_func = requests.post if payload else requests.get kwargs = { "url": "{0}/{1}".format(self.url, query_type), "timeout": self.timeout } if payload: kwargs["headers"] = {"Content-Type": "application/json"} kwargs["data"] = payload try: response = query_func(**kwargs) except requests.exceptions.Timeout: raise GreenwaveError("Greenwave request timed out") except Exception as exc: error_message = "Unspecified greenwave request error " \ '(original exception was: "{0}")'.format(str(exc)) log.exception(error_message) raise GreenwaveError(error_message) try: resp_json = response.json() except ValueError: log.debug("Greenwave response content (status {0}): {1}".format( response.status_code, response.text)) raise GreenwaveError("Greenwave returned invalid JSON.") log.debug('Query to Greenwave (%s) result: status=%d, content="%s"', kwargs["url"], response.status_code, resp_json) if response.status_code == 200: return resp_json try: err_msg = resp_json["message"] except KeyError: err_msg = response.text raise GreenwaveError( "Greenwave returned {0} status code. Message: {1}".format( response.status_code, err_msg))
def load_mmd(yaml, is_file=False): if not yaml: raise UnprocessableEntity('The input modulemd was empty') target_mmd_version = Modulemd.ModuleStreamVersionEnum.TWO try: if is_file: mmd = Modulemd.ModuleStream.read_file(yaml, True) else: mmd = Modulemd.ModuleStream.read_string(to_text_type(yaml), True) mmd.validate() if mmd.get_mdversion() < target_mmd_version: mmd = mmd.upgrade(target_mmd_version) elif mmd.get_mdversion() > target_mmd_version: log.error("Encountered a modulemd file with the version %d", mmd.get_mdversion()) raise UnprocessableEntity( "The modulemd version cannot be greater than {}".format(target_mmd_version)) except ModuleMDError as e: not_found = False if is_file: error = "The modulemd {} is invalid.".format(os.path.basename(yaml)) if os.path.exists(yaml): with open(yaml, "rt") as yaml_hdl: log.debug("Modulemd that failed to load:\n%s", yaml_hdl.read()) else: not_found = True error = "The modulemd file {} was not found.".format(os.path.basename(yaml)) log.error("The modulemd file %s was not found.", yaml) else: error = "The modulemd is invalid." log.debug("Modulemd that failed to load:\n%s", yaml) if "modulemd-error-quark: " in str(e): error = "{} The error was '{}'.".format( error, str(e).split("modulemd-error-quark: ")[-1]) elif "Unknown ModuleStream version" in str(e): error = ( "{}. The modulemd version can't be greater than {}." .format(error, target_mmd_version) ) elif not_found is False: error = "{} Please verify the syntax is correct.".format(error) log.exception(error) raise UnprocessableEntity(error) return mmd
def get_user_membership(self, uid): """ Gets the group membership of a user :param uid: a string of the uid of the user :return: a list of common names of the posixGroups the user is a member of """ ldap_filter = "(memberUid={0})".format(uid) # Only get the groups in the base container/OU self.connection.search(self.base_dn, ldap_filter, search_scope=ldap3.LEVEL, attributes=["cn"]) groups = self.connection.response try: return [group["attributes"]["cn"][0] for group in groups] except KeyError: log.exception( "The LDAP groups could not be determined based on the search results " 'of "{0}"'.format(str(groups))) return []
def _scm_get_latest(pkg): try: # If the modulemd specifies that the 'f25' branch is what # we want to pull from, we need to resolve that f25 branch # to the specific commit available at the time of # submission (now). repo = pkg.get_repository() ref = pkg.get_ref() log.debug("Getting the commit hash for the ref %s on the repo %s", ref, repo) pkgref = module_build_service.common.scm.SCM(repo).get_latest(ref) except Exception as e: log.exception(e) return { "error": "Failed to get the latest commit for %s#%s" % (pkg.get_repository(), pkg.get_ref()) } return {"pkg_name": pkg.get_name(), "pkg_ref": pkgref, "error": None}
def consume(self, message): monitor.messaging_rx_counter.inc() # Sometimes, the messages put into our queue are artificially put there # by other parts of our own codebase. If they are already abstracted # messages, then just use them as-is. If they are not already # instances of our message abstraction base class, then first transform # them before proceeding. if "event" in message: event_info = message else: try: event_info = self.get_abstracted_event_info(message) self.validate_event(event_info) except IgnoreMessage as e: log.debug(str(e)) return if event_info is None: return # Primary work is done here. try: self.process_message(event_info) monitor.messaging_rx_processed_ok_counter.inc() except sqlalchemy.exc.OperationalError as error: monitor.messaging_rx_failed_counter.inc() if "could not translate host name" in str(error): log.exception( "SQLAlchemy can't resolve DNS records. Scheduling fedmsg-hub to shutdown." ) self.shutdown() else: raise except Exception: monitor.messaging_rx_failed_counter.inc() finally: db_session.remove() if self.stop_condition and self.stop_condition(message): self.shutdown()
def koji_import(self, devel=False): """This method imports given module into the configured koji instance as a content generator based build Raises an exception when error is encountered during import :param bool devel: True if the "-devel" module should be created and imported. The "-devel" module build contains only the RPMs which are normally filtered from the module build. If set to False, normal module build respecting the filters is created and imported. """ self.devel = devel session = get_session(self.config) self._load_koji_tag(session) file_dir = self._prepare_file_directory() metadata = self._get_content_generator_metadata(file_dir) try: serverdir = self._upload_outputs(session, metadata, file_dir) try: build_info = session.CGImport(metadata, serverdir) except koji.GenericError as e: if "Build already exists" not in str(e): raise log.warning("Failed to import content generator") build_info = None if conf.koji_cg_tag_build: self._tag_cg_build() if build_info is not None: log.info("Content generator import done.") log.debug(json.dumps(build_info, sort_keys=True, indent=4)) # Only remove the logs if CG import was successful. If it fails, # then we want to keep them around for debugging. log.info("Removing %r", file_dir) shutil.rmtree(file_dir) except Exception as e: log.exception("Content generator import failed: %s", e) raise e
def get_user_oidc(request): """ Returns the client's username and groups based on the OIDC token provided. """ _load_secrets() if "authorization" not in request.headers: raise Unauthorized("No 'authorization' header found.") header = request.headers["authorization"].strip() prefix = "Bearer " if not header.startswith(prefix): raise Unauthorized("Authorization headers must start with %r" % prefix) token = header[len(prefix):].strip() try: data = _get_token_info(token) except Exception as e: error = "Cannot verify OIDC token: %s" % str(e) log.exception(error) raise Exception(error) if not data or "active" not in data or not data["active"]: raise Unauthorized("OIDC token invalid or expired.") if "OIDC_REQUIRED_SCOPE" not in app.config: raise Forbidden("OIDC_REQUIRED_SCOPE must be set in server config.") presented_scopes = data["scope"].split(" ") required_scopes = [ "openid", "https://id.fedoraproject.org/scope/groups", app.config["OIDC_REQUIRED_SCOPE"], ] for scope in required_scopes: if scope not in presented_scopes: raise Unauthorized("Required OIDC scope %r not present: %r" % (scope, presented_scopes)) try: extended_data = _get_user_info(token) except Exception: error = "OpenIDC auth error: Cannot determine the user's groups" log.exception(error) raise Unauthorized(error) username = data["username"] # If the user is part of the whitelist, then the group membership check is skipped if username in conf.allowed_users: groups = set() else: try: groups = set(extended_data["groups"]) except Exception: error = "Could not find groups in UserInfo from OIDC" log.exception("%s (extended_data: %s)", error, extended_data) raise Unauthorized(error) return username, groups
def default_buildroot_groups(cls, db_session, module): try: mmd = module.mmd() resolver = GenericResolver.create(db_session, conf) # Resolve default buildroot groups using the MBS, but only for # non-local modules. groups = resolver.resolve_profiles(mmd, ("buildroot", "srpm-buildroot")) groups = { "build": groups["buildroot"], "srpm-build": groups["srpm-buildroot"] } except ValueError: reason = "Failed to gather buildroot groups from SCM." log.exception(reason) module.transition(db_session, conf, state=BUILD_STATES["failed"], state_reason=reason, failure_type="user") db_session.commit() raise return groups
def init(msg_id, module_build_id, module_build_state): """Called whenever a module enters the 'init' state. :param str msg_id: the original id of the message being handled, which is received from message bus. :param int module_build_id: the module build id. :param int module_build_state: the module build state. """ build = models.ModuleBuild.get_by_id(db_session, module_build_id) state_init = models.BUILD_STATES["init"] if module_build_state == state_init and build.state != state_init: log.warning("Module build %r has moved to %s state already.", build, models.INVERSE_BUILD_STATES[build.state]) log.warning( "Ignore this message %s. Is there something wrong with the frontend" " that sends duplicate messages?", msg_id) return # for MockModuleBuilder, set build logs dir to mock results dir # before build_logs start if conf.system == "mock": build_tag_name = generate_module_build_koji_tag(build) mock_resultsdir = os.path.join(conf.mock_resultsdir, build_tag_name) if not os.path.exists(mock_resultsdir): os.makedirs(mock_resultsdir) build_logs.build_logs_dir = mock_resultsdir build_logs.start(db_session, build) log.info("Start to handle %s which is in init state.", build.mmd().get_nsvc()) error_msg = "" failure_reason = "unspec" try: mmd = build.mmd() record_module_build_arches(mmd, build) arches = [arch.name for arch in build.arches] defaults_added = add_default_modules(mmd) # Get map of packages that have SRPM overrides srpm_overrides = get_module_srpm_overrides(build) # Format the modulemd by putting in defaults and replacing streams that # are branches with commit hashes format_mmd(mmd, build.scmurl, build, db_session, srpm_overrides) record_component_builds(mmd, build) # The ursine.handle_stream_collision_modules is Koji specific. # It is also run only when Ursa Prime is not enabled for the base # module (`if not defaults_added`). if conf.system in ["koji", "test"] and not defaults_added: handle_stream_collision_modules(mmd) # Sets xmd["mbs"]["ursine_rpms"] with RPMs from the buildrequired base modules which # conflict with the RPMs from other buildrequired modules. This is done to prefer modular # RPMs over base module RPMs even if their NVR is lower. if conf.system in ("koji", "test"): handle_collisions_with_base_module_rpms(mmd, arches) else: log.warning( "The necessary conflicts could not be generated due to RHBZ#1693683. " "Some RPMs from the base modules (%s) may end up being used over modular RPMs. " "This may result in different behavior than a production build.", ", ".join(conf.base_module_names)) mmd = record_filtered_rpms(mmd) build.modulemd = mmd_to_str(mmd) build.transition(db_session, conf, models.BUILD_STATES["wait"]) # Catch custom exceptions that we can expose to the user except (UnprocessableEntity, Forbidden, ValidationError, RuntimeError) as e: log.exception(str(e)) error_msg = str(e) failure_reason = "user" except (xmlrpclib.ProtocolError, koji.GenericError) as e: log.exception(str(e)) error_msg = 'Koji communication error: "{0}"'.format(str(e)) failure_reason = "infra" except Exception as e: log.exception(str(e)) error_msg = "An unknown error occurred while validating the modulemd" failure_reason = "user" else: db_session.add(build) db_session.commit() finally: if error_msg: # Rollback changes underway db_session.rollback() build.transition( db_session, conf, models.BUILD_STATES["failed"], state_reason=error_msg, failure_type=failure_reason, ) db_session.commit()
def _get_rpms_in_external_repo(repo_url, arches, cache_dir_name): """ Get the available RPMs in the external repo for the provided arches. :param str repo_url: the URL of the external repo with the "$arch" variable included :param list arches: the list of arches to query the external repo for :param str cache_dir_name: the cache directory name under f"{conf.cache_dir}/dnf" :return: a set of the RPM NEVRAs :rtype: set :raise RuntimeError: if the cache is not writeable or the external repo couldn't be loaded :raises ValueError: if there is no "$arch" variable in repo URL """ if "$arch" not in repo_url: raise ValueError( "The external repo {} does not contain the $arch variable".format( repo_url)) base = dnf.Base() try: dnf_conf = base.conf # Expire the metadata right away so that when a repo is loaded, it will always check to # see if the external repo has been updated dnf_conf.metadata_expire = 0 cache_location = os.path.join(conf.cache_dir, "dnf", cache_dir_name) try: # exist_ok=True can't be used in Python 2 os.makedirs(cache_location, mode=0o0770) except OSError as e: # Don't fail if the directories already exist if e.errno != errno.EEXIST: log.exception("Failed to create the cache directory %s", cache_location) raise RuntimeError("The MBS cache is not writeable.") # Tell DNF to use the cache directory dnf_conf.cachedir = cache_location # Don't skip repos that can't be synchronized dnf_conf.skip_if_unavailable = False dnf_conf.timeout = conf.dnf_timeout # Get rid of everything to be sure it's a blank slate. This doesn't delete the cached repo # data. base.reset(repos=True, goal=True, sack=True) # Add a separate repo for each architecture for arch in arches: # Convert arch to canon_arch. This handles cases where Koji "i686" arch is mapped to # "i386" when generating RPM repository. canon_arch = koji.canonArch(arch) repo_name = "repo_{}".format(canon_arch) repo_arch_url = repo_url.replace("$arch", canon_arch) base.repos.add_new_repo( repo_name, dnf_conf, baseurl=[repo_arch_url], minrate=conf.dnf_minrate, ) try: # Load the repos in parallel base.update_cache() except dnf.exceptions.RepoError: msg = "Failed to load the external repos" log.exception(msg) raise RuntimeError(msg) # dnf will not always raise an error on repo failures, so we check explicitly for repo_name in base.repos: if not base.repos[repo_name].metadata: msg = "Failed to load metadata for repo %s" % repo_name log.exception(msg) raise RuntimeError(msg) base.fill_sack(load_system_repo=False) # Return all the available RPMs nevras = set() for rpm in base.sack.query().available(): rpm_dict = { "arch": rpm.arch, "epoch": rpm.epoch, "name": rpm.name, "release": rpm.release, "version": rpm.version, } nevra = kobo.rpmlib.make_nvra(rpm_dict, force_epoch=True) nevras.add(nevra) finally: base.close() return nevras
def runtimeerror_error(e): """Flask error handler for RuntimeError exceptions""" log.exception("RuntimeError exception raised") return json_error(500, "Internal Server Error", str(e))
def wait(msg_id, module_build_id, module_build_state): """ Called whenever a module enters the 'wait' state. We transition to this state shortly after a modulebuild is first requested. All we do here is request preparation of the buildroot. The kicking off of individual component builds is handled elsewhere, in module_build_service.schedulers.handlers.repos. :param str msg_id: the original id of the message being handled which is received from the message bus. :param int module_build_id: the module build id. :param int module_build_state: the module build state. """ build = models.ModuleBuild.get_by_id(db_session, module_build_id) log.info("Found build=%r from message" % build) log.debug("%r", build.modulemd) if build.state != module_build_state: log.warning( "Note that retrieved module state %r doesn't match message module state %r", build.state, module_build_state, ) # This is ok.. it's a race condition we can ignore. pass try: build_deps = get_module_build_dependencies(build) except ValueError: reason = "Failed to get module info from MBS. Max retries reached." log.exception(reason) build.transition(db_session, conf, state=models.BUILD_STATES["failed"], state_reason=reason, failure_type="infra") db_session.commit() raise tag = generate_module_build_koji_tag(build) log.debug("Found tag=%s for module %r" % (tag, build)) # Hang on to this information for later. We need to know which build is # associated with which koji tag, so that when their repos are regenerated # in koji we can figure out which for which module build that event is # relevant. log.debug("Assigning koji tag=%s to module build" % tag) build.koji_tag = tag if build.scratch: log.debug( "Assigning Content Generator build koji tag is skipped for scratch module build." ) elif conf.koji_cg_tag_build: cg_build_koji_tag = get_content_generator_build_koji_tag(build_deps) log.debug( "Assigning Content Generator build koji tag=%s to module build", cg_build_koji_tag) build.cg_build_koji_tag = cg_build_koji_tag else: log.debug( "It is disabled to tag module build during importing into Koji by Content Generator." ) log.debug( "Skip to assign Content Generator build koji tag to module build.") builder = GenericBuilder.create_from_module(db_session, build, conf) log.debug( "Adding dependencies %s into buildroot for module %s:%s:%s", build_deps.keys(), build.name, build.stream, build.version, ) builder.buildroot_add_repos(build_deps) if not build.component_builds: log.info("There are no components in module %r, skipping build" % build) build.transition(db_session, conf, state=models.BUILD_STATES["build"]) db_session.add(build) db_session.commit() # Return a KojiRepoChange message so that the build can be transitioned to done # in the repos handler from module_build_service.scheduler.handlers.repos import done as repos_done_handler events.scheduler.add(repos_done_handler, ("fake_msg", builder.module_build_tag["name"])) return # If all components in module build will be reused, we don't have to build # module-build-macros, because there won't be any build done. if attempt_to_reuse_all_components(builder, build): log.info( "All components have been reused for module %r, skipping build" % build) build.transition(db_session, conf, state=models.BUILD_STATES["build"]) db_session.add(build) db_session.commit() return [] log.debug("Starting build batch 1") build.batch = 1 db_session.commit() artifact_name = "module-build-macros" component_build = models.ComponentBuild.from_component_name( db_session, artifact_name, build.id) srpm = builder.get_disttag_srpm(disttag=".%s" % get_rpm_release(db_session, build), module_build=build) if not component_build: component_build = models.ComponentBuild( module_id=build.id, package=artifact_name, format="rpms", scmurl=srpm, batch=1, build_time_only=True, ) db_session.add(component_build) # Commit and refresh so that the SQLAlchemy relationships are available db_session.commit() db_session.refresh(component_build) recovered = builder.recover_orphaned_artifact(component_build) if recovered: log.info("Found an existing module-build-macros build") # There was no existing artifact found, so lets submit the build instead else: task_id, state, reason, nvr = builder.build( artifact_name=artifact_name, source=srpm) component_build.task_id = task_id component_build.state = state component_build.reason = reason component_build.nvr = nvr elif not component_build.is_completed: # It's possible that the build succeeded in the builder but some other step failed which # caused module-build-macros to be marked as failed in MBS, so check to see if it exists # first recovered = builder.recover_orphaned_artifact(component_build) if recovered: log.info("Found an existing module-build-macros build") else: task_id, state, reason, nvr = builder.build( artifact_name=artifact_name, source=srpm) component_build.task_id = task_id component_build.state = state component_build.reason = reason component_build.nvr = nvr db_session.add(component_build) build.transition(db_session, conf, state=models.BUILD_STATES["build"]) db_session.add(build) db_session.commit() # We always have to regenerate the repository. if conf.system == "koji": log.info("Regenerating the repository") task_id = builder.koji_session.newRepo( builder.module_build_tag["name"]) build.new_repo_task_id = task_id db_session.commit() else: from module_build_service.scheduler.handlers.repos import done as repos_done_handler events.scheduler.add(repos_done_handler, ("fake_msg", builder.module_build_tag["name"]))
def _koji_rpms_in_tag(self, tag): """ Return the list of koji rpms in a tag. """ log.debug("Listing rpms in koji tag %s", tag) session = get_session(self.config, login=False) try: rpms, builds = session.listTaggedRPMS(tag, latest=True) except koji.GenericError: log.exception("Failed to list rpms in tag %r", tag) # If the tag doesn't exist.. then there are no rpms in that tag. return [] # Module does not contain any RPM, so return an empty list. if not rpms: return [] # Get the exclusivearch, excludearch and license data for each RPM. # The exclusivearch and excludearch lists are set in source RPM from which the RPM # was built. # Create temporary dict with source RPMs in rpm_id:rpms_list_index format. src_rpms = {} binary_rpms = {} for rpm in rpms: if rpm["arch"] == "src": src_rpms[rpm["id"]] = rpm else: binary_rpms[rpm["id"]] = rpm # Prepare the arguments for Koji multicall. # We will call session.getRPMHeaders(...) for each SRC RPM to get exclusivearch, # excludearch and license headers. multicall_kwargs = [ {"rpmID": rpm_id, "headers": ["exclusivearch", "excludearch", "license"]} for rpm_id in src_rpms.keys() ] # For each binary RPM, we only care about the "license" header. multicall_kwargs += [ {"rpmID": rpm_id, "headers": ["license"]} for rpm_id in binary_rpms.keys() ] rpms_headers = koji_retrying_multicall_map( session, session.getRPMHeaders, list_of_kwargs=multicall_kwargs ) # Temporary dict with build_id as a key to find builds easily. builds = {build["build_id"]: build for build in builds} # Create a mapping of build IDs to SRPM NEVRAs so that the for loop below can directly # access these values when adding the `srpm_nevra` key to the returned RPMs build_id_to_srpm_nevra = { srpm["build_id"]: kobo.rpmlib.make_nvra(srpm, force_epoch=True) for srpm in src_rpms.values() } # Handle the multicall result. For each build associated with the source RPM, # store the exclusivearch and excludearch lists. For each RPM, store the 'license' and # also other useful data from the Build associated with the RPM. for rpm, headers in zip(chain(src_rpms.values(), binary_rpms.values()), rpms_headers): if not headers: raise RuntimeError("No RPM headers received from Koji for RPM %s" % rpm["name"]) if "license" not in headers: raise RuntimeError( "No RPM 'license' header received from Koji for RPM %s" % rpm["name"]) build = builds[rpm["build_id"]] if "exclusivearch" in headers and "excludearch" in headers: build["exclusivearch"] = headers["exclusivearch"] build["excludearch"] = headers["excludearch"] rpm["license"] = headers["license"] rpm["srpm_name"] = build["name"] rpm["srpm_nevra"] = build_id_to_srpm_nevra[rpm["build_id"]] rpm["exclusivearch"] = build["exclusivearch"] rpm["excludearch"] = build["excludearch"] return rpms
def koji_multicall_map(koji_session, koji_session_fnc, list_of_args=None, list_of_kwargs=None): """ Calls the `koji_session_fnc` using Koji multicall feature N times based on the list of arguments passed in `list_of_args` and `list_of_kwargs`. Returns list of responses sorted the same way as input args/kwargs. In case of error, the error message is logged and None is returned. For example to get the package ids of "httpd" and "apr" packages: ids = koji_multicall_map(session, session.getPackageID, ["httpd", "apr"]) # ids is now [280, 632] :param KojiSessions koji_session: KojiSession to use for multicall. :param object koji_session_fnc: Python object representing the KojiSession method to call. :param list list_of_args: List of args which are passed to each call of koji_session_fnc. :param list list_of_kwargs: List of kwargs which are passed to each call of koji_session_fnc. """ if list_of_args is None and list_of_kwargs is None: raise ProgrammingError("One of list_of_args or list_of_kwargs must be set.") if ( type(list_of_args) not in [type(None), list] or type(list_of_kwargs) not in [type(None), list] ): raise ProgrammingError("list_of_args and list_of_kwargs must be list or None.") if list_of_kwargs is None: list_of_kwargs = [{}] * len(list_of_args) if list_of_args is None: list_of_args = [[]] * len(list_of_kwargs) if len(list_of_args) != len(list_of_kwargs): raise ProgrammingError("Length of list_of_args and list_of_kwargs must be the same.") koji_session.multicall = True for args, kwargs in zip(list_of_args, list_of_kwargs): if type(args) != list: args = [args] if type(kwargs) != dict: raise ProgrammingError("Every item in list_of_kwargs must be a dict") koji_session_fnc(*args, **kwargs) try: responses = koji_session.multiCall(strict=True) except Exception: log.exception( "Exception raised for multicall of method %r with args %r, %r:", koji_session_fnc, args, kwargs, ) return None if not responses: log.error("Koji did not return response for multicall of %r", koji_session_fnc) return None if type(responses) != list: log.error( "Fault element was returned for multicall of method %r: %r", koji_session_fnc, responses ) return None results = [] # For the response specification, see # https://web.archive.org/web/20060624230303/http://www.xmlrpc.com/discuss/msgReader$1208?mode=topic # Relevant part of this: # Multicall returns an array of responses. There will be one response for each call in # the original array. The result will either be a one-item array containing the result value, # or a struct of the form found inside the standard <fault> element. for response, args, kwargs in zip(responses, list_of_args, list_of_kwargs): if type(response) == list: if not response: log.error( "Empty list returned for multicall of method %r with args %r, %r", koji_session_fnc, args, kwargs ) return None results.append(response[0]) else: log.error( "Unexpected data returned for multicall of method %r with args %r, %r: %r", koji_session_fnc, args, kwargs, response ) return None return results