def _check_buildopts_arches(mmd, arches): """ Returns buildopts arches if valid, or otherwise the arches provided. :param mmd: Module MetaData :param arches: list of architectures :return: list of architectures """ buildopts = mmd.get_buildopts() if not buildopts: return arches try: buildopts_arches = buildopts.get_arches() except AttributeError: # libmodulemd version < 2.8.3 return arches # Must be a subset of the input module arches unsupported_arches = set(buildopts_arches) - set(arches) if unsupported_arches: raise ValidationError( "The following buildopts arches are not supported with these " "buildrequires: %r" % unsupported_arches) if buildopts_arches: log.info( "Setting build arches of %s to %r based on the buildopts arches." % (mmd.get_nsvc(), buildopts_arches)) return buildopts_arches return arches
def retrigger_new_repo_on_failure(): """ Retrigger failed new repo tasks for module builds in the build state. The newRepo task may fail for various reasons outside the scope of MBS. This method will detect this scenario and retrigger the newRepo task if needed to avoid the module build from being stuck in the "build" state. """ if conf.system != "koji": return koji_session = get_session(conf) module_builds = db_session.query(models.ModuleBuild).filter( models.ModuleBuild.state == models.BUILD_STATES["build"], models.ModuleBuild.new_repo_task_id.isnot(None), ).all() for module_build in module_builds: task_info = koji_session.getTaskInfo(module_build.new_repo_task_id) if task_info["state"] in [koji.TASK_STATES["CANCELED"], koji.TASK_STATES["FAILED"]]: log.info( "newRepo task %s for %r failed, starting another one", str(module_build.new_repo_task_id), module_build, ) taginfo = koji_session.getTag(module_build.koji_tag + "-build") module_build.new_repo_task_id = koji_session.newRepo(taginfo["name"]) db_session.commit()
def nudge_module_builds_in_state(state_name, older_than_minutes): """ Finds all the module builds in the `state` with `time_modified` older than `older_than_minutes` and adds fake MBSModule message to the work queue. """ log.info("Looking for module builds stuck in the %s state", state_name) builds = models.ModuleBuild.by_state(db_session, state_name) log.info(" %r module builds in the %s state...", len(builds), state_name) now = datetime.utcnow() time_modified_threshold = timedelta(minutes=older_than_minutes) for build in builds: # Only give builds a nudge if stuck for more than ten minutes if (now - build.time_modified) < time_modified_threshold: continue # Pretend the build is modified, so we don't tight spin. build.time_modified = now db_session.commit() # Fake a message to kickstart the build anew in the consumer state = module_build_service.common.models.BUILD_STATES[state_name] handler = ON_MODULE_CHANGE_HANDLERS[state] handler.delay("internal:mbs.module.state.change", build.id, state)
def get_module_build_dependencies(build): """Used by wait handler to get module's build dependencies :param build: a module build. :type build: :class:`ModuleBuild` :return: the value returned from :meth:`get_module_build_dependencies` according to the configured resolver. :rtype: dict[str, Modulemd.Module] """ resolver = module_build_service.resolver.GenericResolver.create( db_session, conf) if conf.system in ["koji", "test"]: # For Koji backend, query for the module we are going to # build to get the koji_tag and deps from it. log.info("Getting tag for %s:%s:%s", build.name, build.stream, build.version) return resolver.get_module_build_dependencies(build.name, build.stream, build.version, build.context, strict=True) else: # In case of non-koji backend, we want to get the dependencies # of the local module build based on Modulemd.Module, because the # local build is not stored in the external MBS and therefore we # cannot query it using the `query` as for Koji below. return resolver.get_module_build_dependencies(mmd=build.mmd(), strict=True)
def validate(self, message): if conf.messaging == "fedmsg": # If this is a faked internal message, don't bother. if "event" in message: log.info("Skipping crypto validation for %r", message) return # Otherwise, if it is a real message from the network, pass it # through crypto validation. super(MBSConsumer, self).validate(message)
def resolve_profiles(self, mmd, keys): """ Returns a dictionary with keys set according the `keys` parameters and values set to the union of all components defined in all installation profiles matching the key in all buildrequires. If there are some modules loaded by load_local_builds(...), these local modules will be considered when returning the profiles. :param mmd: Modulemd.ModuleStream instance representing the module :param keys: list of modulemd installation profiles to include in the result :return: a dictionary """ results = {} for key in keys: results[key] = set() for module_name, module_info in mmd.get_xmd( )["mbs"]["buildrequires"].items(): local_modules = models.ModuleBuild.local_modules( self.db_session, module_name, module_info["stream"]) if local_modules: local_module = local_modules[0] log.info( "Using local module {0!r} to resolve profiles.".format( local_module)) dep_mmd = local_module.mmd() for key in keys: profile = dep_mmd.get_profile(key) if profile: results[key] |= set(profile.get_rpms()) continue build = models.ModuleBuild.get_build_from_nsvc( self.db_session, module_name, module_info["stream"], module_info["version"], module_info["context"], state=models.BUILD_STATES["ready"], ) if not build: raise UnprocessableEntity( "The module {}:{}:{}:{} was not found".format( module_name, module_info["stream"], module_info["version"], module_info["context"], )) dep_mmd = build.mmd() # Take note of what rpms are in this dep's profile for key in keys: profile = dep_mmd.get_profile(key) if profile: results[key] |= set(profile.get_rpms()) # Return the union of all rpms in all profiles of the given keys return results
def submit_module_build_from_scm(db_session, username, params, allow_local_url=False): url = params["scmurl"] branch = params["branch"] # Translate local paths into file:// URL if allow_local_url and not _url_check_re.match(url): log.info("'{}' is not a valid URL, assuming local path".format(url)) url = os.path.abspath(url) url = "file://" + url mmd, scm = fetch_mmd(url, branch, allow_local_url) return submit_module_build(db_session, username, mmd, params)
def resolve_base_module_virtual_streams(db_session, name, streams): """ Resolve any base module virtual streams and return a copy of `streams` with the resolved values. :param str name: the module name :param str streams: the streams to resolve :return: the resolved streams :rtype: list """ from module_build_service.resolver import GenericResolver resolver = GenericResolver.create(db_session, conf) if name not in conf.base_module_names: return streams new_streams = copy.deepcopy(streams) for i, stream in enumerate(streams): # Ignore streams that start with a minus sign, since those are handled in the # MSE code if stream.startswith("-"): continue # Check if the base module stream is available log.debug('Checking to see if the base module "%s:%s" is available', name, stream) if resolver.get_module_count(name=name, stream=stream) > 0: continue # If the base module stream is not available, check if there's a virtual stream log.debug( 'Checking to see if there is a base module "%s" with the virtual stream "%s"', name, stream, ) base_module_mmd = resolver.get_latest_with_virtual_stream( name=name, virtual_stream=stream ) if not base_module_mmd: # If there isn't this base module stream or virtual stream available, skip it, # and let the dep solving code deal with it like it normally would log.warning( 'There is no base module "%s" with stream/virtual stream "%s"', name, stream, ) continue latest_stream = base_module_mmd.get_stream_name() log.info( 'Replacing the buildrequire "%s:%s" with "%s:%s", since "%s" is a virtual ' "stream", name, stream, name, latest_stream, stream ) new_streams[i] = latest_stream return new_streams
def process_message(self, event_info): # Choose a handler for this message handler, build = self._map_message(db_session, event_info) if handler is None: log.debug("No event handler associated with msg %s", event_info["msg_id"]) return idx = "%s: %s, %s" % (handler.__name__, event_info["event"], event_info["msg_id"]) if handler is no_op_handler: log.debug("Handler is NO_OP: %s", idx) return if not build: log.debug("No module associated with msg %s", event_info["msg_id"]) return MBSConsumer.current_module_build_id = build.id log.info("Calling %s", idx) kwargs = event_info.copy() kwargs.pop("event") try: if conf.celery_broker_url: # handlers are also Celery tasks, when celery_broker_url is configured, # call "delay" method to run the handlers as Celery async tasks func = getattr(handler, "delay") func(**kwargs) else: handler(**kwargs) except Exception as e: log.exception("Could not process message handler.") db_session.rollback() db_session.refresh(build) build.transition( db_session, conf, state=models.BUILD_STATES["failed"], state_reason=str(e), failure_type="infra", ) db_session.commit() # Allow caller to do something when error is occurred. raise finally: MBSConsumer.current_module_build_id = None log.debug("Done with %s", idx)
def cleanup_stale_failed_builds(): """Does various clean up tasks on stale failed module builds""" if conf.system != "koji": return stale_date = datetime.utcnow() - timedelta(days=conf.cleanup_failed_builds_time) stale_module_builds = db_session.query(models.ModuleBuild).filter( models.ModuleBuild.state == models.BUILD_STATES["failed"], models.ModuleBuild.time_modified <= stale_date, ).all() if stale_module_builds: log.info( "%s stale failed module build(s) will be cleaned up", len(stale_module_builds) ) for module in stale_module_builds: log.info("%r is stale and is being cleaned up", module) # Find completed artifacts in the stale build artifacts = [c for c in module.component_builds if c.is_completed] # If there are no completed artifacts, then there is nothing to tag if artifacts: # Set buildroot_connect=False so it doesn't recreate the Koji target and etc. builder = GenericBuilder.create_from_module( db_session, module, conf, buildroot_connect=False ) builder.untag_artifacts([c.nvr for c in artifacts]) # Mark the artifacts as untagged in the database for c in artifacts: c.tagged = False c.tagged_in_final = False db_session.add(c) state_reason = ( "The module was garbage collected since it has failed over {0}" " day(s) ago".format(conf.cleanup_failed_builds_time) ) module.transition( db_session, conf, models.BUILD_STATES["garbage"], state_reason=state_reason, failure_type="user", ) db_session.add(module) db_session.commit()
def delete_old_koji_targets(): """ Deletes targets older than `config.koji_target_delete_time` seconds from Koji to cleanup after the module builds. """ if conf.system != "koji": return log.info("Looking for module builds which Koji target can be removed") now = datetime.utcnow() koji_session = get_session(conf) for target in koji_session.getBuildTargets(): module = db_session.query(models.ModuleBuild).filter( models.ModuleBuild.koji_tag == target["dest_tag_name"], models.ModuleBuild.name.notin_(conf.base_module_names), models.ModuleBuild.state.notin_([ models.BUILD_STATES["init"], models.BUILD_STATES["wait"], models.BUILD_STATES["build"], ]), ).options( load_only("time_completed"), ).first() if module is None: continue # Double-check that the target we are going to remove is prefixed # by our prefix, so we won't remove f26 when there is some garbage # in DB or Koji. for allowed_prefix in conf.koji_tag_prefixes: if target["name"].startswith(allowed_prefix + "-"): break else: log.error("Module %r has Koji target with not allowed prefix.", module) continue delta = now - module.time_completed if delta.total_seconds() > conf.koji_target_delete_time: log.info("Removing target of module %r", module) koji_session.deleteBuildTarget(target["id"])
def generate_module_build_koji_tag(build): """Used by wait handler to get module build koji tag :param build: a module build. :type build: :class:`ModuleBuild` :return: generated koji tag. :rtype: str """ log.info("Getting tag for %s:%s:%s", build.name, build.stream, build.version) if conf.system in ["koji", "test"]: return KojiModuleBuilder.generate_koji_tag( build.name, build.stream, build.version, build.context, scratch=build.scratch, scratch_id=build.id, ) else: return "-".join(["module", build.name, build.stream, build.version])
def process_paused_module_builds(): log.info("Looking for paused module builds in the build state") if at_concurrent_component_threshold(conf): log.debug( "Will not attempt to start paused module builds due to " "the concurrent build threshold being met" ) return ten_minutes = timedelta(minutes=10) # Check for module builds that are in the build state but don't have any active component # builds. Exclude module builds in batch 0. This is likely a build of a module without # components. module_builds = db_session.query(models.ModuleBuild).filter( models.ModuleBuild.state == models.BUILD_STATES["build"], models.ModuleBuild.batch > 0, ).all() for module_build in module_builds: now = datetime.utcnow() # Only give builds a nudge if stuck for more than ten minutes if (now - module_build.time_modified) < ten_minutes: continue # If there are no components in the build state on the module build, # then no possible event will start off new component builds. # But do not try to start new builds when we are waiting for the # repo-regen. if not module_build.current_batch(koji.BUILD_STATES["BUILDING"]): # Initialize the builder... builder = GenericBuilder.create_from_module( db_session, module_build, conf) if has_missed_new_repo_message(module_build, builder.koji_session): log.info(" Processing the paused module build %r", module_build) start_next_batch_build(conf, module_build, builder) # Check if we have met the threshold. if at_concurrent_component_threshold(conf): break
def poll_greenwave(): """Polls Greenwave for all builds in done state""" if greenwave is None: return module_builds = db_session.query(models.ModuleBuild).filter_by( state=models.BUILD_STATES["done"], scratch=False ).all() log.info("Checking Greenwave for %d builds", len(module_builds)) for build in module_builds: if greenwave.check_gating(build): build.transition(db_session, conf, state=models.BUILD_STATES["ready"]) else: build.state_reason = "Gating failed (MBS will retry in {0} seconds)".format( conf.polling_interval ) if greenwave.error_occurred: build.state_reason += " (Error occured while querying Greenwave)" build.time_modified = datetime.utcnow() db_session.commit()
def sync_koji_build_tags(): """ Method checking the "tagged" and "tagged_in_final" attributes of "complete" ComponentBuilds in the current batch of module builds in "building" state against the Koji. In case the Koji shows the build as tagged/tagged_in_final, fake "tagged" message is added to work queue. """ if conf.system != "koji": return koji_session = get_session(conf, login=False) threshold = datetime.utcnow() - timedelta(minutes=10) module_builds = db_session.query(models.ModuleBuild).filter( models.ModuleBuild.time_modified < threshold, models.ModuleBuild.state == models.BUILD_STATES["build"] ).all() for module_build in module_builds: complete_components = module_build.current_batch(koji.BUILD_STATES["COMPLETE"]) for c in complete_components: # In case the component is tagged in the build tag and # also tagged in the final tag (or it is build_time_only # and therefore should not be tagged in final tag), skip it. if c.tagged and (c.tagged_in_final or c.build_time_only): continue log.info( "%r: Component %r is complete, but not tagged in the " "final and/or build tags.", module_build, c, ) # Check in which tags the component is tagged. tag_dicts = koji_session.listTags(c.nvr) tags = [tag_dict["name"] for tag_dict in tag_dicts] # If it is tagged in final tag, but MBS does not think so, # schedule fake message. if not c.tagged_in_final and module_build.koji_tag in tags: log.info( "Apply tag %s to module build %r", module_build.koji_tag, module_build) tagged.delay("internal:sync_koji_build_tags", module_build.koji_tag, c.nvr) # If it is tagged in the build tag, but MBS does not think so, # schedule fake message. build_tag = module_build.koji_tag + "-build" if not c.tagged and build_tag in tags: log.info( "Apply build tag %s to module build %r", build_tag, module_build) tagged.delay("internal:sync_koji_build_tags", build_tag, c.nvr)
def log_summary(): states = sorted(models.BUILD_STATES.items(), key=operator.itemgetter(1)) for name, code in states: query = db_session.query(models.ModuleBuild).filter_by(state=code) count = query.count() if count: log.info(" * %s module builds in the %s state", count, name) if name == "build": for module_build in query.all(): log.info(" * %r", module_build) # First batch is number '1'. for i in range(1, module_build.batch + 1): n = len([c for c in module_build.component_builds if c.batch == i]) log.info(" * %s components in batch %s", n, i)