def _check_buildopts_arches(mmd, arches):
"""
Returns buildopts arches if valid, or otherwise the arches provided.
:param mmd: Module MetaData
:param arches: list of architectures
:return: list of architectures
"""
buildopts = mmd.get_buildopts()
if not buildopts:
return arches
try:
buildopts_arches = buildopts.get_arches()
except AttributeError:
# libmodulemd version < 2.8.3
return arches
# Must be a subset of the input module arches
unsupported_arches = set(buildopts_arches) - set(arches)
if unsupported_arches:
raise ValidationError(
"The following buildopts arches are not supported with these "
"buildrequires: %r" % unsupported_arches)
if buildopts_arches:
log.info(
"Setting build arches of %s to %r based on the buildopts arches." %
(mmd.get_nsvc(), buildopts_arches))
return buildopts_arches
return arches
def retrigger_new_repo_on_failure():
"""
Retrigger failed new repo tasks for module builds in the build state.
The newRepo task may fail for various reasons outside the scope of MBS.
This method will detect this scenario and retrigger the newRepo task
if needed to avoid the module build from being stuck in the "build" state.
"""
if conf.system != "koji":
return
koji_session = get_session(conf)
module_builds = db_session.query(models.ModuleBuild).filter(
models.ModuleBuild.state == models.BUILD_STATES["build"],
models.ModuleBuild.new_repo_task_id.isnot(None),
).all()
for module_build in module_builds:
task_info = koji_session.getTaskInfo(module_build.new_repo_task_id)
if task_info["state"] in [koji.TASK_STATES["CANCELED"], koji.TASK_STATES["FAILED"]]:
log.info(
"newRepo task %s for %r failed, starting another one",
str(module_build.new_repo_task_id), module_build,
)
taginfo = koji_session.getTag(module_build.koji_tag + "-build")
module_build.new_repo_task_id = koji_session.newRepo(taginfo["name"])
db_session.commit()
def nudge_module_builds_in_state(state_name, older_than_minutes):
"""
Finds all the module builds in the `state` with `time_modified` older
than `older_than_minutes` and adds fake MBSModule message to the
work queue.
"""
log.info("Looking for module builds stuck in the %s state", state_name)
builds = models.ModuleBuild.by_state(db_session, state_name)
log.info(" %r module builds in the %s state...", len(builds), state_name)
now = datetime.utcnow()
time_modified_threshold = timedelta(minutes=older_than_minutes)
for build in builds:
# Only give builds a nudge if stuck for more than ten minutes
if (now - build.time_modified) < time_modified_threshold:
continue
# Pretend the build is modified, so we don't tight spin.
build.time_modified = now
db_session.commit()
# Fake a message to kickstart the build anew in the consumer
state = module_build_service.common.models.BUILD_STATES[state_name]
handler = ON_MODULE_CHANGE_HANDLERS[state]
handler.delay("internal:mbs.module.state.change", build.id, state)
def get_module_build_dependencies(build):
"""Used by wait handler to get module's build dependencies
:param build: a module build.
:type build: :class:`ModuleBuild`
:return: the value returned from :meth:`get_module_build_dependencies`
according to the configured resolver.
:rtype: dict[str, Modulemd.Module]
"""
resolver = module_build_service.resolver.GenericResolver.create(
db_session, conf)
if conf.system in ["koji", "test"]:
# For Koji backend, query for the module we are going to
# build to get the koji_tag and deps from it.
log.info("Getting tag for %s:%s:%s", build.name, build.stream,
build.version)
return resolver.get_module_build_dependencies(build.name,
build.stream,
build.version,
build.context,
strict=True)
else:
# In case of non-koji backend, we want to get the dependencies
# of the local module build based on Modulemd.Module, because the
# local build is not stored in the external MBS and therefore we
# cannot query it using the `query` as for Koji below.
return resolver.get_module_build_dependencies(mmd=build.mmd(),
strict=True)
def validate(self, message):
if conf.messaging == "fedmsg":
# If this is a faked internal message, don't bother.
if "event" in message:
log.info("Skipping crypto validation for %r", message)
return
# Otherwise, if it is a real message from the network, pass it
# through crypto validation.
super(MBSConsumer, self).validate(message)
def resolve_profiles(self, mmd, keys):
"""
Returns a dictionary with keys set according the `keys` parameters and values
set to the union of all components defined in all installation profiles matching
the key in all buildrequires. If there are some modules loaded by
load_local_builds(...), these local modules will be considered when returning
the profiles.
:param mmd: Modulemd.ModuleStream instance representing the module
:param keys: list of modulemd installation profiles to include in the result
:return: a dictionary
"""
results = {}
for key in keys:
results[key] = set()
for module_name, module_info in mmd.get_xmd(
)["mbs"]["buildrequires"].items():
local_modules = models.ModuleBuild.local_modules(
self.db_session, module_name, module_info["stream"])
if local_modules:
local_module = local_modules[0]
log.info(
"Using local module {0!r} to resolve profiles.".format(
local_module))
dep_mmd = local_module.mmd()
for key in keys:
profile = dep_mmd.get_profile(key)
if profile:
results[key] |= set(profile.get_rpms())
continue
build = models.ModuleBuild.get_build_from_nsvc(
self.db_session,
module_name,
module_info["stream"],
module_info["version"],
module_info["context"],
state=models.BUILD_STATES["ready"],
)
if not build:
raise UnprocessableEntity(
"The module {}:{}:{}:{} was not found".format(
module_name,
module_info["stream"],
module_info["version"],
module_info["context"],
))
dep_mmd = build.mmd()
# Take note of what rpms are in this dep's profile
for key in keys:
profile = dep_mmd.get_profile(key)
if profile:
results[key] |= set(profile.get_rpms())
# Return the union of all rpms in all profiles of the given keys
return results
def submit_module_build_from_scm(db_session, username, params, allow_local_url=False):
url = params["scmurl"]
branch = params["branch"]
# Translate local paths into file:// URL
if allow_local_url and not _url_check_re.match(url):
log.info("'{}' is not a valid URL, assuming local path".format(url))
url = os.path.abspath(url)
url = "file://" + url
mmd, scm = fetch_mmd(url, branch, allow_local_url)
return submit_module_build(db_session, username, mmd, params)
def resolve_base_module_virtual_streams(db_session, name, streams):
"""
Resolve any base module virtual streams and return a copy of `streams` with the resolved values.
:param str name: the module name
:param str streams: the streams to resolve
:return: the resolved streams
:rtype: list
"""
from module_build_service.resolver import GenericResolver
resolver = GenericResolver.create(db_session, conf)
if name not in conf.base_module_names:
return streams
new_streams = copy.deepcopy(streams)
for i, stream in enumerate(streams):
# Ignore streams that start with a minus sign, since those are handled in the
# MSE code
if stream.startswith("-"):
continue
# Check if the base module stream is available
log.debug('Checking to see if the base module "%s:%s" is available', name, stream)
if resolver.get_module_count(name=name, stream=stream) > 0:
continue
# If the base module stream is not available, check if there's a virtual stream
log.debug(
'Checking to see if there is a base module "%s" with the virtual stream "%s"',
name, stream,
)
base_module_mmd = resolver.get_latest_with_virtual_stream(
name=name, virtual_stream=stream
)
if not base_module_mmd:
# If there isn't this base module stream or virtual stream available, skip it,
# and let the dep solving code deal with it like it normally would
log.warning(
'There is no base module "%s" with stream/virtual stream "%s"',
name, stream,
)
continue
latest_stream = base_module_mmd.get_stream_name()
log.info(
'Replacing the buildrequire "%s:%s" with "%s:%s", since "%s" is a virtual '
"stream",
name, stream, name, latest_stream, stream
)
new_streams[i] = latest_stream
return new_streams
def process_message(self, event_info):
# Choose a handler for this message
handler, build = self._map_message(db_session, event_info)
if handler is None:
log.debug("No event handler associated with msg %s",
event_info["msg_id"])
return
idx = "%s: %s, %s" % (handler.__name__, event_info["event"],
event_info["msg_id"])
if handler is no_op_handler:
log.debug("Handler is NO_OP: %s", idx)
return
if not build:
log.debug("No module associated with msg %s", event_info["msg_id"])
return
MBSConsumer.current_module_build_id = build.id
log.info("Calling %s", idx)
kwargs = event_info.copy()
kwargs.pop("event")
try:
if conf.celery_broker_url:
# handlers are also Celery tasks, when celery_broker_url is configured,
# call "delay" method to run the handlers as Celery async tasks
func = getattr(handler, "delay")
func(**kwargs)
else:
handler(**kwargs)
except Exception as e:
log.exception("Could not process message handler.")
db_session.rollback()
db_session.refresh(build)
build.transition(
db_session,
conf,
state=models.BUILD_STATES["failed"],
state_reason=str(e),
failure_type="infra",
)
db_session.commit()
# Allow caller to do something when error is occurred.
raise
finally:
MBSConsumer.current_module_build_id = None
log.debug("Done with %s", idx)
def cleanup_stale_failed_builds():
"""Does various clean up tasks on stale failed module builds"""
if conf.system != "koji":
return
stale_date = datetime.utcnow() - timedelta(days=conf.cleanup_failed_builds_time)
stale_module_builds = db_session.query(models.ModuleBuild).filter(
models.ModuleBuild.state == models.BUILD_STATES["failed"],
models.ModuleBuild.time_modified <= stale_date,
).all()
if stale_module_builds:
log.info(
"%s stale failed module build(s) will be cleaned up",
len(stale_module_builds)
)
for module in stale_module_builds:
log.info("%r is stale and is being cleaned up", module)
# Find completed artifacts in the stale build
artifacts = [c for c in module.component_builds if c.is_completed]
# If there are no completed artifacts, then there is nothing to tag
if artifacts:
# Set buildroot_connect=False so it doesn't recreate the Koji target and etc.
builder = GenericBuilder.create_from_module(
db_session, module, conf, buildroot_connect=False
)
builder.untag_artifacts([c.nvr for c in artifacts])
# Mark the artifacts as untagged in the database
for c in artifacts:
c.tagged = False
c.tagged_in_final = False
db_session.add(c)
state_reason = (
"The module was garbage collected since it has failed over {0}"
" day(s) ago".format(conf.cleanup_failed_builds_time)
)
module.transition(
db_session,
conf,
models.BUILD_STATES["garbage"],
state_reason=state_reason,
failure_type="user",
)
db_session.add(module)
db_session.commit()
def delete_old_koji_targets():
"""
Deletes targets older than `config.koji_target_delete_time` seconds
from Koji to cleanup after the module builds.
"""
if conf.system != "koji":
return
log.info("Looking for module builds which Koji target can be removed")
now = datetime.utcnow()
koji_session = get_session(conf)
for target in koji_session.getBuildTargets():
module = db_session.query(models.ModuleBuild).filter(
models.ModuleBuild.koji_tag == target["dest_tag_name"],
models.ModuleBuild.name.notin_(conf.base_module_names),
models.ModuleBuild.state.notin_([
models.BUILD_STATES["init"],
models.BUILD_STATES["wait"],
models.BUILD_STATES["build"],
]),
).options(
load_only("time_completed"),
).first()
if module is None:
continue
# Double-check that the target we are going to remove is prefixed
# by our prefix, so we won't remove f26 when there is some garbage
# in DB or Koji.
for allowed_prefix in conf.koji_tag_prefixes:
if target["name"].startswith(allowed_prefix + "-"):
break
else:
log.error("Module %r has Koji target with not allowed prefix.", module)
continue
delta = now - module.time_completed
if delta.total_seconds() > conf.koji_target_delete_time:
log.info("Removing target of module %r", module)
koji_session.deleteBuildTarget(target["id"])
def generate_module_build_koji_tag(build):
"""Used by wait handler to get module build koji tag
:param build: a module build.
:type build: :class:`ModuleBuild`
:return: generated koji tag.
:rtype: str
"""
log.info("Getting tag for %s:%s:%s", build.name, build.stream,
build.version)
if conf.system in ["koji", "test"]:
return KojiModuleBuilder.generate_koji_tag(
build.name,
build.stream,
build.version,
build.context,
scratch=build.scratch,
scratch_id=build.id,
)
else:
return "-".join(["module", build.name, build.stream, build.version])
def process_paused_module_builds():
log.info("Looking for paused module builds in the build state")
if at_concurrent_component_threshold(conf):
log.debug(
"Will not attempt to start paused module builds due to "
"the concurrent build threshold being met"
)
return
ten_minutes = timedelta(minutes=10)
# Check for module builds that are in the build state but don't have any active component
# builds. Exclude module builds in batch 0. This is likely a build of a module without
# components.
module_builds = db_session.query(models.ModuleBuild).filter(
models.ModuleBuild.state == models.BUILD_STATES["build"],
models.ModuleBuild.batch > 0,
).all()
for module_build in module_builds:
now = datetime.utcnow()
# Only give builds a nudge if stuck for more than ten minutes
if (now - module_build.time_modified) < ten_minutes:
continue
# If there are no components in the build state on the module build,
# then no possible event will start off new component builds.
# But do not try to start new builds when we are waiting for the
# repo-regen.
if not module_build.current_batch(koji.BUILD_STATES["BUILDING"]):
# Initialize the builder...
builder = GenericBuilder.create_from_module(
db_session, module_build, conf)
if has_missed_new_repo_message(module_build, builder.koji_session):
log.info(" Processing the paused module build %r", module_build)
start_next_batch_build(conf, module_build, builder)
# Check if we have met the threshold.
if at_concurrent_component_threshold(conf):
break
def poll_greenwave():
"""Polls Greenwave for all builds in done state"""
if greenwave is None:
return
module_builds = db_session.query(models.ModuleBuild).filter_by(
state=models.BUILD_STATES["done"],
scratch=False
).all()
log.info("Checking Greenwave for %d builds", len(module_builds))
for build in module_builds:
if greenwave.check_gating(build):
build.transition(db_session, conf, state=models.BUILD_STATES["ready"])
else:
build.state_reason = "Gating failed (MBS will retry in {0} seconds)".format(
conf.polling_interval
)
if greenwave.error_occurred:
build.state_reason += " (Error occured while querying Greenwave)"
build.time_modified = datetime.utcnow()
db_session.commit()
def sync_koji_build_tags():
"""
Method checking the "tagged" and "tagged_in_final" attributes of
"complete" ComponentBuilds in the current batch of module builds
in "building" state against the Koji.
In case the Koji shows the build as tagged/tagged_in_final,
fake "tagged" message is added to work queue.
"""
if conf.system != "koji":
return
koji_session = get_session(conf, login=False)
threshold = datetime.utcnow() - timedelta(minutes=10)
module_builds = db_session.query(models.ModuleBuild).filter(
models.ModuleBuild.time_modified < threshold,
models.ModuleBuild.state == models.BUILD_STATES["build"]
).all()
for module_build in module_builds:
complete_components = module_build.current_batch(koji.BUILD_STATES["COMPLETE"])
for c in complete_components:
# In case the component is tagged in the build tag and
# also tagged in the final tag (or it is build_time_only
# and therefore should not be tagged in final tag), skip it.
if c.tagged and (c.tagged_in_final or c.build_time_only):
continue
log.info(
"%r: Component %r is complete, but not tagged in the "
"final and/or build tags.",
module_build, c,
)
# Check in which tags the component is tagged.
tag_dicts = koji_session.listTags(c.nvr)
tags = [tag_dict["name"] for tag_dict in tag_dicts]
# If it is tagged in final tag, but MBS does not think so,
# schedule fake message.
if not c.tagged_in_final and module_build.koji_tag in tags:
log.info(
"Apply tag %s to module build %r",
module_build.koji_tag, module_build)
tagged.delay("internal:sync_koji_build_tags", module_build.koji_tag, c.nvr)
# If it is tagged in the build tag, but MBS does not think so,
# schedule fake message.
build_tag = module_build.koji_tag + "-build"
if not c.tagged and build_tag in tags:
log.info(
"Apply build tag %s to module build %r",
build_tag, module_build)
tagged.delay("internal:sync_koji_build_tags", build_tag, c.nvr)
def log_summary():
states = sorted(models.BUILD_STATES.items(), key=operator.itemgetter(1))
for name, code in states:
query = db_session.query(models.ModuleBuild).filter_by(state=code)
count = query.count()
if count:
log.info(" * %s module builds in the %s state", count, name)
if name == "build":
for module_build in query.all():
log.info(" * %r", module_build)
# First batch is number '1'.
for i in range(1, module_build.batch + 1):
n = len([c for c in module_build.component_builds if c.batch == i])
log.info(" * %s components in batch %s", n, i)