def find_module_built_rpms(modules_nsvc):
"""Find out built RPMs of given modules
:param modules_nsvc: a list of modules' NSVC to find out built RPMs for
each of them.
:type modules_nsvc: list[str]
:return: a sorted list of RPMs, each of them is represented as NEVR.
:rtype: list[str]
"""
import kobo.rpmlib
resolver = GenericResolver.create(db_session, conf)
built_rpms = []
koji_session = get_session(conf, login=False)
for nsvc in modules_nsvc:
name, stream, version, context = nsvc.split(":")
module = resolver.get_module(name,
stream,
version,
context,
strict=True)
rpms = koji_session.listTaggedRPMS(module["koji_tag"], latest=True)[0]
built_rpms.extend(
kobo.rpmlib.make_nvr(rpm, force_epoch=True) for rpm in rpms)
# In case there is duplicate NEVRs, ensure every NEVR is unique in the final list.
# And, sometimes, sorted list of RPMs would be easier to read.
return sorted(set(built_rpms))
def create(cls, db_session, owner, module, backend, config, **extra):
"""
:param db_session: SQLAlchemy session object.
:param owner: a string representing who kicked off the builds
:param module: module_build_service.common.models.ModuleBuild instance.
:param backend: a string representing backend e.g. 'koji'
:param config: instance of module_build_service.common.config.Config
Any additional arguments are optional extras which can be passed along
and are implementation-dependent.
"""
# check if the backend is within allowed backends for the used resolver
resolver = GenericResolver.create(db_session, conf)
if not resolver.is_builder_compatible(backend):
raise ValueError(
"Builder backend '{}' is not compatible with resolver backend '{}'. Check your "
"configuration.".format(backend, resolver.backend))
if backend in GenericBuilder.backends:
return GenericBuilder.backends[backend](db_session=db_session,
owner=owner,
module=module,
config=config,
**extra)
else:
raise ValueError("Builder backend='%s' not recognized" % backend)
def resolve_base_module_virtual_streams(db_session, name, streams):
"""
Resolve any base module virtual streams and return a copy of `streams` with the resolved values.
:param str name: the module name
:param str streams: the streams to resolve
:return: the resolved streams
:rtype: list
"""
from module_build_service.resolver import GenericResolver
resolver = GenericResolver.create(db_session, conf)
if name not in conf.base_module_names:
return streams
new_streams = copy.deepcopy(streams)
for i, stream in enumerate(streams):
# Ignore streams that start with a minus sign, since those are handled in the
# MSE code
if stream.startswith("-"):
continue
# Check if the base module stream is available
log.debug('Checking to see if the base module "%s:%s" is available', name, stream)
if resolver.get_module_count(name=name, stream=stream) > 0:
continue
# If the base module stream is not available, check if there's a virtual stream
log.debug(
'Checking to see if there is a base module "%s" with the virtual stream "%s"',
name, stream,
)
base_module_mmd = resolver.get_latest_with_virtual_stream(
name=name, virtual_stream=stream
)
if not base_module_mmd:
# If there isn't this base module stream or virtual stream available, skip it,
# and let the dep solving code deal with it like it normally would
log.warning(
'There is no base module "%s" with stream/virtual stream "%s"',
name, stream,
)
continue
latest_stream = base_module_mmd.get_stream_name()
log.info(
'Replacing the buildrequire "%s:%s" with "%s:%s", since "%s" is a virtual '
"stream",
name, stream, name, latest_stream, stream
)
new_streams[i] = latest_stream
return new_streams
def record_filtered_rpms(mmd):
"""Record filtered RPMs that should not be installed into buildroot
These RPMs are filtered:
* Reads the mmd["xmd"]["buildrequires"] and extends it with "filtered_rpms"
list containing the NVRs of filtered RPMs in a buildrequired module.
:param Modulemd mmd: Modulemd that will be built next.
:rtype: Modulemd.Module
:return: Modulemd extended with the "filtered_rpms" in XMD section.
"""
# Imported here to allow import of utils in GenericBuilder.
from module_build_service.builder import GenericBuilder
from module_build_service.resolver import GenericResolver
resolver = GenericResolver.create(db_session, conf)
builder = GenericBuilder.backends[conf.system]
new_buildrequires = {}
for req_name, req_data in mmd.get_xmd()["mbs"]["buildrequires"].items():
# In case this is module resubmit or local build, the filtered_rpms
# will already be there, so there is no point in generating them again.
if "filtered_rpms" in req_data:
new_buildrequires[req_name] = req_data
continue
# We can just get the first modulemd data from result right here thanks to
# strict=True, so in case the module cannot be found, get_module_modulemds
# raises an exception.
req_mmd = resolver.get_module_modulemds(req_name, req_data["stream"],
req_data["version"],
req_data["context"], True)[0]
# Find out the particular NVR of filtered packages
filtered_rpms = []
rpm_filter = req_mmd.get_rpm_filters()
if rpm_filter:
built_nvrs = builder.get_built_rpms_in_module_build(req_mmd)
for nvr in built_nvrs:
parsed_nvr = kobo.rpmlib.parse_nvr(nvr)
if parsed_nvr["name"] in rpm_filter:
filtered_rpms.append(nvr)
req_data["filtered_rpms"] = filtered_rpms
new_buildrequires[req_name] = req_data
# Replace the old buildrequires with new ones.
xmd = mmd.get_xmd()
xmd["mbs"]["buildrequires"] = new_buildrequires
mmd.set_xmd(xmd)
return mmd
def get_modulemds_from_ursine_content(tag):
"""Get all modules metadata which were added to ursine content
Ursine content is the tag inheritance managed by Ursa-Major by adding
specific modules' koji_tag.
Background of module build based on ursine content:
Each module build buildrequires a platform module, which is a presudo-module
used to connect to an external repository whose packages will be present
in the buildroot. In practice, the external repo is generated from a build
tag which could inherit from a few module koji_tags so that those module's
RPMs could be build dependencies for some specific packages.
So, this function is to find out all module koji_tags from the build tag
and return corresponding module metadata.
:param str tag: a base module's koji_tag.
:return: list of module metadata. Empty list will be returned if no ursine
modules metadata is found.
:rtype: list[Modulemd.Module]
"""
resolver = GenericResolver.create(db_session, conf)
koji_session = get_session(conf, login=False)
repos = koji_session.getExternalRepoList(tag)
build_tags = find_build_tags_from_external_repos(koji_session, repos)
if not build_tags:
log.debug("No external repo containing ursine content is found.")
return []
modulemds = []
for tag in build_tags:
koji_tags = find_module_koji_tags(koji_session, tag)
for koji_tag in koji_tags:
md = resolver.get_modulemd_by_koji_tag(koji_tag)
if md:
modulemds.append(md)
else:
log.warning("No module is found by koji_tag '%s'", koji_tag)
return modulemds
def default_buildroot_groups(cls, db_session, module):
try:
mmd = module.mmd()
resolver = GenericResolver.create(db_session, conf)
# Resolve default buildroot groups using the MBS, but only for
# non-local modules.
groups = resolver.resolve_profiles(mmd,
("buildroot", "srpm-buildroot"))
groups = {
"build": groups["buildroot"],
"srpm-build": groups["srpm-buildroot"]
}
except ValueError:
reason = "Failed to gather buildroot groups from SCM."
log.exception(reason)
module.transition(db_session,
conf,
state=BUILD_STATES["failed"],
state_reason=reason,
failure_type="user")
db_session.commit()
raise
return groups
def get_reusable_module(module):
"""
Returns previous module build of the module `module` in case it can be
used as a source module to get the components to reuse from.
In case there is no such module, returns None.
:param module: the ModuleBuild object of module being built.
:return: ModuleBuild object which can be used for component reuse.
"""
if module.reused_module:
return module.reused_module
mmd = module.mmd()
previous_module_build = None
# The `base_mmds` will contain the list of base modules against which the possible modules
# to reuse are built. There are three options how these base modules are found:
#
# 1) The `conf.allow_only_compatible_base_modules` is False. This means that MBS should
# not try to find any compatible base modules in its DB and simply use the buildrequired
# base module as it is.
# 2) The `conf.allow_only_compatible_base_modules` is True and DBResolver is used. This means
# that MBS should try to find the compatible modules using its database.
# The `get_base_module_mmds` finds out the list of compatible modules and returns mmds of
# all of them.
# 3) The `conf.allow_only_compatible_base_modules` is True and KojiResolver is used. This
# means that MBS should *not* try to find any compatible base modules in its DB, but
# instead just query Koji using KojiResolver later to find out the module to
# reuse. The list of compatible base modules is defined by Koji tag inheritance directly
# in Koji.
# The `get_base_module_mmds` in this case returns just the buildrequired base module.
if conf.allow_only_compatible_base_modules:
log.debug("Checking for compatible base modules")
base_mmds = get_base_module_mmds(db_session, mmd)["ready"]
# Sort the base_mmds based on the stream version, higher version first.
base_mmds.sort(key=lambda mmd: models.ModuleBuild.get_stream_version(
mmd.get_stream_name(), False),
reverse=True)
else:
log.debug("Skipping the check for compatible base modules")
base_mmds = []
for br in module.buildrequires:
if br.name in conf.base_module_names:
base_mmds.append(br.mmd())
for base_mmd in base_mmds:
previous_module_build = (db_session.query(
models.ModuleBuild).filter_by(
name=mmd.get_module_name()).filter_by(
stream=mmd.get_stream_name()).filter_by(
state=models.BUILD_STATES["ready"]).filter(
models.ModuleBuild.scmurl.isnot(None)).order_by(
models.ModuleBuild.time_completed.desc()))
koji_resolver_enabled = base_mmd.get_xmd().get(
"mbs", {}).get("koji_tag_with_modules")
if koji_resolver_enabled:
# Find ModuleBuilds tagged in the Koji tag using KojiResolver.
resolver = GenericResolver.create(db_session, conf, backend="koji")
possible_modules_to_reuse = resolver.get_buildrequired_modules(
module.name, module.stream, base_mmd)
# Limit the query to these modules.
possible_module_ids = [m.id for m in possible_modules_to_reuse]
previous_module_build = previous_module_build.filter(
models.ModuleBuild.id.in_(possible_module_ids))
# Limit the query to modules sharing the same `build_context_no_bms`. That means they
# have the same buildrequirements.
previous_module_build = previous_module_build.filter_by(
build_context_no_bms=module.build_context_no_bms)
else:
# Recompute the build_context with compatible base module stream.
mbs_xmd = mmd.get_xmd()["mbs"]
if base_mmd.get_module_name() not in mbs_xmd["buildrequires"]:
previous_module_build = None
continue
mbs_xmd["buildrequires"][base_mmd.get_module_name()]["stream"] \
= base_mmd.get_stream_name()
build_context = module.calculate_build_context(
mbs_xmd["buildrequires"])
# Limit the query to find only modules sharing the same build_context.
previous_module_build = previous_module_build.filter_by(
build_context=build_context)
# If we are rebuilding with the "changed-and-after" option, then we can't reuse
# components from modules that were built more liberally
if module.rebuild_strategy == "changed-and-after":
previous_module_build = previous_module_build.filter(
models.ModuleBuild.rebuild_strategy.in_(
["all", "changed-and-after"]))
previous_module_build = previous_module_build.first()
if previous_module_build:
break
# The component can't be reused if there isn't a previous build in the done
# or ready state
if not previous_module_build:
log.info("Cannot re-use. %r is the first module build." % module)
return None
module.reused_module_id = previous_module_build.id
db_session.commit()
return previous_module_build
def get_base_module_mmds(db_session, mmd):
"""
Returns list of MMDs of base modules buildrequired by `mmd` including the compatible
old versions of the base module based on the stream version.
:param Modulemd mmd: Input modulemd metadata.
:rtype: dict with lists of Modulemd
:return: Dict with "ready" or "garbage" state name as a key and list of MMDs of base modules
buildrequired by `mmd` as a value.
"""
from module_build_service.common import models
seen = set()
ret = {"ready": [], "garbage": []}
resolver = GenericResolver.create(db_session, conf)
for deps in mmd.get_dependencies():
buildrequires = {
module: deps.get_buildtime_streams(module)
for module in deps.get_buildtime_modules()
}
for name in conf.base_module_names:
if name not in buildrequires.keys():
continue
# Since the query below uses stream_version_lte, we can sort the streams by stream
# version in descending order to not perform unnecessary queries. Otherwise, if the
# streams are f29.1.0 and f29.2.0, then two queries will occur, causing f29.1.0 to be
# returned twice. Only one query for that scenario is necessary.
sorted_desc_streams = sorted(
buildrequires[name],
reverse=True,
key=models.ModuleBuild.get_stream_version)
for stream in sorted_desc_streams:
ns = ":".join([name, stream])
if ns in seen:
continue
# Get the MMD for particular buildrequired name:stream to find out the virtual
# streams according to which we can find the compatible streams later.
# The returned MMDs are all the module builds for name:stream in the highest
# version. Given the base module does not depend on other modules, it can appear
# only in single context and therefore the `mmds` should always contain just
# zero or one module build.
mmds = resolver.get_module_modulemds(name, stream)
if not mmds:
continue
base_mmd = mmds[0]
new_ret = get_compatible_base_module_mmds(resolver,
base_mmd,
ignore_ns=seen)
for state in new_ret.keys():
for mmd_ in new_ret[state]:
mmd_ns = ":".join(
[mmd_.get_module_name(),
mmd_.get_stream_name()])
seen.add(mmd_ns)
ret[state] += new_ret[state]
break
return ret
def _get_mmds_from_requires(
db_session,
requires,
mmds,
recursive=False,
default_streams=None,
raise_if_stream_ambigous=False,
base_module_mmds=None,
):
"""
Helper method for get_mmds_required_by_module_recursively returning
the list of module metadata objects defined by `requires` dict.
:param db_session: SQLAlchemy database session.
:param dict requires: requires or buildrequires in the form {module: [streams]}
:param mmds: Dictionary with already handled name:streams as a keys and lists
of resulting mmds as values.
:param recursive: If True, the requires are checked recursively.
:param dict default_streams: Dict in {module_name: module_stream, ...} format defining
the default stream to choose for module in case when there are multiple streams to
choose from.
:param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case
there are multiple streams for some dependency of module and the module name is not
defined in `default_streams`, so it is not clear which stream should be used.
:param list base_module_mmds: List of modulemd metadata instances. When set, the
returned list contains MMDs build against each base module defined in
`base_module_mmds` list.
:return: Dict with name:stream as a key and list with mmds as value.
"""
default_streams = default_streams or {}
# To be able to call itself recursively, we need to store list of mmds
# we have added to global mmds list in this particular call.
added_mmds = {}
resolver = GenericResolver.create(db_session, conf)
for name, streams in requires.items():
# Base modules are already added to `mmds`.
if name in conf.base_module_names:
continue
streams_to_try = streams
if name in default_streams:
streams_to_try = [default_streams[name]]
elif len(streams_to_try) > 1 and raise_if_stream_ambigous:
raise StreamAmbigous(
"There are multiple streams %r to choose from for module %s." %
(streams_to_try, name))
# For each valid stream, find the last build in a stream and also all
# its contexts and add mmds of these builds to `mmds` and `added_mmds`.
# Of course only do that if we have not done that already in some
# previous call of this method.
for stream in streams:
ns = "%s:%s" % (name, stream)
if ns not in mmds:
mmds[ns] = []
if ns not in added_mmds:
added_mmds[ns] = []
if base_module_mmds:
for base_module_mmd in base_module_mmds:
mmds[ns] += resolver.get_buildrequired_modulemds(
name, stream, base_module_mmd)
else:
mmds[ns] = resolver.get_module_modulemds(name,
stream,
strict=True)
added_mmds[ns] += mmds[ns]
# Get the requires recursively.
if recursive:
for mmd_list in added_mmds.values():
for mmd in mmd_list:
for deps in mmd.get_dependencies():
deps_dict = deps_to_dict(deps, 'runtime')
mmds = _get_mmds_from_requires(
db_session,
deps_dict,
mmds,
True,
base_module_mmds=base_module_mmds)
return mmds
def generate_expanded_mmds(db_session,
mmd,
raise_if_stream_ambigous=False,
default_streams=None):
"""
Returns list with MMDs with buildrequires and requires set according
to module stream expansion rules. These module metadata can be directly
built using MBS.
:param db_session: SQLAlchemy DB session.
:param Modulemd.ModuleStream mmd: Modulemd metadata with original unexpanded module.
:param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case
there are multiple streams for some dependency of module and the module name is not
defined in `default_streams`, so it is not clear which stream should be used.
:param dict default_streams: Dict in {module_name: module_stream, ...} format defining
the default stream to choose for module in case when there are multiple streams to
choose from.
"""
if not default_streams:
default_streams = {}
# Create local copy of mmd, because we will expand its dependencies,
# which would change the module.
current_mmd = mmd.copy()
# MMDResolver expects the input MMD to have no context.
current_mmd.set_context(None)
# Expands the MSE streams. This mainly handles '-' prefix in MSE streams.
expand_mse_streams(db_session, current_mmd, default_streams,
raise_if_stream_ambigous)
# Get the list of all MMDs which this module can be possibly built against
# and add them to MMDResolver.
mmd_resolver = MMDResolver()
mmds_for_resolving = get_mmds_required_by_module_recursively(
db_session, current_mmd, default_streams, raise_if_stream_ambigous)
for m in mmds_for_resolving:
mmd_resolver.add_modules(m)
# Show log.info message with the NSVCs we have added to mmd_resolver.
nsvcs_to_solve = [m.get_nsvc() for m in mmds_for_resolving]
log.info("Starting resolving with following input modules: %r",
nsvcs_to_solve)
# Resolve the dependencies between modules and get the list of all valid
# combinations in which we can build this module.
requires_combinations = mmd_resolver.solve(current_mmd)
log.info("Resolving done, possible requires: %r", requires_combinations)
# This is where we are going to store the generated MMDs.
mmds = []
for requires in requires_combinations:
# Each generated MMD must be new Module object...
mmd_copy = mmd.copy()
xmd = mmd_copy.get_xmd()
# Requires contain the NSVC representing the input mmd.
# The 'context' of this NSVC defines the id of buildrequires/requires
# pair in the mmd.get_dependencies().
dependencies_id = None
# We don't want to depend on ourselves, so store the NSVC of the current_mmd
# to be able to ignore it later.
self_nsvca = None
# Dict to store name:stream pairs from nsvca, so we are able to access it
# easily later.
req_name_stream = {}
# Get the values for dependencies_id, self_nsvca and req_name_stream variables.
for nsvca in requires:
req_name, req_stream, _, req_context, req_arch = nsvca.split(":")
if req_arch == "src":
assert req_name == current_mmd.get_module_name()
assert req_stream == current_mmd.get_stream_name()
assert dependencies_id is None
assert self_nsvca is None
dependencies_id = int(req_context)
self_nsvca = nsvca
continue
req_name_stream[req_name] = req_stream
if dependencies_id is None or self_nsvca is None:
raise RuntimeError("%s:%s not found in requires %r" %
(current_mmd.get_module_name(),
current_mmd.get_stream_name(), requires))
# The name:[streams, ...] pairs do not have to be the same in both
# buildrequires/requires. In case they are the same, we replace the streams
# in requires section with a single stream against which we will build this MMD.
# In case they are not the same, we have to keep the streams as they are in requires
# section. We always replace stream(s) for build-requirement with the one we
# will build this MMD against.
new_deps = Modulemd.Dependencies()
deps = mmd_copy.get_dependencies()[dependencies_id]
deps_requires = deps_to_dict(deps, 'runtime')
deps_buildrequires = deps_to_dict(deps, 'buildtime')
for req_name, req_streams in deps_requires.items():
if req_name not in deps_buildrequires:
# This require is not a buildrequire so just copy this runtime requirement to
# new_dep and don't touch buildrequires
if not req_streams:
new_deps.set_empty_runtime_dependencies_for_module(
req_name)
else:
for req_stream in req_streams:
new_deps.add_runtime_stream(req_name, req_stream)
elif set(req_streams) != set(deps_buildrequires[req_name]):
# Streams in runtime section are not the same as in buildtime section,
# so just copy this runtime requirement to new_dep.
if not req_streams:
new_deps.set_empty_runtime_dependencies_for_module(
req_name)
else:
for req_stream in req_streams:
new_deps.add_runtime_stream(req_name, req_stream)
new_deps.add_buildtime_stream(req_name,
req_name_stream[req_name])
else:
# This runtime requirement has the same streams in both runtime/buildtime
# requires sections, so replace streams in both sections by the one we
# really used in this resolved variant.
new_deps.add_runtime_stream(req_name,
req_name_stream[req_name])
new_deps.add_buildtime_stream(req_name,
req_name_stream[req_name])
# There might be buildrequires which are not in runtime requires list.
# Such buildrequires must be copied to expanded MMD.
for req_name, req_streams in deps_buildrequires.items():
if req_name not in deps_requires:
new_deps.add_buildtime_stream(req_name,
req_name_stream[req_name])
# Set the new dependencies.
mmd_copy.remove_dependencies(deps)
mmd_copy.add_dependencies(new_deps)
# The Modulemd.Dependencies() stores only streams, but to really build this
# module, we need NSVC of buildrequires, so we have to store this data in XMD.
# We also need additional data like for example list of filtered_rpms. We will
# get them using module_build_service.resolver.GenericResolver.resolve_requires,
# so prepare list with NSVCs of buildrequires as an input for this method.
br_list = []
for nsvca in requires:
if nsvca == self_nsvca:
continue
# Remove the arch from nsvca
nsvc = ":".join(nsvca.split(":")[:-1])
br_list.append(nsvc)
# Resolve the buildrequires and store the result in XMD.
if "mbs" not in xmd:
xmd["mbs"] = {}
resolver = GenericResolver.create(db_session, conf)
xmd["mbs"]["buildrequires"] = resolver.resolve_requires(br_list)
xmd["mbs"]["mse"] = True
mmd_copy.set_xmd(xmd)
# Now we have all the info to actually compute context of this module.
context = models.ModuleBuild.contexts_from_mmd(
mmd_to_str(mmd_copy)).context
mmd_copy.set_context(context)
mmds.append(mmd_copy)
return mmds