def test_set_stream(self, main):
cli_cmd = [
"mbs-manager", "build_module_locally", "--set-stream",
"platform:f28", "--file",
staged_data_filename("testmodule-local-build.yaml")
]
self._run_manager_wrapper(cli_cmd)
# Since module_build_service.scheduler.local.main is mocked, MBS does
# not really build the testmodule for this test. Following lines assert
# the fact:
# Module testmodule-local-build is expanded and stored into database,
# and this build has buildrequires platform:f28 and requires
# platform:f28.
# Please note that, the f28 is specified from command line option
# --set-stream, which is the point this test tests.
builds = db_session.query(
models.ModuleBuild).filter_by(name="testmodule-local-build").all()
assert 1 == len(builds)
testmodule_build = builds[0]
mmd_deps = testmodule_build.mmd().get_dependencies()
deps_dict = deps_to_dict(mmd_deps[0], "buildtime")
assert ["f28"] == deps_dict["platform"]
deps_dict = deps_to_dict(mmd_deps[0], "runtime")
assert ["f28"] == deps_dict["platform"]
def _modify_buildtime_streams(db_session, mmd, new_streams_func):
"""
Modify buildtime streams using the input new_streams_func.
:param Modulemd.ModuleStream mmd: the modulemd to apply the overrides on
:param function new_streams: a function that takes the parameters (module_name, module_streams),
and returns the streams that should be set on the buildtime dependency.
"""
deps = mmd.get_dependencies()
for dep in deps:
overridden = False
brs = deps_to_dict(dep, "buildtime")
# There is no way to replace streams, so create a new Dependencies object that will end up
# being a copy, but with the streams replaced if a virtual stream is detected
new_dep = Modulemd.Dependencies()
for name, streams in brs.items():
new_streams = new_streams_func(db_session, name, streams)
if streams != new_streams:
overridden = True
if not new_streams:
new_dep.set_empty_buildtime_dependencies_for_module(name)
else:
for stream in new_streams:
new_dep.add_buildtime_stream(name, stream)
if overridden:
# Copy the runtime streams as is
reqs = deps_to_dict(dep, "runtime")
for name, streams in reqs.items():
if not streams:
new_dep.set_empty_runtime_dependencies_for_module(name)
else:
for stream in streams:
new_dep.add_runtime_stream(name, stream)
# Replace the old Dependencies object with the new one with the overrides
mmd.remove_dependencies(dep)
mmd.add_dependencies(new_dep)
def _apply_dep_overrides(mmd, params):
"""
Apply the dependency override parameters (if specified) on the input modulemd.
:param Modulemd.ModuleStream mmd: the modulemd to apply the overrides on
:param dict params: the API parameters passed in by the user
:raises ValidationError: if one of the overrides doesn't apply
"""
dep_overrides = {
"buildrequires": copy.copy(params.get("buildrequire_overrides", {})),
"requires": copy.copy(params.get("require_overrides", {})),
}
# Parse the module's branch to determine if it should override the stream of the buildrequired
# module defined in conf.br_stream_override_module
branch_search = None
if params.get("branch") and conf.br_stream_override_module and conf.br_stream_override_regexes:
# Only parse the branch for a buildrequire override if the user didn't manually specify an
# override for the module specified in conf.br_stream_override_module
if not dep_overrides["buildrequires"].get(conf.br_stream_override_module):
branch_search = None
for regex in conf.br_stream_override_regexes:
branch_search = re.search(regex, params["branch"])
if branch_search:
log.debug(
"The stream override regex `%s` matched the branch %s",
regex,
params["branch"],
)
break
else:
log.debug('No stream override regexes matched the branch "%s"', params["branch"])
# If a stream was parsed from the branch, then add it as a stream override for the module
# specified in conf.br_stream_override_module
if branch_search:
# Concatenate all the groups that are not None together to get the desired stream.
# This approach is taken in case there are sections to ignore.
# For instance, if we need to parse `el8.0.0` from `rhel-8.0.0`.
parsed_stream = "".join(group for group in branch_search.groups() if group)
if parsed_stream:
dep_overrides["buildrequires"][conf.br_stream_override_module] = [parsed_stream]
log.info(
'The buildrequired stream of "%s" was overriden with "%s" based on the branch "%s"',
conf.br_stream_override_module, parsed_stream, params["branch"],
)
else:
log.warning(
'The regex `%s` only matched empty capture groups on the branch "%s". The regex is '
" invalid and should be rewritten.",
regex, params["branch"],
)
unused_dep_overrides = {
"buildrequires": set(dep_overrides["buildrequires"].keys()),
"requires": set(dep_overrides["requires"].keys()),
}
deps = mmd.get_dependencies()
for dep in deps:
overridden = False
new_dep = Modulemd.Dependencies()
for dep_type, overrides in dep_overrides.items():
if dep_type == "buildrequires":
mmd_dep_type = "buildtime"
else:
mmd_dep_type = "runtime"
# Get the existing streams
reqs = deps_to_dict(dep, mmd_dep_type)
# Get the method to add a new stream for this dependency type
# (e.g. add_buildtime_stream)
add_func = getattr(new_dep, "add_{}_stream".format(mmd_dep_type))
add_empty_func = getattr(
new_dep, "set_empty_{}_dependencies_for_module".format(mmd_dep_type))
for name, streams in reqs.items():
if name in dep_overrides[dep_type]:
streams_to_add = dep_overrides[dep_type][name]
unused_dep_overrides[dep_type].remove(name)
overridden = True
else:
streams_to_add = reqs[name]
if not streams_to_add:
add_empty_func(name)
else:
for stream in streams_to_add:
add_func(name, stream)
if overridden:
# Set the overridden streams
mmd.remove_dependencies(dep)
mmd.add_dependencies(new_dep)
for dep_type in unused_dep_overrides.keys():
# If a stream override was applied from parsing the branch and it wasn't applicable,
# just ignore it
if branch_search and conf.br_stream_override_module in unused_dep_overrides[dep_type]:
unused_dep_overrides[dep_type].remove(conf.br_stream_override_module)
if unused_dep_overrides[dep_type]:
raise ValidationError(
"The {} overrides for the following modules aren't applicable: {}".format(
dep_type[:-1], ", ".join(sorted(unused_dep_overrides[dep_type])))
)
def _get_mmds_from_requires(
db_session,
requires,
mmds,
recursive=False,
default_streams=None,
raise_if_stream_ambigous=False,
base_module_mmds=None,
):
"""
Helper method for get_mmds_required_by_module_recursively returning
the list of module metadata objects defined by `requires` dict.
:param db_session: SQLAlchemy database session.
:param dict requires: requires or buildrequires in the form {module: [streams]}
:param mmds: Dictionary with already handled name:streams as a keys and lists
of resulting mmds as values.
:param recursive: If True, the requires are checked recursively.
:param dict default_streams: Dict in {module_name: module_stream, ...} format defining
the default stream to choose for module in case when there are multiple streams to
choose from.
:param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case
there are multiple streams for some dependency of module and the module name is not
defined in `default_streams`, so it is not clear which stream should be used.
:param list base_module_mmds: List of modulemd metadata instances. When set, the
returned list contains MMDs build against each base module defined in
`base_module_mmds` list.
:return: Dict with name:stream as a key and list with mmds as value.
"""
default_streams = default_streams or {}
# To be able to call itself recursively, we need to store list of mmds
# we have added to global mmds list in this particular call.
added_mmds = {}
resolver = GenericResolver.create(db_session, conf)
for name, streams in requires.items():
# Base modules are already added to `mmds`.
if name in conf.base_module_names:
continue
streams_to_try = streams
if name in default_streams:
streams_to_try = [default_streams[name]]
elif len(streams_to_try) > 1 and raise_if_stream_ambigous:
raise StreamAmbigous(
"There are multiple streams %r to choose from for module %s." %
(streams_to_try, name))
# For each valid stream, find the last build in a stream and also all
# its contexts and add mmds of these builds to `mmds` and `added_mmds`.
# Of course only do that if we have not done that already in some
# previous call of this method.
for stream in streams:
ns = "%s:%s" % (name, stream)
if ns not in mmds:
mmds[ns] = []
if ns not in added_mmds:
added_mmds[ns] = []
if base_module_mmds:
for base_module_mmd in base_module_mmds:
mmds[ns] += resolver.get_buildrequired_modulemds(
name, stream, base_module_mmd)
else:
mmds[ns] = resolver.get_module_modulemds(name,
stream,
strict=True)
added_mmds[ns] += mmds[ns]
# Get the requires recursively.
if recursive:
for mmd_list in added_mmds.values():
for mmd in mmd_list:
for deps in mmd.get_dependencies():
deps_dict = deps_to_dict(deps, 'runtime')
mmds = _get_mmds_from_requires(
db_session,
deps_dict,
mmds,
True,
base_module_mmds=base_module_mmds)
return mmds
def generate_expanded_mmds(db_session,
mmd,
raise_if_stream_ambigous=False,
default_streams=None):
"""
Returns list with MMDs with buildrequires and requires set according
to module stream expansion rules. These module metadata can be directly
built using MBS.
:param db_session: SQLAlchemy DB session.
:param Modulemd.ModuleStream mmd: Modulemd metadata with original unexpanded module.
:param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case
there are multiple streams for some dependency of module and the module name is not
defined in `default_streams`, so it is not clear which stream should be used.
:param dict default_streams: Dict in {module_name: module_stream, ...} format defining
the default stream to choose for module in case when there are multiple streams to
choose from.
"""
if not default_streams:
default_streams = {}
# Create local copy of mmd, because we will expand its dependencies,
# which would change the module.
current_mmd = mmd.copy()
# MMDResolver expects the input MMD to have no context.
current_mmd.set_context(None)
# Expands the MSE streams. This mainly handles '-' prefix in MSE streams.
expand_mse_streams(db_session, current_mmd, default_streams,
raise_if_stream_ambigous)
# Get the list of all MMDs which this module can be possibly built against
# and add them to MMDResolver.
mmd_resolver = MMDResolver()
mmds_for_resolving = get_mmds_required_by_module_recursively(
db_session, current_mmd, default_streams, raise_if_stream_ambigous)
for m in mmds_for_resolving:
mmd_resolver.add_modules(m)
# Show log.info message with the NSVCs we have added to mmd_resolver.
nsvcs_to_solve = [m.get_nsvc() for m in mmds_for_resolving]
log.info("Starting resolving with following input modules: %r",
nsvcs_to_solve)
# Resolve the dependencies between modules and get the list of all valid
# combinations in which we can build this module.
requires_combinations = mmd_resolver.solve(current_mmd)
log.info("Resolving done, possible requires: %r", requires_combinations)
# This is where we are going to store the generated MMDs.
mmds = []
for requires in requires_combinations:
# Each generated MMD must be new Module object...
mmd_copy = mmd.copy()
xmd = mmd_copy.get_xmd()
# Requires contain the NSVC representing the input mmd.
# The 'context' of this NSVC defines the id of buildrequires/requires
# pair in the mmd.get_dependencies().
dependencies_id = None
# We don't want to depend on ourselves, so store the NSVC of the current_mmd
# to be able to ignore it later.
self_nsvca = None
# Dict to store name:stream pairs from nsvca, so we are able to access it
# easily later.
req_name_stream = {}
# Get the values for dependencies_id, self_nsvca and req_name_stream variables.
for nsvca in requires:
req_name, req_stream, _, req_context, req_arch = nsvca.split(":")
if req_arch == "src":
assert req_name == current_mmd.get_module_name()
assert req_stream == current_mmd.get_stream_name()
assert dependencies_id is None
assert self_nsvca is None
dependencies_id = int(req_context)
self_nsvca = nsvca
continue
req_name_stream[req_name] = req_stream
if dependencies_id is None or self_nsvca is None:
raise RuntimeError("%s:%s not found in requires %r" %
(current_mmd.get_module_name(),
current_mmd.get_stream_name(), requires))
# The name:[streams, ...] pairs do not have to be the same in both
# buildrequires/requires. In case they are the same, we replace the streams
# in requires section with a single stream against which we will build this MMD.
# In case they are not the same, we have to keep the streams as they are in requires
# section. We always replace stream(s) for build-requirement with the one we
# will build this MMD against.
new_deps = Modulemd.Dependencies()
deps = mmd_copy.get_dependencies()[dependencies_id]
deps_requires = deps_to_dict(deps, 'runtime')
deps_buildrequires = deps_to_dict(deps, 'buildtime')
for req_name, req_streams in deps_requires.items():
if req_name not in deps_buildrequires:
# This require is not a buildrequire so just copy this runtime requirement to
# new_dep and don't touch buildrequires
if not req_streams:
new_deps.set_empty_runtime_dependencies_for_module(
req_name)
else:
for req_stream in req_streams:
new_deps.add_runtime_stream(req_name, req_stream)
elif set(req_streams) != set(deps_buildrequires[req_name]):
# Streams in runtime section are not the same as in buildtime section,
# so just copy this runtime requirement to new_dep.
if not req_streams:
new_deps.set_empty_runtime_dependencies_for_module(
req_name)
else:
for req_stream in req_streams:
new_deps.add_runtime_stream(req_name, req_stream)
new_deps.add_buildtime_stream(req_name,
req_name_stream[req_name])
else:
# This runtime requirement has the same streams in both runtime/buildtime
# requires sections, so replace streams in both sections by the one we
# really used in this resolved variant.
new_deps.add_runtime_stream(req_name,
req_name_stream[req_name])
new_deps.add_buildtime_stream(req_name,
req_name_stream[req_name])
# There might be buildrequires which are not in runtime requires list.
# Such buildrequires must be copied to expanded MMD.
for req_name, req_streams in deps_buildrequires.items():
if req_name not in deps_requires:
new_deps.add_buildtime_stream(req_name,
req_name_stream[req_name])
# Set the new dependencies.
mmd_copy.remove_dependencies(deps)
mmd_copy.add_dependencies(new_deps)
# The Modulemd.Dependencies() stores only streams, but to really build this
# module, we need NSVC of buildrequires, so we have to store this data in XMD.
# We also need additional data like for example list of filtered_rpms. We will
# get them using module_build_service.resolver.GenericResolver.resolve_requires,
# so prepare list with NSVCs of buildrequires as an input for this method.
br_list = []
for nsvca in requires:
if nsvca == self_nsvca:
continue
# Remove the arch from nsvca
nsvc = ":".join(nsvca.split(":")[:-1])
br_list.append(nsvc)
# Resolve the buildrequires and store the result in XMD.
if "mbs" not in xmd:
xmd["mbs"] = {}
resolver = GenericResolver.create(db_session, conf)
xmd["mbs"]["buildrequires"] = resolver.resolve_requires(br_list)
xmd["mbs"]["mse"] = True
mmd_copy.set_xmd(xmd)
# Now we have all the info to actually compute context of this module.
context = models.ModuleBuild.contexts_from_mmd(
mmd_to_str(mmd_copy)).context
mmd_copy.set_context(context)
mmds.append(mmd_copy)
return mmds
def get_mmds_required_by_module_recursively(db_session,
mmd,
default_streams=None,
raise_if_stream_ambigous=False):
"""
Returns the list of Module metadata objects of all modules required while
building the module defined by `mmd` module metadata. This presumes the
module metadata streams are expanded using `expand_mse_streams(...)`
method.
This method finds out latest versions of all the build-requires of
the `mmd` module and then also all contexts of these latest versions.
For each build-required name:stream:version:context module, it checks
recursively all the "requires" and finds the latest version of each
required module and also all contexts of these latest versions.
:param db_session: SQLAlchemy database session.
:param dict default_streams: Dict in {module_name: module_stream, ...} format defining
the default stream to choose for module in case when there are multiple streams to
choose from.
:param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case
there are multiple streams for some dependency of module and the module name is not
defined in `default_streams`, so it is not clear which stream should be used.
:rtype: list of Modulemd metadata
:return: List of all modulemd metadata of all modules required to build
the module `mmd`.
"""
# We use dict with name:stream as a key and list with mmds as value.
# That way, we can ensure we won't have any duplicate mmds in a resulting
# list and we also don't waste resources on getting the modules we already
# handled from DB.
mmds = {}
# Get the MMDs of all compatible base modules based on the buildrequires.
base_module_mmds = get_base_module_mmds(db_session, mmd)
if not base_module_mmds["ready"]:
base_module_choices = " or ".join(conf.base_module_names)
raise UnprocessableEntity(
"None of the base module ({}) streams in the buildrequires section could be found"
.format(base_module_choices))
# Add base modules to `mmds`.
for base_module in base_module_mmds["ready"]:
ns = ":".join(
[base_module.get_module_name(),
base_module.get_stream_name()])
mmds.setdefault(ns, [])
mmds[ns].append(base_module)
# The currently submitted module build must be built only against "ready" base modules,
# but its dependencies might have been built against some old platform which is already
# EOL ("garbage" state). In order to find such old module builds, we need to include
# also EOL platform streams.
all_base_module_mmds = base_module_mmds["ready"] + base_module_mmds[
"garbage"]
# Get all the buildrequires of the module of interest.
for deps in mmd.get_dependencies():
deps_dict = deps_to_dict(deps, 'buildtime')
mmds = _get_mmds_from_requires(db_session, deps_dict, mmds, False,
default_streams,
raise_if_stream_ambigous,
all_base_module_mmds)
# Now get the requires of buildrequires recursively.
for mmd_key in list(mmds.keys()):
for mmd in mmds[mmd_key]:
for deps in mmd.get_dependencies():
deps_dict = deps_to_dict(deps, 'runtime')
mmds = _get_mmds_from_requires(db_session, deps_dict, mmds,
True, default_streams,
raise_if_stream_ambigous,
all_base_module_mmds)
# Make single list from dict of lists.
res = []
for ns, mmds_list in mmds.items():
if len(mmds_list) == 0:
raise UnprocessableEntity("Cannot find any module builds for %s" %
(ns))
res += mmds_list
return res