def test__get_base_module_mmds_virtual_streams(self, virtual_streams):
"""Ensure the correct results are returned without duplicates."""
init_data(data_size=1, multiple_stream_versions=True)
mmd = load_mmd(read_staged_data("testmodule_v2"))
deps = mmd.get_dependencies()[0]
new_deps = Modulemd.Dependencies()
for stream in deps.get_runtime_streams("platform"):
new_deps.add_runtime_stream("platform", stream)
new_deps.add_buildtime_stream("platform", "f29.2.0")
mmd.remove_dependencies(deps)
mmd.add_dependencies(new_deps)
make_module_in_db("platform:lp29.1.1:12:c11",
virtual_streams=virtual_streams)
mmds = get_base_module_mmds(db_session, mmd)
if virtual_streams == ["f29"]:
expected = {
"platform:f29.0.0", "platform:f29.1.0", "platform:f29.2.0",
"platform:lp29.1.1"
}
else:
expected = {
"platform:f29.0.0", "platform:f29.1.0", "platform:f29.2.0"
}
# Verify no duplicates were returned before doing set operations
assert len(mmds["ready"]) == len(expected)
# Verify the expected ones were returned
actual = set()
for mmd_ in mmds["ready"]:
actual.add("{}:{}".format(mmd_.get_module_name(),
mmd_.get_stream_name()))
assert actual == expected
def test_get_module_build_dependencies_recursive(
self, reuse_component_init_data):
"""
Tests that the buildrequires are returned when it is two layers deep
"""
# Add testmodule2 that requires testmodule
module = models.ModuleBuild.get_by_id(db_session, 3)
mmd = module.mmd()
# Rename the module
mmd = mmd.copy("testmodule2")
mmd.set_version(20180123171545)
deps = Modulemd.Dependencies()
deps.add_runtime_stream("testmodule", "master")
mmd.add_dependencies(deps)
xmd = mmd.get_xmd()
xmd["mbs"]["requires"]["testmodule"] = {
"filtered_rpms": [],
"ref": "620ec77321b2ea7b0d67d82992dda3e1d67055b4",
"stream": "master",
"version": "20180205135154",
}
mmd.set_xmd(xmd)
module.modulemd = mmd_to_str(mmd)
module.name = "testmodule2"
module.version = str(mmd.get_version())
module.koji_tag = "module-ae2adf69caf0e1b6"
db_session.commit()
resolver = mbs_resolver.GenericResolver.create(db_session,
conf,
backend="db")
result = resolver.get_module_build_dependencies(
"testmodule2", "master", "20180123171545", "c40c156c").keys()
assert set(result) == {"module-f28-build"}
def test_get_reusable_component_different_buildrequires_stream(
self, rebuild_strategy):
first_module_build = models.ModuleBuild.get_by_id(db_session, 2)
first_module_build.rebuild_strategy = rebuild_strategy
db_session.commit()
second_module_build = models.ModuleBuild.get_by_id(db_session, 3)
mmd = second_module_build.mmd()
xmd = mmd.get_xmd()
xmd["mbs"]["buildrequires"]["platform"]["stream"] = "different"
deps = Modulemd.Dependencies()
deps.add_buildtime_stream("platform", "different")
deps.add_runtime_stream("platform", "different")
mmd.clear_dependencies()
mmd.add_dependencies(deps)
mmd.set_xmd(xmd)
second_module_build.modulemd = mmd_to_str(mmd)
second_module_build.build_context = \
models.ModuleBuild.contexts_from_mmd(second_module_build.modulemd).build_context
second_module_build.rebuild_strategy = rebuild_strategy
db_session.commit()
plc_rv = get_reusable_component(second_module_build,
"perl-List-Compare")
pt_rv = get_reusable_component(second_module_build, "perl-Tangerine")
tangerine_rv = get_reusable_component(second_module_build, "tangerine")
assert plc_rv is None
assert pt_rv is None
assert tangerine_rv is None
def expand_mse_streams(db_session,
mmd,
default_streams=None,
raise_if_stream_ambigous=False):
"""
Expands streams in both buildrequires/requires sections of MMD.
:param db_session: SQLAlchemy DB session.
:param Modulemd.ModuleStream mmd: Modulemd metadata with original unexpanded module.
:param dict default_streams: Dict in {module_name: module_stream, ...} format defining
the default stream to choose for module in case when there are multiple streams to
choose from.
:param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case
there are multiple streams for some dependency of module and the module name is not
defined in `default_streams`, so it is not clear which stream should be used.
"""
for deps in mmd.get_dependencies():
new_deps = Modulemd.Dependencies()
for name in deps.get_runtime_modules():
streams = deps.get_runtime_streams(name)
new_streams = expand_single_mse_streams(db_session, name, streams,
default_streams,
raise_if_stream_ambigous)
if not new_streams:
new_deps.set_empty_runtime_dependencies_for_module(name)
else:
for stream in new_streams:
new_deps.add_runtime_stream(name, stream)
for name in deps.get_buildtime_modules():
streams = deps.get_buildtime_streams(name)
new_streams = expand_single_mse_streams(db_session, name, streams,
default_streams,
raise_if_stream_ambigous)
if not new_streams:
new_deps.set_empty_buildtime_dependencies_for_module(name)
else:
for stream in new_streams:
new_deps.add_buildtime_stream(name, stream)
# Replace the Dependencies object
mmd.remove_dependencies(deps)
mmd.add_dependencies(new_deps)
def _modify_buildtime_streams(db_session, mmd, new_streams_func):
"""
Modify buildtime streams using the input new_streams_func.
:param Modulemd.ModuleStream mmd: the modulemd to apply the overrides on
:param function new_streams: a function that takes the parameters (module_name, module_streams),
and returns the streams that should be set on the buildtime dependency.
"""
deps = mmd.get_dependencies()
for dep in deps:
overridden = False
brs = deps_to_dict(dep, "buildtime")
# There is no way to replace streams, so create a new Dependencies object that will end up
# being a copy, but with the streams replaced if a virtual stream is detected
new_dep = Modulemd.Dependencies()
for name, streams in brs.items():
new_streams = new_streams_func(db_session, name, streams)
if streams != new_streams:
overridden = True
if not new_streams:
new_dep.set_empty_buildtime_dependencies_for_module(name)
else:
for stream in new_streams:
new_dep.add_buildtime_stream(name, stream)
if overridden:
# Copy the runtime streams as is
reqs = deps_to_dict(dep, "runtime")
for name, streams in reqs.items():
if not streams:
new_dep.set_empty_runtime_dependencies_for_module(name)
else:
for stream in streams:
new_dep.add_runtime_stream(name, stream)
# Replace the old Dependencies object with the new one with the overrides
mmd.remove_dependencies(dep)
mmd.add_dependencies(new_dep)
def test__get_base_module_mmds_virtual_streams_only_major_versions(
self, cfg):
"""Ensure the correct results are returned without duplicates."""
init_data(data_size=1,
multiple_stream_versions=["foo28", "foo29", "foo30"])
# Mark platform:foo28 as garbage to test that it is still considered as compatible.
platform = db_session.query(models.ModuleBuild).filter_by(
name="platform", stream="foo28").first()
platform.state = "garbage"
db_session.add(platform)
db_session.commit()
mmd = load_mmd(read_staged_data("testmodule_v2"))
deps = mmd.get_dependencies()[0]
new_deps = Modulemd.Dependencies()
for stream in deps.get_runtime_streams("platform"):
new_deps.add_runtime_stream("platform", stream)
new_deps.add_buildtime_stream("platform", "foo29")
mmd.remove_dependencies(deps)
mmd.add_dependencies(new_deps)
mmds = get_base_module_mmds(db_session, mmd)
expected = {}
expected["ready"] = {"platform:foo29", "platform:foo30"}
expected["garbage"] = {"platform:foo28"}
# Verify no duplicates were returned before doing set operations
assert len(mmds) == len(expected)
for k in expected.keys():
assert len(mmds[k]) == len(expected[k])
# Verify the expected ones were returned
actual = set()
for mmd_ in mmds[k]:
actual.add("{}:{}".format(mmd_.get_module_name(),
mmd_.get_stream_name()))
assert actual == expected[k]
def make_module(
nsvc,
dependencies=None,
base_module=None,
filtered_rpms=None,
xmd=None,
store_to_db=False,
virtual_streams=None,
arches=None,
):
"""
Creates new models.ModuleBuild defined by `nsvc` string with requires
and buildrequires set according to ``requires_list`` and ``build_requires_list``.
:param str nsvc: name:stream:version:context of a module.
:param dependencies: list of groups of dependencies (requires and buildrequires).
For example, [
{"requires": {"platform": ["f30"]}, "buildrequires": {"platform": ["f30"]}},
...
]
:type dependencies: list[dict]
:param base_module: a base module build required by the new module created.
:type base_module: :class:`ModuleBuild`
:param filtered_rpms: list of filtered RPMs which are added to filter
section in module metadata.
:type filtered_rpms: list[str]
:param dict xmd: a mapping representing XMD section in module metadata. A
custom xmd could be passed for testing a particular scenario and some
default key/value pairs are added if not present.
:param bool store_to_db: whether to store created module metadata to the
database. If set to True, ``db_session`` is required.
:param virtual_streams: List of virtual streams provided by this module.
If set, This requires ``db_session`` and ``store_to_db`` to be set to a
session object and True.
:type virtual_streams: list[str]
:param arches: List of architectures this module is built against. If set
to None, ``["x86_64"]`` is used as a default. If set, likewise
``virtual_stream``.
:type arches: list[str]
:return: New Module Build if set to store module metadata to database,
otherwise the module metadata is returned.
:rtype: ModuleBuild or Modulemd.Module
"""
if store_to_db:
assert db_session is not None
if base_module:
assert db_session is not None
if virtual_streams:
assert store_to_db
if arches:
assert store_to_db
nsvc_regex = re.compile(r"([^:]+):([^:]+):([^:]+)(?::(.+))?")
match = nsvc_regex.match(nsvc)
if not match:
raise ValueError('Argument nsvc is not in format N:S:V or N:S:V:C')
name, stream, version, context = match.groups()
mmd = Modulemd.ModuleStreamV2.new(name, stream)
mmd.set_version(int(version))
if context:
mmd.set_context(context)
mmd.set_summary("foo")
# Test unicode in mmd.
mmd.set_description(u"foo \u2019s")
mmd.add_module_license("GPL")
if filtered_rpms:
for rpm in filtered_rpms:
mmd.add_rpm_filter(rpm)
def _add_require(mmd_deps, require_type, name, streams):
assert isinstance(mmd_deps, Modulemd.Dependencies)
assert require_type in ("requires", "buildrequires")
assert isinstance(streams, (list, tuple))
if require_type == "requires":
add_stream = mmd_deps.add_runtime_stream
set_empty_deps = mmd_deps.set_empty_runtime_dependencies_for_module
else:
add_stream = mmd_deps.add_buildtime_stream
set_empty_deps = mmd_deps.set_empty_buildtime_dependencies_for_module
for stream in streams:
add_stream(name, stream)
else:
set_empty_deps(name)
for dep_group in dependencies or []:
mmd_deps = Modulemd.Dependencies()
# A deps could be {"platform": ["f30"], "python": []}
for require_type, deps in dep_group.items():
for req_name, req_streams in deps.items():
_add_require(mmd_deps, require_type, req_name, req_streams)
mmd.add_dependencies(mmd_deps)
# Caller could pass whole xmd including mbs, but if something is missing,
# default values are given here.
xmd = xmd or {"mbs": {}}
xmd_mbs = xmd["mbs"]
if "buildrequires" not in xmd_mbs:
xmd_mbs["buildrequires"] = {}
if "requires" not in xmd_mbs:
xmd_mbs["requires"] = {}
if "commit" not in xmd_mbs:
xmd_mbs["commit"] = "ref_%s" % context
if "mse" not in xmd_mbs:
xmd_mbs["mse"] = "true"
if virtual_streams:
xmd_mbs["virtual_streams"] = virtual_streams
mmd.set_xmd(xmd)
if not store_to_db:
return mmd
module_build = ModuleBuild(
name=name,
stream=stream,
stream_version=ModuleBuild.get_stream_version(stream),
version=version,
context=context,
state=BUILD_STATES["ready"],
scmurl="https://src.stg.fedoraproject.org/modules/unused.git?#ff1ea79",
batch=1,
owner="Tom Brady",
time_submitted=datetime(2017, 2, 15, 16, 8, 18),
time_modified=datetime(2017, 2, 15, 16, 19, 35),
rebuild_strategy="changed-and-after",
build_context=context,
runtime_context=context,
modulemd=mmd_to_str(mmd),
koji_tag=xmd["mbs"]["koji_tag"] if "koji_tag" in xmd["mbs"] else None,
)
if base_module:
module_build.buildrequires.append(base_module)
db_session.add(module_build)
db_session.commit()
if virtual_streams:
for virtual_stream in virtual_streams:
vs_obj = db_session.query(VirtualStream).filter_by(
name=virtual_stream).first()
if not vs_obj:
vs_obj = VirtualStream(name=virtual_stream)
db_session.add(vs_obj)
db_session.commit()
if vs_obj not in module_build.virtual_streams:
module_build.virtual_streams.append(vs_obj)
db_session.commit()
for arch in arches or ["x86_64"]:
arch_obj = db_session.query(ModuleArch).filter_by(name=arch).first()
if not arch_obj:
arch_obj = ModuleArch(name=arch)
db_session.add(arch_obj)
db_session.commit()
if arch_obj not in module_build.arches:
module_build.arches.append(arch_obj)
db_session.commit()
return module_build
def _apply_dep_overrides(mmd, params):
"""
Apply the dependency override parameters (if specified) on the input modulemd.
:param Modulemd.ModuleStream mmd: the modulemd to apply the overrides on
:param dict params: the API parameters passed in by the user
:raises ValidationError: if one of the overrides doesn't apply
"""
dep_overrides = {
"buildrequires": copy.copy(params.get("buildrequire_overrides", {})),
"requires": copy.copy(params.get("require_overrides", {})),
}
# Parse the module's branch to determine if it should override the stream of the buildrequired
# module defined in conf.br_stream_override_module
branch_search = None
if params.get("branch") and conf.br_stream_override_module and conf.br_stream_override_regexes:
# Only parse the branch for a buildrequire override if the user didn't manually specify an
# override for the module specified in conf.br_stream_override_module
if not dep_overrides["buildrequires"].get(conf.br_stream_override_module):
branch_search = None
for regex in conf.br_stream_override_regexes:
branch_search = re.search(regex, params["branch"])
if branch_search:
log.debug(
"The stream override regex `%s` matched the branch %s",
regex,
params["branch"],
)
break
else:
log.debug('No stream override regexes matched the branch "%s"', params["branch"])
# If a stream was parsed from the branch, then add it as a stream override for the module
# specified in conf.br_stream_override_module
if branch_search:
# Concatenate all the groups that are not None together to get the desired stream.
# This approach is taken in case there are sections to ignore.
# For instance, if we need to parse `el8.0.0` from `rhel-8.0.0`.
parsed_stream = "".join(group for group in branch_search.groups() if group)
if parsed_stream:
dep_overrides["buildrequires"][conf.br_stream_override_module] = [parsed_stream]
log.info(
'The buildrequired stream of "%s" was overriden with "%s" based on the branch "%s"',
conf.br_stream_override_module, parsed_stream, params["branch"],
)
else:
log.warning(
'The regex `%s` only matched empty capture groups on the branch "%s". The regex is '
" invalid and should be rewritten.",
regex, params["branch"],
)
unused_dep_overrides = {
"buildrequires": set(dep_overrides["buildrequires"].keys()),
"requires": set(dep_overrides["requires"].keys()),
}
deps = mmd.get_dependencies()
for dep in deps:
overridden = False
new_dep = Modulemd.Dependencies()
for dep_type, overrides in dep_overrides.items():
if dep_type == "buildrequires":
mmd_dep_type = "buildtime"
else:
mmd_dep_type = "runtime"
# Get the existing streams
reqs = deps_to_dict(dep, mmd_dep_type)
# Get the method to add a new stream for this dependency type
# (e.g. add_buildtime_stream)
add_func = getattr(new_dep, "add_{}_stream".format(mmd_dep_type))
add_empty_func = getattr(
new_dep, "set_empty_{}_dependencies_for_module".format(mmd_dep_type))
for name, streams in reqs.items():
if name in dep_overrides[dep_type]:
streams_to_add = dep_overrides[dep_type][name]
unused_dep_overrides[dep_type].remove(name)
overridden = True
else:
streams_to_add = reqs[name]
if not streams_to_add:
add_empty_func(name)
else:
for stream in streams_to_add:
add_func(name, stream)
if overridden:
# Set the overridden streams
mmd.remove_dependencies(dep)
mmd.add_dependencies(new_dep)
for dep_type in unused_dep_overrides.keys():
# If a stream override was applied from parsing the branch and it wasn't applicable,
# just ignore it
if branch_search and conf.br_stream_override_module in unused_dep_overrides[dep_type]:
unused_dep_overrides[dep_type].remove(conf.br_stream_override_module)
if unused_dep_overrides[dep_type]:
raise ValidationError(
"The {} overrides for the following modules aren't applicable: {}".format(
dep_type[:-1], ", ".join(sorted(unused_dep_overrides[dep_type])))
)
def generate_expanded_mmds(db_session,
mmd,
raise_if_stream_ambigous=False,
default_streams=None):
"""
Returns list with MMDs with buildrequires and requires set according
to module stream expansion rules. These module metadata can be directly
built using MBS.
:param db_session: SQLAlchemy DB session.
:param Modulemd.ModuleStream mmd: Modulemd metadata with original unexpanded module.
:param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case
there are multiple streams for some dependency of module and the module name is not
defined in `default_streams`, so it is not clear which stream should be used.
:param dict default_streams: Dict in {module_name: module_stream, ...} format defining
the default stream to choose for module in case when there are multiple streams to
choose from.
"""
if not default_streams:
default_streams = {}
# Create local copy of mmd, because we will expand its dependencies,
# which would change the module.
current_mmd = mmd.copy()
# MMDResolver expects the input MMD to have no context.
current_mmd.set_context(None)
# Expands the MSE streams. This mainly handles '-' prefix in MSE streams.
expand_mse_streams(db_session, current_mmd, default_streams,
raise_if_stream_ambigous)
# Get the list of all MMDs which this module can be possibly built against
# and add them to MMDResolver.
mmd_resolver = MMDResolver()
mmds_for_resolving = get_mmds_required_by_module_recursively(
db_session, current_mmd, default_streams, raise_if_stream_ambigous)
for m in mmds_for_resolving:
mmd_resolver.add_modules(m)
# Show log.info message with the NSVCs we have added to mmd_resolver.
nsvcs_to_solve = [m.get_nsvc() for m in mmds_for_resolving]
log.info("Starting resolving with following input modules: %r",
nsvcs_to_solve)
# Resolve the dependencies between modules and get the list of all valid
# combinations in which we can build this module.
requires_combinations = mmd_resolver.solve(current_mmd)
log.info("Resolving done, possible requires: %r", requires_combinations)
# This is where we are going to store the generated MMDs.
mmds = []
for requires in requires_combinations:
# Each generated MMD must be new Module object...
mmd_copy = mmd.copy()
xmd = mmd_copy.get_xmd()
# Requires contain the NSVC representing the input mmd.
# The 'context' of this NSVC defines the id of buildrequires/requires
# pair in the mmd.get_dependencies().
dependencies_id = None
# We don't want to depend on ourselves, so store the NSVC of the current_mmd
# to be able to ignore it later.
self_nsvca = None
# Dict to store name:stream pairs from nsvca, so we are able to access it
# easily later.
req_name_stream = {}
# Get the values for dependencies_id, self_nsvca and req_name_stream variables.
for nsvca in requires:
req_name, req_stream, _, req_context, req_arch = nsvca.split(":")
if req_arch == "src":
assert req_name == current_mmd.get_module_name()
assert req_stream == current_mmd.get_stream_name()
assert dependencies_id is None
assert self_nsvca is None
dependencies_id = int(req_context)
self_nsvca = nsvca
continue
req_name_stream[req_name] = req_stream
if dependencies_id is None or self_nsvca is None:
raise RuntimeError("%s:%s not found in requires %r" %
(current_mmd.get_module_name(),
current_mmd.get_stream_name(), requires))
# The name:[streams, ...] pairs do not have to be the same in both
# buildrequires/requires. In case they are the same, we replace the streams
# in requires section with a single stream against which we will build this MMD.
# In case they are not the same, we have to keep the streams as they are in requires
# section. We always replace stream(s) for build-requirement with the one we
# will build this MMD against.
new_deps = Modulemd.Dependencies()
deps = mmd_copy.get_dependencies()[dependencies_id]
deps_requires = deps_to_dict(deps, 'runtime')
deps_buildrequires = deps_to_dict(deps, 'buildtime')
for req_name, req_streams in deps_requires.items():
if req_name not in deps_buildrequires:
# This require is not a buildrequire so just copy this runtime requirement to
# new_dep and don't touch buildrequires
if not req_streams:
new_deps.set_empty_runtime_dependencies_for_module(
req_name)
else:
for req_stream in req_streams:
new_deps.add_runtime_stream(req_name, req_stream)
elif set(req_streams) != set(deps_buildrequires[req_name]):
# Streams in runtime section are not the same as in buildtime section,
# so just copy this runtime requirement to new_dep.
if not req_streams:
new_deps.set_empty_runtime_dependencies_for_module(
req_name)
else:
for req_stream in req_streams:
new_deps.add_runtime_stream(req_name, req_stream)
new_deps.add_buildtime_stream(req_name,
req_name_stream[req_name])
else:
# This runtime requirement has the same streams in both runtime/buildtime
# requires sections, so replace streams in both sections by the one we
# really used in this resolved variant.
new_deps.add_runtime_stream(req_name,
req_name_stream[req_name])
new_deps.add_buildtime_stream(req_name,
req_name_stream[req_name])
# There might be buildrequires which are not in runtime requires list.
# Such buildrequires must be copied to expanded MMD.
for req_name, req_streams in deps_buildrequires.items():
if req_name not in deps_requires:
new_deps.add_buildtime_stream(req_name,
req_name_stream[req_name])
# Set the new dependencies.
mmd_copy.remove_dependencies(deps)
mmd_copy.add_dependencies(new_deps)
# The Modulemd.Dependencies() stores only streams, but to really build this
# module, we need NSVC of buildrequires, so we have to store this data in XMD.
# We also need additional data like for example list of filtered_rpms. We will
# get them using module_build_service.resolver.GenericResolver.resolve_requires,
# so prepare list with NSVCs of buildrequires as an input for this method.
br_list = []
for nsvca in requires:
if nsvca == self_nsvca:
continue
# Remove the arch from nsvca
nsvc = ":".join(nsvca.split(":")[:-1])
br_list.append(nsvc)
# Resolve the buildrequires and store the result in XMD.
if "mbs" not in xmd:
xmd["mbs"] = {}
resolver = GenericResolver.create(db_session, conf)
xmd["mbs"]["buildrequires"] = resolver.resolve_requires(br_list)
xmd["mbs"]["mse"] = True
mmd_copy.set_xmd(xmd)
# Now we have all the info to actually compute context of this module.
context = models.ModuleBuild.contexts_from_mmd(
mmd_to_str(mmd_copy)).context
mmd_copy.set_context(context)
mmds.append(mmd_copy)
return mmds