def test_set_stream(self, main): cli_cmd = [ "mbs-manager", "build_module_locally", "--set-stream", "platform:f28", "--file", staged_data_filename("testmodule-local-build.yaml") ] self._run_manager_wrapper(cli_cmd) # Since module_build_service.scheduler.local.main is mocked, MBS does # not really build the testmodule for this test. Following lines assert # the fact: # Module testmodule-local-build is expanded and stored into database, # and this build has buildrequires platform:f28 and requires # platform:f28. # Please note that, the f28 is specified from command line option # --set-stream, which is the point this test tests. builds = db_session.query( models.ModuleBuild).filter_by(name="testmodule-local-build").all() assert 1 == len(builds) testmodule_build = builds[0] mmd_deps = testmodule_build.mmd().get_dependencies() deps_dict = deps_to_dict(mmd_deps[0], "buildtime") assert ["f28"] == deps_dict["platform"] deps_dict = deps_to_dict(mmd_deps[0], "runtime") assert ["f28"] == deps_dict["platform"]
def _modify_buildtime_streams(db_session, mmd, new_streams_func): """ Modify buildtime streams using the input new_streams_func. :param Modulemd.ModuleStream mmd: the modulemd to apply the overrides on :param function new_streams: a function that takes the parameters (module_name, module_streams), and returns the streams that should be set on the buildtime dependency. """ deps = mmd.get_dependencies() for dep in deps: overridden = False brs = deps_to_dict(dep, "buildtime") # There is no way to replace streams, so create a new Dependencies object that will end up # being a copy, but with the streams replaced if a virtual stream is detected new_dep = Modulemd.Dependencies() for name, streams in brs.items(): new_streams = new_streams_func(db_session, name, streams) if streams != new_streams: overridden = True if not new_streams: new_dep.set_empty_buildtime_dependencies_for_module(name) else: for stream in new_streams: new_dep.add_buildtime_stream(name, stream) if overridden: # Copy the runtime streams as is reqs = deps_to_dict(dep, "runtime") for name, streams in reqs.items(): if not streams: new_dep.set_empty_runtime_dependencies_for_module(name) else: for stream in streams: new_dep.add_runtime_stream(name, stream) # Replace the old Dependencies object with the new one with the overrides mmd.remove_dependencies(dep) mmd.add_dependencies(new_dep)
def _apply_dep_overrides(mmd, params): """ Apply the dependency override parameters (if specified) on the input modulemd. :param Modulemd.ModuleStream mmd: the modulemd to apply the overrides on :param dict params: the API parameters passed in by the user :raises ValidationError: if one of the overrides doesn't apply """ dep_overrides = { "buildrequires": copy.copy(params.get("buildrequire_overrides", {})), "requires": copy.copy(params.get("require_overrides", {})), } # Parse the module's branch to determine if it should override the stream of the buildrequired # module defined in conf.br_stream_override_module branch_search = None if params.get("branch") and conf.br_stream_override_module and conf.br_stream_override_regexes: # Only parse the branch for a buildrequire override if the user didn't manually specify an # override for the module specified in conf.br_stream_override_module if not dep_overrides["buildrequires"].get(conf.br_stream_override_module): branch_search = None for regex in conf.br_stream_override_regexes: branch_search = re.search(regex, params["branch"]) if branch_search: log.debug( "The stream override regex `%s` matched the branch %s", regex, params["branch"], ) break else: log.debug('No stream override regexes matched the branch "%s"', params["branch"]) # If a stream was parsed from the branch, then add it as a stream override for the module # specified in conf.br_stream_override_module if branch_search: # Concatenate all the groups that are not None together to get the desired stream. # This approach is taken in case there are sections to ignore. # For instance, if we need to parse `el8.0.0` from `rhel-8.0.0`. parsed_stream = "".join(group for group in branch_search.groups() if group) if parsed_stream: dep_overrides["buildrequires"][conf.br_stream_override_module] = [parsed_stream] log.info( 'The buildrequired stream of "%s" was overriden with "%s" based on the branch "%s"', conf.br_stream_override_module, parsed_stream, params["branch"], ) else: log.warning( 'The regex `%s` only matched empty capture groups on the branch "%s". The regex is ' " invalid and should be rewritten.", regex, params["branch"], ) unused_dep_overrides = { "buildrequires": set(dep_overrides["buildrequires"].keys()), "requires": set(dep_overrides["requires"].keys()), } deps = mmd.get_dependencies() for dep in deps: overridden = False new_dep = Modulemd.Dependencies() for dep_type, overrides in dep_overrides.items(): if dep_type == "buildrequires": mmd_dep_type = "buildtime" else: mmd_dep_type = "runtime" # Get the existing streams reqs = deps_to_dict(dep, mmd_dep_type) # Get the method to add a new stream for this dependency type # (e.g. add_buildtime_stream) add_func = getattr(new_dep, "add_{}_stream".format(mmd_dep_type)) add_empty_func = getattr( new_dep, "set_empty_{}_dependencies_for_module".format(mmd_dep_type)) for name, streams in reqs.items(): if name in dep_overrides[dep_type]: streams_to_add = dep_overrides[dep_type][name] unused_dep_overrides[dep_type].remove(name) overridden = True else: streams_to_add = reqs[name] if not streams_to_add: add_empty_func(name) else: for stream in streams_to_add: add_func(name, stream) if overridden: # Set the overridden streams mmd.remove_dependencies(dep) mmd.add_dependencies(new_dep) for dep_type in unused_dep_overrides.keys(): # If a stream override was applied from parsing the branch and it wasn't applicable, # just ignore it if branch_search and conf.br_stream_override_module in unused_dep_overrides[dep_type]: unused_dep_overrides[dep_type].remove(conf.br_stream_override_module) if unused_dep_overrides[dep_type]: raise ValidationError( "The {} overrides for the following modules aren't applicable: {}".format( dep_type[:-1], ", ".join(sorted(unused_dep_overrides[dep_type]))) )
def _get_mmds_from_requires( db_session, requires, mmds, recursive=False, default_streams=None, raise_if_stream_ambigous=False, base_module_mmds=None, ): """ Helper method for get_mmds_required_by_module_recursively returning the list of module metadata objects defined by `requires` dict. :param db_session: SQLAlchemy database session. :param dict requires: requires or buildrequires in the form {module: [streams]} :param mmds: Dictionary with already handled name:streams as a keys and lists of resulting mmds as values. :param recursive: If True, the requires are checked recursively. :param dict default_streams: Dict in {module_name: module_stream, ...} format defining the default stream to choose for module in case when there are multiple streams to choose from. :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case there are multiple streams for some dependency of module and the module name is not defined in `default_streams`, so it is not clear which stream should be used. :param list base_module_mmds: List of modulemd metadata instances. When set, the returned list contains MMDs build against each base module defined in `base_module_mmds` list. :return: Dict with name:stream as a key and list with mmds as value. """ default_streams = default_streams or {} # To be able to call itself recursively, we need to store list of mmds # we have added to global mmds list in this particular call. added_mmds = {} resolver = GenericResolver.create(db_session, conf) for name, streams in requires.items(): # Base modules are already added to `mmds`. if name in conf.base_module_names: continue streams_to_try = streams if name in default_streams: streams_to_try = [default_streams[name]] elif len(streams_to_try) > 1 and raise_if_stream_ambigous: raise StreamAmbigous( "There are multiple streams %r to choose from for module %s." % (streams_to_try, name)) # For each valid stream, find the last build in a stream and also all # its contexts and add mmds of these builds to `mmds` and `added_mmds`. # Of course only do that if we have not done that already in some # previous call of this method. for stream in streams: ns = "%s:%s" % (name, stream) if ns not in mmds: mmds[ns] = [] if ns not in added_mmds: added_mmds[ns] = [] if base_module_mmds: for base_module_mmd in base_module_mmds: mmds[ns] += resolver.get_buildrequired_modulemds( name, stream, base_module_mmd) else: mmds[ns] = resolver.get_module_modulemds(name, stream, strict=True) added_mmds[ns] += mmds[ns] # Get the requires recursively. if recursive: for mmd_list in added_mmds.values(): for mmd in mmd_list: for deps in mmd.get_dependencies(): deps_dict = deps_to_dict(deps, 'runtime') mmds = _get_mmds_from_requires( db_session, deps_dict, mmds, True, base_module_mmds=base_module_mmds) return mmds
def generate_expanded_mmds(db_session, mmd, raise_if_stream_ambigous=False, default_streams=None): """ Returns list with MMDs with buildrequires and requires set according to module stream expansion rules. These module metadata can be directly built using MBS. :param db_session: SQLAlchemy DB session. :param Modulemd.ModuleStream mmd: Modulemd metadata with original unexpanded module. :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case there are multiple streams for some dependency of module and the module name is not defined in `default_streams`, so it is not clear which stream should be used. :param dict default_streams: Dict in {module_name: module_stream, ...} format defining the default stream to choose for module in case when there are multiple streams to choose from. """ if not default_streams: default_streams = {} # Create local copy of mmd, because we will expand its dependencies, # which would change the module. current_mmd = mmd.copy() # MMDResolver expects the input MMD to have no context. current_mmd.set_context(None) # Expands the MSE streams. This mainly handles '-' prefix in MSE streams. expand_mse_streams(db_session, current_mmd, default_streams, raise_if_stream_ambigous) # Get the list of all MMDs which this module can be possibly built against # and add them to MMDResolver. mmd_resolver = MMDResolver() mmds_for_resolving = get_mmds_required_by_module_recursively( db_session, current_mmd, default_streams, raise_if_stream_ambigous) for m in mmds_for_resolving: mmd_resolver.add_modules(m) # Show log.info message with the NSVCs we have added to mmd_resolver. nsvcs_to_solve = [m.get_nsvc() for m in mmds_for_resolving] log.info("Starting resolving with following input modules: %r", nsvcs_to_solve) # Resolve the dependencies between modules and get the list of all valid # combinations in which we can build this module. requires_combinations = mmd_resolver.solve(current_mmd) log.info("Resolving done, possible requires: %r", requires_combinations) # This is where we are going to store the generated MMDs. mmds = [] for requires in requires_combinations: # Each generated MMD must be new Module object... mmd_copy = mmd.copy() xmd = mmd_copy.get_xmd() # Requires contain the NSVC representing the input mmd. # The 'context' of this NSVC defines the id of buildrequires/requires # pair in the mmd.get_dependencies(). dependencies_id = None # We don't want to depend on ourselves, so store the NSVC of the current_mmd # to be able to ignore it later. self_nsvca = None # Dict to store name:stream pairs from nsvca, so we are able to access it # easily later. req_name_stream = {} # Get the values for dependencies_id, self_nsvca and req_name_stream variables. for nsvca in requires: req_name, req_stream, _, req_context, req_arch = nsvca.split(":") if req_arch == "src": assert req_name == current_mmd.get_module_name() assert req_stream == current_mmd.get_stream_name() assert dependencies_id is None assert self_nsvca is None dependencies_id = int(req_context) self_nsvca = nsvca continue req_name_stream[req_name] = req_stream if dependencies_id is None or self_nsvca is None: raise RuntimeError("%s:%s not found in requires %r" % (current_mmd.get_module_name(), current_mmd.get_stream_name(), requires)) # The name:[streams, ...] pairs do not have to be the same in both # buildrequires/requires. In case they are the same, we replace the streams # in requires section with a single stream against which we will build this MMD. # In case they are not the same, we have to keep the streams as they are in requires # section. We always replace stream(s) for build-requirement with the one we # will build this MMD against. new_deps = Modulemd.Dependencies() deps = mmd_copy.get_dependencies()[dependencies_id] deps_requires = deps_to_dict(deps, 'runtime') deps_buildrequires = deps_to_dict(deps, 'buildtime') for req_name, req_streams in deps_requires.items(): if req_name not in deps_buildrequires: # This require is not a buildrequire so just copy this runtime requirement to # new_dep and don't touch buildrequires if not req_streams: new_deps.set_empty_runtime_dependencies_for_module( req_name) else: for req_stream in req_streams: new_deps.add_runtime_stream(req_name, req_stream) elif set(req_streams) != set(deps_buildrequires[req_name]): # Streams in runtime section are not the same as in buildtime section, # so just copy this runtime requirement to new_dep. if not req_streams: new_deps.set_empty_runtime_dependencies_for_module( req_name) else: for req_stream in req_streams: new_deps.add_runtime_stream(req_name, req_stream) new_deps.add_buildtime_stream(req_name, req_name_stream[req_name]) else: # This runtime requirement has the same streams in both runtime/buildtime # requires sections, so replace streams in both sections by the one we # really used in this resolved variant. new_deps.add_runtime_stream(req_name, req_name_stream[req_name]) new_deps.add_buildtime_stream(req_name, req_name_stream[req_name]) # There might be buildrequires which are not in runtime requires list. # Such buildrequires must be copied to expanded MMD. for req_name, req_streams in deps_buildrequires.items(): if req_name not in deps_requires: new_deps.add_buildtime_stream(req_name, req_name_stream[req_name]) # Set the new dependencies. mmd_copy.remove_dependencies(deps) mmd_copy.add_dependencies(new_deps) # The Modulemd.Dependencies() stores only streams, but to really build this # module, we need NSVC of buildrequires, so we have to store this data in XMD. # We also need additional data like for example list of filtered_rpms. We will # get them using module_build_service.resolver.GenericResolver.resolve_requires, # so prepare list with NSVCs of buildrequires as an input for this method. br_list = [] for nsvca in requires: if nsvca == self_nsvca: continue # Remove the arch from nsvca nsvc = ":".join(nsvca.split(":")[:-1]) br_list.append(nsvc) # Resolve the buildrequires and store the result in XMD. if "mbs" not in xmd: xmd["mbs"] = {} resolver = GenericResolver.create(db_session, conf) xmd["mbs"]["buildrequires"] = resolver.resolve_requires(br_list) xmd["mbs"]["mse"] = True mmd_copy.set_xmd(xmd) # Now we have all the info to actually compute context of this module. context = models.ModuleBuild.contexts_from_mmd( mmd_to_str(mmd_copy)).context mmd_copy.set_context(context) mmds.append(mmd_copy) return mmds
def get_mmds_required_by_module_recursively(db_session, mmd, default_streams=None, raise_if_stream_ambigous=False): """ Returns the list of Module metadata objects of all modules required while building the module defined by `mmd` module metadata. This presumes the module metadata streams are expanded using `expand_mse_streams(...)` method. This method finds out latest versions of all the build-requires of the `mmd` module and then also all contexts of these latest versions. For each build-required name:stream:version:context module, it checks recursively all the "requires" and finds the latest version of each required module and also all contexts of these latest versions. :param db_session: SQLAlchemy database session. :param dict default_streams: Dict in {module_name: module_stream, ...} format defining the default stream to choose for module in case when there are multiple streams to choose from. :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case there are multiple streams for some dependency of module and the module name is not defined in `default_streams`, so it is not clear which stream should be used. :rtype: list of Modulemd metadata :return: List of all modulemd metadata of all modules required to build the module `mmd`. """ # We use dict with name:stream as a key and list with mmds as value. # That way, we can ensure we won't have any duplicate mmds in a resulting # list and we also don't waste resources on getting the modules we already # handled from DB. mmds = {} # Get the MMDs of all compatible base modules based on the buildrequires. base_module_mmds = get_base_module_mmds(db_session, mmd) if not base_module_mmds["ready"]: base_module_choices = " or ".join(conf.base_module_names) raise UnprocessableEntity( "None of the base module ({}) streams in the buildrequires section could be found" .format(base_module_choices)) # Add base modules to `mmds`. for base_module in base_module_mmds["ready"]: ns = ":".join( [base_module.get_module_name(), base_module.get_stream_name()]) mmds.setdefault(ns, []) mmds[ns].append(base_module) # The currently submitted module build must be built only against "ready" base modules, # but its dependencies might have been built against some old platform which is already # EOL ("garbage" state). In order to find such old module builds, we need to include # also EOL platform streams. all_base_module_mmds = base_module_mmds["ready"] + base_module_mmds[ "garbage"] # Get all the buildrequires of the module of interest. for deps in mmd.get_dependencies(): deps_dict = deps_to_dict(deps, 'buildtime') mmds = _get_mmds_from_requires(db_session, deps_dict, mmds, False, default_streams, raise_if_stream_ambigous, all_base_module_mmds) # Now get the requires of buildrequires recursively. for mmd_key in list(mmds.keys()): for mmd in mmds[mmd_key]: for deps in mmd.get_dependencies(): deps_dict = deps_to_dict(deps, 'runtime') mmds = _get_mmds_from_requires(db_session, deps_dict, mmds, True, default_streams, raise_if_stream_ambigous, all_base_module_mmds) # Make single list from dict of lists. res = [] for ns, mmds_list in mmds.items(): if len(mmds_list) == 0: raise UnprocessableEntity("Cannot find any module builds for %s" % (ns)) res += mmds_list return res