def validate_mmd(mmd): """Validate module metadata If everything is ok, just keep quiet, otherwise error is raised for specific problem. :param mmd: modulemd object representing module metadata. :type mmd: Modulemd.Module :raises Forbidden: if metadata contains module repository but it is not allowed. :raise ValidationError: if the xmd has the "mbs" key set. """ for modname in mmd.get_module_component_names(): mod = mmd.get_module_component(modname) if mod.get_repository() and not conf.modules_allow_repository: raise Forbidden( "Custom module repositories aren't allowed. " "%r bears repository %r" % (modname, mod.get_repository()) ) name = mmd.get_module_name() xmd = mmd.get_xmd() if "mbs" in xmd: if name not in conf.allowed_privileged_module_names: raise ValidationError('The "mbs" xmd field is reserved for MBS') allowed_keys = ["disttag_marking", "koji_tag_arches"] for key in xmd["mbs"].keys(): if key not in allowed_keys: raise ValidationError('The "mbs" xmd field is reserved for MBS') if name in conf.base_module_names: raise ValidationError( 'You cannot build a module named "{}" since it is a base module'.format(name))
def patch(self, api_version, id): username, groups = module_build_service.web.auth.get_user(request) try: r = json.loads(request.get_data().decode("utf-8")) except Exception: log.exception("Invalid JSON submitted") raise ValidationError("Invalid JSON submitted") if "owner" in r: if conf.no_auth is not True: raise ValidationError( "The request contains 'owner' parameter, however NO_AUTH is not allowed" ) elif username == "anonymous": username = r["owner"] self.check_groups(username, groups) module = models.ModuleBuild.query.filter_by(id=id).first() if not module: raise NotFound("No such module found.") if module.owner != username and not (conf.admin_groups & groups): raise Forbidden( "You are not owner of this build and therefore cannot modify it." ) if not r.get("state"): log.error("Invalid JSON submitted") raise ValidationError("Invalid JSON submitted") state = r["state"] valid_input_states = ("failed", str(models.BUILD_STATES["failed"])) if state not in valid_input_states: raise ValidationError( "An invalid state was submitted. Valid states values are: {}". format(", ".join(valid_input_states))) valid_states_to_cancel = ("build", "init", "wait") module_state_name = models.INVERSE_BUILD_STATES[module.state] if module_state_name not in valid_states_to_cancel: log.error( "The user %s attempted to cancel a build in the %s state", username, module_state_name, ) raise ValidationError( "To cancel a module build, it must be in one of the following states: {}" .format(", ".join(valid_states_to_cancel))) module.transition(db.session, conf, models.BUILD_STATES["failed"], "Canceled by %s." % username) db.session.add(module) db.session.commit() return jsonify(module.extended_json(db.session, True, api_version)), 200
def filter_component_builds(flask_request): """ Returns a flask_sqlalchemy.Pagination object based on the request parameters :param request: Flask request object :return: flask_sqlalchemy.Pagination """ search_query = dict() for key in request.args.keys(): # Search by state will be handled separately if key == "state": continue # Only filter on valid database columns if key in models.ComponentBuild.__table__.columns.keys(): if isinstance(models.ComponentBuild.__table__.columns[key].type, sqlalchemy_boolean): search_query[key] = str_to_bool(flask_request.args[key]) else: search_query[key] = flask_request.args[key] # Multiple states can be supplied => or-ing will take place states = flask_request.args.getlist("state") search_states = [] for state in states: if state.isdigit(): search_states.append(state) else: try: import koji except ImportError: raise ValidationError( "Cannot filter by state names because koji isn't installed" ) if state.upper() in koji.BUILD_STATES: search_states.append(koji.BUILD_STATES[state.upper()]) else: raise ValidationError("Invalid state was supplied: %s" % state) # Allow the user to specify the module build ID with a more intuitive key name if "module_build" in flask_request.args: search_query["module_id"] = flask_request.args["module_build"] query = models.ComponentBuild.query if search_query: query = query.filter_by(**search_query) if search_states: query = query.filter(models.ComponentBuild.state.in_(search_states)) query = _add_order_by_clause(flask_request, query, models.ComponentBuild) page = flask_request.args.get("page", 1, type=int) per_page = flask_request.args.get("per_page", 10, type=int) return query.paginate(page, per_page, False)
def __init__(self, url, branch=None, allowed_scm=None, allow_local=False): """ Initialize the SCM object using the specified SCM URL. If url is not in the list of allowed_scm, an error will be raised. :param str url: The unmodified scmurl :param str branch: The optional source control branch. This defaults to "master" when git is used. :param list allowed_scm: The optional list of allowed SCM URL prefixes :param bool allow_local: Allow SCM URLs that start with "file://" :raises: Forbidden or ValidationError """ if allowed_scm: if not (url.startswith(tuple(allowed_scm)) or (allow_local and url.startswith("file://"))): raise Forbidden("%s is not in the list of allowed SCMs" % url) # If we are given the option for the git protocol or the http(s) protocol, # then just use http(s) if re.match(r"(git\+http(?:s)?:\/\/)", url): url = url[4:] url = url.rstrip("/") self.url = url self.sourcedir = None # once we have more than one SCM provider, we will need some more # sophisticated lookup logic for scmtype, schemes in SCM.types.items(): if self.url.startswith(schemes): self.scheme = scmtype break else: raise ValidationError("Invalid SCM URL: %s" % url) # git is the only one supported SCM provider atm if self.scheme == "git": match = re.search( r"^(?P<repository>.*/(?P<name>[^?]*))(\?#(?P<commit>.*))?", url) self.repository = match.group("repository") self.name = match.group("name") self.repository_root = self.repository[:-len(self.name)] if self.name.endswith(".git"): self.name = self.name[:-4] self.commit = match.group("commit") self.branch = branch if branch else "master" self.version = None self._cloned = False else: raise ValidationError("Unhandled SCM scheme: %s" % self.scheme)
def _check_buildopts_arches(mmd, arches): """ Returns buildopts arches if valid, or otherwise the arches provided. :param mmd: Module MetaData :param arches: list of architectures :return: list of architectures """ buildopts = mmd.get_buildopts() if not buildopts: return arches try: buildopts_arches = buildopts.get_arches() except AttributeError: # libmodulemd version < 2.8.3 return arches # Must be a subset of the input module arches unsupported_arches = set(buildopts_arches) - set(arches) if unsupported_arches: raise ValidationError( "The following buildopts arches are not supported with these " "buildrequires: %r" % unsupported_arches) if buildopts_arches: log.info( "Setting build arches of %s to %r based on the buildopts arches." % (mmd.get_nsvc(), buildopts_arches)) return buildopts_arches return arches
def submit_module_build_from_yaml( db_session, username, handle, params, stream=None, skiptests=False ): yaml_file = to_text_type(handle.read()) mmd = load_mmd(yaml_file) dt = datetime.utcfromtimestamp(int(time.time())) if hasattr(handle, "filename"): def_name = str(os.path.splitext(os.path.basename(handle.filename))[0]) elif not mmd.get_module_name(): raise ValidationError( "The module's name was not present in the modulemd file. Please use the " '"module_name" parameter' ) def_version = int(dt.strftime("%Y%m%d%H%M%S")) module_name = mmd.get_module_name() or def_name module_stream = stream or mmd.get_stream_name() or "master" if module_name != mmd.get_module_name() or module_stream != mmd.get_stream_name(): # This is how you set the name and stream in the modulemd mmd = mmd.copy(module_name, module_stream) mmd.set_version(mmd.get_version() or def_version) if skiptests: buildopts = mmd.get_buildopts() or Modulemd.Buildopts() macros = buildopts.get_rpm_macros() or "" buildopts.set_rpm_macros(macros + "\n\n%__spec_check_pre exit 0\n") mmd.set_buildopts(buildopts) return submit_module_build(db_session, username, mmd, params)
def get(self, api_version, id): module = models.ModuleBuild.get_by_id(db.session, id) if not module: raise ValidationError("The module could not be found") if conf.system == "koji": # We are importing KojiContentGenerator here so we can generate the final modulemds. # If we imported this regularly we would have gotten a circular import error. from module_build_service.builder.KojiContentGenerator import KojiContentGenerator # noqa cg = KojiContentGenerator(module, conf) finalmmds = cg.get_final_mmds() else: raise ValidationError( "Configured builder not able to generate final modulemds!") return jsonify(finalmmds), 200
def _is_eol_in_pdc(name, stream): """ Check PDC if the module name:stream is no longer active. """ params = {"type": "module", "global_component": name, "name": stream} url = conf.pdc_url + "/component-branches/" response = requests.get(url, params=params) if not response: raise ValidationError("Failed to talk to PDC {}{}".format( response, response.text)) data = response.json() results = data["results"] if not results: raise ValidationError("No such module {}:{} found at {}".format( name, stream, response.request.url)) # If the module is active, then it is not EOL and vice versa. return not results[0]["active"]
def _dict_from_request(request): if "multipart/form-data" in request.headers.get("Content-Type", ""): data = request.form.to_dict() else: try: data = json.loads(request.get_data().decode("utf-8")) except Exception: log.exception("Invalid JSON submitted") raise ValidationError("Invalid JSON submitted") return data
def _add_order_by_clause(flask_request, query, column_source): """ Orders the given SQLAlchemy query based on the GET arguments provided. :param flask_request: a Flask request object :param query: a SQLAlchemy query object :param column_source: a SQLAlchemy database model :return: a SQLAlchemy query object """ order_by = flask_request.args.getlist("order_by") order_desc_by = flask_request.args.getlist("order_desc_by") # Default to ordering by ID in descending order descending = True requested_order = ["id"] if order_by and order_desc_by: raise ValidationError( "You may not specify both order_by and order_desc_by") elif order_by: descending = False requested_order = order_by elif order_desc_by: descending = True requested_order = order_desc_by column_dict = dict(column_source.__table__.columns) order_args = [] for column_name in requested_order: if column_name not in column_dict: raise ValidationError( 'An invalid ordering key of "{}" was supplied'.format( column_name)) column = column_dict[column_name] # If the version column is provided, cast it as an integer so the sorting is correct if column_name == "version": column = sqlalchemy.cast(column, sqlalchemy.BigInteger) if descending: column = column.desc() order_args.append(column) return query.order_by(*order_args)
def get(self, api_version, id): id_flag = request.args.get("id") if id_flag: endpoint = request.endpoint.split("s_list")[0] raise ValidationError( 'The "id" query option is invalid. Did you mean to go to "{0}"?' .format(url_for(endpoint, api_version=api_version, id=id_flag))) verbose_flag = request.args.get("verbose", "false").lower() short_flag = request.args.get("short", "false").lower() json_func_kwargs = {} json_func_name = "json" if id is None: # Lists all tracked builds p_query = self.query_filter(request) json_data = { "meta": pagination_metadata(p_query, api_version, request.args) } if verbose_flag == "true" or verbose_flag == "1": json_func_name = "extended_json" json_func_kwargs["show_state_url"] = True json_func_kwargs["api_version"] = api_version elif short_flag == "true" or short_flag == "1": if p_query.items and hasattr(p_query.items[0], "short_json"): json_func_name = "short_json" if json_func_name == "json" or json_func_name == "extended_json": # Only ModuleBuild.json and ModuleBuild.extended_json has argument db_session json_func_kwargs["db_session"] = db.session json_data["items"] = [ getattr(item, json_func_name)(**json_func_kwargs) for item in p_query.items ] return jsonify(json_data), 200 else: # Lists details for the specified build instance = self.model.query.filter_by(id=id).first() if instance: if verbose_flag == "true" or verbose_flag == "1": json_func_name = "extended_json" json_func_kwargs["show_state_url"] = True json_func_kwargs["api_version"] = api_version elif short_flag == "true" or short_flag == "1": if getattr(instance, "short_json", None): json_func_name = "short_json" if json_func_name == "json" or json_func_name == "extended_json": # Only ModuleBuild.json and ModuleBuild.extended_json has argument db_session json_func_kwargs["db_session"] = db.session return jsonify( getattr(instance, json_func_name)(**json_func_kwargs)), 200 else: raise NotFound("No such %s found." % self.kind)
def _validate_dep_overrides_format(self, key): """ Validate any dependency overrides provided to the API. :param str key: the override key to validate :raises ValidationError: when the overrides are an invalid format """ if not self.data.get(key): return invalid_override_msg = ( 'The "{}" parameter must be an object with the keys as module ' "names and the values as arrays of streams".format(key)) if not isinstance(self.data[key], dict): raise ValidationError(invalid_override_msg) for streams in self.data[key].values(): if not isinstance(streams, list): raise ValidationError(invalid_override_msg) for stream in streams: if not isinstance(stream, string_types): raise ValidationError(invalid_override_msg)
def _validate_stream_name_for_static_context(module, stream, context, dep_type): """ Validates the the streams of static contexts. :param str module: name of the module :param str stream: name of the stream :param str context: name of the context :param str dep_type: type of the dependencies the module stream belongs to i. e. requires, buildrequires. :return None """ if not stream: raise ValidationError( ("The '{module}' module in '{dep_type}' of the '{context}' static " "context has no stream defined.").format(module=module, dep_type=dep_type, context=context)) stream_type = type(stream) if stream_type is not str: raise ValidationError( ("The module '{module}' in '{dep_type}' of the '{context}' " "static context is of type '{stream_type}' should be " "'str' type.").format(stream=stream, module=module, dep_type=dep_type, context=context, stream_type=stream_type)) if stream.startswith("-"): raise ValidationError(( "The '{stream}' stream of module '{module}' in '{dep_type}' of the" "'{context}' static contexts is using stream expansion. Usage of " "stream expansion with static context is forbidden.").format( stream=stream, module=module, dep_type=dep_type, context=context))
def validate(self, skip_branch=False, skip_optional_params=False): if "scmurl" not in self.data: log.error("Missing scmurl") raise ValidationError("Missing scmurl") url = self.data["scmurl"] allowed_prefix = any(url.startswith(prefix) for prefix in conf.scmurls) if not conf.allow_custom_scmurls and not allowed_prefix: log.error("The submitted scmurl %r is not allowed" % url) raise Forbidden("The submitted scmurl %s is not allowed" % url) if not get_scm_url_re().match(url): log.error("The submitted scmurl %r is not valid" % url) raise ValidationError("The submitted scmurl %s is not valid" % url) if not skip_branch and "branch" not in self.data: log.error("Missing branch") raise ValidationError("Missing branch") if "module_name" in self.data: log.error( "Module name override is only allowed when a YAML file is submitted" ) raise ValidationError( "Module name override is only allowed when a YAML file is submitted" ) if "module_stream" in self.data: log.error( "Stream name override is only allowed when a YAML file is submitted" ) raise ValidationError( "Stream name override is only allowed when a YAML file is submitted" ) if not skip_optional_params: self.validate_optional_params()
def get(self, api_version, id, model): if not model: raise ValidationError( "Model is not set for this log messages endpoint") query = model.query.filter_by(id=id).first().log_messages.order_by( models.LogMessage.time_created.desc()) page = request.args.get("page", 1, type=int) per_page = request.args.get("per_page", 10, type=int) p_query = query.paginate(page, per_page, False) request_args = {"id": id} json_data = { "meta": pagination_metadata(p_query, api_version, request_args) } json_data["messages"] = [ getattr(message, "json")() for message in p_query.items ] return jsonify(json_data), 200
def __init__(self, request, data=None): self.username, self.groups = module_build_service.web.auth.get_user( request) self.data = data or _dict_from_request(request) # canonicalize and validate scratch option if "scratch" in self.data and str_to_bool(str(self.data["scratch"])): self.data["scratch"] = True if conf.modules_allow_scratch is not True: raise Forbidden("Scratch builds are not enabled") else: self.data["scratch"] = False # canonicalize and validate srpms list if "srpms" in self.data and self.data["srpms"]: if not self.data["scratch"]: raise Forbidden( "srpms may only be specified for scratch builds") if not isinstance(self.data["srpms"], list): raise ValidationError("srpms must be specified as a list") else: self.data["srpms"] = []
def verify(self): """ Verifies that the information provided by a user in SCM URL and branch matches the information in SCM repository. For example verifies that the commit hash really belongs to the provided branch. :raises ValidationError """ if not self.sourcedir: raise ProgrammingError("Do .checkout() first.") found = False branches = SCM._run(["git", "branch", "-r", "--contains", self.commit], chdir=self.sourcedir)[1] for branch in branches.decode("utf-8").split("\n"): branch = branch.strip() if branch[len("origin/"):] == self.branch: found = True break if not found: raise ValidationError("Commit %s is not in branch %s." % (self.commit, self.branch))
def submit_module_build(db_session, username, mmd, params): """ Submits new module build. :param db_session: SQLAlchemy session object. :param str username: Username of the build's owner. :param Modulemd.ModuleStream mmd: Modulemd defining the build. :param dict params: the API parameters passed in by the user :rtype: list with ModuleBuild :return: List with submitted module builds. """ log.debug( "Submitted %s module build for %s:%s:%s", ("scratch" if params.get("scratch", False) else "normal"), mmd.get_module_name(), mmd.get_stream_name(), mmd.get_version(), ) raise_if_stream_ambigous = False default_streams = {} # For local builds, we want the user to choose the exact stream using the default_streams # in case there are multiple streams to choose from and raise an exception otherwise. if "local_build" in params: raise_if_stream_ambigous = True # Get the default_streams if set. if "default_streams" in params: default_streams = params["default_streams"] xmd = mmd.get_xmd() # we check if static contexts are enabled by the `contexts` property defined by the user i # as an build option. static_context = "mbs_options" in xmd and "contexts" in xmd["mbs_options"] input_mmds = generate_mmds_from_static_contexts(mmd) if static_context else [mmd] mmds = [] for mmd in input_mmds: validate_mmd(mmd) _apply_dep_overrides(mmd, params) _modify_buildtime_streams(db_session, mmd, resolve_base_module_virtual_streams) _process_support_streams(db_session, mmd, params) mmds += generate_expanded_mmds(db_session, mmd, raise_if_stream_ambigous, default_streams, static_context=static_context) if not mmds: raise ValidationError( "No dependency combination was satisfied. Please verify the " "buildrequires in your modulemd have previously been built." ) modules = [] # True if all module builds are skipped so MBS will actually not rebuild # anything. To keep the backward compatibility, we need to raise an exception # later in the end of this method. all_modules_skipped = True for mmd in mmds: # Prefix the version of the modulemd based on the base module it buildrequires version = get_prefixed_version(mmd) mmd.set_version(version) nsvc = mmd.get_nsvc() log.debug("Checking whether module build already exists: %s.", nsvc) module = models.ModuleBuild.get_build_from_nsvc(db_session, *nsvc.split(":")) if module and not params.get("scratch", False): if module.state != models.BUILD_STATES["failed"]: log.info( "Skipping rebuild of %s, only rebuild of modules in failed state is allowed.", nsvc, ) modules.append(module) continue rebuild_strategy = params.get("rebuild_strategy") if rebuild_strategy and module.rebuild_strategy != rebuild_strategy: raise ValidationError( 'You cannot change the module\'s "rebuild_strategy" when ' "resuming a module build" ) log.debug("Resuming existing module build %r" % module) # Reset all component builds that didn't complete for component in module.component_builds: if not component.is_waiting_for_build and not component.is_completed: component.state = None component.state_reason = None db_session.add(component) module.username = username prev_state = module.previous_non_failed_state if prev_state == models.BUILD_STATES["init"]: transition_to = models.BUILD_STATES["init"] else: transition_to = models.BUILD_STATES["wait"] module.batch = 0 module.transition(db_session, conf, transition_to, "Resubmitted by %s" % username) db_session.commit() log.info("Resumed existing module build in previous state %s" % module.state) else: # make NSVC unique for every scratch build context_suffix = "" if params.get("scratch", False): log.debug("Checking for existing scratch module builds by NSVC") scrmods = models.ModuleBuild.get_scratch_builds_from_nsvc( db_session, *nsvc.split(":")) scrmod_contexts = [scrmod.context for scrmod in scrmods] log.debug( "Found %d previous scratch module build context(s): %s", len(scrmods), ",".join(scrmod_contexts), ) # append incrementing counter to context context_suffix = "_" + str(len(scrmods) + 1) mmd.set_context(mmd.get_context() + context_suffix) else: # In case the branch is defined, check whether user is allowed to submit # non-scratch build from this branch. Note that the branch is always defined # for official builds from SCM, because it is requested in views.py. branch = params.get("branch") if branch: for regex in conf.scratch_build_only_branches: branch_search = re.search(regex, branch) if branch_search: raise ValidationError( "Only scratch module builds can be built from this branch." ) log.debug("Creating new module build") module = models.ModuleBuild.create( db_session, conf, name=mmd.get_module_name(), stream=mmd.get_stream_name(), version=str(mmd.get_version()), modulemd=mmd_to_str(mmd), scmurl=params.get("scmurl"), username=username, rebuild_strategy=params.get("rebuild_strategy"), reused_module_id=params.get("reuse_components_from"), scratch=params.get("scratch"), srpms=params.get("srpms"), ) module.build_context, module.runtime_context, module.context, \ module.build_context_no_bms = module.contexts_from_mmd(module.modulemd) xmd = mmd.get_xmd() if xmd["mbs"].get("static_context"): module.context = mmd.get_context() module.context += context_suffix db_session.commit() notify_on_module_state_change( # Note the state is "init" here... module.json(db_session, show_tasks=False) ) all_modules_skipped = False modules.append(module) log.info('The user "%s" submitted the build "%s"', username, nsvc) if all_modules_skipped: err_msg = ( "Module (state=%s) already exists. Only a new build, resubmission of " "a failed build or build against new buildrequirements is " "allowed." % module.state ) log.error(err_msg) raise Conflict(err_msg) return modules
def filter_module_builds(flask_request): """ Returns a flask_sqlalchemy.Pagination object based on the request parameters :param request: Flask request object :return: flask_sqlalchemy.Pagination """ search_query = dict() special_columns = { "time_submitted", "time_modified", "time_completed", "state", "stream_version_lte", "virtual_stream", } columns = models.ModuleBuild.__table__.columns.keys() for key in set(request.args.keys()) - special_columns: # Only filter on valid database columns but skip columns that are treated specially or # ignored if key in columns: search_query[key] = flask_request.args[key] # Multiple states can be supplied => or-ing will take place states = flask_request.args.getlist("state") search_states = [] for state in states: if state.isdigit(): search_states.append(state) else: if state in models.BUILD_STATES: search_states.append(models.BUILD_STATES[state]) else: raise ValidationError("Invalid state was supplied: %s" % state) nsvc = flask_request.args.get("nsvc", None) if nsvc: nsvc_parts = nsvc.split(":") query_keys = ["name", "stream", "version", "context"] for key, part in zip(query_keys, nsvc_parts): search_query[key] = part rpm = flask_request.args.get("rpm", None) koji_tags = [] if rpm: if conf.system == "koji": # we are importing the koji builder here so we can search for the rpm metadata # from koji. If we imported this regulary we would have gotten a circular import error. from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder # noqa koji_tags = KojiModuleBuilder.get_rpm_module_tag(rpm) else: raise ValidationError( "Configured builder does not allow to search by rpm binary name!" ) query = models.ModuleBuild.query if search_query: query = query.filter_by(**search_query) if search_states: query = query.filter(models.ModuleBuild.state.in_(search_states)) if koji_tags: query = query.filter( models.ModuleBuild.koji_tag.in_(koji_tags)).filter_by( **search_query) # This is used when filtering the date request parameters, but it is here to avoid recompiling utc_iso_datetime_regex = re.compile( r"^(?P<datetime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.\d+)?(?:Z|[-+]00(?::00)?)?$" ) # Filter the query based on date request parameters for item in ("submitted", "modified", "completed"): for context in ("before", "after"): request_arg = "%s_%s" % (item, context) # i.e. submitted_before iso_datetime_arg = request.args.get(request_arg, None) if iso_datetime_arg: iso_datetime_matches = re.match(utc_iso_datetime_regex, iso_datetime_arg) if not iso_datetime_matches or not iso_datetime_matches.group( "datetime"): raise ValidationError( 'An invalid Zulu ISO 8601 timestamp was provided for the "%s" parameter' % request_arg) # Converts the ISO 8601 string to a datetime object for SQLAlchemy to use to filter item_datetime = datetime.strptime( iso_datetime_matches.group("datetime"), "%Y-%m-%dT%H:%M:%S") # Get the database column to filter against column = getattr(models.ModuleBuild, "time_" + item) if context == "after": query = query.filter(column >= item_datetime) elif context == "before": query = query.filter(column <= item_datetime) # Multiple virtual_streams can be supplied for "or" logic filtering virtual_streams = flask_request.args.getlist("virtual_stream") query = models.ModuleBuild._add_virtual_streams_filter( db.session, query, virtual_streams) stream_version_lte = flask_request.args.get("stream_version_lte") if stream_version_lte is not None: invalid_error = ( "An invalid value of stream_version_lte was provided. It must be an " "integer or float greater than or equal to 10000.") try: stream_version_lte = float(stream_version_lte) except (TypeError, ValueError): raise ValidationError(invalid_error) if stream_version_lte < 10000: raise ValidationError(invalid_error) query = models.ModuleBuild._add_stream_version_lte_filter( db.session, query, stream_version_lte) br_joined = False module_br_alias = None for item in ( "base_module_br", "name", "stream", "version", "context", "stream_version", "stream_version_lte", "stream_version_gte", ): if item == "base_module_br": request_arg_name = item else: request_arg_name = "base_module_br_{}".format(item) request_arg = flask_request.args.get(request_arg_name) if not request_arg: continue if not br_joined: module_br_alias = aliased(models.ModuleBuild, name="module_br") # Shorten this table name for clarity in the query below mb_to_br = models.module_builds_to_module_buildrequires # The following joins get added: # JOIN module_builds_to_module_buildrequires # ON module_builds_to_module_buildrequires.module_id = module_builds.id # JOIN module_builds AS module_br # ON module_builds_to_module_buildrequires.module_buildrequire_id = module_br.id query = query.join( mb_to_br, mb_to_br.c.module_id == models.ModuleBuild.id).join( module_br_alias, mb_to_br.c.module_buildrequire_id == module_br_alias.id) br_joined = True if item == "base_module_br": try: name, stream, version, context = flask_request.args[ "base_module_br"].split(":") except ValueError: raise ValidationError( 'The filter argument for "base_module_br" must be in the format of N:S:V:C' ) query = query.filter( module_br_alias.name == name, module_br_alias.stream == stream, module_br_alias.version == version, module_br_alias.context == context, ) elif item.endswith("_lte"): column = getattr(module_br_alias, item[:-4]) query = query.filter(column <= request_arg) elif item.endswith("_gte"): column = getattr(module_br_alias, item[:-4]) query = query.filter(column >= request_arg) else: column = getattr(module_br_alias, item) query = query.filter(column == request_arg) query = _add_order_by_clause(flask_request, query, models.ModuleBuild) page = flask_request.args.get("page", 1, type=int) per_page = flask_request.args.get("per_page", 10, type=int) return query.paginate(page, per_page, False)
def _apply_dep_overrides(mmd, params): """ Apply the dependency override parameters (if specified) on the input modulemd. :param Modulemd.ModuleStream mmd: the modulemd to apply the overrides on :param dict params: the API parameters passed in by the user :raises ValidationError: if one of the overrides doesn't apply """ dep_overrides = { "buildrequires": copy.copy(params.get("buildrequire_overrides", {})), "requires": copy.copy(params.get("require_overrides", {})), } # Parse the module's branch to determine if it should override the stream of the buildrequired # module defined in conf.br_stream_override_module branch_search = None if params.get("branch") and conf.br_stream_override_module and conf.br_stream_override_regexes: # Only parse the branch for a buildrequire override if the user didn't manually specify an # override for the module specified in conf.br_stream_override_module if not dep_overrides["buildrequires"].get(conf.br_stream_override_module): branch_search = None for regex in conf.br_stream_override_regexes: branch_search = re.search(regex, params["branch"]) if branch_search: log.debug( "The stream override regex `%s` matched the branch %s", regex, params["branch"], ) break else: log.debug('No stream override regexes matched the branch "%s"', params["branch"]) # If a stream was parsed from the branch, then add it as a stream override for the module # specified in conf.br_stream_override_module if branch_search: # Concatenate all the groups that are not None together to get the desired stream. # This approach is taken in case there are sections to ignore. # For instance, if we need to parse `el8.0.0` from `rhel-8.0.0`. parsed_stream = "".join(group for group in branch_search.groups() if group) if parsed_stream: dep_overrides["buildrequires"][conf.br_stream_override_module] = [parsed_stream] log.info( 'The buildrequired stream of "%s" was overriden with "%s" based on the branch "%s"', conf.br_stream_override_module, parsed_stream, params["branch"], ) else: log.warning( 'The regex `%s` only matched empty capture groups on the branch "%s". The regex is ' " invalid and should be rewritten.", regex, params["branch"], ) unused_dep_overrides = { "buildrequires": set(dep_overrides["buildrequires"].keys()), "requires": set(dep_overrides["requires"].keys()), } deps = mmd.get_dependencies() for dep in deps: overridden = False new_dep = Modulemd.Dependencies() for dep_type, overrides in dep_overrides.items(): if dep_type == "buildrequires": mmd_dep_type = "buildtime" else: mmd_dep_type = "runtime" # Get the existing streams reqs = deps_to_dict(dep, mmd_dep_type) # Get the method to add a new stream for this dependency type # (e.g. add_buildtime_stream) add_func = getattr(new_dep, "add_{}_stream".format(mmd_dep_type)) add_empty_func = getattr( new_dep, "set_empty_{}_dependencies_for_module".format(mmd_dep_type)) for name, streams in reqs.items(): if name in dep_overrides[dep_type]: streams_to_add = dep_overrides[dep_type][name] unused_dep_overrides[dep_type].remove(name) overridden = True else: streams_to_add = reqs[name] if not streams_to_add: add_empty_func(name) else: for stream in streams_to_add: add_func(name, stream) if overridden: # Set the overridden streams mmd.remove_dependencies(dep) mmd.add_dependencies(new_dep) for dep_type in unused_dep_overrides.keys(): # If a stream override was applied from parsing the branch and it wasn't applicable, # just ignore it if branch_search and conf.br_stream_override_module in unused_dep_overrides[dep_type]: unused_dep_overrides[dep_type].remove(conf.br_stream_override_module) if unused_dep_overrides[dep_type]: raise ValidationError( "The {} overrides for the following modules aren't applicable: {}".format( dep_type[:-1], ", ".join(sorted(unused_dep_overrides[dep_type]))) )
def fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False, mandatory_checks=True): td = None scm = None try: log.debug("Verifying modulemd") td = tempfile.mkdtemp() if whitelist_url: scm = module_build_service.common.scm.SCM(url, branch, [url], allow_local_url) else: scm = module_build_service.common.scm.SCM(url, branch, conf.scmurls, allow_local_url) scm.checkout(td) if not whitelist_url and mandatory_checks: scm.verify() cofn = scm.get_module_yaml() mmd = load_mmd_file(cofn) finally: try: if td is not None: shutil.rmtree(td) except Exception as e: log.warning("Failed to remove temporary directory {!r}: {}".format( td, str(e))) if conf.check_for_eol: if _is_eol_in_pdc(scm.name, scm.branch): raise ValidationError( "Module {}:{} is marked as EOL in PDC.".format( scm.name, scm.branch)) if not mandatory_checks: return mmd, scm # If the name was set in the modulemd, make sure it matches what the scmurl # says it should be if mmd.get_module_name() and mmd.get_module_name() != scm.name: if not conf.allow_name_override_from_scm: raise ValidationError( 'The name "{0}" that is stored in the modulemd is not valid'. format(mmd.get_module_name())) else: # Set the module name mmd = mmd.copy(scm.name) # If the stream was set in the modulemd, make sure it matches what the repo # branch is if mmd.get_stream_name() and mmd.get_stream_name() != scm.branch: if not conf.allow_stream_override_from_scm: raise ValidationError( 'The stream "{0}" that is stored in the modulemd does not match the branch "{1}"' .format(mmd.get_stream_name(), scm.branch)) else: # Set the module stream mmd = mmd.copy(mmd.get_module_name(), scm.branch) # If the version is in the modulemd, throw an exception since the version # since the version is generated by MBS if mmd.get_version(): raise ValidationError( 'The version "{0}" is already defined in the modulemd but it shouldn\'t be since the ' "version is generated based on the commit time".format( mmd.get_version())) else: mmd.set_version(int(scm.version)) return mmd, scm
def record_component_builds(mmd, module, initial_batch=1, previous_buildorder=None, main_mmd=None): # Imported here to allow import of utils in GenericBuilder. from module_build_service.builder import GenericBuilder # When main_mmd is set, merge the metadata from this mmd to main_mmd, # otherwise our current mmd is main_mmd. if main_mmd: # Check for components that are in both MMDs before merging since MBS # currently can't handle that situation. main_mmd_rpms = main_mmd.get_rpm_component_names() mmd_rpms = mmd.get_rpm_component_names() duplicate_components = [ rpm for rpm in main_mmd_rpms if rpm in mmd_rpms ] if duplicate_components: error_msg = ( 'The included module "{0}" in "{1}" have the following ' "conflicting components: {2}".format( mmd.get_module_name(), main_mmd.get_module_name(), ", ".join(duplicate_components))) raise UnprocessableEntity(error_msg) merge_included_mmd(main_mmd, mmd) else: main_mmd = mmd # If the modulemd yaml specifies components, then submit them for build rpm_components = [ mmd.get_rpm_component(name) for name in mmd.get_rpm_component_names() ] module_components = [ mmd.get_module_component(name) for name in mmd.get_module_component_names() ] all_components = list(rpm_components) + list(module_components) if not all_components: return # Get map of packages that have SRPM overrides srpm_overrides = get_module_srpm_overrides(module) rpm_weights = GenericBuilder.get_build_weights( [c.get_name() for c in rpm_components]) all_components.sort(key=lambda x: x.get_buildorder()) # We do not start with batch = 0 here, because the first batch is # reserved for module-build-macros. First real components must be # planned for batch 2 and following. batch = initial_batch for component in all_components: # Increment the batch number when buildorder increases. if previous_buildorder != component.get_buildorder(): previous_buildorder = component.get_buildorder() batch += 1 # If the component is another module, we fetch its modulemd file # and record its components recursively with the initial_batch # set to our current batch, so the components of this module # are built in the right global order. if isinstance(component, Modulemd.ComponentModule): full_url = component.get_repository() + "?#" + component.get_ref() # It is OK to whitelist all URLs here, because the validity # of every URL have been already checked in format_mmd(...). included_mmd = fetch_mmd(full_url, whitelist_url=True)[0] format_mmd(included_mmd, module.scmurl, module, db_session, srpm_overrides) batch = record_component_builds(included_mmd, module, batch, previous_buildorder, main_mmd) continue package = component.get_name() if package in srpm_overrides: component_ref = None full_url = srpm_overrides[package] log.info('Building custom SRPM "{0}"' " for package {1}".format(full_url, package)) else: component_ref = mmd.get_xmd()["mbs"]["rpms"][package]["ref"] full_url = component.get_repository() + "?#" + component_ref # Skip the ComponentBuild if it already exists in database. This can happen # in case of module build resubmition. existing_build = models.ComponentBuild.from_component_name( db_session, package, module.id) if existing_build: # Check that the existing build has the same most important attributes. # This should never be a problem, but it's good to be defensive here so # we do not mess things during resubmition. if (existing_build.batch != batch or existing_build.scmurl != full_url or existing_build.ref != component_ref): raise ValidationError( "Component build %s of module build %s (id: %d) already " "exists in database, but its attributes are different from" " resubmitted one." % (component.get_name(), module.name, module.id)) continue build = models.ComponentBuild(module_id=module.id, package=package, format="rpms", scmurl=full_url, batch=batch, ref=component_ref, weight=rpm_weights[package], buildonly=component.get_buildonly()) db_session.add(build) return batch
def format_mmd(mmd, scmurl, module=None, db_session=None, srpm_overrides=None): """ Prepares the modulemd for the MBS. This does things such as replacing the branches of components with commit hashes and adding metadata in the xmd dictionary. :param mmd: the Modulemd.ModuleStream object to format :param scmurl: the url to the modulemd :param module: When specified together with `session`, the time_modified of a module is updated regularly in case this method takes lot of time. :param db_session: Database session to update the `module`. :param dict srpm_overrides: Mapping of package names to SRPM links for all component packages which have custom SRPM overrides specified. """ srpm_overrides = srpm_overrides or {} xmd = mmd.get_xmd() if "mbs" not in xmd: xmd["mbs"] = {} if "scmurl" not in xmd["mbs"]: xmd["mbs"]["scmurl"] = scmurl or "" if "commit" not in xmd["mbs"]: xmd["mbs"]["commit"] = "" # If module build was submitted via yaml file, there is no scmurl if scmurl: scm = module_build_service.common.scm.SCM(scmurl) # We want to make sure we have the full commit hash for consistency if module_build_service.common.scm.SCM.is_full_commit_hash( scm.scheme, scm.commit): full_scm_hash = scm.commit else: full_scm_hash = scm.get_full_commit_hash() xmd["mbs"]["commit"] = full_scm_hash if mmd.get_rpm_component_names() or mmd.get_module_component_names(): if "rpms" not in xmd["mbs"]: xmd["mbs"]["rpms"] = {} # Add missing data in RPM components for pkgname in mmd.get_rpm_component_names(): pkg = mmd.get_rpm_component(pkgname) # In case of resubmit of existing module which have been # cancelled/failed during the init state, the package # was maybe already handled by MBS, so skip it in this case. if pkgname in xmd["mbs"]["rpms"]: continue if pkg.get_repository() and not conf.rpms_allow_repository: raise Forbidden( "Custom component repositories aren't allowed. " "%r bears repository %r" % (pkgname, pkg.get_repository())) if pkg.get_cache() and not conf.rpms_allow_cache: raise Forbidden("Custom component caches aren't allowed. " "%r bears cache %r" % (pkgname, pkg.get_cache())) if pkg.get_buildafter(): raise ValidationError( 'The usage of "buildafter" is not yet supported') if not pkg.get_repository(): pkg.set_repository(conf.rpms_default_repository + pkgname) if not pkg.get_cache(): pkg.set_cache(conf.rpms_default_cache + pkgname) if not pkg.get_ref(): pkg.set_ref("master") if not pkg.get_arches(): for arch in conf.arches: pkg.add_restricted_arch(arch) # Add missing data in included modules components for modname in mmd.get_module_component_names(): mod = mmd.get_module_component(modname) if mod.get_repository() and not conf.modules_allow_repository: raise Forbidden("Custom module repositories aren't allowed. " "%r bears repository %r" % (modname, mod.get_repository())) if not mod.get_repository(): mod.set_repository(conf.modules_default_repository + modname) if not mod.get_ref(): mod.set_ref("master") # Check that SCM URL is valid and replace potential branches in pkg refs # by real SCM hash and store the result to our private xmd place in modulemd. pool = ThreadPool(20) try: # Filter out the packages which we have already resolved in possible # previous runs of this method (can be caused by module build resubmition) # or which have custom SRPMs and shouldn't be resolved. pkgs_to_resolve = [] for name in mmd.get_rpm_component_names(): if name not in xmd["mbs"]["rpms"]: if name in srpm_overrides: # If this package has a custom SRPM, store an empty # ref entry so no further verification takes place. xmd["mbs"]["rpms"][name] = {"ref": None} else: pkgs_to_resolve.append(mmd.get_rpm_component(name)) async_result = pool.map_async(_scm_get_latest, pkgs_to_resolve) # For modules with lot of components, the _scm_get_latest can take a lot of time. # We need to bump time_modified from time to time, otherwise poller could think # that module is stuck in "init" state and it would send fake "init" message. while not async_result.ready(): async_result.wait(60) if module and db_session: module.time_modified = datetime.utcnow() db_session.commit() pkg_dicts = async_result.get() finally: pool.close() err_msg = "" for pkg_dict in pkg_dicts: if pkg_dict["error"]: err_msg += pkg_dict["error"] + "\n" else: pkg_name = pkg_dict["pkg_name"] pkg_ref = pkg_dict["pkg_ref"] xmd["mbs"]["rpms"][pkg_name] = {"ref": pkg_ref} if err_msg: raise UnprocessableEntity(err_msg) # Set the modified xmd back to the modulemd mmd.set_xmd(xmd)
def validate(self): if ("modulemd" not in self.data and (not hasattr(request, "files") or "yaml" not in request.files)): log.error("Invalid file submitted") raise ValidationError("Invalid file submitted") self.validate_optional_params()
def validate_optional_params(self): forbidden_params = [k for k in self.data if k not in self.valid_params] if forbidden_params: raise ValidationError( "The request contains unspecified parameters: {}".format( ", ".join(forbidden_params))) if not conf.no_auth and "owner" in self.data: raise ValidationError( "The request contains 'owner' parameter, however NO_AUTH is not allowed" ) if not conf.rebuild_strategy_allow_override and "rebuild_strategy" in self.data: raise ValidationError( 'The request contains the "rebuild_strategy" parameter but ' "overriding the default isn't allowed") if "rebuild_strategy" in self.data: if self.data[ "rebuild_strategy"] not in conf.rebuild_strategies_allowed: raise ValidationError( 'The rebuild method of "{0}" is not allowed. Choose from: {1}.' .format(self.data["rebuild_strategy"], ", ".join(conf.rebuild_strategies_allowed))) self._validate_dep_overrides_format("buildrequire_overrides") self._validate_dep_overrides_format("require_overrides") if "reuse_components_from" in self.data: if "rebuild_strategy" in self.data and self.data[ "rebuild_strategy"] == "all": raise ValidationError( 'You cannot specify the parameter "reuse_components_from" when the ' '"rebuild_strategy" parameter is set to "all"') invalid_identifier_msg = ( 'The parameter "reuse_components_from" contains an invalid module identifier' ) if isinstance(self.data["reuse_components_from"], int): reuse_module = models.ModuleBuild.get_by_id( db.session, self.data["reuse_components_from"]) elif isinstance(self.data["reuse_components_from"], string_types): try: n, s, v, c = self.data["reuse_components_from"].split(":") except ValueError: raise ValidationError(invalid_identifier_msg) reuse_module = models.ModuleBuild.get_build_from_nsvc( db.session, n, s, v, c) else: raise ValidationError(invalid_identifier_msg) if not reuse_module: raise ValidationError( 'The module in the parameter "reuse_components_from" could not be found' ) if reuse_module.state != models.BUILD_STATES["ready"]: raise ValidationError( 'The module in the parameter "reuse_components_from" must be in the ready state' ) # Normalize the value so that it simplifies any code that uses this value self.data["reuse_components_from"] = reuse_module.id
def generate_mmds_from_static_contexts(mmd): """ This function preexpands the MSE when static contexts which are defined by the `contexts` property under `mbs_options` in the `xmd` property of the initial modulemd file. :param Modulemd.ModuleStream mmd: Modulemd metadata with the `contexts` property. :return list mmds: list of distinct mmds identified by context. """ xmd = mmd.get_xmd() if not xmd: raise ValidationError( "The 'xmd' property of the modulemd yaml file is empty!") if not xmd["mbs_options"].get("contexts"): raise ValidationError(( "The 'xmd' property of the modulemd yaml file does not contain the" " 'contexts' key.")) contexts = xmd["mbs_options"].get("contexts") if not type(contexts) is dict: raise ValidationError( "The value of the 'contexts' property needs to be a dict.") # remove the information about the `contexts` property from the `mbs_options` as we do not # needed anymore xmd["mbs_options"].pop("contexts") # we check if the `mbs_options` dict is empty after we remove the `contexts` key. # if yes we remove `the mbs_options`. if not xmd["mbs_options"]: xmd.pop("mbs_options") mmd.set_xmd(xmd) mmds = [] for context, dependencies in contexts.items(): # we copy the mmd so we a get a fresh module for each context. mmd_copy = mmd.copy() mmd_copy.set_context(context) if "buildrequires" not in dependencies: raise ValidationError( ("The context '{context}' is missing the" " 'buildrequires' key!").format(context=context)) if "requires" not in dependencies: raise ValidationError( "The context '{context}' is missing the 'requires' key!". format(context=context)) # remove the old deps from the mmd old_module_deps = mmd_copy.get_dependencies() for deps in old_module_deps: mmd_copy.remove_dependencies(deps) module_deps = Modulemd.Dependencies.new() # populate the new requires and buildrequires according to the `contexts` property for module, stream in dependencies["buildrequires"].items(): _validate_stream_name_for_static_context(module, stream, context, "buildrequires") module_deps.add_buildtime_stream(module, stream) for module, stream in dependencies["requires"].items(): _validate_stream_name_for_static_context(module, stream, context, "requires") module_deps.add_runtime_stream(module, stream) mmd_copy.add_dependencies(module_deps) mmds.append(mmd_copy) return mmds