def validate_mmd(mmd):
"""Validate module metadata
If everything is ok, just keep quiet, otherwise error is raised for
specific problem.
:param mmd: modulemd object representing module metadata.
:type mmd: Modulemd.Module
:raises Forbidden: if metadata contains module repository but it is not
allowed.
:raise ValidationError: if the xmd has the "mbs" key set.
"""
for modname in mmd.get_module_component_names():
mod = mmd.get_module_component(modname)
if mod.get_repository() and not conf.modules_allow_repository:
raise Forbidden(
"Custom module repositories aren't allowed. "
"%r bears repository %r" % (modname, mod.get_repository())
)
name = mmd.get_module_name()
xmd = mmd.get_xmd()
if "mbs" in xmd:
if name not in conf.allowed_privileged_module_names:
raise ValidationError('The "mbs" xmd field is reserved for MBS')
allowed_keys = ["disttag_marking", "koji_tag_arches"]
for key in xmd["mbs"].keys():
if key not in allowed_keys:
raise ValidationError('The "mbs" xmd field is reserved for MBS')
if name in conf.base_module_names:
raise ValidationError(
'You cannot build a module named "{}" since it is a base module'.format(name))
def patch(self, api_version, id):
username, groups = module_build_service.web.auth.get_user(request)
try:
r = json.loads(request.get_data().decode("utf-8"))
except Exception:
log.exception("Invalid JSON submitted")
raise ValidationError("Invalid JSON submitted")
if "owner" in r:
if conf.no_auth is not True:
raise ValidationError(
"The request contains 'owner' parameter, however NO_AUTH is not allowed"
)
elif username == "anonymous":
username = r["owner"]
self.check_groups(username, groups)
module = models.ModuleBuild.query.filter_by(id=id).first()
if not module:
raise NotFound("No such module found.")
if module.owner != username and not (conf.admin_groups & groups):
raise Forbidden(
"You are not owner of this build and therefore cannot modify it."
)
if not r.get("state"):
log.error("Invalid JSON submitted")
raise ValidationError("Invalid JSON submitted")
state = r["state"]
valid_input_states = ("failed", str(models.BUILD_STATES["failed"]))
if state not in valid_input_states:
raise ValidationError(
"An invalid state was submitted. Valid states values are: {}".
format(", ".join(valid_input_states)))
valid_states_to_cancel = ("build", "init", "wait")
module_state_name = models.INVERSE_BUILD_STATES[module.state]
if module_state_name not in valid_states_to_cancel:
log.error(
"The user %s attempted to cancel a build in the %s state",
username,
module_state_name,
)
raise ValidationError(
"To cancel a module build, it must be in one of the following states: {}"
.format(", ".join(valid_states_to_cancel)))
module.transition(db.session, conf, models.BUILD_STATES["failed"],
"Canceled by %s." % username)
db.session.add(module)
db.session.commit()
return jsonify(module.extended_json(db.session, True,
api_version)), 200
def filter_component_builds(flask_request):
"""
Returns a flask_sqlalchemy.Pagination object based on the request parameters
:param request: Flask request object
:return: flask_sqlalchemy.Pagination
"""
search_query = dict()
for key in request.args.keys():
# Search by state will be handled separately
if key == "state":
continue
# Only filter on valid database columns
if key in models.ComponentBuild.__table__.columns.keys():
if isinstance(models.ComponentBuild.__table__.columns[key].type,
sqlalchemy_boolean):
search_query[key] = str_to_bool(flask_request.args[key])
else:
search_query[key] = flask_request.args[key]
# Multiple states can be supplied => or-ing will take place
states = flask_request.args.getlist("state")
search_states = []
for state in states:
if state.isdigit():
search_states.append(state)
else:
try:
import koji
except ImportError:
raise ValidationError(
"Cannot filter by state names because koji isn't installed"
)
if state.upper() in koji.BUILD_STATES:
search_states.append(koji.BUILD_STATES[state.upper()])
else:
raise ValidationError("Invalid state was supplied: %s" % state)
# Allow the user to specify the module build ID with a more intuitive key name
if "module_build" in flask_request.args:
search_query["module_id"] = flask_request.args["module_build"]
query = models.ComponentBuild.query
if search_query:
query = query.filter_by(**search_query)
if search_states:
query = query.filter(models.ComponentBuild.state.in_(search_states))
query = _add_order_by_clause(flask_request, query, models.ComponentBuild)
page = flask_request.args.get("page", 1, type=int)
per_page = flask_request.args.get("per_page", 10, type=int)
return query.paginate(page, per_page, False)
def __init__(self, url, branch=None, allowed_scm=None, allow_local=False):
"""
Initialize the SCM object using the specified SCM URL.
If url is not in the list of allowed_scm, an error will be raised.
:param str url: The unmodified scmurl
:param str branch: The optional source control branch. This defaults to "master" when git
is used.
:param list allowed_scm: The optional list of allowed SCM URL prefixes
:param bool allow_local: Allow SCM URLs that start with "file://"
:raises: Forbidden or ValidationError
"""
if allowed_scm:
if not (url.startswith(tuple(allowed_scm)) or
(allow_local and url.startswith("file://"))):
raise Forbidden("%s is not in the list of allowed SCMs" % url)
# If we are given the option for the git protocol or the http(s) protocol,
# then just use http(s)
if re.match(r"(git\+http(?:s)?:\/\/)", url):
url = url[4:]
url = url.rstrip("/")
self.url = url
self.sourcedir = None
# once we have more than one SCM provider, we will need some more
# sophisticated lookup logic
for scmtype, schemes in SCM.types.items():
if self.url.startswith(schemes):
self.scheme = scmtype
break
else:
raise ValidationError("Invalid SCM URL: %s" % url)
# git is the only one supported SCM provider atm
if self.scheme == "git":
match = re.search(
r"^(?P<repository>.*/(?P<name>[^?]*))(\?#(?P<commit>.*))?",
url)
self.repository = match.group("repository")
self.name = match.group("name")
self.repository_root = self.repository[:-len(self.name)]
if self.name.endswith(".git"):
self.name = self.name[:-4]
self.commit = match.group("commit")
self.branch = branch if branch else "master"
self.version = None
self._cloned = False
else:
raise ValidationError("Unhandled SCM scheme: %s" % self.scheme)
def _check_buildopts_arches(mmd, arches):
"""
Returns buildopts arches if valid, or otherwise the arches provided.
:param mmd: Module MetaData
:param arches: list of architectures
:return: list of architectures
"""
buildopts = mmd.get_buildopts()
if not buildopts:
return arches
try:
buildopts_arches = buildopts.get_arches()
except AttributeError:
# libmodulemd version < 2.8.3
return arches
# Must be a subset of the input module arches
unsupported_arches = set(buildopts_arches) - set(arches)
if unsupported_arches:
raise ValidationError(
"The following buildopts arches are not supported with these "
"buildrequires: %r" % unsupported_arches)
if buildopts_arches:
log.info(
"Setting build arches of %s to %r based on the buildopts arches." %
(mmd.get_nsvc(), buildopts_arches))
return buildopts_arches
return arches
def submit_module_build_from_yaml(
db_session, username, handle, params, stream=None, skiptests=False
):
yaml_file = to_text_type(handle.read())
mmd = load_mmd(yaml_file)
dt = datetime.utcfromtimestamp(int(time.time()))
if hasattr(handle, "filename"):
def_name = str(os.path.splitext(os.path.basename(handle.filename))[0])
elif not mmd.get_module_name():
raise ValidationError(
"The module's name was not present in the modulemd file. Please use the "
'"module_name" parameter'
)
def_version = int(dt.strftime("%Y%m%d%H%M%S"))
module_name = mmd.get_module_name() or def_name
module_stream = stream or mmd.get_stream_name() or "master"
if module_name != mmd.get_module_name() or module_stream != mmd.get_stream_name():
# This is how you set the name and stream in the modulemd
mmd = mmd.copy(module_name, module_stream)
mmd.set_version(mmd.get_version() or def_version)
if skiptests:
buildopts = mmd.get_buildopts() or Modulemd.Buildopts()
macros = buildopts.get_rpm_macros() or ""
buildopts.set_rpm_macros(macros + "\n\n%__spec_check_pre exit 0\n")
mmd.set_buildopts(buildopts)
return submit_module_build(db_session, username, mmd, params)
def get(self, api_version, id):
module = models.ModuleBuild.get_by_id(db.session, id)
if not module:
raise ValidationError("The module could not be found")
if conf.system == "koji":
# We are importing KojiContentGenerator here so we can generate the final modulemds.
# If we imported this regularly we would have gotten a circular import error.
from module_build_service.builder.KojiContentGenerator import KojiContentGenerator # noqa
cg = KojiContentGenerator(module, conf)
finalmmds = cg.get_final_mmds()
else:
raise ValidationError(
"Configured builder not able to generate final modulemds!")
return jsonify(finalmmds), 200
def _is_eol_in_pdc(name, stream):
""" Check PDC if the module name:stream is no longer active. """
params = {"type": "module", "global_component": name, "name": stream}
url = conf.pdc_url + "/component-branches/"
response = requests.get(url, params=params)
if not response:
raise ValidationError("Failed to talk to PDC {}{}".format(
response, response.text))
data = response.json()
results = data["results"]
if not results:
raise ValidationError("No such module {}:{} found at {}".format(
name, stream, response.request.url))
# If the module is active, then it is not EOL and vice versa.
return not results[0]["active"]
def _dict_from_request(request):
if "multipart/form-data" in request.headers.get("Content-Type", ""):
data = request.form.to_dict()
else:
try:
data = json.loads(request.get_data().decode("utf-8"))
except Exception:
log.exception("Invalid JSON submitted")
raise ValidationError("Invalid JSON submitted")
return data
def _add_order_by_clause(flask_request, query, column_source):
"""
Orders the given SQLAlchemy query based on the GET arguments provided.
:param flask_request: a Flask request object
:param query: a SQLAlchemy query object
:param column_source: a SQLAlchemy database model
:return: a SQLAlchemy query object
"""
order_by = flask_request.args.getlist("order_by")
order_desc_by = flask_request.args.getlist("order_desc_by")
# Default to ordering by ID in descending order
descending = True
requested_order = ["id"]
if order_by and order_desc_by:
raise ValidationError(
"You may not specify both order_by and order_desc_by")
elif order_by:
descending = False
requested_order = order_by
elif order_desc_by:
descending = True
requested_order = order_desc_by
column_dict = dict(column_source.__table__.columns)
order_args = []
for column_name in requested_order:
if column_name not in column_dict:
raise ValidationError(
'An invalid ordering key of "{}" was supplied'.format(
column_name))
column = column_dict[column_name]
# If the version column is provided, cast it as an integer so the sorting is correct
if column_name == "version":
column = sqlalchemy.cast(column, sqlalchemy.BigInteger)
if descending:
column = column.desc()
order_args.append(column)
return query.order_by(*order_args)
def get(self, api_version, id):
id_flag = request.args.get("id")
if id_flag:
endpoint = request.endpoint.split("s_list")[0]
raise ValidationError(
'The "id" query option is invalid. Did you mean to go to "{0}"?'
.format(url_for(endpoint, api_version=api_version,
id=id_flag)))
verbose_flag = request.args.get("verbose", "false").lower()
short_flag = request.args.get("short", "false").lower()
json_func_kwargs = {}
json_func_name = "json"
if id is None:
# Lists all tracked builds
p_query = self.query_filter(request)
json_data = {
"meta": pagination_metadata(p_query, api_version, request.args)
}
if verbose_flag == "true" or verbose_flag == "1":
json_func_name = "extended_json"
json_func_kwargs["show_state_url"] = True
json_func_kwargs["api_version"] = api_version
elif short_flag == "true" or short_flag == "1":
if p_query.items and hasattr(p_query.items[0], "short_json"):
json_func_name = "short_json"
if json_func_name == "json" or json_func_name == "extended_json":
# Only ModuleBuild.json and ModuleBuild.extended_json has argument db_session
json_func_kwargs["db_session"] = db.session
json_data["items"] = [
getattr(item, json_func_name)(**json_func_kwargs)
for item in p_query.items
]
return jsonify(json_data), 200
else:
# Lists details for the specified build
instance = self.model.query.filter_by(id=id).first()
if instance:
if verbose_flag == "true" or verbose_flag == "1":
json_func_name = "extended_json"
json_func_kwargs["show_state_url"] = True
json_func_kwargs["api_version"] = api_version
elif short_flag == "true" or short_flag == "1":
if getattr(instance, "short_json", None):
json_func_name = "short_json"
if json_func_name == "json" or json_func_name == "extended_json":
# Only ModuleBuild.json and ModuleBuild.extended_json has argument db_session
json_func_kwargs["db_session"] = db.session
return jsonify(
getattr(instance, json_func_name)(**json_func_kwargs)), 200
else:
raise NotFound("No such %s found." % self.kind)
def _validate_dep_overrides_format(self, key):
"""
Validate any dependency overrides provided to the API.
:param str key: the override key to validate
:raises ValidationError: when the overrides are an invalid format
"""
if not self.data.get(key):
return
invalid_override_msg = (
'The "{}" parameter must be an object with the keys as module '
"names and the values as arrays of streams".format(key))
if not isinstance(self.data[key], dict):
raise ValidationError(invalid_override_msg)
for streams in self.data[key].values():
if not isinstance(streams, list):
raise ValidationError(invalid_override_msg)
for stream in streams:
if not isinstance(stream, string_types):
raise ValidationError(invalid_override_msg)
def _validate_stream_name_for_static_context(module, stream, context,
dep_type):
"""
Validates the the streams of static contexts.
:param str module: name of the module
:param str stream: name of the stream
:param str context: name of the context
:param str dep_type: type of the dependencies the module stream belongs to i. e. requires,
buildrequires.
:return None
"""
if not stream:
raise ValidationError(
("The '{module}' module in '{dep_type}' of the '{context}' static "
"context has no stream defined.").format(module=module,
dep_type=dep_type,
context=context))
stream_type = type(stream)
if stream_type is not str:
raise ValidationError(
("The module '{module}' in '{dep_type}' of the '{context}' "
"static context is of type '{stream_type}' should be "
"'str' type.").format(stream=stream,
module=module,
dep_type=dep_type,
context=context,
stream_type=stream_type))
if stream.startswith("-"):
raise ValidationError((
"The '{stream}' stream of module '{module}' in '{dep_type}' of the"
"'{context}' static contexts is using stream expansion. Usage of "
"stream expansion with static context is forbidden.").format(
stream=stream,
module=module,
dep_type=dep_type,
context=context))
def validate(self, skip_branch=False, skip_optional_params=False):
if "scmurl" not in self.data:
log.error("Missing scmurl")
raise ValidationError("Missing scmurl")
url = self.data["scmurl"]
allowed_prefix = any(url.startswith(prefix) for prefix in conf.scmurls)
if not conf.allow_custom_scmurls and not allowed_prefix:
log.error("The submitted scmurl %r is not allowed" % url)
raise Forbidden("The submitted scmurl %s is not allowed" % url)
if not get_scm_url_re().match(url):
log.error("The submitted scmurl %r is not valid" % url)
raise ValidationError("The submitted scmurl %s is not valid" % url)
if not skip_branch and "branch" not in self.data:
log.error("Missing branch")
raise ValidationError("Missing branch")
if "module_name" in self.data:
log.error(
"Module name override is only allowed when a YAML file is submitted"
)
raise ValidationError(
"Module name override is only allowed when a YAML file is submitted"
)
if "module_stream" in self.data:
log.error(
"Stream name override is only allowed when a YAML file is submitted"
)
raise ValidationError(
"Stream name override is only allowed when a YAML file is submitted"
)
if not skip_optional_params:
self.validate_optional_params()
def get(self, api_version, id, model):
if not model:
raise ValidationError(
"Model is not set for this log messages endpoint")
query = model.query.filter_by(id=id).first().log_messages.order_by(
models.LogMessage.time_created.desc())
page = request.args.get("page", 1, type=int)
per_page = request.args.get("per_page", 10, type=int)
p_query = query.paginate(page, per_page, False)
request_args = {"id": id}
json_data = {
"meta": pagination_metadata(p_query, api_version, request_args)
}
json_data["messages"] = [
getattr(message, "json")() for message in p_query.items
]
return jsonify(json_data), 200
def __init__(self, request, data=None):
self.username, self.groups = module_build_service.web.auth.get_user(
request)
self.data = data or _dict_from_request(request)
# canonicalize and validate scratch option
if "scratch" in self.data and str_to_bool(str(self.data["scratch"])):
self.data["scratch"] = True
if conf.modules_allow_scratch is not True:
raise Forbidden("Scratch builds are not enabled")
else:
self.data["scratch"] = False
# canonicalize and validate srpms list
if "srpms" in self.data and self.data["srpms"]:
if not self.data["scratch"]:
raise Forbidden(
"srpms may only be specified for scratch builds")
if not isinstance(self.data["srpms"], list):
raise ValidationError("srpms must be specified as a list")
else:
self.data["srpms"] = []
def verify(self):
"""
Verifies that the information provided by a user in SCM URL and branch
matches the information in SCM repository. For example verifies that
the commit hash really belongs to the provided branch.
:raises ValidationError
"""
if not self.sourcedir:
raise ProgrammingError("Do .checkout() first.")
found = False
branches = SCM._run(["git", "branch", "-r", "--contains", self.commit],
chdir=self.sourcedir)[1]
for branch in branches.decode("utf-8").split("\n"):
branch = branch.strip()
if branch[len("origin/"):] == self.branch:
found = True
break
if not found:
raise ValidationError("Commit %s is not in branch %s." %
(self.commit, self.branch))
def submit_module_build(db_session, username, mmd, params):
"""
Submits new module build.
:param db_session: SQLAlchemy session object.
:param str username: Username of the build's owner.
:param Modulemd.ModuleStream mmd: Modulemd defining the build.
:param dict params: the API parameters passed in by the user
:rtype: list with ModuleBuild
:return: List with submitted module builds.
"""
log.debug(
"Submitted %s module build for %s:%s:%s",
("scratch" if params.get("scratch", False) else "normal"),
mmd.get_module_name(),
mmd.get_stream_name(),
mmd.get_version(),
)
raise_if_stream_ambigous = False
default_streams = {}
# For local builds, we want the user to choose the exact stream using the default_streams
# in case there are multiple streams to choose from and raise an exception otherwise.
if "local_build" in params:
raise_if_stream_ambigous = True
# Get the default_streams if set.
if "default_streams" in params:
default_streams = params["default_streams"]
xmd = mmd.get_xmd()
# we check if static contexts are enabled by the `contexts` property defined by the user i
# as an build option.
static_context = "mbs_options" in xmd and "contexts" in xmd["mbs_options"]
input_mmds = generate_mmds_from_static_contexts(mmd) if static_context else [mmd]
mmds = []
for mmd in input_mmds:
validate_mmd(mmd)
_apply_dep_overrides(mmd, params)
_modify_buildtime_streams(db_session, mmd, resolve_base_module_virtual_streams)
_process_support_streams(db_session, mmd, params)
mmds += generate_expanded_mmds(db_session, mmd, raise_if_stream_ambigous,
default_streams, static_context=static_context)
if not mmds:
raise ValidationError(
"No dependency combination was satisfied. Please verify the "
"buildrequires in your modulemd have previously been built."
)
modules = []
# True if all module builds are skipped so MBS will actually not rebuild
# anything. To keep the backward compatibility, we need to raise an exception
# later in the end of this method.
all_modules_skipped = True
for mmd in mmds:
# Prefix the version of the modulemd based on the base module it buildrequires
version = get_prefixed_version(mmd)
mmd.set_version(version)
nsvc = mmd.get_nsvc()
log.debug("Checking whether module build already exists: %s.", nsvc)
module = models.ModuleBuild.get_build_from_nsvc(db_session, *nsvc.split(":"))
if module and not params.get("scratch", False):
if module.state != models.BUILD_STATES["failed"]:
log.info(
"Skipping rebuild of %s, only rebuild of modules in failed state is allowed.",
nsvc,
)
modules.append(module)
continue
rebuild_strategy = params.get("rebuild_strategy")
if rebuild_strategy and module.rebuild_strategy != rebuild_strategy:
raise ValidationError(
'You cannot change the module\'s "rebuild_strategy" when '
"resuming a module build"
)
log.debug("Resuming existing module build %r" % module)
# Reset all component builds that didn't complete
for component in module.component_builds:
if not component.is_waiting_for_build and not component.is_completed:
component.state = None
component.state_reason = None
db_session.add(component)
module.username = username
prev_state = module.previous_non_failed_state
if prev_state == models.BUILD_STATES["init"]:
transition_to = models.BUILD_STATES["init"]
else:
transition_to = models.BUILD_STATES["wait"]
module.batch = 0
module.transition(db_session, conf, transition_to, "Resubmitted by %s" % username)
db_session.commit()
log.info("Resumed existing module build in previous state %s" % module.state)
else:
# make NSVC unique for every scratch build
context_suffix = ""
if params.get("scratch", False):
log.debug("Checking for existing scratch module builds by NSVC")
scrmods = models.ModuleBuild.get_scratch_builds_from_nsvc(
db_session, *nsvc.split(":"))
scrmod_contexts = [scrmod.context for scrmod in scrmods]
log.debug(
"Found %d previous scratch module build context(s): %s",
len(scrmods), ",".join(scrmod_contexts),
)
# append incrementing counter to context
context_suffix = "_" + str(len(scrmods) + 1)
mmd.set_context(mmd.get_context() + context_suffix)
else:
# In case the branch is defined, check whether user is allowed to submit
# non-scratch build from this branch. Note that the branch is always defined
# for official builds from SCM, because it is requested in views.py.
branch = params.get("branch")
if branch:
for regex in conf.scratch_build_only_branches:
branch_search = re.search(regex, branch)
if branch_search:
raise ValidationError(
"Only scratch module builds can be built from this branch."
)
log.debug("Creating new module build")
module = models.ModuleBuild.create(
db_session,
conf,
name=mmd.get_module_name(),
stream=mmd.get_stream_name(),
version=str(mmd.get_version()),
modulemd=mmd_to_str(mmd),
scmurl=params.get("scmurl"),
username=username,
rebuild_strategy=params.get("rebuild_strategy"),
reused_module_id=params.get("reuse_components_from"),
scratch=params.get("scratch"),
srpms=params.get("srpms"),
)
module.build_context, module.runtime_context, module.context, \
module.build_context_no_bms = module.contexts_from_mmd(module.modulemd)
xmd = mmd.get_xmd()
if xmd["mbs"].get("static_context"):
module.context = mmd.get_context()
module.context += context_suffix
db_session.commit()
notify_on_module_state_change(
# Note the state is "init" here...
module.json(db_session, show_tasks=False)
)
all_modules_skipped = False
modules.append(module)
log.info('The user "%s" submitted the build "%s"', username, nsvc)
if all_modules_skipped:
err_msg = (
"Module (state=%s) already exists. Only a new build, resubmission of "
"a failed build or build against new buildrequirements is "
"allowed." % module.state
)
log.error(err_msg)
raise Conflict(err_msg)
return modules
def filter_module_builds(flask_request):
"""
Returns a flask_sqlalchemy.Pagination object based on the request parameters
:param request: Flask request object
:return: flask_sqlalchemy.Pagination
"""
search_query = dict()
special_columns = {
"time_submitted",
"time_modified",
"time_completed",
"state",
"stream_version_lte",
"virtual_stream",
}
columns = models.ModuleBuild.__table__.columns.keys()
for key in set(request.args.keys()) - special_columns:
# Only filter on valid database columns but skip columns that are treated specially or
# ignored
if key in columns:
search_query[key] = flask_request.args[key]
# Multiple states can be supplied => or-ing will take place
states = flask_request.args.getlist("state")
search_states = []
for state in states:
if state.isdigit():
search_states.append(state)
else:
if state in models.BUILD_STATES:
search_states.append(models.BUILD_STATES[state])
else:
raise ValidationError("Invalid state was supplied: %s" % state)
nsvc = flask_request.args.get("nsvc", None)
if nsvc:
nsvc_parts = nsvc.split(":")
query_keys = ["name", "stream", "version", "context"]
for key, part in zip(query_keys, nsvc_parts):
search_query[key] = part
rpm = flask_request.args.get("rpm", None)
koji_tags = []
if rpm:
if conf.system == "koji":
# we are importing the koji builder here so we can search for the rpm metadata
# from koji. If we imported this regulary we would have gotten a circular import error.
from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder # noqa
koji_tags = KojiModuleBuilder.get_rpm_module_tag(rpm)
else:
raise ValidationError(
"Configured builder does not allow to search by rpm binary name!"
)
query = models.ModuleBuild.query
if search_query:
query = query.filter_by(**search_query)
if search_states:
query = query.filter(models.ModuleBuild.state.in_(search_states))
if koji_tags:
query = query.filter(
models.ModuleBuild.koji_tag.in_(koji_tags)).filter_by(
**search_query)
# This is used when filtering the date request parameters, but it is here to avoid recompiling
utc_iso_datetime_regex = re.compile(
r"^(?P<datetime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.\d+)?(?:Z|[-+]00(?::00)?)?$"
)
# Filter the query based on date request parameters
for item in ("submitted", "modified", "completed"):
for context in ("before", "after"):
request_arg = "%s_%s" % (item, context) # i.e. submitted_before
iso_datetime_arg = request.args.get(request_arg, None)
if iso_datetime_arg:
iso_datetime_matches = re.match(utc_iso_datetime_regex,
iso_datetime_arg)
if not iso_datetime_matches or not iso_datetime_matches.group(
"datetime"):
raise ValidationError(
'An invalid Zulu ISO 8601 timestamp was provided for the "%s" parameter'
% request_arg)
# Converts the ISO 8601 string to a datetime object for SQLAlchemy to use to filter
item_datetime = datetime.strptime(
iso_datetime_matches.group("datetime"),
"%Y-%m-%dT%H:%M:%S")
# Get the database column to filter against
column = getattr(models.ModuleBuild, "time_" + item)
if context == "after":
query = query.filter(column >= item_datetime)
elif context == "before":
query = query.filter(column <= item_datetime)
# Multiple virtual_streams can be supplied for "or" logic filtering
virtual_streams = flask_request.args.getlist("virtual_stream")
query = models.ModuleBuild._add_virtual_streams_filter(
db.session, query, virtual_streams)
stream_version_lte = flask_request.args.get("stream_version_lte")
if stream_version_lte is not None:
invalid_error = (
"An invalid value of stream_version_lte was provided. It must be an "
"integer or float greater than or equal to 10000.")
try:
stream_version_lte = float(stream_version_lte)
except (TypeError, ValueError):
raise ValidationError(invalid_error)
if stream_version_lte < 10000:
raise ValidationError(invalid_error)
query = models.ModuleBuild._add_stream_version_lte_filter(
db.session, query, stream_version_lte)
br_joined = False
module_br_alias = None
for item in (
"base_module_br",
"name",
"stream",
"version",
"context",
"stream_version",
"stream_version_lte",
"stream_version_gte",
):
if item == "base_module_br":
request_arg_name = item
else:
request_arg_name = "base_module_br_{}".format(item)
request_arg = flask_request.args.get(request_arg_name)
if not request_arg:
continue
if not br_joined:
module_br_alias = aliased(models.ModuleBuild, name="module_br")
# Shorten this table name for clarity in the query below
mb_to_br = models.module_builds_to_module_buildrequires
# The following joins get added:
# JOIN module_builds_to_module_buildrequires
# ON module_builds_to_module_buildrequires.module_id = module_builds.id
# JOIN module_builds AS module_br
# ON module_builds_to_module_buildrequires.module_buildrequire_id = module_br.id
query = query.join(
mb_to_br, mb_to_br.c.module_id == models.ModuleBuild.id).join(
module_br_alias,
mb_to_br.c.module_buildrequire_id == module_br_alias.id)
br_joined = True
if item == "base_module_br":
try:
name, stream, version, context = flask_request.args[
"base_module_br"].split(":")
except ValueError:
raise ValidationError(
'The filter argument for "base_module_br" must be in the format of N:S:V:C'
)
query = query.filter(
module_br_alias.name == name,
module_br_alias.stream == stream,
module_br_alias.version == version,
module_br_alias.context == context,
)
elif item.endswith("_lte"):
column = getattr(module_br_alias, item[:-4])
query = query.filter(column <= request_arg)
elif item.endswith("_gte"):
column = getattr(module_br_alias, item[:-4])
query = query.filter(column >= request_arg)
else:
column = getattr(module_br_alias, item)
query = query.filter(column == request_arg)
query = _add_order_by_clause(flask_request, query, models.ModuleBuild)
page = flask_request.args.get("page", 1, type=int)
per_page = flask_request.args.get("per_page", 10, type=int)
return query.paginate(page, per_page, False)
def _apply_dep_overrides(mmd, params):
"""
Apply the dependency override parameters (if specified) on the input modulemd.
:param Modulemd.ModuleStream mmd: the modulemd to apply the overrides on
:param dict params: the API parameters passed in by the user
:raises ValidationError: if one of the overrides doesn't apply
"""
dep_overrides = {
"buildrequires": copy.copy(params.get("buildrequire_overrides", {})),
"requires": copy.copy(params.get("require_overrides", {})),
}
# Parse the module's branch to determine if it should override the stream of the buildrequired
# module defined in conf.br_stream_override_module
branch_search = None
if params.get("branch") and conf.br_stream_override_module and conf.br_stream_override_regexes:
# Only parse the branch for a buildrequire override if the user didn't manually specify an
# override for the module specified in conf.br_stream_override_module
if not dep_overrides["buildrequires"].get(conf.br_stream_override_module):
branch_search = None
for regex in conf.br_stream_override_regexes:
branch_search = re.search(regex, params["branch"])
if branch_search:
log.debug(
"The stream override regex `%s` matched the branch %s",
regex,
params["branch"],
)
break
else:
log.debug('No stream override regexes matched the branch "%s"', params["branch"])
# If a stream was parsed from the branch, then add it as a stream override for the module
# specified in conf.br_stream_override_module
if branch_search:
# Concatenate all the groups that are not None together to get the desired stream.
# This approach is taken in case there are sections to ignore.
# For instance, if we need to parse `el8.0.0` from `rhel-8.0.0`.
parsed_stream = "".join(group for group in branch_search.groups() if group)
if parsed_stream:
dep_overrides["buildrequires"][conf.br_stream_override_module] = [parsed_stream]
log.info(
'The buildrequired stream of "%s" was overriden with "%s" based on the branch "%s"',
conf.br_stream_override_module, parsed_stream, params["branch"],
)
else:
log.warning(
'The regex `%s` only matched empty capture groups on the branch "%s". The regex is '
" invalid and should be rewritten.",
regex, params["branch"],
)
unused_dep_overrides = {
"buildrequires": set(dep_overrides["buildrequires"].keys()),
"requires": set(dep_overrides["requires"].keys()),
}
deps = mmd.get_dependencies()
for dep in deps:
overridden = False
new_dep = Modulemd.Dependencies()
for dep_type, overrides in dep_overrides.items():
if dep_type == "buildrequires":
mmd_dep_type = "buildtime"
else:
mmd_dep_type = "runtime"
# Get the existing streams
reqs = deps_to_dict(dep, mmd_dep_type)
# Get the method to add a new stream for this dependency type
# (e.g. add_buildtime_stream)
add_func = getattr(new_dep, "add_{}_stream".format(mmd_dep_type))
add_empty_func = getattr(
new_dep, "set_empty_{}_dependencies_for_module".format(mmd_dep_type))
for name, streams in reqs.items():
if name in dep_overrides[dep_type]:
streams_to_add = dep_overrides[dep_type][name]
unused_dep_overrides[dep_type].remove(name)
overridden = True
else:
streams_to_add = reqs[name]
if not streams_to_add:
add_empty_func(name)
else:
for stream in streams_to_add:
add_func(name, stream)
if overridden:
# Set the overridden streams
mmd.remove_dependencies(dep)
mmd.add_dependencies(new_dep)
for dep_type in unused_dep_overrides.keys():
# If a stream override was applied from parsing the branch and it wasn't applicable,
# just ignore it
if branch_search and conf.br_stream_override_module in unused_dep_overrides[dep_type]:
unused_dep_overrides[dep_type].remove(conf.br_stream_override_module)
if unused_dep_overrides[dep_type]:
raise ValidationError(
"The {} overrides for the following modules aren't applicable: {}".format(
dep_type[:-1], ", ".join(sorted(unused_dep_overrides[dep_type])))
)
def fetch_mmd(url,
branch=None,
allow_local_url=False,
whitelist_url=False,
mandatory_checks=True):
td = None
scm = None
try:
log.debug("Verifying modulemd")
td = tempfile.mkdtemp()
if whitelist_url:
scm = module_build_service.common.scm.SCM(url, branch, [url],
allow_local_url)
else:
scm = module_build_service.common.scm.SCM(url, branch,
conf.scmurls,
allow_local_url)
scm.checkout(td)
if not whitelist_url and mandatory_checks:
scm.verify()
cofn = scm.get_module_yaml()
mmd = load_mmd_file(cofn)
finally:
try:
if td is not None:
shutil.rmtree(td)
except Exception as e:
log.warning("Failed to remove temporary directory {!r}: {}".format(
td, str(e)))
if conf.check_for_eol:
if _is_eol_in_pdc(scm.name, scm.branch):
raise ValidationError(
"Module {}:{} is marked as EOL in PDC.".format(
scm.name, scm.branch))
if not mandatory_checks:
return mmd, scm
# If the name was set in the modulemd, make sure it matches what the scmurl
# says it should be
if mmd.get_module_name() and mmd.get_module_name() != scm.name:
if not conf.allow_name_override_from_scm:
raise ValidationError(
'The name "{0}" that is stored in the modulemd is not valid'.
format(mmd.get_module_name()))
else:
# Set the module name
mmd = mmd.copy(scm.name)
# If the stream was set in the modulemd, make sure it matches what the repo
# branch is
if mmd.get_stream_name() and mmd.get_stream_name() != scm.branch:
if not conf.allow_stream_override_from_scm:
raise ValidationError(
'The stream "{0}" that is stored in the modulemd does not match the branch "{1}"'
.format(mmd.get_stream_name(), scm.branch))
else:
# Set the module stream
mmd = mmd.copy(mmd.get_module_name(), scm.branch)
# If the version is in the modulemd, throw an exception since the version
# since the version is generated by MBS
if mmd.get_version():
raise ValidationError(
'The version "{0}" is already defined in the modulemd but it shouldn\'t be since the '
"version is generated based on the commit time".format(
mmd.get_version()))
else:
mmd.set_version(int(scm.version))
return mmd, scm
def record_component_builds(mmd,
module,
initial_batch=1,
previous_buildorder=None,
main_mmd=None):
# Imported here to allow import of utils in GenericBuilder.
from module_build_service.builder import GenericBuilder
# When main_mmd is set, merge the metadata from this mmd to main_mmd,
# otherwise our current mmd is main_mmd.
if main_mmd:
# Check for components that are in both MMDs before merging since MBS
# currently can't handle that situation.
main_mmd_rpms = main_mmd.get_rpm_component_names()
mmd_rpms = mmd.get_rpm_component_names()
duplicate_components = [
rpm for rpm in main_mmd_rpms if rpm in mmd_rpms
]
if duplicate_components:
error_msg = (
'The included module "{0}" in "{1}" have the following '
"conflicting components: {2}".format(
mmd.get_module_name(), main_mmd.get_module_name(),
", ".join(duplicate_components)))
raise UnprocessableEntity(error_msg)
merge_included_mmd(main_mmd, mmd)
else:
main_mmd = mmd
# If the modulemd yaml specifies components, then submit them for build
rpm_components = [
mmd.get_rpm_component(name) for name in mmd.get_rpm_component_names()
]
module_components = [
mmd.get_module_component(name)
for name in mmd.get_module_component_names()
]
all_components = list(rpm_components) + list(module_components)
if not all_components:
return
# Get map of packages that have SRPM overrides
srpm_overrides = get_module_srpm_overrides(module)
rpm_weights = GenericBuilder.get_build_weights(
[c.get_name() for c in rpm_components])
all_components.sort(key=lambda x: x.get_buildorder())
# We do not start with batch = 0 here, because the first batch is
# reserved for module-build-macros. First real components must be
# planned for batch 2 and following.
batch = initial_batch
for component in all_components:
# Increment the batch number when buildorder increases.
if previous_buildorder != component.get_buildorder():
previous_buildorder = component.get_buildorder()
batch += 1
# If the component is another module, we fetch its modulemd file
# and record its components recursively with the initial_batch
# set to our current batch, so the components of this module
# are built in the right global order.
if isinstance(component, Modulemd.ComponentModule):
full_url = component.get_repository() + "?#" + component.get_ref()
# It is OK to whitelist all URLs here, because the validity
# of every URL have been already checked in format_mmd(...).
included_mmd = fetch_mmd(full_url, whitelist_url=True)[0]
format_mmd(included_mmd, module.scmurl, module, db_session,
srpm_overrides)
batch = record_component_builds(included_mmd, module, batch,
previous_buildorder, main_mmd)
continue
package = component.get_name()
if package in srpm_overrides:
component_ref = None
full_url = srpm_overrides[package]
log.info('Building custom SRPM "{0}"'
" for package {1}".format(full_url, package))
else:
component_ref = mmd.get_xmd()["mbs"]["rpms"][package]["ref"]
full_url = component.get_repository() + "?#" + component_ref
# Skip the ComponentBuild if it already exists in database. This can happen
# in case of module build resubmition.
existing_build = models.ComponentBuild.from_component_name(
db_session, package, module.id)
if existing_build:
# Check that the existing build has the same most important attributes.
# This should never be a problem, but it's good to be defensive here so
# we do not mess things during resubmition.
if (existing_build.batch != batch
or existing_build.scmurl != full_url
or existing_build.ref != component_ref):
raise ValidationError(
"Component build %s of module build %s (id: %d) already "
"exists in database, but its attributes are different from"
" resubmitted one." %
(component.get_name(), module.name, module.id))
continue
build = models.ComponentBuild(module_id=module.id,
package=package,
format="rpms",
scmurl=full_url,
batch=batch,
ref=component_ref,
weight=rpm_weights[package],
buildonly=component.get_buildonly())
db_session.add(build)
return batch
def format_mmd(mmd, scmurl, module=None, db_session=None, srpm_overrides=None):
"""
Prepares the modulemd for the MBS. This does things such as replacing the
branches of components with commit hashes and adding metadata in the xmd
dictionary.
:param mmd: the Modulemd.ModuleStream object to format
:param scmurl: the url to the modulemd
:param module: When specified together with `session`, the time_modified
of a module is updated regularly in case this method takes lot of time.
:param db_session: Database session to update the `module`.
:param dict srpm_overrides: Mapping of package names to SRPM links for all
component packages which have custom SRPM overrides specified.
"""
srpm_overrides = srpm_overrides or {}
xmd = mmd.get_xmd()
if "mbs" not in xmd:
xmd["mbs"] = {}
if "scmurl" not in xmd["mbs"]:
xmd["mbs"]["scmurl"] = scmurl or ""
if "commit" not in xmd["mbs"]:
xmd["mbs"]["commit"] = ""
# If module build was submitted via yaml file, there is no scmurl
if scmurl:
scm = module_build_service.common.scm.SCM(scmurl)
# We want to make sure we have the full commit hash for consistency
if module_build_service.common.scm.SCM.is_full_commit_hash(
scm.scheme, scm.commit):
full_scm_hash = scm.commit
else:
full_scm_hash = scm.get_full_commit_hash()
xmd["mbs"]["commit"] = full_scm_hash
if mmd.get_rpm_component_names() or mmd.get_module_component_names():
if "rpms" not in xmd["mbs"]:
xmd["mbs"]["rpms"] = {}
# Add missing data in RPM components
for pkgname in mmd.get_rpm_component_names():
pkg = mmd.get_rpm_component(pkgname)
# In case of resubmit of existing module which have been
# cancelled/failed during the init state, the package
# was maybe already handled by MBS, so skip it in this case.
if pkgname in xmd["mbs"]["rpms"]:
continue
if pkg.get_repository() and not conf.rpms_allow_repository:
raise Forbidden(
"Custom component repositories aren't allowed. "
"%r bears repository %r" % (pkgname, pkg.get_repository()))
if pkg.get_cache() and not conf.rpms_allow_cache:
raise Forbidden("Custom component caches aren't allowed. "
"%r bears cache %r" %
(pkgname, pkg.get_cache()))
if pkg.get_buildafter():
raise ValidationError(
'The usage of "buildafter" is not yet supported')
if not pkg.get_repository():
pkg.set_repository(conf.rpms_default_repository + pkgname)
if not pkg.get_cache():
pkg.set_cache(conf.rpms_default_cache + pkgname)
if not pkg.get_ref():
pkg.set_ref("master")
if not pkg.get_arches():
for arch in conf.arches:
pkg.add_restricted_arch(arch)
# Add missing data in included modules components
for modname in mmd.get_module_component_names():
mod = mmd.get_module_component(modname)
if mod.get_repository() and not conf.modules_allow_repository:
raise Forbidden("Custom module repositories aren't allowed. "
"%r bears repository %r" %
(modname, mod.get_repository()))
if not mod.get_repository():
mod.set_repository(conf.modules_default_repository + modname)
if not mod.get_ref():
mod.set_ref("master")
# Check that SCM URL is valid and replace potential branches in pkg refs
# by real SCM hash and store the result to our private xmd place in modulemd.
pool = ThreadPool(20)
try:
# Filter out the packages which we have already resolved in possible
# previous runs of this method (can be caused by module build resubmition)
# or which have custom SRPMs and shouldn't be resolved.
pkgs_to_resolve = []
for name in mmd.get_rpm_component_names():
if name not in xmd["mbs"]["rpms"]:
if name in srpm_overrides:
# If this package has a custom SRPM, store an empty
# ref entry so no further verification takes place.
xmd["mbs"]["rpms"][name] = {"ref": None}
else:
pkgs_to_resolve.append(mmd.get_rpm_component(name))
async_result = pool.map_async(_scm_get_latest, pkgs_to_resolve)
# For modules with lot of components, the _scm_get_latest can take a lot of time.
# We need to bump time_modified from time to time, otherwise poller could think
# that module is stuck in "init" state and it would send fake "init" message.
while not async_result.ready():
async_result.wait(60)
if module and db_session:
module.time_modified = datetime.utcnow()
db_session.commit()
pkg_dicts = async_result.get()
finally:
pool.close()
err_msg = ""
for pkg_dict in pkg_dicts:
if pkg_dict["error"]:
err_msg += pkg_dict["error"] + "\n"
else:
pkg_name = pkg_dict["pkg_name"]
pkg_ref = pkg_dict["pkg_ref"]
xmd["mbs"]["rpms"][pkg_name] = {"ref": pkg_ref}
if err_msg:
raise UnprocessableEntity(err_msg)
# Set the modified xmd back to the modulemd
mmd.set_xmd(xmd)
def validate(self):
if ("modulemd" not in self.data and
(not hasattr(request, "files") or "yaml" not in request.files)):
log.error("Invalid file submitted")
raise ValidationError("Invalid file submitted")
self.validate_optional_params()
def validate_optional_params(self):
forbidden_params = [k for k in self.data if k not in self.valid_params]
if forbidden_params:
raise ValidationError(
"The request contains unspecified parameters: {}".format(
", ".join(forbidden_params)))
if not conf.no_auth and "owner" in self.data:
raise ValidationError(
"The request contains 'owner' parameter, however NO_AUTH is not allowed"
)
if not conf.rebuild_strategy_allow_override and "rebuild_strategy" in self.data:
raise ValidationError(
'The request contains the "rebuild_strategy" parameter but '
"overriding the default isn't allowed")
if "rebuild_strategy" in self.data:
if self.data[
"rebuild_strategy"] not in conf.rebuild_strategies_allowed:
raise ValidationError(
'The rebuild method of "{0}" is not allowed. Choose from: {1}.'
.format(self.data["rebuild_strategy"],
", ".join(conf.rebuild_strategies_allowed)))
self._validate_dep_overrides_format("buildrequire_overrides")
self._validate_dep_overrides_format("require_overrides")
if "reuse_components_from" in self.data:
if "rebuild_strategy" in self.data and self.data[
"rebuild_strategy"] == "all":
raise ValidationError(
'You cannot specify the parameter "reuse_components_from" when the '
'"rebuild_strategy" parameter is set to "all"')
invalid_identifier_msg = (
'The parameter "reuse_components_from" contains an invalid module identifier'
)
if isinstance(self.data["reuse_components_from"], int):
reuse_module = models.ModuleBuild.get_by_id(
db.session, self.data["reuse_components_from"])
elif isinstance(self.data["reuse_components_from"], string_types):
try:
n, s, v, c = self.data["reuse_components_from"].split(":")
except ValueError:
raise ValidationError(invalid_identifier_msg)
reuse_module = models.ModuleBuild.get_build_from_nsvc(
db.session, n, s, v, c)
else:
raise ValidationError(invalid_identifier_msg)
if not reuse_module:
raise ValidationError(
'The module in the parameter "reuse_components_from" could not be found'
)
if reuse_module.state != models.BUILD_STATES["ready"]:
raise ValidationError(
'The module in the parameter "reuse_components_from" must be in the ready state'
)
# Normalize the value so that it simplifies any code that uses this value
self.data["reuse_components_from"] = reuse_module.id
def generate_mmds_from_static_contexts(mmd):
"""
This function preexpands the MSE when static contexts which are defined by the `contexts`
property under `mbs_options` in the `xmd` property of the initial modulemd file.
:param Modulemd.ModuleStream mmd: Modulemd metadata with the `contexts` property.
:return list mmds: list of distinct mmds identified by context.
"""
xmd = mmd.get_xmd()
if not xmd:
raise ValidationError(
"The 'xmd' property of the modulemd yaml file is empty!")
if not xmd["mbs_options"].get("contexts"):
raise ValidationError((
"The 'xmd' property of the modulemd yaml file does not contain the"
" 'contexts' key."))
contexts = xmd["mbs_options"].get("contexts")
if not type(contexts) is dict:
raise ValidationError(
"The value of the 'contexts' property needs to be a dict.")
# remove the information about the `contexts` property from the `mbs_options` as we do not
# needed anymore
xmd["mbs_options"].pop("contexts")
# we check if the `mbs_options` dict is empty after we remove the `contexts` key.
# if yes we remove `the mbs_options`.
if not xmd["mbs_options"]:
xmd.pop("mbs_options")
mmd.set_xmd(xmd)
mmds = []
for context, dependencies in contexts.items():
# we copy the mmd so we a get a fresh module for each context.
mmd_copy = mmd.copy()
mmd_copy.set_context(context)
if "buildrequires" not in dependencies:
raise ValidationError(
("The context '{context}' is missing the"
" 'buildrequires' key!").format(context=context))
if "requires" not in dependencies:
raise ValidationError(
"The context '{context}' is missing the 'requires' key!".
format(context=context))
# remove the old deps from the mmd
old_module_deps = mmd_copy.get_dependencies()
for deps in old_module_deps:
mmd_copy.remove_dependencies(deps)
module_deps = Modulemd.Dependencies.new()
# populate the new requires and buildrequires according to the `contexts` property
for module, stream in dependencies["buildrequires"].items():
_validate_stream_name_for_static_context(module, stream, context,
"buildrequires")
module_deps.add_buildtime_stream(module, stream)
for module, stream in dependencies["requires"].items():
_validate_stream_name_for_static_context(module, stream, context,
"requires")
module_deps.add_runtime_stream(module, stream)
mmd_copy.add_dependencies(module_deps)
mmds.append(mmd_copy)
return mmds
