def submit_module_build(db_session, username, mmd, params):
"""
Submits new module build.
:param db_session: SQLAlchemy session object.
:param str username: Username of the build's owner.
:param Modulemd.ModuleStream mmd: Modulemd defining the build.
:param dict params: the API parameters passed in by the user
:rtype: list with ModuleBuild
:return: List with submitted module builds.
"""
log.debug(
"Submitted %s module build for %s:%s:%s",
("scratch" if params.get("scratch", False) else "normal"),
mmd.get_module_name(),
mmd.get_stream_name(),
mmd.get_version(),
)
validate_mmd(mmd)
raise_if_stream_ambigous = False
default_streams = {}
# For local builds, we want the user to choose the exact stream using the default_streams
# in case there are multiple streams to choose from and raise an exception otherwise.
if "local_build" in params:
raise_if_stream_ambigous = True
# Get the default_streams if set.
if "default_streams" in params:
default_streams = params["default_streams"]
_apply_dep_overrides(mmd, params)
_modify_buildtime_streams(db_session, mmd, resolve_base_module_virtual_streams)
_process_support_streams(db_session, mmd, params)
mmds = generate_expanded_mmds(db_session, mmd, raise_if_stream_ambigous, default_streams)
if not mmds:
raise ValidationError(
"No dependency combination was satisfied. Please verify the "
"buildrequires in your modulemd have previously been built."
)
modules = []
# True if all module builds are skipped so MBS will actually not rebuild
# anything. To keep the backward compatibility, we need to raise an exception
# later in the end of this method.
all_modules_skipped = True
for mmd in mmds:
# Prefix the version of the modulemd based on the base module it buildrequires
version = get_prefixed_version(mmd)
mmd.set_version(version)
nsvc = mmd.get_nsvc()
log.debug("Checking whether module build already exists: %s.", nsvc)
module = models.ModuleBuild.get_build_from_nsvc(db_session, *nsvc.split(":"))
if module and not params.get("scratch", False):
if module.state != models.BUILD_STATES["failed"]:
log.info(
"Skipping rebuild of %s, only rebuild of modules in failed state is allowed.",
nsvc,
)
modules.append(module)
continue
rebuild_strategy = params.get("rebuild_strategy")
if rebuild_strategy and module.rebuild_strategy != rebuild_strategy:
raise ValidationError(
'You cannot change the module\'s "rebuild_strategy" when '
"resuming a module build"
)
log.debug("Resuming existing module build %r" % module)
# Reset all component builds that didn't complete
for component in module.component_builds:
if not component.is_waiting_for_build and not component.is_completed:
component.state = None
component.state_reason = None
db_session.add(component)
module.username = username
prev_state = module.previous_non_failed_state
if prev_state == models.BUILD_STATES["init"]:
transition_to = models.BUILD_STATES["init"]
else:
transition_to = models.BUILD_STATES["wait"]
module.batch = 0
module.transition(db_session, conf, transition_to, "Resubmitted by %s" % username)
db_session.commit()
log.info("Resumed existing module build in previous state %s" % module.state)
else:
# make NSVC unique for every scratch build
context_suffix = ""
if params.get("scratch", False):
log.debug("Checking for existing scratch module builds by NSVC")
scrmods = models.ModuleBuild.get_scratch_builds_from_nsvc(
db_session, *nsvc.split(":"))
scrmod_contexts = [scrmod.context for scrmod in scrmods]
log.debug(
"Found %d previous scratch module build context(s): %s",
len(scrmods), ",".join(scrmod_contexts),
)
# append incrementing counter to context
context_suffix = "_" + str(len(scrmods) + 1)
mmd.set_context(mmd.get_context() + context_suffix)
else:
# In case the branch is defined, check whether user is allowed to submit
# non-scratch build from this branch. Note that the branch is always defined
# for official builds from SCM, because it is requested in views.py.
branch = params.get("branch")
if branch:
for regex in conf.scratch_build_only_branches:
branch_search = re.search(regex, branch)
if branch_search:
raise ValidationError(
"Only scratch module builds can be built from this branch."
)
log.debug("Creating new module build")
module = models.ModuleBuild.create(
db_session,
conf,
name=mmd.get_module_name(),
stream=mmd.get_stream_name(),
version=str(mmd.get_version()),
modulemd=mmd_to_str(mmd),
scmurl=params.get("scmurl"),
username=username,
rebuild_strategy=params.get("rebuild_strategy"),
reused_module_id=params.get("reuse_components_from"),
scratch=params.get("scratch"),
srpms=params.get("srpms"),
)
module.build_context, module.runtime_context, module.context, \
module.build_context_no_bms = module.contexts_from_mmd(module.modulemd)
module.context += context_suffix
db_session.commit()
notify_on_module_state_change(
# Note the state is "init" here...
module.json(db_session, show_tasks=False)
)
all_modules_skipped = False
modules.append(module)
log.info('The user "%s" submitted the build "%s"', username, nsvc)
if all_modules_skipped:
err_msg = (
"Module (state=%s) already exists. Only a new build, resubmission of "
"a failed build or build against new buildrequirements is "
"allowed." % module.state
)
log.error(err_msg)
raise Conflict(err_msg)
return modules
def _process_support_streams(db_session, mmd, params):
"""
Check if any buildrequired base modules require a support stream suffix.
This checks the Red Hat Product Pages to see if the buildrequired base module stream has been
released, if yes, then add the appropriate stream suffix.
:param Modulemd.ModuleStream mmd: the modulemd to apply the overrides on
:param dict params: the API parameters passed in by the user
"""
config_msg = (
'Skipping the release date checks for adding a stream suffix since "%s" '
"is not configured"
)
if not conf.product_pages_url:
log.debug(config_msg, "product_pages_url")
return
elif not conf.product_pages_module_streams:
log.debug(config_msg, "product_pages_module_streams")
return
buildrequire_overrides = params.get("buildrequire_overrides", {})
def is_released_as_per_schedule(pp_release):
"""
Check if the specified scheduled task date has been reached. Returns True if it has.
"""
if not conf.product_pages_schedule_task_name:
log.debug(config_msg, "product_pages_schedule_task_name")
return False
schedule_url = "{}/api/v7/releases/{}/schedule-tasks/?fields=name,date_finish".format(
conf.product_pages_url.rstrip("/"), pp_release)
try:
pp_rv = requests.get(schedule_url, timeout=15)
pp_json = pp_rv.json()
# Catch requests failures and JSON parsing errors
except (requests.exceptions.RequestException, ValueError):
log.exception(
"The query to the Product Pages at %s failed. Assuming it is not available.",
schedule_url,
)
return False
name = conf.product_pages_schedule_task_name.lower().strip()
for task in pp_json:
if task['name'].lower().strip() == name:
task_date = task['date_finish']
if datetime.strptime(task_date, "%Y-%m-%d").date() >= datetime.utcnow().date():
log.debug(
"The task date %s hasn't been reached yet. Not adding a stream suffix.",
task_date
)
return False
return True
# Schedule task not available; rely on GA date
return False
def is_released(pp_release, url):
"""
Check if the stream has been released. Return True if it has.
"""
try:
pp_rv = requests.get(url, timeout=15)
pp_json = pp_rv.json()
# Catch requests failures and JSON parsing errors
except (requests.exceptions.RequestException, ValueError):
log.exception(
"The query to the Product Pages at %s failed. Assuming it is not yet released.",
url,
)
return False
ga_date = pp_json.get("ga_date")
if not ga_date:
log.debug("A release date for the release %s could not be determined", pp_release)
return False
if datetime.strptime(ga_date, "%Y-%m-%d").date() >= datetime.utcnow().date():
log.debug(
"The release %s hasn't been released yet. Not adding a stream suffix.",
ga_date
)
return False
return True
def new_streams_func(db_session, name, streams):
if name not in conf.base_module_names:
log.debug("The module %s is not a base module. Skipping the release date check.", name)
return streams
elif name in buildrequire_overrides:
log.debug(
"The module %s is a buildrequire override. Skipping the release date check.", name)
return streams
new_streams = copy.deepcopy(streams)
for i, stream in enumerate(streams):
for regex, values in conf.product_pages_module_streams.items():
if re.match(regex, stream):
log.debug(
'The regex `%s` from the configuration "product_pages_module_streams" '
"matched the stream %s",
regex, stream,
)
stream_suffix, pp_release_template, pp_major_release_template = values
break
else:
log.debug(
'No regexes in the configuration "product_pages_module_streams" matched the '
"stream %s. Skipping the release date check for this stream.",
stream,
)
continue
if stream.endswith(stream_suffix):
log.debug(
'The stream %s already contains the stream suffix of "%s". Skipping the '
"release date check.",
stream, stream_suffix
)
continue
stream_version = models.ModuleBuild.get_stream_version(stream)
if not stream_version:
log.debug("A stream version couldn't be parsed from %s", stream)
continue
# Convert the stream_version float to an int to make the math below deal with only
# integers
stream_version_int = int(stream_version)
# For example 80000 => 8
x = stream_version_int // 10000
# For example 80100 => 1
y = (stream_version_int - x * 10000) // 100
# For example 80104 => 4
z = stream_version_int - x * 10000 - y * 100
# Check if the stream version is x.0.0
if stream_version_int % 10000 == 0 and pp_major_release_template:
# For example, el8.0.0 => rhel-8-0
pp_release = pp_major_release_template.format(x=x, y=y, z=z)
else:
# For example el8.0.1 => rhel-8-0.1
pp_release = pp_release_template.format(x=x, y=y, z=z)
url = "{}/api/v7/releases/{}/?fields=ga_date".format(
conf.product_pages_url.rstrip("/"), pp_release)
if is_released_as_per_schedule(pp_release):
new_stream = stream + stream_suffix
log.info(
'Replacing the buildrequire "%s:%s" with "%s:%s", since the date is met',
name, stream, name, new_stream
)
new_streams[i] = new_stream
elif is_released(pp_release, url):
new_stream = stream + stream_suffix
log.info(
'Replacing the buildrequire "%s:%s" with "%s:%s", since the stream is released',
name, stream, name, new_stream
)
new_streams[i] = new_stream
return new_streams
_modify_buildtime_streams(db_session, mmd, new_streams_func)
def new_streams_func(db_session, name, streams):
if name not in conf.base_module_names:
log.debug("The module %s is not a base module. Skipping the release date check.", name)
return streams
elif name in buildrequire_overrides:
log.debug(
"The module %s is a buildrequire override. Skipping the release date check.", name)
return streams
new_streams = copy.deepcopy(streams)
for i, stream in enumerate(streams):
for regex, values in conf.product_pages_module_streams.items():
if re.match(regex, stream):
log.debug(
'The regex `%s` from the configuration "product_pages_module_streams" '
"matched the stream %s",
regex, stream,
)
stream_suffix, pp_release_template, pp_major_release_template = values
break
else:
log.debug(
'No regexes in the configuration "product_pages_module_streams" matched the '
"stream %s. Skipping the release date check for this stream.",
stream,
)
continue
if stream.endswith(stream_suffix):
log.debug(
'The stream %s already contains the stream suffix of "%s". Skipping the '
"release date check.",
stream, stream_suffix
)
continue
stream_version = models.ModuleBuild.get_stream_version(stream)
if not stream_version:
log.debug("A stream version couldn't be parsed from %s", stream)
continue
# Convert the stream_version float to an int to make the math below deal with only
# integers
stream_version_int = int(stream_version)
# For example 80000 => 8
x = stream_version_int // 10000
# For example 80100 => 1
y = (stream_version_int - x * 10000) // 100
# For example 80104 => 4
z = stream_version_int - x * 10000 - y * 100
# Check if the stream version is x.0.0
if stream_version_int % 10000 == 0 and pp_major_release_template:
# For example, el8.0.0 => rhel-8-0
pp_release = pp_major_release_template.format(x=x, y=y, z=z)
else:
# For example el8.0.1 => rhel-8-0.1
pp_release = pp_release_template.format(x=x, y=y, z=z)
url = "{}/api/v7/releases/{}/?fields=ga_date".format(
conf.product_pages_url.rstrip("/"), pp_release)
if is_released_as_per_schedule(pp_release):
new_stream = stream + stream_suffix
log.info(
'Replacing the buildrequire "%s:%s" with "%s:%s", since the date is met',
name, stream, name, new_stream
)
new_streams[i] = new_stream
elif is_released(pp_release, url):
new_stream = stream + stream_suffix
log.info(
'Replacing the buildrequire "%s:%s" with "%s:%s", since the stream is released',
name, stream, name, new_stream
)
new_streams[i] = new_stream
return new_streams
def handle_collisions_with_base_module_rpms(mmd, arches):
"""
Find any RPMs in the buildrequired base modules that collide with the buildrequired modules.
If a buildrequired module contains RPMs that overlap with RPMs in the buildrequired base
modules, then the NEVRAs of the overlapping RPMs in the base modules will be added as conflicts
in the input modulemd.
:param Modulemd.ModuleStream mmd: the modulemd to find the collisions
:param list arches: the arches to limit the external repo queries to
:raise RuntimeError: when a Koji query fails
"""
log.info(
"Finding any buildrequired modules that collide with the RPMs in the base modules"
)
bm_tags = set()
non_bm_tags = set()
xmd = mmd.get_xmd()
buildrequires = xmd["mbs"]["buildrequires"]
for name, info in buildrequires.items():
if not info["koji_tag"]:
continue
if name in conf.base_module_names:
bm_tags.add(info["koji_tag"])
else:
non_bm_tags.add(info["koji_tag"])
if not (bm_tags and non_bm_tags):
log.info(
"Skipping the collision check since collisions are not possible with these "
"buildrequires")
return
log.debug(
"Querying Koji for the latest RPMs from the buildrequired base modules from the tags: %s",
", ".join(bm_tags),
)
koji_session = get_session(conf, login=False)
bm_rpms = _get_rpms_from_tags(koji_session, list(bm_tags), arches)
# The keys are base module RPM names and the values are sets of RPM NEVRAs with that name
name_to_nevras = {}
for bm_rpm in bm_rpms:
rpm_name = kobo.rpmlib.parse_nvra(bm_rpm)["name"]
name_to_nevras.setdefault(rpm_name, set())
name_to_nevras[rpm_name].add(bm_rpm)
# Clear this out of RAM as soon as possible since this value can be huge
del bm_rpms
log.debug(
"Querying Koji for the latest RPMs from the other buildrequired modules from the tags: %s",
", ".join(non_bm_tags),
)
# This will contain any NEVRAs of RPMs in the base module tag with the same name as those in the
# buildrequired modules
conflicts = set()
non_bm_rpms = _get_rpms_from_tags(koji_session, list(non_bm_tags), arches)
for rpm in non_bm_rpms:
rpm_name = kobo.rpmlib.parse_nvra(rpm)["name"]
if rpm_name in name_to_nevras:
conflicts = conflicts | name_to_nevras[rpm_name]
# Add the conflicting NEVRAs to `ursine_rpms` so the Conflicts are later generated for them
# in the KojiModuleBuilder.
xmd["mbs"]["ursine_rpms"] = list(conflicts)
mmd.set_xmd(xmd)
def _apply_dep_overrides(mmd, params):
"""
Apply the dependency override parameters (if specified) on the input modulemd.
:param Modulemd.ModuleStream mmd: the modulemd to apply the overrides on
:param dict params: the API parameters passed in by the user
:raises ValidationError: if one of the overrides doesn't apply
"""
dep_overrides = {
"buildrequires": copy.copy(params.get("buildrequire_overrides", {})),
"requires": copy.copy(params.get("require_overrides", {})),
}
# Parse the module's branch to determine if it should override the stream of the buildrequired
# module defined in conf.br_stream_override_module
branch_search = None
if params.get("branch") and conf.br_stream_override_module and conf.br_stream_override_regexes:
# Only parse the branch for a buildrequire override if the user didn't manually specify an
# override for the module specified in conf.br_stream_override_module
if not dep_overrides["buildrequires"].get(conf.br_stream_override_module):
branch_search = None
for regex in conf.br_stream_override_regexes:
branch_search = re.search(regex, params["branch"])
if branch_search:
log.debug(
"The stream override regex `%s` matched the branch %s",
regex,
params["branch"],
)
break
else:
log.debug('No stream override regexes matched the branch "%s"', params["branch"])
# If a stream was parsed from the branch, then add it as a stream override for the module
# specified in conf.br_stream_override_module
if branch_search:
# Concatenate all the groups that are not None together to get the desired stream.
# This approach is taken in case there are sections to ignore.
# For instance, if we need to parse `el8.0.0` from `rhel-8.0.0`.
parsed_stream = "".join(group for group in branch_search.groups() if group)
if parsed_stream:
dep_overrides["buildrequires"][conf.br_stream_override_module] = [parsed_stream]
log.info(
'The buildrequired stream of "%s" was overriden with "%s" based on the branch "%s"',
conf.br_stream_override_module, parsed_stream, params["branch"],
)
else:
log.warning(
'The regex `%s` only matched empty capture groups on the branch "%s". The regex is '
" invalid and should be rewritten.",
regex, params["branch"],
)
unused_dep_overrides = {
"buildrequires": set(dep_overrides["buildrequires"].keys()),
"requires": set(dep_overrides["requires"].keys()),
}
deps = mmd.get_dependencies()
for dep in deps:
overridden = False
new_dep = Modulemd.Dependencies()
for dep_type, overrides in dep_overrides.items():
if dep_type == "buildrequires":
mmd_dep_type = "buildtime"
else:
mmd_dep_type = "runtime"
# Get the existing streams
reqs = deps_to_dict(dep, mmd_dep_type)
# Get the method to add a new stream for this dependency type
# (e.g. add_buildtime_stream)
add_func = getattr(new_dep, "add_{}_stream".format(mmd_dep_type))
add_empty_func = getattr(
new_dep, "set_empty_{}_dependencies_for_module".format(mmd_dep_type))
for name, streams in reqs.items():
if name in dep_overrides[dep_type]:
streams_to_add = dep_overrides[dep_type][name]
unused_dep_overrides[dep_type].remove(name)
overridden = True
else:
streams_to_add = reqs[name]
if not streams_to_add:
add_empty_func(name)
else:
for stream in streams_to_add:
add_func(name, stream)
if overridden:
# Set the overridden streams
mmd.remove_dependencies(dep)
mmd.add_dependencies(new_dep)
for dep_type in unused_dep_overrides.keys():
# If a stream override was applied from parsing the branch and it wasn't applicable,
# just ignore it
if branch_search and conf.br_stream_override_module in unused_dep_overrides[dep_type]:
unused_dep_overrides[dep_type].remove(conf.br_stream_override_module)
if unused_dep_overrides[dep_type]:
raise ValidationError(
"The {} overrides for the following modules aren't applicable: {}".format(
dep_type[:-1], ", ".join(sorted(unused_dep_overrides[dep_type])))
)
def build_task_finalize(msg_id,
task_id,
build_new_state,
build_name,
build_version,
build_release,
module_build_id=None,
state_reason=None):
"""Called when corresponding Koji build task of a component build finishes
When a task finishes, the task could be in state COMPLETE, FAILED or CANCELED.
:param str msg_id: the original id of the message being handled which is
received from the message bus.
:param int task_id: the Koji build task id.
:param int build_new_state: the state of the build. Refer to
``koji.BUILD_STATES`` for details. For this handler, values could be
the corresponding integer value of COMPLETE, FAILED or CANCELED.
:param str build_name: the build name.
:param str build_version: the build version.
:param str build_release: the build release.
:param int module_build_id: optionally set when this event handler is
scheduled from internal rather than just handling the received message.
When set, the value should be the id of module build having the
component build just built by the finished task.
:param str state_reason: optional. When set a reason explicitly, the
corresponding component build will have this reason as the
``state_reason``. Otherwise, a custom reason will be set for a failed
build.
"""
# First, find our ModuleBuild associated with this component, if any.
component_build = models.ComponentBuild.from_component_event(
db_session, task_id, module_id=module_build_id)
nvr = "{}-{}-{}".format(build_name, build_version, build_release)
if not component_build:
log.debug("We have no record of %s", nvr)
return
log.info("Saw relevant component build of %r from %r.", nvr, msg_id)
if state_reason:
state_reason = state_reason
elif build_new_state != koji.BUILD_STATES["COMPLETE"]:
state_reason = "Failed to build artifact {} in Koji".format(build_name)
else:
state_reason = ""
# Mark the state in the db.
component_build.state = build_new_state
component_build.nvr = nvr
component_build.state_reason = state_reason
db_session.commit()
parent = component_build.module_build
# If the macro build failed, then the module is doomed.
if (component_build.package == "module-build-macros"
and build_new_state != koji.BUILD_STATES["COMPLETE"]):
parent.transition(
db_session,
conf,
state=models.BUILD_STATES["failed"],
state_reason=state_reason,
failure_type="user",
)
db_session.commit()
return
if (component_build.buildonly and conf.system in ["koji", "test"]
and build_new_state == koji.BUILD_STATES["COMPLETE"]):
koji_session = get_session(conf, login=False)
rpms = koji_session.listBuildRPMs(component_build.nvr)
mmd = parent.mmd()
for artifact in rpms:
mmd.add_rpm_filter(artifact["name"])
parent.modulemd = mmd_to_str(mmd)
db_session.commit()
parent_current_batch = parent.current_batch()
# If there are no other components still building in a batch,
# we can tag all successfully built components in the batch.
unbuilt_components_in_batch = [
c for c in parent_current_batch
if c.is_waiting_for_build or c.is_building
]
if not unbuilt_components_in_batch:
failed_components_in_batch = [
c for c in parent_current_batch if c.is_unsuccessful
]
built_components_in_batch = [
c for c in parent_current_batch if c.is_completed
]
builder = GenericBuilder.create_from_module(db_session, parent, conf)
if failed_components_in_batch:
log.info(
"Batch done, but not tagging because of failed component builds. Will "
'transition the module to "failed"')
state_reason = "Component(s) {} failed to build.".format(", ".join(
c.package for c in failed_components_in_batch))
parent.transition(
db_session,
conf,
state=models.BUILD_STATES["failed"],
state_reason=state_reason,
failure_type="user",
)
db_session.commit()
return []
elif not built_components_in_batch:
# If there are no successfully built components in a batch, there is nothing to tag.
# The repository won't be regenerated in this case and therefore we generate fake repo
# change message here.
log.info("Batch done. No component to tag")
from module_build_service.scheduler.handlers.repos import done as repos_done_handler
events.scheduler.add(
repos_done_handler,
("fake_msg", builder.module_build_tag["name"]))
else:
built_component_nvrs_in_batch = [
c.nvr for c in built_components_in_batch
]
# tag && add to srpm-build group if neccessary
log.info("Batch done. Tagging %i component(s) in the build tag." %
len(built_component_nvrs_in_batch))
log.debug("%r" % built_component_nvrs_in_batch)
# TODO: install=component_build.build_time_only works here because module-build-macros
# is alone in its batch and the only component with build_time_only set. All other
# batches will have install=False. If we expand to have batches with a mix of
# build_time_only and not components, then this logic will need to change.
builder.buildroot_add_artifacts(
built_component_nvrs_in_batch,
install=component_build.build_time_only)
# Do not tag packages which only belong to the build tag to the dest tag
component_nvrs_to_tag_in_dest = [
c.nvr for c in built_components_in_batch
if c.build_time_only is False
]
log.info("Tagging %i component(s) in the dest tag." %
len(component_nvrs_to_tag_in_dest))
if component_nvrs_to_tag_in_dest:
builder.tag_artifacts(component_nvrs_to_tag_in_dest)
db_session.commit()
elif any(not c.is_building for c in unbuilt_components_in_batch):
# We are not in the middle of the batch building and
# we have some unbuilt components in this batch. We might hit the
# concurrent builds threshold in previous call of continue_batch_build
# done in repos.py:done(...), but because we have just finished one
# build, try to call continue_batch_build again so in case we hit the
# threshold previously, we will submit another build from this batch.
builder = GenericBuilder.create_from_module(db_session, parent, conf)
continue_batch_build(conf, parent, builder)
def add_default_modules(mmd):
"""
Add default modules as buildrequires to the input modulemd.
The base modules that are buildrequired can optionally link their default modules by specifying
a URL to a text file in xmd.mbs.default_modules_url. Any default module that isn't in the
database will be logged and ignored.
:param Modulemd.ModuleStream mmd: the modulemd of the module to add the module defaults to
:raises RuntimeError: if the buildrequired base module isn't in the database or the default
modules list can't be downloaded
"""
log.info("Finding the default modules to include as buildrequires")
xmd = mmd.get_xmd()
buildrequires = xmd["mbs"]["buildrequires"]
defaults_added = False
for module_name in conf.base_module_names:
bm_info = buildrequires.get(module_name)
if bm_info is None:
log.debug(
"The base module %s is not a buildrequire of the submitted module %s",
module_name,
mmd.get_nsvc(),
)
continue
bm = models.ModuleBuild.get_build_from_nsvc(
db_session,
module_name,
bm_info["stream"],
bm_info["version"],
bm_info["context"],
)
bm_nsvc = ":".join([
module_name,
bm_info["stream"],
bm_info["version"],
bm_info["context"],
])
if not bm:
raise RuntimeError(
"Failed to retrieve the module {} from the database".format(
bm_nsvc))
bm_mmd = bm.mmd()
bm_xmd = bm_mmd.get_xmd()
use_default_modules = bm_xmd.get("mbs",
{}).get("use_default_modules", False)
default_modules_scm_url = bm_xmd.get("mbs",
{}).get("default_modules_scm_url")
if not (use_default_modules or default_modules_scm_url):
log.info('The base module %s has no default modules',
bm_mmd.get_nsvc())
continue
# If the base module does not provide a default_modules_scm_url, use the default that is
# configured
default_modules_scm_url = default_modules_scm_url or conf.default_modules_scm_url
default_modules = _get_default_modules(bm.stream,
default_modules_scm_url)
for name, stream in default_modules.items():
ns = "{}:{}".format(name, stream)
if name in buildrequires:
conflicting_stream = buildrequires[name]["stream"]
if stream == conflicting_stream:
log.info("The default module %s is already a buildrequire",
ns)
continue
log.info(
"The default module %s will not be added as a buildrequire since %s:%s "
"is already a buildrequire",
ns,
name,
conflicting_stream,
)
continue
# Query for the latest default module that was built against this base module
resolver = GenericResolver.create(db_session, conf)
base_mmds = get_compatible_base_module_mmds(resolver, bm_mmd)
base_mmds = base_mmds["ready"] + base_mmds["garbage"]
base_mmds.sort(key=lambda mmd: models.ModuleBuild.
get_stream_version(mmd.get_stream_name(), False),
reverse=True)
for base_mmd in base_mmds:
default_module_mmds = resolver.get_buildrequired_modulemds(
name, stream, base_mmd)
if not default_module_mmds:
continue
# We need to ensure that module built against compatible base module stream
# really contains runtime-dependency on the current base module stream.
# For example in Fedora, we can have platform:f30 and platform:f31 base module
# streams. There can be foo:1 module built against platform:f30 which can work with
# any platform ("requires: platform: []"). This module can be configured as default
# module for platform:f28 and we need to support this case, but in the same time we
# cannot simply add any platform:f27 based module to platform:f28.
module_found = False
for default_module_mmd in default_module_mmds:
for deps in default_module_mmd.get_dependencies():
streams = deps.get_runtime_streams(module_name)
if streams is None:
continue
streams = expand_single_mse_streams(
db_session, module_name, streams)
if bm_info["stream"] in streams:
module_found = True
break
else:
log.info(
"Not using module %s as default module, because it does not "
"contain runtime dependency on %s",
default_module_mmd.get_nsvc(), bm_nsvc)
if module_found:
break
else:
log.warning(
"The default module %s from %s is not in the database and couldn't be added as "
"a buildrequire",
ns,
bm_nsvc,
)
continue
# Use resolve_requires since it provides the exact format that is needed for
# mbs.xmd.buildrequires
resolved = resolver.resolve_requires(
[default_module_mmd.get_nsvc()])
nsvc = ":".join([
name, stream, resolved[name]["version"],
resolved[name]["context"]
])
log.info("Adding the default module %s as a buildrequire", nsvc)
buildrequires.update(resolved)
defaults_added = True
if defaults_added:
mmd.set_xmd(xmd)
return defaults_added
def _koji_rpms_in_tag(self, tag):
""" Return the list of koji rpms in a tag. """
log.debug("Listing rpms in koji tag %s", tag)
session = get_session(self.config, login=False)
try:
rpms, builds = session.listTaggedRPMS(tag, latest=True)
except koji.GenericError:
log.exception("Failed to list rpms in tag %r", tag)
# If the tag doesn't exist.. then there are no rpms in that tag.
return []
# Module does not contain any RPM, so return an empty list.
if not rpms:
return []
# Get the exclusivearch, excludearch and license data for each RPM.
# The exclusivearch and excludearch lists are set in source RPM from which the RPM
# was built.
# Create temporary dict with source RPMs in rpm_id:rpms_list_index format.
src_rpms = {}
binary_rpms = {}
for rpm in rpms:
if rpm["arch"] == "src":
src_rpms[rpm["id"]] = rpm
else:
binary_rpms[rpm["id"]] = rpm
# Prepare the arguments for Koji multicall.
# We will call session.getRPMHeaders(...) for each SRC RPM to get exclusivearch,
# excludearch and license headers.
multicall_kwargs = [{
"rpmID":
rpm_id,
"headers": ["exclusivearch", "excludearch", "license"]
} for rpm_id in src_rpms.keys()]
# For each binary RPM, we only care about the "license" header.
multicall_kwargs += [{
"rpmID": rpm_id,
"headers": ["license"]
} for rpm_id in binary_rpms.keys()]
rpms_headers = koji_retrying_multicall_map(
session, session.getRPMHeaders, list_of_kwargs=multicall_kwargs)
# Temporary dict with build_id as a key to find builds easily.
builds = {build["build_id"]: build for build in builds}
# Create a mapping of build IDs to SRPM NEVRAs so that the for loop below can directly
# access these values when adding the `srpm_nevra` key to the returned RPMs
build_id_to_srpm_nevra = {
srpm["build_id"]: kobo.rpmlib.make_nvra(srpm, force_epoch=True)
for srpm in src_rpms.values()
}
# Handle the multicall result. For each build associated with the source RPM,
# store the exclusivearch and excludearch lists. For each RPM, store the 'license' and
# also other useful data from the Build associated with the RPM.
for rpm, headers in zip(chain(src_rpms.values(), binary_rpms.values()),
rpms_headers):
if not headers:
raise RuntimeError(
"No RPM headers received from Koji for RPM %s" %
rpm["name"])
if "license" not in headers:
raise RuntimeError(
"No RPM 'license' header received from Koji for RPM %s" %
rpm["name"])
build = builds[rpm["build_id"]]
if "exclusivearch" in headers and "excludearch" in headers:
build["exclusivearch"] = headers["exclusivearch"]
build["excludearch"] = headers["excludearch"]
rpm["license"] = headers["license"]
rpm["srpm_name"] = build["name"]
rpm["srpm_nevra"] = build_id_to_srpm_nevra[rpm["build_id"]]
rpm["exclusivearch"] = build["exclusivearch"]
rpm["excludearch"] = build["excludearch"]
return rpms
def fail_lost_builds():
# This function is supposed to be handling only the part which can't be
# updated through messaging (e.g. srpm-build failures). Please keep it
# fit `n` slim. We do want rest to be processed elsewhere
# TODO re-use
if conf.system == "koji":
# We don't do this on behalf of users
koji_session = get_session(conf, login=False)
log.info("Querying tasks for statuses:")
res = db_session.query(models.ComponentBuild).filter_by(
state=koji.BUILD_STATES["BUILDING"]
).options(lazyload("module_build")).all()
log.info("Checking status for %s tasks", len(res))
for component_build in res:
log.debug(component_build.json(db_session))
# Don't check tasks which haven't been triggered yet
if not component_build.task_id:
continue
# Don't check tasks for components which have been reused,
# they may have BUILDING state temporarily before we tag them
# to new module tag. Checking them would be waste of resources.
if component_build.reused_component_id:
log.debug(
'Skipping check for task "%s", the component has been reused ("%s").',
component_build.task_id, component_build.reused_component_id
)
continue
task_id = component_build.task_id
log.info('Checking status of task_id "%s"', task_id)
task_info = koji_session.getTaskInfo(task_id)
state_mapping = {
# Cancelled and failed builds should be marked as failed.
koji.TASK_STATES["CANCELED"]: koji.BUILD_STATES["FAILED"],
koji.TASK_STATES["FAILED"]: koji.BUILD_STATES["FAILED"],
# Completed tasks should be marked as complete.
koji.TASK_STATES["CLOSED"]: koji.BUILD_STATES["COMPLETE"],
}
# If it is a closed/completed task, then we can extract the NVR
build_version, build_release = None, None # defaults
if task_info["state"] == koji.TASK_STATES["CLOSED"]:
builds = koji_session.listBuilds(taskID=task_id)
if not builds:
log.warning(
"Task ID %r is closed, but we found no builds in koji.", task_id)
elif len(builds) > 1:
log.warning(
"Task ID %r is closed, but more than one build is present!", task_id)
else:
build_version = builds[0]["version"]
build_release = builds[0]["release"]
log.info(" task %r is in state %r", task_id, task_info["state"])
if task_info["state"] in state_mapping:
build_task_finalize.delay(
msg_id="producer::fail_lost_builds fake msg",
task_id=component_build.task_id,
build_new_state=state_mapping[task_info["state"]],
build_name=component_build.package,
build_release=build_release,
build_version=build_version,
)
elif conf.system == "mock":
pass
def fetch_mmd(url,
branch=None,
allow_local_url=False,
whitelist_url=False,
mandatory_checks=True):
td = None
scm = None
try:
log.debug("Verifying modulemd")
td = tempfile.mkdtemp()
if whitelist_url:
scm = module_build_service.common.scm.SCM(url, branch, [url],
allow_local_url)
else:
scm = module_build_service.common.scm.SCM(url, branch,
conf.scmurls,
allow_local_url)
scm.checkout(td)
if not whitelist_url and mandatory_checks:
scm.verify()
cofn = scm.get_module_yaml()
mmd = load_mmd_file(cofn)
finally:
try:
if td is not None:
shutil.rmtree(td)
except Exception as e:
log.warning("Failed to remove temporary directory {!r}: {}".format(
td, str(e)))
if conf.check_for_eol:
if _is_eol_in_pdc(scm.name, scm.branch):
raise ValidationError(
"Module {}:{} is marked as EOL in PDC.".format(
scm.name, scm.branch))
if not mandatory_checks:
return mmd, scm
# If the name was set in the modulemd, make sure it matches what the scmurl
# says it should be
if mmd.get_module_name() and mmd.get_module_name() != scm.name:
if not conf.allow_name_override_from_scm:
raise ValidationError(
'The name "{0}" that is stored in the modulemd is not valid'.
format(mmd.get_module_name()))
else:
# Set the module name
mmd = mmd.copy(scm.name)
# If the stream was set in the modulemd, make sure it matches what the repo
# branch is
if mmd.get_stream_name() and mmd.get_stream_name() != scm.branch:
if not conf.allow_stream_override_from_scm:
raise ValidationError(
'The stream "{0}" that is stored in the modulemd does not match the branch "{1}"'
.format(mmd.get_stream_name(), scm.branch))
else:
# Set the module stream
mmd = mmd.copy(mmd.get_module_name(), scm.branch)
# If the version is in the modulemd, throw an exception since the version
# since the version is generated by MBS
if mmd.get_version():
raise ValidationError(
'The version "{0}" is already defined in the modulemd but it shouldn\'t be since the '
"version is generated based on the commit time".format(
mmd.get_version()))
else:
mmd.set_version(int(scm.version))
return mmd, scm
def wait(msg_id, module_build_id, module_build_state):
""" Called whenever a module enters the 'wait' state.
We transition to this state shortly after a modulebuild is first requested.
All we do here is request preparation of the buildroot.
The kicking off of individual component builds is handled elsewhere,
in module_build_service.schedulers.handlers.repos.
:param str msg_id: the original id of the message being handled which is
received from the message bus.
:param int module_build_id: the module build id.
:param int module_build_state: the module build state.
"""
build = models.ModuleBuild.get_by_id(db_session, module_build_id)
log.info("Found build=%r from message" % build)
log.debug("%r", build.modulemd)
if build.state != module_build_state:
log.warning(
"Note that retrieved module state %r doesn't match message module state %r",
build.state,
module_build_state,
)
# This is ok.. it's a race condition we can ignore.
pass
try:
build_deps = get_module_build_dependencies(build)
except ValueError:
reason = "Failed to get module info from MBS. Max retries reached."
log.exception(reason)
build.transition(db_session,
conf,
state=models.BUILD_STATES["failed"],
state_reason=reason,
failure_type="infra")
db_session.commit()
raise
tag = generate_module_build_koji_tag(build)
log.debug("Found tag=%s for module %r" % (tag, build))
# Hang on to this information for later. We need to know which build is
# associated with which koji tag, so that when their repos are regenerated
# in koji we can figure out which for which module build that event is
# relevant.
log.debug("Assigning koji tag=%s to module build" % tag)
build.koji_tag = tag
if build.scratch:
log.debug(
"Assigning Content Generator build koji tag is skipped for scratch module build."
)
elif conf.koji_cg_tag_build:
cg_build_koji_tag = get_content_generator_build_koji_tag(build_deps)
log.debug(
"Assigning Content Generator build koji tag=%s to module build",
cg_build_koji_tag)
build.cg_build_koji_tag = cg_build_koji_tag
else:
log.debug(
"It is disabled to tag module build during importing into Koji by Content Generator."
)
log.debug(
"Skip to assign Content Generator build koji tag to module build.")
builder = GenericBuilder.create_from_module(db_session, build, conf)
log.debug(
"Adding dependencies %s into buildroot for module %s:%s:%s",
build_deps.keys(),
build.name,
build.stream,
build.version,
)
builder.buildroot_add_repos(build_deps)
if not build.component_builds:
log.info("There are no components in module %r, skipping build" %
build)
build.transition(db_session, conf, state=models.BUILD_STATES["build"])
db_session.add(build)
db_session.commit()
# Return a KojiRepoChange message so that the build can be transitioned to done
# in the repos handler
from module_build_service.scheduler.handlers.repos import done as repos_done_handler
events.scheduler.add(repos_done_handler,
("fake_msg", builder.module_build_tag["name"]))
return
# If all components in module build will be reused, we don't have to build
# module-build-macros, because there won't be any build done.
if attempt_to_reuse_all_components(builder, build):
log.info(
"All components have been reused for module %r, skipping build" %
build)
build.transition(db_session, conf, state=models.BUILD_STATES["build"])
db_session.add(build)
db_session.commit()
return []
log.debug("Starting build batch 1")
build.batch = 1
db_session.commit()
artifact_name = "module-build-macros"
component_build = models.ComponentBuild.from_component_name(
db_session, artifact_name, build.id)
srpm = builder.get_disttag_srpm(disttag=".%s" %
get_rpm_release(db_session, build),
module_build=build)
if not component_build:
component_build = models.ComponentBuild(
module_id=build.id,
package=artifact_name,
format="rpms",
scmurl=srpm,
batch=1,
build_time_only=True,
)
db_session.add(component_build)
# Commit and refresh so that the SQLAlchemy relationships are available
db_session.commit()
db_session.refresh(component_build)
recovered = builder.recover_orphaned_artifact(component_build)
if recovered:
log.info("Found an existing module-build-macros build")
# There was no existing artifact found, so lets submit the build instead
else:
task_id, state, reason, nvr = builder.build(
artifact_name=artifact_name, source=srpm)
component_build.task_id = task_id
component_build.state = state
component_build.reason = reason
component_build.nvr = nvr
elif not component_build.is_completed:
# It's possible that the build succeeded in the builder but some other step failed which
# caused module-build-macros to be marked as failed in MBS, so check to see if it exists
# first
recovered = builder.recover_orphaned_artifact(component_build)
if recovered:
log.info("Found an existing module-build-macros build")
else:
task_id, state, reason, nvr = builder.build(
artifact_name=artifact_name, source=srpm)
component_build.task_id = task_id
component_build.state = state
component_build.reason = reason
component_build.nvr = nvr
db_session.add(component_build)
build.transition(db_session, conf, state=models.BUILD_STATES["build"])
db_session.add(build)
db_session.commit()
# We always have to regenerate the repository.
if conf.system == "koji":
log.info("Regenerating the repository")
task_id = builder.koji_session.newRepo(
builder.module_build_tag["name"])
build.new_repo_task_id = task_id
db_session.commit()
else:
from module_build_service.scheduler.handlers.repos import done as repos_done_handler
events.scheduler.add(repos_done_handler,
("fake_msg", builder.module_build_tag["name"]))
def get_module_build_dependencies(self,
name=None,
stream=None,
version=None,
context=None,
mmd=None,
strict=False):
"""
Returns a dictionary of koji_tag:[mmd, ...] of all the dependencies of input module.
Although it is expected that single Koji tag always contain just single module build,
it does not have to be a true for Offline local builds which use the local repository
identifier as `koji_tag`.
:kwarg name: a string of a module's name (required if mmd is not set)
:kwarg stream: a string of a module's stream (required if mmd is not set)
:kwarg version: a string of a module's version (required if mmd is not set)
:kwarg context: a string of a module's context (required if mmd is not set)
:kwarg mmd: Modulemd.ModuleStream object. If this is set, the mmd will be used instead of
querying the DB with the name, stream, version, and context.
:kwarg strict: Normally this function returns None if no module can be
found. If strict=True, then an UnprocessableEntity is raised.
:return: a dictionary
"""
if mmd:
log.debug(
"get_module_build_dependencies(mmd={0!r} strict={1!r})".format(
mmd, strict))
elif any(x is None for x in [name, stream, version, context]):
raise RuntimeError(
"The name, stream, version, and/or context weren't specified")
else:
version = str(version)
log.debug(
"get_module_build_dependencies({0}, strict={1!r})".format(
", ".join([name, stream,
str(version), context]), strict))
module_tags = {}
if mmd:
queried_mmd = mmd
nsvc = ":".join([
mmd.get_module_name(),
mmd.get_stream_name(),
str(mmd.get_version()),
mmd.get_context() or models.DEFAULT_MODULE_CONTEXT,
])
else:
build = models.ModuleBuild.get_build_from_nsvc(
self.db_session, name, stream, version, context)
if not build:
raise UnprocessableEntity("The module {} was not found".format(
":".join([name, stream, version, context])))
queried_mmd = build.mmd()
nsvc = ":".join([name, stream, version, context])
xmd_mbs = queried_mmd.get_xmd().get("mbs", {})
if "buildrequires" not in xmd_mbs:
raise RuntimeError(
"The module {} did not contain its modulemd or did not have "
"its xmd attribute filled out in MBS".format(nsvc))
buildrequires = xmd_mbs["buildrequires"]
for br_name, details in buildrequires.items():
build = models.ModuleBuild.get_build_from_nsvc(
self.db_session,
br_name,
details["stream"],
details["version"],
details["context"],
state=models.BUILD_STATES["ready"],
)
if not build:
raise RuntimeError(
"Buildrequired module %s %r does not exist in MBS db" %
(br_name, details))
# If the buildrequire is a meta-data only module with no Koji tag set, then just
# skip it
if build.koji_tag is None:
continue
module_tags.setdefault(build.koji_tag, [])
module_tags[build.koji_tag].append(build.mmd())
return module_tags
