def test_retire_build(self, prompt_bool, overrides, identifier,
changed_count):
prompt_bool.return_value = True
module_builds = (db_session.query(ModuleBuild).filter_by(
state=BUILD_STATES["ready"]).order_by(ModuleBuild.id.desc()).all())
# Verify our assumption of the amount of ModuleBuilds in database
assert len(module_builds) == 3
for x, build in enumerate(module_builds):
build.name = "spam"
build.stream = "eggs"
build.version = "ham"
build.context = str(x)
for attr, value in overrides.items():
setattr(module_builds[0], attr, value)
db_session.commit()
retire(identifier)
retired_module_builds = (db_session.query(ModuleBuild).filter_by(
state=BUILD_STATES["garbage"]).order_by(
ModuleBuild.id.desc()).all())
assert len(retired_module_builds) == changed_count
for x in range(changed_count):
assert retired_module_builds[x].id == module_builds[x].id
assert retired_module_builds[x].state == BUILD_STATES["garbage"]
def test_set_stream(self, main):
cli_cmd = [
"mbs-manager", "build_module_locally", "--set-stream",
"platform:f28", "--file",
staged_data_filename("testmodule-local-build.yaml")
]
self._run_manager_wrapper(cli_cmd)
# Since module_build_service.scheduler.local.main is mocked, MBS does
# not really build the testmodule for this test. Following lines assert
# the fact:
# Module testmodule-local-build is expanded and stored into database,
# and this build has buildrequires platform:f28 and requires
# platform:f28.
# Please note that, the f28 is specified from command line option
# --set-stream, which is the point this test tests.
builds = db_session.query(
models.ModuleBuild).filter_by(name="testmodule-local-build").all()
assert 1 == len(builds)
testmodule_build = builds[0]
mmd_deps = testmodule_build.mmd().get_dependencies()
deps_dict = deps_to_dict(mmd_deps[0], "buildtime")
assert ["f28"] == deps_dict["platform"]
deps_dict = deps_to_dict(mmd_deps[0], "runtime")
assert ["f28"] == deps_dict["platform"]
def retrigger_new_repo_on_failure():
"""
Retrigger failed new repo tasks for module builds in the build state.
The newRepo task may fail for various reasons outside the scope of MBS.
This method will detect this scenario and retrigger the newRepo task
if needed to avoid the module build from being stuck in the "build" state.
"""
if conf.system != "koji":
return
koji_session = get_session(conf)
module_builds = db_session.query(models.ModuleBuild).filter(
models.ModuleBuild.state == models.BUILD_STATES["build"],
models.ModuleBuild.new_repo_task_id.isnot(None),
).all()
for module_build in module_builds:
task_info = koji_session.getTaskInfo(module_build.new_repo_task_id)
if task_info["state"] in [koji.TASK_STATES["CANCELED"], koji.TASK_STATES["FAILED"]]:
log.info(
"newRepo task %s for %r failed, starting another one",
str(module_build.new_repo_task_id), module_build,
)
taginfo = koji_session.getTag(module_build.koji_tag + "-build")
module_build.new_repo_task_id = koji_session.newRepo(taginfo["name"])
db_session.commit()
def test_get_buildrequired_modules(self, ClientSession):
koji_session = ClientSession.return_value
# We will ask for testmodule:master, but there is also testmodule:2 in a tag.
koji_session.listTagged.return_value = [{
"build_id": 123,
"name": "testmodule",
"version": "2",
"release": "820181219174508.9edba152",
"tag_name": "foo-test"
}, {
"build_id": 124,
"name": "testmodule",
"version": "master",
"release": "20170109091357.7c29193d",
"tag_name": "foo-test"
}]
koji_session.multiCall.return_value = [
[build] for build in koji_session.listTagged.return_value
]
self._create_test_modules()
platform = db_session.query(ModuleBuild).filter_by(
stream="f30.1.3").one()
resolver = mbs_resolver.GenericResolver.create(db_session,
conf,
backend="koji")
result = resolver.get_buildrequired_modules("testmodule", "master",
platform.mmd())
nvrs = {m.nvr_string for m in result}
assert nvrs == {"testmodule-master-20170109091357.7c29193d"}
def at_concurrent_component_threshold(config):
"""
Determines if the number of concurrent component builds has reached
the configured threshold
:param config: Module Build Service configuration object
:return: boolean representing if there are too many concurrent builds at
this time
"""
# We must not check it for "mock" backend.
# It would lead to multiple calls of continue_batch_build method and
# creation of multiple worker threads there. Mock backend uses thread-id
# to create and identify mock buildroot and for mock backend, we must
# build whole module in this single continue_batch_build call to keep
# the number of created buildroots low. The concurrent build limit
# for mock backend is secured by setting max_workers in
# ThreadPoolExecutor to num_concurrent_builds.
if conf.system == "mock":
return False
import koji # Placed here to avoid py2/py3 conflicts...
# Components which are reused should not be counted in, because
# we do not submit new build for them. They are in BUILDING state
# just internally in MBS to be handled by
# scheduler.handlers.components.complete.
if config.num_concurrent_builds:
count = db_session.query(models.ComponentBuild).filter_by(
state=koji.BUILD_STATES["BUILDING"], reused_component_id=None).count()
if config.num_concurrent_builds <= count:
return True
return False
def test_get_buildrequired_modulemds(self):
mmd = load_mmd(tests.read_staged_data("platform"))
mmd = mmd.copy(mmd.get_module_name(), "f30.1.3")
import_mmd(db_session, mmd)
platform_f300103 = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one()
mmd = tests.make_module("testmodule:master:20170109091357:123")
build = ModuleBuild(
name="testmodule",
stream="master",
version=20170109091357,
state=5,
build_context="dd4de1c346dcf09ce77d38cd4e75094ec1c08ec3",
runtime_context="ec4de1c346dcf09ce77d38cd4e75094ec1c08ef7",
context="7c29193d",
koji_tag="module-testmodule-master-20170109091357-7c29193d",
scmurl="https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79",
batch=3,
owner="Dr. Pepper",
time_submitted=datetime(2018, 11, 15, 16, 8, 18),
time_modified=datetime(2018, 11, 15, 16, 19, 35),
rebuild_strategy="changed-and-after",
modulemd=mmd_to_str(mmd),
)
build.buildrequires.append(platform_f300103)
db_session.add(build)
db_session.commit()
resolver = mbs_resolver.GenericResolver.create(db_session, conf, backend="db")
result = resolver.get_buildrequired_modulemds(
"testmodule", "master", platform_f300103.mmd())
nsvcs = {m.get_nsvc() for m in result}
assert nsvcs == {"testmodule:master:20170109091357:123"}
def test_get_buildrequired_modulemds_kojiresolver(self, mock_session):
"""
Test that MBSResolver uses KojiResolver as input when KojiResolver is enabled for
the base module.
"""
mock_session.get.return_value = Mock(ok=True)
mock_session.get.return_value.json.return_value = {
"items": [
{
"name": "nodejs",
"stream": "10",
"version": 2,
"context": "c1",
"modulemd":
mmd_to_str(tests.make_module("nodejs:10:2:c1"), ),
},
],
"meta": {
"next": None
},
}
resolver = mbs_resolver.GenericResolver.create(db_session,
conf,
backend="mbs")
platform = db_session.query(
module_build_service.common.models.ModuleBuild).filter_by(
id=1).one()
platform_mmd = platform.mmd()
platform_xmd = platform_mmd.get_xmd()
platform_xmd["mbs"]["koji_tag_with_modules"] = "module-f29-build"
platform_mmd.set_xmd(platform_xmd)
with patch.object(resolver, "get_buildrequired_koji_builds"
) as get_buildrequired_koji_builds:
get_buildrequired_koji_builds.return_value = [{
"build_id":
124,
"name":
"nodejs",
"version":
"10",
"release":
"2.c1",
"tag_name":
"foo-test"
}]
result = resolver.get_buildrequired_modulemds(
"nodejs", "10", platform_mmd)
get_buildrequired_koji_builds.assert_called_once()
assert 1 == len(result)
mmd = result[0]
assert "nodejs" == mmd.get_module_name()
assert "10" == mmd.get_stream_name()
assert 2 == mmd.get_version()
assert "c1" == mmd.get_context()
def test_corresponding_module_build_id_does_not_exist_in_db(self, ClientSession,
require_platform_and_default_arch):
fake_module_build_id, = db_session.query(func.max(ModuleBuild.id)).first()
ClientSession.return_value.getBuild.return_value = {
"extra": {"typeinfo": {"module": {"module_build_service_id": fake_module_build_id + 1}}}
}
assert get_corresponding_module_build("n-v-r") is None
def test_do_not_handle_a_duplicate_late_init_message(self):
build = db_session.query(ModuleBuild).filter(
ModuleBuild.name == "testmodule").one()
build.state = BUILD_STATES["wait"]
db_session.commit()
with patch.object(module_build_service.scheduler.handlers.modules,
"log") as log:
self.fn("msg-id-123", build.id, BUILD_STATES["init"])
assert 2 == log.warning.call_count
def _create_test_modules(self, koji_tag_with_modules="foo-test"):
mmd = load_mmd(tests.read_staged_data("platform"))
mmd = mmd.copy(mmd.get_module_name(), "f30.1.3")
import_mmd(db_session, mmd)
platform = db_session.query(ModuleBuild).filter_by(
stream="f30.1.3").one()
if koji_tag_with_modules:
platform = db_session.query(ModuleBuild).filter_by(
stream="f30.1.3").one()
platform_mmd = platform.mmd()
platform_xmd = platform_mmd.get_xmd()
platform_xmd["mbs"][
"koji_tag_with_modules"] = koji_tag_with_modules
platform_mmd.set_xmd(platform_xmd)
platform.modulemd = mmd_to_str(platform_mmd)
for context in ["7c29193d", "7c29193e"]:
mmd = tests.make_module("testmodule:master:20170109091357:" +
context)
build = ModuleBuild(
name="testmodule",
stream="master",
version=20170109091357,
state=5,
build_context="dd4de1c346dcf09ce77d38cd4e75094ec1c08ec3",
runtime_context="ec4de1c346dcf09ce77d38cd4e75094ec1c08ef7",
context=context,
koji_tag="module-testmodule-master-20170109091357-" + context,
scmurl=
"https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79",
batch=3,
owner="Dr. Pepper",
time_submitted=datetime(2018, 11, 15, 16, 8, 18),
time_modified=datetime(2018, 11, 15, 16, 19, 35),
rebuild_strategy="changed-and-after",
modulemd=mmd_to_str(mmd),
)
build.buildrequires.append(platform)
db_session.add(build)
db_session.commit()
def sync_koji_build_tags():
"""
Method checking the "tagged" and "tagged_in_final" attributes of
"complete" ComponentBuilds in the current batch of module builds
in "building" state against the Koji.
In case the Koji shows the build as tagged/tagged_in_final,
fake "tagged" message is added to work queue.
"""
if conf.system != "koji":
return
koji_session = get_session(conf, login=False)
threshold = datetime.utcnow() - timedelta(minutes=10)
module_builds = db_session.query(models.ModuleBuild).filter(
models.ModuleBuild.time_modified < threshold,
models.ModuleBuild.state == models.BUILD_STATES["build"]
).all()
for module_build in module_builds:
complete_components = module_build.current_batch(koji.BUILD_STATES["COMPLETE"])
for c in complete_components:
# In case the component is tagged in the build tag and
# also tagged in the final tag (or it is build_time_only
# and therefore should not be tagged in final tag), skip it.
if c.tagged and (c.tagged_in_final or c.build_time_only):
continue
log.info(
"%r: Component %r is complete, but not tagged in the "
"final and/or build tags.",
module_build, c,
)
# Check in which tags the component is tagged.
tag_dicts = koji_session.listTags(c.nvr)
tags = [tag_dict["name"] for tag_dict in tag_dicts]
# If it is tagged in final tag, but MBS does not think so,
# schedule fake message.
if not c.tagged_in_final and module_build.koji_tag in tags:
log.info(
"Apply tag %s to module build %r",
module_build.koji_tag, module_build)
tagged.delay("internal:sync_koji_build_tags", module_build.koji_tag, c.nvr)
# If it is tagged in the build tag, but MBS does not think so,
# schedule fake message.
build_tag = module_build.koji_tag + "-build"
if not c.tagged and build_tag in tags:
log.info(
"Apply build tag %s to module build %r",
build_tag, module_build)
tagged.delay("internal:sync_koji_build_tags", build_tag, c.nvr)
def test_retire_build_confirm_prompt(self, prompt_bool, confirm_prompt,
confirm_arg, confirm_expected):
prompt_bool.return_value = confirm_prompt
module_builds = db_session.query(ModuleBuild).filter_by(
state=BUILD_STATES["ready"]).all()
# Verify our assumption of the amount of ModuleBuilds in database
assert len(module_builds) == 3
for x, build in enumerate(module_builds):
build.name = "spam" + str(x) if x > 0 else "spam"
build.stream = "eggs"
db_session.commit()
retire("spam:eggs", confirm_arg)
retired_module_builds = (db_session.query(ModuleBuild).filter_by(
state=BUILD_STATES["garbage"]).all())
expected_changed_count = 1 if confirm_expected else 0
assert len(retired_module_builds) == expected_changed_count
def test_find_the_module_build(self, ClientSession, require_platform_and_default_arch):
expected_module_build = (
db_session.query(ModuleBuild).filter(ModuleBuild.name == "platform").first()
)
ClientSession.return_value.getBuild.return_value = {
"extra": {"typeinfo": {"module": {"module_build_service_id": expected_module_build.id}}}
}
build = get_corresponding_module_build("n-v-r")
assert expected_module_build.id == build.id
assert expected_module_build.name == build.name
def test_init_basic(self, create_builder):
builder = mock.Mock()
builder.get_disttag_srpm.return_value = "some srpm disttag"
builder.build.return_value = 1234, 1, "", None
builder.module_build_tag = {"name": "some-tag-build"}
create_builder.return_value = builder
module_build_id = db_session.query(ModuleBuild).first().id
with patch("module_build_service.resolver.GenericResolver.create"):
module_build_service.scheduler.handlers.modules.wait(
msg_id="msg-id-1",
module_build_id=module_build_id,
module_build_state="some state")
def test_cleanup_stale_failed_builds(self, create_builder, dbg):
""" Test that one of the two module builds gets to the garbage state when running
cleanup_stale_failed_builds.
"""
builder = mock.MagicMock()
create_builder.return_value = builder
module_build_one = models.ModuleBuild.get_by_id(db_session, 2)
module_build_one.state = models.BUILD_STATES["failed"]
module_build_one.time_modified = datetime.utcnow() - timedelta(
days=conf.cleanup_failed_builds_time + 1)
module_build_two = models.ModuleBuild.get_by_id(db_session, 3)
module_build_two.time_modified = datetime.utcnow()
module_build_two.state = models.BUILD_STATES["failed"]
failed_component = db_session.query(models.ComponentBuild).filter_by(
package="tangerine", module_id=3).one()
failed_component.state = koji.BUILD_STATES["FAILED"]
failed_component.tagged = False
failed_component.tagged_in_final = False
db_session.commit()
producer.cleanup_stale_failed_builds()
db_session.refresh(module_build_two)
# Make sure module_build_one was transitioned to garbage
assert module_build_one.state == models.BUILD_STATES["garbage"]
state_reason = (
"The module was garbage collected since it has failed over {0} day(s) ago"
.format(conf.cleanup_failed_builds_time)
)
assert module_build_one.state_reason == state_reason
# Make sure all the components are marked as untagged in the database
for component in module_build_one.component_builds:
assert not component.tagged
assert not component.tagged_in_final
# Make sure module_build_two stayed the same
assert module_build_two.state == models.BUILD_STATES["failed"]
# Make sure the builds were untagged
builder.untag_artifacts.assert_called_once()
args, _ = builder.untag_artifacts.call_args
expected = [
"module-build-macros-0.1-1.module+0+d027b723",
"perl-List-Compare-0.53-5.module+0+d027b723",
"perl-Tangerine-0.23-1.module+0+d027b723",
"tangerine-0.22-3.module+0+d027b723",
]
assert expected == sorted(args[0])
def test_get_buildrequired_modulemds_multiple_versions_contexts(
self, ClientSession):
koji_session = ClientSession.return_value
# We will ask for testmodule:2, but it is not in database, so it should raise
# ValueError later.
koji_session.listTagged.return_value = [{
"build_id": 124,
"name": "testmodule",
"version": "master",
"release": "20160110091357.7c29193d",
"tag_name": "foo-test"
}, {
"build_id": 124,
"name": "testmodule",
"version": "master",
"release": "20170109091357.7c29193d",
"tag_name": "foo-test"
}, {
"build_id": 124,
"name": "testmodule",
"version": "master",
"release": "20170109091357.7c29193e",
"tag_name": "foo-test"
}, {
"build_id": 124,
"name": "testmodule",
"version": "master",
"release": "20160109091357.7c29193d",
"tag_name": "foo-test"
}]
koji_session.multiCall.return_value = [
[build] for build in koji_session.listTagged.return_value
]
self._create_test_modules()
platform = db_session.query(ModuleBuild).filter_by(
stream="f30.1.3").one()
resolver = mbs_resolver.GenericResolver.create(db_session,
conf,
backend="koji")
result = resolver.get_buildrequired_modulemds("testmodule", "master",
platform.mmd())
nsvcs = {m.get_nsvc() for m in result}
assert nsvcs == {
"testmodule:master:20170109091357:7c29193d",
"testmodule:master:20170109091357:7c29193e"
}
def log_summary():
states = sorted(models.BUILD_STATES.items(), key=operator.itemgetter(1))
for name, code in states:
query = db_session.query(models.ModuleBuild).filter_by(state=code)
count = query.count()
if count:
log.info(" * %s module builds in the %s state", count, name)
if name == "build":
for module_build in query.all():
log.info(" * %r", module_build)
# First batch is number '1'.
for i in range(1, module_build.batch + 1):
n = len([c for c in module_build.component_builds if c.batch == i])
log.info(" * %s components in batch %s", n, i)
def retire(identifier, confirm=False):
""" Retire module build(s) by placing them into 'garbage' state.
"""
# Parse identifier and build query
parts = identifier.split(":")
if len(parts) < 2:
raise ValueError("Identifier must contain at least NAME:STREAM")
if len(parts) >= 5:
raise ValueError("Too many parts in identifier")
filter_by_kwargs = {
"state": models.BUILD_STATES["ready"],
"name": parts[0],
"stream": parts[1]
}
if len(parts) >= 3:
filter_by_kwargs["version"] = parts[2]
if len(parts) >= 4:
filter_by_kwargs["context"] = parts[3]
# Find module builds to retire
module_builds = db_session.query(
models.ModuleBuild).filter_by(**filter_by_kwargs).all()
if not module_builds:
logging.info("No module builds found.")
return
logging.info("Found %d module builds:", len(module_builds))
for build in module_builds:
logging.info(
"\t%s", ":".join(
(build.name, build.stream, build.version, build.context)))
# Prompt for confirmation
is_confirmed = confirm or prompt_bool("Retire {} module builds?".format(
len(module_builds)))
if not is_confirmed:
logging.info("Module builds were NOT retired.")
return
# Retire module builds
for build in module_builds:
build.transition(db_session, conf, models.BUILD_STATES["garbage"],
"Module build retired")
db_session.commit()
logging.info("Module builds retired.")
def test_get_reusable_module_when_reused_module_not_set(self):
module = db_session.query(models.ModuleBuild)\
.filter_by(name="testmodule")\
.order_by(models.ModuleBuild.id.desc())\
.first()
module.state = models.BUILD_STATES["build"]
db_session.commit()
assert not module.reused_module
reusable_module = get_reusable_module(module)
assert module.reused_module
assert reusable_module.id == module.reused_module_id
def test_get_compatible_base_module_modulemds_fallback_to_dbresolver(self):
tests.init_data(1, multiple_stream_versions=True)
resolver = mbs_resolver.GenericResolver.create(db_session,
conf,
backend="koji")
platform = db_session.query(ModuleBuild).filter_by(
name="platform", stream="f29.1.0").one()
platform_mmd = platform.mmd()
result = resolver.get_compatible_base_module_modulemds(
platform_mmd,
stream_version_lte=True,
virtual_streams=["f29"],
states=[BUILD_STATES["ready"]])
assert len(result) == 2
def test_newrepo_still_building_components(self, create_builder,
koji_get_session, dbg):
"""
Test that newRepo is called in the expected times.
"""
koji_session = mock.MagicMock()
koji_session.getTag = lambda tag_name: {"name": tag_name}
koji_session.getTaskInfo.return_value = {
"state": koji.TASK_STATES["CLOSED"]
}
koji_session.newRepo.return_value = 123456
koji_get_session.return_value = koji_session
builder = mock.MagicMock()
builder.koji_session = koji_session
builder.buildroot_ready.return_value = False
builder.module_build_tag = {
"name": "module-testmodule-master-20170219191323-c40c156c-build"
}
create_builder.return_value = builder
module_build = module_build_service.common.models.ModuleBuild.get_by_id(
db_session, 3)
module_build.batch = 2
component = db_session.query(
module_build_service.common.models.ComponentBuild).filter_by(
package="perl-Tangerine", module_id=module_build.id).one()
component.state = koji.BUILD_STATES["BUILDING"]
component.nvr = "perl-Tangerine-0.23-1.module+0+d027b723"
db_session.commit()
# Tag the perl-List-Compare component to the buildroot.
module_build_service.scheduler.handlers.tags.tagged(
msg_id="id",
tag_name="module-testmodule-master-20170219191323-c40c156c-build",
build_nvr="perl-Tangerine-0.23-1.module+0+d027b723",
)
# Tag the perl-List-Compare component to final tag.
module_build_service.scheduler.handlers.tags.tagged(
msg_id="id",
tag_name="module-testmodule-master-20170219191323-c40c156c",
build_nvr="perl-Tangerine-0.23-1.module+0+d027b723",
)
# newRepo should not be called, because perl-List-Compare has not been
# built yet.
assert not koji_session.newRepo.called
def test_get_buildrequired_modulemds_fallback_to_db_resolver(self):
self._create_test_modules(koji_tag_with_modules=None)
platform = db_session.query(ModuleBuild).filter_by(
stream="f30.1.3").one()
resolver = mbs_resolver.GenericResolver.create(db_session,
conf,
backend="koji")
result = resolver.get_buildrequired_modulemds("testmodule", "master",
platform.mmd())
nsvcs = {m.get_nsvc() for m in result}
assert nsvcs == {
"testmodule:master:20170109091357:7c29193d",
"testmodule:master:20170109091357:7c29193e"
}
def raise_for_failed_build(module_build_ids):
"""
Raises an exception if any module build from `module_build_ids` list is in failed state.
This function also calls "failed" handler before raises an exception.
:param list module_build_ids: List of module build IDs (int) to build locally.
"""
builds = db_session.query(models.ModuleBuild).filter(
models.ModuleBuild.id.in_(module_build_ids)).all()
has_failed_build = False
for build in builds:
if build.state == models.BUILD_STATES["failed"]:
modules_failed_handler("fake_msg_id", build.id, "failed")
has_failed_build = True
if has_failed_build:
raise ValueError("Local module build failed.")
def test_get_compatible_base_module_modulemds_stream_versions(self, stream_versions):
tests.init_data(1, multiple_stream_versions=True)
resolver = mbs_resolver.GenericResolver.create(db_session, conf, backend="db")
platform = db_session.query(ModuleBuild).filter_by(name="platform", stream="f29.1.0").one()
platform_mmd = platform.mmd()
result = resolver.get_compatible_base_module_modulemds(
platform_mmd, stream_version_lte=stream_versions, virtual_streams=["f29"],
states=[models.BUILD_STATES["ready"]])
nsvcs = {mmd.get_nsvc() for mmd in result}
if stream_versions:
assert nsvcs == {"platform:f29.1.0:3:00000000", "platform:f29.0.0:3:00000000"}
else:
assert nsvcs == {
"platform:f29.1.0:3:00000000",
"platform:f29.0.0:3:00000000",
"platform:f29.2.0:3:00000000"
}
def test_get_rpm_release_platform_stream_override():
scheduler_init_data(1)
# Set the disttag_marking override on the platform
platform = (db_session.query(models.ModuleBuild).filter_by(
name="platform", stream="f28").first())
platform_mmd = platform.mmd()
platform_xmd = platform_mmd.get_xmd()
platform_xmd["mbs"]["disttag_marking"] = "fedora28"
platform_mmd.set_xmd(platform_xmd)
platform.modulemd = mmd_to_str(platform_mmd)
db_session.add(platform)
db_session.commit()
build_one = models.ModuleBuild.get_by_id(db_session, 2)
release = utils.get_rpm_release(db_session, build_one)
assert release == "module+fedora28+2+814cfa39"
def cleanup_stale_failed_builds():
"""Does various clean up tasks on stale failed module builds"""
if conf.system != "koji":
return
stale_date = datetime.utcnow() - timedelta(days=conf.cleanup_failed_builds_time)
stale_module_builds = db_session.query(models.ModuleBuild).filter(
models.ModuleBuild.state == models.BUILD_STATES["failed"],
models.ModuleBuild.time_modified <= stale_date,
).all()
if stale_module_builds:
log.info(
"%s stale failed module build(s) will be cleaned up",
len(stale_module_builds)
)
for module in stale_module_builds:
log.info("%r is stale and is being cleaned up", module)
# Find completed artifacts in the stale build
artifacts = [c for c in module.component_builds if c.is_completed]
# If there are no completed artifacts, then there is nothing to tag
if artifacts:
# Set buildroot_connect=False so it doesn't recreate the Koji target and etc.
builder = GenericBuilder.create_from_module(
db_session, module, conf, buildroot_connect=False
)
builder.untag_artifacts([c.nvr for c in artifacts])
# Mark the artifacts as untagged in the database
for c in artifacts:
c.tagged = False
c.tagged_in_final = False
db_session.add(c)
state_reason = (
"The module was garbage collected since it has failed over {0}"
" day(s) ago".format(conf.cleanup_failed_builds_time)
)
module.transition(
db_session,
conf,
models.BUILD_STATES["garbage"],
state_reason=state_reason,
failure_type="user",
)
db_session.add(module)
db_session.commit()
def test_get_empty_buildrequired_modulemds(self, request_session):
resolver = mbs_resolver.GenericResolver.create(db_session,
conf,
backend="mbs")
request_session.get.return_value = Mock(ok=True)
request_session.get.return_value.json.return_value = {
"items": [],
"meta": {
"next": None
}
}
platform = db_session.query(
module_build_service.common.models.ModuleBuild).filter_by(
id=1).one()
result = resolver.get_buildrequired_modulemds("nodejs", "10",
platform.mmd())
assert [] == result
def record_module_build_arches(mmd, build):
"""
Finds out the list of build arches against which the ModuleBuld `build` should be built
and records them to `build.arches`.
:param Modulemd mmd: The MMD file associated with a ModuleBuild.
:param ModuleBuild build: The ModuleBuild.
"""
arches = get_build_arches(mmd, conf)
for arch in arches:
arch_obj = db_session.query(
models.ModuleArch).filter_by(name=arch).first()
if not arch_obj:
arch_obj = models.ModuleArch(name=arch)
if arch_obj not in build.arches:
build.arches.append(arch_obj)
db_session.commit()
def test_resolve_profiles_local_module(self, local_builds, conf_system):
"""
Test that profiles get resolved recursively on local builds
"""
# This test requires a platform module loaded from local rather than
# the one added to database.
platform = db_session.query(models.ModuleBuild).filter(
models.ModuleBuild.name == "platform"
).one()
db_session.delete(platform)
db_session.commit()
load_local_builds(["platform:f28"])
mmd = models.ModuleBuild.get_by_id(db_session, 2).mmd()
resolver = mbs_resolver.GenericResolver.create(db_session, conf, backend="mbs")
result = resolver.resolve_profiles(mmd, ("buildroot", "srpm-buildroot"))
expected = {"buildroot": {"foo"}, "srpm-buildroot": {"bar"}}
assert result == expected
def test_get_reusable_module_when_reused_module_already_set(self):
modules = db_session.query(models.ModuleBuild)\
.filter_by(name="testmodule")\
.order_by(models.ModuleBuild.id.desc())\
.limit(2).all()
build_module = modules[0]
reused_module = modules[1]
build_module.state = models.BUILD_STATES["build"]
build_module.reused_module_id = reused_module.id
db_session.commit()
assert build_module.reused_module
assert reused_module == build_module.reused_module
reusable_module = get_reusable_module(build_module)
assert build_module.reused_module
assert reusable_module.id == build_module.reused_module_id
assert reusable_module.id == reused_module.id