def test_start_next_batch_build_reuse_some(self, mock_sbc,
default_buildroot_groups):
"""
Tests that start_next_batch_build:
1) Increments module.batch.
2) Can reuse all components in the batch that it can.
3) Returns proper further_work messages for reused components.
4) Builds the remaining components
5) Handling the further_work messages lead to proper tagging of
reused components.
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.batch = 1
plc_component = models.ComponentBuild.from_component_name(
db_session, "perl-List-Compare", 3)
plc_component.ref = "5ceea46add2366d8b8c5a623a2fb563b625b9abd"
builder = mock.MagicMock()
builder.recover_orphaned_artifact.return_value = []
start_next_batch_build(conf, module_build, builder)
# Batch number should increase.
assert module_build.batch == 2
# Make sure we only have one message returned for the one reused component
assert len(events.scheduler.queue) == 1
# The KojiBuildChange message in further_work should have build_new_state
# set to COMPLETE, but the current component build state in the DB should be set
# to BUILDING, so KojiBuildChange message handler handles the change
# properly.
event_info = events.scheduler.queue[0][3]
assert event_info == ('reuse_component: fake msg', 90276227, 1,
'perl-Tangerine', '0.23', '1.module+0+d027b723',
3, 'Reused component from previous module build')
component_build = models.ComponentBuild.from_component_event(
db_session,
task_id=event_info[1],
module_id=event_info[6],
)
assert component_build.state == koji.BUILD_STATES["BUILDING"]
assert component_build.package == "perl-Tangerine"
assert component_build.reused_component_id is not None
# Make sure perl-List-Compare is set to the build state as well but not reused
assert plc_component.state == koji.BUILD_STATES["BUILDING"]
assert plc_component.reused_component_id is None
mock_sbc.assert_called_once()
def test_start_next_batch_build_rebuild_strategy_all(
self, mock_rm, mock_sbc, default_buildroot_groups):
"""
Tests that start_next_batch_build can't reuse any components in the batch because the
rebuild method is set to "all".
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.rebuild_strategy = "all"
module_build.batch = 1
builder = mock.MagicMock()
builder.recover_orphaned_artifact.return_value = []
start_next_batch_build(conf, module_build, builder)
# Batch number should increase.
assert module_build.batch == 2
# No component reuse messages should be returned
assert len(events.scheduler.queue) == 0
# Make sure that both components in the batch were submitted
assert len(mock_sbc.mock_calls) == 2
def test_start_next_batch_build_smart_scheduling(self, mock_sbc,
default_buildroot_groups):
"""
Tests that components with the longest build time will be scheduled first
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.batch = 1
pt_component = models.ComponentBuild.from_component_name(
db_session, "perl-Tangerine", 3)
pt_component.ref = "6ceea46add2366d8b8c5a623b2fb563b625bfabe"
plc_component = models.ComponentBuild.from_component_name(
db_session, "perl-List-Compare", 3)
plc_component.ref = "5ceea46add2366d8b8c5a623a2fb563b625b9abd"
# Components are by default built by component id. To find out that weight is respected,
# we have to set bigger weight to component with lower id.
pt_component.weight = 3 if pt_component.id < plc_component.id else 4
plc_component.weight = 4 if pt_component.id < plc_component.id else 3
builder = mock.MagicMock()
builder.recover_orphaned_artifact.return_value = []
start_next_batch_build(conf, module_build, builder)
# Batch number should increase.
assert module_build.batch == 2
# Make sure we don't have any messages returned since no components should be reused
assert len(events.scheduler.queue) == 0
# Make sure both components are set to the build state but not reused
assert pt_component.state == koji.BUILD_STATES["BUILDING"]
assert pt_component.reused_component_id is None
assert plc_component.state == koji.BUILD_STATES["BUILDING"]
assert plc_component.reused_component_id is None
# Test the order of the scheduling
expected_calls = [
mock.call(db_session, builder, plc_component),
mock.call(db_session, builder, pt_component)
]
assert mock_sbc.mock_calls == expected_calls
def test_start_next_batch_build_reuse(self, default_buildroot_groups):
"""
Tests that start_next_batch_build:
1) Increments module.batch.
2) Can reuse all components in batch
3) Returns proper further_work messages for reused components.
4) Returns the fake Repo change message
5) Handling the further_work messages lead to proper tagging of
reused components.
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.batch = 1
builder = mock.MagicMock()
builder.module_build_tag = {"name": "module-fedora-27-build"}
start_next_batch_build(conf, module_build, builder)
# Batch number should increase.
assert module_build.batch == 2
# buildsys.build.state.change messages in further_work should have
# build_new_state set to COMPLETE, but the current component build
# state should be set to BUILDING, so KojiBuildChange message handler
# handles the change properly.
for event in events.scheduler.queue:
event_info = event[3]
if event_info[0].startswith("reuse_component"):
assert event_info[2] == koji.BUILD_STATES["COMPLETE"]
component_build = models.ComponentBuild.from_component_event(
db_session, task_id=event_info[1], module_id=event_info[6])
assert component_build.state == koji.BUILD_STATES["BUILDING"]
# When we handle these KojiBuildChange messages, MBS should tag all
# the components just once.
events.scheduler.run()
# Check that packages have been tagged just once.
assert len(DummyModuleBuilder.TAGGED_COMPONENTS) == 2
def process_paused_module_builds():
log.info("Looking for paused module builds in the build state")
if at_concurrent_component_threshold(conf):
log.debug(
"Will not attempt to start paused module builds due to "
"the concurrent build threshold being met"
)
return
ten_minutes = timedelta(minutes=10)
# Check for module builds that are in the build state but don't have any active component
# builds. Exclude module builds in batch 0. This is likely a build of a module without
# components.
module_builds = db_session.query(models.ModuleBuild).filter(
models.ModuleBuild.state == models.BUILD_STATES["build"],
models.ModuleBuild.batch > 0,
).all()
for module_build in module_builds:
now = datetime.utcnow()
# Only give builds a nudge if stuck for more than ten minutes
if (now - module_build.time_modified) < ten_minutes:
continue
# If there are no components in the build state on the module build,
# then no possible event will start off new component builds.
# But do not try to start new builds when we are waiting for the
# repo-regen.
if not module_build.current_batch(koji.BUILD_STATES["BUILDING"]):
# Initialize the builder...
builder = GenericBuilder.create_from_module(
db_session, module_build, conf)
if has_missed_new_repo_message(module_build, builder.koji_session):
log.info(" Processing the paused module build %r", module_build)
start_next_batch_build(conf, module_build, builder)
# Check if we have met the threshold.
if at_concurrent_component_threshold(conf):
break
def test_start_next_batch_continue(self, mock_sbc,
default_buildroot_groups):
"""
Tests that start_next_batch_build does not start new batch when
there are unbuilt components in the current one.
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.batch = 2
# The component was reused when the batch first started
building_component = module_build.current_batch()[0]
building_component.state = koji.BUILD_STATES["BUILDING"]
db_session.commit()
builder = mock.MagicMock()
start_next_batch_build(conf, module_build, builder)
# Batch number should not increase.
assert module_build.batch == 2
# Make sure start build was called for the second component which wasn't reused
mock_sbc.assert_called_once()
# No further work should be returned
assert len(events.scheduler.queue) == 0
def done(msg_id, tag_name):
"""Called whenever koji rebuilds a repo, any repo.
:param str msg_id: the original id of the message being handled which is
received from the message bus.
:param str tag_name: the tag name from which the repo is generated.
"""
# First, find our ModuleBuild associated with this repo, if any.
if conf.system in ("koji", "test") and not tag_name.endswith("-build"):
log.debug("Tag %r does not end with '-build' suffix, ignoring",
tag_name)
return
tag = tag_name[:-6] if tag_name.endswith("-build") else tag_name
module_build = models.ModuleBuild.get_by_tag(db_session, tag_name)
if not module_build:
log.debug("No module build found associated with koji tag %r" % tag)
return
# It is possible that we have already failed.. but our repo is just being
# routinely regenerated. Just ignore that. If module_build_service says the module is
# dead, then the module is dead.
if module_build.state == models.BUILD_STATES["failed"]:
log.info("Ignoring repo regen for already failed %r" % module_build)
return
# If there are no components in this module build, then current_batch will be empty
if module_build.component_builds:
current_batch = module_build.current_batch()
else:
current_batch = []
# Get the list of untagged components in current/previous batches which
# have been built successfully
if conf.system in ("koji", "test") and current_batch:
if any(c.is_completed and not c.is_tagged
for c in module_build.up_to_current_batch()):
log.info(
"Ignoring repo regen, because not all components are tagged.")
return
if all(c.is_waiting_for_build for c in current_batch):
log.info(
"Ignoring repo regen because no components have started in the batch."
)
return
# If any in the current batch are still running.. just wait.
running = [c for c in current_batch if c.is_building]
if running:
log.info(
"%r has %r of %r components still building in this batch (%r total)",
module_build, len(running), len(current_batch),
len(module_build.component_builds))
return
# Assemble the list of all successful components in the batch.
good = [c for c in current_batch if c.is_completed]
# If *none* of the components completed for this batch, then obviously the
# module fails. However! We shouldn't reach this scenario. There is
# logic over in the component handler which should fail the module build
# first before we ever get here. This is here as a race condition safety
# valve.
if module_build.component_builds and not good:
state_reason = "Component(s) {} failed to build.".format(", ".join(
c.package for c in current_batch if c.is_unsuccessful))
module_build.transition(db_session,
conf,
models.BUILD_STATES["failed"],
state_reason,
failure_type="infra")
db_session.commit()
log.warning("Odd! All components in batch failed for %r." %
module_build)
return
groups = GenericBuilder.default_buildroot_groups(db_session, module_build)
builder = GenericBuilder.create(
db_session,
module_build.owner,
module_build,
conf.system,
conf,
tag_name=tag,
components=[c.package for c in module_build.component_builds],
)
builder.buildroot_connect(groups)
# If we have reached here then we know the following things:
#
# - All components in this batch have finished (failed or succeeded)
# - One or more succeeded.
# - They have been regenerated back into the buildroot.
#
# So now we can either start a new batch if there are still some to build
# or, if everything is built successfully, then we can bless the module as
# complete.
has_unbuilt_components = any(c.is_unbuilt
for c in module_build.component_builds)
has_failed_components = any(c.is_unsuccessful
for c in module_build.component_builds)
if has_unbuilt_components and not has_failed_components:
# Ok, for the subset of builds that did complete successfully, check to
# see if they are in the buildroot before starting new batch.
artifacts = [component_build.nvr for component_build in good]
if not builder.buildroot_ready(artifacts):
log.info("Not all of %r are in the buildroot. Waiting." %
artifacts)
return
# Try to start next batch build, because there are still unbuilt
# components in a module.
start_next_batch_build(conf, module_build, builder)
else:
if has_failed_components:
state_reason = "Component(s) {} failed to build.".format(", ".join(
c.package for c in module_build.component_builds
if c.is_unsuccessful))
module_build.transition(
db_session,
conf,
state=models.BUILD_STATES["failed"],
state_reason=state_reason,
failure_type="user",
)
else:
# Tell the external buildsystem to wrap up (CG import, createrepo, etc.)
module_build.time_completed = datetime.utcnow()
builder.finalize(succeeded=True)
module_build.transition(db_session,
conf,
state=models.BUILD_STATES["done"])
db_session.commit()
def test_start_next_batch_build_rebuild_strategy_only_changed(
self, mock_rm, mock_sbc, default_buildroot_groups):
"""
Tests that start_next_batch_build reuses all unchanged components in the batch because the
rebuild method is set to "only-changed". This means that one component is reused in batch
2, and even though the other component in batch 2 changed and was rebuilt, the component
in batch 3 can be reused.
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.rebuild_strategy = "only-changed"
module_build.batch = 1
# perl-List-Compare changed
plc_component = models.ComponentBuild.from_component_name(
db_session, "perl-List-Compare", 3)
plc_component.ref = "5ceea46add2366d8b8c5a623a2fb563b625b9abd"
builder = mock.MagicMock()
builder.recover_orphaned_artifact.return_value = []
start_next_batch_build(conf, module_build, builder)
# Batch number should increase
assert module_build.batch == 2
# Make sure we only have one message returned for the one reused component
assert len(events.scheduler.queue) == 1
# The buildsys.build.state.change message in further_work should have
# build_new_state set to COMPLETE, but the current component build state
# in the DB should be set to BUILDING, so the build state change handler
# handles the change properly.
event_info = events.scheduler.queue[0][3]
assert event_info == ('reuse_component: fake msg', 90276227, 1,
'perl-Tangerine', '0.23', '1.module+0+d027b723',
3, 'Reused component from previous module build')
component_build = models.ComponentBuild.from_component_event(
db_session,
task_id=event_info[1],
module_id=event_info[6],
)
assert component_build.state == koji.BUILD_STATES["BUILDING"]
assert component_build.package == "perl-Tangerine"
assert component_build.reused_component_id is not None
# Make sure perl-List-Compare is set to the build state as well but not reused
assert plc_component.state == koji.BUILD_STATES["BUILDING"]
assert plc_component.reused_component_id is None
mock_sbc.assert_called_once()
mock_sbc.reset_mock()
# Complete the build
plc_component.state = koji.BUILD_STATES["COMPLETE"]
pt_component = models.ComponentBuild.from_component_name(
db_session, "perl-Tangerine", 3)
pt_component.state = koji.BUILD_STATES["COMPLETE"]
events.scheduler.reset()
# Start the next build batch
start_next_batch_build(conf, module_build, builder)
# Batch number should increase
assert module_build.batch == 3
# Verify that tangerine was reused even though perl-Tangerine was rebuilt in the previous
# batch
event_info = events.scheduler.queue[0][3]
assert event_info == ('reuse_component: fake msg', 90276315, 1,
'tangerine', '0.22', '3.module+0+d027b723', 3,
'Reused component from previous module build')
component_build = models.ComponentBuild.from_component_event(
db_session,
task_id=event_info[1],
module_id=event_info[6],
)
assert component_build.state == koji.BUILD_STATES["BUILDING"]
assert component_build.package == "tangerine"
assert component_build.reused_component_id is not None
mock_sbc.assert_not_called()