def test_start_next_batch_build_reuse_some(self, mock_sbc, default_buildroot_groups): """ Tests that start_next_batch_build: 1) Increments module.batch. 2) Can reuse all components in the batch that it can. 3) Returns proper further_work messages for reused components. 4) Builds the remaining components 5) Handling the further_work messages lead to proper tagging of reused components. """ module_build = models.ModuleBuild.get_by_id(db_session, 3) module_build.batch = 1 plc_component = models.ComponentBuild.from_component_name( db_session, "perl-List-Compare", 3) plc_component.ref = "5ceea46add2366d8b8c5a623a2fb563b625b9abd" builder = mock.MagicMock() builder.recover_orphaned_artifact.return_value = [] start_next_batch_build(conf, module_build, builder) # Batch number should increase. assert module_build.batch == 2 # Make sure we only have one message returned for the one reused component assert len(events.scheduler.queue) == 1 # The KojiBuildChange message in further_work should have build_new_state # set to COMPLETE, but the current component build state in the DB should be set # to BUILDING, so KojiBuildChange message handler handles the change # properly. event_info = events.scheduler.queue[0][3] assert event_info == ('reuse_component: fake msg', 90276227, 1, 'perl-Tangerine', '0.23', '1.module+0+d027b723', 3, 'Reused component from previous module build') component_build = models.ComponentBuild.from_component_event( db_session, task_id=event_info[1], module_id=event_info[6], ) assert component_build.state == koji.BUILD_STATES["BUILDING"] assert component_build.package == "perl-Tangerine" assert component_build.reused_component_id is not None # Make sure perl-List-Compare is set to the build state as well but not reused assert plc_component.state == koji.BUILD_STATES["BUILDING"] assert plc_component.reused_component_id is None mock_sbc.assert_called_once()
def test_start_next_batch_build_rebuild_strategy_all( self, mock_rm, mock_sbc, default_buildroot_groups): """ Tests that start_next_batch_build can't reuse any components in the batch because the rebuild method is set to "all". """ module_build = models.ModuleBuild.get_by_id(db_session, 3) module_build.rebuild_strategy = "all" module_build.batch = 1 builder = mock.MagicMock() builder.recover_orphaned_artifact.return_value = [] start_next_batch_build(conf, module_build, builder) # Batch number should increase. assert module_build.batch == 2 # No component reuse messages should be returned assert len(events.scheduler.queue) == 0 # Make sure that both components in the batch were submitted assert len(mock_sbc.mock_calls) == 2
def test_start_next_batch_build_smart_scheduling(self, mock_sbc, default_buildroot_groups): """ Tests that components with the longest build time will be scheduled first """ module_build = models.ModuleBuild.get_by_id(db_session, 3) module_build.batch = 1 pt_component = models.ComponentBuild.from_component_name( db_session, "perl-Tangerine", 3) pt_component.ref = "6ceea46add2366d8b8c5a623b2fb563b625bfabe" plc_component = models.ComponentBuild.from_component_name( db_session, "perl-List-Compare", 3) plc_component.ref = "5ceea46add2366d8b8c5a623a2fb563b625b9abd" # Components are by default built by component id. To find out that weight is respected, # we have to set bigger weight to component with lower id. pt_component.weight = 3 if pt_component.id < plc_component.id else 4 plc_component.weight = 4 if pt_component.id < plc_component.id else 3 builder = mock.MagicMock() builder.recover_orphaned_artifact.return_value = [] start_next_batch_build(conf, module_build, builder) # Batch number should increase. assert module_build.batch == 2 # Make sure we don't have any messages returned since no components should be reused assert len(events.scheduler.queue) == 0 # Make sure both components are set to the build state but not reused assert pt_component.state == koji.BUILD_STATES["BUILDING"] assert pt_component.reused_component_id is None assert plc_component.state == koji.BUILD_STATES["BUILDING"] assert plc_component.reused_component_id is None # Test the order of the scheduling expected_calls = [ mock.call(db_session, builder, plc_component), mock.call(db_session, builder, pt_component) ] assert mock_sbc.mock_calls == expected_calls
def test_start_next_batch_build_reuse(self, default_buildroot_groups): """ Tests that start_next_batch_build: 1) Increments module.batch. 2) Can reuse all components in batch 3) Returns proper further_work messages for reused components. 4) Returns the fake Repo change message 5) Handling the further_work messages lead to proper tagging of reused components. """ module_build = models.ModuleBuild.get_by_id(db_session, 3) module_build.batch = 1 builder = mock.MagicMock() builder.module_build_tag = {"name": "module-fedora-27-build"} start_next_batch_build(conf, module_build, builder) # Batch number should increase. assert module_build.batch == 2 # buildsys.build.state.change messages in further_work should have # build_new_state set to COMPLETE, but the current component build # state should be set to BUILDING, so KojiBuildChange message handler # handles the change properly. for event in events.scheduler.queue: event_info = event[3] if event_info[0].startswith("reuse_component"): assert event_info[2] == koji.BUILD_STATES["COMPLETE"] component_build = models.ComponentBuild.from_component_event( db_session, task_id=event_info[1], module_id=event_info[6]) assert component_build.state == koji.BUILD_STATES["BUILDING"] # When we handle these KojiBuildChange messages, MBS should tag all # the components just once. events.scheduler.run() # Check that packages have been tagged just once. assert len(DummyModuleBuilder.TAGGED_COMPONENTS) == 2
def process_paused_module_builds(): log.info("Looking for paused module builds in the build state") if at_concurrent_component_threshold(conf): log.debug( "Will not attempt to start paused module builds due to " "the concurrent build threshold being met" ) return ten_minutes = timedelta(minutes=10) # Check for module builds that are in the build state but don't have any active component # builds. Exclude module builds in batch 0. This is likely a build of a module without # components. module_builds = db_session.query(models.ModuleBuild).filter( models.ModuleBuild.state == models.BUILD_STATES["build"], models.ModuleBuild.batch > 0, ).all() for module_build in module_builds: now = datetime.utcnow() # Only give builds a nudge if stuck for more than ten minutes if (now - module_build.time_modified) < ten_minutes: continue # If there are no components in the build state on the module build, # then no possible event will start off new component builds. # But do not try to start new builds when we are waiting for the # repo-regen. if not module_build.current_batch(koji.BUILD_STATES["BUILDING"]): # Initialize the builder... builder = GenericBuilder.create_from_module( db_session, module_build, conf) if has_missed_new_repo_message(module_build, builder.koji_session): log.info(" Processing the paused module build %r", module_build) start_next_batch_build(conf, module_build, builder) # Check if we have met the threshold. if at_concurrent_component_threshold(conf): break
def test_start_next_batch_continue(self, mock_sbc, default_buildroot_groups): """ Tests that start_next_batch_build does not start new batch when there are unbuilt components in the current one. """ module_build = models.ModuleBuild.get_by_id(db_session, 3) module_build.batch = 2 # The component was reused when the batch first started building_component = module_build.current_batch()[0] building_component.state = koji.BUILD_STATES["BUILDING"] db_session.commit() builder = mock.MagicMock() start_next_batch_build(conf, module_build, builder) # Batch number should not increase. assert module_build.batch == 2 # Make sure start build was called for the second component which wasn't reused mock_sbc.assert_called_once() # No further work should be returned assert len(events.scheduler.queue) == 0
def done(msg_id, tag_name): """Called whenever koji rebuilds a repo, any repo. :param str msg_id: the original id of the message being handled which is received from the message bus. :param str tag_name: the tag name from which the repo is generated. """ # First, find our ModuleBuild associated with this repo, if any. if conf.system in ("koji", "test") and not tag_name.endswith("-build"): log.debug("Tag %r does not end with '-build' suffix, ignoring", tag_name) return tag = tag_name[:-6] if tag_name.endswith("-build") else tag_name module_build = models.ModuleBuild.get_by_tag(db_session, tag_name) if not module_build: log.debug("No module build found associated with koji tag %r" % tag) return # It is possible that we have already failed.. but our repo is just being # routinely regenerated. Just ignore that. If module_build_service says the module is # dead, then the module is dead. if module_build.state == models.BUILD_STATES["failed"]: log.info("Ignoring repo regen for already failed %r" % module_build) return # If there are no components in this module build, then current_batch will be empty if module_build.component_builds: current_batch = module_build.current_batch() else: current_batch = [] # Get the list of untagged components in current/previous batches which # have been built successfully if conf.system in ("koji", "test") and current_batch: if any(c.is_completed and not c.is_tagged for c in module_build.up_to_current_batch()): log.info( "Ignoring repo regen, because not all components are tagged.") return if all(c.is_waiting_for_build for c in current_batch): log.info( "Ignoring repo regen because no components have started in the batch." ) return # If any in the current batch are still running.. just wait. running = [c for c in current_batch if c.is_building] if running: log.info( "%r has %r of %r components still building in this batch (%r total)", module_build, len(running), len(current_batch), len(module_build.component_builds)) return # Assemble the list of all successful components in the batch. good = [c for c in current_batch if c.is_completed] # If *none* of the components completed for this batch, then obviously the # module fails. However! We shouldn't reach this scenario. There is # logic over in the component handler which should fail the module build # first before we ever get here. This is here as a race condition safety # valve. if module_build.component_builds and not good: state_reason = "Component(s) {} failed to build.".format(", ".join( c.package for c in current_batch if c.is_unsuccessful)) module_build.transition(db_session, conf, models.BUILD_STATES["failed"], state_reason, failure_type="infra") db_session.commit() log.warning("Odd! All components in batch failed for %r." % module_build) return groups = GenericBuilder.default_buildroot_groups(db_session, module_build) builder = GenericBuilder.create( db_session, module_build.owner, module_build, conf.system, conf, tag_name=tag, components=[c.package for c in module_build.component_builds], ) builder.buildroot_connect(groups) # If we have reached here then we know the following things: # # - All components in this batch have finished (failed or succeeded) # - One or more succeeded. # - They have been regenerated back into the buildroot. # # So now we can either start a new batch if there are still some to build # or, if everything is built successfully, then we can bless the module as # complete. has_unbuilt_components = any(c.is_unbuilt for c in module_build.component_builds) has_failed_components = any(c.is_unsuccessful for c in module_build.component_builds) if has_unbuilt_components and not has_failed_components: # Ok, for the subset of builds that did complete successfully, check to # see if they are in the buildroot before starting new batch. artifacts = [component_build.nvr for component_build in good] if not builder.buildroot_ready(artifacts): log.info("Not all of %r are in the buildroot. Waiting." % artifacts) return # Try to start next batch build, because there are still unbuilt # components in a module. start_next_batch_build(conf, module_build, builder) else: if has_failed_components: state_reason = "Component(s) {} failed to build.".format(", ".join( c.package for c in module_build.component_builds if c.is_unsuccessful)) module_build.transition( db_session, conf, state=models.BUILD_STATES["failed"], state_reason=state_reason, failure_type="user", ) else: # Tell the external buildsystem to wrap up (CG import, createrepo, etc.) module_build.time_completed = datetime.utcnow() builder.finalize(succeeded=True) module_build.transition(db_session, conf, state=models.BUILD_STATES["done"]) db_session.commit()
def test_start_next_batch_build_rebuild_strategy_only_changed( self, mock_rm, mock_sbc, default_buildroot_groups): """ Tests that start_next_batch_build reuses all unchanged components in the batch because the rebuild method is set to "only-changed". This means that one component is reused in batch 2, and even though the other component in batch 2 changed and was rebuilt, the component in batch 3 can be reused. """ module_build = models.ModuleBuild.get_by_id(db_session, 3) module_build.rebuild_strategy = "only-changed" module_build.batch = 1 # perl-List-Compare changed plc_component = models.ComponentBuild.from_component_name( db_session, "perl-List-Compare", 3) plc_component.ref = "5ceea46add2366d8b8c5a623a2fb563b625b9abd" builder = mock.MagicMock() builder.recover_orphaned_artifact.return_value = [] start_next_batch_build(conf, module_build, builder) # Batch number should increase assert module_build.batch == 2 # Make sure we only have one message returned for the one reused component assert len(events.scheduler.queue) == 1 # The buildsys.build.state.change message in further_work should have # build_new_state set to COMPLETE, but the current component build state # in the DB should be set to BUILDING, so the build state change handler # handles the change properly. event_info = events.scheduler.queue[0][3] assert event_info == ('reuse_component: fake msg', 90276227, 1, 'perl-Tangerine', '0.23', '1.module+0+d027b723', 3, 'Reused component from previous module build') component_build = models.ComponentBuild.from_component_event( db_session, task_id=event_info[1], module_id=event_info[6], ) assert component_build.state == koji.BUILD_STATES["BUILDING"] assert component_build.package == "perl-Tangerine" assert component_build.reused_component_id is not None # Make sure perl-List-Compare is set to the build state as well but not reused assert plc_component.state == koji.BUILD_STATES["BUILDING"] assert plc_component.reused_component_id is None mock_sbc.assert_called_once() mock_sbc.reset_mock() # Complete the build plc_component.state = koji.BUILD_STATES["COMPLETE"] pt_component = models.ComponentBuild.from_component_name( db_session, "perl-Tangerine", 3) pt_component.state = koji.BUILD_STATES["COMPLETE"] events.scheduler.reset() # Start the next build batch start_next_batch_build(conf, module_build, builder) # Batch number should increase assert module_build.batch == 3 # Verify that tangerine was reused even though perl-Tangerine was rebuilt in the previous # batch event_info = events.scheduler.queue[0][3] assert event_info == ('reuse_component: fake msg', 90276315, 1, 'tangerine', '0.22', '3.module+0+d027b723', 3, 'Reused component from previous module build') component_build = models.ComponentBuild.from_component_event( db_session, task_id=event_info[1], module_id=event_info[6], ) assert component_build.state == koji.BUILD_STATES["BUILDING"] assert component_build.package == "tangerine" assert component_build.reused_component_id is not None mock_sbc.assert_not_called()