def makeScheduler(self, **kwargs): sched = self.attachScheduler(timed.Nightly(**kwargs), self.OBJECTID, overrideBuildsetMethods=True) self.master.db.insertTestData( [fakedb.Builder(name=bname) for bname in kwargs.get("builderNames", [])]) # add a Clock to help checking timing issues self.clock = sched._reactor = task.Clock() self.clock.advance(self.localtime_offset) # get to 0 min past the hour self.addBuildsetCallTimes = [] def recordTimes(timeList, method): def timedMethod(**kw): timeList.append(self.clock.seconds() - self.localtime_offset) return method(**kw) return timedMethod sched.addBuildsetForSourceStampsWithDefaults = recordTimes( self.addBuildsetCallTimes, sched.addBuildsetForSourceStampsWithDefaults) sched.addBuildsetForChanges = recordTimes( self.addBuildsetCallTimes, sched.addBuildsetForChanges) # see self.assertConsumingChanges self.consumingChanges = None def startConsumingChanges(**kwargs): self.consumingChanges = kwargs return defer.succeed(None) sched.startConsumingChanges = startConsumingChanges return sched
def test_create_scheduler_changes(self): s = timed.Nightly(name="tsched", builderNames=['tbuild'], onlyIfChanged=True) s.parent = DummyParent(self.dbc) d = self.dbc.addSchedulers([s]) # Add some changes for i in range(10): c = Change(who='just a guy', files=[], comments="") d.addCallback(lambda res: self.dbc.addChangeToDatabase(c)) def runScheduler(res): return s.run() d.addCallback(runScheduler) def checkTables(res): # Check that we have the number of changes we think we should have self.assertEquals(len(self.dbc.changes), 10) # Check that there are entries in scheduler_changes important, unimportant = self.dbc.classified_changes.get( s.schedulerid, ([], [])) self.assertEquals(len(important + unimportant), 10) d.addCallback(checkTables) return d
def makeScheduler(self, firstBuildDuration=0, **kwargs): sched = self.attachScheduler(timed.Nightly(**kwargs), self.OBJECTID, overrideBuildsetMethods=True) # add a Clock to help checking timing issues self.clock = sched._reactor = task.Clock() self.clock.advance(self.localtime_offset) # get to 0 min past the hour # keep track of builds in self.events self.events = [] def addBuildsetForLatest(reason='', external_idstring='', branch=None, repository='', project=''): self.assertIn('scheduler named', reason) isFirst = (self.events == []) self.events.append('B(%s)@%d' % ( branch, # show the offset as seconds past the GMT hour self.clock.seconds() - self.localtime_offset)) if isFirst and firstBuildDuration: d = defer.Deferred() self.clock.callLater(firstBuildDuration, d.callback, None) return d else: return defer.succeed(None) sched.addBuildsetForLatest = addBuildsetForLatest origAddBuildsetForChanges = sched.addBuildsetForChanges def addBuildsetForChanges(reason='', external_idstring=None, changeids=[]): self.events.append('B%s@%d' % ( repr(changeids).replace(' ', ''), # show the offset as seconds past the GMT hour self.clock.seconds() - self.localtime_offset)) return origAddBuildsetForChanges( reason=reason, external_idstring=external_idstring, changeids=changeids) sched.addBuildsetForChanges = addBuildsetForChanges # see self.assertConsumingChanges self.consumingChanges = None def startConsumingChanges(**kwargs): self.consumingChanges = kwargs return defer.succeed(None) sched.startConsumingChanges = startConsumingChanges return sched
def makeScheduler(self, firstBuildDuration=0, **kwargs): sched = self.attachScheduler(timed.Nightly(**kwargs), self.SCHEDULERID) # add a Clock to help checking timing issues self.clock = sched._reactor = task.Clock() # keep track of builds in self.events self.events = [] def addBuildsetForLatest(reason='', external_idstring='', branch=None, repository='', project=''): self.assertIn('scheduler named', reason) isFirst = (self.events == []) self.events.append('B(%s)@%d' % (branch, self.clock.seconds())) if isFirst and firstBuildDuration: d = defer.Deferred() self.clock.callLater(firstBuildDuration, d.callback, None) return d else: return defer.succeed(None) sched.addBuildsetForLatest = addBuildsetForLatest def addBuildsetForChanges(reason='', external_idstring='', changeids=[]): self.events.append( 'B%s@%d' % ( ` changeids `.replace(' ', ''), self.clock.seconds())) return defer.succeed(None) sched.addBuildsetForChanges = addBuildsetForChanges # see self.assertConsumingChanges self.consumingChanges = None def startConsumingChanges(**kwargs): self.consumingChanges = kwargs return defer.succeed(None) sched.startConsumingChanges = startConsumingChanges return sched
def test_expire_old_scheduler_changes(self): s = timed.Nightly( name="tsched", builderNames=['tbuild'], ) s.parent = DummyParent(self.dbc) # Hack the scheduler so that it always runs def _check_timer(t): now = time.time() s._maybe_start_build(t) s.update_last_build(t, now) # reschedule for the next timer return now + 10 s._check_timer = _check_timer d = self.dbc.addSchedulers([s]) # Add a changes c = Change(who='just a guy', files=[], comments="") d.addCallback(lambda res: self.dbc.addChangeToDatabase(c)) # Add some dummy scheduler_changes def addSchedulerChanges(res): for i in range(100): self.dbc.classified_changes.setdefault(s.schedulerid, ([], []))[0].append(c) d.addCallback(addSchedulerChanges) def runScheduler(res): return s.run() d.addCallback(runScheduler) def checkTables(res): # Check that there are no entries in scheduler_changes important, unimportant = self.dbc.classified_changes.get( s.schedulerid, ([], [])) self.assertEquals(len(important + unimportant), 0) d.addCallback(checkTables) return d
def SetupBuildersAndSchedulers(c, builders, slaves, ActiveMaster): """Set up builders and schedulers for the build master.""" # List of dicts for every builder. builder_dicts = [] # Builder names by scheduler. builders_by_scheduler = {s: [] for s in SCHEDULERS} # Maps a triggering builder to its triggered builders. triggered_builders = collections.defaultdict(list) def process_builder(builder, is_trybot=False): """Create a dict for the given builder and place its name in the appropriate scheduler list. """ builder_name = builder['name'] if is_trybot: builder_name = builder_name_schema.TrybotName(builder_name) # Categorize the builder based on its role. try: category = builder_name_schema.DictForBuilderName( builder_name)['role'] subcategory = builder_name.split( builder_name_schema.BUILDER_NAME_SEP)[1] category = '|'.join((category, subcategory)) except ValueError: # Assume that all builders whose names don't play by our rules are named # upstream and are therefore canaries. category = builder_name_schema.BUILDER_ROLE_CANARY builder_dict = { 'name': builder_name, 'gatekeeper': builder.get('gatekeeper_categories', ''), 'auto_reboot': builder.get('auto_reboot', DEFAULT_AUTO_REBOOT), 'slavenames': slaves.GetSlavesName(builder=builder['name']), 'category': category, 'recipe': builder.get('recipe', DEFAULT_RECIPE), } builder_dicts.append(builder_dict) parent_builder = builder.get('triggered_by') if parent_builder is not None: assert builder.get('scheduler') is None if is_trybot: parent_builder = builder_name_schema.TrybotName(parent_builder) triggered_builders[parent_builder].append(builder_name) elif is_trybot: builders_by_scheduler[TRY_SCHEDULER_NAME].append(builder_name) else: scheduler = builder.get('scheduler', PERCOMMIT_SCHEDULER_NAME) builders_by_scheduler[scheduler].append(builder_name) # Create builders and trybots. for builder in builders: process_builder(builder) if builder.get('do_trybot', DEFAULT_DO_TRYBOT): process_builder(builder, is_trybot=True) # Verify that all parent builders exist. all_nontriggered_builders = set( builders_by_scheduler[PERCOMMIT_SCHEDULER_NAME]).union( set(builders_by_scheduler[TRY_SCHEDULER_NAME])) trigger_parents = set(triggered_builders.keys()) nonexistent_parents = trigger_parents - all_nontriggered_builders if nonexistent_parents: raise Exception('Could not find parent builders: %s' % ', '.join(nonexistent_parents)) # Create the schedulers. def trigger_name(parent_builder): """Given a parent builder name, return a triggerable scheduler name.""" return 'triggers_%s' % parent_builder c['schedulers'] = [] s = Scheduler(name=PERCOMMIT_SCHEDULER_NAME, branch=POLLING_BRANCH, treeStableTimer=60, builderNames=builders_by_scheduler[PERCOMMIT_SCHEDULER_NAME]) c['schedulers'].append(s) s = timed.Nightly( name=PERIODIC_15MINS_SCHEDULER_NAME, branch=POLLING_BRANCH, builderNames=builders_by_scheduler[PERIODIC_15MINS_SCHEDULER_NAME], minute=[i * 15 for i in xrange(60 / 15)], hour='*', dayOfMonth='*', month='*', dayOfWeek='*') c['schedulers'].append(s) for parent, builders_to_trigger in triggered_builders.iteritems(): c['schedulers'].append( Triggerable(name=trigger_name(parent), builderNames=builders_to_trigger)) pools = BuildersPools(TRY_SCHEDULER_NAME) pools[TRY_SCHEDULER_NAME].extend(builders_by_scheduler[TRY_SCHEDULER_NAME]) c['schedulers'].append( TryJobRietveld(name=TRY_SCHEDULER_NAME, code_review_sites={ TRY_SCHEDULER_PROJECT: ActiveMaster.code_review_site }, pools=pools, project=TRY_SCHEDULER_PROJECT, filter_master=True)) # Create the BuildFactorys. annotator = annotator_factory.AnnotatorFactory() for builder_dict in builder_dicts: triggers = ([trigger_name(builder_dict['name'])] if builder_dict['name'] in triggered_builders else None) builder_dict['factory'] = annotator.BaseFactory(builder_dict['recipe'], triggers=triggers) # Finished! c['builders'] = builder_dicts
treeStableTimer=None, builderNames=['release'])) c['schedulers'].append( SingleBranchScheduler(name="features", change_filter=filter.ChangeFilter(branch_re='.+'), treeStableTimer=None, builderNames=['continuously'])) c['schedulers'].append( ForceScheduler(name="force", builderNames=['nightly', 'weekly'])) c['schedulers'].append( timed.Nightly(name='sch_nightly', branch='develop', builderNames=['nightly'], hour=config.NIGHTLY_TIME[0], minute=config.NIGHTLY_TIME[1], onlyIfChanged=True)) c['schedulers'].append( timed.Nightly(name='sch_weekly', branch='develop', builderNames=['weekly'], dayOfWeek=config.WEEKLY_TIME[2], hour=config.WEEKLY_TIME[0], minute=config.WEEKLY_TIME[1], onlyIfChanged=True)) ####### BUILDERS from buildbot.process.factory import BuildFactory
def Update(self, c): for builder in self._builders: # Update the schedulers with the builder. schedulers = builder['schedulers'] if schedulers: for scheduler in schedulers: self._schedulers[scheduler]['builders'].append(builder['name']) # Construct the category. categories = [] if builder.get('category', None): categories.append(builder['category']) if builder.get('gatekeeper', None): categories.extend(builder['gatekeeper'].split('|')) category = '|'.join(categories) # Append the builder to the list. new_builder = {'name': builder['name'], 'factory': self._factories[builder['factory']], 'category': category, 'auto_reboot': builder['auto_reboot']} if builder['builddir']: new_builder['builddir'] = builder['builddir'] c['builders'].append(new_builder) c['builders'].sort(key=lambda builder: builder['name']) # Process the main schedulers. for s_name in self._schedulers: scheduler = self._schedulers[s_name] if scheduler['type'] == 'Scheduler': def filter_fn(change, builders): """Filters out if change.comments contains certain keywords. The change is filtered out if the commit message contains: * SKIP_BUILDBOT_SUBSTRING or * RUN_BUILDERS_REGEX when the scheduler does not contain any of the specified builders Args: change: An instance of changes.Change. builders: Sequence of strings. The builders that are run by this scheduler. Returns: If the change should be filtered out (i.e. not run by the buildbot code) then False is returned else True is returned. """ if SKIP_BUILDBOT_SUBSTRING in change.comments: return False match_obj = RUN_BUILDERS_RE_COMPILED.search(change.comments) if builders and match_obj: for builder_to_run in match_obj.group(1).split(','): if builder_to_run.strip() in builders: break else: return False return True skia_change_filter = SkiaChangeFilter( builders=scheduler['builders'], branch=skia_vars.GetGlobalVariable('master_branch_name'), filter_fn=filter_fn) instance = Scheduler(name=s_name, treeStableTimer=scheduler['treeStableTimer'], builderNames=scheduler['builders'], change_filter=skia_change_filter) c['schedulers'].append(instance) self._schedulers[s_name]['instance'] = instance # Process the periodic schedulers. for s_name in self._schedulers: scheduler = self._schedulers[s_name] if scheduler['type'] == 'PeriodicScheduler': instance = timed.Nightly( name=s_name, branch=skia_vars.GetGlobalVariable('master_branch_name'), builderNames=scheduler['builders'], minute=scheduler['minute'], hour=scheduler['hour'], dayOfMonth=scheduler['dayOfMonth'], month=scheduler['month'], dayOfWeek=scheduler['dayOfWeek']) c['schedulers'].append(instance) self._schedulers[s_name]['instance'] = instance # Process the Rietveld-based try schedulers. for s_name in self._schedulers: scheduler = self._schedulers[s_name] if scheduler['type'] == 'TryJobRietveld': pools = BuildersPools(s_name) pools[s_name].extend(scheduler['builders']) instance = try_job_rietveld.TryJobRietveld( name=s_name, pools=pools, last_good_urls={'skia': None}, code_review_sites={'skia': config_private.CODE_REVIEW_SITE}, project='skia') c['schedulers'].append(instance) self._schedulers[s_name]['instance'] = instance # Process the svn-based try schedulers. for s_name in self._schedulers: scheduler = self._schedulers[s_name] if scheduler['type'] == 'TryJobSubversion': pools = BuildersPools(s_name) pools[s_name].extend(scheduler['builders']) instance = try_job_svn.TryJobSubversion( name=s_name, svn_url=TRY_SVN_BASEURL, last_good_urls={'skia': None}, code_review_sites={'skia': config_private.CODE_REVIEW_SITE}, pools=pools) c['schedulers'].append(instance) self._schedulers[s_name]['instance'] = instance # Process the dependent schedulers. for s_name in self._schedulers: scheduler = self._schedulers[s_name] if scheduler['type'] == 'Dependent': instance = Dependent( s_name, self._schedulers[scheduler['parent']]['instance'], scheduler['builders']) c['schedulers'].append(instance) self._schedulers[s_name]['instance'] = instance
break return is_important scheduler_win_Server_Business_build = SingleBranchScheduler(name = "win_Server_Business", change_filter = ChangeFilter(branch_fn=dev_branch_fn_git), treeStableTimer = 5, builderNames = ["win_Server_Business"], fileIsImportant = isImportantWin_Server_BusinessWindows, onlyImportant = True, properties={'owner':['*****@*****.**']}) Win_Server_Business_time = timed.Nightly( name = 'Win_Server_Business', builderNames = ["win_Server_Business"], branch = "trunk/goland", hour = [200], minute = 200, dayOfWeek = [0, 1, 2, 3, 4]) win_Server_Business_force_builder= ForceScheduler( name="all_win_Server_Business", builderNames=["win_Server_Business"]) ################################# Factory ################################ import dvdfab_factory_win_Server_Business f_win_Server_Business = BuildFactory() win_Server_Business = dvdfab_factory_win_Server_Business.Factory(f_win_Server_Business) f_win_Server_Business = win_Server_Business.f_build()
def SetupBuildersAndSchedulers(c, builders, slaves, ActiveMaster): """Set up builders and schedulers for the build master.""" # List of dicts for every builder. builder_dicts = [] # Builder names by scheduler. builders_by_scheduler = {s: [] for s in SCHEDULERS} # Maps a triggering builder to its triggered builders. triggered_builders = collections.defaultdict(list) def process_builder(builder, is_trybot=False): """Create a dict for the given builder and place its name in the appropriate scheduler list. """ builder_name = builder['name'] if is_trybot: builder_name = builder_name_schema.TrybotName(builder_name) # Categorize the builder based on its role. try: category = builder_name_schema.DictForBuilderName( builder_name)['role'] subcategory = builder_name.split( builder_name_schema.BUILDER_NAME_SEP)[1] category = '|'.join((category, subcategory)) except ValueError: # Assume that all builders whose names don't play by our rules are named # upstream and are therefore canaries. category = builder_name_schema.BUILDER_ROLE_CANARY properties = builder.get('properties', {}) cc = builder.get('cc') if cc: if isinstance(cc, basestring): cc = [cc] properties['owners'] = cc builder_dict = { 'name': builder_name, 'auto_reboot': builder.get('auto_reboot', DEFAULT_AUTO_REBOOT), 'slavenames': slaves.GetSlavesName(builder=builder['name']), 'category': category, 'recipe': builder.get('recipe', DEFAULT_RECIPE), 'properties': properties, 'mergeRequests': builder.get('can_merge_requests', CanMergeBuildRequests), 'slavebuilddir': SLAVE_WORKDIR, } builder_dicts.append(builder_dict) parent_builder = builder.get('triggered_by') if parent_builder is not None: assert builder.get('scheduler') is None if is_trybot: parent_builder = builder_name_schema.TrybotName(parent_builder) triggered_builders[parent_builder].append(builder_name) else: scheduler = builder.get('scheduler', BUILDBUCKET_SCHEDULER_NAME) # Setting the scheduler to BUILDBUCKET_SCHEDULER_NAME indicates that # BuildBucket is the only way to schedule builds for this bot; just # pretend to add a scheduler in those cases. builders_by_scheduler[scheduler].append(builder_name) # Create builders and trybots. for builder in builders: if builder.get('trybot_only', DEFAULT_TRYBOT_ONLY): # trybot_only=True should only be used in combination with do_trybot=True # Also, the buildername then needs to already have the '-Trybot' suffix. assert builder.get('do_trybot', DEFAULT_DO_TRYBOT) assert builder['name'] == builder_name_schema.TrybotName( builder['name']) else: process_builder(builder) if builder.get('do_trybot', DEFAULT_DO_TRYBOT): process_builder(builder, is_trybot=True) # Verify that all parent builders exist. all_nontriggered_builders = set( builders_by_scheduler[BUILDBUCKET_SCHEDULER_NAME]) trigger_parents = set(triggered_builders.keys()) nonexistent_parents = trigger_parents - all_nontriggered_builders if nonexistent_parents: raise Exception('Could not find parent builders: %s' % ', '.join(nonexistent_parents)) # Create the schedulers. infra_change_filter = change_filter.ChangeFilter( project='buildbot', repository=global_constants.INFRA_REPO) skia_master_only_change_filter = change_filter.ChangeFilter( project='skia', repository=ActiveMaster.repo_url, branch=MASTER_BRANCH) c['schedulers'] = [] s = Scheduler( name=MASTER_ONLY_SCHEDULER_NAME, treeStableTimer=60, change_filter=skia_master_only_change_filter, builderNames=builders_by_scheduler[MASTER_ONLY_SCHEDULER_NAME]) c['schedulers'].append(s) s = timed.Nightly( name=PERIODIC_15MINS_SCHEDULER_NAME, branch=MASTER_BRANCH, builderNames=builders_by_scheduler[PERIODIC_15MINS_SCHEDULER_NAME], minute=[i * 15 for i in xrange(60 / 15)], hour='*', dayOfMonth='*', month='*', dayOfWeek='*') c['schedulers'].append(s) s = timed.Nightly( name=NIGHTLY_SCHEDULER_NAME, branch=MASTER_BRANCH, builderNames=builders_by_scheduler[NIGHTLY_SCHEDULER_NAME], minute=0, hour=22, dayOfMonth='*', month='*', dayOfWeek='*') c['schedulers'].append(s) s = timed.Nightly( name=WEEKLY_SCHEDULER_NAME, branch=MASTER_BRANCH, builderNames=builders_by_scheduler[WEEKLY_SCHEDULER_NAME], minute=0, hour=0, dayOfMonth='*', month='*', dayOfWeek=6) # Sunday (Monday = 0). c['schedulers'].append(s) s = AnyBranchScheduler( name=INFRA_PERCOMMIT_SCHEDULER_NAME, treeStableTimer=0, change_filter=infra_change_filter, builderNames=builders_by_scheduler[INFRA_PERCOMMIT_SCHEDULER_NAME]) c['schedulers'].append(s) # Don't add triggerable schedulers for triggered_builders; triggers are now # handled on the slave-side through buildbucket. # Create the BuildFactorys. annotator = annotator_factory.AnnotatorFactory(ActiveMaster) for builder_dict in builder_dicts: factory = annotator.BaseFactory(builder_dict['recipe'], timeout=2400) factory.properties.update(builder_dict['properties'], 'BuildFactory') builder_dict['factory'] = factory # Finished! c['builders'] = builder_dicts
from buildbot.schedulers import basic, timed, triggerable ####### SCHEDULERS # Configure the Schedulers, which decide how to react to incoming changes. schedulers = [] nightly = timed.Nightly(name="Nightly", branch=None, builderNames=["System_Update"], minute=0, hour=1) schedulers.append(nightly) weekly = timed.Nightly( name="Weekly", branch=None, minute=0, hour=2, dayOfWeek=7, # 0 = Monday, 7 = Sunday builderNames=["JaCoCo_ITs_Linux"]) schedulers.append(weekly) schedulers.append( basic.SingleBranchScheduler( name="JaCoCo_Checkin", branch=None, treeStableTimer=1, builderNames=[] # TODO ))
def Config(self): return { 'projectName': "Clementine", 'projectURL': "http://www.clementine-player.org/", 'buildbotURL': "http://buildbot.clementine-player.org/", 'slavePortnum': 9989, 'slaves': self.slaves, 'builders': self.builders, 'change_source': [ builders.GitPoller("Android-Remote"), builders.GitPoller("Clementine"), builders.GitPoller("Dependencies"), builders.GitPoller("Website"), ], 'status': [ html.WebStatus( http_port="tcp:8010", authz=authz.Authz( forceBuild=True, forceAllBuilds=True, stopBuild=True, stopAllBuilds=True, cancelPendingBuild=True, cancelAllPendingBuilds=True, stopChange=True, ), ), mail.MailNotifier( fromaddr="*****@*****.**", lookup="gmail.com", mode="failing", ), ], 'schedulers': [ basic.SingleBranchScheduler( name="automatic", change_filter=filter.ChangeFilter(project="clementine", branch="master"), treeStableTimer=2 * 60, builderNames=self.auto_builder_names, ), basic.SingleBranchScheduler( name="dependencies", change_filter=filter.ChangeFilter(project="dependencies", branch="master"), treeStableTimer=2 * 60, builderNames=[ 'Mac Dependencies', 'Windows Dependencies', ], ), basic.SingleBranchScheduler( name="website", change_filter=filter.ChangeFilter(project="website", branch="master"), treeStableTimer=2 * 60, builderNames=[ "Transifex website POT push", ], ), basic.SingleBranchScheduler( name="android-remote", change_filter=filter.ChangeFilter(project="android-remote", branch="master"), treeStableTimer=2 * 60, builderNames=[ "Android Remote", ], ), forcesched.ForceScheduler( name="force", reason=forcesched.FixedParameter(name="reason", default="force build"), branch=forcesched.StringParameter(name="branch", default="master"), revision=forcesched.FixedParameter(name="revision", default=""), repository=forcesched.FixedParameter(name="repository", default=""), project=forcesched.FixedParameter(name="project", default=""), properties=[], builderNames=[x['name'] for x in self.builders], ), timed.Nightly( name="transifex_pull", change_filter=filter.ChangeFilter(project="clementine", branch="master"), hour=10, minute=0, dayOfWeek=0, branch="master", builderNames=[ "Transifex PO pull", "Transifex website PO pull", "Transifex Android PO pull", ], ), ], }