def testSuccess(self): S = buildset.BuildSet a,b = FakeBuilder(), FakeBuilder() # this time, both builds succeed source = sourcestamp.SourceStamp() s = S(["a","b"], source, "forced build") s.start([a,b]) st = s.status self.failUnlessEqual(st.getSourceStamp(), source) self.failUnlessEqual(st.getReason(), "forced build") self.failUnlessEqual(st.getBuilderNames(), ["a","b"]) self.failIf(st.isFinished()) builderstatus_a = builder.BuilderStatus("a") bsa = builder.BuildStatus(builderstatus_a, 1) bsa.setResults(builder.SUCCESS) a.requests[0].finished(bsa) builderstatus_b = builder.BuilderStatus("b") bsb = builder.BuildStatus(builderstatus_b, 1) bsb.setResults(builder.SUCCESS) b.requests[0].finished(bsb) self.failUnless(st.isFinished()) self.failUnlessEqual(st.getResults(), builder.SUCCESS)
def testStepStatistics(self): status = builder.BuildStatus(builder.BuilderStatus("test"), 123) status.addStepWithName('step1') status.addStepWithName('step2') status.addStepWithName('step3') status.addStepWithName('step4') steps = status.getSteps() (step1, step2, step3, step4) = steps step1.setStatistic('test-prop', 1) step3.setStatistic('test-prop', 2) step4.setStatistic('test-prop', 4) step1.setStatistic('other-prop', 27) # Just to have some other properties around self.failUnlessEqual(step1.getStatistic('test-prop'), 1, 'Retrieve an existing property') self.failUnlessEqual(step1.getStatistic('test-prop', 99), 1, "Don't default an existing property") self.failUnlessEqual(step2.getStatistic('test-prop', 99), 99, 'Default a non-existant property') self.failUnlessEqual( status.getSummaryStatistic('test-prop', operator.add), 7, 'Sum property across the build') self.failUnlessEqual( status.getSummaryStatistic('test-prop', operator.add, 13), 20, 'Sum property across the build with initial value')
def testAB(self): self.setupSimple() c = Change('author', ['some/file.dtd'], 'comment', branch='dir', properties={'locale': 'ab'}) c.number = 1 self.scheduler.addChange(c) self.failUnlessEqual(len(self.master.sets), 1) bset = self.master.sets[0] props = bset.getProperties() self.assertEqual(props['locale'], 'ab') self.assertEqual(props['branch'], 'dir') self.assertEqual(props['tree'], 'dir-compare') self.failUnlessEqual(bset.builderNames, ['dir-compare']) ftb = FakeBuilder('dir-compare') bset.start([ftb]) self.failUnlessEqual(len(ftb.requests), 1) st = bset.status self.failIf(st.isFinished()) builder = builderstatus.BuilderStatus('dir-compare') build = builderstatus.BuildStatus(builder, 1) build.setResults(builderstatus.SUCCESS) ftb.requests[0].finished(build) self.failUnless(st.isFinished())
def test_d_ini(self): self.setupSimple() c = Change('author', ['test-app/locales/l10n.ini'], 'comment', branch='test-branch') c.number = 1 self.scheduler.addChange(c) c = Change('author', ['test-app/locales/en-US/app.dtd'], 'comment', branch='test-branch') c.number = 2 self.scheduler.addChange(c) self.failUnlessEqual(len(self.master.sets), 1) bset = self.master.sets[0] self.failUnlessEqual(bset.builderNames, ['tree-builds']) ftb = FakeBuilder('tree-builds') bset.start([ftb]) self.failUnlessEqual(len(ftb.requests), 1) st = bset.status self.failIf(st.isFinished()) builder = builderstatus.BuilderStatus('tree-builds') build = builderstatus.BuildStatus(builder, 1) build.setResults(builderstatus.SUCCESS) ftb.requests[0].finished(build) self.failUnless(st.isFinished()) self.failUnless(self.scheduler.dSubmitBuildsets) self.scheduler.dSubmitBuildsets.cancel() pendings = self.scheduler.pendings self.failUnlessEqual(len(pendings), 2) self.failUnlessEqual(len(pendings[('test', 'de')]), 1) self.failUnlessEqual(len(pendings[('test', 'fr')]), 1)
def builderAdded(self, name, basedir, category=None, friendly_name=None, description=None, project=None): """ @rtype: L{BuilderStatus} """ filename = os.path.join(self.basedir, basedir, "builder") log.msg("trying to load status pickle from %s" % filename) builder_status = None if friendly_name is None: friendly_name = name try: with open(filename, "rb") as f: builder_status = load(f) builder_status.master = self.master builder_status.basedir = os.path.join(self.basedir, basedir) # (bug #1068) if we need to upgrade, we probably need to rewrite # this pickle, too. We determine this by looking at the list of # Versioned objects that have been unpickled, and (after doUpgrade) # checking to see if any of them set wasUpgraded. The Versioneds' # upgradeToVersionNN methods all set this. versioneds = styles.versionedsToUpgrade styles.doUpgrade() if True in [ hasattr(o, 'wasUpgraded') for o in versioneds.values() ]: log.msg("re-writing upgraded builder pickle") builder_status.saveYourself() except IOError: log.msg("no saved status pickle, creating a new one") except: log.msg("error while loading status pickle, creating a new one") log.msg("error follows:") klog.err_json() if not builder_status: builder_status = builder.BuilderStatus(name, category, self.master, friendly_name, description, project=project) builder_status.addPointEvent(["builder", "created"]) log.msg("added builder %s in category %s" % (name, category)) # an unpickled object might not have category set from before, # so set it here to make sure builder_status.category = category builder_status.description = description builder_status.master = self.master builder_status.basedir = os.path.join(self.basedir, basedir) builder_status.name = name # it might have been updated builder_status.setStatus(self) builder_status.friendly_name = friendly_name if not os.path.isdir(builder_status.basedir): os.makedirs(builder_status.basedir) builder_status.determineNextBuildNumber() builder_status.setBigState("offline") for t in self.watchers: self.announceNewBuilder(t, name, builder_status) return builder_status
def setupBuilder(self, buildername, category=None): b = builder.BuilderStatus(buildername=buildername, category=category) # Ackwardly, Status sets this member variable. b.basedir = os.path.abspath(self.mktemp()) os.mkdir(b.basedir) # Otherwise, builder.nextBuildNumber is not defined. b.determineNextBuildNumber() return b
def makeBuilderStatus(self): """ Return a new BuilderStatus. """ return builder.BuilderStatus(buildername='testing-builder', tags=None, master=fakemaster.make_master(), description=None)
def setupBuilder(self, buildername, tags=None, description=None): self.master = fakemaster.make_master() self.master.basedir = '/basedir' b = builder.BuilderStatus(buildername, tags, self.master, description) # Ackwardly, Status sets this member variable. b.basedir = os.path.abspath(self.mktemp()) os.mkdir(b.basedir) # Otherwise, builder.nextBuildNumber is not defined. b.determineNextBuildNumber() return b
def setupBuilder(self, buildername, category=None): b = builder.BuilderStatus(buildername=buildername, category=category) # Awkwardly, Status sets this member variable. b.basedir = os.path.abspath(self.mktemp()) os.mkdir(b.basedir) # Otherwise, builder.nextBuildNumber is not defined. b.determineNextBuildNumber() # Must initialize these fields before pickling. b.currentBigState = 'idle' b.status = 'idle' return b
def setUp(self): self.master = fakemaster.make_master(wantDb=True, testcase="TestBuilderStatus") katana = { 'katana-buildbot': { 'project': 'general', 'display_name': 'Katana buildbot', 'defaultbranch': 'katana', 'repository': 'https://github.com/Unity-Technologies/buildbot.git', 'display_repository': 'https://github.com/Unity-Technologies/buildbot.git', 'branch': ['katana', 'master', 'staging'] } } self.project = ProjectConfig(name="Katana", codebases=[katana]) self.master.getProject = lambda x: self.project self.getProjects = lambda: {'Katana': self.project} self.master.db.builds.getLastBuildsNumbers = lambda buildername, sourcestamps, results, num_builds: \ defer.succeed([38]) self.builder_status = builder.BuilderStatus(buildername="builder-01", category=None, master=self.master) self.builder_status.nextBuildNumber = 39 self.builder_status.master = self.master self.builder_status.buildCache = Mock() self.builder_status.buildCache.cache = {} def getCachedBuild(number): build_status = BuildStatus(self.builder_status, self.master, number) build_status.finished = 1422441501.21 build_status.reason = 'A build was forced by user@localhost' build_status.slavename = 'build-slave-01' build_status.results = SUCCESS build_status.sources = [ SourceStamp( branch='katana', codebase='katana-buildbot', revision='804d540eac7b90022130d34616a8f8336fe5691a') ] return build_status self.builder_status.buildCache.get = getCachedBuild self.builder_status.saveYourself = lambda skipBuilds: True
def setupBuilder(self, buildername, description=None): m = fakemaster.make_master() b = builder.BuilderStatus(buildername=buildername, tags=None, master=m, description=description) # Awkwardly, Status sets this member variable. b.basedir = os.path.abspath(self.mktemp()) os.mkdir(b.basedir) # Otherwise, builder.nextBuildNumber is not defined. b.determineNextBuildNumber() # Must initialize these fields before pickling. b.currentBigState = 'idle' b.status = 'idle' return b
def setUp(self): self.builder = FakeBuilder() self.builder_status = builder.BuilderStatus("fakebuilder") self.builder_status.basedir = "test_properties" self.builder_status.nextBuildNumber = 5 rmdirRecursive(self.builder_status.basedir) os.mkdir(self.builder_status.basedir) self.build_status = self.builder_status.newBuild() req = base.BuildRequest("reason", SourceStamp(branch="branch2", revision=1234)) self.build = base.Build([req]) self.build.setBuilder(self.builder) self.build.setupStatus(self.build_status) self.build.setupSlaveBuilder(FakeSlaveBuilder())
def builderAdded(self, name, basedir, tags=None, description=None): """ @rtype: L{BuilderStatus} """ filename = os.path.join(self.basedir, basedir, "builder") log.msg("trying to load status pickle from %s" % filename) builder_status = None try: with open(filename, "rb") as f: builder_status = pickle.load(f) builder_status.master = self.master # (bug #1068) if we need to upgrade, we probably need to rewrite # this pickle, too. We determine this by looking at the list of # Versioned objects that have been unpickled, and (after doUpgrade) # checking to see if any of them set wasUpgraded. The Versioneds' # upgradeToVersionNN methods all set this. versioneds = styles.versionedsToUpgrade styles.doUpgrade() if True in [ hasattr(o, 'wasUpgraded') for o in itervalues(versioneds) ]: log.msg("re-writing upgraded builder pickle") builder_status.saveYourself() except IOError: log.msg("no saved status pickle, creating a new one") except Exception: log.err("error while loading status pickle, creating a new one") if not builder_status: builder_status = builder.BuilderStatus(name, tags, self.master, description) builder_status.addPointEvent(["builder", "created"]) log.msg("added builder %s with tags %r" % (name, tags)) # an unpickled object might not have tags set from before, # so set it here to make sure builder_status.setTags(tags) builder_status.description = description builder_status.master = self.master builder_status.basedir = os.path.join(self.basedir, basedir) builder_status.name = name # it might have been updated builder_status.status = self builder_status.setBigState("offline") for t in self.watchers: self.announceNewBuilder(t, name, builder_status) return builder_status
def setUp(self): rmtree("test_steps") self.builder = FakeBuilder() self.builder_status = builder.BuilderStatus("fakebuilder") self.builder_status.basedir = "test_steps" self.builder_status.nextBuildNumber = 0 os.mkdir(self.builder_status.basedir) self.build_status = self.builder_status.newBuild() req = base.BuildRequest("reason", SourceStamp(), 'test_builder') self.build = base.Build([req]) self.build.build_status = self.build_status # fake it self.build.builder = self.builder self.build.slavebuilder = FakeSlaveBuilder() self.remote = FakeRemote() self.finished = 0
def builderAdded(self, name, basedir, tags=None, description=None): """ @rtype: L{BuilderStatus} """ builder_status = builder.BuilderStatus(name, tags, self.master, description) builder_status.setTags(tags) builder_status.description = description builder_status.master = self.master builder_status.basedir = os.path.join(self.basedir, basedir) builder_status.name = name # it might have been updated builder_status.status = self builder_status.setBigState("offline") return builder_status
def setUp(self): self.builder = FakeBuilder() self.builder_status = builder.BuilderStatus("fakebuilder") self.basedir = self.mktemp() os.mkdir(self.basedir) self.builder_status.basedir = self.basedir self.builder_status.nextBuildNumber = 5 self.build_status = self.builder_status.newBuild() req = BuildRequest("reason", SourceStamp(branch="branch2", revision="1234"), 'test_builder', properties=Properties(scheduler="fakescheduler")) self.build = base.Build([req]) self.build.build_status = self.build_status self.build.setBuilder(self.builder) self.build.setupProperties() self.build.setupSlaveBuilder(FakeSlaveBuilder())
def testAddResults(self): b = builder.BuildStatus(builder.BuilderStatus("test"), 12) testname = ("buildbot", "test", "test_status", "Results", "testAddResults") r1 = builder.TestResult( name=testname, results=builder.SUCCESS, text=["passed"], logs={'output': ""}, ) b.addTestResult(r1) res = b.getTestResults() self.failUnlessEqual(res.keys(), [testname]) t = res[testname] self.failUnless(interfaces.ITestResult.providedBy(t)) self.failUnlessEqual(t.getName(), testname) self.failUnlessEqual(t.getResults(), builder.SUCCESS) self.failUnlessEqual(t.getText(), ["passed"]) self.failUnlessEqual(t.getLogs(), {'output': ""})
def main(args): if args.list_masters: masterpairs = get_masters() pp_masters(masterpairs) return 0 if args.master_dir: config = read_config(args.master_dir, args.master_cfg) else: path = choose_master(args.mastername) if not path: return 2 config = read_config(path, args.master_cfg) if not config: return 2 mastername = config['BuildmasterConfig']['properties']['mastername'] builders = dup_slaves(config['BuildmasterConfig']['builders']) if args.list_builders: pp_builders(builders, mastername) return 0 my_builder = choose(builders, args.spec) if args.spec and 'hostname' in args.spec: slavename = args.spec['hostname'] elif (args.spec and 'either' in args.spec) and (args.spec['either'] != my_builder['name']): slavename = args.spec['either'] else: slavename = my_builder['slavename'] if not my_builder: return 2 my_factory = my_builder['factory'] steplist = generate_steplist(my_factory) if args.list_steps: print print 'listing steps in %s/%s:' % (mastername, my_builder['name']) print for step in steplist: if hasattr(args, 'step_regex') and not args.step_regex.search(step.name): print '-', step.name, '[skipped]' elif hasattr(args, 'stepreject_regex') and (args.stepreject_regex.search( step.name)): print '-', step.name, '[skipped]' else: print '*', step.name return 0 if not args.annotate: print >> sys.stderr, 'using %s builder \'%s\'' % (mastername, my_builder['name']) if args.build_properties: buildsetup = args.build_properties else: buildsetup = {} buildsetup['revision'] = '%d' % args.revision buildsetup['branch'] = 'src' build = base.Build([FakeRequest(buildsetup)]) safename = buildbot.util.safeTranslate(my_builder['name']) if hasattr(args, 'builderpath'): basepath = args.builderpath else: basepath = safename basedir = os.path.join('..', '..', '..', 'slave', basepath) build.basedir = basedir builderstatus = builder.BuilderStatus('test') builderstatus.nextBuildNumber = 2 builderstatus.basedir = basedir my_builder['builddir'] = safename my_builder['slavebuilddir'] = safename mybuilder = real_builder.Builder(my_builder, builderstatus) build.setBuilder(mybuilder) build_status = build_module.BuildStatus(builderstatus, 1) build_status.setProperty('blamelist', [], 'Build') build_status.setProperty('mastername', mastername, 'Build') build_status.setProperty('slavename', slavename, 'Build') build_status.setProperty('gtest_filter', [], 'Build') # if build_properties are set on the CLI, overwrite the defaults # set above when build.setupProperties is called buildprops = Properties() if args.build_properties: buildprops.update(args.build_properties, 'Botmaster') mybuilder.setBotmaster(FakeBotmaster(mastername, buildprops)) mylogger = LogClass(args.log) buildslave = FakeSlave(safename, slavename) buildslave.addUpdateAction(mylogger.log_to_file) build.build_status = build_status build.setupSlaveBuilder(buildslave) build.setupProperties() if args.output_build_properties: print print 'build properties:' print propertiesToJSON(build.getProperties()) if args.output_factory_properties: print print 'factory properties:' print propertiesToJSON(my_factory.properties) if args.output_build_properties or args.output_factory_properties: return 0 process_steps(steplist, build, buildslave, build_status, basedir) commands = get_commands(steplist) run_status = ReturnStatus() start_time = time.clock() commands_executed = 0 for command in commands: if hasattr(args, 'step_regex'): if not args.step_regex.search(command['name']): if not args.annotate: print >> sys.stderr, 'skipping step: ' + command['name'] continue if hasattr(args, 'stepreject_regex'): if args.stepreject_regex.search(command['name']): if not args.annotate: print >> sys.stderr, 'skipping step: ' + command['name'] continue if not args.annotate: print >> sys.stderr, 'running step: %s' % command['name'] else: print '@@@BUILD_STEP %s@@@' % command['name'] print >> args.log, '(in %s): %s' % (command['workdir'], shell_quote(command['command'])) mydir = os.getcwd() myenv = os.environ os.chdir(command['workdir']) # python docs says this might cause leaks on FreeBSD/OSX for envar in command['env']: os.environ[envar] = command['env'][envar] ret = chromium_utils.RunCommand(command['command'], filter_obj=mylogger, print_cmd=False) os.chdir(mydir) os.environ = myenv commands_executed += 1 if ret != 0: return 2 end_time = time.clock() if not args.annotate: print >> sys.stderr, '%d commands completed (%0.2fs).' % ( commands_executed, end_time - start_time) else: if commands_executed < 1: print '0 commands executed.' return run_status.code
def testBuildSet(self): S = buildset.BuildSet a,b = FakeBuilder(), FakeBuilder() # two builds, the first one fails, the second one succeeds. The # waitUntilSuccess watcher fires as soon as the first one fails, # while the waitUntilFinished watcher doesn't fire until all builds # are complete. source = sourcestamp.SourceStamp() s = S(["a","b"], source, "forced build") s.start([a,b]) self.failUnlessEqual(len(a.requests), 1) self.failUnlessEqual(len(b.requests), 1) r1 = a.requests[0] self.failUnlessEqual(r1.reason, s.reason) self.failUnlessEqual(r1.source, s.source) st = s.status self.failUnlessEqual(st.getSourceStamp(), source) self.failUnlessEqual(st.getReason(), "forced build") self.failUnlessEqual(st.getBuilderNames(), ["a","b"]) self.failIf(st.isFinished()) brs = st.getBuildRequests() self.failUnlessEqual(len(brs), 2) res = [] d1 = s.waitUntilSuccess() d1.addCallback(lambda r: res.append(("success", r))) d2 = s.waitUntilFinished() d2.addCallback(lambda r: res.append(("finished", r))) self.failUnlessEqual(res, []) # the first build finishes here, with FAILURE builderstatus_a = builder.BuilderStatus("a") bsa = builder.BuildStatus(builderstatus_a, 1) bsa.setResults(builder.FAILURE) a.requests[0].finished(bsa) # any FAILURE flunks the BuildSet immediately, so the # waitUntilSuccess deferred fires right away. However, the # waitUntilFinished deferred must wait until all builds have # completed. self.failUnlessEqual(len(res), 1) self.failUnlessEqual(res[0][0], "success") bss = res[0][1] self.failUnless(interfaces.IBuildSetStatus(bss, None)) self.failUnlessEqual(bss.getResults(), builder.FAILURE) # here we finish the second build builderstatus_b = builder.BuilderStatus("b") bsb = builder.BuildStatus(builderstatus_b, 1) bsb.setResults(builder.SUCCESS) b.requests[0].finished(bsb) # .. which ought to fire the waitUntilFinished deferred self.failUnlessEqual(len(res), 2) self.failUnlessEqual(res[1][0], "finished") self.failUnlessEqual(res[1][1], bss) # and finish the BuildSet overall self.failUnless(st.isFinished()) self.failUnlessEqual(st.getResults(), builder.FAILURE)
def testAdaptation(self): b = builder.BuilderStatus("bname") b2 = client.makeRemote(b) self.failUnless(isinstance(b2, client.RemoteBuilder)) b3 = client.makeRemote(None) self.failUnless(b3 is None)
def MockBuild(my_builder, buildsetup, mastername, slavename, basepath=None, build_properties=None, slavedir=None): """Given a builder object and configuration, mock a Buildbot setup around it. This sets up a mock BuildMaster, BuildSlave, Build, BuildStatus, and all other superstructure required for BuildSteps inside the provided builder to render properly. These BuildSteps are returned to the user in an array. It additionally returns the build object (in order to get its properties if desired). buildsetup is passed straight into the FakeSource's init method and contains sourcestamp information (revision, branch, etc). basepath is the directory of the build (what goes under build/slave/, for example 'Chromium_Linux_Builder'. It is nominally inferred from the builder name, but it can be overridden. This is useful when pointing the buildrunner at a different builder than what it's running under. build_properties will update and override build_properties after all builder-derived defaults have been set. """ my_factory = my_builder['factory'] steplist = ListSteps(my_factory) build = base.Build([FakeRequest(buildsetup)]) safename = buildbot.util.safeTranslate(my_builder['name']) my_builder['builddir'] = safename my_builder.setdefault('slavebuilddir', safename) workdir_root = None if not slavedir: workdir_root = os.path.join(SCRIPT_DIR, '..', '..', 'slave', my_builder['slavebuilddir']) if not basepath: basepath = safename if not slavedir: slavedir = os.path.join(SCRIPT_DIR, '..', '..', 'slave') basedir = os.path.join(slavedir, basepath) build.basedir = basedir if not workdir_root: workdir_root = basedir builderstatus = builder.BuilderStatus('test') builderstatus.basedir = basedir buildnumber = build_properties.get('buildnumber', 1) builderstatus.nextBuildNumber = buildnumber + 1 mybuilder = real_builder.Builder(my_builder, builderstatus) build.setBuilder(mybuilder) build_status = build_module.BuildStatus(builderstatus, buildnumber) build_status.setProperty('blamelist', [], 'Build') build_status.setProperty('mastername', mastername, 'Build') build_status.setProperty('slavename', slavename, 'Build') build_status.setProperty('gtest_filter', [], 'Build') build_status.setProperty('extra_args', [], 'Build') build_status.setProperty('build_id', buildnumber, 'Build') # if build_properties are passed in, overwrite the defaults above: buildprops = Properties() if build_properties: buildprops.update(build_properties, 'Botmaster') mybuilder.setBotmaster(FakeBotmaster(mastername, buildprops)) buildslave = FakeSlave(safename, my_builder.get('slavebuilddir'), slavename) build.build_status = build_status build.setupSlaveBuilder(buildslave) build.setupProperties() process_steps(steplist, build, buildslave, build_status, workdir_root) return steplist, build