def setupProperties(self): props = interfaces.IProperties(self) # give the properties a reference back to this build props.build = self # start with global properties from the configuration master = self.builder.botmaster.master props.updateFromProperties(master.config.properties) # from the SourceStamps, which have properties via Change for change in self.allChanges(): props.updateFromProperties(change.properties) # and finally, get any properties from requests (this is the path # through which schedulers will send us properties) for rq in self.requests: props.updateFromProperties(rq.properties) # now set some properties of our own, corresponding to the # build itself props.setProperty("buildnumber", self.build_status.number, "Build") if self.sources and len(self.sources) == 1: # old interface for backwards compatibility source = self.sources[0] props.setProperty("branch", source.branch, "Build") props.setProperty("revision", source.revision, "Build") props.setProperty("repository", source.repository, "Build") props.setProperty("codebase", source.codebase, "Build") props.setProperty("project", source.project, "Build") self.builder.setupProperties(props)
def _flushProperties(self, results): # `results` is just passed on to the next callback props = interfaces.IProperties(self) properties = props.getProperties().asList() for name, value, source in properties: yield self.master.data.updates.setBuildProperty( self.buildid, name, value, source) defer.returnValue(results)
def setupProperties(self): props = interfaces.IProperties(self) # give the properties a reference back to this build props.build = self # start with global properties from the configuration props.updateFromProperties(self.master.config.properties) # from the SourceStamps, which have properties via Change for change in self.allChanges(): props.updateFromProperties(change.properties) # and finally, get any properties from requests (this is the path # through which schedulers will send us properties) for rq in self.requests: props.updateFromProperties(rq.properties) self.builder.setupProperties(props)
def getSummaryStatistic(self, name, summary_fn, initial_value=_sentinel): step_stats_list = [ st.getStatistic(name) for st in self.executedSteps if st.hasStatistic(name)] if initial_value is self._sentinel: return reduce(summary_fn, step_stats_list) return reduce(summary_fn, step_stats_list, initial_value) @defer.inlineCallbacks def getUrl(self): builder_id = yield self.builder.getBuilderId() defer.returnValue(getURLForBuild(self.master, builder_id, self.number)) def waitUntilFinished(self): return self.master.mq.waitUntilEvent( ('builds', str(self.buildid), 'finished'), lambda: self.finished) # IBuildControl def getStatus(self): return self.build_status # stopBuild is defined earlier components.registerAdapter( lambda build: interfaces.IProperties(build.properties), Build, interfaces.IProperties)
def setupStep(self, step, worker_version=None, worker_env=None, buildFiles=None, wantDefaultWorkdir=True, wantData=True, wantDb=False, wantMq=False): """ Set up C{step} for testing. This begins by using C{step} as a factory to create a I{new} step instance, thereby testing that the factory arguments are handled correctly. It then creates a comfortable environment for the worker to run in, replete with a fake build and a fake worker. As a convenience, it can set the step's workdir with C{'wkdir'}. @param worker_version: worker version to present, as a dictionary mapping command name to version. A command name of '*' will apply for all commands. @param worker_env: environment from the worker at worker startup @param wantData(bool): Set to True to add data API connector to master. Default value: True. @param wantDb(bool): Set to True to add database connector to master. Default value: False. @param wantMq(bool): Set to True to add mq connector to master. Default value: False. """ if worker_version is None: worker_version = { '*': '99.99' } if worker_env is None: worker_env = dict() if buildFiles is None: buildFiles = list() factory = interfaces.IBuildStepFactory(step) step = self.step = factory.buildStep() self.master = fakemaster.make_master(wantData=wantData, wantDb=wantDb, wantMq=wantMq, testcase=self) # mock out the reactor for updateSummary's debouncing self.debounceClock = task.Clock() self.master.reactor = self.debounceClock # set defaults if wantDefaultWorkdir: step.workdir = step._workdir or 'wkdir' # step.build b = self.build = fakebuild.FakeBuild(master=self.master) b.allFiles = lambda: buildFiles b.master = self.master def getWorkerVersion(cmd, oldversion): if cmd in worker_version: return worker_version[cmd] if '*' in worker_version: return worker_version['*'] return oldversion b.getWorkerCommandVersion = getWorkerVersion b.workerEnvironment = worker_env.copy() step.setBuild(b) # watch for properties being set self.properties = interfaces.IProperties(b) # step.progress step.progress = mock.Mock(name="progress") # step.worker self.worker = step.worker = worker.FakeWorker(self.master) # step overrides def addLog(name, type='s', logEncoding=None): _log = logfile.FakeLogFile(name, step) self.step.logs[name] = _log return defer.succeed(_log) step.addLog = addLog step.addLog_newStyle = addLog def addHTMLLog(name, html): _log = logfile.FakeLogFile(name, step) html = bytes2NativeString(html) _log.addStdout(html) return defer.succeed(None) step.addHTMLLog = addHTMLLog def addCompleteLog(name, text): _log = logfile.FakeLogFile(name, step) self.step.logs[name] = _log _log.addStdout(text) return defer.succeed(None) step.addCompleteLog = addCompleteLog step.logobservers = self.logobservers = {} def addLogObserver(logname, observer): self.logobservers.setdefault(logname, []).append(observer) observer.step = step step.addLogObserver = addLogObserver # add any observers defined in the constructor, before this # monkey-patch for n, o in step._pendingLogObservers: addLogObserver(n, o) # expectations self.exp_result = None self.exp_state_string = None self.exp_properties = {} self.exp_missing_properties = [] self.exp_logfiles = {} self.exp_hidden = False self.exp_exception = None # check that the step's name is not None self.assertNotEqual(step.name, None) return step
def runCommand(self, c): c.buildslave = self.buildslave d = c.run(self, self.remote) return d @staticmethod def _maybeEvaluate(value, *args, **kwargs): if callable(value): value = value(*args, **kwargs) return value components.registerAdapter(BuildStep._getStepFactory, BuildStep, interfaces.IBuildStepFactory) components.registerAdapter(lambda step: interfaces.IProperties(step.build), BuildStep, interfaces.IProperties) class OutputProgressObserver(LogObserver): length = 0 def __init__(self, name): self.name = name def logChunk(self, build, step, log, channel, text): self.length += len(text) self.step.setProgress(self.name, self.length) class LoggingBuildStep(BuildStep):
def describe(self, done=False): # old-style steps expect this function to exist assert not self.isNewStyle() desc = self._describe(done) if not desc: return [] if self.descriptionSuffix: desc += self.descriptionSuffix return desc components.registerAdapter( BuildStep._getStepFactory, BuildStep, interfaces.IBuildStepFactory) components.registerAdapter( lambda step: interfaces.IProperties(step.build), BuildStep, interfaces.IProperties) class LoggingBuildStep(BuildStep): progressMetrics = ('output',) logfiles = {} parms = BuildStep.parms + ['logfiles', 'lazylogfiles', 'log_eval_func'] cmd = None renderables = ['logfiles', 'lazylogfiles'] def __init__(self, logfiles=None, lazylogfiles=False, log_eval_func=None, *args, **kwargs):
def setupStep(self, step, slave_version={'*': "99.99"}, slave_env={}): """ Set up C{step} for testing. This begins by using C{step} as a factory to create a I{new} step instance, thereby testing that the the factory arguments are handled correctly. It then creates a comfortable environment for the slave to run in, repleate with a fake build and a fake slave. As a convenience, it calls the step's setDefaultWorkdir method with C{'wkdir'}. @param slave_version: slave version to present, as a dictionary mapping command name to version. A command name of '*' will apply for all commands. @param slave_env: environment from the slave at slave startup """ factory = interfaces.IBuildStepFactory(step) step = self.step = factory.buildStep() self.master = fakemaster.make_master(testcase=self) # step.build b = self.build = fakebuild.FakeBuild() b.master = self.master def getSlaveVersion(cmd, oldversion): if cmd in slave_version: return slave_version[cmd] if '*' in slave_version: return slave_version['*'] return oldversion b.getSlaveCommandVersion = getSlaveVersion b.slaveEnvironment = slave_env.copy() step.setBuild(b) # watch for properties being set self.properties = interfaces.IProperties(b) # step.progress step.progress = mock.Mock(name="progress") # step.buildslave self.master = fakemaster.make_master(testcase=self) self.buildslave = step.buildslave = slave.FakeSlave(self.master) # step.step_status ss = self.step_status = mock.Mock(name="step_status") ss.status_text = None ss.logs = {} def ss_setText(strings): ss.status_text = strings ss.setText = ss_setText ss.getLogs = lambda: ss.logs.values() self.step_statistics = {} ss.setStatistic = self.step_statistics.__setitem__ ss.getStatistic = self.step_statistics.get ss.hasStatistic = self.step_statistics.__contains__ self.step.setStepStatus(ss) # step overrides def addLog(name): l = remotecommand.FakeLogFile(name, step) ss.logs[name] = l return l step.addLog = addLog def addHTMLLog(name, html): l = remotecommand.FakeLogFile(name, step) l.addStdout(html) ss.logs[name] = l return l step.addHTMLLog = addHTMLLog def addCompleteLog(name, text): l = remotecommand.FakeLogFile(name, step) l.addStdout(text) ss.logs[name] = l return l step.addCompleteLog = addCompleteLog step.logobservers = self.logobservers = {} def addLogObserver(logname, observer): self.logobservers.setdefault(logname, []).append(observer) observer.step = step step.addLogObserver = addLogObserver # add any observers defined in the constructor, before this monkey-patch for n, o in step._pendingLogObservers: addLogObserver(n, o) # set defaults step.setDefaultWorkdir('wkdir') # expectations self.exp_outcome = None self.exp_properties = {} self.exp_missing_properties = [] self.exp_logfiles = {} self.exp_hidden = False # check that the step's name is not None self.assertNotEqual(step.name, None) return step
def createEmail(self, msgdict, builderName, title, results, builds=None, patches=None, logs=None): text = msgdict['body'].encode(ENCODING) type = msgdict['type'] if 'subject' in msgdict: subject = msgdict['subject'].encode(ENCODING) else: subject = self.subject % { 'result': Results[results], 'projectName': title, 'title': title, 'builder': builderName, } assert '\n' not in subject, \ "Subject cannot contain newlines" assert type in ('plain', 'html'), \ "'%s' message type must be 'plain' or 'html'." % type if patches or logs: m = MIMEMultipart() m.attach(MIMEText(text, type, ENCODING)) else: m = Message() m.set_payload(text, ENCODING) m.set_type("text/%s" % type) m['Date'] = formatdate(localtime=True) m['Subject'] = subject m['From'] = self.fromaddr # m['To'] is added later if patches: for (i, patch) in enumerate(patches): a = self.patch_to_attachment(patch, i) m.attach(a) if logs: for log in logs: name = "%s.%s" % (log.getStep().getName(), log.getName()) if (self._shouldAttachLog(log.getName()) or self._shouldAttachLog(name)): text = log.getText() if not isinstance(text, unicode): text = text.decode(LOG_ENCODING) a = MIMEText(text.encode(ENCODING), _charset=ENCODING) a.add_header('Content-Disposition', "attachment", filename=name) m.attach(a) #@todo: is there a better way to do this? # Add any extra headers that were requested, doing WithProperties # interpolation if only one build was given if self.extraHeaders: for k, v in self.extraHeaders.items(): if len(builds) == 1: k = interfaces.IProperties(builds[0]).render(k) if k in m: twlog.msg("Warning: Got header " + k + " in self.extraHeaders " "but it already exists in the Message - " "not adding it.") if len(builds) == 1: m[k] = interfaces.IProperties(builds[0]).render(v) else: m[k] = v return m
self.build_status.buildFinished() if self.progress and results == SUCCESS: # XXX: also test a 'timing consistent' flag? log.msg(" setting expectations for next time") self.builder.setExpectations(self.progress) eventually(self.releaseLocks) self.deferred.callback(self) self.deferred = None def releaseLocks(self): if self.locks: log.msg("releaseLocks(%s): %s" % (self, self.locks)) for lock, access in self.locks: if lock.isOwner(self, access): lock.release(self, access) else: # This should only happen if we've been interrupted assert self.stopped # IBuildControl def getStatus(self): return self.build_status # stopBuild is defined earlier components.registerAdapter( lambda build: interfaces.IProperties(build.build_status), Build, interfaces.IProperties)
def setupStep(self, step, worker_version=None, worker_env=None, buildFiles=None, wantDefaultWorkdir=True): """ Set up C{step} for testing. This begins by using C{step} as a factory to create a I{new} step instance, thereby testing that the factory arguments are handled correctly. It then creates a comfortable environment for the worker to run in, replete with a fake build and a fake worker. As a convenience, it can set the step's workdir with C{'wkdir'}. @param worker_version: worker version to present, as a dictionary mapping command name to version. A command name of '*' will apply for all commands. @param worker_env: environment from the worker at worker startup """ if worker_version is None: worker_version = {'*': '99.99'} if worker_env is None: worker_env = dict() if buildFiles is None: buildFiles = list() factory = interfaces.IBuildStepFactory(step) step = self.step = factory.buildStep() # set defaults if wantDefaultWorkdir: step.workdir = step._workdir or 'wkdir' # step.build b = self.build = fakebuild.FakeBuild(master=self.master) b.allFiles = lambda: buildFiles b.master = self.master def getWorkerVersion(cmd, oldversion): if cmd in worker_version: return worker_version[cmd] if '*' in worker_version: return worker_version['*'] return oldversion b.getWorkerCommandVersion = getWorkerVersion b.workerEnvironment = worker_env.copy() step.setBuild(b) # watch for properties being set self.properties = interfaces.IProperties(b) # step.progress step.progress = mock.Mock(name="progress") # step.worker self.worker = step.worker = worker.FakeWorker(self.master) self.worker.attached(None) # step overrides def addLog(name, type='s', logEncoding=None): _log = logfile.FakeLogFile(name, step) self.step.logs[name] = _log return defer.succeed(_log) step.addLog = addLog step.addLog_newStyle = addLog def addHTMLLog(name, html): _log = logfile.FakeLogFile(name, step) html = bytes2unicode(html) _log.addStdout(html) return defer.succeed(None) step.addHTMLLog = addHTMLLog def addCompleteLog(name, text): _log = logfile.FakeLogFile(name, step) self.step.logs[name] = _log _log.addStdout(text) return defer.succeed(None) step.addCompleteLog = addCompleteLog step.logobservers = self.logobservers = {} def addLogObserver(logname, observer): self.logobservers.setdefault(logname, []).append(observer) observer.step = step step.addLogObserver = addLogObserver # add any observers defined in the constructor, before this # monkey-patch for n, o in step._pendingLogObservers: addLogObserver(n, o) self._got_test_result_sets = [] self._next_test_result_set_id = 1000 def add_test_result_set(description, category, value_unit): self._got_test_result_sets.append( (description, category, value_unit)) setid = self._next_test_result_set_id self._next_test_result_set_id += 1 return defer.succeed(setid) step.addTestResultSet = add_test_result_set self._got_test_results = [] def add_test_result(setid, value, test_name=None, test_code_path=None, line=None, duration_ns=None): self._got_test_results.append( (setid, value, test_name, test_code_path, line, duration_ns)) step.addTestResult = add_test_result # expectations self.exp_result = None self.exp_state_string = None self.exp_properties = {} self.exp_missing_properties = [] self.exp_logfiles = {} self.exp_hidden = False self.exp_exception = None self._exp_test_result_sets = [] self._exp_test_results = [] # check that the step's name is not None self.assertNotEqual(step.name, None) return step
def setupStep(self, step, slave_version={'*': "99.99"}, slave_env={}, buildFiles=[]): """ Set up C{step} for testing. This begins by using C{step} as a factory to create a I{new} step instance, thereby testing that the the factory arguments are handled correctly. It then creates a comfortable environment for the slave to run in, replete with a fake build and a fake slave. As a convenience, it calls the step's setDefaultWorkdir method with C{'wkdir'}. @param slave_version: slave version to present, as a dictionary mapping command name to version. A command name of '*' will apply for all commands. @param slave_env: environment from the slave at slave startup """ factory = interfaces.IBuildStepFactory(step) step = self.step = factory.buildStep() self.master = fakemaster.make_master(wantData=True, testcase=self) # step.build b = self.build = fakebuild.FakeBuild(master=self.master) b.allFiles = lambda: buildFiles b.master = self.master def getSlaveVersion(cmd, oldversion): if cmd in slave_version: return slave_version[cmd] if '*' in slave_version: return slave_version['*'] return oldversion b.getSlaveCommandVersion = getSlaveVersion b.slaveEnvironment = slave_env.copy() step.setBuild(b) # watch for properties being set self.properties = interfaces.IProperties(b) # step.progress step.progress = mock.Mock(name="progress") # step.buildslave self.buildslave = step.buildslave = slave.FakeSlave(self.master) # step overrides def addLog(name, type='s', logEncoding=None): l = logfile.FakeLogFile(name, step) self.step.logs[name] = l return defer.succeed(l) step.addLog = addLog step.addLog_newStyle = addLog def addHTMLLog(name, html): l = logfile.FakeLogFile(name, step) l.addStdout(html) return defer.succeed(None) step.addHTMLLog = addHTMLLog def addCompleteLog(name, text): l = logfile.FakeLogFile(name, step) self.step.logs[name] = l l.addStdout(text) return defer.succeed(None) step.addCompleteLog = addCompleteLog step.logobservers = self.logobservers = {} def addLogObserver(logname, observer): self.logobservers.setdefault(logname, []).append(observer) observer.step = step step.addLogObserver = addLogObserver # add any observers defined in the constructor, before this # monkey-patch for n, o in step._pendingLogObservers: addLogObserver(n, o) # set defaults step.setDefaultWorkdir('wkdir') # expectations self.exp_result = None self.exp_state_strings = None self.exp_properties = {} self.exp_missing_properties = [] self.exp_logfiles = {} self.exp_hidden = False # check that the step's name is not None self.assertNotEqual(step.name, None) # mock out the reactor for updateSummary's debouncing self.debounceClock = task.Clock() step.updateSummary._reactor = self.debounceClock return step
def setupStep(self, step, slave_version="99.99", slave_env={}): """ Set up C{step} for testing. This begins by using C{step} as a factory to create a I{new} step instance, thereby testing that the the factory arguments are handled correctly. It then creates a comfortable environment for the slave to run in, repleate with a fake build and a fake slave. As a convenience, it calls the step's setDefaultWorkdir method with C{'wkdir'}. @param slave_version: slave version to present; defaults to "99.99" @param slave_env: environment from the slave at slave startup """ # yes, Virginia, "factory" refers both to the tuple and its first # element TODO: fix that up factory, args = step.getStepFactory() step = self.step = factory(**args) # step.build b = self.build = fakebuild.FakeBuild() b.getSlaveCommandVersion = lambda command, oldversion: slave_version b.slaveEnvironment = slave_env.copy() step.setBuild(b) # watch for properties being set self.properties = interfaces.IProperties(b) # step.progress step.progress = mock.Mock(name="progress") # step.buildslave step.buildslave = mock.Mock(name="buildslave") # step.step_status ss = self.step_status = mock.Mock(name="step_status") ss.status_text = None ss.logs = {} def ss_setText(strings): ss.status_text = strings ss.setText = ss_setText ss.getLogs = lambda: ss.logs.values() self.step_statistics = {} ss.setStatistic = self.step_statistics.__setitem__ ss.getStatistic = self.step_statistics.get ss.hasStatistic = self.step_statistics.__contains__ self.step.setStepStatus(ss) # step overrides def addLog(name): l = remotecommand.FakeLogFile(name) ss.logs[name] = l return l step.addLog = addLog def addHTMLLog(name, html): l = remotecommand.FakeLogFile(name) l.addStdout(html) ss.logs[name] = l return l step.addHTMLLog = addHTMLLog def addCompleteLog(name, text): l = remotecommand.FakeLogFile(name) l.addStdout(text) ss.logs[name] = l return l step.addCompleteLog = addCompleteLog # set defaults step.setDefaultWorkdir('wkdir') # expectations self.exp_outcome = None self.exp_properties = {} self.exp_missing_properties = [] self.exp_logfiles = {} return step