def asVersion(self): """ Convert the version data in this item to a L{twisted.python.versions.Version}. """ return versions.Version(self.package, self.major, self.minor, self.micro)
from buildbot import util from buildbot.process.build import Build from buildbot.process.buildstep import BuildStep from buildbot.steps.download_secret_to_worker import DownloadSecretsToWorker from buildbot.steps.download_secret_to_worker import RemoveWorkerFileSecret from buildbot.steps.shell import Compile from buildbot.steps.shell import Configure from buildbot.steps.shell import PerlModuleTest from buildbot.steps.shell import ShellCommand from buildbot.steps.shell import Test from buildbot.steps.source.cvs import CVS from buildbot.steps.source.svn import SVN # deprecated, use BuildFactory.addStep @deprecate.deprecated(versions.Version("buildbot", 0, 8, 6)) def s(steptype, **kwargs): # convenience function for master.cfg files, to create step # specification tuples return interfaces.IBuildStepFactory(steptype(**kwargs)) class BuildFactory(util.ComparableMixin): """ @cvar buildClass: class to use when creating builds @type buildClass: L{buildbot.process.build.Build} """ buildClass = Build useProgress = 1 workdir = "build"
# Copyright (c) Twisted Matrix Laboratories. # See LICENSE for details. # This is an auto-generated file. Do not edit it. """ Provides Twisted version information. """ from twisted.python import versions version = versions.Version('twisted.news', 15, 1, 0)
def patch_gatherResults(): if twisted.version < versions.Version('twisted', 11, 1, 0): from buildbot.monkeypatches import gatherResults gatherResults.patch()
# Copyright (c) Twisted Matrix Laboratories. # See LICENSE for details. # This is an auto-generated file. Do not edit it. """ Provides Twisted version information. """ from twisted.python import versions version = versions.Version('twisted.words', 15, 2, 0)
class BuildStep(results.ResultComputingConfigMixin, properties.PropertiesMixin, util.ComparableMixin): # Note that the BuildStep is at the same time a template from which per-build steps are # constructed. This works by creating a new IBuildStepFactory in __new__, retrieving it via # get_step_factory() and then calling buildStep() on that factory. alwaysRun = False doStepIf = True hideStepIf = False compare_attrs = ("_factory", ) # properties set on a build step are, by nature, always runtime properties set_runtime_properties = True renderables = results.ResultComputingConfigMixin.resultConfig + [ 'alwaysRun', 'description', 'descriptionDone', 'descriptionSuffix', 'doStepIf', 'hideStepIf', 'workdir', ] # 'parms' holds a list of all the parameters we care about, to allow # users to instantiate a subclass of BuildStep with a mixture of # arguments, some of which are for us, some of which are for the subclass # (or a delegate of the subclass, like how ShellCommand delivers many # arguments to the RemoteShellCommand that it creates). Such delegating # subclasses will use this list to figure out which arguments are meant # for us and which should be given to someone else. parms = [ 'alwaysRun', 'description', 'descriptionDone', 'descriptionSuffix', 'doStepIf', 'flunkOnFailure', 'flunkOnWarnings', 'haltOnFailure', 'updateBuildSummaryPolicy', 'hideStepIf', 'locks', 'logEncoding', 'name', 'progressMetrics', 'useProgress', 'warnOnFailure', 'warnOnWarnings', 'workdir', ] name = "generic" description = None # set this to a list of short strings to override descriptionDone = None # alternate description when the step is complete descriptionSuffix = None # extra information to append to suffix updateBuildSummaryPolicy = None locks = [] progressMetrics = () # 'time' is implicit useProgress = True # set to False if step is really unpredictable build = None step_status = None progress = None logEncoding = None cmd = None rendered = False # true if attributes are rendered _workdir = None _waitingForLocks = False def _run_finished_hook(self): return None # override in tests def __init__(self, **kwargs): self.worker = None for p in self.__class__.parms: if p in kwargs: setattr(self, p, kwargs.pop(p)) if kwargs: config.error( "{}.__init__ got unexpected keyword argument(s) {}".format( self.__class__, list(kwargs))) self._pendingLogObservers = [] if not isinstance(self.name, str) and not IRenderable.providedBy( self.name): config.error( "BuildStep name must be a string or a renderable object: " "%r" % (self.name, )) if isinstance(self.description, str): self.description = [self.description] if isinstance(self.descriptionDone, str): self.descriptionDone = [self.descriptionDone] if isinstance(self.descriptionSuffix, str): self.descriptionSuffix = [self.descriptionSuffix] if self.updateBuildSummaryPolicy is None: # compute default value for updateBuildSummaryPolicy self.updateBuildSummaryPolicy = [EXCEPTION, RETRY, CANCELLED] if self.flunkOnFailure or self.haltOnFailure or self.warnOnFailure: self.updateBuildSummaryPolicy.append(FAILURE) if self.warnOnWarnings or self.flunkOnWarnings: self.updateBuildSummaryPolicy.append(WARNINGS) if self.updateBuildSummaryPolicy is False: self.updateBuildSummaryPolicy = [] if self.updateBuildSummaryPolicy is True: self.updateBuildSummaryPolicy = ALL_RESULTS if not isinstance(self.updateBuildSummaryPolicy, list): config.error("BuildStep updateBuildSummaryPolicy must be " "a list of result ids or boolean but it is %r" % (self.updateBuildSummaryPolicy, )) self._acquiringLocks = [] self.stopped = False self.master = None self.statistics = {} self.logs = {} self._running = False self.stepid = None self.results = None self._start_unhandled_deferreds = None self._test_result_submitters = {} def __new__(klass, *args, **kwargs): self = object.__new__(klass) self._factory = _BuildStepFactory(klass, *args, **kwargs) return self def __str__(self): args = [repr(x) for x in self._factory.args] args.extend( [str(k) + "=" + repr(v) for k, v in self._factory.kwargs.items()]) return "{}({})".format(self.__class__.__name__, ", ".join(args)) __repr__ = __str__ def setBuild(self, build): self.build = build self.master = self.build.master def setWorker(self, worker): self.worker = worker @deprecate.deprecated(versions.Version("buildbot", 0, 9, 0)) def setDefaultWorkdir(self, workdir): if self._workdir is None: self._workdir = workdir @property def workdir(self): # default the workdir appropriately if self._workdir is not None or self.build is None: return self._workdir else: # see :ref:`Factory-Workdir-Functions` for details on how to # customize this if callable(self.build.workdir): try: return self.build.workdir(self.build.sources) except AttributeError as e: # if the callable raises an AttributeError # python thinks it is actually workdir that is not existing. # python will then swallow the attribute error and call # __getattr__ from worker_transition _, _, traceback = sys.exc_info() raise CallableAttributeError(e).with_traceback(traceback) # we re-raise the original exception by changing its type, # but keeping its stacktrace else: return self.build.workdir @workdir.setter def workdir(self, workdir): self._workdir = workdir def getProperties(self): return self.build.getProperties() def get_step_factory(self): return self._factory def setupProgress(self): # this function temporarily does nothing pass def setProgress(self, metric, value): # this function temporarily does nothing pass def getCurrentSummary(self): if self.description is not None: stepsumm = util.join_list(self.description) if self.descriptionSuffix: stepsumm += ' ' + util.join_list(self.descriptionSuffix) else: stepsumm = 'running' return {'step': stepsumm} def getResultSummary(self): if self.descriptionDone is not None or self.description is not None: stepsumm = util.join_list(self.descriptionDone or self.description) if self.descriptionSuffix: stepsumm += ' ' + util.join_list(self.descriptionSuffix) else: stepsumm = 'finished' if self.results != SUCCESS: stepsumm += ' ({})'.format(Results[self.results]) return {'step': stepsumm} @defer.inlineCallbacks def getBuildResultSummary(self): summary = yield self.getResultSummary() if self.results in self.updateBuildSummaryPolicy and \ 'build' not in summary and 'step' in summary: summary['build'] = summary['step'] return summary @debounce.method(wait=1) @defer.inlineCallbacks def updateSummary(self): def methodInfo(m): lines = inspect.getsourcelines(m) return "\nat {}:{}:\n {}".format(inspect.getsourcefile(m), lines[1], "\n".join(lines[0])) if not self._running: summary = yield self.getResultSummary() if not isinstance(summary, dict): raise TypeError('getResultSummary must return a dictionary: ' + methodInfo(self.getResultSummary)) else: summary = yield self.getCurrentSummary() if not isinstance(summary, dict): raise TypeError( 'getCurrentSummary must return a dictionary: ' + methodInfo(self.getCurrentSummary)) stepResult = summary.get('step', 'finished') if not isinstance(stepResult, str): raise TypeError("step result string must be unicode (got %r)" % (stepResult, )) if self.stepid is not None: stepResult = self.build.properties.cleanupTextFromSecrets( stepResult) yield self.master.data.updates.setStepStateString( self.stepid, stepResult) if not self._running: buildResult = summary.get('build', None) if buildResult and not isinstance(buildResult, str): raise TypeError("build result string must be unicode") # updateSummary gets patched out for old-style steps, so keep a copy we can # call internally for such steps realUpdateSummary = updateSummary @defer.inlineCallbacks def addStep(self): # create and start the step, noting that the name may be altered to # ensure uniqueness self.name = yield self.build.render(self.name) self.build.setUniqueStepName(self) self.stepid, self.number, self.name = yield self.master.data.updates.addStep( buildid=self.build.buildid, name=util.bytes2unicode(self.name)) yield self.master.data.updates.startStep(self.stepid) @defer.inlineCallbacks def startStep(self, remote): self.remote = remote yield self.addStep() self.locks = yield self.build.render(self.locks) # convert all locks into their real form botmaster = self.build.builder.botmaster self.locks = yield botmaster.getLockFromLockAccesses( self.locks, self.build.config_version) # then narrow WorkerLocks down to the worker that this build is being # run on self.locks = [(l.getLockForWorker(self.build.workerforbuilder.worker), la) for l, la in self.locks] for l, la in self.locks: if l in self.build.locks: log.msg(("Hey, lock {} is claimed by both a Step ({}) and the" " parent Build ({})").format(l, self, self.build)) raise RuntimeError("lock claimed by both Step and Build") try: # set up locks yield self.acquireLocks() if self.stopped: raise BuildStepCancelled # render renderables in parallel renderables = [] accumulateClassList(self.__class__, 'renderables', renderables) def setRenderable(res, attr): setattr(self, attr, res) dl = [] for renderable in renderables: d = self.build.render(getattr(self, renderable)) d.addCallback(setRenderable, renderable) dl.append(d) yield defer.gatherResults(dl) self.rendered = True # we describe ourselves only when renderables are interpolated self.realUpdateSummary() # check doStepIf (after rendering) if isinstance(self.doStepIf, bool): doStep = self.doStepIf else: doStep = yield self.doStepIf(self) # run -- or skip -- the step if doStep: yield self.addTestResultSets() try: self._running = True self.results = yield self.run() finally: self._running = False else: self.results = SKIPPED # NOTE: all of these `except` blocks must set self.results immediately! except BuildStepCancelled: self.results = CANCELLED except BuildStepFailed: self.results = FAILURE except error.ConnectionLost: self.results = RETRY except Exception: self.results = EXCEPTION why = Failure() log.err(why, "BuildStep.failed; traceback follows") yield self.addLogWithFailure(why) if self.stopped and self.results != RETRY: # We handle this specially because we don't care about # the return code of an interrupted command; we know # that this should just be exception due to interrupt # At the same time we must respect RETRY status because it's used # to retry interrupted build due to some other issues for example # due to worker lost if self.results != CANCELLED: self.results = EXCEPTION # determine whether we should hide this step hidden = self.hideStepIf if callable(hidden): try: hidden = hidden(self.results, self) except Exception: why = Failure() log.err(why, "hidden callback failed; traceback follows") yield self.addLogWithFailure(why) self.results = EXCEPTION hidden = False yield self.master.data.updates.finishStep(self.stepid, self.results, hidden) # perform final clean ups success = yield self._cleanup_logs() if not success: self.results = EXCEPTION # update the summary one last time, make sure that completes, # and then don't update it any more. self.realUpdateSummary() yield self.realUpdateSummary.stop() for sub in self._test_result_submitters.values(): yield sub.finish() self.releaseLocks() return self.results @defer.inlineCallbacks def _cleanup_logs(self): all_success = True not_finished_logs = [ v for (k, v) in self.logs.items() if not v.finished ] finish_logs = yield defer.DeferredList( [v.finish() for v in not_finished_logs], consumeErrors=True) for success, res in finish_logs: if not success: log.err(res, "when trying to finish a log") all_success = False for log_ in self.logs.values(): if log_.had_errors(): all_success = False return all_success def addTestResultSets(self): return defer.succeed(None) @defer.inlineCallbacks def addTestResultSet(self, description, category, value_unit): sub = TestResultSubmitter() yield sub.setup(self, description, category, value_unit) setid = sub.get_test_result_set_id() self._test_result_submitters[setid] = sub return setid def addTestResult(self, setid, value, test_name=None, test_code_path=None, line=None, duration_ns=None): self._test_result_submitters[setid].add_test_result( value, test_name=test_name, test_code_path=test_code_path, line=line, duration_ns=duration_ns) def acquireLocks(self, res=None): if not self.locks: return defer.succeed(None) if self.stopped: return defer.succeed(None) log.msg("acquireLocks(step {}, locks {})".format(self, self.locks)) for lock, access in self.locks: for waited_lock, _, _ in self._acquiringLocks: if lock is waited_lock: continue if not lock.isAvailable(self, access): self._waitingForLocks = True log.msg("step {} waiting for lock {}".format(self, lock)) d = lock.waitUntilMaybeAvailable(self, access) self._acquiringLocks.append((lock, access, d)) d.addCallback(self.acquireLocks) return d # all locks are available, claim them all for lock, access in self.locks: lock.claim(self, access) self._acquiringLocks = [] self._waitingForLocks = False return defer.succeed(None) @defer.inlineCallbacks def run(self): self._start_deferred = defer.Deferred() unhandled = self._start_unhandled_deferreds = [] self._sync_addlog_deferreds = [] try: # here's where we set things up for backward compatibility for # old-style steps, using monkey patches so that new-style steps # aren't bothered by any of this equipment # monkey-patch self.step_status.{setText,setText2} back into # existence for old steps, signalling an update to the summary self.step_status = BuildStepStatus() self.step_status.setText = lambda text: self.realUpdateSummary() self.step_status.setText2 = lambda text: self.realUpdateSummary() # monkey-patch in support for old statistics functions self.step_status.setStatistic = self.setStatistic self.step_status.getStatistic = self.getStatistic self.step_status.hasStatistic = self.hasStatistic # monkey-patch an addLog that returns an write-only, sync log self.addLog = self.addLog_oldStyle self._logFileWrappers = {} # and a getLog that returns a read-only, sync log, captured by # LogObservers installed by addLog_oldStyle self.getLog = self.getLog_oldStyle # old-style steps shouldn't be calling updateSummary def updateSummary(): assert 0, 'updateSummary is only valid on new-style steps' self.updateSummary = updateSummary results = yield self.start() if results is not None: self._start_deferred.callback(results) results = yield self._start_deferred finally: # hook for tests # assert so that it is only run in non optimized mode assert self._run_finished_hook() is None # wait until all the sync logs have been actually created before # finishing yield defer.DeferredList(self._sync_addlog_deferreds, consumeErrors=True) self._start_deferred = None unhandled = self._start_unhandled_deferreds self.realUpdateSummary() # Wait for any possibly-unhandled deferreds. If any fail, change the # result to EXCEPTION and log. while unhandled: self._start_unhandled_deferreds = [] unhandled_results = yield defer.DeferredList( unhandled, consumeErrors=True) for success, res in unhandled_results: if not success: log.err( res, "from an asynchronous method executed in an old-style step" ) results = EXCEPTION unhandled = self._start_unhandled_deferreds return results def finished(self, results): assert self._start_deferred, \ "finished() can only be called from old steps implementing start()" self._start_deferred.callback(results) def failed(self, why): assert self._start_deferred, \ "failed() can only be called from old steps implementing start()" self._start_deferred.errback(why) def isNewStyle(self): # **temporary** method until new-style steps are the only supported style return self.run.__func__ is not BuildStep.run def start(self): # New-style classes implement 'run'. # Old-style classes implemented 'start'. Advise them to do 'run' # instead. raise NotImplementedError("your subclass must implement run()") def interrupt(self, reason): self.stopped = True if self._acquiringLocks: for (lock, access, d) in self._acquiringLocks: lock.stopWaitingUntilAvailable(self, access, d) self._acquiringLocks = [] if self._waitingForLocks: self.addCompleteLog('cancelled while waiting for locks', str(reason)) else: self.addCompleteLog('cancelled', str(reason)) if self.cmd: d = self.cmd.interrupt(reason) d.addErrback(log.err, 'while cancelling command') def releaseLocks(self): log.msg("releaseLocks({}): {}".format(self, self.locks)) for lock, access in self.locks: if lock.isOwner(self, access): lock.release(self, access) else: # This should only happen if we've been interrupted assert self.stopped # utility methods that BuildSteps may find useful def workerVersion(self, command, oldversion=None): return self.build.getWorkerCommandVersion(command, oldversion) def workerVersionIsOlderThan(self, command, minversion): sv = self.build.getWorkerCommandVersion(command, None) if sv is None: return True if [int(s) for s in sv.split(".") ] < [int(m) for m in minversion.split(".")]: return True return False def checkWorkerHasCommand(self, command): if not self.workerVersion(command): message = "worker is too old, does not know about {}".format( command) raise WorkerSetupError(message) def getWorkerName(self): return self.build.getWorkerName() def addLog(self, name, type='s', logEncoding=None): if self.stepid is None: raise BuildStepCancelled d = self.master.data.updates.addLog(self.stepid, util.bytes2unicode(name), str(type)) @d.addCallback def newLog(logid): return self._newLog(name, type, logid, logEncoding) return d addLog_newStyle = addLog def addLog_oldStyle(self, name, type='s', logEncoding=None): # create a logfile instance that acts like old-style status logfiles # begin to create a new-style logfile loog_d = self.addLog_newStyle(name, type, logEncoding) self._start_unhandled_deferreds.append(loog_d) # and wrap the deferred that will eventually fire with that logfile # into a write-only logfile instance wrapper = SyncLogFileWrapper(self, name, loog_d) self._logFileWrappers[name] = wrapper return wrapper def getLog(self, name): return self.logs[name] def getLog_oldStyle(self, name): return self._logFileWrappers[name] @_maybeUnhandled @defer.inlineCallbacks def addCompleteLog(self, name, text): if self.stepid is None: raise BuildStepCancelled logid = yield self.master.data.updates.addLog(self.stepid, util.bytes2unicode(name), 't') _log = self._newLog(name, 't', logid) yield _log.addContent(text) yield _log.finish() @_maybeUnhandled @defer.inlineCallbacks def addHTMLLog(self, name, html): if self.stepid is None: raise BuildStepCancelled logid = yield self.master.data.updates.addLog(self.stepid, util.bytes2unicode(name), 'h') _log = self._newLog(name, 'h', logid) html = bytes2unicode(html) yield _log.addContent(html) yield _log.finish() @defer.inlineCallbacks def addLogWithFailure(self, why, logprefix=""): # helper for showing exceptions to the users try: yield self.addCompleteLog(logprefix + "err.text", why.getTraceback()) yield self.addHTMLLog(logprefix + "err.html", formatFailure(why)) except Exception: log.err(Failure(), "error while formatting exceptions") def addLogWithException(self, why, logprefix=""): return self.addLogWithFailure(Failure(why), logprefix) def addLogObserver(self, logname, observer): assert interfaces.ILogObserver.providedBy(observer) observer.setStep(self) self._pendingLogObservers.append((logname, observer)) self._connectPendingLogObservers() def _newLog(self, name, type, logid, logEncoding=None): if not logEncoding: logEncoding = self.logEncoding if not logEncoding: logEncoding = self.master.config.logEncoding log = plog.Log.new(self.master, name, type, logid, logEncoding) self.logs[name] = log self._connectPendingLogObservers() return log def _connectPendingLogObservers(self): for logname, observer in self._pendingLogObservers[:]: if logname in self.logs: observer.setLog(self.logs[logname]) self._pendingLogObservers.remove((logname, observer)) @_maybeUnhandled @defer.inlineCallbacks def addURL(self, name, url): yield self.master.data.updates.addStepURL(self.stepid, str(name), str(url)) return None @defer.inlineCallbacks def runCommand(self, command): self.cmd = command command.worker = self.worker try: res = yield command.run(self, self.remote, self.build.builder.name) finally: self.cmd = None return res def hasStatistic(self, name): return name in self.statistics def getStatistic(self, name, default=None): return self.statistics.get(name, default) def getStatistics(self): return self.statistics.copy() def setStatistic(self, name, value): self.statistics[name] = value def _describe(self, done=False): # old-style steps expect this function to exist assert not self.isNewStyle() return [] def describe(self, done=False): # old-style steps expect this function to exist assert not self.isNewStyle() desc = self._describe(done) if not desc: return [] if self.descriptionSuffix: desc += self.descriptionSuffix return desc def warn_deprecated_if_oldstyle_subclass(self, name): if self.__class__.__name__ != name: warn_deprecated( '2.9.0', ('Subclassing old-style step {0} in {1} is deprecated, ' 'please migrate to new-style equivalent {0}NewStyle').format( name, self.__class__.__name__))
# Copyright (c) Twisted Matrix Laboratories. # See LICENSE for details. # This is an auto-generated file. Do not edit it. """ Provides Twisted version information. """ from twisted.python import versions version = versions.Version('twisted.news', 14, 0, 0)
# This is an auto-generated file. Do not edit it. from twisted.python import versions version = versions.Version('twisted.runner', 10, 0, 0)
# Copyright (c) Twisted Matrix Laboratories. # See LICENSE for details. # This is an auto-generated file. Do not edit it. """ Provides Twisted version information. """ from twisted.python import versions version = versions.Version('twisted.web', 13, 2, 0)
# Copyright (c) Twisted Matrix Laboratories. # See LICENSE for details. # This is an auto-generated file. Do not edit it. """ Provides Twisted version information. """ from twisted.python import versions version = versions.Version('twisted.conch', 15, 2, 0)
# This is an auto-generated file. Do not edit it. from twisted.python import versions version = versions.Version('twisted.mail', 8, 2, 0)
# Copyright (c) Twisted Matrix Laboratories. # See LICENSE for details. # This is an auto-generated file. Do not edit it. """ Provides Twisted version information. """ from twisted.python import versions version = versions.Version('twisted', 14, 0, 0)
s = s + '\n' return s def isMultiline(s): """Returns True if this string has a newline in it.""" return (string.find(s, '\n') != -1) def endsInNewline(s): """Returns True if this string ends in a newline.""" return (s[-len('\n'):] == '\n') deprecate.deprecatedModuleAttribute(versions.Version("Twisted", 10, 2, 0), "Please use inspect.getdoc instead.", __name__, "docstringLStrip") def docstringLStrip(docstring): """ Gets rid of unsightly lefthand docstring whitespace residue. You'd think someone would have done this already, but apparently not in 1.5.2. BUT since we're all using Python 2.1 now, use L{inspect.getdoc} instead. I{This function should go away soon.} """
# This is an auto-generated file. Do not edit it. from twisted.python import versions version = versions.Version('txweb2', 9, 0, 0)
class BuildStep(results.ResultComputingConfigMixin, properties.PropertiesMixin, WorkerAPICompatMixin): implements(interfaces.IBuildStep) alwaysRun = False doStepIf = True hideStepIf = False # properties set on a build step are, by nature, always runtime properties set_runtime_properties = True renderables = results.ResultComputingConfigMixin.resultConfig + [ 'alwaysRun', 'description', 'descriptionDone', 'descriptionSuffix', 'doStepIf', 'hideStepIf', 'workdir', ] # 'parms' holds a list of all the parameters we care about, to allow # users to instantiate a subclass of BuildStep with a mixture of # arguments, some of which are for us, some of which are for the subclass # (or a delegate of the subclass, like how ShellCommand delivers many # arguments to the RemoteShellCommand that it creates). Such delegating # subclasses will use this list to figure out which arguments are meant # for us and which should be given to someone else. parms = [ 'alwaysRun', 'description', 'descriptionDone', 'descriptionSuffix', 'doStepIf', 'flunkOnFailure', 'flunkOnWarnings', 'haltOnFailure', 'hideStepIf', 'locks', 'logEncoding', 'name', 'progressMetrics', 'useProgress', 'warnOnFailure', 'warnOnWarnings', 'workdir', ] name = "generic" description = None # set this to a list of short strings to override descriptionDone = None # alternate description when the step is complete descriptionSuffix = None # extra information to append to suffix locks = [] progressMetrics = () # 'time' is implicit useProgress = True # set to False if step is really unpredictable build = None step_status = None progress = None logEncoding = None cmd = None rendered = False # true if attributes are rendered _workdir = None _waitingForLocks = False _run_finished_hook = lambda self: None # for tests def __init__(self, **kwargs): self.worker = None self._registerOldWorkerAttr("worker", name="buildslave") for p in self.__class__.parms: if p in kwargs: setattr(self, p, kwargs.pop(p)) if kwargs: config.error("%s.__init__ got unexpected keyword argument(s) %s" % (self.__class__, list(kwargs))) self._pendingLogObservers = [] if not isinstance(self.name, str): config.error("BuildStep name must be a string: %r" % (self.name, )) if isinstance(self.description, str): self.description = [self.description] if isinstance(self.descriptionDone, str): self.descriptionDone = [self.descriptionDone] if isinstance(self.descriptionSuffix, str): self.descriptionSuffix = [self.descriptionSuffix] self._acquiringLock = None self.stopped = False self.master = None self.statistics = {} self.logs = {} self._running = False self.stepid = None self._start_unhandled_deferreds = None def __new__(klass, *args, **kwargs): self = object.__new__(klass) self._factory = _BuildStepFactory(klass, *args, **kwargs) return self def setBuild(self, build): self.build = build self.master = self.build.master def setWorker(self, worker): self.worker = worker deprecatedWorkerClassMethod(locals(), setWorker, compat_name="setBuildSlave") @deprecate.deprecated(versions.Version("buildbot", 0, 9, 0)) def setDefaultWorkdir(self, workdir): if self._workdir is None: self._workdir = workdir @property def workdir(self): # default the workdir appropriately if self._workdir is not None or self.build is None: return self._workdir else: # see :ref:`Factory-Workdir-Functions` for details on how to # customize this if callable(self.build.workdir): return self.build.workdir(self.build.sources) else: return self.build.workdir @workdir.setter def workdir(self, workdir): self._workdir = workdir def addFactoryArguments(self, **kwargs): # this is here for backwards compatibility pass def _getStepFactory(self): return self._factory def setupProgress(self): # this function temporarily does nothing pass def setProgress(self, metric, value): # this function temporarily does nothing pass def getCurrentSummary(self): if self.description is not None: stepsumm = util.join_list(self.description) if self.descriptionSuffix: stepsumm += u' ' + util.join_list(self.descriptionSuffix) else: stepsumm = u'running' return {u'step': stepsumm} def getResultSummary(self): if self.descriptionDone is not None or self.description is not None: stepsumm = util.join_list(self.descriptionDone or self.description) if self.descriptionSuffix: stepsumm += u' ' + util.join_list(self.descriptionSuffix) else: stepsumm = u'finished' if self.results != SUCCESS: stepsumm += u' (%s)' % Results[self.results] return {u'step': stepsumm} @debounce.method(wait=1) @defer.inlineCallbacks def updateSummary(self): def methodInfo(m): import inspect lines = inspect.getsourcelines(m) return "\nat %s:%s:\n %s" % (inspect.getsourcefile(m), lines[1], "\n".join(lines[0])) if not self._running: summary = yield self.getResultSummary() if not isinstance(summary, dict): raise TypeError('getResultSummary must return a dictionary: ' + methodInfo(self.getCurrentSummary)) else: summary = yield self.getCurrentSummary() if not isinstance(summary, dict): raise TypeError( 'getCurrentSummary must return a dictionary: ' + methodInfo(self.getCurrentSummary)) stepResult = summary.get('step', u'finished') if not isinstance(stepResult, unicode): raise TypeError("step result string must be unicode (got %r)" % (stepResult, )) if self.stepid is not None: yield self.master.data.updates.setStepStateString( self.stepid, stepResult) if not self._running: buildResult = summary.get('build', None) if buildResult and not isinstance(buildResult, unicode): raise TypeError("build result string must be unicode") # updateSummary gets patched out for old-style steps, so keep a copy we can # call internally for such steps realUpdateSummary = updateSummary @defer.inlineCallbacks def startStep(self, remote): self.remote = remote # create and start the step, noting that the name may be altered to # ensure uniqueness self.stepid, self.number, self.name = yield self.master.data.updates.addStep( buildid=self.build.buildid, name=util.ascii2unicode(self.name)) yield self.master.data.updates.startStep(self.stepid) self.locks = yield self.build.render(self.locks) # convert all locks into their real form self.locks = [ (self.build.builder.botmaster.getLockFromLockAccess(access), access) for access in self.locks ] # then narrow WorkerLocks down to the worker that this build is being # run on self.locks = [(l.getLock(self.build.workerforbuilder.worker), la) for l, la in self.locks] for l, la in self.locks: if l in self.build.locks: log.msg("Hey, lock %s is claimed by both a Step (%s) and the" " parent Build (%s)" % (l, self, self.build)) raise RuntimeError("lock claimed by both Step and Build") try: # set up locks yield self.acquireLocks() if self.stopped: raise BuildStepCancelled # check doStepIf if isinstance(self.doStepIf, bool): doStep = self.doStepIf else: doStep = yield self.doStepIf(self) # render renderables in parallel renderables = [] accumulateClassList(self.__class__, 'renderables', renderables) def setRenderable(res, attr): setattr(self, attr, res) dl = [] for renderable in renderables: d = self.build.render(getattr(self, renderable)) d.addCallback(setRenderable, renderable) dl.append(d) yield defer.gatherResults(dl) self.rendered = True # we describe ourselves only when renderables are interpolated self.realUpdateSummary() # run -- or skip -- the step if doStep: try: self._running = True self.results = yield self.run() finally: self._running = False else: self.results = SKIPPED # NOTE: all of these `except` blocks must set self.results immediately! except BuildStepCancelled: self.results = CANCELLED except BuildStepFailed: self.results = FAILURE except error.ConnectionLost: self.results = RETRY except Exception: self.results = EXCEPTION why = Failure() log.err(why, "BuildStep.failed; traceback follows") yield self.addLogWithFailure(why) if self.stopped and self.results != RETRY: # We handle this specially because we don't care about # the return code of an interrupted command; we know # that this should just be exception due to interrupt # At the same time we must respect RETRY status because it's used # to retry interrupted build due to some other issues for example # due to worker lost if self.results != CANCELLED: self.results = EXCEPTION # update the summary one last time, make sure that completes, # and then don't update it any more. self.realUpdateSummary() yield self.realUpdateSummary.stop() # determine whether we should hide this step hidden = self.hideStepIf if callable(hidden): try: hidden = hidden(self.results, self) except Exception: why = Failure() log.err(why, "hidden callback failed; traceback follows") yield self.addLogWithFailure(why) self.results = EXCEPTION hidden = False yield self.master.data.updates.finishStep(self.stepid, self.results, hidden) self.releaseLocks() defer.returnValue(self.results) def acquireLocks(self, res=None): self._acquiringLock = None if not self.locks: return defer.succeed(None) if self.stopped: return defer.succeed(None) log.msg("acquireLocks(step %s, locks %s)" % (self, self.locks)) for lock, access in self.locks: if not lock.isAvailable(self, access): self._waitingForLocks = True log.msg("step %s waiting for lock %s" % (self, lock)) d = lock.waitUntilMaybeAvailable(self, access) d.addCallback(self.acquireLocks) self._acquiringLock = (lock, access, d) return d # all locks are available, claim them all for lock, access in self.locks: lock.claim(self, access) self._waitingForLocks = False return defer.succeed(None) @defer.inlineCallbacks def run(self): self._start_deferred = defer.Deferred() unhandled = self._start_unhandled_deferreds = [] self._sync_addlog_deferreds = [] try: # here's where we set things up for backward compatibility for # old-style steps, using monkey patches so that new-style steps # aren't bothered by any of this equipment # monkey-patch self.step_status.{setText,setText2} back into # existence for old steps, signalling an update to the summary self.step_status = BuildStepStatus() self.step_status.setText = lambda text: self.realUpdateSummary() self.step_status.setText2 = lambda text: self.realUpdateSummary() # monkey-patch in support for old statistics functions self.step_status.setStatistic = self.setStatistic self.step_status.getStatistic = self.getStatistic self.step_status.hasStatistic = self.hasStatistic # monkey-patch an addLog that returns an write-only, sync log self.addLog = self.addLog_oldStyle self._logFileWrappers = {} # and a getLog that returns a read-only, sync log, captured by # LogObservers installed by addLog_oldStyle self.getLog = self.getLog_oldStyle # old-style steps shouldn't be calling updateSummary def updateSummary(): assert 0, 'updateSummary is only valid on new-style steps' self.updateSummary = updateSummary results = yield self.start() if results is not None: self._start_deferred.callback(results) results = yield self._start_deferred finally: # hook for tests # assert so that it is only run in non optimized mode assert self._run_finished_hook() is None # wait until all the sync logs have been actually created before # finishing yield defer.DeferredList(self._sync_addlog_deferreds, consumeErrors=True) self._start_deferred = None unhandled = self._start_unhandled_deferreds self._start_unhandled_deferreds = None self.realUpdateSummary() # Wait for any possibly-unhandled deferreds. If any fail, change the # result to EXCEPTION and log. if unhandled: unhandled_results = yield defer.DeferredList(unhandled, consumeErrors=True) for success, res in unhandled_results: if not success: log.err( res, "from an asynchronous method executed in an old-style step" ) results = EXCEPTION defer.returnValue(results) def finished(self, results): assert self._start_deferred, \ "finished() can only be called from old steps implementing start()" self._start_deferred.callback(results) def failed(self, why): assert self._start_deferred, \ "failed() can only be called from old steps implementing start()" self._start_deferred.errback(why) def isNewStyle(self): # **temporary** method until new-style steps are the only supported style return self.run.im_func is not BuildStep.run.im_func def start(self): # New-style classes implement 'run'. # Old-style classes implemented 'start'. Advise them to do 'run' # instead. raise NotImplementedError("your subclass must implement run()") def interrupt(self, reason): # TODO: consider adding an INTERRUPTED or STOPPED status to use # instead of FAILURE, might make the text a bit more clear. # 'reason' can be a Failure, or text self.stopped = True if self._acquiringLock: lock, access, d = self._acquiringLock lock.stopWaitingUntilAvailable(self, access, d) d.callback(None) if self._waitingForLocks: self.addCompleteLog('cancelled while waiting for locks', str(reason)) else: self.addCompleteLog('cancelled', str(reason)) if self.cmd: d = self.cmd.interrupt(reason) d.addErrback(log.err, 'while cancelling command') def releaseLocks(self): log.msg("releaseLocks(%s): %s" % (self, self.locks)) for lock, access in self.locks: if lock.isOwner(self, access): lock.release(self, access) else: # This should only happen if we've been interrupted assert self.stopped # utility methods that BuildSteps may find useful def workerVersion(self, command, oldversion=None): return self.build.getWorkerCommandVersion(command, oldversion) deprecatedWorkerClassMethod(locals(), workerVersion) def workerVersionIsOlderThan(self, command, minversion): sv = self.build.getWorkerCommandVersion(command, None) if sv is None: return True if map(int, sv.split(".")) < map(int, minversion.split(".")): return True return False deprecatedWorkerClassMethod(locals(), workerVersionIsOlderThan) def checkWorkerHasCommand(self, command): if not self.workerVersion(command): message = "worker is too old, does not know about %s" % command raise WorkerTooOldError(message) deprecatedWorkerClassMethod(locals(), checkWorkerHasCommand) def getWorkerName(self): return self.build.getWorkerName() deprecatedWorkerClassMethod(locals(), getWorkerName) def addLog(self, name, type='s', logEncoding=None): d = self.master.data.updates.addLog(self.stepid, util.ascii2unicode(name), unicode(type)) @d.addCallback def newLog(logid): return self._newLog(name, type, logid, logEncoding) return d addLog_newStyle = addLog def addLog_oldStyle(self, name, type='s', logEncoding=None): # create a logfile instance that acts like old-style status logfiles # begin to create a new-style logfile loog_d = self.addLog_newStyle(name, type, logEncoding) self._start_unhandled_deferreds.append(loog_d) # and wrap the deferred that will eventually fire with that logfile # into a write-only logfile instance wrapper = SyncLogFileWrapper(self, name, loog_d) self._logFileWrappers[name] = wrapper return wrapper def getLog(self, name): return self.logs[name] def getLog_oldStyle(self, name): return self._logFileWrappers[name] @_maybeUnhandled @defer.inlineCallbacks def addCompleteLog(self, name, text): log.msg("addCompleteLog(%s)" % name) logid = yield self.master.data.updates.addLog(self.stepid, util.ascii2unicode(name), u't') l = self._newLog(name, u't', logid) yield l.addContent(text) yield l.finish() @_maybeUnhandled @defer.inlineCallbacks def addHTMLLog(self, name, html): log.msg("addHTMLLog(%s)" % name) logid = yield self.master.data.updates.addLog(self.stepid, util.ascii2unicode(name), u'h') l = self._newLog(name, u'h', logid) yield l.addContent(html) yield l.finish() @defer.inlineCallbacks def addLogWithFailure(self, why, logprefix=""): # helper for showing exceptions to the users try: yield self.addCompleteLog(logprefix + "err.text", why.getTraceback()) yield self.addHTMLLog(logprefix + "err.html", formatFailure(why)) except Exception: log.err(Failure(), "error while formatting exceptions") def addLogWithException(self, why, logprefix=""): return self.addLogWithFailure(Failure(why), logprefix) def addLogObserver(self, logname, observer): assert interfaces.ILogObserver.providedBy(observer) observer.setStep(self) self._pendingLogObservers.append((logname, observer)) self._connectPendingLogObservers() def _newLog(self, name, type, logid, logEncoding=None): if not logEncoding: logEncoding = self.logEncoding if not logEncoding: logEncoding = self.master.config.logEncoding log = plog.Log.new(self.master, name, type, logid, logEncoding) self.logs[name] = log self._connectPendingLogObservers() return log def _connectPendingLogObservers(self): for logname, observer in self._pendingLogObservers[:]: if logname in self.logs: observer.setLog(self.logs[logname]) self._pendingLogObservers.remove((logname, observer)) @_maybeUnhandled @defer.inlineCallbacks def addURL(self, name, url): yield self.master.data.updates.addStepURL(self.stepid, unicode(name), unicode(url)) defer.returnValue(None) @defer.inlineCallbacks def runCommand(self, command): self.cmd = command command.worker = self.worker try: res = yield command.run(self, self.remote, self.build.builder.name) finally: self.cmd = None defer.returnValue(res) def hasStatistic(self, name): return name in self.statistics def getStatistic(self, name, default=None): return self.statistics.get(name, default) def getStatistics(self): return self.statistics.copy() def setStatistic(self, name, value): self.statistics[name] = value def _describe(self, done=False): # old-style steps expect this function to exist assert not self.isNewStyle() return [] def describe(self, done=False): # old-style steps expect this function to exist assert not self.isNewStyle() desc = self._describe(done) if not desc: return [] if self.descriptionSuffix: desc += self.descriptionSuffix return desc
def patch_bug4881(): # this bug was only present in Twisted-10.2.0 if twisted.version == versions.Version('twisted', 10, 2, 0): from buildbot.monkeypatches import bug4881 bug4881.patch()
class TestStart(misc.StdoutAssertionsMixin, dirs.DirsMixin, unittest.TestCase): def setUp(self): self.setUpDirs('basedir') with open(os.path.join('basedir', 'buildbot.tac'), 'wt') as f: f.write(fake_master_tac) self.setUpStdoutAssertions() def tearDown(self): self.tearDownDirs() # tests def test_start_not_basedir(self): self.assertEqual(start.start(mkconfig(basedir='doesntexist')), 1) self.assertInStdout('invalid buildmaster directory') def runStart(self, **config): args = [ '-c', 'from buildbot.scripts.start import start; import sys; ' 'sys.exit(start(%r))' % ( mkconfig(**config),), ] env = os.environ.copy() env['PYTHONPATH'] = os.pathsep.join(sys.path) return getProcessOutputAndValue(sys.executable, args=args, env=env) @defer.inlineCallbacks def test_start_no_daemon(self): (_, err, rc) = yield self.runStart(nodaemon=True) # on python 3.5, cryptography loudly complains to upgrade if sys.version_info[:2] != (3, 5): self.assertEqual((err, rc), (b'', 0)) @defer.inlineCallbacks def test_start_quiet(self): res = yield self.runStart(quiet=True) # on python 3.5, cryptography loudly complains to upgrade if sys.version_info[:2] != (3, 5): self.assertEqual(res, (b'', b'', 0)) @skipUnlessPlatformIs('posix') @defer.inlineCallbacks def test_start_timeout_nonnumber(self): (out, err, rc) = yield self.runStart(start_timeout='a') self.assertEqual((rc, err), (1, b'')) self.assertSubstring(b'Start timeout must be a number\n', out) @skipUnlessPlatformIs('posix') @defer.inlineCallbacks def test_start_timeout_number_string(self): # integer values from command-line options come in as strings res = yield self.runStart(start_timeout='10') self.assertEqual(res, (mock.ANY, b'', 0)) @skipUnlessPlatformIs('posix') @defer.inlineCallbacks def test_start(self): try: (out, err, rc) = yield self.runStart() self.assertEqual((rc, err), (0, b'')) self.assertSubstring(b'buildmaster appears to have (re)started correctly', out) finally: # wait for the pidfile to go away after the reactor.stop # in buildbot.tac takes effect pidfile = os.path.join('basedir', 'twistd.pid') while os.path.exists(pidfile): time.sleep(0.01) if twisted.version <= versions.Version('twisted', 9, 0, 0): test_start.skip = test_start_quiet.skip = "Skipping due to suprious PotentialZombieWarning."
import os import gc from twisted.web import http from twisted.internet import address from twisted.python.logfile import DailyLogFile from twisted.python.monkey import MonkeyPatcher from twisted.python import versions, filepath, log from mamba.utils import borg from mamba.http import headers from mamba.core import packages from mamba import _version as _mamba_version from mamba.application import controller, model _app_ver = versions.Version('Application', 0, 1, 0) _app_project_ver = versions.Version('Project', 0, 1, 0) class ApplicationError(Exception): """ApplicationError raises when an error occurs """ class Mamba(borg.Borg): """ This object is just a global configuration for mamba applications that act as the central object on them and is able to act as a central registry. It inherits from the :class: `~mamba.utils.borg.Borg` so you can just instantiate a new object of this class and it will share all the information between instances.
# This is an auto-generated file. Use Epsilon/bin/release-divmod to update. from twisted.python import versions version = versions.Version(__name__[:__name__.rfind('.')], 0, 9, 32)
def asTwistedVersion(packageName, versionString): from twisted.python import versions import re return versions.Version( packageName, *map(int, re.match(r"[0-9.]*", versionString).group().split(".")[:3]))
# This is an auto-generated file. Do not edit it. from twisted.python import versions version = versions.Version('twisted.web', 12, 1, 0)
# This is an auto-generated file. Do not edit it. from twisted.python import versions version = versions.Version('twisted.names', 12, 0, 0)
# Copyright (c) Twisted Matrix Laboratories. # See LICENSE for details. # This is an auto-generated file. Do not edit it. """ Provides Twisted version information. """ from twisted.python import versions version = versions.Version('twisted.runner', 13, 1, 0)
def patch_bug5079(): # this bug will hopefully be patched in Twisted-12.0.0 if twisted.version < versions.Version('twisted', 12, 0, 0): from buildslave.monkeypatches import bug5079 bug5079.patch()
class TestStart(misc.StdoutAssertionsMixin, dirs.DirsMixin, unittest.TestCase): def setUp(self): self.setUpDirs('basedir') with open(os.path.join('basedir', 'buildbot.tac'), 'wt') as f: f.write(fake_master_tac) self.setUpStdoutAssertions() def tearDown(self): self.tearDownDirs() # tests def test_start_not_basedir(self): self.assertEqual(start.start(mkconfig(basedir='doesntexist')), 1) self.assertInStdout('invalid buildmaster directory') def runStart(self, **config): args = [ '-c', 'from buildbot.scripts.start import start; start(%r)' % (mkconfig(**config), ), ] env = os.environ.copy() env['PYTHONPATH'] = os.pathsep.join(sys.path) return getProcessOutputAndValue(sys.executable, args=args, env=env) @skipIfPythonVersionIsLess((2, 7)) def test_start_no_daemon(self): d = self.runStart(nodaemon=True) @d.addCallback def cb(res): self.assertEquals(res, ('', '', 0)) print(res) return d @skipIfPythonVersionIsLess((2, 7)) def test_start_quiet(self): d = self.runStart(quiet=True) @d.addCallback def cb(res): self.assertEquals(res, ('', '', 0)) print(res) return d @flaky(bugNumber=2760) @skipUnlessPlatformIs('posix') def test_start(self): d = self.runStart() @d.addCallback def cb(xxx_todo_changeme): (out, err, rc) = xxx_todo_changeme self.assertEqual((rc, err), (0, '')) self.assertSubstring('BuildMaster is running', out) @d.addBoth def flush(x): # wait for the pidfile to go away after the reactor.stop # in buildbot.tac takes effect pidfile = os.path.join('basedir', 'twistd.pid') while os.path.exists(pidfile): time.sleep(0.01) return x return d if twisted.version <= versions.Version('twisted', 9, 0, 0): test_start.skip = test_start_quiet.skip = "Skipping due to suprious PotentialZombieWarning."
"""Walk up the Python tree from method 'meth', finding its class, its module and all containing packages.""" containers = [] containers.append(meth.im_class) moduleName = meth.im_class.__module__ while moduleName is not None: module = sys.modules.get(moduleName, None) if module is None: module = __import__(moduleName) containers.append(module) moduleName = getattr(module, '__module__', None) return containers deprecate.deprecatedModuleAttribute( versions.Version("Twisted", 12, 3, 0), "This function never worked correctly. Implement lookup on your own.", __name__, "getPythonContainers") def _runSequentially(callables, stopOnFirstError=False): """ Run the given callables one after the other. If a callable returns a Deferred, wait until it has finished before running the next callable. @param callables: An iterable of callables that take no parameters. @param stopOnFirstError: If True, then stop running callables as soon as one raises an exception or fires an errback. False by default. @return: A L{Deferred} that fires a list of C{(flag, value)} tuples. Each
# This is an auto-generated file. Do not edit it. from twisted.python import versions version = versions.Version('twisted.pair', 10, 1, 0)
# Copyright (c) Twisted Matrix Laboratories. # See LICENSE for details. # This is an auto-generated file. Do not edit it. """ Provides Twisted version information. """ from twisted.python import versions version = versions.Version('twisted.mail', 13, 1, 0)
# This is an auto-generated file. Do not edit it. from twisted.python import versions version = versions.Version('twisted.web', 11, 0, 0)
def asTwistedVersion(packageName, versionString): return versions.Version(packageName, *map(int, versionString.split(".")))