def test_worst_status(self):
self.assertEqual(
results.WARNINGS,
results.worst_status(results.SUCCESS, results.WARNINGS))
self.assertEqual(
results.CANCELLED,
results.worst_status(results.SKIPPED, results.CANCELLED))
def evaluateCommand(self, cmd):
result = cmd.results()
if (self.maxWarnCount is not None and self.warnCount > self.maxWarnCount):
result = worst_status(result, FAILURE)
elif self.warnCount:
result = worst_status(result, WARNINGS)
return result
def evaluateCommand(self, cmd):
result = cmd.results()
if (self.maxWarnCount is not None and self.warnCount > self.maxWarnCount):
result = worst_status(result, FAILURE)
elif self.warnCount:
result = worst_status(result, WARNINGS)
return result
def buildFinished(self, text, results):
"""This method must be called when the last Step has completed. It
marks the Build as complete and returns the Builder to the 'idle'
state.
It takes two arguments which describe the overall build status:
text, results. 'results' is one of the possible results (see buildbot.process.results).
If 'results' is SUCCESS or WARNINGS, we will permit any dependant
builds to start. If it is 'FAILURE', those builds will be
abandoned."""
self.stopBuildConsumer.stopConsuming()
self.finished = True
if self.conn:
self.subs.unsubscribe()
self.subs = None
self.conn = None
log.msg(" %s: build finished" % self)
self.results = worst_status(self.results, results)
self.build_status.setText(text)
self.build_status.setResults(self.results)
self.build_status.buildFinished()
eventually(self.releaseLocks)
self.deferred.callback(self)
self.deferred = None
def buildFinished(self, text, results):
"""This method must be called when the last Step has completed. It
marks the Build as complete and returns the Builder to the 'idle'
state.
It takes two arguments which describe the overall build status:
text, results. 'results' is one of the possible results (see buildbot.process.results).
If 'results' is SUCCESS or WARNINGS, we will permit any dependant
builds to start. If it is 'FAILURE', those builds will be
abandoned."""
self.stopBuildConsumer.stopConsuming()
self.finished = True
if self.conn:
self.subs.unsubscribe()
self.subs = None
self.conn = None
log.msg(" %s: build finished" % self)
self.results = worst_status(self.results, results)
self.build_status.setText(text)
self.build_status.setResults(self.results)
self.build_status.buildFinished()
eventually(self.releaseLocks)
self.deferred.callback(self)
self.deferred = None
def runPopulateSecrets(self):
result = SUCCESS
for path, secretvalue in self.secret_to_be_populated:
if not isinstance(path, str):
raise ValueError("Secret path %s is not a string" % path)
self.secret_to_be_interpolated = secretvalue
res = yield self.downloadFileContentToWorker(path, self.secret_to_be_interpolated, mode=stat.S_IRUSR | stat.S_IWUSR)
result = worst_status(result, res)
return result
def maybeBuildsetComplete(self, bsid, _reactor=reactor):
brdicts = yield self.master.db.buildrequests.getBuildRequests(
bsid=bsid, complete=False)
# if there are incomplete buildrequests, bail out
if brdicts:
return
brdicts = yield self.master.db.buildrequests.getBuildRequests(bsid=bsid)
# figure out the overall results of the buildset:
cumulative_results = SUCCESS
for brdict in brdicts:
cumulative_results = worst_status(
cumulative_results, brdict['results'])
# get a copy of the buildset
bsdict = yield self.master.db.buildsets.getBuildset(bsid)
# if it's already completed, we're late to the game, and there's
# nothing to do.
#
# NOTE: there's still a strong possibility of a race condition here,
# which would cause buildset being completed twice.
# in this case, the db layer will detect that and raise AlreadyCompleteError
if bsdict['complete']:
return
# mark it as completed in the database
complete_at = epoch2datetime(int(_reactor.seconds()))
try:
yield self.master.db.buildsets.completeBuildset(bsid,
cumulative_results, complete_at=complete_at)
except AlreadyCompleteError:
return
# get the sourcestamps for the message
# get each of the sourcestamps for this buildset (sequentially)
bsdict = yield self.master.db.buildsets.getBuildset(bsid)
sourcestamps = []
for ssid in bsdict['sourcestamps']:
sourcestamps.append(
copy.deepcopy(
(yield self.master.data.get(('sourcestamps', str(ssid))))
)
)
msg = dict(
bsid=bsid,
external_idstring=bsdict['external_idstring'],
reason=bsdict['reason'],
sourcestamps=sourcestamps,
submitted_at=bsdict['submitted_at'],
complete=True,
complete_at=complete_at,
results=cumulative_results)
# TODO: properties=properties)
self.produceEvent(msg, "complete")
def maybeBuildsetComplete(self, bsid):
brdicts = yield self.master.db.buildrequests.getBuildRequests(
bsid=bsid, complete=False)
# if there are incomplete buildrequests, bail out
if brdicts:
return
brdicts = yield self.master.db.buildrequests.getBuildRequests(bsid=bsid)
# figure out the overall results of the buildset:
cumulative_results = SUCCESS
for brdict in brdicts:
cumulative_results = worst_status(
cumulative_results, brdict['results'])
# get a copy of the buildset
bsdict = yield self.master.db.buildsets.getBuildset(bsid)
# if it's already completed, we're late to the game, and there's
# nothing to do.
#
# NOTE: there's still a strong possibility of a race condition here,
# which would cause buildset being completed twice.
# in this case, the db layer will detect that and raise AlreadyCompleteError
if bsdict['complete']:
return
# mark it as completed in the database
complete_at = epoch2datetime(int(self.master.reactor.seconds()))
try:
yield self.master.db.buildsets.completeBuildset(bsid,
cumulative_results, complete_at=complete_at)
except AlreadyCompleteError:
return
# get the sourcestamps for the message
# get each of the sourcestamps for this buildset (sequentially)
bsdict = yield self.master.db.buildsets.getBuildset(bsid)
sourcestamps = []
for ssid in bsdict['sourcestamps']:
sourcestamps.append(
copy.deepcopy(
(yield self.master.data.get(('sourcestamps', str(ssid))))
)
)
msg = dict(
bsid=bsid,
external_idstring=bsdict['external_idstring'],
reason=bsdict['reason'],
sourcestamps=sourcestamps,
submitted_at=bsdict['submitted_at'],
complete=True,
complete_at=complete_at,
results=cumulative_results)
# TODO: properties=properties)
self.produceEvent(msg, "complete")
def runPopulateSecrets(self):
result = SUCCESS
for path, secretvalue in self.secret_to_be_populated:
if not isinstance(path, str):
raise ValueError("Secret path %s is not a string" % path)
self.secret_to_be_interpolated = secretvalue
res = yield self.downloadFileContentToWorker(path, self.secret_to_be_interpolated, mode=stat.S_IRUSR | stat.S_IWUSR)
result = worst_status(result, res)
defer.returnValue(result)
def maybeBuildsetComplete(self, bsid, _reactor=reactor):
brdicts = yield self.master.db.buildrequests.getBuildRequests(
bsid=bsid, complete=False)
# if there are incomplete buildrequests, bail out
if brdicts:
return
brdicts = yield self.master.db.buildrequests.getBuildRequests(bsid=bsid)
# figure out the overall results of the buildset:
cumulative_results = SUCCESS
for brdict in brdicts:
cumulative_results = worst_status(
cumulative_results, brdict['results'])
# get a copy of the buildset
bsdict = yield self.master.db.buildsets.getBuildset(bsid)
# if it's already completed, we're late to the game, and there's
# nothing to do.
#
# NOTE: there's still a strong possibility of a race condition here,
# which would cause two buildset.$bsid.complete messages to be sent.
# That's an acceptable risk, and a necessary consequence of this
# denormalized representation of a buildset's state.
if bsdict['complete']:
return
# mark it as completed in the database
complete_at = epoch2datetime(int(_reactor.seconds()))
yield self.master.db.buildsets.completeBuildset(bsid,
cumulative_results, complete_at=complete_at)
# get the sourcestamps for the message
# get each of the sourcestamps for this buildset (sequentially)
bsdict = yield self.master.db.buildsets.getBuildset(bsid)
sourcestamps = []
for ssid in bsdict['sourcestamps']:
sourcestamps.append(
copy.deepcopy(
(yield self.master.data.get(('sourcestamps', str(ssid))))
)
)
msg = dict(
bsid=bsid,
external_idstring=bsdict['external_idstring'],
reason=bsdict['reason'],
sourcestamps=sourcestamps,
submitted_at=bsdict['submitted_at'],
complete=True,
complete_at=complete_at,
results=cumulative_results)
# TODO: properties=properties)
self.produceEvent(msg, "complete")
def worstStatus(self, overall_results, rclist):
for was_cb, results in rclist:
if isinstance(results, tuple):
results, _ = results
if not was_cb:
yield self.addLogWithFailure(results)
results = EXCEPTION
overall_results = worst_status(overall_results, results)
defer.returnValue(overall_results)
def worstStatus(self, overall_results, rclist):
for was_cb, results in rclist:
if isinstance(results, tuple):
results, _ = results
if not was_cb:
yield self.addLogWithFailure(results)
results = EXCEPTION
overall_results = worst_status(overall_results, results)
defer.returnValue(overall_results)
def maybeBuildsetComplete(self, bsid, _reactor=reactor):
brdicts = yield self.master.db.buildrequests.getBuildRequests(
bsid=bsid, complete=False)
# if there are incomplete buildrequests, bail out
if brdicts:
return
brdicts = yield self.master.db.buildrequests.getBuildRequests(
bsid=bsid)
# figure out the overall results of the buildset:
cumulative_results = SUCCESS
for brdict in brdicts:
cumulative_results = worst_status(cumulative_results,
brdict['results'])
# get a copy of the buildset
bsdict = yield self.master.db.buildsets.getBuildset(bsid)
# if it's already completed, we're late to the game, and there's
# nothing to do.
#
# NOTE: there's still a strong possibility of a race condition here,
# which would cause two buildset.$bsid.complete messages to be sent.
# That's an acceptable risk, and a necessary consequence of this
# denormalized representation of a buildset's state.
if bsdict['complete']:
return
# mark it as completed in the database
complete_at = epoch2datetime(int(_reactor.seconds()))
yield self.master.db.buildsets.completeBuildset(
bsid, cumulative_results, complete_at=complete_at)
# get the sourcestamps for the message
# get each of the sourcestamps for this buildset (sequentially)
bsdict = yield self.master.db.buildsets.getBuildset(bsid)
sourcestamps = [
copy.deepcopy((yield self.master.data.get(
('sourcestamps', str(ssid)))))
for ssid in bsdict['sourcestamps']
]
msg = dict(bsid=bsid,
external_idstring=bsdict['external_idstring'],
reason=bsdict['reason'],
sourcestamps=sourcestamps,
submitted_at=bsdict['submitted_at'],
complete=True,
complete_at=complete_at,
results=cumulative_results)
# TODO: properties=properties)
self.produceEvent(msg, "complete")
def buildFinished(self, text, results):
"""This method must be called when the last Step has completed. It
marks the Build as complete and returns the Builder to the 'idle'
state.
It takes two arguments which describe the overall build status:
text, results. 'results' is one of the possible results (see buildbot.process.results).
If 'results' is SUCCESS or WARNINGS, we will permit any dependent
builds to start. If it is 'FAILURE', those builds will be
abandoned.
This method never throws."""
try:
self.stopBuildConsumer.stopConsuming()
self.finished = True
if self.conn:
self.subs.unsubscribe()
self.subs = None
self.conn = None
log.msg(" {}: build finished".format(self))
self.results = worst_status(self.results, results)
self.build_status.setText(text)
self.build_status.setResults(self.results)
self.build_status.buildFinished()
eventually(self.releaseLocks)
metrics.MetricCountEvent.log('active_builds', -1)
yield self.master.data.updates.setBuildStateString(
self.buildid, bytes2unicode(" ".join(text)))
yield self.master.data.updates.finishBuild(self.buildid,
self.results)
if self.results == EXCEPTION:
# When a build has an exception, put the worker in quarantine for a few seconds
# to make sure we try next build with another worker
self.workerforbuilder.worker.putInQuarantine()
elif self.results != RETRY:
# This worker looks sane if status is neither retry or exception
# Avoid a race in case the build step reboot the worker
if self.workerforbuilder.worker is not None:
self.workerforbuilder.worker.resetQuarantine()
# mark the build as finished
self.workerforbuilder.buildFinished()
self.builder.buildFinished(self, self.workerforbuilder)
self._tryScheduleBuildsAfterLockUnlock(build_finished=True)
except Exception:
log.err(
None, 'from finishing a build; this is a '
'serious error - please file a bug at http://buildbot.net')
def test_worst_status(self):
res = range(len(results.Results))
res.sort(
cmp=lambda a, b: 1 if (results.worst_status(a, b) == a) else -1)
self.assertEqual(res, [
results.SKIPPED,
results.SUCCESS,
results.WARNINGS,
results.FAILURE,
results.EXCEPTION,
results.RETRY,
results.CANCELLED,
])
def regex_log_evaluator(cmd, _, regexes):
worst = cmd.results()
for err, possible_status in regexes:
# worst_status returns the worse of the two status' passed to it.
# we won't be changing "worst" unless possible_status is worse than it,
# so we don't even need to check the log if that's the case
if worst_status(worst, possible_status) == possible_status:
if isinstance(err, string_types):
err = re.compile(".*%s.*" % err, re.DOTALL)
for l in itervalues(cmd.logs):
if err.search(l.getText()):
worst = possible_status
return worst
def regex_log_evaluator(cmd, _, regexes):
worst = cmd.results()
for err, possible_status in regexes:
# worst_status returns the worse of the two status' passed to it.
# we won't be changing "worst" unless possible_status is worse than it,
# so we don't even need to check the log if that's the case
if worst_status(worst, possible_status) == possible_status:
if isinstance(err, (basestring)):
err = re.compile(".*%s.*" % err, re.DOTALL)
for l in itervalues(cmd.logs):
if err.search(l.getText()):
worst = possible_status
return worst
def buildFinished(self, text, results):
"""This method must be called when the last Step has completed. It
marks the Build as complete and returns the Builder to the 'idle'
state.
It takes two arguments which describe the overall build status:
text, results. 'results' is one of the possible results (see buildbot.process.results).
If 'results' is SUCCESS or WARNINGS, we will permit any dependent
builds to start. If it is 'FAILURE', those builds will be
abandoned."""
try:
self.stopBuildConsumer.stopConsuming()
self.finished = True
if self.conn:
self.subs.unsubscribe()
self.subs = None
self.conn = None
log.msg(" %s: build finished" % self)
self.results = worst_status(self.results, results)
self.build_status.setText(text)
self.build_status.setResults(self.results)
self.build_status.buildFinished()
eventually(self.releaseLocks)
metrics.MetricCountEvent.log('active_builds', -1)
yield self.master.data.updates.setBuildStateString(self.buildid,
bytes2unicode(" ".join(text)))
yield self.master.data.updates.finishBuild(self.buildid, self.results)
if self.results == EXCEPTION:
# When a build has an exception, put the worker in quarantine for a few seconds
# to make sure we try next build with another worker
self.workerforbuilder.worker.putInQuarantine()
elif self.results != RETRY:
# This worker looks sane if status is neither retry or exception
# Avoid a race in case the build step reboot the worker
if self.workerforbuilder.worker is not None:
self.workerforbuilder.worker.resetQuarantine()
# mark the build as finished
self.workerforbuilder.buildFinished()
self.builder.buildFinished(self, self.workerforbuilder)
self._tryScheduleBuildsAfterLockUnlock(build_finished=True)
except Exception:
log.err(None, 'from finishing a build; this is a '
'serious error - please file a bug at http://buildbot.net')
def worstStatus(self, overall_results, rclist, unimportant_brids):
for was_cb, results in rclist:
if isinstance(results, tuple):
results, brids_dict = results
if not was_cb:
yield self.addLogWithFailure(results)
results = EXCEPTION
# brids_dict.values() represents the list of brids kicked by a certain scheduler.
# We want to ignore the result of ANY brid that was kicked off
# by an UNimportant scheduler.
if set(unimportant_brids).issuperset(set(brids_dict.values())):
continue
overall_results = worst_status(overall_results, results)
return overall_results
def worstStatus(self, overall_results, rclist, unimportant_brids):
for was_cb, results in rclist:
if isinstance(results, tuple):
results, brids_dict = results
if not was_cb:
yield self.addLogWithFailure(results)
results = EXCEPTION
# brids_dict.values() rapresents the list of brids kicked by a certain scheduler.
# We want to ignore the result of ANY brid that was kicked off
# by an UNimportant scheduler.
if set(unimportant_brids).issuperset(set(brids_dict.values())):
continue
overall_results = worst_status(overall_results, results)
defer.returnValue(overall_results)
def _updateLastAnnotatedStepStatus(self, status):
debuglog(">>> AnnotatedCommand::_updateLastAnnotatedStepStatus(%r)" %
status)
# Alway update the common annotate command status.
self.annotate_status = results.worst_status(self.annotate_status,
status)
s = self._getLastAnnotatedStep()
if s is not None:
s.updateStatus(status)
if self.halt_on_failure and status in [
results.FAILURE, results.EXCEPTION
]:
if self.cmd:
self.cmd.interrupt("Failed annotated step")
def buildFinished(self, text, results):
"""This method must be called when the last Step has completed. It
marks the Build as complete and returns the Builder to the 'idle'
state.
It takes two arguments which describe the overall build status:
text, results. 'results' is one of the possible results (see buildbot.process.results).
If 'results' is SUCCESS or WARNINGS, we will permit any dependent
builds to start. If it is 'FAILURE', those builds will be
abandoned."""
try:
self.stopBuildConsumer.stopConsuming()
self.finished = True
if self.conn:
self.subs.unsubscribe()
self.subs = None
self.conn = None
log.msg(" %s: build finished" % self)
self.results = worst_status(self.results, results)
self.build_status.setText(text)
self.build_status.setResults(self.results)
self.build_status.buildFinished()
eventually(self.releaseLocks)
metrics.MetricCountEvent.log('active_builds', -1)
yield self.master.data.updates.setBuildStateString(
self.buildid, u'finished')
yield self.master.data.updates.finishBuild(self.buildid,
self.results)
# mark the build as finished
self.workerforbuilder.buildFinished()
self.builder.buildFinished(self, self.workerforbuilder)
except Exception:
log.err(
None, 'from finishing a build; this is a '
'serious error - please file a bug at http://buildbot.net')
def buildFinished(self, text, results):
"""This method must be called when the last Step has completed. It
marks the Build as complete and returns the Builder to the 'idle'
state.
It takes two arguments which describe the overall build status:
text, results. 'results' is one of the possible results (see buildbot.process.results).
If 'results' is SUCCESS or WARNINGS, we will permit any dependent
builds to start. If it is 'FAILURE', those builds will be
abandoned."""
try:
self.stopBuildConsumer.stopConsuming()
self.finished = True
if self.conn:
self.subs.unsubscribe()
self.subs = None
self.conn = None
log.msg(" %s: build finished" % self)
self.results = worst_status(self.results, results)
self.build_status.setText(text)
self.build_status.setResults(self.results)
self.build_status.buildFinished()
eventually(self.releaseLocks)
metrics.MetricCountEvent.log('active_builds', -1)
yield self.master.data.updates.setBuildStateString(self.buildid,
bytes2unicode(" ".join(text)))
yield self.master.data.updates.finishBuild(self.buildid, self.results)
# mark the build as finished
self.workerforbuilder.buildFinished()
self.builder.buildFinished(self, self.workerforbuilder)
except Exception:
log.err(None, 'from finishing a build; this is a '
'serious error - please file a bug at http://buildbot.net')
def updateStatus(self, status):
self.results = results.worst_status(self.results, status)
def runAnnotatedCommands(self):
try:
# Create a preamble log for primary annotate step.
self.preamble_log = yield self.addLog('preamble')
cmd = yield self.makeRemoteShellCommand(command=self.command,
stdioLogName='stdio')
d1 = self.runCommand(cmd)
d2 = defer.maybeDeferred(self._walkOverScheduledAnnotatedSteps)
@d1.addBoth
def cb(r):
self._annotated_finished = True
try:
# In some cases we can get the empty queue after the check.
# Just catch and pass the exception.
if self.annotated_steps:
# Current processing step.
last_step = self.annotated_steps[0]
cmd_results = cmd.results()
# We can get CANCELLED for two cases:
# 1. when UI stop button was pressed. In that case just pass this status
# into the annotated step as the result status.
# 2. and when the remote command was canceled because of failed annotated step.
# For that situation we finish the annotated step with its current status.
if cmd_results == results.CANCELLED and last_step.results in [
results.FAILURE, results.EXCEPTION
]:
# Finish the annotated step with its current status.
last_step.requestFinish()
else:
last_step.requestFinish(cmd_results)
except IndexError:
pass
return r
@d2.addErrback
def cbErrback(r):
debuglog(
"+++ AnnotatedCommand::runAnnotatedCommands(): error callback with exception. "
"Terminate remote command.")
self.annotate_status = results.EXCEPTION
if self.cmd:
self.cmd.interrupt("Annotated step exception")
return r
# Wait until both -- the remote command and the annotated step processing loop -- get finished.
yield defer.DeferredList([d1, d2])
except Exception:
why = failure.Failure()
logging.err(why, "AnnotatedCommand.failed; traceback follows")
yield self.addLogWithFailure(why)
# This case when the remote command has been canceled by failed annotated command.
if cmd.results() == results.CANCELLED and self.annotate_status in [
results.FAILURE, results.EXCEPTION
]:
return self.annotate_status
# Regular case.
return results.worst_status(cmd.results(), self.annotate_status)
def test_worst_status(self):
self.assertEqual(results.WARNINGS,
results.worst_status(results.SUCCESS, results.WARNINGS))
self.assertEqual(results.CANCELLED,
results.worst_status(results.SKIPPED, results.CANCELLED))
def evaluateCommand(self, cmd):
result = super().evaluateCommand(cmd)
if any(self.has_error(line) for line in self.loggedWarnings):
return worst_status(FAILURE, result)
return result
