def decide_and_remove_changes(self, t, important, unimportant):
"""
Based on Scheduler.decide_and_remove_changes.
If we have n or more important changes, we should trigger jobs.
If more than idleTimeout has elapsed since the last change, we should trigger jobs.
"""
if not important:
return None
nImportant = len(important)
if nImportant < self.n:
if not self.idleTimeout:
log.msg("%s: skipping with %i/%i important changes since no idle timeout" %
(self.name, nImportant, self.n))
return
most_recent = max([c.when for c in important])
elapsed = int(now() - most_recent)
if self.idleTimeout and elapsed < self.idleTimeout:
# Haven't hit the timeout yet, so let's wait more
log.msg("%s: skipping with %i/%i important changes since only %i/%is have elapsed" %
(self.name, nImportant, self.n, elapsed, self.idleTimeout))
return now() + (self.idleTimeout - elapsed)
log.msg("%s: triggering with %i/%i important changes since %is have elapsed" % (self.name, nImportant, self.n, elapsed))
else:
log.msg("%s: triggering since we have %i/%i important changes" % (self.name, nImportant, self.n))
return Scheduler.decide_and_remove_changes(self, t, important, unimportant)
def checkActivity(self):
self.activityTimer = None
if self.lastActivity + self.keepaliveInterval < now():
log.msg("BotFactory.checkActivity: nothing from master for "
"%d secs" % (now() - self.lastActivity))
self.perspective.broker.transport.loseConnection()
return
self.startTimers()
def update(self):
if now() < self.when:
next = time.strftime("%H:%M:%S", time.localtime(self.when))
secs = "[%d secs]" % (self.when - now())
self.label.set_text("%s\n%s\n%s" % (self.text, next, secs))
return True # restart timer
else:
# done
self.label.set_text("%s\n[soon]\n[overdue]" % (self.text,))
self.timer = None
return False
def create_builds(self, to_create, t):
db = self.parent.db
for builderName, count in to_create:
ss = self.ssFunc(builderName)
ssid = db.get_sourcestampid(ss, t)
for i in range(0, count):
self.create_buildset(ssid, "scheduler", t, builderNames=[builderName])
# Try again in a bit
self.lastCheck = now()
return now() + self.pollInterval
def get_initial_state(self, max_changeid):
log.msg('%s: get_initial_state()' % self.log_prefix)
# Keep initial state of builders in upstreamBuilders
# and operate on remainingBuilders to simplify comparison
# on reconfig
return {
"upstreamBuilders": self.upstreamBuilders,
"remainingBuilders": self.upstreamBuilders,
"lastCheck": now(),
"lastReset": now(),
}
def reportBuild(self, builder, buildnum):
"""Returns True if this build should be reported for this contact
(eliminating duplicates), and also records the report for later"""
for w, b, n in self.reported_builds:
if b == builder and n == buildnum:
return False
self.reported_builds.append([util.now(), builder, buildnum])
# clean the reported builds
horizon = util.now() - 60
while self.reported_builds and self.reported_builds[0][0] < horizon:
self.reported_builds.pop(0)
# and return True, since this is a new one
return True
def get_sync_connection(self):
# This is a wrapper around spec.get_sync_connection that maintains a
# single connection to the database for synchronous usage. It will get
# a new connection if the existing one has been idle for more than
# max_idle seconds.
if self._nonpool_max_idle is not None:
now = util.now()
if self._nonpool_lastused and self._nonpool_lastused + self._nonpool_max_idle < now:
self._nonpool = None
if not self._nonpool:
self._nonpool = self._spec.get_sync_connection()
self._nonpool_lastused = util.now()
return self._nonpool
def streamProcessStopped(self):
self.process = None
# if the service is stopped, don't try to restart the process
if not self.wantProcess or reactor._stopped:
return
now = util.now()
if now - self.lastStreamProcessStart < \
self.STREAM_GOOD_CONNECTION_TIME:
# bad startup; start the stream process again after a timeout,
# and then increase the timeout
log.msg(
"'gerrit stream-events' failed; restarting after %ds"
% round(self.streamProcessTimeout))
reactor.callLater(
self.streamProcessTimeout, self.startStreamProcess)
self.streamProcessTimeout *= self.STREAM_BACKOFF_EXPONENT
if self.streamProcessTimeout > self.STREAM_BACKOFF_MAX:
self.streamProcessTimeout = self.STREAM_BACKOFF_MAX
else:
# good startup, but lost connection; restart immediately,
# and set the timeout to its minimum
# make sure we log the reconnection, so that it might be detected
# and network connectivity fixed
log.msg("gerrit stream-events lost connection. Reconnecting...")
self.startStreamProcess()
self.streamProcessTimeout = self.STREAM_BACKOFF_MIN
def render(self, request):
status = request.site.buildbot_service.getStatus()
cxt = dict(project_url = status.getProjectURL(),
project_name = status.getProjectName(),
stylesheet = path_to_root(request) + 'default.css',
version = version,
time = time.strftime("%a %d %b %Y %H:%M:%S",
time.localtime(util.now())),
tz = time.tzname[time.localtime()[-1]],
metatags = [],
title = 'BuildBot')
if self.dirs is None:
directory = os.listdir(self.path)
directory.sort()
else:
directory = self.dirs
dirs, files = self._getFilesAndDirectories(directory)
cxt['path'] = cgi.escape(urllib.unquote(request.uri))
cxt['directories'] = dirs
cxt['files'] = files
template = request.site.buildbot_service.templates.get_template("directory.html")
data = template.render(**cxt)
if isinstance(data, unicode):
data = data.encode("utf-8")
return data
def finished(self, sig, rc):
self.elapsedTime = util.now(self._reactor) - self.startTime
log.msg("command finished with signal %s, exit code %s, elapsedTime: %0.6f" % (sig,rc,self.elapsedTime))
for w in self.logFileWatchers:
# this will send the final updates
w.stop()
self._sendBuffers()
if sig is not None:
rc = -1
if self.sendRC:
if sig is not None:
self.sendStatus(
{'header': "process killed by signal %d\n" % sig})
self.sendStatus({'rc': rc})
self.sendStatus({'header': "elapsedTime=%0.6f\n" % self.elapsedTime})
if self.timer:
self.timer.cancel()
self.timer = None
if self.maxTimer:
self.maxTimer.cancel()
self.maxTimer = None
if self.buftimer:
self.buftimer.cancel()
self.buftimer = None
d = self.deferred
self.deferred = None
if d:
d.callback(rc)
else:
log.msg("Hey, command %s finished twice" % self)
def emit_status(self, which):
b = self.getBuilder(which)
str = "%s: " % which
state, builds = b.getState()
str += state
if state == "idle":
last = b.getLastFinishedBuild()
if last:
start,finished = last.getTimes()
str += ", last build %s ago: %s" % \
(self.convertTime(int(util.now() - finished)), " ".join(last.getText()))
if state == "building":
t = []
for build in builds:
step = build.getCurrentStep()
if step:
s = "(%s)" % " ".join(step.getText())
else:
s = "(no current step)"
ETA = build.getETA()
if ETA is not None:
s += " [ETA %s]" % self.convertTime(ETA)
t.append(s)
str += ", ".join(t)
self.send(str)
def stepFinished(self, results):
self.finished = util.now()
self.results = results
cld = [] # deferreds for log compression
logCompressionLimit = self.master.config.logCompressionLimit
for loog in self.logs:
if not loog.isFinished():
loog.finish()
# if log compression is on, and it's a real LogFile,
# HTMLLogFiles aren't files
if logCompressionLimit is not False and \
isinstance(loog, LogFile):
if os.path.getsize(loog.getFilename()) > logCompressionLimit:
loog_deferred = loog.compressLog()
if loog_deferred:
cld.append(loog_deferred)
for r in self.updates.keys():
if self.updates[r] is not None:
self.updates[r].cancel()
del self.updates[r]
watchers = self.finishedWatchers
self.finishedWatchers = []
for w in watchers:
w.callback(self)
if cld:
return defer.DeferredList(cld)
def getBaseRevision(self):
# this depends upon our local clock and the repository's clock being
# reasonably synchronized with each other. We express everything in
# UTC because the '%z' format specifier for strftime doesn't always
# work.
self.baserev = time.strftime("%Y-%m-%d %H:%M:%S +0000", time.gmtime(now()))
return defer.succeed(None)
def _claim_buildreqs(self, t, available_slaves):
# return a dict mapping slave -> (brid,ssid)
now = util.now()
old = now - self.RECLAIM_INTERVAL
requests = self.db.get_unclaimed_buildrequests(self.name, old,
self.master_name,
self.master_incarnation,
t)
assignments = {}
while requests and available_slaves:
sb = self._choose_slave(available_slaves)
if not sb:
log.msg("%s: want to start build, but we don't have a remote"
% self)
break
available_slaves.remove(sb)
breq = self._choose_build(requests)
if not breq:
log.msg("%s: went to start build, but nextBuild said not to"
% self)
break
requests.remove(breq)
merged_requests = [breq]
for other_breq in requests[:]:
if (self.mergeRequests and
self.botmaster.shouldMergeRequests(self, breq, other_breq)
):
requests.remove(other_breq)
merged_requests.append(other_breq)
assignments[sb] = merged_requests
brids = [br.id for br in merged_requests]
self.db.claim_buildrequests(now, self.master_name,
self.master_incarnation, brids, t)
return assignments
def _getBuildable(self, t):
now = util.now()
old = now - self.RECLAIM_INTERVAL
return self.db.get_unclaimed_buildrequests(self.name, old,
self.master_name,
self.master_incarnation,
t)
def command_LAST(self, args):
# FIXME: NEED TO THINK ABOUT!
args = self.splitArgs(args)
if len(args) == 0:
builders = yield self.getAllBuilders()
elif len(args) == 1:
builder = yield self.getBuilder(buildername=args[0])
if not builder:
raise UsageError("no such builder")
builders = [builder]
else:
raise UsageError("try 'last <builder>'")
for builder in builders:
lastBuild = yield self.getLastCompletedBuild(builder['builderid'])
if not lastBuild:
status = "(no builds run since last restart)"
else:
complete_at = lastBuild['complete_at']
if complete_at:
complete_at = util.datetime2epoch(complete_at)
ago = self.convertTime(int(util.now() - complete_at))
else:
ago = "??"
status = lastBuild['state_string']
status = 'last build %s ago: %s' % (ago, status)
self.send("last build [%s]: %s" % (builder['name'], status))
def periodicCheck(_reactor=reactor):
# Measure how much garbage we have
garbage_count = len(gc.garbage)
MetricCountEvent.log('gc.garbage', garbage_count, absolute=True)
if garbage_count == 0:
level = ALARM_OK
else:
level = ALARM_WARN
MetricAlarmEvent.log('gc.garbage', level=level)
if resource:
r = resource.getrusage(resource.RUSAGE_SELF)
attrs = ['ru_utime', 'ru_stime', 'ru_maxrss', 'ru_ixrss', 'ru_idrss',
'ru_isrss', 'ru_minflt', 'ru_majflt', 'ru_nswap',
'ru_inblock', 'ru_oublock', 'ru_msgsnd', 'ru_msgrcv',
'ru_nsignals', 'ru_nvcsw', 'ru_nivcsw']
for i,a in enumerate(attrs):
# Linux versions prior to 2.6.32 didn't report this value, but we
# can calculate it from /proc/<pid>/statm
v = r[i]
if a == 'ru_maxrss' and v == 0:
v = _get_rss() * resource.getpagesize() / 1024
MetricCountEvent.log('resource.%s' % a, v, absolute=True)
MetricCountEvent.log('resource.pagesize', resource.getpagesize(), absolute=True)
# Measure the reactor delay
then = util.now(_reactor)
dt = 0.1
def cb():
now = util.now(_reactor)
delay = (now - then) - dt
MetricTimeEvent.log("reactorDelay", delay)
_reactor.callLater(dt, cb)
def getCommonBuildInfo(self, req, b):
cxt = {}
cxt['number'] = b.getNumber()
if not b.isFinished():
step = b.getCurrentStep()
if not step:
cxt['current_step'] = "[waiting for Lock]"
else:
if step.isWaitingForLocks():
cxt['current_step'] = "%s [waiting for Lock]" % step.getName()
else:
cxt['current_step'] = step.getName()
when = b.getETA()
if when is not None:
cxt['when'] = util.formatInterval(when)
cxt['when_time'] = time.strftime("%H:%M:%S",
time.localtime(time.time() + when))
cxt['result_css'] = "building"
else:
cxt['result_css'] = css_classes[b.getResults()]
(start, end) = b.getTimes()
cxt['start'] = time.ctime(start)
if end:
cxt['end'] = time.ctime(end)
cxt['elapsed'] = util.formatInterval(end - start)
else:
now = util.now()
cxt['elapsed'] = util.formatInterval(now - start)
return cxt
def finish(self):
"""This stops the 'time' metric and marks the step as finished
overall. It should be called after the last .setProgress has been
done for each axis."""
if self.debug: print "StepProgress.finish[%s]" % self.name
self.stopTime = util.now()
self.buildProgress.stepFinished(self.name)
def __init__(self, who, files, comments, isdir=0, links=None,
revision=None, when=None, branch=None, category=None,
revlink='', properties={}, repository='', project=''):
self.who = who
self.comments = comments
self.isdir = isdir
if links is None:
links = []
self.links = links
def none_or_unicode(x):
if x is None: return x
return unicode(x)
self.revision = none_or_unicode(revision)
if when is None:
when = util.now()
self.when = when
self.branch = none_or_unicode(branch)
self.category = none_or_unicode(category)
self.revlink = revlink
self.properties = Properties()
self.properties.update(properties, "Change")
self.repository = repository
self.project = project
# keep a sorted list of the files, for easier display
self.files = files[:]
self.files.sort()
def finishStep(self, section, status=None):
"""Mark the specified step as 'finished.'"""
# Update status if set as an argument.
if status is not None:
section['status'] = status
self.ensureStepIsStarted(section)
# Final update of text.
updateText(section)
# Add timing info.
section['ended'] = section.get('ended', util.now())
started = section['started']
ended = section['ended']
msg = '\n\n' + '-' * 80 + '\n'
msg += '\n'.join([
'started: %s' % time.ctime(started),
'ended: %s' % time.ctime(ended),
'duration: %s' % util.formatInterval(ended - started),
'', # So we get a final \n
])
section['log'].addHeader(msg)
# Change status (unless handling the preamble).
if len(self.sections) != 1:
section['step'].stepFinished(section['status'])
# Finish log.
section['log'].finish()
def fixupLast(self, status=None):
# Potentially start initial section here, as initial section might have
# no output at all.
self.initialSection()
last = self.sections[-1]
# Update status if set as an argument.
if status is not None:
last['status'] = status
# Final update of text.
self.updateText()
# Add timing info.
(start, end) = self.command.step_status.getTimes()
msg = '\n\n' + '-' * 80 + '\n'
if start is None:
msg += 'Not Started\n'
else:
if end is None:
end = util.now()
msg += '\n'.join([
'started: %s' % time.ctime(start),
'ended: %s' % time.ctime(end),
'duration: %s' % util.formatInterval(end - start),
'', # So we get a final \n
])
last['log'].addStdout(msg)
# Change status (unless handling the preamble).
if len(self.sections) != 1:
last['step'].stepFinished(last['status'])
# Finish log.
last['log'].finish()
def streamProcessStopped(self):
self.process = None
# if the service is stopped, don't try to restart the process
if not self.wantProcess:
log.msg("service is not running; not reconnecting")
return
now = util.now()
if now - self.lastStreamProcessStart < \
self.STREAM_GOOD_CONNECTION_TIME:
# bad startup; start the stream process again after a timeout,
# and then increase the timeout
log.msg(
"'gerrit stream-events' failed; restarting after %ds"
% round(self.streamProcessTimeout))
reactor.callLater(
self.streamProcessTimeout, self.startStreamProcess)
self.streamProcessTimeout *= self.STREAM_BACKOFF_EXPONENT
if self.streamProcessTimeout > self.STREAM_BACKOFF_MAX:
self.streamProcessTimeout = self.STREAM_BACKOFF_MAX
else:
# good startup, but lost connection; restart immediately,
# and set the timeout to its minimum
self.startStreamProcess()
self.streamProcessTimeout = self.STREAM_BACKOFF_MIN
def footer(self, s, req):
# TODO: this stuff should be generated by a template of some sort
projectURL = s.getProjectURL()
projectName = s.getProjectName()
data = '<hr /><div class="footer">\n'
welcomeurl = self.path_to_root(req) + "index.html"
data += '[<a href="%s">welcome</a>]\n' % welcomeurl
data += "<br />\n"
data += '<a href="http://buildbot.sourceforge.net/">Buildbot</a>'
data += "-%s " % version
if projectName:
data += "working for the "
if projectURL:
data += "<a href=\"%s\">%s</a> project." % (projectURL,
projectName)
else:
data += "%s project." % projectName
data += "<br />\n"
data += ("Page built: " +
time.strftime("%a %d %b %Y %H:%M:%S",
time.localtime(util.now()))
+ "\n")
data += '</div>\n'
return data
def processRequest(self, t):
db = self.parent.db
state = self.get_state(t)
# Check for new builds completed since lastCheck
lastCheck = state['lastCheck']
remainingBuilders = state['remainingBuilders']
n = now()
newBuilds = self.findNewBuilds(db, t, lastCheck)
state['lastCheck'] = n
for builder, ssid in newBuilds:
if builder in remainingBuilders:
remainingBuilders.remove(builder)
if remainingBuilders:
state['remainingBuilders'] = remainingBuilders
else:
ss = SourceStamp(branch=self.branch)
ssid = db.get_sourcestampid(ss, t)
# Start a build!
log.msg(
'%s: new buildset: branch=%s, ssid=%s, builders: %s' \
% (self.log_prefix, self.branch, ssid,
', '.join(self.builderNames)))
self.create_buildset(ssid, "downstream", t)
# Reset the list of builders we're waiting for
state = self.get_initial_state(None)
self.set_state(t, state)
def content(self, req, cxt):
s = self.step_status
b = s.getBuild()
logs = cxt['logs'] = []
for l in s.getLogs():
# FIXME: If the step name has a / in it, this is broken
# either way. If we quote it but say '/'s are safe,
# it chops up the step name. If we quote it and '/'s
# are not safe, it escapes the / that separates the
# step name from the log number.
logs.append({'has_contents': l.hasContents(),
'name': l.getName(),
'link': req.childLink("logs/%s" % urllib.quote(l.getName())) })
start, end = s.getTimes()
if start:
cxt['start'] = ctime(start)
if end:
cxt['end'] = ctime(end)
cxt['elapsed'] = util.formatInterval(end - start)
else:
cxt['end'] = "Not Finished"
cxt['elapsed'] = util.formatInterval(util.now() - start)
cxt.update(dict(builder_link = path_to_builder(req, b.getBuilder()),
build_link = path_to_build(req, b),
b = b,
s = s,
result_css = css_classes[s.getResults()[0]]))
template = req.site.buildbot_service.templates.get_template("buildstep.html");
return template.render(**cxt)
def request_line(self, build_request, req):
when = time.strftime("%b %d %H:%M:%S", time.localtime(build_request.getSubmitTime()))
delay = util.formatInterval(util.now() - build_request.getSubmitTime())
changes = build_request.source.changes
if changes:
change_strings = []
for c in changes:
change_strings.append("<a href=\"%s\">%s</a>" % (path_to_change(req, c), c.who))
if len(change_strings) == 1:
reason = "change by %s" % change_strings[0]
else:
reason = "changes by %s" % ", ".join(change_strings)
elif build_request.source.revision:
reason = build_request.source.revision
else:
reason = "no changes specified"
if self.builder_control is not None:
cancelURL = path_to_builder(req, self.builder_status) + '/cancelbuild'
cancelButton = '''
<form action="%s" class="command cancelbuild" style="display:inline" method="post">
<input type="hidden" name="id" value="%s" />
<input type="submit" value="Cancel Build" />
</form>''' % (cancelURL, id(build_request))
else:
cancelButton = ""
return "<font size=\"-1\">(%s, waiting %s)</font>%s%s" % (when, delay, cancelButton, reason)
def displayPage(self, request, status, builder_list, all_builds, revisions,
categories, repository, branch, debug_info):
"""Display the console page."""
# Build the main template directory with all the informations we have.
subs = dict()
subs["branch"] = branch or 'trunk'
subs["repository"] = repository
if categories:
subs["categories"] = ' '.join(categories)
subs["time"] = time.strftime("%a %d %b %Y %H:%M:%S",
time.localtime(util.now()))
subs["debugInfo"] = debug_info
subs["ANYBRANCH"] = ANYBRANCH
if builder_list:
builders = builder_list
else:
builders = {}
subs['builders'] = builders
subs['revisions'] = []
# For each revision we show one line
for revision in revisions:
r = {}
# Fill the dictionary with this new information
r['id'] = revision.revision
r['link'] = revision.revlink
if (skia_vars.GetGlobalVariable('commit_bot_username') in revision.who
and 'Author: ' in revision.comments):
who = revision.comments.split('Author: ')[1].split('\n')[0]
who += ' (commit-bot)'
else:
who = revision.who
r['who'] = utils.FixGitSvnEmail(who)
r['date'] = revision.date
r['comments'] = revision.comments
r['repository'] = revision.repository
r['project'] = revision.project
# Display the status for all builders.
(builds, details) = self.displayStatusLine(builder_list,
all_builds,
revision,
debug_info)
r['builds'] = builds
r['details'] = details
# Calculate the td span for the comment and the details.
r["span"] = sum ([len(builder_list[category]) \
for category in builder_list]) + 2
subs['revisions'].append(r)
#
# Display the footer of the page.
#
debug_info["load_time"] = time.time() - debug_info["load_time"]
return subs
def body(self, req):
s = self.step_status
b = s.getBuild()
builder_name = b.getBuilder().getName()
build_num = b.getNumber()
data = ""
data += '<h1>BuildStep <a href="%s">%s</a>:' % (path_to_builder(req, b.getBuilder()), builder_name)
data += '<a href="%s">#%d</a>' % (path_to_build(req, b), build_num)
data += ":%s</h1>\n" % s.getName()
if s.isFinished():
data += "<h2>Finished</h2>\n" "<p>%s</p>\n" % html.escape("%s" % s.getText())
else:
data += "<h2>Not Finished</h2>\n" "<p>ETA %s seconds</p>\n" % s.getETA()
exp = s.getExpectations()
if exp:
data += "<h2>Expectations</h2>\n" "<ul>\n"
for e in exp:
data += "<li>%s: current=%s, target=%s</li>\n" % (html.escape(e[0]), e[1], e[2])
data += "</ul>\n"
(start, end) = s.getTimes()
if not start:
start_text = end_text = elapsed = "Not Started"
else:
start_text = ctime(start)
if end:
end_text = ctime(end)
elapsed = util.formatInterval(end - start)
else:
end_text = "Not Finished"
elapsed = util.formatInterval(util.now() - start)
data += "<h2>Timing</h2>\n"
data += "<table>\n"
data += "<tr><td>Start</td><td>%s</td></tr>\n" % start_text
data += "<tr><td>End</td><td>%s</td></tr>\n" % end_text
data += "<tr><td>Elapsed</td><td>%s</td></tr>\n" % elapsed
data += "</table>\n"
logs = s.getLogs()
if logs:
data += "<h2>Logs</h2>\n" "<ul>\n"
for logfile in logs:
if logfile.hasContents():
# FIXME: If the step name has a / in it, this is broken
# either way. If we quote it but say '/'s are safe,
# it chops up the step name. If we quote it and '/'s
# are not safe, it escapes the / that separates the
# step name from the log number.
logname = logfile.getName()
logurl = req.childLink("logs/%s" % urllib.quote(logname))
data += '<li><a href="%s">%s</a></li>\n' % (logurl, html.escape(logname))
else:
data += "<li>%s</li>\n" % html.escape(logname)
data += "</ul>\n"
return data
def buildStarted(self, build):
"""The Build has been set up and is about to be started. It can now
be safely queried, so it is time to announce the new build."""
self.started = util.now()
# now that we're ready to report status, let the BuilderStatus tell
# the world about us
self.builder.buildStarted(self)
def initialSection(self):
"""Initializes the annotator's sections.
Annotator uses a list of dictionaries which hold information stuch as status
and logs for each added step. This method populates the section list with an
entry referencing the original buildbot step."""
if self.sections:
return
# Add a log section for output before the first section heading.
preamble = self.command.addLog('preamble')
self.addSection(self.command.name, self.command.step_status)
self.sections[0]['log'] = preamble
self.sections[0]['started'] = util.now()
self.cursor = self.sections[0]
def execute(self, *args, **kw):
start_time = util.now()
sleep_time = 0.1
while True:
try:
query_start_time = util.now()
result = self.cursor.execute(*args, **kw)
end_time = util.now()
if end_time - query_start_time > 2:
log.msg("Long query (%is): %s" %
((end_time - query_start_time), str((args, kw))))
return result
except self.dbapi.OperationalError, e:
if e.args[0] == 'database is locked':
# Retry
log.msg("Retrying query %s" % str((args, kw)))
now = util.now()
if start_time + self.max_retry_time < now:
raise TimeoutError("Exceeded timeout trying to do %s" %
str((args, kw)))
self.sleep(sleep_time)
sleep_time = max(self.max_sleep_time, sleep_time * 2)
continue
raise
def __getstate__(self):
d = styles.Versioned.__getstate__(self)
# for now, a serialized Build is always "finished". We will never
# save unfinished builds.
if not self.finished:
d['finished'] = util.now()
# TODO: push an "interrupted" step so it is clear that the build
# was interrupted. The builder will have a 'shutdown' event, but
# someone looking at just this build will be confused as to why
# the last log is truncated.
for k in [
'builder', 'watchers', 'updates', 'finishedWatchers', 'master'
]:
if k in d: del d[k]
return d
def remoteComplete(self, maybeFailure):
if self._startTime and self._remoteElapsed:
delta = (util.now() - self._startTime) - self._remoteElapsed
metrics.MetricTimeEvent.log("RemoteCommand.overhead", delta)
for name, loog in self.logs.items():
if self._closeWhenFinished[name]:
if maybeFailure:
loog = yield self._unwrap(loog)
yield loog.addHeader("\nremoteFailed: %s" % maybeFailure)
else:
log.msg("closing log %s" % loog)
loog.finish()
if maybeFailure:
raise maybeFailure
def _set_wakeup_timer(self):
if not self._timers:
if self._wakeup_timer:
self._wakeup_timer.cancel()
self._wakeup_timer = None
return
when = min(self._timers.values())
# to avoid waking too frequently, this could be:
# delay=max(when-now,OCD_MINIMUM_DELAY)
# but that delays unrelated jobs that want to wake few seconds apart
delay = max(0, when - util.now(self._reactor))
if self._wakeup_timer:
self._wakeup_timer.reset(delay)
else:
self._wakeup_timer = self._reactor.callLater(delay, self._wakeup)
def formatETA(self, prefix, eta):
if eta is None:
return []
if eta < 60:
return ["< 1 min"]
eta_parts = ["~"]
eta_secs = eta
if eta_secs > 3600:
eta_parts.append("%d hrs" % (eta_secs / 3600))
eta_secs %= 3600
if eta_secs > 60:
eta_parts.append("%d mins" % (eta_secs / 60))
eta_secs %= 60
abstime = time.strftime("%H:%M", time.localtime(util.now()+eta))
return [prefix, " ".join(eta_parts), "at %s" % abstime]
def _start(self):
if 'stdio' not in self.logs or 'stdio' not in self.delayedLogs:
log.msg("RemoteCommand (%s) is running a command, but "
"it isn't being logged to anything. This seems unusual." %
self)
self.updates = {}
self._startTime = util.now()
# This method only initiates the remote command.
# We will receive remote_update messages as the command runs.
# We will get a single remote_complete when it finishes.
# We should fire self.deferred when the command is done.
d = self.remote.callRemote("startCommand", self, self.commandID,
self.remote_command, self.args)
return d
def initialSection(self):
if self.sections:
return
# Add a log section for output before the first section heading.
preamble = self.command.addLog('preamble')
self.sections.append({
'name': 'preamble',
'step': self.command.step_status.getBuild().steps[-1],
'log': preamble,
'status': builder.SUCCESS,
'links': [],
'step_summary_text': [],
'step_text': [],
'started': util.now(),
})
def run(self):
if self.lastCheck + self.pollInterval > now():
# Try again later
return (self.lastCheck + self.pollInterval + 1)
db = self.parent.db
to_create = []
for builderName in self.builderNames:
n = len(db.get_pending_brids_for_builder(builderName))
num_to_create = self.numPending - n
if num_to_create <= 0:
continue
to_create.append( (builderName, num_to_create) )
d = db.runInteraction(lambda t: self.create_builds(to_create, t))
return d
def test_get_steps_with_hidden_steps_object(
self, get_logs_mock, get_prepare_url_mock, path_to_step_mock
):
path_to_step_mock.return_value = 'example-path'
get_logs_mock.return_value = {}
get_prepare_url_mock.return_value = {}
start_time = now()
steps_list = [
BuildStepStub("Step 1", start_time - 100, start_time + 100, [results.SUCCESS], True, True, True, False),
BuildStepStub("Step 2", start_time - 50, start_time + 200, [results.EXCEPTION], True, True, True, False),
]
steps = yield get_steps(steps_list, "", None)
self.assertEqual(len(steps), 0)
def getBox(self, status, brcounts):
# getState() returns offline, idle, or building
state, builds = self.original.getState()
# look for upcoming builds. We say the state is "waiting" if the
# builder is otherwise idle and there is a scheduler which tells us a
# build will be performed some time in the near future. TODO: this
# functionality used to be in BuilderStatus.. maybe this code should
# be merged back into it.
upcoming = []
builderName = self.original.getName()
for s in status.getSchedulers():
if builderName in s.listBuilderNames():
upcoming.extend(s.getPendingBuildTimes())
if state == "idle" and upcoming:
state = "waiting"
if state == "building":
text = ["building"]
if builds:
for b in builds:
eta = b.getETA()
text.extend(self.formatETA("ETA in", eta))
elif state == "offline":
text = ["offline"]
elif state == "idle":
text = ["idle"]
elif state == "waiting":
text = ["waiting"]
else:
# just in case I add a state and forget to update this
text = [state]
# TODO: for now, this pending/upcoming stuff is in the "current
# activity" box, but really it should go into a "next activity" row
# instead. The only times it should show up in "current activity" is
# when the builder is otherwise idle.
# are any builds pending? (waiting for a slave to be free)
brcount = brcounts[builderName]
if brcount:
text.append("%d pending" % brcount)
for t in upcoming:
if t is not None:
eta = t - util.now()
text.extend(self.formatETA("next in", eta))
return Box(text, class_="Activity " + state)
def content(self, req, cxt):
s = self.step_status
b = s.getBuild()
logs = cxt['logs'] = []
for l in s.getLogs():
# FIXME: If the step name has a / in it, this is broken
# either way. If we quote it but say '/'s are safe,
# it chops up the step name. If we quote it and '/'s
# are not safe, it escapes the / that separates the
# step name from the log number.
logs.append({
'has_contents':
l.old_hasContents(),
'name':
l.getName(),
'link':
req.childLink("logs/%s" % urllib.quote(l.getName()))
})
stepStatistics = s.getStatistics()
statistics = cxt['statistics'] = []
for stat in stepStatistics:
statistics.append({'name': stat, 'value': stepStatistics[stat]})
start, end = s.getTimes()
if start:
cxt['start'] = ctime(start)
if end:
cxt['end'] = ctime(end)
cxt['elapsed'] = util.formatInterval(end - start)
else:
cxt['end'] = "Not Finished"
cxt['elapsed'] = util.formatInterval(util.now() - start)
cxt.update(
dict(builder_link=path_to_builder(req, b.getBuilder()),
build_link=path_to_build(req, b),
b=b,
s=s,
result_css=css_classes[s.getResults()[0]]))
template = req.site.buildbot_service.templates.get_template(
"buildstep.html")
return template.render(**cxt)
def printStatus(self):
names = self.buildRequests.keys()
names.sort()
for n in names:
if n not in self.outstanding:
# the build is finished, and we have results
code, text = self.results[n]
t = builder.Results[code]
if text:
t += " (%s)" % " ".join(text)
elif self.builds[n]:
t = self.currentStep[n] or "building"
if self.ETA[n]:
t += " [ETA %ds]" % (self.ETA[n] - now())
else:
t = "no build"
self.announce("%s: %s" % (n, t))
self.announce("")
def _loop_start(self):
if self._everything_needs_to_run:
self._everything_needs_to_run = False
self._timers.clear() ; self._set_wakeup_timer()
self._remaining = list(self.get_processors())
else:
self._remaining = []
now = util.now(self._reactor)
all_processors = self.get_processors()
for p in list(self._timers.keys()):
if self._timers[p] <= now:
del self._timers[p]
# don't run a processor that was removed while it still
# had a timer running
if p in all_processors:
self._remaining.append(p)
# consider sorting by 'when'
self._loop_next()
def body(self, request):
"""Calls default body method and prepends the Announce file"""
# Limit access to the X last days. Buildbot doesn't scale well.
# TODO(maruel) Throttle the requests instead or just fix the code to cache
# data.
stop_gap_in_seconds = 60 * 60 * 24 * 2
earliest_accepted_time = util.now() - stop_gap_in_seconds
# Will throw a TypeError if last_time is not a number.
last_time = int(request.args.get('last_time', [0])[0])
if (last_time and last_time < earliest_accepted_time
and not request.args.get('force', [False])[0]):
return """To prevent DOS of the waterfall, heavy request like this
are blocked. If you know what you are doing, ask a Chromium trooper
how to bypass the protection."""
else:
data = waterfall.WaterfallStatusResource.body(self, request)
return "%s %s" % (GetAnnounce(request.site.resource), data)
def asDict(self, request):
builders = request.args.get('builder', ())
build_fields = request.args.get('build_field', ())
current_builds = RequestArgToBool(request, 'current_builds', False)
completed_builds = self._CountOrCachedRequestArg(
request, 'completed_builds')
pending_builds = RequestArgToBool(request, 'pending_builds', False)
slaves = RequestArgToBool(request, 'slaves', False)
builder_names = self.status.getBuilderNames()
if builders:
builder_regex = re.compile('|'.join(
fnmatch.translate(b) for b in builders))
builder_names = [
b for b in builder_names if builder_regex.match(b)
]
# Collect child endpoint data.
wfd = defer.waitForDeferred(
defer.maybeDeferred(JsonResource.asDict, self, request))
yield wfd
response = wfd.getResult()
# Collect builder data.
wfd = defer.waitForDeferred(
defer.gatherResults([
self._getBuilderData(self.status.getBuilder(builder_name),
current_builds, completed_builds,
pending_builds, build_fields)
for builder_name in builder_names
]))
yield wfd
response['builders'] = wfd.getResult()
# Add slave data.
if slaves:
response['slaves'] = self._getAllSlavesData()
# Add timestamp and return.
response['timestamp'] = now()
response['accepting_builds'] = bool(
self.status.master.botmaster.brd.running)
yield response
def getContext(self, request):
status = self.getStatus(request)
rootpath = path_to_root(request)
locale_enc = locale.getdefaultlocale()[1]
if locale_enc is not None:
locale_tz = unicode(time.tzname[time.localtime()[-1]], locale_enc)
else:
locale_tz = unicode(time.tzname[time.localtime()[-1]])
return dict(project_url=status.getProjectURL(),
project_name=status.getProjectName(),
stylesheet=rootpath + 'default.css',
path_to_root=rootpath,
version=version,
time=time.strftime("%a %d %b %Y %H:%M:%S",
time.localtime(util.now())),
tz=locale_tz,
metatags=[],
title=self.getTitle(request),
welcomeurl=rootpath)
def __init__(self,
who,
files,
comments,
isdir=0,
links=[],
revision=None,
when=None,
branch=None):
self.who = who
self.files = files
self.comments = comments
self.isdir = isdir
self.links = links
self.revision = revision
if when is None:
when = util.now()
self.when = when
self.branch = branch
def printStatus(self):
try:
names = sorted(self.buildRequests.keys())
for n in names:
if n not in self.outstanding:
# the build is finished, and we have results
code, text = self.results[n]
t = Results[code]
if text:
t += " ({})".format(" ".join(text))
elif self.builds[n]:
t = self.currentStep[n] or "building"
if self.ETA[n]:
t += " [ETA {}s]".format(self.ETA[n] - now())
else:
t = "no build"
self.announce("{}: {}".format(n, t))
self.announce("")
except Exception:
log.err(None, "printing status")
def periodicCheck(_reactor=reactor):
try:
# Measure how much garbage we have
garbage_count = len(gc.garbage)
MetricCountEvent.log('gc.garbage', garbage_count, absolute=True)
if garbage_count == 0:
level = ALARM_OK
else:
level = ALARM_WARN
MetricAlarmEvent.log('gc.garbage', level=level)
if resource:
r = resource.getrusage(resource.RUSAGE_SELF)
attrs = [
'ru_utime', 'ru_stime', 'ru_maxrss', 'ru_ixrss', 'ru_idrss',
'ru_isrss', 'ru_minflt', 'ru_majflt', 'ru_nswap', 'ru_inblock',
'ru_oublock', 'ru_msgsnd', 'ru_msgrcv', 'ru_nsignals',
'ru_nvcsw', 'ru_nivcsw'
]
for i, a in enumerate(attrs):
# Linux versions prior to 2.6.32 didn't report this value, but we
# can calculate it from /proc/<pid>/statm
v = r[i]
if a == 'ru_maxrss' and v == 0:
v = _get_rss() * resource.getpagesize() / 1024
MetricCountEvent.log('resource.%s' % a, v, absolute=True)
MetricCountEvent.log('resource.pagesize',
resource.getpagesize(),
absolute=True)
# Measure the reactor delay
then = util.now(_reactor)
dt = 0.1
def cb():
now = util.now(_reactor)
delay = (now - then) - dt
MetricTimeEvent.log("reactorDelay", delay)
_reactor.callLater(dt, cb)
except Exception:
log.err(None, "while collecting VM metrics")
def getPending(self, request):
nr = self.build_status.getNumber()
status = self.getStatus(request)
job = status.getBuilder(self.build_status.getBuilder().getName() +
"-job")
builds = []
pending = yield job.getPendingBuildRequestStatuses()
for b in pending:
source = yield b.getSourceStamp()
submitTime = yield b.getSubmitTime()
bsid = yield b.getBsid()
properties = yield \
b.master.db.buildsets.getBuildsetProperties(bsid)
if properties["spawned_by"][0] != nr:
continue
info = {}
info['number'] = "?"
env = info['environment'] = {}
for name, value in properties.items():
value, source = value
if source != ".travis.yml":
continue
env[name] = value
# How long has it been pending?
info['start'] = time.strftime("%b %d %H:%M:%S",
time.localtime(submitTime))
info['elapsed'] = util.formatInterval(util.now() - submitTime)
info['result_css'] = "pending"
builds.append(info)
defer.returnValue(builds)
def __init__(self, who, files, comments, isdir=0, links=None,
revision=None, when=None, branch=None, category=None,
revlink='', properties={}, repository='', project='',
_fromChdict=False):
# skip all this madness if we're being built from the database
if _fromChdict:
return
self.who = who
self.comments = comments
self.isdir = isdir
if links is None:
links = []
self.links = links
def none_or_unicode(x):
if x is None: return x
return unicode(x)
self.revision = none_or_unicode(revision)
now = util.now()
if when is None:
self.when = now
elif when > now:
# this happens when the committing system has an incorrect clock, for example.
# handle it gracefully
log.msg("received a Change with when > now; assuming the change happened now")
self.when = now
else:
self.when = when
self.branch = none_or_unicode(branch)
self.category = none_or_unicode(category)
self.revlink = revlink
self.properties = Properties()
self.properties.update(properties, "Change")
self.repository = repository
self.project = project
# keep a sorted list of the files, for easier display
self.files = files[:]
self.files.sort()
def remoteComplete(self, maybeFailure):
if self._startTime and self._remoteElapsed:
delta = (util.now() - self._startTime) - self._remoteElapsed
metrics.MetricTimeEvent.log("RemoteCommand.overhead", delta)
for name, loog in self.logs.items():
if self._closeWhenFinished[name]:
if maybeFailure:
loog = yield self._unwrap(loog)
yield loog.addHeader("\nremoteFailed: %s" % maybeFailure)
else:
log.msg("closing log %s" % loog)
loog.finish()
if maybeFailure:
# workaround http://twistedmatrix.com/trac/ticket/5507
# CopiedFailure cannot be raised back, this make debug difficult
if isinstance(maybeFailure, pb.CopiedFailure):
maybeFailure.value = RemoteException("%s: %s\n%s" % (
maybeFailure.type, maybeFailure.value, maybeFailure.traceback))
maybeFailure.type = RemoteException
maybeFailure.raiseException()
def __init__(self,
who,
files,
comments,
isdir=0,
links=None,
revision=None,
when=None,
branch=None,
category=None,
revlink='',
properties={},
repository='',
project=''):
self.who = who
self.comments = comments
self.isdir = isdir
if links is None:
links = []
self.links = links
def none_or_unicode(x):
if x is None: return x
return unicode(x)
self.revision = none_or_unicode(revision)
if when is None:
when = util.now()
self.when = when
self.branch = none_or_unicode(branch)
self.category = none_or_unicode(category)
self.revlink = revlink
self.properties = Properties()
self.properties.update(properties, "Change")
self.repository = repository
self.project = project
# keep a sorted list of the files, for easier display
self.files = files[:]
self.files.sort()
def streamProcessStopped(self):
self.process = None
# if the service is stopped, don't try to restart the process
if not self.wantProcess:
log.msg("service is not running; not reconnecting")
return
now = util.now()
if now - self.lastStreamProcessStart < self.STREAM_GOOD_CONNECTION_TIME:
# bad startup; start the stream process again after a timeout, and then
# increase the timeout
log.msg("'gerrit stream-events' failed; restarting after %ds" % round(self.streamProcessTimeout))
reactor.callLater(self.streamProcessTimeout, self.startStreamProcess)
self.streamProcessTimeout *= self.STREAM_BACKOFF_EXPONENT
if self.streamProcessTimeout > self.STREAM_BACKOFF_MAX:
self.streamProcessTimeout = self.STREAM_BACKOFF_MAX
else:
# good startup, but lost connection; restart immediately, and set the timeout
# to its minimum
self.startStreamProcess()
self.streamProcessTimeout = self.STREAM_BACKOFF_MIN
def emit_status(self, which):
b = self.getBuilder(which)
str = "%s: " % which
state, builds = b.getState()
str += state
if state == "idle":
last = b.getLastFinishedBuild()
if last:
start, finished = last.getTimes()
str += ", last build %s secs ago: %s" % \
(int(util.now() - finished), " ".join(last.getText()))
if state == "building":
t = []
for build in builds:
step = build.getCurrentStep()
s = "(%s)" % " ".join(step.getText())
ETA = build.getETA()
if ETA is not None:
s += " [ETA %s]" % self.convertTime(ETA)
t.append(s)
str += ", ".join(t)
self.send(str)
def remaining(self):
if self.startTime == None:
return self.expectedTime
if self.stopTime != None:
return 0 # already finished
# TODO: replace this with cleverness that graphs each metric vs.
# time, then finds the inverse function. Will probably need to save
# a timestamp with each setProgress update, when finished, go back
# and find the 2% transition points, then save those 50 values in a
# list. On the next build, do linear interpolation between the two
# closest samples to come up with a percentage represented by that
# metric.
# TODO: If no other metrics are available, just go with elapsed
# time. Given the non-time-uniformity of text output from most
# steps, this would probably be better than the text-percentage
# scheme currently implemented.
if self.expectedTime is not None:
# fall back to pure time
return self.expectedTime - (util.now() - self.startTime)
return None # no idea
def body(self, req):
s = self.getStatus(req)
data = ""
data += "<h1>Builders</h1>\n"
# TODO: this is really basic. It should be expanded to include a
# brief one-line summary of the builder (perhaps with whatever the
# builder is currently doing)
data += "<ol>\n"
for bname in s.getBuilderNames():
data += (' <li><a href="%s">%s</a></li>\n' % (req.childLink(
urllib.quote(bname, safe='')), urllib.quote(bname, safe='')))
data += "</ol>\n"
# TODO: this stuff should be generated by a template of some sort
projectURL = s.getProjectURL()
projectName = s.getProjectName()
data += '<hr /><div class="footer">\n'
welcomeurl = self.path_to_root(req) + "index.html"
data += '[<a href="%s">welcome</a>]\n' % welcomeurl
data += "<br />\n"
data += '<a href="http://buildbot.sourceforge.net/">Buildbot</a>'
data += "-%s " % version
if projectName:
data += "working for the "
if projectURL:
data += "<a href=\"%s\">%s</a> project." % (projectURL,
projectName)
else:
data += "%s project." % projectName
data += "<br />\n"
data += ("Page built: " + time.strftime(
"%a %d %b %Y %H:%M:%S", time.localtime(util.now())) + "\n")
data += '</div>\n'
return data
def getContext(self, request):
status = self.getStatus(request)
rootpath = path_to_root(request)
locale_enc = locale.getdefaultlocale()[1]
if locale_enc is not None:
locale_tz = unicode(time.tzname[time.localtime()[-1]], locale_enc)
else:
locale_tz = unicode(time.tzname[time.localtime()[-1]])
return dict(title_url = status.getTitleURL(),
title = status.getTitle(),
stylesheet = rootpath + 'default.css',
path_to_root = rootpath,
version = version,
time = time.strftime("%a %d %b %Y %H:%M:%S",
time.localtime(util.now())),
tz = locale_tz,
metatags = [],
pageTitle = self.getPageTitle(request),
welcomeurl = rootpath,
authz = self.getAuthz(request),
request = request,
alert_msg = request.args.get("alert_msg", [""])[0],
)
def __init__(self, who, files, comments, isdir=0, links=None,
revision=None, when=None, branch=None, category=None,
repository='', revlink='', properties={}):
self.who = who
self.comments = comments
self.isdir = isdir
if links is None:
links = []
self.links = links
self.revision = revision
if when is None:
when = util.now()
self.when = when
self.branch = branch
self.category = category
self.repository = repository
self.revlink = revlink
self.properties = Properties()
self.properties.update(properties, "Change")
# keep a sorted list of the files, for easier display
self.files = files[:]
self.files.sort()