def logFinished(self, build, step, log):
""" Update log to be in synchronized, final state """
step.db_step['stdout'] = ''.join(log.readlines(LOG_CHANNEL_STDOUT))
step.db_step['stderr'] = ''.join(log.readlines(LOG_CHANNEL_STDERR))
step.db_step['headers'] = ''.join(log.readlines(LOG_CHANNEL_HEADER))
step.db_step['output'] = ''.join(log.getTextWithHeaders())
self.database.steps.save(step.db_step)
def createSummary(self, log):
my_builder = self.getProperty("builddir")
# Server is forcing a clobber
forcedClobberRe = re.compile('%s:Server is forcing a clobber' %
my_builder)
# We are looking for something like :
# More than 604800.0 seconds have passed since our last clobber
periodicClobberRe = re.compile(
'%s:More than [\d+\.]+ seconds have passed since our last clobber'
% my_builder)
# We don't have clobber data. This usually means we've been purged before
purgedClobberRe = re.compile("%s:Our last clobber date:.*None" %
my_builder)
self.setProperty('forced_clobber', False, 'MozillaClobberer')
self.setProperty('periodic_clobber', False, 'MozillaClobberer')
self.setProperty('purged_clobber', False, 'MozillaClobberer')
clobberType = None
for line in log.readlines():
if forcedClobberRe.search(line):
self.setProperty('forced_clobber', True, 'MozillaClobberer')
clobberType = "forced"
elif periodicClobberRe.search(line):
self.setProperty('periodic_clobber', True, 'MozillaClobberer')
clobberType = "periodic"
elif purgedClobberRe.search(line):
self.setProperty('purged_clobber', True, 'MozillaClobberer')
clobberType = "free-space"
if clobberType != None:
summary = "TinderboxPrint: %s clobber" % clobberType
self.addCompleteLog('clobberer', summary)
def createSummary(self, log):
bytes = ""
diff = ""
for line in log.readlines():
if '__codesize:' in line:
rawBytes = line.split(':')[1].rstrip()
bytes = formatBytes(rawBytes)
elif '__codesizeDiff:' in line:
diffData = line.split(':')[1].rstrip()
# if we anything but '+0' here, we print additional data
if diffData[0:2] != '+0':
diff = diffData
z = 'Z'
zLong = "codesighs"
if self.type == 'base':
z = 'mZ'
zLong = "codesighs_embed"
self.setProperty('testresults', [(z, zLong, rawBytes, bytes)])
if self.tbPrint:
slug = '%s:%s' % (z, bytes)
summary = 'TinderboxPrint:%s\n' % slug
self.addCompleteLog(slug, summary)
if diff:
# buildbot chokes if we put all the data in the short log
slug = '%sdiff' % z
summary = 'TinderboxPrint:%s:%s\n' % (slug, diff)
self.addCompleteLog(slug, summary)
def createSummary(self, log):
my_builder = self.getProperty("builddir")
# Server is forcing a clobber
forcedClobberRe = re.compile('%s:Server is forcing a clobber' % my_builder)
# We are looking for something like :
# More than 604800.0 seconds have passed since our last clobber
periodicClobberRe = re.compile('%s:More than [\d+\.]+ seconds have passed since our last clobber' % my_builder)
# We don't have clobber data. This usually means we've been purged before
purgedClobberRe = re.compile("%s:Our last clobber date:.*None" % my_builder)
self.setProperty('forced_clobber', False, 'MozillaClobberer')
self.setProperty('periodic_clobber', False, 'MozillaClobberer')
self.setProperty('purged_clobber', False, 'MozillaClobberer')
clobberType = None
for line in log.readlines():
if forcedClobberRe.search(line):
self.setProperty('forced_clobber', True, 'MozillaClobberer')
clobberType = "forced"
elif periodicClobberRe.search(line):
self.setProperty('periodic_clobber', True, 'MozillaClobberer')
clobberType = "periodic"
elif purgedClobberRe.search(line):
self.setProperty('purged_clobber', True, 'MozillaClobberer')
clobberType = "free-space"
if clobberType != None:
summary = "TinderboxPrint: %s clobber" % clobberType
self.addCompleteLog('clobberer', summary)
def parse_timings(log):
# scan the log, measure time consumed per test, show a sorted list
# with the most time-consuming test at the top
last_test = None
tests = []
test_re = re.compile(r'^(allmydata\.test\.\S+) \.\.\.')
time_re = re.compile(r'^\(([\d\.]+) secs\)')
for line in log.readlines():
line = line.strip()
mo = test_re.search(line)
if mo:
last_test = mo.group(1)
continue
if not last_test:
continue
mo = time_re.search(line.strip())
if mo:
t0 = float(mo.group(1))
tests.append( (t0, last_test) )
last_test = None
tests.sort()
tests.reverse()
if tests:
timings = "\n".join(["%7s seconds: %s" % (("%.3f" % t[0]), t[1])
for t in tests]) + "\n"
return timings
return None
def createSummary(self, log):
bytes = ""
diff = ""
for line in log.readlines():
if '__codesize:' in line:
rawBytes = line.split(':')[1].rstrip()
bytes = formatBytes(rawBytes)
elif '__codesizeDiff:' in line:
diffData = line.split(':')[1].rstrip()
# if we anything but '+0' here, we print additional data
if diffData[0:2] != '+0':
diff = diffData
z = 'Z'
zLong = "codesighs"
if self.type == 'base':
z = 'mZ'
zLong = "codesighs_embed"
self.setProperty('testresults', [(z, zLong, rawBytes, bytes)])
if self.tbPrint:
slug = '%s:%s' % (z, bytes)
summary = 'TinderboxPrint:%s\n' % slug
self.addCompleteLog(slug, summary)
if diff:
# buildbot chokes if we put all the data in the short log
slug = '%sdiff' % z
summary = 'TinderboxPrint:%s:%s\n' % (slug, diff)
self.addCompleteLog(slug, summary)
def createSummary(self, log):
ver_re = re.compile("wrote '([^']+)' into src/allmydata/_version.py")
for line in log.readlines():
m = ver_re.search(line)
if m:
self.version = m.group(1)
self.setProperty("tahoe-version", self.version)
return
def createSummary(self, log):
summary = []
for line in log.readlines():
if "RETURN:" in line:
summary.append(line.replace("RETURN:", "TinderboxPrint:"))
if "FAIL:" in line:
summary.append(line.replace("FAIL:", "TinderboxPrint:FAIL:"))
self.addCompleteLog('summary', "\n".join(summary))
def createSummary(self, log):
summary = []
for line in log.readlines():
if "RETURN:" in line:
summary.append(line.replace("RETURN:", "TinderboxPrint:"))
if "FAIL:" in line:
summary.append(line.replace("FAIL:", "TinderboxPrint:FAIL:"))
self.addCompleteLog('summary', "\n".join(summary))
def createSummary(self, log):
ver_re = re.compile("^allmydata-tahoe: ([^ ]+)")
for line in log.readlines():
m = ver_re.search(line)
if m:
self.tahoeversion = m.group(1).split(',')[0]
self.setProperty("tahoe-version", self.tahoeversion)
return
if not hasattr(self, 'tahoeversion'):
return FAILURE # "Tahoe version could not be found."
def createSummary(self, log):
self.counts = {}
count_re = re.compile(r"^(\w+): (\d+)$")
for line in log.readlines():
m = count_re.search(line)
if m:
name = m.group(1)
value = int(m.group(2))
self.setProperty("line-count-%s" % name, value)
self.counts[name] = value
def createSummary(self, log):
# create a logfile with the de-duped DeprecationWarning messages
warnings = set()
warn_re = re.compile(r'DeprecationWarning: ')
for line in log.readlines(): # add stderr
line = line.strip()
mo = warn_re.search(line)
if mo:
warnings.add(line)
if warnings:
self.addCompleteLog("warnings", "\n".join(sorted(warnings))+"\n")
def createSummary(self, log):
python_re = re.compile(r'^python: (\S+)\s')
twisted_re = re.compile(r'^buildbot: .* Twisted version: (\S+)')
self.tool_versions = []
for line in log.readlines():
line = line.strip()
mo = python_re.search(line)
if mo:
self.tool_versions.append( ("py", mo.group(1)) )
mo = twisted_re.search(line)
if mo:
self.tool_versions.append( ("tw", mo.group(1)) )
def createSummary(self, log):
clobberTypeRE = re.compile(r"TinderboxPrint: (\w+) clobber")
clobberTypePerformed = None
for line in log.readlines():
m = clobberTypeRE.match(line)
if m:
clobberTypePerformed = m.group(1)
break
for clobberType in ('forced', 'periodic', 'purged'):
wasPerformed = clobberType == clobberTypePerformed
self.setProperty('%s_clobber' % clobberType, wasPerformed, 'MozillaClobberer')
def createSummary(self, log):
clobberTypeRE = re.compile(r"TinderboxPrint: (\w+) clobber")
clobberTypePerformed = None
for line in log.readlines():
m = clobberTypeRE.match(line)
if m:
clobberTypePerformed = m.group(1)
break
for clobberType in ('forced', 'periodic', 'purged'):
wasPerformed = clobberType == clobberTypePerformed
self.setProperty('%s_clobber' % clobberType, wasPerformed,
'MozillaClobberer')
def commandComplete(self, cmd):
log = cmd.logs['stdio']
for line in log.readlines():
mt = self.filename_rx.search(line)
if mt:
filename = mt.group(1).strip()
if self.filesuffix is not None:
filename += self.filesuffix
if self.fileprefix is not None:
filename = self.fileprefix + filename
self.setProperty('filename', filename)
self.setProperty('basefilename', filename[filename.rfind(self.sep) + 1:])
break
Compile.commandComplete(self, cmd)
def createSummary(self, log):
lines=log.readlines()
for line in lines:
if line.startswith('url:'):
items=line.split()
url=items[1]
items=items[2:]
desc=''
for item in items:
desc="%s %s" % (desc,item)
self.addURL(desc, url)
if line.startswith("message:"):
message = line[len("message:"):].strip()
self.messages.append(message)
if line.startswith("buildbot_status:"):
# valid values: [ SUCCESS | FAILURE | WARNINGS ]
self.script_status = line[len("buildbot_status:"):].strip()
def createSummary(self, log):
self.counts = {}
count_re = re.compile(r"^([\w ]+): ([\d\.]+)$")
namemap = {"total files": "count-files",
"total source lines": "source-lines",
"total covered lines": "covered-lines",
"total uncovered lines": "uncovered-lines",
"lines added": "lines-added",
"lines removed": "lines-removed",
"total coverage percentage": "coverage-percentage",
}
for line in log.readlines():
m = count_re.search(line.strip())
if m:
name = namemap.get(m.group(1), m.group(1))
if "percentage" in name:
value = float(m.group(2))
else:
value = int(m.group(2))
self.setProperty("coverage-" + name, value)
self.counts[name] = value
def createSummary(self, log):
summary = "######################## BLOAT STATISTICS\n"
totalLineList = []
leaks = 0
bloat = 0
for line in log.readlines():
summary += line
if leaks == 0 and bloat == 0:
if "TOTAL" in line:
m = re.search('TOTAL\s+(\d+)\s+[\-\d\.]+\%\s+(\d+)',
line)
leaks = int(m.group(1))
bloat = int(m.group(2))
summary += "######################## END BLOAT STATISTICS\n\n"
# Scrape for leak/bloat totals from TOTAL line
# TOTAL 23 0% 876224
summary += "leaks = %d\n" % leaks
summary += "bloat = %d\n" % bloat
leaksAbbr = "%sRLk" % self.testnameprefix
leaksTestname = ("%srefcnt_leaks" % self.testnameprefix).replace(' ', '_')
leaksTestnameLabel = "%srefcnt Leaks" % self.testnameprefix
if self.tbPrint:
tinderLink = tinderboxPrint(leaksTestname,
leaksTestnameLabel,
0,
'bytes',
leaksAbbr,
formatBytes(leaks,3)
)
summary += tinderLink
self.setProperty('leaks',leaks)
self.setProperty('bloat',bloat)
self.setProperty('testresults', [(leaksAbbr, leaksTestname, leaks, formatBytes(leaks,3))])
self.addCompleteLog(leaksAbbr + ":" + formatBytes(leaks,3),
summary)
def createSummary(self, log):
summary = "######################## BLOAT STATISTICS\n"
totalLineList = []
leaks = 0
bloat = 0
for line in log.readlines():
summary += line
if leaks == 0 and bloat == 0:
if "TOTAL" in line:
m = re.search('TOTAL\s+(\d+)\s+[\-\d\.]+\%\s+(\d+)', line)
leaks = int(m.group(1))
bloat = int(m.group(2))
summary += "######################## END BLOAT STATISTICS\n\n"
# Scrape for leak/bloat totals from TOTAL line
# TOTAL 23 0% 876224
summary += "leaks = %d\n" % leaks
summary += "bloat = %d\n" % bloat
leaksAbbr = "%sRLk" % self.testnameprefix
leaksTestname = ("%srefcnt_leaks" % self.testnameprefix).replace(
' ', '_')
leaksTestnameLabel = "%srefcnt Leaks" % self.testnameprefix
if self.tbPrint:
tinderLink = tinderboxPrint(leaksTestname, leaksTestnameLabel, 0,
'bytes', leaksAbbr,
formatBytes(leaks, 3))
summary += tinderLink
self.setProperty('leaks', leaks)
self.setProperty('bloat', bloat)
self.setProperty(
'testresults',
[(leaksAbbr, leaksTestname, leaks, formatBytes(leaks, 3))])
self.addCompleteLog(leaksAbbr + ":" + formatBytes(leaks, 3), summary)
def createSummary(self, log):
leakStats = {}
leakStats['old'] = {}
leakStats['new'] = {}
summary = self.testname + " trace-malloc bloat test: leakstats\n"
lkAbbr = "%sLk" % self.testnameprefix
lkTestname = ("%strace_malloc_leaks" % self.testnameprefix).replace(' ','_')
mhAbbr = "%sMH" % self.testnameprefix
mhTestname = ("%strace_malloc_maxheap" % self.testnameprefix).replace(' ','_')
aAbbr = "%sA" % self.testnameprefix
aTestname = ("%strace_malloc_allocs" % self.testnameprefix).replace(' ','_')
resultSet = 'new'
for line in log.readlines():
summary += line
m = self.leaksAllocsRe.search(line)
if m:
leakStats[resultSet]['leaks'] = m.group(1)
leakStats[resultSet]['leakedAllocs'] = m.group(2)
continue
m = self.heapRe.search(line)
if m:
leakStats[resultSet]['mhs'] = m.group(1)
continue
m = self.bytesAllocsRe.search(line)
if m:
leakStats[resultSet]['bytes'] = m.group(1)
leakStats[resultSet]['allocs'] = m.group(2)
continue
for key in ('leaks', 'leakedAllocs', 'mhs', 'bytes', 'allocs'):
if key not in leakStats['new']:
self.addCompleteLog('summary',
'Unable to parse leakstats output')
return
lk = formatBytes(leakStats['new']['leaks'],3)
mh = formatBytes(leakStats['new']['mhs'],3)
a = formatCount(leakStats['new']['allocs'],3)
self.setProperty('testresults', [ \
(lkAbbr, lkTestname, leakStats['new']['leaks'], lk), \
(mhAbbr, mhTestname, leakStats['new']['mhs'], mh), \
(aAbbr, aTestname, leakStats['new']['allocs'], a)])
self.setProperty('leakStats',leakStats)
slug = "%s: %s, %s: %s, %s: %s" % (lkAbbr, lk, mhAbbr, mh, aAbbr, a)
logText = ""
if self.tbPrint and self.testname.startswith("current"):
logText += tinderboxPrint(lkTestname,
"Total Bytes malloc'ed and not free'd",
0,
"bytes",
lkAbbr,
lk)
logText += tinderboxPrint(mhTestname,
"Maximum Heap Size",
0,
"bytes",
mhAbbr,
mh)
logText += tinderboxPrint(aTestname,
"Allocations - number of calls to malloc and friends",
0,
"count",
aAbbr,
a)
else:
logText += "%s: %s\n%s: %s\n%s: %s\n" % (lkAbbr, lk, mhAbbr, mh, aAbbr, a)
self.addCompleteLog(slug, logText)
def createSummary(self, log):
leakStats = {}
leakStats['old'] = {}
leakStats['new'] = {}
summary = self.testname + " trace-malloc bloat test: leakstats\n"
lkAbbr = "%sLk" % self.testnameprefix
lkTestname = ("%strace_malloc_leaks" % self.testnameprefix).replace(
' ', '_')
mhAbbr = "%sMH" % self.testnameprefix
mhTestname = ("%strace_malloc_maxheap" % self.testnameprefix).replace(
' ', '_')
aAbbr = "%sA" % self.testnameprefix
aTestname = ("%strace_malloc_allocs" % self.testnameprefix).replace(
' ', '_')
resultSet = 'new'
for line in log.readlines():
summary += line
m = self.leaksAllocsRe.search(line)
if m:
leakStats[resultSet]['leaks'] = m.group(1)
leakStats[resultSet]['leakedAllocs'] = m.group(2)
continue
m = self.heapRe.search(line)
if m:
leakStats[resultSet]['mhs'] = m.group(1)
continue
m = self.bytesAllocsRe.search(line)
if m:
leakStats[resultSet]['bytes'] = m.group(1)
leakStats[resultSet]['allocs'] = m.group(2)
continue
for key in ('leaks', 'leakedAllocs', 'mhs', 'bytes', 'allocs'):
if key not in leakStats['new']:
self.addCompleteLog('summary',
'Unable to parse leakstats output')
return
lk = formatBytes(leakStats['new']['leaks'], 3)
mh = formatBytes(leakStats['new']['mhs'], 3)
a = formatCount(leakStats['new']['allocs'], 3)
self.setProperty('testresults', [ \
(lkAbbr, lkTestname, leakStats['new']['leaks'], lk), \
(mhAbbr, mhTestname, leakStats['new']['mhs'], mh), \
(aAbbr, aTestname, leakStats['new']['allocs'], a)])
self.setProperty('leakStats', leakStats)
slug = "%s: %s, %s: %s, %s: %s" % (lkAbbr, lk, mhAbbr, mh, aAbbr, a)
logText = ""
if self.tbPrint and self.testname.startswith("current"):
logText += tinderboxPrint(lkTestname,
"Total Bytes malloc'ed and not free'd",
0, "bytes", lkAbbr, lk)
logText += tinderboxPrint(mhTestname, "Maximum Heap Size", 0,
"bytes", mhAbbr, mh)
logText += tinderboxPrint(
aTestname,
"Allocations - number of calls to malloc and friends", 0,
"count", aAbbr, a)
else:
logText += "%s: %s\n%s: %s\n%s: %s\n" % (lkAbbr, lk, mhAbbr, mh,
aAbbr, a)
self.addCompleteLog(slug, logText)