def _haskeyword(self, keyword):
if keyword == 'core':
return self.parent.regrtest.core
if keyword not in ('error', 'ok', 'timeout'):
return super(ReallyRunFileExternal, self).haskeyword(keyword)
if self._resultcache is None:
from pypy.tool.pytest.overview import ResultCache
self.__class__._resultcache = rc = ResultCache()
rc.parselatest()
result = self._resultcache.getlatestrelevant(self.fspath.purebasename)
if not result: return False
if keyword == 'timeout': return result.istimeout()
if keyword == 'error': return result.iserror()
if keyword == 'ok': return result.isok()
assert False, "should not be there"
def __init__(self, resultdir):
self.resultcache = ResultCache(resultdir)
class HtmlReport(object):
def __init__(self, resultdir):
self.resultcache = ResultCache(resultdir)
def parselatest(self):
self.resultcache.parselatest()
#
# rendering
#
def render_latest_table(self, results):
table = html.table(
[html.th(x, align='left')
for x in ("failure", "filename", "revision",
"user", "platform", "elapsed",
"options", "last error line"
)],
)
r = results[:]
def f(x, y):
xnum = x.isok() and 1 or (x.istimeout() and 2 or 3)
ynum = y.isok() and 1 or (y.istimeout() and 2 or 3)
res = -cmp(xnum, ynum)
if res == 0:
return cmp(x['execution-time'], y['execution-time'])
return res
r.sort(f)
for result in r:
table.append(self.render_result_row(result))
return table
def render_result_row(self, result):
dp = py.path.local(result['fspath'])
options = " ".join([x for x in result.get('options', []) if x!= 'core'])
if not options:
options = NBSP
failureratio = 100 * (1.0 - result.ratio_of_passed())
self.data[result.testname] = failureratio
return html.tr(
html.td("%.2f%%" % failureratio,
style = "background-color: %s" % (getresultcolor(result),)),
html.td(self.render_test_references(result)),
html.td(result['pypy-revision']),
html.td(result['userhost'][:15]),
html.td(result['platform']),
html.td("%.2fs" % result['execution-time']),
html.td(options),
html.td(result.repr_short_error() or NBSP)
)
def getrelpath(self, p):
return p.relto(self.indexpath.dirpath())
def render_test_references(self, result):
dest = self.make_single_test_result(result)
modified = result.ismodifiedtest() and " [mod]" or ""
return html.div(html.a(result.path.purebasename + modified,
href=self.getrelpath(dest)),
style="background-color: transparent")
def make_single_test_result(self, result):
cache = self.indexpath.dirpath('.cache', result['userhost'][:15])
cache.ensure(dir=1)
dest = cache.join(result.path.basename).new(ext='.html')
doc = ViewResult(result)
doc.writetopath(dest)
return dest
def getcorelists(self):
def iscore(result):
return 'core' in result.get('options', [])
coretests = []
noncoretests = []
for name in self.resultcache.getnames():
result = self.resultcache.getlatestrelevant(name)
if iscore(result):
coretests.append(result)
else:
noncoretests.append(result)
return coretests, noncoretests
# generate html files
#
def makeindex(self, indexpath, detail="PyPy - latest"):
self.indexpath = indexpath
self.data = {}
doc = Document(title='pypy test results')
body = doc.body
coretests, noncoretests = self.getcorelists()
body.append(html.h2("%s compliance test results - "
"core tests" % detail))
body.append(self.render_test_summary('core', coretests))
body.append(self.render_latest_table(coretests))
body.append(
html.h2("%s compliance test results - non-core tests" % detail))
body.append(self.render_test_summary('noncore', noncoretests))
body.append(self.render_latest_table(noncoretests))
doc.writetopath(indexpath)
datapath = indexpath.dirpath().join('data')
d = datapath.open('w')
print >>d, "data = ",
pprint.pprint(self.data, stream=d)
d.close()
self.data = None
def render_test_summary(self, tag, tests):
ok = len([x for x in tests if x.isok()])
err = len([x for x in tests if x.iserror()])
to = len([x for x in tests if x.istimeout()])
numtests = ok + err + to
assert numtests == len(tests)
assert numtests
t = html.table()
sum100 = numtests / 100.0
def row(*args):
return html.tr(*[html.td(arg) for arg in args])
sum_passed = sum([x.ratio_of_passed() for x in tests])
compliancy = sum_passed/sum100
self.data['%s-compliancy' % tag] = compliancy
t.append(row(html.b("tests compliancy"),
html.b("%.2f%%" % (compliancy,))))
passed = ok/sum100
self.data['%s-passed' % tag] = passed
t.append(row("testmodules passed completely", "%.2f%%" % passed))
failed = err/sum100
self.data['%s-failed' % tag] = failed
t.append(row("testmodules (partially) failed", "%.2f%%" % failed))
timedout = to/sum100
self.data['%s-timedout' % tag] = timedout
t.append(row("testmodules timeout", "%.2f%%" % timedout))
return t
def setup_class(cls):
if not testresultdir.check(dir=1):
py.test.skip("testresult directory not checked out")
cls.rc = ResultCache(testresultdir)
cls.rc.parselatest()
def __init__(self, resultdir):
self.resultcache = ResultCache(resultdir)
class HtmlReport(object):
def __init__(self, resultdir):
self.resultcache = ResultCache(resultdir)
def parselatest(self):
self.resultcache.parselatest()
#
# rendering
#
def render_latest_table(self, results):
table = html.table([
html.th(x, align='left')
for x in ("failure", "filename", "revision", "user", "platform",
"elapsed", "options", "last error line")
], )
r = results[:]
def f(x, y):
xnum = x.isok() and 1 or (x.istimeout() and 2 or 3)
ynum = y.isok() and 1 or (y.istimeout() and 2 or 3)
res = -cmp(xnum, ynum)
if res == 0:
return cmp(x['execution-time'], y['execution-time'])
return res
r.sort(f)
for result in r:
table.append(self.render_result_row(result))
return table
def render_result_row(self, result):
dp = py.path.local(result['fspath'])
options = " ".join(
[x for x in result.get('options', []) if x != 'core'])
if not options:
options = NBSP
failureratio = 100 * (1.0 - result.ratio_of_passed())
self.data[result.testname] = failureratio
return html.tr(
html.td("%.2f%%" % failureratio,
style="background-color: %s" % (getresultcolor(result), )),
html.td(self.render_test_references(result)),
html.td(result['pypy-revision']), html.td(result['userhost'][:15]),
html.td(result['platform']),
html.td("%.2fs" % result['execution-time']), html.td(options),
html.td(result.repr_short_error() or NBSP))
def getrelpath(self, p):
return p.relto(self.indexpath.dirpath())
def render_test_references(self, result):
dest = self.make_single_test_result(result)
#XXX: ask hg for differences between test and vendor branch
modified = result.ismodifiedtest() and " [mod]" or ""
return html.div(html.a(result.path.purebasename + modified,
href=self.getrelpath(dest)),
style="background-color: transparent")
def make_single_test_result(self, result):
cache = self.indexpath.dirpath('.cache', result['userhost'][:15])
cache.ensure(dir=1)
dest = cache.join(result.path.basename).new(ext='.html')
doc = ViewResult(result)
doc.writetopath(dest)
return dest
def getcorelists(self):
def iscore(result):
return 'core' in result.get('options', [])
coretests = []
noncoretests = []
for name in self.resultcache.getnames():
result = self.resultcache.getlatestrelevant(name)
if iscore(result):
coretests.append(result)
else:
noncoretests.append(result)
return coretests, noncoretests
# generate html files
#
def makeindex(self, indexpath, detail="PyPy - latest"):
self.indexpath = indexpath
self.data = {}
doc = Document(title='pypy test results')
body = doc.body
coretests, noncoretests = self.getcorelists()
body.append(
html.h2("%s compliance test results - "
"core tests" % detail))
body.append(self.render_test_summary('core', coretests))
body.append(self.render_latest_table(coretests))
body.append(
html.h2("%s compliance test results - non-core tests" % detail))
body.append(self.render_test_summary('noncore', noncoretests))
body.append(self.render_latest_table(noncoretests))
doc.writetopath(indexpath)
datapath = indexpath.dirpath().join('data')
d = datapath.open('w')
print >> d, "data = ",
pprint.pprint(self.data, stream=d)
d.close()
self.data = None
def render_test_summary(self, tag, tests):
ok = len([x for x in tests if x.isok()])
err = len([x for x in tests if x.iserror()])
to = len([x for x in tests if x.istimeout()])
numtests = ok + err + to
assert numtests == len(tests)
assert numtests
t = html.table()
sum100 = numtests / 100.0
def row(*args):
return html.tr(*[html.td(arg) for arg in args])
sum_passed = sum([x.ratio_of_passed() for x in tests])
compliancy = sum_passed / sum100
self.data['%s-compliancy' % tag] = compliancy
t.append(
row(html.b("tests compliancy"), html.b("%.2f%%" % (compliancy, ))))
passed = ok / sum100
self.data['%s-passed' % tag] = passed
t.append(row("testmodules passed completely", "%.2f%%" % passed))
failed = err / sum100
self.data['%s-failed' % tag] = failed
t.append(row("testmodules (partially) failed", "%.2f%%" % failed))
timedout = to / sum100
self.data['%s-timedout' % tag] = timedout
t.append(row("testmodules timeout", "%.2f%%" % timedout))
return t
