def _haskeyword(self, keyword): if keyword == 'core': return self.parent.regrtest.core if keyword not in ('error', 'ok', 'timeout'): return super(ReallyRunFileExternal, self).haskeyword(keyword) if self._resultcache is None: from pypy.tool.pytest.overview import ResultCache self.__class__._resultcache = rc = ResultCache() rc.parselatest() result = self._resultcache.getlatestrelevant(self.fspath.purebasename) if not result: return False if keyword == 'timeout': return result.istimeout() if keyword == 'error': return result.iserror() if keyword == 'ok': return result.isok() assert False, "should not be there"
def __init__(self, resultdir): self.resultcache = ResultCache(resultdir)
class HtmlReport(object): def __init__(self, resultdir): self.resultcache = ResultCache(resultdir) def parselatest(self): self.resultcache.parselatest() # # rendering # def render_latest_table(self, results): table = html.table( [html.th(x, align='left') for x in ("failure", "filename", "revision", "user", "platform", "elapsed", "options", "last error line" )], ) r = results[:] def f(x, y): xnum = x.isok() and 1 or (x.istimeout() and 2 or 3) ynum = y.isok() and 1 or (y.istimeout() and 2 or 3) res = -cmp(xnum, ynum) if res == 0: return cmp(x['execution-time'], y['execution-time']) return res r.sort(f) for result in r: table.append(self.render_result_row(result)) return table def render_result_row(self, result): dp = py.path.local(result['fspath']) options = " ".join([x for x in result.get('options', []) if x!= 'core']) if not options: options = NBSP failureratio = 100 * (1.0 - result.ratio_of_passed()) self.data[result.testname] = failureratio return html.tr( html.td("%.2f%%" % failureratio, style = "background-color: %s" % (getresultcolor(result),)), html.td(self.render_test_references(result)), html.td(result['pypy-revision']), html.td(result['userhost'][:15]), html.td(result['platform']), html.td("%.2fs" % result['execution-time']), html.td(options), html.td(result.repr_short_error() or NBSP) ) def getrelpath(self, p): return p.relto(self.indexpath.dirpath()) def render_test_references(self, result): dest = self.make_single_test_result(result) modified = result.ismodifiedtest() and " [mod]" or "" return html.div(html.a(result.path.purebasename + modified, href=self.getrelpath(dest)), style="background-color: transparent") def make_single_test_result(self, result): cache = self.indexpath.dirpath('.cache', result['userhost'][:15]) cache.ensure(dir=1) dest = cache.join(result.path.basename).new(ext='.html') doc = ViewResult(result) doc.writetopath(dest) return dest def getcorelists(self): def iscore(result): return 'core' in result.get('options', []) coretests = [] noncoretests = [] for name in self.resultcache.getnames(): result = self.resultcache.getlatestrelevant(name) if iscore(result): coretests.append(result) else: noncoretests.append(result) return coretests, noncoretests # generate html files # def makeindex(self, indexpath, detail="PyPy - latest"): self.indexpath = indexpath self.data = {} doc = Document(title='pypy test results') body = doc.body coretests, noncoretests = self.getcorelists() body.append(html.h2("%s compliance test results - " "core tests" % detail)) body.append(self.render_test_summary('core', coretests)) body.append(self.render_latest_table(coretests)) body.append( html.h2("%s compliance test results - non-core tests" % detail)) body.append(self.render_test_summary('noncore', noncoretests)) body.append(self.render_latest_table(noncoretests)) doc.writetopath(indexpath) datapath = indexpath.dirpath().join('data') d = datapath.open('w') print >>d, "data = ", pprint.pprint(self.data, stream=d) d.close() self.data = None def render_test_summary(self, tag, tests): ok = len([x for x in tests if x.isok()]) err = len([x for x in tests if x.iserror()]) to = len([x for x in tests if x.istimeout()]) numtests = ok + err + to assert numtests == len(tests) assert numtests t = html.table() sum100 = numtests / 100.0 def row(*args): return html.tr(*[html.td(arg) for arg in args]) sum_passed = sum([x.ratio_of_passed() for x in tests]) compliancy = sum_passed/sum100 self.data['%s-compliancy' % tag] = compliancy t.append(row(html.b("tests compliancy"), html.b("%.2f%%" % (compliancy,)))) passed = ok/sum100 self.data['%s-passed' % tag] = passed t.append(row("testmodules passed completely", "%.2f%%" % passed)) failed = err/sum100 self.data['%s-failed' % tag] = failed t.append(row("testmodules (partially) failed", "%.2f%%" % failed)) timedout = to/sum100 self.data['%s-timedout' % tag] = timedout t.append(row("testmodules timeout", "%.2f%%" % timedout)) return t
def setup_class(cls): if not testresultdir.check(dir=1): py.test.skip("testresult directory not checked out") cls.rc = ResultCache(testresultdir) cls.rc.parselatest()
class HtmlReport(object): def __init__(self, resultdir): self.resultcache = ResultCache(resultdir) def parselatest(self): self.resultcache.parselatest() # # rendering # def render_latest_table(self, results): table = html.table([ html.th(x, align='left') for x in ("failure", "filename", "revision", "user", "platform", "elapsed", "options", "last error line") ], ) r = results[:] def f(x, y): xnum = x.isok() and 1 or (x.istimeout() and 2 or 3) ynum = y.isok() and 1 or (y.istimeout() and 2 or 3) res = -cmp(xnum, ynum) if res == 0: return cmp(x['execution-time'], y['execution-time']) return res r.sort(f) for result in r: table.append(self.render_result_row(result)) return table def render_result_row(self, result): dp = py.path.local(result['fspath']) options = " ".join( [x for x in result.get('options', []) if x != 'core']) if not options: options = NBSP failureratio = 100 * (1.0 - result.ratio_of_passed()) self.data[result.testname] = failureratio return html.tr( html.td("%.2f%%" % failureratio, style="background-color: %s" % (getresultcolor(result), )), html.td(self.render_test_references(result)), html.td(result['pypy-revision']), html.td(result['userhost'][:15]), html.td(result['platform']), html.td("%.2fs" % result['execution-time']), html.td(options), html.td(result.repr_short_error() or NBSP)) def getrelpath(self, p): return p.relto(self.indexpath.dirpath()) def render_test_references(self, result): dest = self.make_single_test_result(result) #XXX: ask hg for differences between test and vendor branch modified = result.ismodifiedtest() and " [mod]" or "" return html.div(html.a(result.path.purebasename + modified, href=self.getrelpath(dest)), style="background-color: transparent") def make_single_test_result(self, result): cache = self.indexpath.dirpath('.cache', result['userhost'][:15]) cache.ensure(dir=1) dest = cache.join(result.path.basename).new(ext='.html') doc = ViewResult(result) doc.writetopath(dest) return dest def getcorelists(self): def iscore(result): return 'core' in result.get('options', []) coretests = [] noncoretests = [] for name in self.resultcache.getnames(): result = self.resultcache.getlatestrelevant(name) if iscore(result): coretests.append(result) else: noncoretests.append(result) return coretests, noncoretests # generate html files # def makeindex(self, indexpath, detail="PyPy - latest"): self.indexpath = indexpath self.data = {} doc = Document(title='pypy test results') body = doc.body coretests, noncoretests = self.getcorelists() body.append( html.h2("%s compliance test results - " "core tests" % detail)) body.append(self.render_test_summary('core', coretests)) body.append(self.render_latest_table(coretests)) body.append( html.h2("%s compliance test results - non-core tests" % detail)) body.append(self.render_test_summary('noncore', noncoretests)) body.append(self.render_latest_table(noncoretests)) doc.writetopath(indexpath) datapath = indexpath.dirpath().join('data') d = datapath.open('w') print >> d, "data = ", pprint.pprint(self.data, stream=d) d.close() self.data = None def render_test_summary(self, tag, tests): ok = len([x for x in tests if x.isok()]) err = len([x for x in tests if x.iserror()]) to = len([x for x in tests if x.istimeout()]) numtests = ok + err + to assert numtests == len(tests) assert numtests t = html.table() sum100 = numtests / 100.0 def row(*args): return html.tr(*[html.td(arg) for arg in args]) sum_passed = sum([x.ratio_of_passed() for x in tests]) compliancy = sum_passed / sum100 self.data['%s-compliancy' % tag] = compliancy t.append( row(html.b("tests compliancy"), html.b("%.2f%%" % (compliancy, )))) passed = ok / sum100 self.data['%s-passed' % tag] = passed t.append(row("testmodules passed completely", "%.2f%%" % passed)) failed = err / sum100 self.data['%s-failed' % tag] = failed t.append(row("testmodules (partially) failed", "%.2f%%" % failed)) timedout = to / sum100 self.data['%s-timedout' % tag] = timedout t.append(row("testmodules timeout", "%.2f%%" % timedout)) return t