def renderPageBody(self): test = self.test if self.currentView() == "test_results": return self.renderIndividualTestResults() if self.currentView() == "test_runs": testRuns = self.testManager.database.TestRun.lookupAll(test=test) if not testRuns: return card("No runs of this test") return HtmlGeneration.grid(self.gridForTestList_(testRuns)) if self.currentView() == "test_definition": return card( '<pre class="language-yaml"><code class="line-numbers">%s</code></pre>' % cgi.escape( algebraic_to_json.encode_and_dump_as_yaml(self.testManager.definitionForTest(test)) ) ) if self.currentView() == "test_dependencies": grid = self.allTestDependencyGrid() if not grid: return card("No dependencies") return HtmlGeneration.grid(grid)
def renderPageBody(self): deployments = sorted( self.testManager.database.Deployment.lookupAll(isAlive=True), key=lambda d: d.createdTimestamp) grid = [["COMMIT", "TEST", "BOOTED AT", "UP FOR", "CLIENTS", "", ""]] for d in deployments: row = [] commit = self.testManager.oldestCommitForTest(d.test) repo = commit.repo row.append(self.contextFor(commit).renderLink()) row.append(d.test.testDefinitionSummary.name) row.append(time.asctime(time.gmtime(d.createdTimestamp))) row.append(secondsUpToString(time.time() - d.createdTimestamp)) row.append( str( self.testManager.streamForDeployment( d._identity).clientCount())) row.append(self.connectDeploymentLink(d)) row.append(self.shutdownDeploymentLink(d)) grid.append(row) return HtmlGeneration.grid(grid)
def renderProjectAndFilterCrossGrid(self): projects = set() configurations = set() for t in self.allTests(): if t.testDefinitionSummary.type == "Test" and self.shouldIncludeTest( t): projects.add(t.testDefinitionSummary.project) configurations.add(t.testDefinitionSummary.configuration) renderer = TestGridRenderer.TestGridRenderer( sorted(projects), lambda p: [ t for t in self.allTests() if t.testDefinitionSummary.project == p and self.shouldIncludeTest(t) ], lambda group: self.contextFor( ComboContexts.CommitAndFilter(self.commit, group, "") ).renderNavbarLink(textOverride=group), lambda group, row: self.contextFor( ComboContexts.CommitAndFilter(self.commit, group, row)). urlString(), lambda test: test.testDefinitionSummary.configuration) grid = [["PROJECT"] + renderer.headers()] for p in sorted(projects): gridrow = renderer.gridRow(p) grid.append([ self.contextFor( ComboContexts.CommitAndFilter( self.commit, "", p)).renderLink(textOverride=p) ] + gridrow) return HtmlGeneration.grid(grid)
def artifactsForTestRunGrid(self): testRun = self.testRun grid = [["Artifact", "Size"]] if testRun.test.testDefinitionSummary.type == "Build": for artifact in testRun.test.testDefinitionSummary.artifacts: full_name = testRun.test.testDefinitionSummary.name + ("/" + artifact if artifact else "") build_key = self.renderer.artifactStorage.sanitizeName(full_name) + ".tar.gz" if self.renderer.artifactStorage.build_exists(testRun.test.hash, build_key): grid.append([ HtmlGeneration.link(full_name + ".tar.gz", self.renderer.buildDownloadUrl(testRun.test.hash, build_key)), HtmlGeneration.bytesToHumanSize(self.renderer.artifactStorage.build_size(testRun.test.hash, build_key)) ]) for artifactName, sizeInBytes in self.renderer.artifactStorage.testResultKeysForWithSizes(testRun.test.hash, testRun._identity): name = self.renderer.artifactStorage.unsanitizeName(artifactName) if not name.startswith(ArtifactStorage.TEST_LOG_NAME_PREFIX): grid.append([ HtmlGeneration.link( name, self.renderer.testResultDownloadUrl(testRun._identity, artifactName) ), HtmlGeneration.bytesToHumanSize(sizeInBytes) ]) if not grid: return card("No Test Artifacts produced") return HtmlGeneration.grid(grid)
def renderPageBody(self): headers, grid = self.grid() if not headers: res = card("No repos found") else: res = HtmlGeneration.grid(headers + grid, header_rows=len(headers)) return res
def renderPageBody(self): mm = self.testManager.machine_management if self.options.get("amiLogs"): ami,hash = self.options.get("amiLogs").split("_") url = mm.amiConfigLogUrl(ami,hash) if url: raise cherrypy.HTTPRedirect(url) else: return HtmlGeneration.card("No logs available") if self.options.get("amiSetupScript"): ami,hash = self.options.get("amiSetupScript").split("_") url = mm.amiConfigLogUrl(ami,hash, "InstallScript") if url: raise cherrypy.HTTPRedirect(url) else: return HtmlGeneration.card("No script available") osConfigs = set( list(mm.windowsOsConfigsAvailable) + list(mm.windowsOsConfigsBeingCreated) + list(mm.invalidWindowsOsConfigs) ) grid = [["BaseAmi", "Hash", "Status", "", ""]] for osConfig in sorted(osConfigs, key=lambda c: (c.ami, c.setupHash)): ami,contentHash = osConfig.ami, osConfig.setupHash status = ( "OK" if osConfig in mm.windowsOsConfigsAvailable else "In progress" if osConfig in mm.windowsOsConfigsBeingCreated else "Invalid" ) if status in ("OK", "Invalid"): logsButton = HtmlGeneration.Link( self.withOptions(amiLogs=ami+"_"+contentHash).urlString(), "Logs", is_button=True, button_style='btn-primary btn-xs' ).render() else: logsButton = "" scriptButton = HtmlGeneration.Link( self.withOptions(amiSetupScript=ami+"_"+contentHash).urlString(), "Setup Script", is_button=True, button_style='btn-primary btn-xs' ).render() grid.append([ami,contentHash,status, logsButton, scriptButton]) return HtmlGeneration.grid(grid)
def renderPageBody(self): view = self.currentView() if view == "branches": headers, grid = self.grid() return HtmlGeneration.grid(headers+grid, header_rows=len(headers)) if view == "configuration": return self.configurationView() if view == "logs": return self.logsView()
def logsView(self): grid = [["Timestamp", "Message"]] log = self.repo.branchCreateLogs if not log: return card("No logs so far") while log and len(grid) < 100: grid.append([time.asctime(time.gmtime(log.timestamp)), "<pre>" + cgi.escape(log.msg) + "</pre>"]) log = log.prior return HtmlGeneration.grid(grid)
def renderProjectAndFilterCrossGridOverCommits(self, configFilter): projects = set() for t in self.allTests(): if t.testDefinitionSummary.type == "Test" and self.shouldIncludeTest( t): projects.add(t.testDefinitionSummary.project) commits = [self.commit] while len(commits) < self.commitsToRender( ) and commits[-1].data and commits[-1].data.parents: commits.append(commits[-1].data.parents[-1]) grid = [] renderers = [] for c in commits: def makeRenderer(commit): return TestGridRenderer.TestGridRenderer( sorted(projects), lambda p: [ t for t in self.allTests() if self.shouldIncludeTest( t) and t.testDefinitionSummary.project == p and t. testDefinitionSummary.configuration == configFilter ], lambda group: self.contextFor( ComboContexts.CommitAndFilter(commit, configFilter, group)).renderLink( textOverride=group, includeRepo=False, includeBranch=False), lambda group, row: self.contextFor( ComboContexts.CommitAndFilter(commit, group, row) ).urlString(), lambda test: "") renderers.append(makeRenderer(c)) grid = [[""] + [renderer.headers()[0] for renderer in renderers]] for project in sorted(projects): gridrow = [renderer.gridRow(project)[0] for renderer in renderers] grid.append([ self.contextFor( ComboContexts.CommitAndFilter(self.commit, configFilter, project)).renderLink( textOverride=project) ] + gridrow) return HtmlGeneration.grid(grid)
def renderIndividualTestResults(self): #show broken out tests over the last N commits rows = [self.testRun] def rowLinkFun(row): return self.contextFor(row).renderLink(includeCommit=False, includeTest=False) def testFun(row): return [row] def cellUrlFun(testGroup, row): return None def rowContextFun(row): return row renderer = IndividualTestGridRenderer.IndividualTestGridRenderer( rows, self, testFun, cellUrlFun, rowContextFun ) grid = [["Test Run", "Logs", "Elapsed (Min)", "Status", ""] + renderer.headers()] for testRun in rows: row = [rowLinkFun(testRun),self.renderer.testLogsButton(testRun._identity)] if testRun.endTimestamp > 0.0: elapsed = (testRun.endTimestamp - testRun.startedTimestamp) / 60.0 else: elapsed = (time.time() - testRun.startedTimestamp) / 60.0 row.append("%.2f" % elapsed) if testRun.endTimestamp > 0.0: row.append("passed" if testRun.success else "failed") else: row.append("running") row.append(" ") grid.append(row + renderer.gridRow(testRun)) grid = HtmlGeneration.transposeGrid(grid) return HtmlGeneration.grid(grid, dataTables=True, header_rows=5)
def renderPageBody(self): view = self.options.get("view", "commits") if view == "commits": return self.testDisplayForCommits( self.testManager.commitsToDisplayForBranch( self.branch, self.maxCommitCount())) elif view == "pins": pinGrid = self.pinGridWithUpdateButtons(self.branch) if len(pinGrid) > 1: pinContents = (HtmlGeneration.grid(pinGrid)) else: pinContents = card("Branch has no pins.") return pinContents
def renderRepoReferencesGrid(self): lines = [["refname", "Commit", "Target Branch"]] if not self.commit.data: return card("Commit data not loaded yet.") if not self.commit.data.repos: return card("Commit has no references to external repos.") for refname, repoRef in sorted(self.commit.data.repos.iteritems()): if repoRef.matches.Pin: lines.append([ refname, self.renderPinReference(refname, repoRef), repoRef.branchname() if repoRef.branchname() else "" ]) return HtmlGeneration.grid(lines)
def renderTestSuitesSummary(self, builds=False): commit = self.commit tests = self.allTests() if builds: tests = [ t for t in tests if t.testDefinitionSummary.type == "Build" and self.shouldIncludeTest(t) ] else: tests = [ t for t in tests if t.testDefinitionSummary.type == "Test" and self.shouldIncludeTest(t) ] if not tests: if commit.data.noTestsFound: return card("Commit defined no test definition file.") raw_text, extension = self.testManager.getRawTestFileForCommit( commit) if not raw_text: return card( "Commit defined no tests because the test-definitions file is empty." ) elif commit.data.testDefinitionsError: return card( "<div>Commit defined no tests or builds. Maybe look at the test definitions? Error was</div><pre><code>%s</code></pre>" % commit.data.testDefinitionsError) else: if self.projectFilter and self.configFilter: return card( "Commit defined no %s for project '%s' and configuration '%s'." % ("builds" if builds else "tests", self.projectFilter, self.configFilter)) if self.projectFilter: return card( "Commit defined no %s for project '%s'." % ("builds" if builds else "tests", self.projectFilter)) if self.configFilter: return card( "Commit defined no %s for configuration %s." % ("builds" if builds else "tests", self.configFilter)) return card("Commit defined no %s." % ("builds" if builds else "tests")) tests = sorted(tests, key=lambda test: test.testDefinitionSummary.name) if builds: grid = [[ "BUILD", "HASH", "", "PROJECT", "CONFIGURATION", "STATUS", "STAGE", "RUNS", "RUNTIME", "", "DEPENDENCIES" ]] else: grid = [[ "SUITE", "HASH", "", "PROJECT", "CONFIGURATION", "STATUS", "RUNS", "TARGET_RUNS", "TEST_CT", "FAILURE_CT", "AVG_RUNTIME", "", "DEPENDENCIES" ]] if self.options.get("show_disabled"): grid[0].append("Disabled") grid[0].append("Calculated Priority") for t in tests: row = [] row.append(self.contextFor(t).renderLink(includeCommit=False)) row.append(t.hash[:8]) row.append( HtmlGeneration.Link( self.contextFor(t).bootTestOrEnvUrl(), "BOOT", is_button=True, new_tab=True, button_style=self.renderer.disable_if_cant_write( 'btn-primary btn-xs'))) row.append(t.testDefinitionSummary.project) row.append(t.testDefinitionSummary.configuration) row.append( TestSummaryRenderer.TestSummaryRenderer( [t], "", ignoreIndividualTests=True).renderSummary()) all_tests = list( self.testManager.database.TestRun.lookupAll(test=t)) all_noncanceled_tests = [ testRun for testRun in all_tests if not testRun.canceled ] all_running_tests = [ testRun for testRun in all_noncanceled_tests if testRun.endTimestamp == 0.0 ] finished_tests = [ testRun for testRun in all_noncanceled_tests if testRun.endTimestamp > 0.0 ] if builds: if not all_running_tests or not t.testDefinitionSummary.artifacts: row.append("") else: completed = len(all_running_tests[0].artifactsCompleted) row.append( "%s / %s" % (completed, len(t.testDefinitionSummary.artifacts)) + ([ " (" + x + ")" for x in t.testDefinitionSummary.artifacts ] + [""])[completed]) row.append(str(t.totalRuns)) if not builds: row.append(self.renderIncreaseSuiteTargetCount(t)) if t.totalRuns: if not builds: if t.totalRuns == 1: #don't want to convert these to floats row.append("%d" % t.totalTestCount) row.append("%d" % t.totalFailedTestCount) else: row.append(str(t.totalTestCount / float(t.totalRuns))) row.append( str(t.totalFailedTestCount / float(t.totalRuns))) if finished_tests: row.append( HtmlGeneration.secondsUpToString( sum([ testRun.endTimestamp - testRun.startedTimestamp for testRun in finished_tests ]) / len(finished_tests))) else: row.append("") else: if not builds: row.append("") row.append("") if all_noncanceled_tests: row.append( HtmlGeneration.secondsUpToString( sum([ time.time() - testRun.startedTimestamp for testRun in all_noncanceled_tests ]) / len(all_noncanceled_tests)) + " so far") else: row.append("") runButtons = [] for testRun in all_noncanceled_tests[:5]: runButtons.append( self.renderer.testLogsButton(testRun._identity).render()) if len(all_noncanceled_tests) > 5: runButtons.append(" and %s more" % (len(all_noncanceled_tests) - 5)) row.append(" ".join(runButtons)) row.append(self.testDependencySummary(t)) if self.options.get("show_disabled"): row.append( "Disabled" if t.testDefinitionSummary.disabled else "") row.append(str(t.calculatedPriority)) grid.append(row) return HtmlGeneration.grid(grid)
def renderTestResultsGrid(self): projectFilter = self.projectFilter configFilter = self.configFilter projects = set() configurations = set() for t in self.allTests(): if self.shouldIncludeTest(t): projects.add(t.testDefinitionSummary.project) configurations.add(t.testDefinitionSummary.configuration) if not projectFilter and len(projects) == 1: projectFilter = list(projects)[0] if not configFilter and len(configurations) == 1: configFilter = list(configurations)[0] if not (projectFilter or configFilter): return self.renderProjectAndFilterCrossGrid() if configFilter and not projectFilter: return self.renderProjectAndFilterCrossGridOverCommits( configFilter) if not configurations or not projects: return card("No tests defined.") if configFilter: #show broken out tests over the last N commits rows = [self.commit] while len(rows) < self.commitsToRender( ) and rows[-1].data and rows[-1].data.parents: rows.append(rows[-1].data.parents[-1]) def rowLinkFun(row): return self.contextFor( ComboContexts.CommitAndFilter( row, configFilter, projectFilter)).withOptions(**self.options).renderLink( includeRepo=False, includeBranch=False) def testFun(row): for t in self.testManager.allTestsForCommit(row): if self.shouldIncludeTest( t) and t.testDefinitionSummary.type == "Test": yield t def cellUrlFun(testGroup, row): return self.contextFor( ComboContexts.CommitAndFilter( row, configFilter, projectFilter)).withOptions( **self.options).withOptions( testGroup=testGroup).urlString() def rowContextFun(row): return ComboContexts.CommitAndFilter(row, configFilter, projectFilter) else: #show tests over configurations rows = sorted(configurations) def rowLinkFun(row): return self.contextFor( ComboContexts.CommitAndFilter( self.commit, row, projectFilter)).withOptions( **self.options).renderNavbarLink(textOverride=row) def testFun(row): for t in self.testManager.allTestsForCommit(self.commit): if self.shouldIncludeTest( t ) and t.testDefinitionSummary.type == "Test" and t.testDefinitionSummary.configuration == row: yield t def cellUrlFun(testGroup, row): return self.contextFor( ComboContexts.CommitAndFilter( self.commit, row, projectFilter)).withOptions( **self.options).withOptions( testGroup=testGroup).urlString() def rowContextFun(row): return ComboContexts.CommitAndFilter(self.commit, row, projectFilter) renderer = IndividualTestGridRenderer.IndividualTestGridRenderer( rows, self, testFun, cellUrlFun, rowContextFun) grid = [[""] + renderer.headers()] for row in rows: grid.append([rowLinkFun(row)] + renderer.gridRow(row)) grid = HtmlGeneration.transposeGrid(grid) return HtmlGeneration.grid(grid, dataTables=True)
def renderPageBody(self): machines = self.testManager.database.Machine.lookupAll(isAlive=True) grid = [[ "MachineID", "Hardware", "OS", "UP FOR", "STATUS", "LASTMSG", "COMMIT", "TEST", "LOGS", "CANCEL", "" ]] for m in sorted(machines, key=lambda m: -m.bootTime): row = [] row.append(m.machineId) row.append("%s cores, %s GB" % (m.hardware.cores, m.hardware.ram_gb)) if m.os.matches.WindowsVM: row.append("Win(%s, %s)" % (m.os.ami, m.os.setupHash[:8])) elif m.os.matches.LinuxVM: row.append("Linux(%s, %s)" % (m.os.ami, m.os.setupHash[:8])) elif m.os.matches.LinuxWithDocker: row.append("LinuxDocker()") elif m.os.matches.WindowsWithDocker: row.append("WindowsDocker()") else: row.append("Unknown") row.append( HtmlGeneration.secondsUpToString(time.time() - m.bootTime)) if m.firstHeartbeat < 1.0: row.append( '<span class="octicon octicon-watch" aria-hidden="true"></span>' ) elif time.time() - m.lastHeartbeat < 60: row.append( '<span class="octicon octicon-check" aria-hidden="true"' + ' data-toggle="tooltip" data-placement="right" title="Heartbeat %s seconds ago" ' % (int(time.time() - m.lastHeartbeat)) + '></span>') else: row.append( '<span class="octicon octicon-issue-opened" aria-hidden="true"' + ' data-toggle="tooltip" data-placement="right" title="Heartbeat %s seconds ago" ' % (int(time.time() - m.lastHeartbeat)) + '></span>') row.append(m.lastHeartbeatMsg) tests = self.testManager.database.TestRun.lookupAll( runningOnMachine=m) deployments = self.testManager.database.Deployment.lookupAll( runningOnMachine=m) if len(tests) + len(deployments) > 1: row.append("ERROR: multiple test runs/deployments") elif tests: commit = self.testManager.oldestCommitForTest(tests[0].test) try: row.append(self.contextFor(commit).renderLink()) except: row.append("") row.append( self.renderer.testRunLink( tests[0], tests[0].test.testDefinitionSummary.name)) row.append(self.renderer.testLogsButton(tests[0]._identity)) row.append( self.renderer.cancelTestRunButton(tests[0]._identity)) elif deployments: commit = self.testManager.oldestCommitForTest( deployments[0].test) try: row.append(self.contextFor(commit).renderLink()) except: row.append("") d = deployments[0] row.append("DEPLOYMENT") grid.append(row) return HtmlGeneration.grid(grid)
def renderPageBody(self): if self.options.get("context", "") == "dropdown-menu": items = [] for testRun in self.relevantTestRuns(): for path, sz in self.renderer.artifactStorage.testResultKeysAndSizesForIndividualTest( testRun.test.hash, testRun._identity, self.individualTestName): contents = os.path.basename( path) + " (" + HtmlGeneration.bytesToHumanSize( sz) + ")" if sz: items.append( '<a class="dropdown-item" href="{link}" title="{title}">{contents}</a>' .format(link=self.renderer.testResultDownloadUrl( testRun._identity, path), title=os.path.basename(path), contents=contents)) else: items.append( '<span class="dropdown-item disabled text-muted">{contents}</span>' .format(contents=contents)) return "".join(items) else: grid = [["Test Run", "Failure", "File", "Size"]] for testRun in [ t for t in self.relevantTestRuns() if not t.canceled and t.endTimestamp ]: try: index = testRun.testNames.test_names.index( self.individualTestName) passFail = True if testRun.testFailures[index] else False except: passFail = None if passFail is not None: pathsAndSizes = self.renderer.artifactStorage.testResultKeysAndSizesForIndividualTest( testRun.test.hash, testRun._identity, self.individualTestName) for path, sz in pathsAndSizes: grid.append([ self.contextFor(testRun).renderLink(False, False), "OK" if passFail is True else "FAIL" if passFail is False else "", HtmlGeneration.link( os.path.basename(path), self.renderer.testResultDownloadUrl( testRun._identity, path)), HtmlGeneration.bytesToHumanSize(sz) ]) if not pathsAndSizes: grid.append([ self.contextFor(testRun).renderLink(False, False), "FAIL" if passFail is True else "OK" if passFail is False else "", '<span class="text-muted">%s</span>' % "No artifacts", "" ]) return HtmlGeneration.grid(grid, dataTables=True)
def testDisplayForCommits(self, commits): commit_string = "" detail_divs = "" ids_to_resize = [] branches = {} commits = [c for c in commits if c.data] commit_hashes = {c.hash: c for c in commits} children = {c.hash: [] for c in commits} parents = {} for c in commits: parents[c.hash] = [ p.hash for p in c.data.parents if p.hash in commit_hashes ] for p in parents[c.hash]: children[p].append(c.hash) for c in commits: if not parents[c.hash]: branchname = "branch_%s" % len(branches) commit_string += 'var %s = gitgraph.branch("%s");\n' % ( branchname, branchname) branches[c.hash] = branchname #we need to walk the commits from bottom to top. E.g. the ones with no parents go first. order = {} unordered_parents = {h: set(parents[h]) for h in parents} edges = [h for h in unordered_parents if not unordered_parents[h]] while len(order) < len(commits): e = edges.pop() order[e] = max([order[p] + 1 for p in parents[e]] + [0]) for c in children[e]: unordered_parents[c].discard(e) if not unordered_parents[c]: edges.append(c) commits = sorted(commits, key=lambda c: order[c.hash]) for commit_ix, c in enumerate(commits): commit_string += "//%s -- %s\n" % (commit_ix, c.hash) parentsWeHave = parents[c.hash] if len(parentsWeHave) == 0: #push a commit onto the branch our_branch = branches[c.hash] commit_string += "%s.commit({sha1: '%s', message: '%s', detailId: 'commit_%s'});\n" % ( branches[c.hash], c.hash, c.data.subject.replace("\\", "\\\\").replace( "'", "\\'"), c.hash) elif len(parentsWeHave) == 1: #push a commit onto the branch our_branch = branches[(parentsWeHave[0], c.hash)] commit_string += "%s.commit({sha1: '%s', message: '%s', detailId: 'commit_%s'});\n" % ( our_branch, c.hash, c.data.subject.replace( "\\", "\\\\").replace("'", "\\'"), c.hash) else: our_branch = branches[(parentsWeHave[0], c.hash)] other_branch = branches[(parentsWeHave[1], c.hash)] commit_string += "%s.merge(%s, {sha1: '%s', message: '%s', detailId: 'commit_%s'}).delete();" % ( other_branch, our_branch, c.hash, c.data.subject.replace("\\", "\\\\").replace( "'", "\\'"), c.hash) if len(children[c.hash]) == 0: #nothing to do - this is terminal pass elif len(children[c.hash]) == 1: #one child gets to use this branch branches[(c.hash, children[c.hash][0])] = our_branch else: #this is a fork - one child gets to use the branch, and everyone else needs to get a fork branches[(c.hash, children[c.hash][0])] = our_branch for other_child in children[c.hash][1:]: branchname = "branch_%s" % len(branches) commit_string += 'var %s = %s.branch("%s");\n' % ( branchname, our_branch, branchname) branches[(c.hash, other_child)] = branchname gridRenderer = self.getGridRenderer(commits) grid = [["COMMIT"] + gridRenderer.headers() + [""]] for c in reversed(commits): gridrow = self.getBranchCommitRow(c, gridRenderer) grid.append(gridrow) grid = HtmlGeneration.grid(grid, rowHeightOverride=36) canvas = HtmlGeneration.gitgraph_canvas_setup(commit_string, grid) return detail_divs + canvas