def createMasterReport(self, outdir):
report_filename = "index.html"
f = open(os.path.join(outdir, report_filename), 'w')
f.write('<!DOCTYPE html><html><body><head><meta charset="utf-8" /><style lang="css">%s %s</style><script language="javascript">%s</script></head>' % (FuzzingFactory.css_common, FuzzingFactory.css_master, self.js_master()))
f.write('<h1>WebSockets Protocol Test Report</h1>')
f.write('<p id="intro">Test summary report generated on</p>')
f.write('<p id="intro" style="margin-left: 80px;"><i>%s</i></p>' % utcnow())
f.write('<p id="intro">by <a href="%s">Autobahn</a> WebSockets.</p>' % "http://www.tavendo.de/autobahn")
f.write('<p id="intro"><a href="#" onclick="toggleClose();">Toggle Close Results</a></p>')
f.write('<h2>Test Results</h2>')
f.write('<table id="agent_case_results">')
## sorted list of agents for which test cases where run
##
agentList = sorted(self.agents.keys())
## create list ordered list of case Ids
##
cl = []
for c in Cases:
t = caseClasstoIdTuple(c)
cl.append((t, caseIdTupletoId(t)))
cl = sorted(cl)
caseList = []
for c in cl:
caseList.append(c[1])
lastCaseCategory = None
lastCaseSubCategory = None
for caseId in caseList:
caseCategoryIndex = caseId.split('.')[0]
caseCategory = CaseCategories.get(caseCategoryIndex, "Misc")
caseSubCategoryIndex = '.'.join(caseId.split('.')[:2])
caseSubCategory = CaseSubCategories.get(caseSubCategoryIndex, None)
## Category row
##
if caseCategory != lastCaseCategory:
f.write('<tr id="case_category_row">')
f.write('<td id="case_category">%s %s</td>' % (caseCategoryIndex, caseCategory))
for agentId in agentList:
f.write('<td class="agent close_flex" colspan="2">%s</td>' % agentId)
f.write('</tr>')
lastCaseCategory = caseCategory
lastCaseSubCategory = None
if caseSubCategory != lastCaseSubCategory:
f.write('<tr id="case_subcategory_row">')
f.write('<td class="case_subcategory" colspan="%d">%s %s</td>' % (len(agentList)*2 + 1, caseSubCategoryIndex, caseSubCategory))
lastCaseSubCategory = caseSubCategory
f.write('<tr id="agent_case_result_row">')
f.write('<td id="case"><a href="#case_desc_%s">Case %s</a></td>' % (caseId.replace('.', '_'), caseId))
## Agent/Case Result
##
for agentId in agentList:
if self.agents[agentId].has_key(caseId):
case = self.agents[agentId][caseId]
agent_case_report_file = self.makeAgentCaseReportFilename(agentId, caseId)
if case["behavior"] == Case.OK:
td_text = "Pass"
td_class = "case_ok"
elif case["behavior"] == Case.NON_STRICT:
td_text = "Non-Strict"
td_class = "case_non_strict"
elif case["behavior"] == Case.NO_CLOSE:
td_text = "No Close"
td_class = "case_no_close"
else:
td_text = "Fail"
td_class = "case_failed"
if case["behaviorClose"] == Case.OK:
ctd_text = "%s" % str(case["remoteCloseCode"])
ctd_class = "case_ok"
elif case["behaviorClose"] == Case.FAILED_BY_CLIENT:
ctd_text = "%s" % str(case["remoteCloseCode"])
ctd_class = "case_almost"
elif case["behaviorClose"] == Case.WRONG_CODE:
ctd_text = "%s" % str(case["remoteCloseCode"])
ctd_class = "case_non_strict"
elif case["behaviorClose"] == Case.UNCLEAN:
ctd_text = "Unclean"
ctd_class = "case_failed"
else:
ctd_text = "Fail"
ctd_class = "case_failed"
if case["reportTime"]:
f.write('<td class="%s"><a href="%s">%s</a><br/><span id="case_duration">%s ms</span></td><td class="close close_hide %s"><span class="close_code">%s</span></td>' % (td_class, agent_case_report_file, td_text, case["duration"],ctd_class,ctd_text))
else:
f.write('<td class="%s"><a href="%s">%s</a></td><td class="close close_hide %s"><span class="close_code">%s</span></td>' % (td_class, agent_case_report_file, td_text,ctd_class,ctd_text))
else:
f.write('<td class="case_missing close_flex" colspan="2">Missing</td>')
f.write("</tr>")
f.write("</table>")
f.write('<h2>Test Cases</h2>')
for caseId in caseList:
CCase = CasesById[caseId]
f.write('<a name="case_desc_%s"></a>' % caseId.replace('.', '_'))
f.write('<h3 id="case_desc_title">Case %s</h2>' % caseId)
f.write('<p id="case_desc"><i>Description</i><br/><br/> %s</p>' % CCase.DESCRIPTION)
f.write('<p id="case_expect"><i>Expectation</i><br/><br/> %s</p>' % CCase.EXPECTATION)
f.write("</body></html>")
f.close()
return report_filename
def createMasterReportHTML(self, outdir):
"""
Create report master HTML file.
:param outdir: Directory where to create file.
:type outdir: str
:returns: str -- Name of created file.
"""
## open report file in create / write-truncate mode
##
report_filename = "index.html"
f = open(os.path.join(outdir, report_filename), 'w')
## write HTML
##
f.write('<!DOCTYPE html>\n')
f.write('<html>\n')
f.write(' <head>\n')
f.write(' <meta charset="utf-8" />\n')
f.write(' <style lang="css">%s</style>\n' % CSS_COMMON)
f.write(' <style lang="css">%s</style>\n' % CSS_MASTER_REPORT)
f.write(' <script language="javascript">%s</script>\n' % JS_MASTER_REPORT % {"agents_cnt": len(self.agents.keys())})
f.write(' </head>\n')
f.write(' <body>\n')
f.write(' <a href="#"><div id="toggle_button" class="unselectable" onclick="toggleClose();">Toggle Details</div></a>\n')
f.write(' <a name="top"></a>\n')
f.write(' <br/>\n')
## top logos
f.write(' <center><a href="http://autobahn.ws/testsuite" title="Autobahn WebSockets Testsuite"><img src="http://autobahn.ws/static/img/ws_protocol_test_report.png" border="0" width="820" height="46" alt="Autobahn WebSockets Testsuite Report"></img></a></center>\n')
f.write(' <center><a href="http://autobahn.ws" title="Autobahn WebSockets"> <img src="http://autobahn.ws/static/img/ws_protocol_test_report_autobahn.png" border="0" width="300" height="68" alt="Autobahn WebSockets"> </img></a></center>\n')
## write report header
##
f.write(' <div id="master_report_header" class="block">\n')
f.write(' <p id="intro">Summary report generated on %s (UTC) by <a href="%s">Autobahn WebSockets Testsuite</a> v%s/v%s.</p>\n' % (utcnow(), "http://autobahn.ws/testsuite", autobahntestsuite.version, autobahn.version))
f.write("""
<table id="case_outcome_desc">
<tr>
<td class="case_ok">Pass</td>
<td class="outcome_desc">Test case was executed and passed successfully.</td>
</tr>
<tr>
<td class="case_non_strict">Non-Strict</td>
<td class="outcome_desc">Test case was executed and passed non-strictly.
A non-strict behavior is one that does not adhere to a SHOULD-behavior as described in the protocol specification or
a well-defined, canonical behavior that appears to be desirable but left open in the protocol specification.
An implementation with non-strict behavior is still conformant to the protocol specification.</td>
</tr>
<tr>
<td class="case_failed">Fail</td>
<td class="outcome_desc">Test case was executed and failed. An implementation which fails a test case - other
than a performance/limits related one - is non-conforming to a MUST-behavior as described in the protocol specification.</td>
</tr>
<tr>
<td class="case_info">Info</td>
<td class="outcome_desc">Informational test case which detects certain implementation behavior left unspecified by the spec
but nevertheless potentially interesting to implementors.</td>
</tr>
<tr>
<td class="case_missing">Missing</td>
<td class="outcome_desc">Test case is missing, either because it was skipped via the test suite configuration
or deactivated, i.e. because the implementation does not implement the tested feature or breaks during running
the test case.</td>
</tr>
</table>
""")
f.write(' </div>\n')
## write big agent/case report table
##
f.write(' <table id="agent_case_results">\n')
## sorted list of agents for which test cases where run
##
agentList = sorted(self.agents.keys())
## create list ordered list of case Ids
##
cl = []
for c in Cases:
t = self.CaseSet.caseClasstoIdTuple(c)
cl.append((t, self.CaseSet.caseIdTupletoId(t)))
cl = sorted(cl)
caseList = []
for c in cl:
caseList.append(c[1])
lastCaseCategory = None
lastCaseSubCategory = None
for caseId in caseList:
caseCategoryIndex = caseId.split('.')[0]
caseCategory = CaseCategories.get(caseCategoryIndex, "Misc")
caseSubCategoryIndex = '.'.join(caseId.split('.')[:2])
caseSubCategory = CaseSubCategories.get(caseSubCategoryIndex, None)
## Category/Agents row
##
if caseCategory != lastCaseCategory or (self.repeatAgentRowPerSubcategory and caseSubCategory != lastCaseSubCategory):
f.write(' <tr class="case_category_row">\n')
f.write(' <td class="case_category">%s %s</td>\n' % (caseCategoryIndex, caseCategory))
for agentId in agentList:
f.write(' <td class="agent close_flex" colspan="2">%s</td>\n' % agentId)
f.write(' </tr>\n')
lastCaseCategory = caseCategory
lastCaseSubCategory = None
## Subcategory row
##
if caseSubCategory != lastCaseSubCategory:
f.write(' <tr class="case_subcategory_row">\n')
f.write(' <td class="case_subcategory" colspan="%d">%s %s</td>\n' % (len(agentList) * 2 + 1, caseSubCategoryIndex, caseSubCategory))
f.write(' </tr>\n')
lastCaseSubCategory = caseSubCategory
## Cases row
##
f.write(' <tr class="agent_case_result_row">\n')
f.write(' <td class="case"><a href="#case_desc_%s">Case %s</a></td>\n' % (caseId.replace('.', '_'), caseId))
## Case results
##
for agentId in agentList:
if self.agents[agentId].has_key(caseId):
case = self.agents[agentId][caseId]
if case["behavior"] != Case.UNIMPLEMENTED:
agent_case_report_file = self.makeAgentCaseReportFilename(agentId, caseId, ext = 'html')
if case["behavior"] == Case.OK:
td_text = "Pass"
td_class = "case_ok"
elif case["behavior"] == Case.NON_STRICT:
td_text = "Non-Strict"
td_class = "case_non_strict"
elif case["behavior"] == Case.NO_CLOSE:
td_text = "No Close"
td_class = "case_no_close"
elif case["behavior"] == Case.INFORMATIONAL:
td_text = "Info"
td_class = "case_info"
else:
td_text = "Fail"
td_class = "case_failed"
if case["behaviorClose"] == Case.OK:
ctd_text = "%s" % str(case["remoteCloseCode"])
ctd_class = "case_ok"
elif case["behaviorClose"] == Case.FAILED_BY_CLIENT:
ctd_text = "%s" % str(case["remoteCloseCode"])
ctd_class = "case_almost"
elif case["behaviorClose"] == Case.WRONG_CODE:
ctd_text = "%s" % str(case["remoteCloseCode"])
ctd_class = "case_non_strict"
elif case["behaviorClose"] == Case.UNCLEAN:
ctd_text = "Unclean"
ctd_class = "case_failed"
elif case["behaviorClose"] == Case.INFORMATIONAL:
ctd_text = "%s" % str(case["remoteCloseCode"])
ctd_class = "case_info"
else:
ctd_text = "Fail"
ctd_class = "case_failed"
detail = ""
if case["reportTime"]:
detail += "%d ms" % case["duration"]
if case["reportCompressionRatio"] and case["trafficStats"] is not None:
crIn = case["trafficStats"]["incomingCompressionRatio"]
crOut = case["trafficStats"]["outgoingCompressionRatio"]
detail += " [%s/%s]" % ("%.3f" % crIn if crIn is not None else "-", "%.3f" % crOut if crOut is not None else "-")
if detail != "":
f.write(' <td class="%s"><a href="%s">%s</a><br/><span class="case_duration">%s</span></td><td class="close close_hide %s"><span class="close_code">%s</span></td>\n' % (td_class, agent_case_report_file, td_text, detail, ctd_class, ctd_text))
else:
f.write(' <td class="%s"><a href="%s">%s</a></td><td class="close close_hide %s"><span class="close_code">%s</span></td>\n' % (td_class, agent_case_report_file, td_text, ctd_class, ctd_text))
else:
f.write(' <td class="case_unimplemented close_flex" colspan="2">Unimplemented</td>\n')
else:
f.write(' <td class="case_missing close_flex" colspan="2">Missing</td>\n')
f.write(" </tr>\n")
f.write(" </table>\n")
f.write(" <br/><hr/>\n")
## Case descriptions
##
f.write(' <div id="test_case_descriptions">\n')
for caseId in caseList:
CCase = self.CaseSet.CasesById[caseId]
f.write(' <br/>\n')
f.write(' <a name="case_desc_%s"></a>\n' % caseId.replace('.', '_'))
f.write(' <h2>Case %s</h2>\n' % caseId)
f.write(' <a class="up" href="#top">Up</a>\n')
f.write(' <p class="case_text_block case_desc"><b>Case Description</b><br/><br/>%s</p>\n' % CCase.DESCRIPTION)
f.write(' <p class="case_text_block case_expect"><b>Case Expectation</b><br/><br/>%s</p>\n' % CCase.EXPECTATION)
f.write(' </div>\n')
f.write(" <br/><hr/>\n")
## end of HTML
##
f.write(" </body>\n")
f.write("</html>\n")
## close created HTML file and return filename
##
f.close()
return report_filename
def createMasterReport(self, outdir):
report_filename = "index.html"
f = open(os.path.join(outdir, report_filename), 'w')
f.write('<!DOCTYPE html><html><body><head><meta charset="utf-8" /><style lang="css">%s %s</style></head>' % (FuzzingFactory.css_common, FuzzingFactory.css_master))
f.write('<h1>WebSockets Protocol Test Report</h1>')
f.write('<p id="intro">Test summary report generated on</p>')
f.write('<p id="intro" style="margin-left: 80px;"><i>%s</i></p>' % getUtcNow())
f.write('<p id="intro">by <a href="%s">Autobahn</a> WebSockets.</p>' % "http://www.tavendo.de/autobahn")
f.write('<h2>Test Results</h2>')
f.write('<table id="agent_case_results">')
## sorted list of agents for which test cases where run
##
agentList = sorted(self.agents.keys())
## create list of case indexes order by case ID
##
cl = []
i = 1
for c in Cases:
cl.append((caseClasstoIdTuple(c) , i))
i += 1
cl = sorted(cl)
caseList = []
for c in cl:
caseList.append(c[1])
lastCaseCategory = None
lastCaseSubCategory = None
for caseNo in caseList:
## Case ID and category
##
caseId = caseClasstoId(Cases[caseNo - 1])
caseCategoryIndex = caseId.split('.')[0]
caseCategory = CaseCategories.get(caseCategoryIndex, "Misc")
caseSubCategoryIndex = '.'.join(caseId.split('.')[:2])
caseSubCategory = CaseSubCategories.get(caseSubCategoryIndex, None)
## Category row
##
if caseCategory != lastCaseCategory:
f.write('<tr id="case_category_row">')
f.write('<td id="case_category">%s %s</td>' % (caseCategoryIndex, caseCategory))
for agentId in agentList:
f.write('<td id="agent">%s</td>' % agentId)
f.write('</tr>')
lastCaseCategory = caseCategory
lastCaseSubCategory = None
if caseSubCategory != lastCaseSubCategory:
f.write('<tr id="case_subcategory_row">')
f.write('<td id="case_subcategory" colspan="%d">%s %s</td>' % (len(agentList) + 1, caseSubCategoryIndex, caseSubCategory))
lastCaseSubCategory = caseSubCategory
f.write('<tr id="agent_case_result_row">')
f.write('<td id="case"><a href="#case_desc_%d">Case %s</a></td>' % (caseNo, caseId))
## Agent/Case Result
##
for agentId in agentList:
if self.agents[agentId].has_key(caseNo):
case = self.agents[agentId][caseNo]
agent_case_report_file = self.makeAgentCaseReportFilename(agentId, caseNo)
if case["behavior"] == Case.OK:
td_text = "Pass"
td_class = "case_ok"
elif case["behavior"] == Case.NON_STRICT:
td_text = "Non-Strict"
td_class = "case_non_strict"
else:
td_text = "Fail"
td_class = "case_failed"
if case["reportTime"]:
f.write('<td id="%s"><a href="%s">%s</a><br/><span id="case_duration">%s ms</span></td>' % (td_class, agent_case_report_file, td_text, case["duration"]))
else:
f.write('<td id="%s"><a href="%s">%s</a></td>' % (td_class, agent_case_report_file, td_text))
else:
f.write('<td id="case_missing">Missing</td>')
f.write("</tr>")
f.write("</table>")
f.write('<h2>Test Cases</h2>')
for caseNo in caseList:
CCase = Cases[caseNo - 1]
f.write('<a name="case_desc_%d"></a>' % caseNo)
f.write('<h3 id="case_desc_title">Case %s</h2>' % caseClasstoId(CCase))
f.write('<p id="case_desc"><i>Description</i><br/><br/> %s</p>' % CCase.DESCRIPTION)
f.write('<p id="case_expect"><i>Expectation</i><br/><br/> %s</p>' % CCase.EXPECTATION)
f.write("</body></html>")
f.close()
return report_filename