def createMasterReport(self, outdir):
report_filename = "index.html"
f = open(os.path.join(outdir, report_filename), 'w')
f.write('<!DOCTYPE html><html><body><head><meta charset="utf-8" /><style lang="css">%s %s</style><script language="javascript">%s</script></head>' % (FuzzingFactory.css_common, FuzzingFactory.css_master, self.js_master()))
f.write('<h1>WebSockets Protocol Test Report</h1>')
f.write('<p id="intro">Test summary report generated on</p>')
f.write('<p id="intro" style="margin-left: 80px;"><i>%s</i></p>' % utcnow())
f.write('<p id="intro">by <a href="%s">Autobahn</a> WebSockets.</p>' % "http://www.tavendo.de/autobahn")
f.write('<p id="intro"><a href="#" onclick="toggleClose();">Toggle Close Results</a></p>')
f.write('<h2>Test Results</h2>')
f.write('<table id="agent_case_results">')
## sorted list of agents for which test cases where run
##
agentList = sorted(self.agents.keys())
## create list ordered list of case Ids
##
cl = []
for c in Cases:
t = caseClasstoIdTuple(c)
cl.append((t, caseIdTupletoId(t)))
cl = sorted(cl)
caseList = []
for c in cl:
caseList.append(c[1])
lastCaseCategory = None
lastCaseSubCategory = None
for caseId in caseList:
caseCategoryIndex = caseId.split('.')[0]
caseCategory = CaseCategories.get(caseCategoryIndex, "Misc")
caseSubCategoryIndex = '.'.join(caseId.split('.')[:2])
caseSubCategory = CaseSubCategories.get(caseSubCategoryIndex, None)
## Category row
##
if caseCategory != lastCaseCategory:
f.write('<tr id="case_category_row">')
f.write('<td id="case_category">%s %s</td>' % (caseCategoryIndex, caseCategory))
for agentId in agentList:
f.write('<td class="agent close_flex" colspan="2">%s</td>' % agentId)
f.write('</tr>')
lastCaseCategory = caseCategory
lastCaseSubCategory = None
if caseSubCategory != lastCaseSubCategory:
f.write('<tr id="case_subcategory_row">')
f.write('<td class="case_subcategory" colspan="%d">%s %s</td>' % (len(agentList)*2 + 1, caseSubCategoryIndex, caseSubCategory))
lastCaseSubCategory = caseSubCategory
f.write('<tr id="agent_case_result_row">')
f.write('<td id="case"><a href="#case_desc_%s">Case %s</a></td>' % (caseId.replace('.', '_'), caseId))
## Agent/Case Result
##
for agentId in agentList:
if self.agents[agentId].has_key(caseId):
case = self.agents[agentId][caseId]
agent_case_report_file = self.makeAgentCaseReportFilename(agentId, caseId)
if case["behavior"] == Case.OK:
td_text = "Pass"
td_class = "case_ok"
elif case["behavior"] == Case.NON_STRICT:
td_text = "Non-Strict"
td_class = "case_non_strict"
elif case["behavior"] == Case.NO_CLOSE:
td_text = "No Close"
td_class = "case_no_close"
else:
td_text = "Fail"
td_class = "case_failed"
if case["behaviorClose"] == Case.OK:
ctd_text = "%s" % str(case["remoteCloseCode"])
ctd_class = "case_ok"
elif case["behaviorClose"] == Case.FAILED_BY_CLIENT:
ctd_text = "%s" % str(case["remoteCloseCode"])
ctd_class = "case_almost"
elif case["behaviorClose"] == Case.WRONG_CODE:
ctd_text = "%s" % str(case["remoteCloseCode"])
ctd_class = "case_non_strict"
elif case["behaviorClose"] == Case.UNCLEAN:
ctd_text = "Unclean"
ctd_class = "case_failed"
else:
ctd_text = "Fail"
ctd_class = "case_failed"
if case["reportTime"]:
f.write('<td class="%s"><a href="%s">%s</a><br/><span id="case_duration">%s ms</span></td><td class="close close_hide %s"><span class="close_code">%s</span></td>' % (td_class, agent_case_report_file, td_text, case["duration"],ctd_class,ctd_text))
else:
f.write('<td class="%s"><a href="%s">%s</a></td><td class="close close_hide %s"><span class="close_code">%s</span></td>' % (td_class, agent_case_report_file, td_text,ctd_class,ctd_text))
else:
f.write('<td class="case_missing close_flex" colspan="2">Missing</td>')
f.write("</tr>")
f.write("</table>")
f.write('<h2>Test Cases</h2>')
for caseId in caseList:
CCase = CasesById[caseId]
f.write('<a name="case_desc_%s"></a>' % caseId.replace('.', '_'))
f.write('<h3 id="case_desc_title">Case %s</h2>' % caseId)
f.write('<p id="case_desc"><i>Description</i><br/><br/> %s</p>' % CCase.DESCRIPTION)
f.write('<p id="case_expect"><i>Expectation</i><br/><br/> %s</p>' % CCase.EXPECTATION)
f.write("</body></html>")
f.close()
return report_filename
def createMasterReport(self, outdir):
"""
Create report master HTML file.
:param outdir: Directory where to create file.
:type outdir: str
:returns: str -- Name of created file.
"""
## open report file in create / write-truncate mode
##
report_filename = "index.html"
f = open(os.path.join(outdir, report_filename), "w")
## write HTML
##
f.write("<!DOCTYPE html>\n")
f.write("<html>\n")
f.write(" <head>\n")
f.write(' <meta charset="utf-8" />\n')
f.write(' <style lang="css">%s</style>\n' % CSS_COMMON)
f.write(' <style lang="css">%s</style>\n' % CSS_MASTER_REPORT)
f.write(
' <script language="javascript">%s</script>\n'
% JS_MASTER_REPORT
% {"agents_cnt": len(self.agents.keys())}
)
f.write(" </head>\n")
f.write(" <body>\n")
f.write(
' <a href="#"><div id="toggle_button" class="unselectable" onclick="toggleClose();">Toggle Details</div></a>\n'
)
f.write(' <a name="top"></a>\n')
f.write(" <br/>\n")
## top logos
f.write(
' <center><img src="http://www.tavendo.de/static/autobahn/ws_protocol_test_report.png" border="0" width="820" height="46" alt="WebSockets Protocol Test Report"></img></a></center>\n'
)
f.write(
' <center><a href="http://www.tavendo.de/autobahn" title="Autobahn WebSockets"><img src="http://www.tavendo.de/static/autobahn/ws_protocol_test_report_autobahn.png" border="0" width="300" height="68" alt="Autobahn WebSockets"></img></a></center>\n'
)
## write report header
##
f.write(' <div id="master_report_header" class="block">\n')
f.write(
' <p id="intro">Summary report generated on %s (UTC) by <a href="%s">Autobahn WebSockets</a> v%s.</p>\n'
% (utcnow(), "http://www.tavendo.de/autobahn", str(autobahn.version))
)
f.write(
"""
<table id="case_outcome_desc">
<tr>
<td class="case_ok">Pass</td>
<td class="outcome_desc">Test case was executed and passed successfully.</td>
</tr>
<tr>
<td class="case_non_strict">Non-Strict</td>
<td class="outcome_desc">Test case was executed and passed non-strictly.
A non-strict behavior is one that does not adhere to a SHOULD-behavior as described in the protocol specification or
a well-defined, canonical behavior that appears to be desirable but left open in the protocol specification.
An implementation with non-strict behavior is still conformant to the protocol specification.</td>
</tr>
<tr>
<td class="case_failed">Fail</td>
<td class="outcome_desc">Test case was executed and failed. An implementation which fails a test case - other
than a performance/limits related one - is non-conforming to a MUST-behavior as described in the protocol specification.</td>
</tr>
<tr>
<td class="case_missing">Missing</td>
<td class="outcome_desc">Test case is missing, either because it was skipped via the test suite configuration
or deactivated, i.e. because the implementation does not implement the tested feature or breaks during running
the test case.</td>
</tr>
</table>
"""
)
f.write(" </div>\n")
## write big agent/case report table
##
f.write(' <table id="agent_case_results">\n')
## sorted list of agents for which test cases where run
##
agentList = sorted(self.agents.keys())
## create list ordered list of case Ids
##
cl = []
for c in Cases:
t = caseClasstoIdTuple(c)
cl.append((t, caseIdTupletoId(t)))
cl = sorted(cl)
caseList = []
for c in cl:
caseList.append(c[1])
lastCaseCategory = None
lastCaseSubCategory = None
for caseId in caseList:
caseCategoryIndex = caseId.split(".")[0]
caseCategory = CaseCategories.get(caseCategoryIndex, "Misc")
caseSubCategoryIndex = ".".join(caseId.split(".")[:2])
caseSubCategory = CaseSubCategories.get(caseSubCategoryIndex, None)
## Category/Agents row
##
if caseCategory != lastCaseCategory or (
self.repeatAgentRowPerSubcategory and caseSubCategory != lastCaseSubCategory
):
f.write(' <tr class="case_category_row">\n')
f.write(' <td class="case_category">%s %s</td>\n' % (caseCategoryIndex, caseCategory))
for agentId in agentList:
f.write(' <td class="agent close_flex" colspan="2">%s</td>\n' % agentId)
f.write(" </tr>\n")
lastCaseCategory = caseCategory
lastCaseSubCategory = None
## Subcategory row
##
if caseSubCategory != lastCaseSubCategory:
f.write(' <tr class="case_subcategory_row">\n')
f.write(
' <td class="case_subcategory" colspan="%d">%s %s</td>\n'
% (len(agentList) * 2 + 1, caseSubCategoryIndex, caseSubCategory)
)
f.write(" </tr>\n")
lastCaseSubCategory = caseSubCategory
## Cases row
##
f.write(' <tr class="agent_case_result_row">\n')
f.write(
' <td class="case"><a href="#case_desc_%s">Case %s</a></td>\n'
% (caseId.replace(".", "_"), caseId)
)
## Case results
##
for agentId in agentList:
if self.agents[agentId].has_key(caseId):
case = self.agents[agentId][caseId]
agent_case_report_file = self.makeAgentCaseReportFilename(agentId, caseId)
if case["behavior"] == Case.OK:
td_text = "Pass"
td_class = "case_ok"
elif case["behavior"] == Case.NON_STRICT:
td_text = "Non-Strict"
td_class = "case_non_strict"
elif case["behavior"] == Case.NO_CLOSE:
td_text = "No Close"
td_class = "case_no_close"
elif case["behavior"] == Case.INFORMATIONAL:
td_text = "Info"
td_class = "case_info"
else:
td_text = "Fail"
td_class = "case_failed"
if case["behaviorClose"] == Case.OK:
ctd_text = "%s" % str(case["remoteCloseCode"])
ctd_class = "case_ok"
elif case["behaviorClose"] == Case.FAILED_BY_CLIENT:
ctd_text = "%s" % str(case["remoteCloseCode"])
ctd_class = "case_almost"
elif case["behaviorClose"] == Case.WRONG_CODE:
ctd_text = "%s" % str(case["remoteCloseCode"])
ctd_class = "case_non_strict"
elif case["behaviorClose"] == Case.UNCLEAN:
ctd_text = "Unclean"
ctd_class = "case_failed"
elif case["behaviorClose"] == Case.INFORMATIONAL:
ctd_text = "%s" % str(case["remoteCloseCode"])
ctd_class = "case_info"
else:
ctd_text = "Fail"
ctd_class = "case_failed"
if case["reportTime"]:
f.write(
' <td class="%s"><a href="%s">%s</a><br/><span class="case_duration">%s ms</span></td><td class="close close_hide %s"><span class="close_code">%s</span></td>\n'
% (td_class, agent_case_report_file, td_text, case["duration"], ctd_class, ctd_text)
)
else:
f.write(
' <td class="%s"><a href="%s">%s</a></td><td class="close close_hide %s"><span class="close_code">%s</span></td>\n'
% (td_class, agent_case_report_file, td_text, ctd_class, ctd_text)
)
else:
f.write(' <td class="case_missing close_flex" colspan="2">Missing</td>\n')
f.write(" </tr>\n")
f.write(" </table>\n")
f.write(" <br/><hr/>\n")
## Case descriptions
##
f.write(' <div id="test_case_descriptions">\n')
for caseId in caseList:
CCase = CasesById[caseId]
f.write(" <br/>\n")
f.write(' <a name="case_desc_%s"></a>\n' % caseId.replace(".", "_"))
f.write(" <h2>Case %s</h2>\n" % caseId)
f.write(' <a class="up" href="#top">Up</a>\n')
f.write(
' <p class="case_text_block case_desc"><b>Case Description</b><br/><br/>%s</p>\n'
% CCase.DESCRIPTION
)
f.write(
' <p class="case_text_block case_expect"><b>Case Expectation</b><br/><br/>%s</p>\n'
% CCase.EXPECTATION
)
f.write(" </div>\n")
f.write(" <br/><hr/>\n")
## end of HTML
##
f.write(" </body>\n")
f.write("</html>\n")
## close created HTML file and return filename
##
f.close()
return report_filename
def onOpen(self):
if self.runCase:
cc = caseClasstoIdTuple(self.runCase.__class__)
## IE10 crashes on these
##
if self.caseAgent.find("MSIE") >= 0 and (cc[0:3] in [(6, 4, 3), (6, 4, 5)] or
cc[0:2] in [(2, 5)] or
cc[0:1][0] in [3, 4, 5]):
print "Skipping test case for IE10 (crashes) !!!"
self.runCase = None
self.sendClose()
return
## Chrome crashes on these
##
if self.caseAgent.find("Chrome") >= 0 and cc[0:3] in [(6, 4, 3), (6, 4, 5)]:
print "Skipping forever sending data after invalid UTF-8 for Chrome (crashes) !!!"
self.runCase = None
self.sendClose()
return
## FF7 crashes on these
##
if self.caseAgent.find("Firefox/7") >= 0 and cc[0:2] == (9, 3):
print "Skipping fragmented message test case for Firefox/7 (crashes) !!!"
self.runCase = None
self.sendClose()
return
## FF7 crashes on these
##
if self.caseAgent.find("Firefox/7") >= 0 and cc[0:3] in [(6, 4, 2), (6, 4, 3), (6, 4, 4), (6, 4, 5)]:
print "Skipping invalid UTF-8 test for Firefox/7 (crashes) !!!"
self.runCase = None
self.sendClose()
return
## FF does not yet implement binary messages
##
if self.caseAgent.find("Firefox") >= 0 and cc[0:2] in [(1, 2), (9, 2), (9, 4), (9, 6), (9, 8)]:
print "Skipping binary message test case for Firefox !!!"
self.runCase = None
self.sendClose()
return
self.caseStart = time.time()
self.runCase.onOpen()
elif self.path == "/updateReports":
self.factory.createReports()
self.sendClose()
elif self.path == "/getCaseCount":
self.sendMessage(json.dumps(len(self.factory.specCases)))
self.sendClose()
else:
pass
def createMasterReport(self, outdir):
report_filename = "index.html"
f = open(os.path.join(outdir, report_filename), 'w')
f.write('<!DOCTYPE html><html><body><head><meta charset="utf-8" /><style lang="css">%s %s</style></head>' % (FuzzingFactory.css_common, FuzzingFactory.css_master))
f.write('<h1>WebSockets Protocol Test Report</h1>')
f.write('<p id="intro">Test summary report generated on</p>')
f.write('<p id="intro" style="margin-left: 80px;"><i>%s</i></p>' % getUtcNow())
f.write('<p id="intro">by <a href="%s">Autobahn</a> WebSockets.</p>' % "http://www.tavendo.de/autobahn")
f.write('<h2>Test Results</h2>')
f.write('<table id="agent_case_results">')
## sorted list of agents for which test cases where run
##
agentList = sorted(self.agents.keys())
## create list of case indexes order by case ID
##
cl = []
i = 1
for c in Cases:
cl.append((caseClasstoIdTuple(c) , i))
i += 1
cl = sorted(cl)
caseList = []
for c in cl:
caseList.append(c[1])
lastCaseCategory = None
lastCaseSubCategory = None
for caseNo in caseList:
## Case ID and category
##
caseId = caseClasstoId(Cases[caseNo - 1])
caseCategoryIndex = caseId.split('.')[0]
caseCategory = CaseCategories.get(caseCategoryIndex, "Misc")
caseSubCategoryIndex = '.'.join(caseId.split('.')[:2])
caseSubCategory = CaseSubCategories.get(caseSubCategoryIndex, None)
## Category row
##
if caseCategory != lastCaseCategory:
f.write('<tr id="case_category_row">')
f.write('<td id="case_category">%s %s</td>' % (caseCategoryIndex, caseCategory))
for agentId in agentList:
f.write('<td id="agent">%s</td>' % agentId)
f.write('</tr>')
lastCaseCategory = caseCategory
lastCaseSubCategory = None
if caseSubCategory != lastCaseSubCategory:
f.write('<tr id="case_subcategory_row">')
f.write('<td id="case_subcategory" colspan="%d">%s %s</td>' % (len(agentList) + 1, caseSubCategoryIndex, caseSubCategory))
lastCaseSubCategory = caseSubCategory
f.write('<tr id="agent_case_result_row">')
f.write('<td id="case"><a href="#case_desc_%d">Case %s</a></td>' % (caseNo, caseId))
## Agent/Case Result
##
for agentId in agentList:
if self.agents[agentId].has_key(caseNo):
case = self.agents[agentId][caseNo]
agent_case_report_file = self.makeAgentCaseReportFilename(agentId, caseNo)
if case["behavior"] == Case.OK:
td_text = "Pass"
td_class = "case_ok"
elif case["behavior"] == Case.NON_STRICT:
td_text = "Non-Strict"
td_class = "case_non_strict"
else:
td_text = "Fail"
td_class = "case_failed"
if case["reportTime"]:
f.write('<td id="%s"><a href="%s">%s</a><br/><span id="case_duration">%s ms</span></td>' % (td_class, agent_case_report_file, td_text, case["duration"]))
else:
f.write('<td id="%s"><a href="%s">%s</a></td>' % (td_class, agent_case_report_file, td_text))
else:
f.write('<td id="case_missing">Missing</td>')
f.write("</tr>")
f.write("</table>")
f.write('<h2>Test Cases</h2>')
for caseNo in caseList:
CCase = Cases[caseNo - 1]
f.write('<a name="case_desc_%d"></a>' % caseNo)
f.write('<h3 id="case_desc_title">Case %s</h2>' % caseClasstoId(CCase))
f.write('<p id="case_desc"><i>Description</i><br/><br/> %s</p>' % CCase.DESCRIPTION)
f.write('<p id="case_expect"><i>Expectation</i><br/><br/> %s</p>' % CCase.EXPECTATION)
f.write("</body></html>")
f.close()
return report_filename