def table_row(self, test): name = test['name'].split('/')[-1] suite_name = test['relpath'].split('/')[0] app_name = os.path.basename(test['here']) path_name = test['here'].split('gaiatest')[1] link = self._test_href + test['relpath'] run_link = self._test_href + suite_name class_link = self._test_href + os.path.dirname(test['relpath']) flame = Device('Flame') desktop = Device('desktop') flame_test = self._get_test(self._flame, name) desktop_test = self._get_test(self._desktop, name) if flame_test: if flame_test['expected'] == 'fail': flame.status = 'XFailed' flame.tooltip = 'Test is expected to fail' elif 'disabled' in flame_test.keys(): flame.status = 'Disabled' flame.tooltip = flame_test['disabled'] elif flame_test['expected'] == 'pass': flame.status = 'Enabled' flame.tooltip = 'Test is expected to pass' if desktop_test: if desktop_test.has_key('skip-if') and re.match('device\s*==\s*"desktop"', desktop_test['skip-if']): pass elif desktop_test['expected'] == 'fail': desktop.status = 'XFailed' desktop.tooltip = 'Test is expected to fail' elif 'disabled' in desktop_test.keys(): desktop.status = 'Disabled' desktop.tooltip = desktop_test['disabled'] elif desktop_test['expected'] == 'pass': desktop.status = 'Enabled' desktop.tooltip = 'Test is expected to pass' self.test_logs.append( html.tr([ html.td( html.a(name, href_=link, target_='_blank'), class_='col-name', title=path_name), html.td( html.a(suite_name, href_=run_link, target_='_blank'), class_='col-run'), html.td( html.a(app_name, href_=class_link, target_='_blank'), class_='col-class'), html.td(flame.status, class_='col-flame ' + flame.status, title_=flame.tooltip), html.td(desktop.status, class_='col-desktop ' + desktop.status, title_=desktop.tooltip) ], class_='results-table-row') )
def _extract_html(result, test_name, test_class='', duration=0, debug=None, output=''): additional_html = [] debug = debug or {} links_html = [] result_map = { 'KNOWN-FAIL': 'expected failure', 'PASS': '******', 'UNEXPECTED-FAIL': 'failure', 'UNEXPECTED-PASS': '******'} if result.upper() in ['SKIPPED', 'UNEXPECTED-FAIL', 'KNOWN-FAIL', 'ERROR']: if debug.get('screenshot'): screenshot = 'data:image/png;base64,{}'.format(debug['screenshot']) additional_html.append(html.div( html.a(html.img(src=screenshot), href="#"), class_='screenshot')) for name, content in debug.items(): try: if 'screenshot' in name: href = '#' else: # use base64 to avoid that some browser (such as Firefox, Opera) # treats '#' as the start of another link if the data URL contains. # use 'charset=utf-8' to show special characters like Chinese. href = 'data:text/plain;charset=utf-8;base64,{}'.format( base64.b64encode(content)) links_html.append(html.a( name.title(), class_=name, href=href, target='_blank')) links_html.append(' ') except: pass log = html.div(class_='log') for line in output.splitlines(): separator = line.startswith(' ' * 10) if separator: log.append(line[:80]) else: if line.lower().find("error") != -1 or line.lower().find("exception") != -1: log.append(html.span(raw(cgi.escape(line)), class_='error')) else: log.append(raw(cgi.escape(line))) log.append(html.br()) additional_html.append(log) test_logs.append(html.tr([ html.td(result_map.get(result, result).title(), class_='col-result'), html.td(test_class, class_='col-class'), html.td(test_name, class_='col-name'), html.td(str(duration), class_='col-duration'), html.td(links_html, class_='col-links'), html.td(additional_html, class_='debug')], class_=result_map.get(result, result).lower() + ' results-table-row'))
def _extract_html(result, test_name, test_class="", duration=0, debug=None, output=""): additional_html = [] debug = debug or {} links_html = [] result_map = { "KNOWN-FAIL": "expected failure", "PASS": "******", "UNEXPECTED-FAIL": "failure", "UNEXPECTED-PASS": "******", } if result.upper() in ["SKIPPED", "UNEXPECTED-FAIL", "KNOWN-FAIL", "ERROR"]: if debug.get("screenshot"): screenshot = "data:image/png;base64,%s" % debug["screenshot"] additional_html.append(html.div(html.a(html.img(src=screenshot), href="#"), class_="screenshot")) for name, content in debug.items(): try: if "screenshot" in name: href = "#" else: # use base64 to avoid that some browser (such as Firefox, Opera) # treats '#' as the start of another link if the data URL contains. # use 'charset=utf-8' to show special characters like Chinese. href = "data:text/plain;charset=utf-8;base64,%s" % base64.b64encode(content) links_html.append(html.a(name.title(), class_=name, href=href, target="_blank")) links_html.append(" ") except: pass log = html.div(class_="log") for line in output.splitlines(): separator = line.startswith(" " * 10) if separator: log.append(line[:80]) else: if line.lower().find("error") != -1 or line.lower().find("exception") != -1: log.append(html.span(raw(cgi.escape(line)), class_="error")) else: log.append(raw(cgi.escape(line))) log.append(html.br()) additional_html.append(log) test_logs.append( html.tr( [ html.td(result_map.get(result, result).title(), class_="col-result"), html.td(test_class, class_="col-class"), html.td(test_name, class_="col-name"), html.td(str(duration), class_="col-duration"), html.td(links_html, class_="col-links"), html.td(additional_html, class_="debug"), ], class_=result_map.get(result, result).lower() + " results-table-row", ) )
def generate_html(self, results_list): tests = sum([results.testsRun for results in results_list]) failures = sum([len(results.failures) for results in results_list]) expected_failures = sum([len(results.expectedFailures) for results in results_list]) skips = sum([len(results.skipped) for results in results_list]) + len(self.manifest_skipped_tests) errors = sum([len(results.errors) for results in results_list]) passes = sum([results.passed for results in results_list]) unexpected_passes = sum([len(results.unexpectedSuccesses) for results in results_list]) test_time = self.elapsedtime test_logs = [] def _extract_html_from_result(result): _extract_html( result=result.result, test_name=result.name, test_class=result.test_class, duration=round(result.duration, 1), debug=result.debug, output='\n'.join(result.output)) def _extract_html_from_skipped_manifest_test(test): _extract_html( result='skipped', test_name=test['name'], output=test.get('disabled')) def _extract_html(result, test_name, test_class='', duration=0, debug=None, output=''): additional_html = [] debug = debug or {} links_html = [] result_map = { 'KNOWN-FAIL': 'expected failure', 'PASS': '******', 'UNEXPECTED-FAIL': 'failure', 'UNEXPECTED-PASS': '******'} if result.upper() in ['SKIPPED', 'UNEXPECTED-FAIL', 'KNOWN-FAIL', 'ERROR']: if debug.get('screenshot'): screenshot = 'data:image/png;base64,%s' % debug['screenshot'] additional_html.append(html.div( html.a(html.img(src=screenshot), href="#"), class_='screenshot')) for name, content in debug.items(): try: if 'screenshot' in name: href = '#' else: # use base64 to avoid that some browser (such as Firefox, Opera) # treats '#' as the start of another link if the data URL contains. # use 'charset=utf-8' to show special characters like Chinese. href = 'data:text/plain;charset=utf-8;base64,%s' % base64.b64encode(content) links_html.append(html.a( name.title(), class_=name, href=href, target='_blank')) links_html.append(' ') except: pass log = html.div(class_='log') for line in output.splitlines(): separator = line.startswith(' ' * 10) if separator: log.append(line[:80]) else: if line.lower().find("error") != -1 or line.lower().find("exception") != -1: log.append(html.span(raw(cgi.escape(line)), class_='error')) else: log.append(raw(cgi.escape(line))) log.append(html.br()) additional_html.append(log) test_logs.append(html.tr([ html.td(result_map.get(result, result).title(), class_='col-result'), html.td(test_class, class_='col-class'), html.td(test_name, class_='col-name'), html.td(str(duration), class_='col-duration'), html.td(links_html, class_='col-links'), html.td(additional_html, class_='debug')], class_=result_map.get(result, result).lower() + ' results-table-row')) for results in results_list: [_extract_html_from_result(test) for test in results.tests] for test in self.manifest_skipped_tests: _extract_html_from_skipped_manifest_test(test) generated = datetime.datetime.now() date_format = '%d %b %Y %H:%M:%S' version = {} if self.capabilities: version.update({ 'application_buildid': self.capabilities.get('appBuildId'), 'application_version': self.capabilities.get('version'), 'device_id': self.capabilities.get('device')}) if self.bin or self.capabilities.get('device') != 'desktop': version.update(mozversion.get_version( binary=self.bin, sources=self.sources, dm_type=os.environ.get('DM_TRANS', 'adb'))) configuration = { 'Gecko version': version.get('application_version'), 'Gecko build': version.get('application_buildid'), 'Gecko revision': version.get('application_revision'), 'Gaia date': version.get('gaia_date') and time.strftime(date_format, time.localtime( int(version.get('gaia_date')))), 'Device identifier': version.get('device_id'), 'Device firmware (base)': version.get('device_firmware_version_base'), 'Device firmware (date)': version.get('device_firmware_date') and time.strftime(date_format, time.localtime( int(version.get('device_firmware_date')))), 'Device firmware (incremental)': version.get('device_firmware_version_incremental'), 'Device firmware (release)': version.get('device_firmware_version_release')} if version.get('application_changeset') and version.get('application_repository'): configuration['Gecko revision'] = html.a( version.get('application_changeset'), href='/'.join([version.get('application_repository'), version.get('application_changeset')]), target='_blank') if version.get('gaia_changeset'): configuration['Gaia revision'] = html.a( version.get('gaia_changeset')[:12], href='https://github.com/mozilla-b2g/gaia/commit/%s' % version.get('gaia_changeset'), target='_blank') doc = html.html( html.head( html.meta(charset='utf-8'), html.title('Test Report'), #TODO: must redisgn this to use marionette's resourcs, instead of the caller folder's html.style(raw(pkg_resources.resource_string( __name__, os.path.sep.join(['resources', 'htmlreport', 'style.css']))), type='text/css')), html.body( html.script(raw(pkg_resources.resource_string( __name__, os.path.sep.join(['resources', 'htmlreport', 'jquery.js']))), type='text/javascript'), html.script(raw(pkg_resources.resource_string( __name__, os.path.sep.join(['resources', 'htmlreport', 'main.js']))), type='text/javascript'), html.p('Report generated on %s at %s by %s version %s' % ( generated.strftime('%d-%b-%Y'), generated.strftime('%H:%M:%S'), self.html_name, self.html_version)), html.h2('Configuration'), html.table( [html.tr(html.td(k), html.td(v)) for k, v in sorted(configuration.items()) if v], id='configuration'), html.h2('Summary'), html.p('%i tests ran in %i seconds.' % (tests, test_time), html.br(), html.span('%i passed' % passes, class_='passed'), ', ', html.span('%i skipped' % skips, class_='skipped'), ', ', html.span('%i failed' % failures, class_='failed'), ', ', html.span('%i errors' % errors, class_='error'), '.', html.br(), html.span('%i expected failures' % expected_failures, class_='expected failure'), ', ', html.span('%i unexpected passes' % unexpected_passes, class_='unexpected pass'), '.'), html.h2('Results'), html.table([html.thead( html.tr([ html.th('Result', class_='sortable', col='result'), html.th('Class', class_='sortable', col='class'), html.th('Test Name', class_='sortable', col='name'), html.th('Duration', class_='sortable numeric', col='duration'), html.th('Links')]), id='results-table-head'), html.tbody(test_logs, id='results-table-body')], id='results-table'))) return doc.unicode(indent=2)
def generate_html(self, results_list): tests = sum([results.testsRun for results in results_list]) failures = sum([len(results.failures) for results in results_list]) expected_failures = sum([len(results.expectedFailures) for results in results_list]) skips = sum([len(results.skipped) for results in results_list]) + len(self.manifest_skipped_tests) errors = sum([len(results.errors) for results in results_list]) passes = sum([results.passed for results in results_list]) unexpected_passes = sum([len(results.unexpectedSuccesses) for results in results_list]) test_time = self.elapsedtime test_logs = [] def _extract_html_from_result(result): _extract_html( result=result.result, test_name=result.name, test_class=result.test_class, duration=round(result.duration, 1), debug=result.debug, output="\n".join(result.output), ) def _extract_html_from_skipped_manifest_test(test): _extract_html(result="skipped", test_name=test["name"], output=test.get("disabled")) def _extract_html(result, test_name, test_class="", duration=0, debug=None, output=""): additional_html = [] debug = debug or {} links_html = [] result_map = { "KNOWN-FAIL": "expected failure", "PASS": "******", "UNEXPECTED-FAIL": "failure", "UNEXPECTED-PASS": "******", } if result.upper() in ["SKIPPED", "UNEXPECTED-FAIL", "KNOWN-FAIL", "ERROR"]: if debug.get("screenshot"): screenshot = "data:image/png;base64,%s" % debug["screenshot"] additional_html.append(html.div(html.a(html.img(src=screenshot), href="#"), class_="screenshot")) for name, content in debug.items(): try: if "screenshot" in name: href = "#" else: # use base64 to avoid that some browser (such as Firefox, Opera) # treats '#' as the start of another link if the data URL contains. # use 'charset=utf-8' to show special characters like Chinese. href = "data:text/plain;charset=utf-8;base64,%s" % base64.b64encode(content) links_html.append(html.a(name.title(), class_=name, href=href, target="_blank")) links_html.append(" ") except: pass log = html.div(class_="log") for line in output.splitlines(): separator = line.startswith(" " * 10) if separator: log.append(line[:80]) else: if line.lower().find("error") != -1 or line.lower().find("exception") != -1: log.append(html.span(raw(cgi.escape(line)), class_="error")) else: log.append(raw(cgi.escape(line))) log.append(html.br()) additional_html.append(log) test_logs.append( html.tr( [ html.td(result_map.get(result, result).title(), class_="col-result"), html.td(test_class, class_="col-class"), html.td(test_name, class_="col-name"), html.td(str(duration), class_="col-duration"), html.td(links_html, class_="col-links"), html.td(additional_html, class_="debug"), ], class_=result_map.get(result, result).lower() + " results-table-row", ) ) for results in results_list: [_extract_html_from_result(test) for test in results.tests] for test in self.manifest_skipped_tests: _extract_html_from_skipped_manifest_test(test) generated = datetime.datetime.now() date_format = "%d %b %Y %H:%M:%S" version = {} if self.capabilities: version.update( { "application_buildid": self.capabilities.get("appBuildId"), "application_version": self.capabilities.get("version"), "device_id": self.capabilities.get("device"), } ) if self.bin or self.capabilities.get("device") != "desktop": version.update( mozversion.get_version(binary=self.bin, sources=self.sources, dm_type=os.environ.get("DM_TRANS", "adb")) ) configuration = { "Goanna version": version.get("application_version"), "Goanna build": version.get("application_buildid"), "Goanna revision": version.get("application_revision"), "Gaia date": version.get("gaia_date") and time.strftime(date_format, time.localtime(int(version.get("gaia_date")))), "Device identifier": version.get("device_id"), "Device firmware (base)": version.get("device_firmware_version_base"), "Device firmware (date)": version.get("device_firmware_date") and time.strftime(date_format, time.localtime(int(version.get("device_firmware_date")))), "Device firmware (incremental)": version.get("device_firmware_version_incremental"), "Device firmware (release)": version.get("device_firmware_version_release"), } if version.get("application_changeset") and version.get("application_repository"): configuration["Goanna revision"] = html.a( version.get("application_changeset"), href="/".join([version.get("application_repository"), version.get("application_changeset")]), target="_blank", ) if version.get("gaia_changeset"): configuration["Gaia revision"] = html.a( version.get("gaia_changeset")[:12], href="https://github.com/mozilla-b2g/gaia/commit/%s" % version.get("gaia_changeset"), target="_blank", ) doc = html.html( html.head( html.meta(charset="utf-8"), html.title("Test Report"), # TODO: must redisgn this to use marionette's resourcs, instead of the caller folder's html.style( raw( pkg_resources.resource_string( __name__, os.path.sep.join(["resources", "htmlreport", "style.css"]) ) ), type="text/css", ), ), html.body( html.script( raw( pkg_resources.resource_string( __name__, os.path.sep.join(["resources", "htmlreport", "jquery.js"]) ) ), type="text/javascript", ), html.script( raw( pkg_resources.resource_string( __name__, os.path.sep.join(["resources", "htmlreport", "main.js"]) ) ), type="text/javascript", ), html.p( "Report generated on %s at %s by %s version %s" % ( generated.strftime("%d-%b-%Y"), generated.strftime("%H:%M:%S"), self.html_name, self.html_version, ) ), html.h2("Configuration"), html.table( [html.tr(html.td(k), html.td(v)) for k, v in sorted(configuration.items()) if v], id="configuration" ), html.h2("Summary"), html.p( "%i tests ran in %i seconds." % (tests, test_time), html.br(), html.span("%i passed" % passes, class_="passed"), ", ", html.span("%i skipped" % skips, class_="skipped"), ", ", html.span("%i failed" % failures, class_="failed"), ", ", html.span("%i errors" % errors, class_="error"), ".", html.br(), html.span("%i expected failures" % expected_failures, class_="expected failure"), ", ", html.span("%i unexpected passes" % unexpected_passes, class_="unexpected pass"), ".", ), html.h2("Results"), html.table( [ html.thead( html.tr( [ html.th("Result", class_="sortable", col="result"), html.th("Class", class_="sortable", col="class"), html.th("Test Name", class_="sortable", col="name"), html.th("Duration", class_="sortable numeric", col="duration"), html.th("Links"), ] ), id="results-table-head", ), html.tbody(test_logs, id="results-table-body"), ], id="results-table", ), ), ) return doc.unicode(indent=2)
def generate_html(self, results_list): tests = sum([results.testsRun for results in results_list]) failures = sum([len(results.failures) for results in results_list]) expected_failures = sum( [len(results.expectedFailures) for results in results_list]) skips = sum([len(results.skipped) for results in results_list]) + len( self.manifest_skipped_tests) errors = sum([len(results.errors) for results in results_list]) passes = sum([results.passed for results in results_list]) unexpected_passes = sum( [len(results.unexpectedSuccesses) for results in results_list]) test_time = self.elapsedtime test_logs = [] def _extract_html_from_result(result): _extract_html(result=result.result, test_name=result.name, test_class=result.test_class, duration=round(result.duration, 1), debug=result.debug, output='\n'.join(result.output)) def _extract_html_from_skipped_manifest_test(test): _extract_html(result='skipped', test_name=test['name'], output=test.get('disabled')) def _extract_html(result, test_name, test_class='', duration=0, debug=None, output=''): additional_html = [] debug = debug or {} links_html = [] result_map = { 'KNOWN-FAIL': 'expected failure', 'PASS': '******', 'UNEXPECTED-FAIL': 'failure', 'UNEXPECTED-PASS': '******' } if result.upper() in [ 'SKIPPED', 'UNEXPECTED-FAIL', 'KNOWN-FAIL', 'ERROR' ]: if debug.get('screenshot'): screenshot = 'data:image/png;base64,%s' % debug[ 'screenshot'] additional_html.append( html.div(html.a(html.img(src=screenshot), href="#"), class_='screenshot')) for name, content in debug.items(): try: if 'screenshot' in name: href = '#' else: # use base64 to avoid that some browser (such as Firefox, Opera) # treats '#' as the start of another link if the data URL contains. # use 'charset=utf-8' to show special characters like Chinese. href = 'data:text/plain;charset=utf-8;base64,%s' % base64.b64encode( content) links_html.append( html.a(name.title(), class_=name, href=href, target='_blank')) links_html.append(' ') except: pass log = html.div(class_='log') for line in output.splitlines(): separator = line.startswith(' ' * 10) if separator: log.append(line[:80]) else: if line.lower().find("error") != -1 or line.lower( ).find("exception") != -1: log.append( html.span(raw(cgi.escape(line)), class_='error')) else: log.append(raw(cgi.escape(line))) log.append(html.br()) additional_html.append(log) test_logs.append( html.tr([ html.td(result_map.get(result, result).title(), class_='col-result'), html.td(test_class, class_='col-class'), html.td(test_name, class_='col-name'), html.td(str(duration), class_='col-duration'), html.td(links_html, class_='col-links'), html.td(additional_html, class_='debug') ], class_=result_map.get(result, result).lower() + ' results-table-row')) for results in results_list: [_extract_html_from_result(test) for test in results.tests] for test in self.manifest_skipped_tests: _extract_html_from_skipped_manifest_test(test) generated = datetime.datetime.now() date_format = '%d %b %Y %H:%M:%S' version = {} if self.capabilities: version.update({ 'application_buildid': self.capabilities.get('appBuildId'), 'application_version': self.capabilities.get('version'), 'device_id': self.capabilities.get('device') }) if self.bin or self.capabilities.get('device') != 'desktop': version.update( mozversion.get_version(binary=self.bin, sources=self.sources, dm_type=os.environ.get( 'DM_TRANS', 'adb'))) configuration = { 'Gecko version': version.get('application_version'), 'Gecko build': version.get('application_buildid'), 'Gecko revision': version.get('application_revision'), 'Gaia date': version.get('gaia_date') and time.strftime( date_format, time.localtime(int(version.get('gaia_date')))), 'Device identifier': version.get('device_id'), 'Device firmware (date)': version.get('device_firmware_date') and time.strftime( date_format, time.localtime(int(version.get('device_firmware_date')))), 'Device firmware (incremental)': version.get('device_firmware_version_incremental'), 'Device firmware (release)': version.get('device_firmware_version_release') } if version.get('application_changeset') and version.get( 'application_repository'): configuration['Gecko revision'] = html.a( version.get('application_changeset'), href='/'.join([ version.get('application_repository'), version.get('application_changeset') ]), target='_blank') if version.get('gaia_changeset'): configuration['Gaia revision'] = html.a( version.get('gaia_changeset')[:12], href='https://github.com/mozilla-b2g/gaia/commit/%s' % version.get('gaia_changeset'), target='_blank') doc = html.html( html.head( html.meta(charset='utf-8'), html.title('Test Report'), #TODO: must redisgn this to use marionette's resourcs, instead of the caller folder's html.style(raw( pkg_resources.resource_string( __name__, os.path.sep.join( ['resources', 'htmlreport', 'style.css']))), type='text/css')), html.body( html.script(raw( pkg_resources.resource_string( __name__, os.path.sep.join( ['resources', 'htmlreport', 'jquery.js']))), type='text/javascript'), html.script(raw( pkg_resources.resource_string( __name__, os.path.sep.join( ['resources', 'htmlreport', 'main.js']))), type='text/javascript'), html.p('Report generated on %s at %s by %s version %s' % (generated.strftime('%d-%b-%Y'), generated.strftime('%H:%M:%S'), self.html_name, self.html_version)), html.h2('Configuration'), html.table([ html.tr(html.td(k), html.td(v)) for k, v in sorted(configuration.items()) if v ], id='configuration'), html.h2('Summary'), html.p( '%i tests ran in %i seconds.' % (tests, test_time), html.br(), html.span('%i passed' % passes, class_='passed'), ', ', html.span('%i skipped' % skips, class_='skipped'), ', ', html.span('%i failed' % failures, class_='failed'), ', ', html.span('%i errors' % errors, class_='error'), '.', html.br(), html.span('%i expected failures' % expected_failures, class_='expected failure'), ', ', html.span('%i unexpected passes' % unexpected_passes, class_='unexpected pass'), '.'), html.h2('Results'), html.table([ html.thead(html.tr([ html.th('Result', class_='sortable', col='result'), html.th('Class', class_='sortable', col='class'), html.th('Test Name', class_='sortable', col='name'), html.th('Duration', class_='sortable numeric', col='duration'), html.th('Links') ]), id='results-table-head'), html.tbody(test_logs, id='results-table-body') ], id='results-table'))) return doc.unicode(indent=2)