def test_overall_stats(self):
data = ResultData()
data.add_results('chrome', [
self._build_result('error'),
self._build_result('error'),
self._build_result('fail'),
self._build_result('fail'),
self._build_result('fail'),
self._build_result('pass'),
self._build_result('pass'),
self._build_result('pass'),
self._build_result('pass'),
self._build_result('skip'),
self._build_result('skip'),
self._build_result('skip'),
self._build_result('skip'),
self._build_result('skip'),
])
data.add_results('firefox', [
self._build_result('error'),
self._build_result('fail'),
self._build_result('pass'),
self._build_result('skip'),
])
# Expect that the overall stats (no browser specified)
# is the sum of all browser-specific stats
expected_stats = self._build_stats(
num_error=3, num_failed=4,
num_passed=5, num_skipped=6
)
self.assertEqual(data.stats(None), expected_stats)
def test_stats(self):
# Tuples relating input results to stats
cases = [
([], self._build_stats()),
([self._build_result('error')], self._build_stats(num_error=1)),
([self._build_result('fail')], self._build_stats(num_failed=1)),
([self._build_result('skip')], self._build_stats(num_skipped=1)),
([self._build_result('pass')], self._build_stats(num_passed=1)),
([
self._build_result('error'),
self._build_result('error'),
self._build_result('fail'),
self._build_result('fail'),
self._build_result('fail'),
self._build_result('pass'),
self._build_result('pass'),
self._build_result('pass'),
self._build_result('pass'),
self._build_result('skip'),
self._build_result('skip'),
self._build_result('skip'),
self._build_result('skip'),
self._build_result('skip'),
], self._build_stats(
num_error=2, num_failed=3,
num_passed=4, num_skipped=5
))
]
for input_results, expected_stats in cases:
data = ResultData()
data.add_results('chrome', input_results)
self.assertEqual(data.stats('chrome'), expected_stats)
def testadd_results(self):
data = ResultData()
input_results = [self._build_result('pass'), self._build_result('fail')]
data.add_results('chrome', input_results)
self.assertEqual(data.browsers(), ['chrome'])
self.assertEqual(data.test_results('chrome'), input_results)
self.assertEqual(data.all_passed(), False)
def run(self):
"""
Execute each available test suite page and record whether
each test passed/failed.
If configured, will write test results using reporters.
Returns a `ResultData` object containing the test results.
"""
# Start the suite page server running on a local port
self._suite_page_server.start()
# Create an object to hold results data for all browsers
results_data = ResultData()
try:
for browser in self._browser_list:
# Run the test suite with one of our browsers
results_data.add_results(
browser.name(),
self._run_with_browser(browser)
)
# After all browsers have loaded their pages,
# Block until all coverage data received
# (if coverage is not configured, this will
# not do anything).
try:
self._report_coverage_data = self._suite_page_server.all_coverage_data()
# If we timed out, log it, but don't exit with failure
except TimeoutError:
msg = dedent("""
Did not receive all coverage data. No coverage reports will be written.
(This sometimes occurs when JSCover does not have enough memory to run.)
""").strip()
LOGGER.warning(msg)
# Re-raise any exceptions that occur
except:
raise
# Stop the suite page server, freeing up the port
finally:
self._suite_page_server.stop()
# Generate test result reports
for reporter in self._result_reporters:
reporter.write_report(results_data)
return results_data
class ResultReporterTestCase(TestCase):
"""
Base class for reporter tests.
"""
# Subclasses override this to specify the class under test
REPORTER_CLASS = None
def setUp(self):
self.result_data = ResultData()
def add_result(self, browser_name, group_name, test_name, status, detail):
"""
Add test results for the browser with `browser_name`.
`group_name`: name of the test group (e.g. 'Adder tests')
`test_name`: name of the specific test case
(e.g. 'it should start at zero')
`status`: pass | fail | skip
`detail`: details of the test case (e.g. a stack trace)
"""
self.result_data.add_results(
browser_name, [{
'test_group': group_name,
'test_name': test_name,
'status': status,
'detail': detail
}]
)
def assert_report(self, expected_report):
"""
Assert that the console report matches `expected_report`.
"""
assert_long_str_equal(self.build_report(), expected_report, strip=True)
def build_report(self):
"""
Build a report.
"""
output = StringIO()
self.REPORTER_CLASS(output).write_report(self.result_data)
return output.getvalue()
def test_all_passed(self):
# Tuples relating all_passed to input results from each browser.
cases = [
(True, [], []),
(True, [self._build_result('pass')], []),
(True, [self._build_result('pass'), self._build_result('pass')], []),
(True, [self._build_result('pass')], [self._build_result('pass')]),
(False, [self._build_result('fail')], [self._build_result('pass')]),
(False, [self._build_result('error')], [self._build_result('pass')]),
(False, [self._build_result('fail')], [self._build_result('fail')]),
(False, [self._build_result('error')], [self._build_result('fail')]),
(False, [self._build_result('error')], [self._build_result('error')]),
]
for (expect_passed, chrome_results, firefox_results) in cases:
data = ResultData()
data.add_results('chrome', chrome_results)
data.add_results('firefox', firefox_results)
self.assertEqual(data.all_passed(), expect_passed)