def test_overall_stats(self): data = ResultData() data.add_results('chrome', [ self._build_result('error'), self._build_result('error'), self._build_result('fail'), self._build_result('fail'), self._build_result('fail'), self._build_result('pass'), self._build_result('pass'), self._build_result('pass'), self._build_result('pass'), self._build_result('skip'), self._build_result('skip'), self._build_result('skip'), self._build_result('skip'), self._build_result('skip'), ]) data.add_results('firefox', [ self._build_result('error'), self._build_result('fail'), self._build_result('pass'), self._build_result('skip'), ]) # Expect that the overall stats (no browser specified) # is the sum of all browser-specific stats expected_stats = self._build_stats( num_error=3, num_failed=4, num_passed=5, num_skipped=6 ) self.assertEqual(data.stats(None), expected_stats)
def test_stats(self): # Tuples relating input results to stats cases = [ ([], self._build_stats()), ([self._build_result('error')], self._build_stats(num_error=1)), ([self._build_result('fail')], self._build_stats(num_failed=1)), ([self._build_result('skip')], self._build_stats(num_skipped=1)), ([self._build_result('pass')], self._build_stats(num_passed=1)), ([ self._build_result('error'), self._build_result('error'), self._build_result('fail'), self._build_result('fail'), self._build_result('fail'), self._build_result('pass'), self._build_result('pass'), self._build_result('pass'), self._build_result('pass'), self._build_result('skip'), self._build_result('skip'), self._build_result('skip'), self._build_result('skip'), self._build_result('skip'), ], self._build_stats( num_error=2, num_failed=3, num_passed=4, num_skipped=5 )) ] for input_results, expected_stats in cases: data = ResultData() data.add_results('chrome', input_results) self.assertEqual(data.stats('chrome'), expected_stats)
def testadd_results(self): data = ResultData() input_results = [self._build_result('pass'), self._build_result('fail')] data.add_results('chrome', input_results) self.assertEqual(data.browsers(), ['chrome']) self.assertEqual(data.test_results('chrome'), input_results) self.assertEqual(data.all_passed(), False)
def run(self): """ Execute each available test suite page and record whether each test passed/failed. If configured, will write test results using reporters. Returns a `ResultData` object containing the test results. """ # Start the suite page server running on a local port self._suite_page_server.start() # Create an object to hold results data for all browsers results_data = ResultData() try: for browser in self._browser_list: # Run the test suite with one of our browsers results_data.add_results( browser.name(), self._run_with_browser(browser) ) # After all browsers have loaded their pages, # Block until all coverage data received # (if coverage is not configured, this will # not do anything). try: self._report_coverage_data = self._suite_page_server.all_coverage_data() # If we timed out, log it, but don't exit with failure except TimeoutError: msg = dedent(""" Did not receive all coverage data. No coverage reports will be written. (This sometimes occurs when JSCover does not have enough memory to run.) """).strip() LOGGER.warning(msg) # Re-raise any exceptions that occur except: raise # Stop the suite page server, freeing up the port finally: self._suite_page_server.stop() # Generate test result reports for reporter in self._result_reporters: reporter.write_report(results_data) return results_data
class ResultReporterTestCase(TestCase): """ Base class for reporter tests. """ # Subclasses override this to specify the class under test REPORTER_CLASS = None def setUp(self): self.result_data = ResultData() def add_result(self, browser_name, group_name, test_name, status, detail): """ Add test results for the browser with `browser_name`. `group_name`: name of the test group (e.g. 'Adder tests') `test_name`: name of the specific test case (e.g. 'it should start at zero') `status`: pass | fail | skip `detail`: details of the test case (e.g. a stack trace) """ self.result_data.add_results( browser_name, [{ 'test_group': group_name, 'test_name': test_name, 'status': status, 'detail': detail }] ) def assert_report(self, expected_report): """ Assert that the console report matches `expected_report`. """ assert_long_str_equal(self.build_report(), expected_report, strip=True) def build_report(self): """ Build a report. """ output = StringIO() self.REPORTER_CLASS(output).write_report(self.result_data) return output.getvalue()
def test_all_passed(self): # Tuples relating all_passed to input results from each browser. cases = [ (True, [], []), (True, [self._build_result('pass')], []), (True, [self._build_result('pass'), self._build_result('pass')], []), (True, [self._build_result('pass')], [self._build_result('pass')]), (False, [self._build_result('fail')], [self._build_result('pass')]), (False, [self._build_result('error')], [self._build_result('pass')]), (False, [self._build_result('fail')], [self._build_result('fail')]), (False, [self._build_result('error')], [self._build_result('fail')]), (False, [self._build_result('error')], [self._build_result('error')]), ] for (expect_passed, chrome_results, firefox_results) in cases: data = ResultData() data.add_results('chrome', chrome_results) data.add_results('firefox', firefox_results) self.assertEqual(data.all_passed(), expect_passed)