def summarized_results(self, port, expected, passing, flaky, extra_tests=[], extra_expectations=None): tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html', 'failures/expected/wontfix.html'] if extra_tests: tests.extend(extra_tests) expectations = '' if extra_expectations: expectations += extra_expectations paths, rs, exp = self.get_result_summary(port, tests, expectations) if expected: rs.add(self.get_result('passes/text.html', test_expectations.PASS), expected) rs.add(self.get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected) rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), expected) elif passing: rs.add(self.get_result('passes/text.html'), expected) rs.add(self.get_result('failures/expected/timeout.html'), expected) rs.add(self.get_result('failures/expected/crash.html'), expected) else: rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), expected) rs.add(self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected) rs.add(self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected) for test in extra_tests: rs.add(self.get_result(test, test_expectations.CRASH), expected) retry = rs if flaky: paths, retry, exp = self.get_result_summary(port, tests, expectations) retry.add(self.get_result('passes/text.html'), True) retry.add(self.get_result('failures/expected/timeout.html'), True) retry.add(self.get_result('failures/expected/crash.html'), True) unexpected_results = manager.summarize_results(port, exp, rs, retry, test_timings={}, only_unexpected=True, interrupted=False) expected_results = manager.summarize_results(port, exp, rs, retry, test_timings={}, only_unexpected=False, interrupted=False) return expected_results, unexpected_results
def get_unexpected_results(expected, passing, flaky): """Return an unexpected results summary matching the input description. There are a lot of different combinations of test results that can be tested; this routine produces various combinations based on the values of the input flags. Args expected: whether the tests ran as expected passing: whether the tests should all pass flaky: whether the tests should be flaky (if False, they produce the same results on both runs; if True, they all pass on the second run). """ paths, rs, exp = self.get_result_summary(tests, expectations) if expected: rs.add( self.get_result('passes/text.html', test_expectations.PASS), expected) rs.add( self.get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected) rs.add( self.get_result('failures/expected/crash.html', test_expectations.CRASH), expected) elif passing: rs.add(self.get_result('passes/text.html'), expected) rs.add(self.get_result('failures/expected/timeout.html'), expected) rs.add(self.get_result('failures/expected/crash.html'), expected) else: rs.add( self.get_result('passes/text.html', test_expectations.TIMEOUT), expected) rs.add( self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected) rs.add( self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected) retry = rs if flaky: paths, retry, exp = self.get_result_summary( tests, expectations) retry.add(self.get_result('passes/text.html'), True) retry.add(self.get_result('failures/expected/timeout.html'), True) retry.add(self.get_result('failures/expected/crash.html'), True) unexpected_results = manager.summarize_results( self._port, exp, rs, retry, test_timings={}, only_unexpected=True, interrupted=False) return unexpected_results
def summarized_results(self, port, expected, passing, flaky, extra_tests=[], extra_expectations=None): tests = [ "passes/text.html", "failures/expected/timeout.html", "failures/expected/crash.html", "failures/expected/wontfix.html", ] if extra_tests: tests.extend(extra_tests) expectations = "" if extra_expectations: expectations += extra_expectations test_is_slow = False paths, rs, exp = self.get_result_summary(port, tests, expectations) if expected: rs.add(self.get_result("passes/text.html", test_expectations.PASS), expected, test_is_slow) rs.add(self.get_result("failures/expected/timeout.html", test_expectations.TIMEOUT), expected, test_is_slow) rs.add(self.get_result("failures/expected/crash.html", test_expectations.CRASH), expected, test_is_slow) elif passing: rs.add(self.get_result("passes/text.html"), expected, test_is_slow) rs.add(self.get_result("failures/expected/timeout.html"), expected, test_is_slow) rs.add(self.get_result("failures/expected/crash.html"), expected, test_is_slow) else: rs.add(self.get_result("passes/text.html", test_expectations.TIMEOUT), expected, test_is_slow) rs.add(self.get_result("failures/expected/timeout.html", test_expectations.CRASH), expected, test_is_slow) rs.add(self.get_result("failures/expected/crash.html", test_expectations.TIMEOUT), expected, test_is_slow) for test in extra_tests: rs.add(self.get_result(test, test_expectations.CRASH), expected, test_is_slow) retry = rs if flaky: paths, retry, exp = self.get_result_summary(port, tests, expectations) retry.add(self.get_result("passes/text.html"), True, test_is_slow) retry.add(self.get_result("failures/expected/timeout.html"), True, test_is_slow) retry.add(self.get_result("failures/expected/crash.html"), True, test_is_slow) unexpected_results = summarize_results(port, exp, rs, retry, only_unexpected=True) expected_results = summarize_results(port, exp, rs, retry, only_unexpected=False) return expected_results, unexpected_results
def summarized_results(self, port, expected, passing, flaky, extra_tests=[], extra_expectations=None): tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html', 'failures/expected/wontfix.html'] if extra_tests: tests.extend(extra_tests) expectations = '' if extra_expectations: expectations += extra_expectations test_is_slow = False paths, rs, exp = self.get_result_summary(port, tests, expectations) if expected: rs.add(self.get_result('passes/text.html', test_expectations.PASS), expected, test_is_slow) rs.add(self.get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected, test_is_slow) rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), expected, test_is_slow) elif passing: rs.add(self.get_result('passes/text.html'), expected, test_is_slow) rs.add(self.get_result('failures/expected/timeout.html'), expected, test_is_slow) rs.add(self.get_result('failures/expected/crash.html'), expected, test_is_slow) else: rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), expected, test_is_slow) rs.add(self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected, test_is_slow) rs.add(self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected, test_is_slow) for test in extra_tests: rs.add(self.get_result(test, test_expectations.CRASH), expected, test_is_slow) retry = rs if flaky: paths, retry, exp = self.get_result_summary(port, tests, expectations) retry.add(self.get_result('passes/text.html'), True, test_is_slow) retry.add(self.get_result('failures/expected/timeout.html'), True, test_is_slow) retry.add(self.get_result('failures/expected/crash.html'), True, test_is_slow) unexpected_results = manager.summarize_results(port, exp, rs, retry, test_timings={}, only_unexpected=True, interrupted=False) expected_results = manager.summarize_results(port, exp, rs, retry, test_timings={}, only_unexpected=False, interrupted=False) return expected_results, unexpected_results
def get_unexpected_results(self, port, expected, passing, flaky): tests = [ 'passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html' ] expectations = '' paths, rs, exp = self.get_result_summary(port, tests, expectations) if expected: rs.add(self.get_result('passes/text.html', test_expectations.PASS), expected) rs.add( self.get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected) rs.add( self.get_result('failures/expected/crash.html', test_expectations.CRASH), expected) elif passing: rs.add(self.get_result('passes/text.html'), expected) rs.add(self.get_result('failures/expected/timeout.html'), expected) rs.add(self.get_result('failures/expected/crash.html'), expected) else: rs.add( self.get_result('passes/text.html', test_expectations.TIMEOUT), expected) rs.add( self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected) rs.add( self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected) retry = rs if flaky: paths, retry, exp = self.get_result_summary( port, tests, expectations) retry.add(self.get_result('passes/text.html'), True) retry.add(self.get_result('failures/expected/timeout.html'), True) retry.add(self.get_result('failures/expected/crash.html'), True) unexpected_results = manager.summarize_results(port, exp, rs, retry, test_timings={}, only_unexpected=True, interrupted=False) return unexpected_results
def get_unexpected_results(expected, passing, flaky): """Return an unexpected results summary matching the input description. There are a lot of different combinations of test results that can be tested; this routine produces various combinations based on the values of the input flags. Args expected: whether the tests ran as expected passing: whether the tests should all pass flaky: whether the tests should be flaky (if False, they produce the same results on both runs; if True, they all pass on the second run). """ port = MockHost().port_factory.get('test') test_is_slow = False paths, rs, exp = self.get_result_summary(port, tests, expectations) if expected: rs.add(self.get_result('passes/text.html', test_expectations.PASS), expected, test_is_slow) rs.add(self.get_result('failures/expected/audio.html', test_expectations.AUDIO), expected, test_is_slow) rs.add(self.get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected, test_is_slow) rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), expected, test_is_slow) elif passing: rs.add(self.get_result('passes/text.html'), expected, test_is_slow) rs.add(self.get_result('failures/expected/audio.html'), expected, test_is_slow) rs.add(self.get_result('failures/expected/timeout.html'), expected, test_is_slow) rs.add(self.get_result('failures/expected/crash.html'), expected, test_is_slow) else: rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), expected, test_is_slow) rs.add(self.get_result('failures/expected/audio.html', test_expectations.CRASH), expected, test_is_slow) rs.add(self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected, test_is_slow) rs.add(self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected, test_is_slow) retry = rs if flaky: paths, retry, exp = self.get_result_summary(port, tests, expectations) retry.add(self.get_result('passes/text.html'), True, test_is_slow) retry.add(self.get_result('failures/expected/audio.html'), True, test_is_slow) retry.add(self.get_result('failures/expected/timeout.html'), True, test_is_slow) retry.add(self.get_result('failures/expected/crash.html'), True, test_is_slow) return manager.summarize_results(port, exp, rs, retry)
def get_unexpected_results(expected, passing, flaky): """Return an unexpected results summary matching the input description. There are a lot of different combinations of test results that can be tested; this routine produces various combinations based on the values of the input flags. Args expected: whether the tests ran as expected passing: whether the tests should all pass flaky: whether the tests should be flaky (if False, they produce the same results on both runs; if True, they all pass on the second run). """ paths, rs, exp = self.get_result_summary(tests, expectations) if expected: rs.add(self.get_result("passes/text.html", test_expectations.PASS), expected) rs.add(self.get_result("failures/expected/timeout.html", test_expectations.TIMEOUT), expected) rs.add(self.get_result("failures/expected/crash.html", test_expectations.CRASH), expected) elif passing: rs.add(self.get_result("passes/text.html"), expected) rs.add(self.get_result("failures/expected/timeout.html"), expected) rs.add(self.get_result("failures/expected/crash.html"), expected) else: rs.add(self.get_result("passes/text.html", test_expectations.TIMEOUT), expected) rs.add(self.get_result("failures/expected/timeout.html", test_expectations.CRASH), expected) rs.add(self.get_result("failures/expected/crash.html", test_expectations.TIMEOUT), expected) retry = rs if flaky: paths, retry, exp = self.get_result_summary(tests, expectations) retry.add(self.get_result("passes/text.html"), True) retry.add(self.get_result("failures/expected/timeout.html"), True) retry.add(self.get_result("failures/expected/crash.html"), True) unexpected_results = manager.summarize_results( self._port, exp, rs, retry, test_timings={}, only_unexpected=True, interrupted=False ) return unexpected_results
def get_unexpected_results(self, port, expected, passing, flaky): tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html'] expectations = '' paths, rs, exp = self.get_result_summary(port, tests, expectations) if expected: rs.add(self.get_result('passes/text.html', test_expectations.PASS), expected) rs.add(self.get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected) rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), expected) elif passing: rs.add(self.get_result('passes/text.html'), expected) rs.add(self.get_result('failures/expected/timeout.html'), expected) rs.add(self.get_result('failures/expected/crash.html'), expected) else: rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), expected) rs.add(self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected) rs.add(self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected) retry = rs if flaky: paths, retry, exp = self.get_result_summary(port, tests, expectations) retry.add(self.get_result('passes/text.html'), True) retry.add(self.get_result('failures/expected/timeout.html'), True) retry.add(self.get_result('failures/expected/crash.html'), True) unexpected_results = manager.summarize_results(port, exp, rs, retry, test_timings={}, only_unexpected=True, interrupted=False) return unexpected_results