def summarized_results(self, port, expected, passing, flaky, extra_tests=[], extra_expectations=None):
tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html', 'failures/expected/wontfix.html']
if extra_tests:
tests.extend(extra_tests)
expectations = ''
if extra_expectations:
expectations += extra_expectations
paths, rs, exp = self.get_result_summary(port, tests, expectations)
if expected:
rs.add(self.get_result('passes/text.html', test_expectations.PASS), expected)
rs.add(self.get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected)
rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), expected)
elif passing:
rs.add(self.get_result('passes/text.html'), expected)
rs.add(self.get_result('failures/expected/timeout.html'), expected)
rs.add(self.get_result('failures/expected/crash.html'), expected)
else:
rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), expected)
rs.add(self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected)
rs.add(self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected)
for test in extra_tests:
rs.add(self.get_result(test, test_expectations.CRASH), expected)
retry = rs
if flaky:
paths, retry, exp = self.get_result_summary(port, tests, expectations)
retry.add(self.get_result('passes/text.html'), True)
retry.add(self.get_result('failures/expected/timeout.html'), True)
retry.add(self.get_result('failures/expected/crash.html'), True)
unexpected_results = manager.summarize_results(port, exp, rs, retry, test_timings={}, only_unexpected=True, interrupted=False)
expected_results = manager.summarize_results(port, exp, rs, retry, test_timings={}, only_unexpected=False, interrupted=False)
return expected_results, unexpected_results
def get_unexpected_results(expected, passing, flaky):
"""Return an unexpected results summary matching the input description.
There are a lot of different combinations of test results that
can be tested; this routine produces various combinations based
on the values of the input flags.
Args
expected: whether the tests ran as expected
passing: whether the tests should all pass
flaky: whether the tests should be flaky (if False, they
produce the same results on both runs; if True, they
all pass on the second run).
"""
paths, rs, exp = self.get_result_summary(tests, expectations)
if expected:
rs.add(
self.get_result('passes/text.html',
test_expectations.PASS), expected)
rs.add(
self.get_result('failures/expected/timeout.html',
test_expectations.TIMEOUT), expected)
rs.add(
self.get_result('failures/expected/crash.html',
test_expectations.CRASH), expected)
elif passing:
rs.add(self.get_result('passes/text.html'), expected)
rs.add(self.get_result('failures/expected/timeout.html'),
expected)
rs.add(self.get_result('failures/expected/crash.html'),
expected)
else:
rs.add(
self.get_result('passes/text.html',
test_expectations.TIMEOUT), expected)
rs.add(
self.get_result('failures/expected/timeout.html',
test_expectations.CRASH), expected)
rs.add(
self.get_result('failures/expected/crash.html',
test_expectations.TIMEOUT), expected)
retry = rs
if flaky:
paths, retry, exp = self.get_result_summary(
tests, expectations)
retry.add(self.get_result('passes/text.html'), True)
retry.add(self.get_result('failures/expected/timeout.html'),
True)
retry.add(self.get_result('failures/expected/crash.html'),
True)
unexpected_results = manager.summarize_results(
self._port,
exp,
rs,
retry,
test_timings={},
only_unexpected=True,
interrupted=False)
return unexpected_results
def summarized_results(self, port, expected, passing, flaky, extra_tests=[], extra_expectations=None):
tests = [
"passes/text.html",
"failures/expected/timeout.html",
"failures/expected/crash.html",
"failures/expected/wontfix.html",
]
if extra_tests:
tests.extend(extra_tests)
expectations = ""
if extra_expectations:
expectations += extra_expectations
test_is_slow = False
paths, rs, exp = self.get_result_summary(port, tests, expectations)
if expected:
rs.add(self.get_result("passes/text.html", test_expectations.PASS), expected, test_is_slow)
rs.add(self.get_result("failures/expected/timeout.html", test_expectations.TIMEOUT), expected, test_is_slow)
rs.add(self.get_result("failures/expected/crash.html", test_expectations.CRASH), expected, test_is_slow)
elif passing:
rs.add(self.get_result("passes/text.html"), expected, test_is_slow)
rs.add(self.get_result("failures/expected/timeout.html"), expected, test_is_slow)
rs.add(self.get_result("failures/expected/crash.html"), expected, test_is_slow)
else:
rs.add(self.get_result("passes/text.html", test_expectations.TIMEOUT), expected, test_is_slow)
rs.add(self.get_result("failures/expected/timeout.html", test_expectations.CRASH), expected, test_is_slow)
rs.add(self.get_result("failures/expected/crash.html", test_expectations.TIMEOUT), expected, test_is_slow)
for test in extra_tests:
rs.add(self.get_result(test, test_expectations.CRASH), expected, test_is_slow)
retry = rs
if flaky:
paths, retry, exp = self.get_result_summary(port, tests, expectations)
retry.add(self.get_result("passes/text.html"), True, test_is_slow)
retry.add(self.get_result("failures/expected/timeout.html"), True, test_is_slow)
retry.add(self.get_result("failures/expected/crash.html"), True, test_is_slow)
unexpected_results = summarize_results(port, exp, rs, retry, only_unexpected=True)
expected_results = summarize_results(port, exp, rs, retry, only_unexpected=False)
return expected_results, unexpected_results
def summarized_results(self, port, expected, passing, flaky, extra_tests=[], extra_expectations=None):
tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html', 'failures/expected/wontfix.html']
if extra_tests:
tests.extend(extra_tests)
expectations = ''
if extra_expectations:
expectations += extra_expectations
test_is_slow = False
paths, rs, exp = self.get_result_summary(port, tests, expectations)
if expected:
rs.add(self.get_result('passes/text.html', test_expectations.PASS), expected, test_is_slow)
rs.add(self.get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected, test_is_slow)
rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), expected, test_is_slow)
elif passing:
rs.add(self.get_result('passes/text.html'), expected, test_is_slow)
rs.add(self.get_result('failures/expected/timeout.html'), expected, test_is_slow)
rs.add(self.get_result('failures/expected/crash.html'), expected, test_is_slow)
else:
rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), expected, test_is_slow)
rs.add(self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected, test_is_slow)
rs.add(self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected, test_is_slow)
for test in extra_tests:
rs.add(self.get_result(test, test_expectations.CRASH), expected, test_is_slow)
retry = rs
if flaky:
paths, retry, exp = self.get_result_summary(port, tests, expectations)
retry.add(self.get_result('passes/text.html'), True, test_is_slow)
retry.add(self.get_result('failures/expected/timeout.html'), True, test_is_slow)
retry.add(self.get_result('failures/expected/crash.html'), True, test_is_slow)
unexpected_results = manager.summarize_results(port, exp, rs, retry, test_timings={}, only_unexpected=True, interrupted=False)
expected_results = manager.summarize_results(port, exp, rs, retry, test_timings={}, only_unexpected=False, interrupted=False)
return expected_results, unexpected_results
def get_unexpected_results(self, port, expected, passing, flaky):
tests = [
'passes/text.html', 'failures/expected/timeout.html',
'failures/expected/crash.html'
]
expectations = ''
paths, rs, exp = self.get_result_summary(port, tests, expectations)
if expected:
rs.add(self.get_result('passes/text.html', test_expectations.PASS),
expected)
rs.add(
self.get_result('failures/expected/timeout.html',
test_expectations.TIMEOUT), expected)
rs.add(
self.get_result('failures/expected/crash.html',
test_expectations.CRASH), expected)
elif passing:
rs.add(self.get_result('passes/text.html'), expected)
rs.add(self.get_result('failures/expected/timeout.html'), expected)
rs.add(self.get_result('failures/expected/crash.html'), expected)
else:
rs.add(
self.get_result('passes/text.html', test_expectations.TIMEOUT),
expected)
rs.add(
self.get_result('failures/expected/timeout.html',
test_expectations.CRASH), expected)
rs.add(
self.get_result('failures/expected/crash.html',
test_expectations.TIMEOUT), expected)
retry = rs
if flaky:
paths, retry, exp = self.get_result_summary(
port, tests, expectations)
retry.add(self.get_result('passes/text.html'), True)
retry.add(self.get_result('failures/expected/timeout.html'), True)
retry.add(self.get_result('failures/expected/crash.html'), True)
unexpected_results = manager.summarize_results(port,
exp,
rs,
retry,
test_timings={},
only_unexpected=True,
interrupted=False)
return unexpected_results
def get_unexpected_results(expected, passing, flaky):
"""Return an unexpected results summary matching the input description.
There are a lot of different combinations of test results that
can be tested; this routine produces various combinations based
on the values of the input flags.
Args
expected: whether the tests ran as expected
passing: whether the tests should all pass
flaky: whether the tests should be flaky (if False, they
produce the same results on both runs; if True, they
all pass on the second run).
"""
port = MockHost().port_factory.get('test')
test_is_slow = False
paths, rs, exp = self.get_result_summary(port, tests, expectations)
if expected:
rs.add(self.get_result('passes/text.html', test_expectations.PASS), expected, test_is_slow)
rs.add(self.get_result('failures/expected/audio.html', test_expectations.AUDIO), expected, test_is_slow)
rs.add(self.get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected, test_is_slow)
rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), expected, test_is_slow)
elif passing:
rs.add(self.get_result('passes/text.html'), expected, test_is_slow)
rs.add(self.get_result('failures/expected/audio.html'), expected, test_is_slow)
rs.add(self.get_result('failures/expected/timeout.html'), expected, test_is_slow)
rs.add(self.get_result('failures/expected/crash.html'), expected, test_is_slow)
else:
rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), expected, test_is_slow)
rs.add(self.get_result('failures/expected/audio.html', test_expectations.CRASH), expected, test_is_slow)
rs.add(self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected, test_is_slow)
rs.add(self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected, test_is_slow)
retry = rs
if flaky:
paths, retry, exp = self.get_result_summary(port, tests, expectations)
retry.add(self.get_result('passes/text.html'), True, test_is_slow)
retry.add(self.get_result('failures/expected/audio.html'), True, test_is_slow)
retry.add(self.get_result('failures/expected/timeout.html'), True, test_is_slow)
retry.add(self.get_result('failures/expected/crash.html'), True, test_is_slow)
return manager.summarize_results(port, exp, rs, retry)
def get_unexpected_results(expected, passing, flaky):
"""Return an unexpected results summary matching the input description.
There are a lot of different combinations of test results that
can be tested; this routine produces various combinations based
on the values of the input flags.
Args
expected: whether the tests ran as expected
passing: whether the tests should all pass
flaky: whether the tests should be flaky (if False, they
produce the same results on both runs; if True, they
all pass on the second run).
"""
paths, rs, exp = self.get_result_summary(tests, expectations)
if expected:
rs.add(self.get_result("passes/text.html", test_expectations.PASS), expected)
rs.add(self.get_result("failures/expected/timeout.html", test_expectations.TIMEOUT), expected)
rs.add(self.get_result("failures/expected/crash.html", test_expectations.CRASH), expected)
elif passing:
rs.add(self.get_result("passes/text.html"), expected)
rs.add(self.get_result("failures/expected/timeout.html"), expected)
rs.add(self.get_result("failures/expected/crash.html"), expected)
else:
rs.add(self.get_result("passes/text.html", test_expectations.TIMEOUT), expected)
rs.add(self.get_result("failures/expected/timeout.html", test_expectations.CRASH), expected)
rs.add(self.get_result("failures/expected/crash.html", test_expectations.TIMEOUT), expected)
retry = rs
if flaky:
paths, retry, exp = self.get_result_summary(tests, expectations)
retry.add(self.get_result("passes/text.html"), True)
retry.add(self.get_result("failures/expected/timeout.html"), True)
retry.add(self.get_result("failures/expected/crash.html"), True)
unexpected_results = manager.summarize_results(
self._port, exp, rs, retry, test_timings={}, only_unexpected=True, interrupted=False
)
return unexpected_results
def get_unexpected_results(self, port, expected, passing, flaky):
tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html']
expectations = ''
paths, rs, exp = self.get_result_summary(port, tests, expectations)
if expected:
rs.add(self.get_result('passes/text.html', test_expectations.PASS), expected)
rs.add(self.get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected)
rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), expected)
elif passing:
rs.add(self.get_result('passes/text.html'), expected)
rs.add(self.get_result('failures/expected/timeout.html'), expected)
rs.add(self.get_result('failures/expected/crash.html'), expected)
else:
rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), expected)
rs.add(self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected)
rs.add(self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected)
retry = rs
if flaky:
paths, retry, exp = self.get_result_summary(port, tests, expectations)
retry.add(self.get_result('passes/text.html'), True)
retry.add(self.get_result('failures/expected/timeout.html'), True)
retry.add(self.get_result('failures/expected/crash.html'), True)
unexpected_results = manager.summarize_results(port, exp, rs, retry, test_timings={}, only_unexpected=True, interrupted=False)
return unexpected_results
