def test_print_unexpected_results_with_options(self):
port = MockHost().port_factory.get('test', options=MockOptions(pixel_tests=False, world_leaks=False))
DASHED_LINE = "-" * 78 + "\n"
port._options.pixel_tests = True
port._options.world_leaks = False
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port, expected=True, passing=False, flaky=False)
printer.print_unexpected_results(summary)
self.assertEqual(out.getvalue(), DASHED_LINE)
port._options.pixel_tests = False
port._options.world_leaks = True
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port, expected=True, passing=False, flaky=False)
printer.print_unexpected_results(summary)
self.assertEqual(out.getvalue(), DASHED_LINE)
port._options.pixel_tests = True
port._options.world_leaks = True
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port, expected=True, passing=False, flaky=False)
printer.print_unexpected_results(summary)
self.assertEqual(out.getvalue(), DASHED_LINE)
def test_print_results(self):
port = MockHost().port_factory.get('test')
printer, out = self.get_printer()
initial_results = test_run_results_unittest.run_results(port)
full_summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False)
failing_summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False, only_include_failing=True)
details = test_run_results.RunDetails(failing_summary['num_regressions'], full_summary, failing_summary, initial_results, None)
printer.print_results(details)
self.assertTrue(out.getvalue().find('but passed') != -1)
def test_print_results(self):
port = MockHost().port_factory.get('test')
printer, out = self.get_printer()
initial_results = test_run_results_unittest.run_results(port)
full_summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False)
failing_summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False, only_include_failing=True)
details = test_run_results.RunDetails(failing_summary['num_regressions'], full_summary, failing_summary, initial_results, None)
printer.print_results(details)
self.assertNotEmpty(out)
def test_print_results(self):
port = MockHost().port_factory.get("test")
printer, out = self.get_printer()
initial_results = test_run_results_unittest.run_results(port)
full_summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False)
failing_summary = test_run_results_unittest.summarized_results(
port, expected=False, passing=True, flaky=False, only_include_failing=True
)
details = test_run_results.RunDetails(
failing_summary["num_regressions"], full_summary, failing_summary, initial_results, None
)
printer.print_results(details)
self.assertTrue(out.getvalue().find("but passed") != -1)
def test_print_unexpected_results_fail_on_retry_also(self):
port = MockHost().port_factory.get('test')
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=True, fail_on_retry=True)
printer.print_unexpected_results(summary)
output = out.getvalue()
self.assertIn('Regressions: Unexpected crashes (1)\n failures/expected/timeout.html [ Crash Failure ]', output)
def test_print_results(self):
port = MockHost().port_factory.get('test', options=MockOptions(pixel_tests=False, world_leaks=False))
printer, out = self.get_printer()
initial_results = test_run_results_unittest.run_results(port)
summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False)
details = test_run_results.RunDetails(summary['num_regressions'], summary, initial_results, None)
printer.print_results(details)
self.assertNotEmpty(out)
def test_print_unexpected_results_fail_on_retry_also(self):
port = MockHost().port_factory.get('test')
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(
port, expected=False, passing=False, flaky=False, fail_on_retry=True)
printer.print_unexpected_results(summary)
output = out.getvalue()
self.assertIn('Regressions: Unexpected crashes (1)\n failures/expected/audio.html [ Crash Leak Leak Leak ]', output)
self.assertIn(
'Regressions: Unexpected text-only failures (1)\n failures/expected/timeout.html [ Failure Failure Crash Leak ]', output)
def test_print_unexpected_results(self):
port = MockHost().port_factory.get('test')
printer, out = self.get_printer()
# test everything running as expected
DASHED_LINE = "-" * 78 + "\n"
summary = test_run_results_unittest.summarized_results(port,
expected=True,
passing=False,
flaky=False)
printer.print_unexpected_results(summary)
self.assertEqual(out.getvalue(), DASHED_LINE)
# test failures
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port,
expected=False,
passing=False,
flaky=False)
printer.print_unexpected_results(summary)
self.assertNotEmpty(out)
# test unexpected flaky
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port,
expected=False,
passing=False,
flaky=True)
printer.print_unexpected_results(summary)
self.assertNotEmpty(out)
self.assertNotIn('failures/expected/crash.html', out.getvalue())
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port,
expected=False,
passing=False,
flaky=False)
printer.print_unexpected_results(summary)
self.assertNotEmpty(out)
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port,
expected=False,
passing=False,
flaky=False)
printer.print_unexpected_results(summary)
self.assertNotEmpty(out)
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port,
expected=False,
passing=True,
flaky=False)
printer.print_unexpected_results(summary)
output = out.getvalue()
self.assertTrue(output)
self.assertTrue('Skip' not in output)
def test_print_unexpected_results(self):
port = MockHost().port_factory.get('test',
options=MockOptions(
pixel_tests=False,
world_leaks=False))
printer, out = self.get_printer()
# test everything running as expected
DASHED_LINE = "-" * 78 + "\n"
summary = test_run_results_unittest.summarized_results(port,
expected=True,
passing=False,
flaky=False)
printer.print_unexpected_results(summary)
self.assertEqual(out.getvalue(), DASHED_LINE)
# test failures
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port,
expected=False,
passing=False,
flaky=False)
printer.print_unexpected_results(summary)
self.assertNotEmpty(out)
# test unexpected flaky
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port,
expected=False,
passing=False,
flaky=True)
printer.print_unexpected_results(summary)
self.assertNotEmpty(out)
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port,
expected=False,
passing=False,
flaky=False)
printer.print_unexpected_results(summary)
self.assertNotEmpty(out)
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port,
expected=False,
passing=False,
flaky=False)
printer.print_unexpected_results(summary)
self.assertNotEmpty(out)
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port,
expected=False,
passing=True,
flaky=False)
printer.print_unexpected_results(summary)
self.assertNotEmpty(out)
def test_print_unexpected_results_fail_on_retry_also(self):
port = MockHost().port_factory.get("test")
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(
port, expected=False, passing=False, flaky=False, fail_on_retry=True
)
printer.print_unexpected_results(summary)
output = out.getvalue()
self.assertIn(
"Regressions: Unexpected crashes (1)\n" " failures/expected/audio.html [ Crash Leak Leak Leak ]", output
)
self.assertIn(
"Regressions: Unexpected text-only failures (1)\n"
" failures/expected/timeout.html [ Failure Failure Crash Leak ]",
output,
)
def test_print_unexpected_results(self):
port = MockHost().port_factory.get('test')
printer, out = self.get_printer()
# test everything running as expected
DASHED_LINE = "-" * 78 + "\n"
summary = test_run_results_unittest.summarized_results(port, expected=True, passing=False, flaky=False)
printer.print_unexpected_results(summary)
self.assertEqual(out.getvalue(), DASHED_LINE)
# test failures
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=False)
printer.print_unexpected_results(summary)
self.assertNotEmpty(out)
# test unexpected flaky
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=True)
printer.print_unexpected_results(summary)
self.assertNotEmpty(out)
self.assertNotIn('failures/expected/crash.html', out.getvalue())
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=False)
printer.print_unexpected_results(summary)
self.assertNotEmpty(out)
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=False)
printer.print_unexpected_results(summary)
self.assertNotEmpty(out)
printer, out = self.get_printer()
summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False)
printer.print_unexpected_results(summary)
output = out.getvalue()
self.assertTrue(output)
self.assertTrue(output.find('Skip') == -1)
