def test_print_unexpected_results_with_options(self): port = MockHost().port_factory.get('test', options=MockOptions(pixel_tests=False, world_leaks=False)) DASHED_LINE = "-" * 78 + "\n" port._options.pixel_tests = True port._options.world_leaks = False printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=True, passing=False, flaky=False) printer.print_unexpected_results(summary) self.assertEqual(out.getvalue(), DASHED_LINE) port._options.pixel_tests = False port._options.world_leaks = True printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=True, passing=False, flaky=False) printer.print_unexpected_results(summary) self.assertEqual(out.getvalue(), DASHED_LINE) port._options.pixel_tests = True port._options.world_leaks = True printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=True, passing=False, flaky=False) printer.print_unexpected_results(summary) self.assertEqual(out.getvalue(), DASHED_LINE)
def test_print_results(self): port = MockHost().port_factory.get('test') printer, out = self.get_printer() initial_results = test_run_results_unittest.run_results(port) full_summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False) failing_summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False, only_include_failing=True) details = test_run_results.RunDetails(failing_summary['num_regressions'], full_summary, failing_summary, initial_results, None) printer.print_results(details) self.assertTrue(out.getvalue().find('but passed') != -1)
def test_print_results(self): port = MockHost().port_factory.get('test') printer, out = self.get_printer() initial_results = test_run_results_unittest.run_results(port) full_summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False) failing_summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False, only_include_failing=True) details = test_run_results.RunDetails(failing_summary['num_regressions'], full_summary, failing_summary, initial_results, None) printer.print_results(details) self.assertNotEmpty(out)
def test_print_results(self): port = MockHost().port_factory.get("test") printer, out = self.get_printer() initial_results = test_run_results_unittest.run_results(port) full_summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False) failing_summary = test_run_results_unittest.summarized_results( port, expected=False, passing=True, flaky=False, only_include_failing=True ) details = test_run_results.RunDetails( failing_summary["num_regressions"], full_summary, failing_summary, initial_results, None ) printer.print_results(details) self.assertTrue(out.getvalue().find("but passed") != -1)
def test_print_unexpected_results_fail_on_retry_also(self): port = MockHost().port_factory.get('test') printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=True, fail_on_retry=True) printer.print_unexpected_results(summary) output = out.getvalue() self.assertIn('Regressions: Unexpected crashes (1)\n failures/expected/timeout.html [ Crash Failure ]', output)
def test_print_results(self): port = MockHost().port_factory.get('test', options=MockOptions(pixel_tests=False, world_leaks=False)) printer, out = self.get_printer() initial_results = test_run_results_unittest.run_results(port) summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False) details = test_run_results.RunDetails(summary['num_regressions'], summary, initial_results, None) printer.print_results(details) self.assertNotEmpty(out)
def test_print_unexpected_results_fail_on_retry_also(self): port = MockHost().port_factory.get('test') printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results( port, expected=False, passing=False, flaky=False, fail_on_retry=True) printer.print_unexpected_results(summary) output = out.getvalue() self.assertIn('Regressions: Unexpected crashes (1)\n failures/expected/audio.html [ Crash Leak Leak Leak ]', output) self.assertIn( 'Regressions: Unexpected text-only failures (1)\n failures/expected/timeout.html [ Failure Failure Crash Leak ]', output)
def test_print_unexpected_results(self): port = MockHost().port_factory.get('test') printer, out = self.get_printer() # test everything running as expected DASHED_LINE = "-" * 78 + "\n" summary = test_run_results_unittest.summarized_results(port, expected=True, passing=False, flaky=False) printer.print_unexpected_results(summary) self.assertEqual(out.getvalue(), DASHED_LINE) # test failures printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=False) printer.print_unexpected_results(summary) self.assertNotEmpty(out) # test unexpected flaky printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=True) printer.print_unexpected_results(summary) self.assertNotEmpty(out) self.assertNotIn('failures/expected/crash.html', out.getvalue()) printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=False) printer.print_unexpected_results(summary) self.assertNotEmpty(out) printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=False) printer.print_unexpected_results(summary) self.assertNotEmpty(out) printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False) printer.print_unexpected_results(summary) output = out.getvalue() self.assertTrue(output) self.assertTrue('Skip' not in output)
def test_print_unexpected_results(self): port = MockHost().port_factory.get('test', options=MockOptions( pixel_tests=False, world_leaks=False)) printer, out = self.get_printer() # test everything running as expected DASHED_LINE = "-" * 78 + "\n" summary = test_run_results_unittest.summarized_results(port, expected=True, passing=False, flaky=False) printer.print_unexpected_results(summary) self.assertEqual(out.getvalue(), DASHED_LINE) # test failures printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=False) printer.print_unexpected_results(summary) self.assertNotEmpty(out) # test unexpected flaky printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=True) printer.print_unexpected_results(summary) self.assertNotEmpty(out) printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=False) printer.print_unexpected_results(summary) self.assertNotEmpty(out) printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=False) printer.print_unexpected_results(summary) self.assertNotEmpty(out) printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False) printer.print_unexpected_results(summary) self.assertNotEmpty(out)
def test_print_unexpected_results_fail_on_retry_also(self): port = MockHost().port_factory.get("test") printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results( port, expected=False, passing=False, flaky=False, fail_on_retry=True ) printer.print_unexpected_results(summary) output = out.getvalue() self.assertIn( "Regressions: Unexpected crashes (1)\n" " failures/expected/audio.html [ Crash Leak Leak Leak ]", output ) self.assertIn( "Regressions: Unexpected text-only failures (1)\n" " failures/expected/timeout.html [ Failure Failure Crash Leak ]", output, )
def test_print_unexpected_results(self): port = MockHost().port_factory.get('test') printer, out = self.get_printer() # test everything running as expected DASHED_LINE = "-" * 78 + "\n" summary = test_run_results_unittest.summarized_results(port, expected=True, passing=False, flaky=False) printer.print_unexpected_results(summary) self.assertEqual(out.getvalue(), DASHED_LINE) # test failures printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=False) printer.print_unexpected_results(summary) self.assertNotEmpty(out) # test unexpected flaky printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=True) printer.print_unexpected_results(summary) self.assertNotEmpty(out) self.assertNotIn('failures/expected/crash.html', out.getvalue()) printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=False) printer.print_unexpected_results(summary) self.assertNotEmpty(out) printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=False) printer.print_unexpected_results(summary) self.assertNotEmpty(out) printer, out = self.get_printer() summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False) printer.print_unexpected_results(summary) output = out.getvalue() self.assertTrue(output) self.assertTrue(output.find('Skip') == -1)