def run(port, options, args, regular_output=sys.stderr, buildbot_output=sys.stdout): try: warnings = _set_up_derived_options(port, options) printer = printing.Printer(port, options, regular_output, buildbot_output, logger=logging.getLogger()) for warning in warnings: _log.warning(warning) if options.lint_test_files: return lint(port, options) # We wrap any parts of the run that are slow or likely to raise exceptions # in a try/finally to ensure that we clean up the logging configuration. unexpected_result_count = -1 manager = Manager(port, options, printer) printer.print_config() unexpected_result_count = manager.run(args) _log.debug("Testing completed, Exit status: %d" % unexpected_result_count) except Exception: exception_type, exception_value, exception_traceback = sys.exc_info() if exception_type not in (KeyboardInterrupt, TestRunInterruptedException, WorkerException): print >> sys.stderr, '\n%s raised: %s' % (exception_type.__name__, exception_value) stack_utils.log_traceback(_log.error, exception_traceback) raise finally: printer.cleanup() return unexpected_result_count
def get_manager_with_tests(test_names): port = Mock() # FIXME: Use a tighter mock. port.TEST_PATH_SEPARATOR = '/' manager = Manager(port, options=MockOptions(http=True), printer=Mock()) manager._test_files = set(test_names) manager._test_files_list = test_names return manager
def test_interrupt_if_at_failure_limits(self): port = Mock() # FIXME: This should be a tighter mock. port.TEST_PATH_SEPARATOR = '/' port._filesystem = MockFileSystem() manager = Manager(port=port, options=MockOptions(), printer=Mock()) manager._options = MockOptions(exit_after_n_failures=None, exit_after_n_crashes_or_timeouts=None) result_summary = ResultSummary(expectations=Mock(), test_files=[]) result_summary.unexpected_failures = 100 result_summary.unexpected_crashes = 50 result_summary.unexpected_timeouts = 50 # No exception when the exit_after* options are None. manager._interrupt_if_at_failure_limits(result_summary) # No exception when we haven't hit the limit yet. manager._options.exit_after_n_failures = 101 manager._options.exit_after_n_crashes_or_timeouts = 101 manager._interrupt_if_at_failure_limits(result_summary) # Interrupt if we've exceeded either limit: manager._options.exit_after_n_crashes_or_timeouts = 10 self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary) manager._options.exit_after_n_crashes_or_timeouts = None manager._options.exit_after_n_failures = 10 exception = self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary)
def test_fallback_path_in_config(self): options = self.get_options() host = MockHost() port = host.port_factory.get('test-mac-leopard', options=options) printer = self.get_printer() manager = Manager(port, port.options, printer) manager.print_config() self.assertTrue('Baseline search path: test-mac-leopard -> test-mac-snowleopard -> generic' in printer.output)
def run(port, options, args, logging_stream): try: printer = printing.Printer(port, options, logging_stream, logger=logging.getLogger()) _set_up_derived_options(port, options) manager = Manager(port, options, printer) printer.print_config(port.results_directory()) run_details = manager.run(args) _log.debug("Testing completed, Exit status: %d" % run_details.exit_code) return run_details finally: printer.cleanup()
def test_update_summary_with_result(self): host = MockHost() port = host.port_factory.get('test-win-xp') test = 'failures/expected/reftest.html' port.test_expectations = lambda: 'WONTFIX : failures/expected/reftest.html = IMAGE' expectations = TestExpectations(port, tests=[test]) # Reftests expected to be image mismatch should be respected when pixel_tests=False. manager = Manager(port=port, options=MockOptions(pixel_tests=False, exit_after_n_failures=None, exit_after_n_crashes_or_timeouts=None), printer=Mock()) manager._expectations = expectations result_summary = ResultSummary(expectations=expectations, test_files=[test]) result = TestResult(test_name=test, failures=[test_failures.FailureReftestMismatchDidNotOccur()]) manager._update_summary_with_result(result_summary, result) self.assertEquals(1, result_summary.expected) self.assertEquals(0, result_summary.unexpected)
def get_manager(): host = MockHost() port = host.port_factory.get('test-mac-leopard') manager = Manager(port, options=MockOptions(test_list=None, http=True), printer=Mock()) return manager
def run(port, options, args, logging_stream): logger = logging.getLogger() logger.setLevel(logging.DEBUG if options.debug_rwt_logging else logging.INFO) try: printer = printing.Printer(port, options, logging_stream, logger=logger) _set_up_derived_options(port, options) manager = Manager(port, options, printer) printer.print_config(port.results_directory()) run_details = manager.run(args) _log.debug("Testing completed, Exit status: %d" % run_details.exit_code) return run_details finally: printer.cleanup()
def get_manager(): port = Mock() # FIXME: Use a tighter mock. port.TEST_PATH_SEPARATOR = '/' manager = Manager(port, options=MockOptions(http=True, max_locked_shards=1), printer=Mock()) return manager
def _print_expectations(port, options, args, logging_stream): logger = logging.getLogger() logger.setLevel(logging.DEBUG if options.debug_rwt_logging else logging.INFO) try: printer = printing.Printer(port, options, logging_stream, logger=logger) _set_up_derived_options(port, options) manager = Manager(port, options, printer) exit_code = manager.print_expectations(args) _log.debug("Printing expectations completed, Exit status: %d", exit_code) return exit_code except Exception as error: _log.error('Error printing expectations: {}'.format(error)) finally: printer.cleanup() return -1
def get_manager(port): manager = Manager(port, options=optparse.Values({ 'http': True, 'max_locked_shards': 1 }), printer=FakePrinter()) return manager
def get_manager(): port = Mock() # FIXME: Use a tighter mock. port.TEST_PATH_SEPARATOR = '/' port.web_platform_test_server_doc_root = get_wpt_doc_root manager = Manager(port, options=MockOptions(http=True), printer=Mock()) return manager
def run(port, options, args, regular_output=sys.stderr, buildbot_output=sys.stdout): try: printer = printing.Printer(port, options, regular_output, buildbot_output, logger=logging.getLogger()) _set_up_derived_options(port, options) if options.lint_test_files: return lint(port, options) manager = Manager(port, options, printer) printer.print_config(port.results_directory()) unexpected_result_count = manager.run(args) _log.debug("Testing completed, Exit status: %d" % unexpected_result_count) return unexpected_result_count finally: printer.cleanup()
def get_manager_with_tests(test_names): host = MockHost() port = host.port_factory.get() manager = Manager(port, options=MockOptions(test_list=None, http=True), printer=Mock()) manager.collect_tests(test_names) return manager
def get_manager(): host = MockHost() port = host.port_factory.get('test-mac-leopard') manager = Manager(port, options=MockOptions(http=True, max_locked_shards=1), printer=FakePrinter()) return manager
def test_servers_started(self): def start_http_server(number_of_servers=None): self.http_started = True def start_websocket_server(): self.websocket_started = True def stop_http_server(): self.http_stopped = True def stop_websocket_server(): self.websocket_stopped = True host = MockHost() port = host.port_factory.get('test-mac-leopard') port.start_http_server = start_http_server port.start_websocket_server = start_websocket_server port.stop_http_server = stop_http_server port.stop_websocket_server = stop_websocket_server self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False manager = Manager(port=port, options=MockOptions(http=True), printer=Mock()) manager._test_files = ['http/tests/pass.txt'] manager.start_servers_with_lock(number_of_servers=4) self.assertEquals(self.http_started, True) self.assertEquals(self.websocket_started, False) manager.stop_servers_with_lock() self.assertEquals(self.http_stopped, True) self.assertEquals(self.websocket_stopped, False) self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False manager = Manager(port=port, options=MockOptions(http=True), printer=Mock()) manager._test_files = ['websocket/pass.txt'] manager.start_servers_with_lock(number_of_servers=4) self.assertEquals(self.http_started, True) self.assertEquals(self.websocket_started, True) manager.stop_servers_with_lock() self.assertEquals(self.http_stopped, True) self.assertEquals(self.websocket_stopped, True) self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False manager = Manager(port=port, options=MockOptions(http=True), printer=Mock()) manager._test_files = ['perf/foo/test.html'] manager.start_servers_with_lock(number_of_servers=4) self.assertEquals(self.http_started, False) self.assertEquals(self.websocket_started, False) manager.stop_servers_with_lock() self.assertEquals(self.http_stopped, False) self.assertEquals(self.websocket_stopped, False)
def test_interrupt_if_at_failure_limits(self): port = Mock() port.TEST_PATH_SEPARATOR = '/' port._filesystem = filesystem_mock.MockFileSystem() manager = Manager(port=port, options=MockOptions(), printer=Mock()) manager._options = MockOptions(exit_after_n_failures=None, exit_after_n_crashes_or_timeouts=None) result_summary = ResultSummary(expectations=Mock(), test_files=[]) result_summary.unexpected_failures = 100 result_summary.unexpected_crashes = 50 result_summary.unexpected_timeouts = 50 # No exception when the exit_after* options are None. manager._interrupt_if_at_failure_limits(result_summary) # No exception when we haven't hit the limit yet. manager._options.exit_after_n_failures = 101 manager._options.exit_after_n_crashes_or_timeouts = 101 manager._interrupt_if_at_failure_limits(result_summary) # Interrupt if we've exceeded either limit: manager._options.exit_after_n_crashes_or_timeouts = 10 self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary) manager._options.exit_after_n_crashes_or_timeouts = None manager._options.exit_after_n_failures = 10 exception = self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary)
def get_manager(): host = MockHost() port = host.port_factory.get() port.TEST_PATH_SEPARATOR = '/' port.web_platform_test_server_doc_root = get_wpt_doc_root manager = Manager(port, options=MockOptions(http=True), printer=Mock()) return manager
def get_manager(): host = MockHost() port = MockCustomDevicePort(host) manager = Manager(port, options=MockOptions( test_list=['fast/test-starship/lasers.html'], http=True), printer=Mock()) return manager
def get_manager(): host = MockHost() port = host.port_factory.get() manager = Manager(port, options=MockOptions(test_list=None, http=True, max_locked_shards=1), printer=Mock()) return manager
def get_manager(): host = MockHost() port = host.port_factory.get('test-mac-mac10.10') manager = Manager(port, options=optparse.Values({ 'http': True, 'max_locked_shards': 1 }), printer=FakePrinter()) return manager
def run(port, options, args, regular_output=sys.stderr, buildbot_output=sys.stdout): try: warnings = _set_up_derived_options(port, options) printer = printing.Printer(port, options, regular_output, buildbot_output, logger=logging.getLogger()) for warning in warnings: _log.warning(warning) if options.lint_test_files: return lint(port, options) # We wrap any parts of the run that are slow or likely to raise exceptions # in a try/finally to ensure that we clean up the logging configuration. unexpected_result_count = -1 manager = Manager(port, options, printer) printer.print_config(port.results_directory()) unexpected_result_count = manager.run(args) _log.debug("Testing completed, Exit status: %d" % unexpected_result_count) except Exception: exception_type, exception_value, exception_traceback = sys.exc_info() if exception_type not in (KeyboardInterrupt, TestRunInterruptedException, WorkerException): print >> sys.stderr, '\n%s raised: %s' % (exception_type.__name__, exception_value) stack_utils.log_traceback(_log.error, exception_traceback) raise finally: printer.cleanup() return unexpected_result_count
def test_fallback_path_in_config(self): options = self.get_options() host = MockHost() port = host.port_factory.get('test-mac-leopard', options=options) printer = self.get_printer() manager = Manager(port, options, printer) manager.print_config() self.assertTrue( 'Baseline search path: test-mac-leopard -> test-mac-snowleopard -> generic' in printer.output)
def run(port, options, args, regular_output=sys.stderr, buildbot_output=sys.stdout): try: warnings = _set_up_derived_options(port, options) printer = printing.Printer(port, options, regular_output, buildbot_output, logger=logging.getLogger()) for warning in warnings: _log.warning(warning) if options.help_printing: printer.help_printing() printer.cleanup() return 0 if options.lint_test_files: return lint(port, options) # We wrap any parts of the run that are slow or likely to raise exceptions # in a try/finally to ensure that we clean up the logging configuration. unexpected_result_count = -1 manager = Manager(port, options, printer) printer.print_config() printer.write_update("Collecting tests ...") try: manager.collect_tests(args) except IOError, e: if e.errno == errno.ENOENT: return -1 raise printer.write_update("Checking build ...") if not port.check_build(manager.needs_servers()): _log.error("Build check failed") return -1 printer.write_update("Parsing expectations ...") manager.parse_expectations() unexpected_result_count = manager.run() _log.debug("Testing completed, Exit status: %d" % unexpected_result_count)
def test_ensure_manifest_copies_new_manifest(self): host = MockHost() port = host.port_factory.get() manifest_path = '/mock-checkout/third_party/WebKit/LayoutTests/external/wpt/MANIFEST.json' self.assertFalse(port.host.filesystem.exists(manifest_path)) manager = Manager(port, options=optparse.Values({'max_locked_shards': 1}), printer=FakePrinter()) manager._ensure_manifest() self.assertTrue(port.host.filesystem.exists(manifest_path)) webkit_base = '/mock-checkout/third_party/WebKit' self.assertEqual(port.host.executive.calls, [[ 'python', webkit_base + '/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/manifest', '--work', '--tests-root', webkit_base + '/LayoutTests/external/wpt', ]])
def test_update_summary_with_result(self): host = MockHost() port = host.port_factory.get('test-win-xp') test = 'failures/expected/reftest.html' port.test_expectations = lambda: 'WONTFIX : failures/expected/reftest.html = IMAGE' expectations = TestExpectations(port, tests=[test]) # Reftests expected to be image mismatch should be respected when pixel_tests=False. manager = Manager(port=port, options=MockOptions( pixel_tests=False, exit_after_n_failures=None, exit_after_n_crashes_or_timeouts=None), printer=Mock()) manager._expectations = expectations result_summary = ResultSummary(expectations=expectations, test_files=[test]) result = TestResult( test_name=test, failures=[test_failures.FailureReftestMismatchDidNotOccur()]) manager._update_summary_with_result(result_summary, result) self.assertEquals(1, result_summary.expected) self.assertEquals(0, result_summary.unexpected)
def test_interrupt_if_at_failure_limits(self): port = Mock() # FIXME: This should be a tighter mock. port.TEST_PATH_SEPARATOR = '/' port._filesystem = MockFileSystem() manager = Manager(port=port, options=MockOptions(), printer=Mock()) manager._options = MockOptions(exit_after_n_failures=None, exit_after_n_crashes_or_timeouts=None) manager._test_files = ['foo/bar.html', 'baz.html'] result_summary = ResultSummary(expectations=Mock(), test_files=manager._test_files) result_summary.unexpected_failures = 100 result_summary.unexpected_crashes = 50 result_summary.unexpected_timeouts = 50 # No exception when the exit_after* options are None. manager._interrupt_if_at_failure_limits(result_summary) # No exception when we haven't hit the limit yet. manager._options.exit_after_n_failures = 101 manager._options.exit_after_n_crashes_or_timeouts = 101 manager._interrupt_if_at_failure_limits(result_summary) # Interrupt if we've exceeded either limit: manager._options.exit_after_n_crashes_or_timeouts = 10 self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary) self.assertEquals(result_summary.results['foo/bar.html'].type, test_expectations.SKIP) self.assertEquals(result_summary.results['baz.html'].type, test_expectations.SKIP) manager._options.exit_after_n_crashes_or_timeouts = None manager._options.exit_after_n_failures = 10 exception = self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary)
def run(port, options, args, regular_output=sys.stderr, buildbot_output=sys.stdout): warnings = _set_up_derived_options(port, options) printer = printing.Printer(port, options, regular_output, buildbot_output, logger=logging.getLogger()) for warning in warnings: _log.warning(warning) if options.help_printing: printer.help_printing() printer.cleanup() return 0 if options.lint_test_files: return lint(port, options) # We wrap any parts of the run that are slow or likely to raise exceptions # in a try/finally to ensure that we clean up the logging configuration. unexpected_result_count = -1 try: manager = Manager(port, options, printer) manager.print_config() printer.print_update("Collecting tests ...") try: manager.collect_tests(args) except IOError, e: if e.errno == errno.ENOENT: return -1 raise printer.print_update("Checking build ...") if not port.check_build(manager.needs_servers()): _log.error("Build check failed") return -1 printer.print_update("Parsing expectations ...") manager.parse_expectations() unexpected_result_count = manager.run() _log.debug("Testing completed, Exit status: %d" % unexpected_result_count)
def run(port, options, args, regular_output=sys.stderr, buildbot_output=sys.stdout): """Run the tests. Args: port: Port object for port-specific behavior options: a dictionary of command line options args: a list of sub directories or files to test regular_output: a stream-like object that we can send logging/debug output to buildbot_output: a stream-like object that we can write all output that is intended to be parsed by the buildbot to Returns: the number of unexpected results that occurred, or -1 if there is an error. """ warnings = _set_up_derived_options(port, options) printer = printing.Printer(port, options, regular_output, buildbot_output, configure_logging=True) for w in warnings: _log.warning(w) if options.help_printing: printer.help_printing() printer.cleanup() return 0 # We wrap any parts of the run that are slow or likely to raise exceptions # in a try/finally to ensure that we clean up the logging configuration. num_unexpected_results = -1 try: manager = Manager(port, options, printer) manager.print_config() printer.print_update("Collecting tests ...") try: manager.collect_tests(args) except IOError, e: if e.errno == errno.ENOENT: return -1 raise if options.lint_test_files: return manager.lint() printer.print_update("Checking build ...") if not port.check_build(manager.needs_servers()): _log.error("Build check failed") return -1 printer.print_update("Parsing expectations ...") manager.parse_expectations() result_summary = manager.set_up_run() if result_summary: num_unexpected_results = manager.run(result_summary) manager.clean_up_run() _log.debug("Testing completed, Exit status: %d" % num_unexpected_results)
def get_manager_with_tests(test_names): port = Mock() # FIXME: Use a tighter mock. port.TEST_PATH_SEPARATOR = '/' manager = Manager(port, options=MockOptions(http=True, max_locked_shards=1), printer=Mock()) manager._test_names = test_names return manager
def get_manager_with_tests(test_names): host = MockHost() port = host.port_factory.get('test-mac-leopard') manager = Manager(port, options=MockOptions(test_list=None, http=True, max_locked_shards=1), printer=Mock()) manager._collect_tests(test_names) return manager
def _run_tests(port, options, args, printer): _set_up_derived_options(port, options, args) manager = Manager(port, options, printer) printer.print_config(port.results_directory()) return manager.run(args)
def get_manager(port): manager = Manager(port, options=MockOptions(http=True, max_locked_shards=1), printer=FakePrinter()) return manager
def __init__(self, ref_tests, **kwargs): Manager.__init__(self, **kwargs) self._ref_tests = ref_tests
def get_manager_with_tests(test_names): port = layout_tests.port.get() manager = Manager(port, options=MockOptions(test_list=None, http=True), printer=Mock()) manager.collect_tests(test_names) return manager