def run(port, options, args, regular_output=sys.stderr, buildbot_output=sys.stdout):
try:
warnings = _set_up_derived_options(port, options)
printer = printing.Printer(port, options, regular_output, buildbot_output, logger=logging.getLogger())
for warning in warnings:
_log.warning(warning)
if options.lint_test_files:
return lint(port, options)
# We wrap any parts of the run that are slow or likely to raise exceptions
# in a try/finally to ensure that we clean up the logging configuration.
unexpected_result_count = -1
manager = Manager(port, options, printer)
printer.print_config()
unexpected_result_count = manager.run(args)
_log.debug("Testing completed, Exit status: %d" % unexpected_result_count)
except Exception:
exception_type, exception_value, exception_traceback = sys.exc_info()
if exception_type not in (KeyboardInterrupt, TestRunInterruptedException, WorkerException):
print >> sys.stderr, '\n%s raised: %s' % (exception_type.__name__, exception_value)
stack_utils.log_traceback(_log.error, exception_traceback)
raise
finally:
printer.cleanup()
return unexpected_result_count
def get_manager_with_tests(test_names):
port = Mock() # FIXME: Use a tighter mock.
port.TEST_PATH_SEPARATOR = '/'
manager = Manager(port, options=MockOptions(http=True), printer=Mock())
manager._test_files = set(test_names)
manager._test_files_list = test_names
return manager
def test_interrupt_if_at_failure_limits(self):
port = Mock() # FIXME: This should be a tighter mock.
port.TEST_PATH_SEPARATOR = '/'
port._filesystem = MockFileSystem()
manager = Manager(port=port, options=MockOptions(), printer=Mock())
manager._options = MockOptions(exit_after_n_failures=None, exit_after_n_crashes_or_timeouts=None)
result_summary = ResultSummary(expectations=Mock(), test_files=[])
result_summary.unexpected_failures = 100
result_summary.unexpected_crashes = 50
result_summary.unexpected_timeouts = 50
# No exception when the exit_after* options are None.
manager._interrupt_if_at_failure_limits(result_summary)
# No exception when we haven't hit the limit yet.
manager._options.exit_after_n_failures = 101
manager._options.exit_after_n_crashes_or_timeouts = 101
manager._interrupt_if_at_failure_limits(result_summary)
# Interrupt if we've exceeded either limit:
manager._options.exit_after_n_crashes_or_timeouts = 10
self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary)
manager._options.exit_after_n_crashes_or_timeouts = None
manager._options.exit_after_n_failures = 10
exception = self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary)
def test_fallback_path_in_config(self):
options = self.get_options()
host = MockHost()
port = host.port_factory.get('test-mac-leopard', options=options)
printer = self.get_printer()
manager = Manager(port, port.options, printer)
manager.print_config()
self.assertTrue('Baseline search path: test-mac-leopard -> test-mac-snowleopard -> generic' in printer.output)
def run(port, options, args, logging_stream):
try:
printer = printing.Printer(port, options, logging_stream, logger=logging.getLogger())
_set_up_derived_options(port, options)
manager = Manager(port, options, printer)
printer.print_config(port.results_directory())
run_details = manager.run(args)
_log.debug("Testing completed, Exit status: %d" % run_details.exit_code)
return run_details
finally:
printer.cleanup()
def test_update_summary_with_result(self):
host = MockHost()
port = host.port_factory.get('test-win-xp')
test = 'failures/expected/reftest.html'
port.test_expectations = lambda: 'WONTFIX : failures/expected/reftest.html = IMAGE'
expectations = TestExpectations(port, tests=[test])
# Reftests expected to be image mismatch should be respected when pixel_tests=False.
manager = Manager(port=port, options=MockOptions(pixel_tests=False, exit_after_n_failures=None, exit_after_n_crashes_or_timeouts=None), printer=Mock())
manager._expectations = expectations
result_summary = ResultSummary(expectations=expectations, test_files=[test])
result = TestResult(test_name=test, failures=[test_failures.FailureReftestMismatchDidNotOccur()])
manager._update_summary_with_result(result_summary, result)
self.assertEquals(1, result_summary.expected)
self.assertEquals(0, result_summary.unexpected)
def get_manager():
host = MockHost()
port = host.port_factory.get('test-mac-leopard')
manager = Manager(port,
options=MockOptions(test_list=None, http=True),
printer=Mock())
return manager
def run(port, options, args, logging_stream):
logger = logging.getLogger()
logger.setLevel(logging.DEBUG if options.debug_rwt_logging else logging.INFO)
try:
printer = printing.Printer(port, options, logging_stream, logger=logger)
_set_up_derived_options(port, options)
manager = Manager(port, options, printer)
printer.print_config(port.results_directory())
run_details = manager.run(args)
_log.debug("Testing completed, Exit status: %d" % run_details.exit_code)
return run_details
finally:
printer.cleanup()
def get_manager():
port = Mock() # FIXME: Use a tighter mock.
port.TEST_PATH_SEPARATOR = '/'
manager = Manager(port,
options=MockOptions(http=True,
max_locked_shards=1),
printer=Mock())
return manager
def _print_expectations(port, options, args, logging_stream):
logger = logging.getLogger()
logger.setLevel(logging.DEBUG if options.debug_rwt_logging else logging.INFO)
try:
printer = printing.Printer(port, options, logging_stream, logger=logger)
_set_up_derived_options(port, options)
manager = Manager(port, options, printer)
exit_code = manager.print_expectations(args)
_log.debug("Printing expectations completed, Exit status: %d", exit_code)
return exit_code
except Exception as error:
_log.error('Error printing expectations: {}'.format(error))
finally:
printer.cleanup()
return -1
def get_manager(port):
manager = Manager(port,
options=optparse.Values({
'http': True,
'max_locked_shards': 1
}),
printer=FakePrinter())
return manager
def get_manager():
port = Mock() # FIXME: Use a tighter mock.
port.TEST_PATH_SEPARATOR = '/'
port.web_platform_test_server_doc_root = get_wpt_doc_root
manager = Manager(port,
options=MockOptions(http=True),
printer=Mock())
return manager
def run(port, options, args, regular_output=sys.stderr, buildbot_output=sys.stdout):
try:
printer = printing.Printer(port, options, regular_output, buildbot_output, logger=logging.getLogger())
_set_up_derived_options(port, options)
if options.lint_test_files:
return lint(port, options)
manager = Manager(port, options, printer)
printer.print_config(port.results_directory())
unexpected_result_count = manager.run(args)
_log.debug("Testing completed, Exit status: %d" % unexpected_result_count)
return unexpected_result_count
finally:
printer.cleanup()
def get_manager_with_tests(test_names):
host = MockHost()
port = host.port_factory.get()
manager = Manager(port,
options=MockOptions(test_list=None, http=True),
printer=Mock())
manager.collect_tests(test_names)
return manager
def get_manager():
host = MockHost()
port = host.port_factory.get('test-mac-leopard')
manager = Manager(port,
options=MockOptions(http=True,
max_locked_shards=1),
printer=FakePrinter())
return manager
def _print_expectations(port, options, args, logging_stream):
logger = logging.getLogger()
logger.setLevel(logging.DEBUG if options.debug_rwt_logging else logging.INFO)
try:
printer = printing.Printer(port, options, logging_stream, logger=logger)
_set_up_derived_options(port, options)
manager = Manager(port, options, printer)
exit_code = manager.print_expectations(args)
_log.debug("Printing expectations completed, Exit status: %d", exit_code)
return exit_code
except Exception as error:
_log.error('Error printing expectations: {}'.format(error))
finally:
printer.cleanup()
return -1
def test_servers_started(self):
def start_http_server(number_of_servers=None):
self.http_started = True
def start_websocket_server():
self.websocket_started = True
def stop_http_server():
self.http_stopped = True
def stop_websocket_server():
self.websocket_stopped = True
host = MockHost()
port = host.port_factory.get('test-mac-leopard')
port.start_http_server = start_http_server
port.start_websocket_server = start_websocket_server
port.stop_http_server = stop_http_server
port.stop_websocket_server = stop_websocket_server
self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
manager = Manager(port=port, options=MockOptions(http=True), printer=Mock())
manager._test_files = ['http/tests/pass.txt']
manager.start_servers_with_lock(number_of_servers=4)
self.assertEquals(self.http_started, True)
self.assertEquals(self.websocket_started, False)
manager.stop_servers_with_lock()
self.assertEquals(self.http_stopped, True)
self.assertEquals(self.websocket_stopped, False)
self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
manager = Manager(port=port, options=MockOptions(http=True), printer=Mock())
manager._test_files = ['websocket/pass.txt']
manager.start_servers_with_lock(number_of_servers=4)
self.assertEquals(self.http_started, True)
self.assertEquals(self.websocket_started, True)
manager.stop_servers_with_lock()
self.assertEquals(self.http_stopped, True)
self.assertEquals(self.websocket_stopped, True)
self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
manager = Manager(port=port, options=MockOptions(http=True), printer=Mock())
manager._test_files = ['perf/foo/test.html']
manager.start_servers_with_lock(number_of_servers=4)
self.assertEquals(self.http_started, False)
self.assertEquals(self.websocket_started, False)
manager.stop_servers_with_lock()
self.assertEquals(self.http_stopped, False)
self.assertEquals(self.websocket_stopped, False)
def test_interrupt_if_at_failure_limits(self):
port = Mock()
port.TEST_PATH_SEPARATOR = '/'
port._filesystem = filesystem_mock.MockFileSystem()
manager = Manager(port=port, options=MockOptions(), printer=Mock())
manager._options = MockOptions(exit_after_n_failures=None,
exit_after_n_crashes_or_timeouts=None)
result_summary = ResultSummary(expectations=Mock(), test_files=[])
result_summary.unexpected_failures = 100
result_summary.unexpected_crashes = 50
result_summary.unexpected_timeouts = 50
# No exception when the exit_after* options are None.
manager._interrupt_if_at_failure_limits(result_summary)
# No exception when we haven't hit the limit yet.
manager._options.exit_after_n_failures = 101
manager._options.exit_after_n_crashes_or_timeouts = 101
manager._interrupt_if_at_failure_limits(result_summary)
# Interrupt if we've exceeded either limit:
manager._options.exit_after_n_crashes_or_timeouts = 10
self.assertRaises(TestRunInterruptedException,
manager._interrupt_if_at_failure_limits,
result_summary)
manager._options.exit_after_n_crashes_or_timeouts = None
manager._options.exit_after_n_failures = 10
exception = self.assertRaises(TestRunInterruptedException,
manager._interrupt_if_at_failure_limits,
result_summary)
def get_manager():
host = MockHost()
port = host.port_factory.get()
port.TEST_PATH_SEPARATOR = '/'
port.web_platform_test_server_doc_root = get_wpt_doc_root
manager = Manager(port,
options=MockOptions(http=True),
printer=Mock())
return manager
def get_manager():
host = MockHost()
port = MockCustomDevicePort(host)
manager = Manager(port,
options=MockOptions(
test_list=['fast/test-starship/lasers.html'],
http=True),
printer=Mock())
return manager
def get_manager():
host = MockHost()
port = host.port_factory.get()
manager = Manager(port,
options=MockOptions(test_list=None,
http=True,
max_locked_shards=1),
printer=Mock())
return manager
def get_manager():
host = MockHost()
port = host.port_factory.get('test-mac-mac10.10')
manager = Manager(port,
options=optparse.Values({
'http': True,
'max_locked_shards': 1
}),
printer=FakePrinter())
return manager
def run(port,
options,
args,
regular_output=sys.stderr,
buildbot_output=sys.stdout):
try:
warnings = _set_up_derived_options(port, options)
printer = printing.Printer(port,
options,
regular_output,
buildbot_output,
logger=logging.getLogger())
for warning in warnings:
_log.warning(warning)
if options.lint_test_files:
return lint(port, options)
# We wrap any parts of the run that are slow or likely to raise exceptions
# in a try/finally to ensure that we clean up the logging configuration.
unexpected_result_count = -1
manager = Manager(port, options, printer)
printer.print_config(port.results_directory())
unexpected_result_count = manager.run(args)
_log.debug("Testing completed, Exit status: %d" %
unexpected_result_count)
except Exception:
exception_type, exception_value, exception_traceback = sys.exc_info()
if exception_type not in (KeyboardInterrupt,
TestRunInterruptedException,
WorkerException):
print >> sys.stderr, '\n%s raised: %s' % (exception_type.__name__,
exception_value)
stack_utils.log_traceback(_log.error, exception_traceback)
raise
finally:
printer.cleanup()
return unexpected_result_count
def test_fallback_path_in_config(self):
options = self.get_options()
host = MockHost()
port = host.port_factory.get('test-mac-leopard', options=options)
printer = self.get_printer()
manager = Manager(port, options, printer)
manager.print_config()
self.assertTrue(
'Baseline search path: test-mac-leopard -> test-mac-snowleopard -> generic'
in printer.output)
def run(port,
options,
args,
regular_output=sys.stderr,
buildbot_output=sys.stdout):
try:
warnings = _set_up_derived_options(port, options)
printer = printing.Printer(port,
options,
regular_output,
buildbot_output,
logger=logging.getLogger())
for warning in warnings:
_log.warning(warning)
if options.help_printing:
printer.help_printing()
printer.cleanup()
return 0
if options.lint_test_files:
return lint(port, options)
# We wrap any parts of the run that are slow or likely to raise exceptions
# in a try/finally to ensure that we clean up the logging configuration.
unexpected_result_count = -1
manager = Manager(port, options, printer)
printer.print_config()
printer.write_update("Collecting tests ...")
try:
manager.collect_tests(args)
except IOError, e:
if e.errno == errno.ENOENT:
return -1
raise
printer.write_update("Checking build ...")
if not port.check_build(manager.needs_servers()):
_log.error("Build check failed")
return -1
printer.write_update("Parsing expectations ...")
manager.parse_expectations()
unexpected_result_count = manager.run()
_log.debug("Testing completed, Exit status: %d" %
unexpected_result_count)
def test_ensure_manifest_copies_new_manifest(self):
host = MockHost()
port = host.port_factory.get()
manifest_path = '/mock-checkout/third_party/WebKit/LayoutTests/external/wpt/MANIFEST.json'
self.assertFalse(port.host.filesystem.exists(manifest_path))
manager = Manager(port,
options=optparse.Values({'max_locked_shards': 1}),
printer=FakePrinter())
manager._ensure_manifest()
self.assertTrue(port.host.filesystem.exists(manifest_path))
webkit_base = '/mock-checkout/third_party/WebKit'
self.assertEqual(port.host.executive.calls, [[
'python',
webkit_base +
'/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/manifest',
'--work',
'--tests-root',
webkit_base + '/LayoutTests/external/wpt',
]])
def test_update_summary_with_result(self):
host = MockHost()
port = host.port_factory.get('test-win-xp')
test = 'failures/expected/reftest.html'
port.test_expectations = lambda: 'WONTFIX : failures/expected/reftest.html = IMAGE'
expectations = TestExpectations(port, tests=[test])
# Reftests expected to be image mismatch should be respected when pixel_tests=False.
manager = Manager(port=port,
options=MockOptions(
pixel_tests=False,
exit_after_n_failures=None,
exit_after_n_crashes_or_timeouts=None),
printer=Mock())
manager._expectations = expectations
result_summary = ResultSummary(expectations=expectations,
test_files=[test])
result = TestResult(
test_name=test,
failures=[test_failures.FailureReftestMismatchDidNotOccur()])
manager._update_summary_with_result(result_summary, result)
self.assertEquals(1, result_summary.expected)
self.assertEquals(0, result_summary.unexpected)
def test_interrupt_if_at_failure_limits(self):
port = Mock() # FIXME: This should be a tighter mock.
port.TEST_PATH_SEPARATOR = '/'
port._filesystem = MockFileSystem()
manager = Manager(port=port, options=MockOptions(), printer=Mock())
manager._options = MockOptions(exit_after_n_failures=None,
exit_after_n_crashes_or_timeouts=None)
manager._test_files = ['foo/bar.html', 'baz.html']
result_summary = ResultSummary(expectations=Mock(),
test_files=manager._test_files)
result_summary.unexpected_failures = 100
result_summary.unexpected_crashes = 50
result_summary.unexpected_timeouts = 50
# No exception when the exit_after* options are None.
manager._interrupt_if_at_failure_limits(result_summary)
# No exception when we haven't hit the limit yet.
manager._options.exit_after_n_failures = 101
manager._options.exit_after_n_crashes_or_timeouts = 101
manager._interrupt_if_at_failure_limits(result_summary)
# Interrupt if we've exceeded either limit:
manager._options.exit_after_n_crashes_or_timeouts = 10
self.assertRaises(TestRunInterruptedException,
manager._interrupt_if_at_failure_limits,
result_summary)
self.assertEquals(result_summary.results['foo/bar.html'].type,
test_expectations.SKIP)
self.assertEquals(result_summary.results['baz.html'].type,
test_expectations.SKIP)
manager._options.exit_after_n_crashes_or_timeouts = None
manager._options.exit_after_n_failures = 10
exception = self.assertRaises(TestRunInterruptedException,
manager._interrupt_if_at_failure_limits,
result_summary)
def run(port, options, args, regular_output=sys.stderr, buildbot_output=sys.stdout):
warnings = _set_up_derived_options(port, options)
printer = printing.Printer(port, options, regular_output, buildbot_output, logger=logging.getLogger())
for warning in warnings:
_log.warning(warning)
if options.help_printing:
printer.help_printing()
printer.cleanup()
return 0
if options.lint_test_files:
return lint(port, options)
# We wrap any parts of the run that are slow or likely to raise exceptions
# in a try/finally to ensure that we clean up the logging configuration.
unexpected_result_count = -1
try:
manager = Manager(port, options, printer)
manager.print_config()
printer.print_update("Collecting tests ...")
try:
manager.collect_tests(args)
except IOError, e:
if e.errno == errno.ENOENT:
return -1
raise
printer.print_update("Checking build ...")
if not port.check_build(manager.needs_servers()):
_log.error("Build check failed")
return -1
printer.print_update("Parsing expectations ...")
manager.parse_expectations()
unexpected_result_count = manager.run()
_log.debug("Testing completed, Exit status: %d" % unexpected_result_count)
def test_servers_started(self):
def start_http_server(number_of_servers=None):
self.http_started = True
def start_websocket_server():
self.websocket_started = True
def stop_http_server():
self.http_stopped = True
def stop_websocket_server():
self.websocket_stopped = True
host = MockHost()
port = host.port_factory.get('test-mac-leopard')
port.start_http_server = start_http_server
port.start_websocket_server = start_websocket_server
port.stop_http_server = stop_http_server
port.stop_websocket_server = stop_websocket_server
self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
manager = Manager(port=port, options=MockOptions(http=True), printer=Mock())
manager._test_files = ['http/tests/pass.txt']
manager.start_servers_with_lock(number_of_servers=4)
self.assertEquals(self.http_started, True)
self.assertEquals(self.websocket_started, False)
manager.stop_servers_with_lock()
self.assertEquals(self.http_stopped, True)
self.assertEquals(self.websocket_stopped, False)
self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
manager = Manager(port=port, options=MockOptions(http=True), printer=Mock())
manager._test_files = ['websocket/pass.txt']
manager.start_servers_with_lock(number_of_servers=4)
self.assertEquals(self.http_started, True)
self.assertEquals(self.websocket_started, True)
manager.stop_servers_with_lock()
self.assertEquals(self.http_stopped, True)
self.assertEquals(self.websocket_stopped, True)
self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
manager = Manager(port=port, options=MockOptions(http=True), printer=Mock())
manager._test_files = ['perf/foo/test.html']
manager.start_servers_with_lock(number_of_servers=4)
self.assertEquals(self.http_started, False)
self.assertEquals(self.websocket_started, False)
manager.stop_servers_with_lock()
self.assertEquals(self.http_stopped, False)
self.assertEquals(self.websocket_stopped, False)
def run(port, options, args, regular_output=sys.stderr,
buildbot_output=sys.stdout):
"""Run the tests.
Args:
port: Port object for port-specific behavior
options: a dictionary of command line options
args: a list of sub directories or files to test
regular_output: a stream-like object that we can send logging/debug
output to
buildbot_output: a stream-like object that we can write all output that
is intended to be parsed by the buildbot to
Returns:
the number of unexpected results that occurred, or -1 if there is an
error.
"""
warnings = _set_up_derived_options(port, options)
printer = printing.Printer(port, options, regular_output, buildbot_output,
configure_logging=True)
for w in warnings:
_log.warning(w)
if options.help_printing:
printer.help_printing()
printer.cleanup()
return 0
# We wrap any parts of the run that are slow or likely to raise exceptions
# in a try/finally to ensure that we clean up the logging configuration.
num_unexpected_results = -1
try:
manager = Manager(port, options, printer)
manager.print_config()
printer.print_update("Collecting tests ...")
try:
manager.collect_tests(args)
except IOError, e:
if e.errno == errno.ENOENT:
return -1
raise
if options.lint_test_files:
return manager.lint()
printer.print_update("Checking build ...")
if not port.check_build(manager.needs_servers()):
_log.error("Build check failed")
return -1
printer.print_update("Parsing expectations ...")
manager.parse_expectations()
result_summary = manager.set_up_run()
if result_summary:
num_unexpected_results = manager.run(result_summary)
manager.clean_up_run()
_log.debug("Testing completed, Exit status: %d" %
num_unexpected_results)
def get_manager_with_tests(test_names):
port = Mock() # FIXME: Use a tighter mock.
port.TEST_PATH_SEPARATOR = '/'
manager = Manager(port, options=MockOptions(http=True, max_locked_shards=1), printer=Mock())
manager._test_names = test_names
return manager
def get_manager_with_tests(test_names):
host = MockHost()
port = host.port_factory.get('test-mac-leopard')
manager = Manager(port, options=MockOptions(test_list=None, http=True, max_locked_shards=1), printer=Mock())
manager._collect_tests(test_names)
return manager
def _run_tests(port, options, args, printer):
_set_up_derived_options(port, options, args)
manager = Manager(port, options, printer)
printer.print_config(port.results_directory())
return manager.run(args)
def get_manager_with_tests(test_names):
host = MockHost()
port = host.port_factory.get()
manager = Manager(port, options=MockOptions(test_list=None, http=True), printer=Mock())
manager.collect_tests(test_names)
return manager
def get_manager(port):
manager = Manager(port,
options=MockOptions(http=True,
max_locked_shards=1),
printer=FakePrinter())
return manager
def get_manager_with_tests(test_names):
port = Mock() # FIXME: Use a tighter mock.
port.TEST_PATH_SEPARATOR = '/'
manager = Manager(port, options=MockOptions(http=True, max_locked_shards=1), printer=Mock())
manager._test_names = test_names
return manager
def get_manager_with_tests(test_names):
host = MockHost()
port = host.port_factory.get('test-mac-leopard')
manager = Manager(port, options=MockOptions(test_list=None, http=True, max_locked_shards=1), printer=Mock())
manager._collect_tests(test_names)
return manager
def __init__(self, ref_tests, **kwargs):
Manager.__init__(self, **kwargs)
self._ref_tests = ref_tests
def run(port,
options,
args,
regular_output=sys.stderr,
buildbot_output=sys.stdout):
"""Run the tests.
Args:
port: Port object for port-specific behavior
options: a dictionary of command line options
args: a list of sub directories or files to test
regular_output: a stream-like object that we can send logging/debug
output to
buildbot_output: a stream-like object that we can write all output that
is intended to be parsed by the buildbot to
Returns:
the number of unexpected results that occurred, or -1 if there is an
error.
"""
warnings = _set_up_derived_options(port, options)
printer = printing.Printer(port,
options,
regular_output,
buildbot_output,
configure_logging=True)
for w in warnings:
_log.warning(w)
if options.help_printing:
printer.help_printing()
printer.cleanup()
return 0
# We wrap any parts of the run that are slow or likely to raise exceptions
# in a try/finally to ensure that we clean up the logging configuration.
num_unexpected_results = -1
try:
manager = Manager(port, options, printer)
manager.print_config()
printer.print_update("Collecting tests ...")
try:
manager.collect_tests(args)
except IOError, e:
if e.errno == errno.ENOENT:
return -1
raise
if options.lint_test_files:
return manager.lint()
printer.print_update("Checking build ...")
if not port.check_build(manager.needs_servers()):
_log.error("Build check failed")
return -1
printer.print_update("Parsing expectations ...")
manager.parse_expectations()
result_summary = manager.set_up_run()
if result_summary:
num_unexpected_results = manager.run(result_summary)
manager.clean_up_run()
_log.debug("Testing completed, Exit status: %d" %
num_unexpected_results)
def _run_tests(port, options, args, printer):
_set_up_derived_options(port, options, args)
manager = Manager(port, options, printer)
printer.print_config(port.results_directory())
return manager.run(args)
def get_manager_with_tests(test_names):
port = layout_tests.port.get()
manager = Manager(port, options=MockOptions(test_list=None, http=True), printer=Mock())
manager.collect_tests(test_names)
return manager