def run(port, options, args, logging_stream, stdout):
logger = logging.getLogger()
logger.setLevel(
logging.DEBUG if options.debug_rwt_logging else logging.INFO)
printer = printing.Printer(port, options, logging_stream, logger=logger)
try:
run_details = _run_tests(port, options, args, printer)
printer.flush()
if (not options.dry_run
and (run_details.exit_code not in exit_codes.ERROR_CODES
or run_details.exit_code == exit_codes.EARLY_EXIT_STATUS)
and not run_details.initial_results.keyboard_interrupted):
bot_printer = buildbot_results.BuildBotPrinter(
stdout, options.debug_rwt_logging)
bot_printer.print_results(run_details)
stdout.flush()
_log.debug('')
_log.debug('Testing completed, Exit status: %d', run_details.exit_code)
return run_details
finally:
printer.cleanup()
def run(port, options, args, regular_output=sys.stderr, buildbot_output=sys.stdout):
try:
warnings = _set_up_derived_options(port, options)
printer = printing.Printer(port, options, regular_output, buildbot_output, logger=logging.getLogger())
for warning in warnings:
_log.warning(warning)
if options.lint_test_files:
return lint(port, options)
# We wrap any parts of the run that are slow or likely to raise exceptions
# in a try/finally to ensure that we clean up the logging configuration.
unexpected_result_count = -1
manager = Manager(port, options, printer)
printer.print_config(port.results_directory())
unexpected_result_count = manager.run(args)
_log.debug("Testing completed, Exit status: %d" % unexpected_result_count)
except Exception:
exception_type, exception_value, exception_traceback = sys.exc_info()
if exception_type not in (KeyboardInterrupt, TestRunInterruptedException, WorkerException):
print >> sys.stderr, '\n%s raised: %s' % (exception_type.__name__, exception_value)
stack_utils.log_traceback(_log.error, exception_traceback)
raise
finally:
printer.cleanup()
return unexpected_result_count
def run(self):
# We need to create a new Host object here because this is
# running in a new process and we can't require the parent's
# Host to be pickleable and passed to the child.
if self._platform_name.startswith('test'):
host = MockHost()
else:
host = Host()
host._initialize_scm()
options = self._options
port_obj = host.port_factory.get(self._platform_name, options)
# The unix multiprocessing implementation clones the
# log handler configuration into the child processes,
# but the win implementation doesn't.
configure_logging = (sys.platform == 'win32')
# FIXME: this won't work if the calling process is logging
# somewhere other than sys.stderr and sys.stdout, but I'm not sure
# if this will be an issue in practice.
printer = printing.Printer(port_obj, options, sys.stderr,
sys.stdout, configure_logging)
self._client.run(port_obj)
printer.cleanup()
def run(port, options, args, logging_stream, stdout):
logger = logging.getLogger()
logger.setLevel(logging.DEBUG if options.debug_rwt_logging else logging.INFO)
printer = printing.Printer(port, options, logging_stream, logger=logger)
try:
run_details = _run_tests(port, options, args, printer)
printer.flush()
if (not options.dry_run and
(run_details.exit_code not in test_run_results.ERROR_CODES or
run_details.exit_code == test_run_results.EARLY_EXIT_STATUS) and
not run_details.initial_results.keyboard_interrupted):
bot_printer = buildbot_results.BuildBotPrinter(stdout, options.debug_rwt_logging)
bot_printer.print_results(run_details)
stdout.flush()
_log.debug("Generating dashboard...")
gen_dash_board = DashBoardGenerator(port)
gen_dash_board.generate()
_log.debug("Dashboard generated.")
_log.debug("")
_log.debug("Testing completed, Exit status: %d", run_details.exit_code)
# Temporary process dump for debugging windows timeout issues, see crbug.com/522396.
_log.debug("")
_log.debug("Process dump:")
for process in port.host.executive.process_dump():
_log.debug("\t%s", process)
return run_details
finally:
printer.cleanup()
def get_printer(self, args=None):
args = args or []
printing_options = printing.print_options()
option_parser = optparse.OptionParser(option_list=printing_options)
options, args = option_parser.parse_args(args)
host = MockHost()
self._port = host.port_factory.get('test', options)
regular_output = StringIO.StringIO()
printer = printing.Printer(self._port, options, regular_output)
return printer, regular_output
def run(port,
options,
args,
regular_output=sys.stderr,
buildbot_output=sys.stdout):
warnings = _set_up_derived_options(port, options)
printer = printing.Printer(port,
options,
regular_output,
buildbot_output,
configure_logging=True)
for warning in warnings:
_log.warning(warning)
if options.help_printing:
printer.help_printing()
printer.cleanup()
return 0
if options.lint_test_files:
return lint(port, options, test_expectations.TestExpectations)
# We wrap any parts of the run that are slow or likely to raise exceptions
# in a try/finally to ensure that we clean up the logging configuration.
unexpected_result_count = -1
try:
manager = Manager(port, options, printer)
manager.print_config()
printer.print_update("Collecting tests ...")
try:
manager.collect_tests(args)
except IOError, e:
if e.errno == errno.ENOENT:
return -1
raise
printer.print_update("Checking build ...")
if not port.check_build(manager.needs_servers()):
_log.error("Build check failed")
return -1
printer.print_update("Parsing expectations ...")
manager.parse_expectations()
result_summary = manager.set_up_run()
if result_summary:
unexpected_result_count = manager.run(result_summary)
manager.clean_up_run()
_log.debug("Testing completed, Exit status: %d" %
unexpected_result_count)
def test_perf_locking(tester):
options, args = run_webkit_tests.parse_args(['--platform=test', '--print=nothing', '--no-http', 'passes', 'perf/'])
host = MockHost()
port = host.port_factory.get(port_name=options.platform, options=options)
run_webkit_tests._set_up_derived_options(port, options)
printer = printing.Printer(port, options, StringIO.StringIO(), StringIO.StringIO())
manager = LockCheckingManager(port, options, printer, tester, False)
manager.collect_tests(args)
manager.parse_expectations()
num_unexpected_results = manager.run()
printer.cleanup()
tester.assertEquals(num_unexpected_results, 0)
def run(port, options, args, logging_stream, stdout):
logger = logging.getLogger()
logger.setLevel(logging.DEBUG if options.debug_rwt_logging else logging.INFO)
printer = printing.Printer(port, options, logging_stream, logger=logger)
try:
run_details = _run_tests(port, options, args, printer)
printer.flush()
_log.debug('')
_log.debug('Testing completed. Exit status: %d', run_details.exit_code)
return run_details
finally:
printer.cleanup()
def run(port, options, args, logging_stream):
logger = logging.getLogger()
logger.setLevel(logging.DEBUG if options.debug_rwt_logging else logging.INFO)
try:
printer = printing.Printer(port, options, logging_stream, logger=logger)
_set_up_derived_options(port, options)
manager = Manager(port, options, printer)
printer.print_config(port.results_directory())
run_details = manager.run(args)
_log.debug("Testing completed, Exit status: %d" % run_details.exit_code)
return run_details
finally:
printer.cleanup()
def get_printer(self, args=None, tty=False):
args = args or []
printing_options = printing.print_options()
option_parser = optparse.OptionParser(option_list=printing_options)
options, args = option_parser.parse_args(args)
self._port = port.get('test', options)
nproc = 2
regular_output = array_stream.ArrayStream(tty=tty)
buildbot_output = array_stream.ArrayStream()
printer = printing.Printer(self._port,
options,
regular_output,
buildbot_output,
configure_logging=True)
return printer, regular_output, buildbot_output
def run(self):
options = self._options
port_obj = layout_tests.port.get(self._platform_name, options)
# The unix multiprocessing implementation clones the
# log handler configuration into the child processes,
# but the win implementation doesn't.
configure_logging = (sys.platform == 'win32')
# FIXME: this won't work if the calling process is logging
# somewhere other than sys.stderr and sys.stdout, but I'm not sure
# if this will be an issue in practice.
printer = printing.Printer(port_obj, options, sys.stderr,
sys.stdout, configure_logging)
self._client.run(port_obj)
printer.cleanup()
def _print_expectations(port, options, args, logging_stream):
logger = logging.getLogger()
logger.setLevel(logging.DEBUG if options.debug_rwt_logging else logging.INFO)
try:
printer = printing.Printer(port, options, logging_stream, logger=logger)
_set_up_derived_options(port, options)
manager = Manager(port, options, printer)
exit_code = manager.print_expectations(args)
_log.debug("Printing expectations completed, Exit status: %d", exit_code)
return exit_code
except Exception as error:
_log.error('Error printing expectations: {}'.format(error))
finally:
printer.cleanup()
return -1
def __init__(self,
perf_tests_dir,
regular_output=sys.stderr,
buildbot_output=sys.stdout,
args=None):
self._perf_tests_dir = perf_tests_dir
self._buildbot_output = buildbot_output
self._options, self._args = self._parse_args(args)
self._host = Host()
self._host._initialize_scm()
self._port = self._host.port_factory.get(self._options.platform,
self._options)
self._printer = printing.Printer(self._port,
self._options,
regular_output,
buildbot_output,
configure_logging=False)
self._webkit_base_dir_len = len(self._port.webkit_base())
def test_http_locking(tester):
class LockCheckingManager(Manager):
def __init__(self, port, options, printer):
super(LockCheckingManager,
self).__init__(port, options, printer)
self._finished_list_called = False
def handle_finished_list(self, source, list_name, num_tests,
elapsed_time):
if not self._finished_list_called:
tester.assertEquals(list_name, 'locked_tests')
tester.assertTrue(self._remaining_locked_shards)
tester.assertTrue(self._has_http_lock)
super(LockCheckingManager,
self).handle_finished_list(source, list_name, num_tests,
elapsed_time)
if not self._finished_list_called:
tester.assertEquals(self._remaining_locked_shards, [])
tester.assertFalse(self._has_http_lock)
self._finished_list_called = True
options, args = run_webkit_tests.parse_args([
'--platform=test', '--print=nothing', 'http/tests/passes', 'passes'
])
host = MockHost()
port = host.port_factory.get(port_name=options.platform,
options=options)
run_webkit_tests._set_up_derived_options(port, options)
printer = printing.Printer(port,
options,
StringIO.StringIO(),
StringIO.StringIO(),
configure_logging=False)
manager = LockCheckingManager(port, options, printer)
manager.collect_tests(args)
manager.parse_expectations()
result_summary = manager.set_up_run()
num_unexpected_results = manager.run(result_summary)
manager.clean_up_run()
printer.cleanup()
tester.assertEquals(num_unexpected_results, 0)
def lint(port, options, logging_stream):
host = port.host
logging.getLogger().setLevel(
logging.DEBUG if options.debug_rwt_logging else logging.INFO)
printer = printing.Printer(port,
options,
logging_stream,
logger=logging.getLogger())
if options.platform:
ports_to_lint = [port]
else:
ports_to_lint = [
host.port_factory.get(name)
for name in host.port_factory.all_port_names()
]
files_linted = set()
lint_failed = False
for port_to_lint in ports_to_lint:
expectations_dict = port_to_lint.expectations_dict()
# FIXME: This won't work if multiple ports share a TestExpectations file but support different modifiers in the file.
for expectations_file in expectations_dict.keys():
if expectations_file in files_linted:
continue
try:
test_expectations.TestExpectations(
port_to_lint,
expectations_to_lint={
expectations_file: expectations_dict[expectations_file]
})
except test_expectations.ParseError, e:
lint_failed = True
_log.error('')
for warning in e.warnings:
_log.error(warning)
_log.error('')
files_linted.add(expectations_file)
def __init__(self,
regular_output=sys.stderr,
buildbot_output=sys.stdout,
args=None,
port=None):
self._buildbot_output = buildbot_output
self._options, self._args = PerfTestsRunner._parse_args(args)
if port:
self._port = port
self._host = self._port.host
else:
self._host = Host()
self._port = self._host.port_factory.get(self._options.platform,
self._options)
self._host._initialize_scm()
self._printer = printing.Printer(self._port,
self._options,
regular_output,
buildbot_output,
configure_logging=False)
self._webkit_base_dir_len = len(self._port.webkit_base())
self._base_path = self._port.perf_tests_dir()
self._results = {}
self._timestamp = time.time()
def run(port,
options,
args,
regular_output=sys.stderr,
buildbot_output=sys.stdout):
"""Run the tests.
Args:
port: Port object for port-specific behavior
options: a dictionary of command line options
args: a list of sub directories or files to test
regular_output: a stream-like object that we can send logging/debug
output to
buildbot_output: a stream-like object that we can write all output that
is intended to be parsed by the buildbot to
Returns:
the number of unexpected results that occurred, or -1 if there is an
error.
"""
warnings = _set_up_derived_options(port, options)
printer = printing.Printer(port,
options,
regular_output,
buildbot_output,
configure_logging=True)
for w in warnings:
_log.warning(w)
if options.help_printing:
printer.help_printing()
printer.cleanup()
return 0
# We wrap any parts of the run that are slow or likely to raise exceptions
# in a try/finally to ensure that we clean up the logging configuration.
num_unexpected_results = -1
try:
manager = Manager(port, options, printer)
manager.print_config()
printer.print_update("Collecting tests ...")
try:
manager.collect_tests(args)
except IOError, e:
if e.errno == errno.ENOENT:
return -1
raise
if options.lint_test_files:
return manager.lint()
printer.print_update("Checking build ...")
if not port.check_build(manager.needs_servers()):
_log.error("Build check failed")
return -1
printer.print_update("Parsing expectations ...")
manager.parse_expectations()
result_summary = manager.set_up_run()
if result_summary:
num_unexpected_results = manager.run(result_summary)
manager.clean_up_run()
_log.debug("Testing completed, Exit status: %d" %
num_unexpected_results)
