def test_basic(self):
options, args = parse_args(tests_included=True)
logging_stream = StringIO.StringIO()
host = MockHost()
port_obj = host.port_factory.get(options.platform, options)
details = run_webkit_tests.run(port_obj, options, args, logging_stream)
# These numbers will need to be updated whenever we add new tests.
self.assertEqual(details.initial_results.total, test.TOTAL_TESTS)
self.assertEqual(details.initial_results.expected_skips, test.TOTAL_SKIPS)
self.assertEqual(len(details.initial_results.unexpected_results_by_name), test.UNEXPECTED_PASSES + test.UNEXPECTED_FAILURES)
self.assertEqual(details.exit_code, test.UNEXPECTED_FAILURES)
self.assertEqual(details.retry_results.total, test.TOTAL_RETRIES)
one_line_summary = "%d tests ran as expected, %d didn't:\n" % (
details.initial_results.total - details.initial_results.expected_skips - len(details.initial_results.unexpected_results_by_name),
len(details.initial_results.unexpected_results_by_name))
self.assertTrue(one_line_summary in logging_stream.buflist)
# Ensure the results were summarized properly.
self.assertEqual(details.summarized_results['num_regressions'], details.exit_code)
# Ensure the image diff percentage is in the results.
self.assertEqual(details.summarized_results['tests']['failures']['expected']['image.html']['image_diff_percent'], 1)
# Ensure the results were written out and displayed.
full_results_text = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
json_to_eval = full_results_text.replace("ADD_RESULTS(", "").replace(");", "")
self.assertEqual(json.loads(json_to_eval), details.summarized_results)
self.assertEqual(host.user.opened_urls, [path.abspath_to_uri(MockHost().platform, '/tmp/layout-test-results/results.html')])
def test_basic(self):
options, args = parse_args(tests_included=True)
logging_stream = StringIO.StringIO()
host = MockHost()
port_obj = host.port_factory.get(options.platform, options)
details = run_webkit_tests.run(port_obj, options, args, logging_stream)
# These numbers will need to be updated whenever we add new tests.
self.assertEqual(details.initial_results.total, test.TOTAL_TESTS)
self.assertEqual(details.initial_results.expected_skips, test.TOTAL_SKIPS)
self.assertEqual(len(details.initial_results.unexpected_results_by_name), test.UNEXPECTED_PASSES + test.UNEXPECTED_FAILURES)
self.assertEqual(details.exit_code, test.UNEXPECTED_FAILURES)
self.assertEqual(details.retry_results.total, test.TOTAL_RETRIES)
one_line_summary = "%d tests ran as expected, %d didn't:\n" % (
details.initial_results.total - details.initial_results.expected_skips - len(details.initial_results.unexpected_results_by_name),
len(details.initial_results.unexpected_results_by_name))
self.assertTrue(one_line_summary in logging_stream.buflist)
# Ensure the results were summarized properly.
self.assertEqual(details.summarized_results['num_regressions'], details.exit_code)
# Ensure the image diff percentage is in the results.
self.assertEqual(details.summarized_results['tests']['failures']['expected']['image.html']['image_diff_percent'], 1)
# Ensure the results were written out and displayed.
full_results_text = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
json_to_eval = full_results_text.replace("ADD_RESULTS(", "").replace(");", "")
self.assertEqual(json.loads(json_to_eval), details.summarized_results)
self.assertEqual(host.user.opened_urls, [path.abspath_to_uri(MockHost().platform, '/tmp/layout-test-results/results.html')])
def logging_run(extra_args=None, port_obj=None, tests_included=False):
extra_args = extra_args or []
args = ['--no-record-results']
if not '--platform' in extra_args:
args.extend(['--platform', 'test'])
if not '--child-processes' in extra_args:
args.extend(['--worker-model', 'inline'])
args.extend(extra_args)
if not tests_included:
args.extend(['passes',
'http/tests',
'websocket/tests',
'failures/expected/*'])
oc = outputcapture.OutputCapture()
try:
oc.capture_output()
options, parsed_args = run_webkit_tests.parse_args(args)
user = MockUser()
if not port_obj:
port_obj = port.get(port_name=options.platform, options=options,
user=user)
buildbot_output = array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
res = run_webkit_tests.run(port_obj, options, parsed_args,
buildbot_output=buildbot_output,
regular_output=regular_output)
finally:
oc.restore_output()
return (res, buildbot_output, regular_output, user)
def test_verbose_in_child_processes(self):
# When we actually run multiple processes, we may have to reconfigure logging in the
# child process (e.g., on win32) and we need to make sure that works and we still
# see the verbose log output. However, we can't use logging_run() because using
# outputcapture to capture stdout and stderr latter results in a nonpicklable host.
# Test is flaky on Windows: https://bugs.webkit.org/show_bug.cgi?id=98559
if not self.should_test_processes:
return
options, parsed_args = parse_args(['--verbose', '--fully-parallel', '--child-processes', '2', 'passes/text.html', 'passes/image.html'], tests_included=True, print_nothing=False)
host = MockHost()
port_obj = host.port_factory.get(port_name=options.platform, options=options)
logging_stream = StringIO.StringIO()
run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream)
self.assertTrue('text.html passed' in logging_stream.getvalue())
self.assertTrue('image.html passed' in logging_stream.getvalue())
def passing_run(args, port_obj=None, logging_included=False):
if not logging_included:
args.extend(['--print', 'nothing'])
options, args = run_webkit_tests.parse_args(args)
if port_obj is None:
port_obj = port.get(options.platform, options)
res = run_webkit_tests.run(port_obj, options, args)
return res == 0
def passing_run(args, port_obj=None, logging_included=False):
if not logging_included:
args.extend(['--print', 'nothing'])
options, args = run_webkit_tests.parse_args(args)
if port_obj is None:
port_obj = port.get(options.platform, options)
res = run_webkit_tests.run(port_obj, options, args)
return res == 0
def passing_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, filesystem=None):
options, parsed_args = parse_args(extra_args, record_results, tests_included)
if not port_obj:
port_obj = port.get(port_name=options.platform, options=options, user=mocktool.MockUser(), filesystem=filesystem)
buildbot_output = array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
res = run_webkit_tests.run(port_obj, options, parsed_args, buildbot_output=buildbot_output, regular_output=regular_output)
return res == 0 and regular_output.empty() and buildbot_output.empty()
def test_verbose_in_child_processes(self):
# When we actually run multiple processes, we may have to reconfigure logging in the
# child process (e.g., on win32) and we need to make sure that works and we still
# see the verbose log output. However, we can't use logging_run() because using
# outputcapture to capture stdout and stderr latter results in a nonpicklable host.
# Test is flaky on Windows: https://bugs.webkit.org/show_bug.cgi?id=98559
if not self.should_test_processes:
return
options, parsed_args = parse_args(['--verbose', '--fully-parallel', '--child-processes', '2', 'passes/text.html', 'passes/image.html'], tests_included=True, print_nothing=False)
host = MockHost()
port_obj = host.port_factory.get(port_name=options.platform, options=options)
logging_stream = StringIO.StringIO()
run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream)
self.assertTrue('text.html passed' in logging_stream.getvalue())
self.assertTrue('image.html passed' in logging_stream.getvalue())
def passing_run(extra_args=None, port_obj=None, record_results=False,
tests_included=False, filesystem=None):
options, parsed_args = parse_args(extra_args, record_results,
tests_included)
if not port_obj:
port_obj = port.get(port_name=options.platform, options=options,
user=mocktool.MockUser(), filesystem=filesystem)
res = run_webkit_tests.run(port_obj, options, parsed_args)
return res == 0
def logging_run(args):
options, args = run_webkit_tests.parse_args(args)
port_obj = port.get(options.platform, options)
buildbot_output = array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
res = run_webkit_tests.run(port_obj, options, args,
buildbot_output=buildbot_output,
regular_output=regular_output)
return (res, buildbot_output, regular_output)
def logging_run(args):
options, args = run_webkit_tests.parse_args(args)
port_obj = port.get(options.platform, options)
buildbot_output = array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
res = run_webkit_tests.run(port_obj, options, args,
buildbot_output=buildbot_output,
regular_output=regular_output)
return (res, buildbot_output, regular_output)
def passing_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, host=None):
options, parsed_args = parse_args(extra_args, record_results, tests_included)
if not port_obj:
host = host or MockHost()
port_obj = host.port_factory.get(port_name=options.platform, options=options)
buildbot_output = StringIO.StringIO()
regular_output = StringIO.StringIO()
res = run_webkit_tests.run(port_obj, options, parsed_args, buildbot_output=buildbot_output, regular_output=regular_output)
return res == 0 and not regular_output.getvalue() and not buildbot_output.getvalue()
def run_and_capture(port_obj, options, parsed_args, shared_port=True):
if shared_port:
port_obj.host.port_factory.get = lambda *args, **kwargs: port_obj
oc = outputcapture.OutputCapture()
try:
oc.capture_output()
logging_stream = StringIO.StringIO()
run_details = run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream)
finally:
oc.restore_output()
return (run_details, logging_stream)
def run_and_capture(port_obj, options, parsed_args, shared_port=True):
if shared_port:
port_obj.host.port_factory.get = lambda *args, **kwargs: port_obj
oc = outputcapture.OutputCapture()
try:
oc.capture_output()
logging_stream = StringIO.StringIO()
run_details = run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream)
finally:
oc.restore_output()
return (run_details, logging_stream)
def passing_run(extra_args=None, port_obj=None, tests_included=False, host=None, shared_port=True):
options, parsed_args = parse_args(extra_args, tests_included)
if not port_obj:
host = host or MockHost()
port_obj = host.port_factory.get(port_name=options.platform, options=options)
if shared_port:
port_obj.host.port_factory.get = lambda *args, **kwargs: port_obj
logging_stream = StringIO.StringIO()
run_details = run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream)
return run_details.exit_code == 0
def passing_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, filesystem=None):
options, parsed_args = parse_args(extra_args, record_results, tests_included)
filesystem = filesystem or unit_test_filesystem()
if not port_obj:
host = MockHost()
port_obj = host.port_factory.get(port_name=options.platform, options=options, filesystem=filesystem)
buildbot_output = array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
res = run_webkit_tests.run(
port_obj, options, parsed_args, buildbot_output=buildbot_output, regular_output=regular_output
)
return res == 0 and regular_output.empty() and buildbot_output.empty()
def passing_run(extra_args=None, port_obj=None, tests_included=False, host=None, shared_port=True):
options, parsed_args = parse_args(extra_args, tests_included)
if not port_obj:
host = host or MockHost()
port_obj = host.port_factory.get(port_name=options.platform, options=options)
if shared_port:
port_obj.host.port_factory.get = lambda *args, **kwargs: port_obj
logging_stream = StringIO.StringIO()
run_details = run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream)
return run_details.exit_code == 0
def run_and_capture(port_obj, options, parsed_args):
oc = outputcapture.OutputCapture()
try:
oc.capture_output()
buildbot_output = StringIO.StringIO()
regular_output = StringIO.StringIO()
res = run_webkit_tests.run(port_obj, options, parsed_args,
buildbot_output=buildbot_output,
regular_output=regular_output)
finally:
oc.restore_output()
return (res, buildbot_output, regular_output)
def passing_run(extra_args=None,
port_obj=None,
record_results=False,
tests_included=False,
filesystem=None):
options, parsed_args = parse_args(extra_args, record_results,
tests_included)
if not port_obj:
port_obj = port.get(port_name=options.platform,
options=options,
user=mocktool.MockUser(),
filesystem=filesystem)
res = run_webkit_tests.run(port_obj, options, parsed_args)
return res == 0
def run_and_capture(port_obj, options, parsed_args):
oc = outputcapture.OutputCapture()
try:
oc.capture_output()
buildbot_output = array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
res = run_webkit_tests.run(port_obj,
options,
parsed_args,
buildbot_output=buildbot_output,
regular_output=regular_output)
finally:
oc.restore_output()
return (res, buildbot_output, regular_output)
def logging_run(args=[], tests_included=False):
new_args = ['--no-record-results']
if not '--platform' in args:
new_args.extend(['--platform', 'test'])
new_args.extend(args)
if not tests_included:
new_args.extend(['passes',
'http/tests'
'websocket/tests',
'failures/expected/*'])
options, parsed_args = run_webkit_tests.parse_args(new_args)
port_obj = port.get(options.platform, options)
buildbot_output = array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
res = run_webkit_tests.run(port_obj, options, parsed_args,
buildbot_output=buildbot_output,
regular_output=regular_output)
return (res, buildbot_output, regular_output)
def passing_run(args=[],
port_obj=None,
record_results=False,
tests_included=False):
new_args = ['--print', 'nothing']
if not '--platform' in args:
new_args.extend(['--platform', 'test'])
if not record_results:
new_args.append('--no-record-results')
new_args.extend(args)
if not tests_included:
# We use the glob to test that globbing works.
new_args.extend(
['passes', 'http/tests', 'websocket/tests', 'failures/expected/*'])
options, parsed_args = run_webkit_tests.parse_args(new_args)
if port_obj is None:
port_obj = port.get(options.platform, options)
res = run_webkit_tests.run(port_obj, options, parsed_args)
return res == 0
def passing_run(args=[], port_obj=None, record_results=False,
tests_included=False):
new_args = ['--print', 'nothing']
if not '--platform' in args:
new_args.extend(['--platform', 'test'])
if not record_results:
new_args.append('--no-record-results')
new_args.extend(args)
if not tests_included:
# We use the glob to test that globbing works.
new_args.extend(['passes',
'http/tests',
'websocket/tests',
'failures/expected/*'])
options, parsed_args = run_webkit_tests.parse_args(new_args)
if port_obj is None:
port_obj = port.get(options.platform, options)
res = run_webkit_tests.run(port_obj, options, parsed_args)
return res == 0
def logging_run(args=[], tests_included=False):
new_args = ['--no-record-results']
if not '--platform' in args:
new_args.extend(['--platform', 'test'])
new_args.extend(args)
if not tests_included:
new_args.extend(
['passes', 'http/tests'
'websocket/tests', 'failures/expected/*'])
options, parsed_args = run_webkit_tests.parse_args(new_args)
port_obj = port.get(options.platform, options)
buildbot_output = array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
res = run_webkit_tests.run(port_obj,
options,
parsed_args,
buildbot_output=buildbot_output,
regular_output=regular_output)
return (res, buildbot_output, regular_output)
def passing_run(extra_args=None,
port_obj=None,
record_results=False,
tests_included=False,
filesystem=None):
options, parsed_args = parse_args(extra_args, record_results,
tests_included)
if not port_obj:
port_obj = port.get(port_name=options.platform,
options=options,
user=mocktool.MockUser(),
filesystem=filesystem)
buildbot_output = array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
res = run_webkit_tests.run(port_obj,
options,
parsed_args,
buildbot_output=buildbot_output,
regular_output=regular_output)
return res == 0 and regular_output.empty() and buildbot_output.empty()
def get_test_results(args, host=None):
options, parsed_args = parse_args(args, tests_included=True)
host = host or MockHost()
port_obj = host.port_factory.get(port_name=options.platform, options=options)
oc = outputcapture.OutputCapture()
oc.capture_output()
logging_stream = StringIO.StringIO()
try:
run_details = run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream)
finally:
oc.restore_output()
all_results = []
if run_details.initial_results:
all_results.extend(run_details.initial_results.all_results)
if run_details.retry_results:
all_results.extend(run_details.retry_results.all_results)
return all_results
def get_test_results(args, host=None):
options, parsed_args = parse_args(args, tests_included=True)
host = host or MockHost()
port_obj = host.port_factory.get(port_name=options.platform, options=options)
oc = outputcapture.OutputCapture()
oc.capture_output()
logging_stream = StringIO.StringIO()
try:
run_details = run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream)
finally:
oc.restore_output()
all_results = []
if run_details.initial_results:
all_results.extend(run_details.initial_results.all_results)
if run_details.retry_results:
all_results.extend(run_details.retry_results.all_results)
return all_results
def passing_run(extra_args=None, port_obj=None, record_results=False,
tests_included=False):
extra_args = extra_args or []
args = ['--print', 'nothing']
if not '--platform' in extra_args:
args.extend(['--platform', 'test'])
if not record_results:
args.append('--no-record-results')
if not '--child-processes' in extra_args:
args.extend(['--worker-model', 'inline'])
args.extend(extra_args)
if not tests_included:
# We use the glob to test that globbing works.
args.extend(['passes',
'http/tests',
'websocket/tests',
'failures/expected/*'])
options, parsed_args = run_webkit_tests.parse_args(args)
if not port_obj:
port_obj = port.get(port_name=options.platform, options=options,
user=MockUser())
res = run_webkit_tests.run(port_obj, options, parsed_args)
return res == 0