def get_port_for_run(args):
options, parsed_args = run_webkit_tests.parse_args(args)
test_port = ImageDiffTestPort(options=options,
user=mocktool.MockUser())
res = passing_run(args, port_obj=test_port, tests_included=True)
self.assertTrue(res)
return test_port
def parse_args(extra_args=None,
record_results=False,
tests_included=False,
new_results=False,
print_nothing=True):
extra_args = extra_args or []
if print_nothing:
args = ['--print', 'nothing']
else:
args = []
if not '--platform' in extra_args:
args.extend(['--platform', 'test'])
if not record_results:
args.append('--no-record-results')
if not new_results:
args.append('--no-new-test-results')
if not '--child-processes' in extra_args and not '--worker-model' in extra_args:
args.extend(['--worker-model', 'inline'])
args.extend(extra_args)
if not tests_included:
# We use the glob to test that globbing works.
args.extend(
['passes', 'http/tests', 'websocket/tests', 'failures/expected/*'])
return run_webkit_tests.parse_args(args)
def test_http_locking(tester):
class LockCheckingManager(Manager):
def __init__(self, port, options, printer):
super(LockCheckingManager, self).__init__(port, options, printer)
self._finished_list_called = False
def handle_finished_list(self, source, list_name, num_tests, elapsed_time):
if not self._finished_list_called:
tester.assertEquals(list_name, 'locked_tests')
tester.assertTrue(self._remaining_locked_shards)
tester.assertTrue(self._has_http_lock)
super(LockCheckingManager, self).handle_finished_list(source, list_name, num_tests, elapsed_time)
if not self._finished_list_called:
tester.assertEquals(self._remaining_locked_shards, [])
tester.assertFalse(self._has_http_lock)
self._finished_list_called = True
options, args = run_webkit_tests.parse_args(['--platform=test', '--print=nothing', 'http/tests/passes', 'passes'])
host = MockHost()
port = host.port_factory.get(port_name=options.platform, options=options)
run_webkit_tests._set_up_derived_options(port, options)
printer = printing.Printer(port, options, StringIO.StringIO(), StringIO.StringIO(), configure_logging=False)
manager = LockCheckingManager(port, options, printer)
manager.collect_tests(args)
manager.parse_expectations()
result_summary = manager.set_up_run()
num_unexpected_results = manager.run(result_summary)
manager.clean_up_run()
printer.cleanup()
tester.assertEquals(num_unexpected_results, 0)
def get_port_for_run(args):
options, parsed_args = run_webkit_tests.parse_args(args)
host = MockHost()
test_port = ImageDiffTestPort(host, options=options)
res = passing_run(args, port_obj=test_port, tests_included=True)
self.assertTrue(res)
return test_port
def logging_run(extra_args=None, port_obj=None, tests_included=False):
extra_args = extra_args or []
args = ['--no-record-results']
if not '--platform' in extra_args:
args.extend(['--platform', 'test'])
if not '--child-processes' in extra_args:
args.extend(['--worker-model', 'inline'])
args.extend(extra_args)
if not tests_included:
args.extend(['passes',
'http/tests',
'websocket/tests',
'failures/expected/*'])
oc = outputcapture.OutputCapture()
try:
oc.capture_output()
options, parsed_args = run_webkit_tests.parse_args(args)
user = MockUser()
if not port_obj:
port_obj = port.get(port_name=options.platform, options=options,
user=user)
buildbot_output = array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
res = run_webkit_tests.run(port_obj, options, parsed_args,
buildbot_output=buildbot_output,
regular_output=regular_output)
finally:
oc.restore_output()
return (res, buildbot_output, regular_output, user)
def get_port_for_run(args):
options, parsed_args = run_webkit_tests.parse_args(args)
host = MockHost()
test_port = ImageDiffTestPort(host, options=options)
res = passing_run(args, port_obj=test_port, tests_included=True)
self.assertTrue(res)
return test_port
def _runner(self, port=None):
# FIXME: we shouldn't have to use run_webkit_tests.py to get the options we need.
options = run_webkit_tests.parse_args(['--platform', 'test-mac-mac10.11'])[0]
options.child_processes = '1'
host = MockHost()
port = port or host.port_factory.get(options.platform, options=options)
return LockCheckingRunner(port, options, FakePrinter(), self, True)
def passing_run(args, port_obj=None, logging_included=False):
if not logging_included:
args.extend(['--print', 'nothing'])
options, args = run_webkit_tests.parse_args(args)
if port_obj is None:
port_obj = port.get(options.platform, options)
res = run_webkit_tests.run(port_obj, options, args)
return res == 0
def _runner(self, port=None):
# FIXME: we shouldn't have to use run_webkit_tests.py to get the options we need.
options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopard'])[0]
options.child_processes = '1'
host = MockHost()
port = port or host.port_factory.get(options.platform, options=options)
return LayoutTestRunner(options, port, FakePrinter(), port.results_directory(), lambda test_name: False)
def _runner(self, port=None):
# FIXME: we shouldn't have to use run_webkit_tests.py to get the options we need.
options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopard'])[0]
options.child_processes = '1'
host = MockHost()
port = port or host.port_factory.get(options.platform, options=options)
return LayoutTestRunner(options, port, FakePrinter(), port.results_directory())
def _runner(self, port=None):
# FIXME: we shouldn't have to use run_webkit_tests.py to get the options we need.
options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopard'])[0]
options.child_processes = '1'
host = MockHost()
port = port or host.port_factory.get(options.platform, options=options)
return LockCheckingRunner(port, options, FakePrinter(), self, True)
def passing_run(args, port_obj=None, logging_included=False):
if not logging_included:
args.extend(['--print', 'nothing'])
options, args = run_webkit_tests.parse_args(args)
if port_obj is None:
port_obj = port.get(options.platform, options)
res = run_webkit_tests.run(port_obj, options, args)
return res == 0
def logging_run(args):
options, args = run_webkit_tests.parse_args(args)
port_obj = port.get(options.platform, options)
buildbot_output = array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
res = run_webkit_tests.run(port_obj, options, args,
buildbot_output=buildbot_output,
regular_output=regular_output)
return (res, buildbot_output, regular_output)
def logging_run(args):
options, args = run_webkit_tests.parse_args(args)
port_obj = port.get(options.platform, options)
buildbot_output = array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
res = run_webkit_tests.run(port_obj, options, args,
buildbot_output=buildbot_output,
regular_output=regular_output)
return (res, buildbot_output, regular_output)
def test_perf_locking(tester):
options, args = run_webkit_tests.parse_args(['--platform=test', '--print=nothing', '--no-http', 'passes', 'perf/'])
host = MockHost()
port = host.port_factory.get(port_name=options.platform, options=options)
run_webkit_tests._set_up_derived_options(port, options)
printer = printing.Printer(port, options, StringIO.StringIO(), StringIO.StringIO())
manager = LockCheckingManager(port, options, printer, tester, False)
manager.collect_tests(args)
manager.parse_expectations()
num_unexpected_results = manager.run()
printer.cleanup()
tester.assertEquals(num_unexpected_results, 0)
def test_perf_locking(tester):
options, args = run_webkit_tests.parse_args(['--platform=test', '--print=nothing', '--no-http', 'passes', 'perf/'])
host = MockHost()
port = host.port_factory.get(port_name=options.platform, options=options)
run_webkit_tests._set_up_derived_options(port, options)
printer = printing.Printer(port, options, StringIO.StringIO(), StringIO.StringIO())
manager = LockCheckingManager(port, options, printer, tester, False)
manager.collect_tests(args)
manager.parse_expectations()
num_unexpected_results = manager.run()
printer.cleanup()
tester.assertEquals(num_unexpected_results, 0)
def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False):
extra_args = extra_args or []
args = [
'--print', 'nothing',
'--platform', 'test',
'--no-record-results',
'--worker-model', 'inline']
args.extend(extra_args)
if not tests_included:
# Not including http tests since they get run out of order (that
# behavior has its own test, see test_get_test_file_queue)
args.extend(['passes', 'failures'])
options, parsed_args = run_webkit_tests.parse_args(args)
user = MockUser()
test_batches = []
class RecordingTestDriver(TestDriver):
def __init__(self, port, worker_number):
TestDriver.__init__(self, port, worker_number)
self._current_test_batch = None
def poll(self):
# So that we don't create a new driver for every test
return None
def stop(self):
self._current_test_batch = None
def run_test(self, test_input):
if self._current_test_batch is None:
self._current_test_batch = []
test_batches.append(self._current_test_batch)
test_name = self._port.relative_test_filename(test_input.filename)
self._current_test_batch.append(test_name)
return TestDriver.run_test(self, test_input)
class RecordingTestPort(TestPort):
def create_driver(self, worker_number):
return RecordingTestDriver(self, worker_number)
recording_port = RecordingTestPort(options=options, user=user)
logging_run(extra_args=args, port_obj=recording_port, tests_included=True)
if flatten_batches:
return list(itertools.chain(*test_batches))
return test_batches
def test_missing_and_unexpected_results_with_custom_exit_code(self):
# Test that we update expectations in place. If the expectation
# is missing, update the expected generic location.
class CustomExitCodePort(test.TestPort):
def exit_code_from_summarized_results(self, unexpected_results):
return unexpected_results['num_regressions'] + unexpected_results['num_missing']
host = MockHost()
options, parsed_args = run_webkit_tests.parse_args(['--pixel-tests', '--no-new-test-results'])
test_port = CustomExitCodePort(host, options=options)
details, err, _ = logging_run(['--no-show-results',
'failures/expected/missing_image.html',
'failures/unexpected/missing_text.html',
'failures/unexpected/text-image-checksum.html'],
tests_included=True, host=host, port_obj=test_port)
self.assertEqual(details.exit_code, 2)
def test_missing_and_unexpected_results_with_custom_exit_code(self):
# Test that we update expectations in place. If the expectation
# is missing, update the expected generic location.
class CustomExitCodePort(test.TestPort):
def exit_code_from_summarized_results(self, unexpected_results):
return unexpected_results['num_regressions'] + unexpected_results['num_missing']
host = MockHost()
options, parsed_args = run_webkit_tests.parse_args(['--pixel-tests', '--no-new-test-results'])
test_port = CustomExitCodePort(host, options=options)
details, err, _ = logging_run(['--no-show-results',
'failures/expected/missing_image.html',
'failures/unexpected/missing_text.html',
'failures/unexpected/text-image-checksum.html'],
tests_included=True, host=host, port_obj=test_port)
self.assertEqual(details.exit_code, 2)
def test_missing_and_unexpected_results_with_custom_exit_code(self):
# Test that we update expectations in place. If the expectation
# is missing, update the expected generic location.
fs = unit_test_filesystem()
class CustomExitCodePort(TestPort):
def exit_code_from_summarized_results(self, unexpected_results):
return unexpected_results['num_regressions'] + unexpected_results['num_missing']
options, parsed_args = run_webkit_tests.parse_args(['--pixel-tests', '--no-new-test-results'])
test_port = CustomExitCodePort(options=options, user=mocktool.MockUser())
res, out, err, _ = logging_run(['--no-show-results',
'failures/expected/missing_image.html',
'failures/unexpected/missing_text.html',
'failures/unexpected/text-image-checksum.html'],
tests_included=True, filesystem=fs, record_results=True, port_obj=test_port)
self.assertEquals(res, 2)
def logging_run(args=[], tests_included=False):
new_args = ['--no-record-results']
if not '--platform' in args:
new_args.extend(['--platform', 'test'])
new_args.extend(args)
if not tests_included:
new_args.extend(['passes',
'http/tests'
'websocket/tests',
'failures/expected/*'])
options, parsed_args = run_webkit_tests.parse_args(new_args)
port_obj = port.get(options.platform, options)
buildbot_output = array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
res = run_webkit_tests.run(port_obj, options, parsed_args,
buildbot_output=buildbot_output,
regular_output=regular_output)
return (res, buildbot_output, regular_output)
def parse_args(extra_args=None, tests_included=False, new_results=False, print_nothing=True):
extra_args = extra_args or []
args = []
if not '--platform' in extra_args:
args.extend(['--platform', 'test'])
if not new_results:
args.append('--no-new-test-results')
if not '--child-processes' in extra_args:
args.extend(['--child-processes', 1])
args.extend(extra_args)
if not tests_included:
# We use the glob to test that globbing works.
args.extend(['passes',
'http/tests',
'websocket/tests',
'failures/expected/*'])
return run_webkit_tests.parse_args(args)
def passing_run(args=[], port_obj=None, record_results=False,
tests_included=False):
new_args = ['--print', 'nothing']
if not '--platform' in args:
new_args.extend(['--platform', 'test'])
if not record_results:
new_args.append('--no-record-results')
new_args.extend(args)
if not tests_included:
# We use the glob to test that globbing works.
new_args.extend(['passes',
'http/tests',
'websocket/tests',
'failures/expected/*'])
options, parsed_args = run_webkit_tests.parse_args(new_args)
if port_obj is None:
port_obj = port.get(options.platform, options)
res = run_webkit_tests.run(port_obj, options, parsed_args)
return res == 0
def passing_run(args=[],
port_obj=None,
record_results=False,
tests_included=False):
new_args = ['--print', 'nothing']
if not '--platform' in args:
new_args.extend(['--platform', 'test'])
if not record_results:
new_args.append('--no-record-results')
new_args.extend(args)
if not tests_included:
# We use the glob to test that globbing works.
new_args.extend(
['passes', 'http/tests', 'websocket/tests', 'failures/expected/*'])
options, parsed_args = run_webkit_tests.parse_args(new_args)
if port_obj is None:
port_obj = port.get(options.platform, options)
res = run_webkit_tests.run(port_obj, options, parsed_args)
return res == 0
def test_http_locking(tester):
class LockCheckingManager(Manager):
def __init__(self, port, options, printer):
super(LockCheckingManager,
self).__init__(port, options, printer)
self._finished_list_called = False
def handle_finished_list(self, source, list_name, num_tests,
elapsed_time):
if not self._finished_list_called:
tester.assertEquals(list_name, 'locked_tests')
tester.assertTrue(self._remaining_locked_shards)
tester.assertTrue(self._has_http_lock)
super(LockCheckingManager,
self).handle_finished_list(source, list_name, num_tests,
elapsed_time)
if not self._finished_list_called:
tester.assertEquals(self._remaining_locked_shards, [])
tester.assertFalse(self._has_http_lock)
self._finished_list_called = True
options, args = run_webkit_tests.parse_args([
'--platform=test', '--print=nothing', 'http/tests/passes', 'passes'
])
host = MockHost()
port = host.port_factory.get(port_name=options.platform,
options=options)
run_webkit_tests._set_up_derived_options(port, options)
printer = printing.Printer(port,
options,
StringIO.StringIO(),
StringIO.StringIO(),
configure_logging=False)
manager = LockCheckingManager(port, options, printer)
manager.collect_tests(args)
manager.parse_expectations()
result_summary = manager.set_up_run()
num_unexpected_results = manager.run(result_summary)
manager.clean_up_run()
printer.cleanup()
tester.assertEquals(num_unexpected_results, 0)
def logging_run(args=[], tests_included=False):
new_args = ['--no-record-results']
if not '--platform' in args:
new_args.extend(['--platform', 'test'])
new_args.extend(args)
if not tests_included:
new_args.extend(
['passes', 'http/tests'
'websocket/tests', 'failures/expected/*'])
options, parsed_args = run_webkit_tests.parse_args(new_args)
port_obj = port.get(options.platform, options)
buildbot_output = array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
res = run_webkit_tests.run(port_obj,
options,
parsed_args,
buildbot_output=buildbot_output,
regular_output=regular_output)
return (res, buildbot_output, regular_output)
def parse_args(extra_args=None, record_results=False, tests_included=False, new_results=False, print_nothing=True):
extra_args = extra_args or []
if print_nothing:
args = ["--print", "nothing"]
else:
args = []
if not "--platform" in extra_args:
args.extend(["--platform", "test"])
if not record_results:
args.append("--no-record-results")
if not new_results:
args.append("--no-new-test-results")
if not "--child-processes" in extra_args and not "--worker-model" in extra_args:
args.extend(["--worker-model", "inline"])
args.extend(extra_args)
if not tests_included:
# We use the glob to test that globbing works.
args.extend(["passes", "http/tests", "websocket/tests", "failures/expected/*"])
return run_webkit_tests.parse_args(args)
def parse_args(extra_args=None, record_results=False, tests_included=False,
print_nothing=True):
extra_args = extra_args or []
if print_nothing:
args = ['--print', 'nothing']
else:
args = []
if not '--platform' in extra_args:
args.extend(['--platform', 'test'])
if not record_results:
args.append('--no-record-results')
if not '--child-processes' in extra_args:
args.extend(['--worker-model', 'old-inline'])
args.extend(extra_args)
if not tests_included:
# We use the glob to test that globbing works.
args.extend(['passes',
'http/tests',
'websocket/tests',
'failures/expected/*'])
return run_webkit_tests.parse_args(args)
def test_missing_and_unexpected_results_with_custom_exit_code(self):
# Test that we update expectations in place. If the expectation
# is missing, update the expected generic location.
class CustomExitCodePort(TestPort):
def exit_code_from_summarized_results(self, unexpected_results):
return unexpected_results["num_regressions"] + unexpected_results["num_missing"]
host = MockHost()
options, parsed_args = run_webkit_tests.parse_args(["--pixel-tests", "--no-new-test-results"])
test_port = CustomExitCodePort(host, options=options)
res, out, err, _ = logging_run(
[
"--no-show-results",
"failures/expected/missing_image.html",
"failures/unexpected/missing_text.html",
"failures/unexpected/text-image-checksum.html",
],
tests_included=True,
host=host,
record_results=True,
port_obj=test_port,
)
self.assertEquals(res, 2)
def passing_run(extra_args=None, port_obj=None, record_results=False,
tests_included=False):
extra_args = extra_args or []
args = ['--print', 'nothing']
if not '--platform' in extra_args:
args.extend(['--platform', 'test'])
if not record_results:
args.append('--no-record-results')
if not '--child-processes' in extra_args:
args.extend(['--worker-model', 'inline'])
args.extend(extra_args)
if not tests_included:
# We use the glob to test that globbing works.
args.extend(['passes',
'http/tests',
'websocket/tests',
'failures/expected/*'])
options, parsed_args = run_webkit_tests.parse_args(args)
if not port_obj:
port_obj = port.get(port_name=options.platform, options=options,
user=MockUser())
res = run_webkit_tests.run(port_obj, options, parsed_args)
return res == 0
def get_port_for_run(args):
options, parsed_args = run_webkit_tests.parse_args(args)
test_port = ImageDiffTestPort(options=options, user=mocktool.MockUser())
res = passing_run(args, port_obj=test_port, tests_included=True)
self.assertTrue(res)
return test_port
def get_port_for_run(args):
options, parsed_args = run_webkit_tests.parse_args(args)
test_port = ImageDiffTestPort(options=options, user=MockUser())
passing_run(args, port_obj=test_port, tests_included=True)
return test_port