def test_basic(self): options, args = parse_args(tests_included=True) logging_stream = StringIO.StringIO() host = MockHost() port_obj = host.port_factory.get(options.platform, options) details = run_webkit_tests.run(port_obj, options, args, logging_stream) # These numbers will need to be updated whenever we add new tests. self.assertEqual(details.initial_results.total, test.TOTAL_TESTS) self.assertEqual(details.initial_results.expected_skips, test.TOTAL_SKIPS) self.assertEqual(len(details.initial_results.unexpected_results_by_name), test.UNEXPECTED_PASSES + test.UNEXPECTED_FAILURES) self.assertEqual(details.exit_code, test.UNEXPECTED_FAILURES) self.assertEqual(details.retry_results.total, test.TOTAL_RETRIES) one_line_summary = "%d tests ran as expected, %d didn't:\n" % ( details.initial_results.total - details.initial_results.expected_skips - len(details.initial_results.unexpected_results_by_name), len(details.initial_results.unexpected_results_by_name)) self.assertTrue(one_line_summary in logging_stream.buflist) # Ensure the results were summarized properly. self.assertEqual(details.summarized_results['num_regressions'], details.exit_code) # Ensure the image diff percentage is in the results. self.assertEqual(details.summarized_results['tests']['failures']['expected']['image.html']['image_diff_percent'], 1) # Ensure the results were written out and displayed. full_results_text = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json') json_to_eval = full_results_text.replace("ADD_RESULTS(", "").replace(");", "") self.assertEqual(json.loads(json_to_eval), details.summarized_results) self.assertEqual(host.user.opened_urls, [path.abspath_to_uri(MockHost().platform, '/tmp/layout-test-results/results.html')])
def test_basic(self): options, args = parse_args(tests_included=True) logging_stream = StringIO.StringIO() host = MockHost() port_obj = host.port_factory.get(options.platform, options) details = run_webkit_tests.run(port_obj, options, args, logging_stream) # These numbers will need to be updated whenever we add new tests. self.assertEqual(details.initial_results.total, test.TOTAL_TESTS) self.assertEqual(details.initial_results.expected_skips, test.TOTAL_SKIPS) self.assertEqual(len(details.initial_results.unexpected_results_by_name), test.UNEXPECTED_PASSES + test.UNEXPECTED_FAILURES) self.assertEqual(details.exit_code, test.UNEXPECTED_FAILURES) self.assertEqual(details.retry_results.total, test.TOTAL_RETRIES) one_line_summary = "%d tests ran as expected, %d didn't:\n" % ( details.initial_results.total - details.initial_results.expected_skips - len(details.initial_results.unexpected_results_by_name), len(details.initial_results.unexpected_results_by_name)) self.assertTrue(one_line_summary in logging_stream.buflist) # Ensure the results were summarized properly. self.assertEqual(details.summarized_results['num_regressions'], details.exit_code) # Ensure the image diff percentage is in the results. self.assertEqual(details.summarized_results['tests']['failures']['expected']['image.html']['image_diff_percent'], 1) # Ensure the results were written out and displayed. full_results_text = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json') json_to_eval = full_results_text.replace("ADD_RESULTS(", "").replace(");", "") self.assertEqual(json.loads(json_to_eval), details.summarized_results) self.assertEqual(host.user.opened_urls, [path.abspath_to_uri(MockHost().platform, '/tmp/layout-test-results/results.html')])
def logging_run(extra_args=None, port_obj=None, tests_included=False): extra_args = extra_args or [] args = ['--no-record-results'] if not '--platform' in extra_args: args.extend(['--platform', 'test']) if not '--child-processes' in extra_args: args.extend(['--worker-model', 'inline']) args.extend(extra_args) if not tests_included: args.extend(['passes', 'http/tests', 'websocket/tests', 'failures/expected/*']) oc = outputcapture.OutputCapture() try: oc.capture_output() options, parsed_args = run_webkit_tests.parse_args(args) user = MockUser() if not port_obj: port_obj = port.get(port_name=options.platform, options=options, user=user) buildbot_output = array_stream.ArrayStream() regular_output = array_stream.ArrayStream() res = run_webkit_tests.run(port_obj, options, parsed_args, buildbot_output=buildbot_output, regular_output=regular_output) finally: oc.restore_output() return (res, buildbot_output, regular_output, user)
def test_verbose_in_child_processes(self): # When we actually run multiple processes, we may have to reconfigure logging in the # child process (e.g., on win32) and we need to make sure that works and we still # see the verbose log output. However, we can't use logging_run() because using # outputcapture to capture stdout and stderr latter results in a nonpicklable host. # Test is flaky on Windows: https://bugs.webkit.org/show_bug.cgi?id=98559 if not self.should_test_processes: return options, parsed_args = parse_args(['--verbose', '--fully-parallel', '--child-processes', '2', 'passes/text.html', 'passes/image.html'], tests_included=True, print_nothing=False) host = MockHost() port_obj = host.port_factory.get(port_name=options.platform, options=options) logging_stream = StringIO.StringIO() run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream) self.assertTrue('text.html passed' in logging_stream.getvalue()) self.assertTrue('image.html passed' in logging_stream.getvalue())
def passing_run(args, port_obj=None, logging_included=False): if not logging_included: args.extend(['--print', 'nothing']) options, args = run_webkit_tests.parse_args(args) if port_obj is None: port_obj = port.get(options.platform, options) res = run_webkit_tests.run(port_obj, options, args) return res == 0
def passing_run(args, port_obj=None, logging_included=False): if not logging_included: args.extend(['--print', 'nothing']) options, args = run_webkit_tests.parse_args(args) if port_obj is None: port_obj = port.get(options.platform, options) res = run_webkit_tests.run(port_obj, options, args) return res == 0
def passing_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, filesystem=None): options, parsed_args = parse_args(extra_args, record_results, tests_included) if not port_obj: port_obj = port.get(port_name=options.platform, options=options, user=mocktool.MockUser(), filesystem=filesystem) buildbot_output = array_stream.ArrayStream() regular_output = array_stream.ArrayStream() res = run_webkit_tests.run(port_obj, options, parsed_args, buildbot_output=buildbot_output, regular_output=regular_output) return res == 0 and regular_output.empty() and buildbot_output.empty()
def test_verbose_in_child_processes(self): # When we actually run multiple processes, we may have to reconfigure logging in the # child process (e.g., on win32) and we need to make sure that works and we still # see the verbose log output. However, we can't use logging_run() because using # outputcapture to capture stdout and stderr latter results in a nonpicklable host. # Test is flaky on Windows: https://bugs.webkit.org/show_bug.cgi?id=98559 if not self.should_test_processes: return options, parsed_args = parse_args(['--verbose', '--fully-parallel', '--child-processes', '2', 'passes/text.html', 'passes/image.html'], tests_included=True, print_nothing=False) host = MockHost() port_obj = host.port_factory.get(port_name=options.platform, options=options) logging_stream = StringIO.StringIO() run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream) self.assertTrue('text.html passed' in logging_stream.getvalue()) self.assertTrue('image.html passed' in logging_stream.getvalue())
def passing_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, filesystem=None): options, parsed_args = parse_args(extra_args, record_results, tests_included) if not port_obj: port_obj = port.get(port_name=options.platform, options=options, user=mocktool.MockUser(), filesystem=filesystem) res = run_webkit_tests.run(port_obj, options, parsed_args) return res == 0
def logging_run(args): options, args = run_webkit_tests.parse_args(args) port_obj = port.get(options.platform, options) buildbot_output = array_stream.ArrayStream() regular_output = array_stream.ArrayStream() res = run_webkit_tests.run(port_obj, options, args, buildbot_output=buildbot_output, regular_output=regular_output) return (res, buildbot_output, regular_output)
def logging_run(args): options, args = run_webkit_tests.parse_args(args) port_obj = port.get(options.platform, options) buildbot_output = array_stream.ArrayStream() regular_output = array_stream.ArrayStream() res = run_webkit_tests.run(port_obj, options, args, buildbot_output=buildbot_output, regular_output=regular_output) return (res, buildbot_output, regular_output)
def passing_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, host=None): options, parsed_args = parse_args(extra_args, record_results, tests_included) if not port_obj: host = host or MockHost() port_obj = host.port_factory.get(port_name=options.platform, options=options) buildbot_output = StringIO.StringIO() regular_output = StringIO.StringIO() res = run_webkit_tests.run(port_obj, options, parsed_args, buildbot_output=buildbot_output, regular_output=regular_output) return res == 0 and not regular_output.getvalue() and not buildbot_output.getvalue()
def run_and_capture(port_obj, options, parsed_args, shared_port=True): if shared_port: port_obj.host.port_factory.get = lambda *args, **kwargs: port_obj oc = outputcapture.OutputCapture() try: oc.capture_output() logging_stream = StringIO.StringIO() run_details = run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream) finally: oc.restore_output() return (run_details, logging_stream)
def run_and_capture(port_obj, options, parsed_args, shared_port=True): if shared_port: port_obj.host.port_factory.get = lambda *args, **kwargs: port_obj oc = outputcapture.OutputCapture() try: oc.capture_output() logging_stream = StringIO.StringIO() run_details = run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream) finally: oc.restore_output() return (run_details, logging_stream)
def passing_run(extra_args=None, port_obj=None, tests_included=False, host=None, shared_port=True): options, parsed_args = parse_args(extra_args, tests_included) if not port_obj: host = host or MockHost() port_obj = host.port_factory.get(port_name=options.platform, options=options) if shared_port: port_obj.host.port_factory.get = lambda *args, **kwargs: port_obj logging_stream = StringIO.StringIO() run_details = run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream) return run_details.exit_code == 0
def passing_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, filesystem=None): options, parsed_args = parse_args(extra_args, record_results, tests_included) filesystem = filesystem or unit_test_filesystem() if not port_obj: host = MockHost() port_obj = host.port_factory.get(port_name=options.platform, options=options, filesystem=filesystem) buildbot_output = array_stream.ArrayStream() regular_output = array_stream.ArrayStream() res = run_webkit_tests.run( port_obj, options, parsed_args, buildbot_output=buildbot_output, regular_output=regular_output ) return res == 0 and regular_output.empty() and buildbot_output.empty()
def passing_run(extra_args=None, port_obj=None, tests_included=False, host=None, shared_port=True): options, parsed_args = parse_args(extra_args, tests_included) if not port_obj: host = host or MockHost() port_obj = host.port_factory.get(port_name=options.platform, options=options) if shared_port: port_obj.host.port_factory.get = lambda *args, **kwargs: port_obj logging_stream = StringIO.StringIO() run_details = run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream) return run_details.exit_code == 0
def run_and_capture(port_obj, options, parsed_args): oc = outputcapture.OutputCapture() try: oc.capture_output() buildbot_output = StringIO.StringIO() regular_output = StringIO.StringIO() res = run_webkit_tests.run(port_obj, options, parsed_args, buildbot_output=buildbot_output, regular_output=regular_output) finally: oc.restore_output() return (res, buildbot_output, regular_output)
def passing_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, filesystem=None): options, parsed_args = parse_args(extra_args, record_results, tests_included) if not port_obj: port_obj = port.get(port_name=options.platform, options=options, user=mocktool.MockUser(), filesystem=filesystem) res = run_webkit_tests.run(port_obj, options, parsed_args) return res == 0
def run_and_capture(port_obj, options, parsed_args): oc = outputcapture.OutputCapture() try: oc.capture_output() buildbot_output = array_stream.ArrayStream() regular_output = array_stream.ArrayStream() res = run_webkit_tests.run(port_obj, options, parsed_args, buildbot_output=buildbot_output, regular_output=regular_output) finally: oc.restore_output() return (res, buildbot_output, regular_output)
def logging_run(args=[], tests_included=False): new_args = ['--no-record-results'] if not '--platform' in args: new_args.extend(['--platform', 'test']) new_args.extend(args) if not tests_included: new_args.extend(['passes', 'http/tests' 'websocket/tests', 'failures/expected/*']) options, parsed_args = run_webkit_tests.parse_args(new_args) port_obj = port.get(options.platform, options) buildbot_output = array_stream.ArrayStream() regular_output = array_stream.ArrayStream() res = run_webkit_tests.run(port_obj, options, parsed_args, buildbot_output=buildbot_output, regular_output=regular_output) return (res, buildbot_output, regular_output)
def passing_run(args=[], port_obj=None, record_results=False, tests_included=False): new_args = ['--print', 'nothing'] if not '--platform' in args: new_args.extend(['--platform', 'test']) if not record_results: new_args.append('--no-record-results') new_args.extend(args) if not tests_included: # We use the glob to test that globbing works. new_args.extend( ['passes', 'http/tests', 'websocket/tests', 'failures/expected/*']) options, parsed_args = run_webkit_tests.parse_args(new_args) if port_obj is None: port_obj = port.get(options.platform, options) res = run_webkit_tests.run(port_obj, options, parsed_args) return res == 0
def passing_run(args=[], port_obj=None, record_results=False, tests_included=False): new_args = ['--print', 'nothing'] if not '--platform' in args: new_args.extend(['--platform', 'test']) if not record_results: new_args.append('--no-record-results') new_args.extend(args) if not tests_included: # We use the glob to test that globbing works. new_args.extend(['passes', 'http/tests', 'websocket/tests', 'failures/expected/*']) options, parsed_args = run_webkit_tests.parse_args(new_args) if port_obj is None: port_obj = port.get(options.platform, options) res = run_webkit_tests.run(port_obj, options, parsed_args) return res == 0
def logging_run(args=[], tests_included=False): new_args = ['--no-record-results'] if not '--platform' in args: new_args.extend(['--platform', 'test']) new_args.extend(args) if not tests_included: new_args.extend( ['passes', 'http/tests' 'websocket/tests', 'failures/expected/*']) options, parsed_args = run_webkit_tests.parse_args(new_args) port_obj = port.get(options.platform, options) buildbot_output = array_stream.ArrayStream() regular_output = array_stream.ArrayStream() res = run_webkit_tests.run(port_obj, options, parsed_args, buildbot_output=buildbot_output, regular_output=regular_output) return (res, buildbot_output, regular_output)
def passing_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, filesystem=None): options, parsed_args = parse_args(extra_args, record_results, tests_included) if not port_obj: port_obj = port.get(port_name=options.platform, options=options, user=mocktool.MockUser(), filesystem=filesystem) buildbot_output = array_stream.ArrayStream() regular_output = array_stream.ArrayStream() res = run_webkit_tests.run(port_obj, options, parsed_args, buildbot_output=buildbot_output, regular_output=regular_output) return res == 0 and regular_output.empty() and buildbot_output.empty()
def get_test_results(args, host=None): options, parsed_args = parse_args(args, tests_included=True) host = host or MockHost() port_obj = host.port_factory.get(port_name=options.platform, options=options) oc = outputcapture.OutputCapture() oc.capture_output() logging_stream = StringIO.StringIO() try: run_details = run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream) finally: oc.restore_output() all_results = [] if run_details.initial_results: all_results.extend(run_details.initial_results.all_results) if run_details.retry_results: all_results.extend(run_details.retry_results.all_results) return all_results
def get_test_results(args, host=None): options, parsed_args = parse_args(args, tests_included=True) host = host or MockHost() port_obj = host.port_factory.get(port_name=options.platform, options=options) oc = outputcapture.OutputCapture() oc.capture_output() logging_stream = StringIO.StringIO() try: run_details = run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream) finally: oc.restore_output() all_results = [] if run_details.initial_results: all_results.extend(run_details.initial_results.all_results) if run_details.retry_results: all_results.extend(run_details.retry_results.all_results) return all_results
def passing_run(extra_args=None, port_obj=None, record_results=False, tests_included=False): extra_args = extra_args or [] args = ['--print', 'nothing'] if not '--platform' in extra_args: args.extend(['--platform', 'test']) if not record_results: args.append('--no-record-results') if not '--child-processes' in extra_args: args.extend(['--worker-model', 'inline']) args.extend(extra_args) if not tests_included: # We use the glob to test that globbing works. args.extend(['passes', 'http/tests', 'websocket/tests', 'failures/expected/*']) options, parsed_args = run_webkit_tests.parse_args(args) if not port_obj: port_obj = port.get(port_name=options.platform, options=options, user=MockUser()) res = run_webkit_tests.run(port_obj, options, parsed_args) return res == 0