def logging_run(args): options, args = run_webkit_tests.parse_args(args) port_obj = port.get(options.platform, options) buildbot_output = array_stream.ArrayStream() regular_output = array_stream.ArrayStream() res = run_webkit_tests.run(port_obj, options, args, buildbot_output=buildbot_output, regular_output=regular_output) return (res, buildbot_output, regular_output)
def create_runner(self, buildbot_output=None, args=[], regular_output=None): buildbot_output = buildbot_output or array_stream.ArrayStream() regular_output = regular_output or array_stream.ArrayStream() options, parsed_args = PerfTestsRunner._parse_args(args) test_port = TestPort(host=MockHost(), options=options) test_port.create_driver = lambda worker_number=None, no_timeout=False: MainTest.TestDriver() runner = PerfTestsRunner(regular_output, buildbot_output, args=args, port=test_port) runner._host.filesystem.maybe_make_directory(runner._base_path, 'inspector') runner._host.filesystem.maybe_make_directory(runner._base_path, 'Bindings') runner._host.filesystem.maybe_make_directory(runner._base_path, 'Parser') return runner
def run_and_capture(port_obj, options, parsed_args): oc = outputcapture.OutputCapture() try: oc.capture_output() buildbot_output = array_stream.ArrayStream() regular_output = array_stream.ArrayStream() res = run_webkit_tests.run(port_obj, options, parsed_args, buildbot_output=buildbot_output, regular_output=regular_output) finally: oc.restore_output() return (res, buildbot_output, regular_output)
def get_printer(self, args=None, tty=False): args = args or [] printing_options = printing.print_options() option_parser = optparse.OptionParser(option_list=printing_options) options, args = option_parser.parse_args(args) self._port = port.get('test', options) nproc = 2 regular_output = array_stream.ArrayStream(tty=tty) buildbot_output = array_stream.ArrayStream() printer = printing.Printer(self._port, options, regular_output, buildbot_output, configure_logging=True) return printer, regular_output, buildbot_output
def get_printer(self, args=None, single_threaded=False, is_fully_parallel=False): printing_options = printing.print_options() option_parser = optparse.OptionParser(option_list=printing_options) options, args = option_parser.parse_args(args) self._port = port.get('test', options) nproc = 2 if single_threaded: nproc = 1 regular_output = array_stream.ArrayStream() buildbot_output = array_stream.ArrayStream() printer = printing.Printer(self._port, options, regular_output, buildbot_output, single_threaded, is_fully_parallel) return printer, regular_output, buildbot_output
def test_run_test_set(self): buildbot_output = array_stream.ArrayStream() runner = self.create_runner(buildbot_output) port = MainTest.TestPort() tests = ['pass.html', 'silent.html', 'failed.html', 'tonguey.html', 'timeout.html', 'crash.html'] unexpected_result_count = runner._run_tests_set(tests, port) self.assertEqual(unexpected_result_count, len(tests) - 1) self.assertEqual(len(buildbot_output.get()), 1) self.assertEqual(buildbot_output.get()[0], 'RESULT group_name: test_name= 42 ms\n')
def test_run_test_set_for_parser_tests(self): buildbot_output = array_stream.ArrayStream() runner = self.create_runner(buildbot_output) tests = [runner._base_path + '/Bindings/event-target-wrapper.html', runner._base_path + '/Parser/some-parser.html'] unexpected_result_count = runner._run_tests_set(tests, runner._port) self.assertEqual(unexpected_result_count, 0) self.assertEqual(buildbot_output.get()[0], 'RESULT Bindings: event-target-wrapper= 1489.05 ms\n') self.assertEqual(buildbot_output.get()[1], 'median= 1487.0 ms, stdev= 14.46 ms, min= 1471.0 ms, max= 1510.0 ms\n') self.assertEqual(buildbot_output.get()[2], 'RESULT Parser: some-parser= 1100.0 ms\n') self.assertEqual(buildbot_output.get()[3], 'median= 1101.0 ms, stdev= 11.0 ms, min= 1080.0 ms, max= 1120.0 ms\n')
def logging_run(args=[], tests_included=False): new_args = ['--no-record-results'] if not '--platform' in args: new_args.extend(['--platform', 'test']) new_args.extend(args) if not tests_included: new_args.extend( ['passes', 'http/tests' 'websocket/tests', 'failures/expected/*']) options, parsed_args = run_webkit_tests.parse_args(new_args) port_obj = port.get(options.platform, options) buildbot_output = array_stream.ArrayStream() regular_output = array_stream.ArrayStream() res = run_webkit_tests.run(port_obj, options, parsed_args, buildbot_output=buildbot_output, regular_output=regular_output) return (res, buildbot_output, regular_output)
def test_run_test_set(self): buildbot_output = array_stream.ArrayStream() runner = self.create_runner(buildbot_output) dirname = runner._base_path + '/inspector/' tests = [dirname + 'pass.html', dirname + 'silent.html', dirname + 'failed.html', dirname + 'tonguey.html', dirname + 'timeout.html', dirname + 'crash.html'] unexpected_result_count = runner._run_tests_set(tests, runner._port) self.assertEqual(unexpected_result_count, len(tests) - 1) self.assertEqual(len(buildbot_output.get()), 1) self.assertEqual(buildbot_output.get()[0], 'RESULT group_name: test_name= 42 ms\n')
def passing_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, filesystem=None): options, parsed_args = parse_args(extra_args, record_results, tests_included) if not port_obj: port_obj = port.get(port_name=options.platform, options=options, user=mocktool.MockUser(), filesystem=filesystem) buildbot_output = array_stream.ArrayStream() regular_output = array_stream.ArrayStream() res = run_webkit_tests.run(port_obj, options, parsed_args, buildbot_output=buildbot_output, regular_output=regular_output) return res == 0 and regular_output.empty() and buildbot_output.empty()
def test_timeout(self): # Because the timeout shows up as a wedged thread, this also tests # log_wedged_worker(). oc = outputcapture.OutputCapture() stdout, stderr = oc.capture_output() logger = message_broker._log astream = array_stream.ArrayStream() handler = TestHandler(astream) logger.addHandler(handler) interrupted = self.run_one_thread('Timeout') stdout, stderr = oc.restore_output() self.assertFalse(interrupted) logger.handlers.remove(handler) self.assertTrue( 'All remaining threads are wedged, bailing out.' in astream.get())
def test_run_test_set_with_json_output(self): buildbot_output = array_stream.ArrayStream() runner = self.create_runner(buildbot_output, args=['--output-json-path=/mock-checkout/output.json']) runner._host.filesystem.files[runner._base_path + '/inspector/pass.html'] = True runner._host.filesystem.files[runner._base_path + '/Bindings/event-target-wrapper.html'] = True runner._timestamp = 123456789 self.assertEqual(runner.run(), 0) self.assertEqual(len(buildbot_output.get()), 3) self.assertEqual(buildbot_output.get()[0], 'RESULT Bindings: event-target-wrapper= 1489.05 ms\n') self.assertEqual(buildbot_output.get()[1], 'median= 1487.0 ms, stdev= 14.46 ms, min= 1471.0 ms, max= 1510.0 ms\n') self.assertEqual(buildbot_output.get()[2], 'RESULT group_name: test_name= 42 ms\n') self.assertEqual(json.loads(runner._host.filesystem.files['/mock-checkout/output.json']), { "timestamp": 123456789, "results": {"Bindings/event-target-wrapper": {"max": 1510, "avg": 1489.05, "median": 1487, "min": 1471, "stdev": 14.46}, "group_name:test_name": 42}, "revision": 1234})
def test_configure_logging(self): # FIXME: We need to figure out how to reset the basic logger. # FIXME: If other testing classes call logging.basicConfig() then # FIXME: these calls become no-ops and we can't control the # FIXME: configuration to test things properly. options, args = get_options([]) stream = array_stream.ArrayStream() printing.configure_logging(options, stream) logging.info("this should be logged") # self.assertFalse(stream.empty()) stream.reset() logging.debug("this should not be logged") # self.assertTrue(stream.empty()) stream.reset() options, args = get_options(['--verbose']) printing.configure_logging(options, stream) logging.debug("this should be logged")
def test_configure_logging(self): options, args = get_options([]) stream = array_stream.ArrayStream() handler = printing._configure_logging(stream, options.verbose) logging.info("this should be logged") self.assertFalse(stream.empty()) stream.reset() logging.debug("this should not be logged") self.assertTrue(stream.empty()) printing._restore_logging(handler) stream.reset() options, args = get_options(['--verbose']) handler = printing._configure_logging(stream, options.verbose) logging.debug("this should be logged") self.assertFalse(stream.empty()) printing._restore_logging(handler)
def test_log_wedged_thread(self): oc = outputcapture.OutputCapture() oc.capture_output() logger = run_webkit_tests._log astream = array_stream.ArrayStream() handler = TestHandler(astream) logger.addHandler(handler) starting_queue = Queue.Queue() stopping_queue = Queue.Queue() child_thread = TestThread(starting_queue, stopping_queue) child_thread.start() msg = starting_queue.get() run_webkit_tests._log_wedged_thread(child_thread) stopping_queue.put('') child_thread.join(timeout=1.0) self.assertFalse(astream.empty()) self.assertFalse(child_thread.isAlive()) oc.restore_output()
def test_upload_json(self): regular_output = array_stream.ArrayStream() runner = self.create_runner(regular_output=regular_output) runner._host.filesystem.files['/mock-checkout/some.json'] = 'some content' called = [] upload_single_text_file_throws = False upload_single_text_file_return_value = StringIO.StringIO('OK') class MockFileUploader: def __init__(mock, url, timeout): self.assertEqual(url, 'https://some.host/api/test/report') self.assertTrue(isinstance(timeout, int) and timeout) called.append('FileUploader') def upload_single_text_file(mock, filesystem, content_type, filename): self.assertEqual(filesystem, runner._host.filesystem) self.assertEqual(content_type, 'application/json') self.assertEqual(filename, 'some.json') called.append('upload_single_text_file') if upload_single_text_file_throws: raise "Some exception" return upload_single_text_file_return_value runner._upload_json('some.host', 'some.json', MockFileUploader) self.assertEqual(called, ['FileUploader', 'upload_single_text_file']) output = OutputCapture() output.capture_output() upload_single_text_file_return_value = StringIO.StringIO('Some error') runner._upload_json('some.host', 'some.json', MockFileUploader) _, _, logs = output.restore_output() self.assertEqual(logs, 'Uploaded JSON but got a bad response:\nSome error\n') # Throwing an exception upload_single_text_file shouldn't blow up _upload_json called = [] upload_single_text_file_throws = True runner._upload_json('some.host', 'some.json', MockFileUploader) self.assertEqual(called, ['FileUploader', 'upload_single_text_file'])
def create_runner(self, buildbot_output=None): buildbot_output = buildbot_output or array_stream.ArrayStream() regular_output = array_stream.ArrayStream() return PerfTestsRunner('', regular_output, buildbot_output, args=[])