def logging_run(args):
options, args = run_webkit_tests.parse_args(args)
port_obj = port.get(options.platform, options)
buildbot_output = array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
res = run_webkit_tests.run(port_obj, options, args,
buildbot_output=buildbot_output,
regular_output=regular_output)
return (res, buildbot_output, regular_output)
def create_runner(self, buildbot_output=None, args=[], regular_output=None):
buildbot_output = buildbot_output or array_stream.ArrayStream()
regular_output = regular_output or array_stream.ArrayStream()
options, parsed_args = PerfTestsRunner._parse_args(args)
test_port = TestPort(host=MockHost(), options=options)
test_port.create_driver = lambda worker_number=None, no_timeout=False: MainTest.TestDriver()
runner = PerfTestsRunner(regular_output, buildbot_output, args=args, port=test_port)
runner._host.filesystem.maybe_make_directory(runner._base_path, 'inspector')
runner._host.filesystem.maybe_make_directory(runner._base_path, 'Bindings')
runner._host.filesystem.maybe_make_directory(runner._base_path, 'Parser')
return runner
def run_and_capture(port_obj, options, parsed_args):
oc = outputcapture.OutputCapture()
try:
oc.capture_output()
buildbot_output = array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
res = run_webkit_tests.run(port_obj,
options,
parsed_args,
buildbot_output=buildbot_output,
regular_output=regular_output)
finally:
oc.restore_output()
return (res, buildbot_output, regular_output)
def get_printer(self, args=None, tty=False):
args = args or []
printing_options = printing.print_options()
option_parser = optparse.OptionParser(option_list=printing_options)
options, args = option_parser.parse_args(args)
self._port = port.get('test', options)
nproc = 2
regular_output = array_stream.ArrayStream(tty=tty)
buildbot_output = array_stream.ArrayStream()
printer = printing.Printer(self._port,
options,
regular_output,
buildbot_output,
configure_logging=True)
return printer, regular_output, buildbot_output
def get_printer(self, args=None, single_threaded=False,
is_fully_parallel=False):
printing_options = printing.print_options()
option_parser = optparse.OptionParser(option_list=printing_options)
options, args = option_parser.parse_args(args)
self._port = port.get('test', options)
nproc = 2
if single_threaded:
nproc = 1
regular_output = array_stream.ArrayStream()
buildbot_output = array_stream.ArrayStream()
printer = printing.Printer(self._port, options, regular_output,
buildbot_output, single_threaded,
is_fully_parallel)
return printer, regular_output, buildbot_output
def test_run_test_set(self):
buildbot_output = array_stream.ArrayStream()
runner = self.create_runner(buildbot_output)
port = MainTest.TestPort()
tests = ['pass.html', 'silent.html', 'failed.html', 'tonguey.html', 'timeout.html', 'crash.html']
unexpected_result_count = runner._run_tests_set(tests, port)
self.assertEqual(unexpected_result_count, len(tests) - 1)
self.assertEqual(len(buildbot_output.get()), 1)
self.assertEqual(buildbot_output.get()[0], 'RESULT group_name: test_name= 42 ms\n')
def test_run_test_set_for_parser_tests(self):
buildbot_output = array_stream.ArrayStream()
runner = self.create_runner(buildbot_output)
tests = [runner._base_path + '/Bindings/event-target-wrapper.html', runner._base_path + '/Parser/some-parser.html']
unexpected_result_count = runner._run_tests_set(tests, runner._port)
self.assertEqual(unexpected_result_count, 0)
self.assertEqual(buildbot_output.get()[0], 'RESULT Bindings: event-target-wrapper= 1489.05 ms\n')
self.assertEqual(buildbot_output.get()[1], 'median= 1487.0 ms, stdev= 14.46 ms, min= 1471.0 ms, max= 1510.0 ms\n')
self.assertEqual(buildbot_output.get()[2], 'RESULT Parser: some-parser= 1100.0 ms\n')
self.assertEqual(buildbot_output.get()[3], 'median= 1101.0 ms, stdev= 11.0 ms, min= 1080.0 ms, max= 1120.0 ms\n')
def logging_run(args=[], tests_included=False):
new_args = ['--no-record-results']
if not '--platform' in args:
new_args.extend(['--platform', 'test'])
new_args.extend(args)
if not tests_included:
new_args.extend(
['passes', 'http/tests'
'websocket/tests', 'failures/expected/*'])
options, parsed_args = run_webkit_tests.parse_args(new_args)
port_obj = port.get(options.platform, options)
buildbot_output = array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
res = run_webkit_tests.run(port_obj,
options,
parsed_args,
buildbot_output=buildbot_output,
regular_output=regular_output)
return (res, buildbot_output, regular_output)
def test_run_test_set(self):
buildbot_output = array_stream.ArrayStream()
runner = self.create_runner(buildbot_output)
dirname = runner._base_path + '/inspector/'
tests = [dirname + 'pass.html', dirname + 'silent.html', dirname + 'failed.html',
dirname + 'tonguey.html', dirname + 'timeout.html', dirname + 'crash.html']
unexpected_result_count = runner._run_tests_set(tests, runner._port)
self.assertEqual(unexpected_result_count, len(tests) - 1)
self.assertEqual(len(buildbot_output.get()), 1)
self.assertEqual(buildbot_output.get()[0], 'RESULT group_name: test_name= 42 ms\n')
def passing_run(extra_args=None,
port_obj=None,
record_results=False,
tests_included=False,
filesystem=None):
options, parsed_args = parse_args(extra_args, record_results,
tests_included)
if not port_obj:
port_obj = port.get(port_name=options.platform,
options=options,
user=mocktool.MockUser(),
filesystem=filesystem)
buildbot_output = array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
res = run_webkit_tests.run(port_obj,
options,
parsed_args,
buildbot_output=buildbot_output,
regular_output=regular_output)
return res == 0 and regular_output.empty() and buildbot_output.empty()
def test_timeout(self):
# Because the timeout shows up as a wedged thread, this also tests
# log_wedged_worker().
oc = outputcapture.OutputCapture()
stdout, stderr = oc.capture_output()
logger = message_broker._log
astream = array_stream.ArrayStream()
handler = TestHandler(astream)
logger.addHandler(handler)
interrupted = self.run_one_thread('Timeout')
stdout, stderr = oc.restore_output()
self.assertFalse(interrupted)
logger.handlers.remove(handler)
self.assertTrue(
'All remaining threads are wedged, bailing out.' in astream.get())
def test_run_test_set_with_json_output(self):
buildbot_output = array_stream.ArrayStream()
runner = self.create_runner(buildbot_output, args=['--output-json-path=/mock-checkout/output.json'])
runner._host.filesystem.files[runner._base_path + '/inspector/pass.html'] = True
runner._host.filesystem.files[runner._base_path + '/Bindings/event-target-wrapper.html'] = True
runner._timestamp = 123456789
self.assertEqual(runner.run(), 0)
self.assertEqual(len(buildbot_output.get()), 3)
self.assertEqual(buildbot_output.get()[0], 'RESULT Bindings: event-target-wrapper= 1489.05 ms\n')
self.assertEqual(buildbot_output.get()[1], 'median= 1487.0 ms, stdev= 14.46 ms, min= 1471.0 ms, max= 1510.0 ms\n')
self.assertEqual(buildbot_output.get()[2], 'RESULT group_name: test_name= 42 ms\n')
self.assertEqual(json.loads(runner._host.filesystem.files['/mock-checkout/output.json']), {
"timestamp": 123456789, "results":
{"Bindings/event-target-wrapper": {"max": 1510, "avg": 1489.05, "median": 1487, "min": 1471, "stdev": 14.46},
"group_name:test_name": 42},
"revision": 1234})
def test_configure_logging(self):
# FIXME: We need to figure out how to reset the basic logger.
# FIXME: If other testing classes call logging.basicConfig() then
# FIXME: these calls become no-ops and we can't control the
# FIXME: configuration to test things properly.
options, args = get_options([])
stream = array_stream.ArrayStream()
printing.configure_logging(options, stream)
logging.info("this should be logged")
# self.assertFalse(stream.empty())
stream.reset()
logging.debug("this should not be logged")
# self.assertTrue(stream.empty())
stream.reset()
options, args = get_options(['--verbose'])
printing.configure_logging(options, stream)
logging.debug("this should be logged")
def test_configure_logging(self):
options, args = get_options([])
stream = array_stream.ArrayStream()
handler = printing._configure_logging(stream, options.verbose)
logging.info("this should be logged")
self.assertFalse(stream.empty())
stream.reset()
logging.debug("this should not be logged")
self.assertTrue(stream.empty())
printing._restore_logging(handler)
stream.reset()
options, args = get_options(['--verbose'])
handler = printing._configure_logging(stream, options.verbose)
logging.debug("this should be logged")
self.assertFalse(stream.empty())
printing._restore_logging(handler)
def test_log_wedged_thread(self):
oc = outputcapture.OutputCapture()
oc.capture_output()
logger = run_webkit_tests._log
astream = array_stream.ArrayStream()
handler = TestHandler(astream)
logger.addHandler(handler)
starting_queue = Queue.Queue()
stopping_queue = Queue.Queue()
child_thread = TestThread(starting_queue, stopping_queue)
child_thread.start()
msg = starting_queue.get()
run_webkit_tests._log_wedged_thread(child_thread)
stopping_queue.put('')
child_thread.join(timeout=1.0)
self.assertFalse(astream.empty())
self.assertFalse(child_thread.isAlive())
oc.restore_output()
def test_upload_json(self):
regular_output = array_stream.ArrayStream()
runner = self.create_runner(regular_output=regular_output)
runner._host.filesystem.files['/mock-checkout/some.json'] = 'some content'
called = []
upload_single_text_file_throws = False
upload_single_text_file_return_value = StringIO.StringIO('OK')
class MockFileUploader:
def __init__(mock, url, timeout):
self.assertEqual(url, 'https://some.host/api/test/report')
self.assertTrue(isinstance(timeout, int) and timeout)
called.append('FileUploader')
def upload_single_text_file(mock, filesystem, content_type, filename):
self.assertEqual(filesystem, runner._host.filesystem)
self.assertEqual(content_type, 'application/json')
self.assertEqual(filename, 'some.json')
called.append('upload_single_text_file')
if upload_single_text_file_throws:
raise "Some exception"
return upload_single_text_file_return_value
runner._upload_json('some.host', 'some.json', MockFileUploader)
self.assertEqual(called, ['FileUploader', 'upload_single_text_file'])
output = OutputCapture()
output.capture_output()
upload_single_text_file_return_value = StringIO.StringIO('Some error')
runner._upload_json('some.host', 'some.json', MockFileUploader)
_, _, logs = output.restore_output()
self.assertEqual(logs, 'Uploaded JSON but got a bad response:\nSome error\n')
# Throwing an exception upload_single_text_file shouldn't blow up _upload_json
called = []
upload_single_text_file_throws = True
runner._upload_json('some.host', 'some.json', MockFileUploader)
self.assertEqual(called, ['FileUploader', 'upload_single_text_file'])
def create_runner(self, buildbot_output=None):
buildbot_output = buildbot_output or array_stream.ArrayStream()
regular_output = array_stream.ArrayStream()
return PerfTestsRunner('', regular_output, buildbot_output, args=[])
