def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self):
self.tool.executive = MockExecutive()
port = self.tool.port_factory.get('test-win-win7')
self._write(
port.host.filesystem.join(
port.layout_tests_dir(),
'platform/test-win-win10/failures/expected/image-expected.txt'),
'original win10 result')
oc = OutputCapture()
try:
options = optparse.Values({
'optimize': True,
'builder': 'MOCK Win10',
'suffixes': 'txt',
'verbose': True,
'test': 'failures/expected/image.html',
'results_directory': None,
'build_number': None
})
oc.capture_output()
self.command.execute(options, [], self.tool)
finally:
out, _, _ = oc.restore_output()
self.assertMultiLineEqual(
self._read(self.tool.filesystem.join(
port.layout_tests_dir(),
'platform/test-win-win10/failures/expected/image-expected.txt')),
'MOCK Web result, convert 404 to None=True')
self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(
port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt')))
self.assertMultiLineEqual(
out, '{"remove-lines": [{"test": "failures/expected/image.html", "builder": "MOCK Win10"}]}\n')
def test_convert_for_webkit_nothing_to_convert(self):
"""Tests convert_for_webkit() using a basic test that has nothing to convert."""
test_html = """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>CSS Test: DESCRIPTION OF TEST</title>
<link rel="author" title="NAME_OF_AUTHOR"
href="mailto:EMAIL OR http://CONTACT_PAGE"/>
<link rel="help" href="RELEVANT_SPEC_SECTION"/>
<meta name="assert" content="TEST ASSERTION"/>
<style type="text/css"><![CDATA[
CSS FOR TEST
]]></style>
</head>
<body>
CONTENT OF TEST
</body>
</html>
"""
converter = _W3CTestConverter(DUMMY_PATH, DUMMY_FILENAME, None)
oc = OutputCapture()
oc.capture_output()
try:
converter.feed(test_html)
converter.close()
converted = converter.output()
finally:
oc.restore_output()
self.verify_no_conversion_happened(converted, test_html)
def run_test(self,
tests,
expected_stdout,
platform='test-win-win7',
**kwargs):
options_defaults = {
'all': False,
'csv': False,
'full': False,
'platform': platform,
'include_keyword': [],
'exclude_keyword': [],
'paths': False,
}
options_defaults.update(kwargs)
options = optparse.Values(dict(**options_defaults))
tool = MockBlinkTool()
tool.port_factory.all_port_names = lambda: [
'test-linux-trusty', 'test-linux-precise', 'test-mac-mac10.11',
'test-mac-mac10.10', 'test-win-win10', 'test-win-win7'
]
command = PrintExpectations()
oc = OutputCapture()
try:
oc.capture_output()
command.execute(options, tests, tool)
finally:
stdout, _, _ = oc.restore_output()
self.assertMultiLineEqual(stdout, expected_stdout)
def test_convert_for_webkit_properties_only(self):
"""Tests convert_for_webkit() using a test that has 2 prefixed properties: 1 in a style block + 1 inline style."""
test_html = """<html>
<head>
<link href="/resources/testharness.css" rel="stylesheet" type="text/css">
<script src="/resources/testharness.js"></script>
<style type="text/css">
#block1 { @test0@: propvalue; }
</style>
</head>
<body>
<div id="elem1" style="@test1@: propvalue;"></div>
</body>
</html>
"""
fake_dir_path = self.fake_dir_path('harnessandprops')
converter = _W3CTestConverter(fake_dir_path, DUMMY_FILENAME, None)
test_content = self.generate_test_content(converter.prefixed_properties, 1, test_html)
oc = OutputCapture()
oc.capture_output()
try:
converter.feed(test_content[1])
converter.close()
converted = converter.output()
finally:
oc.restore_output()
self.verify_conversion_happened(converted)
self.verify_prefixed_properties(converted, test_content[0])
class OutputCaptureTest(unittest.TestCase):
def setUp(self):
self.output = OutputCapture()
def log_all_levels(self):
_log.info('INFO')
_log.warning('WARN')
_log.error('ERROR')
_log.critical('CRITICAL')
def assert_logged(self, expected_logs):
actual_stdout, actual_stderr, actual_logs = self.output.restore_output(
)
self.assertEqual('', actual_stdout)
self.assertEqual('', actual_stderr)
self.assertMultiLineEqual(expected_logs, actual_logs)
def test_initial_log_level(self):
self.output.capture_output()
self.log_all_levels()
self.assert_logged('INFO\nWARN\nERROR\nCRITICAL\n')
def test_set_log_level(self):
self.output.set_log_level(logging.ERROR)
self.output.capture_output()
self.log_all_levels()
self.output.set_log_level(logging.WARN)
self.log_all_levels()
self.assert_logged('ERROR\nCRITICAL\nWARN\nERROR\nCRITICAL\n')
def test_start_cmd(self):
# Fails on win - see https://bugs.webkit.org/show_bug.cgi?id=84726
if sys.platform in ('cygwin', 'win32'):
return
def fake_pid(_):
host.filesystem.write_text_file('/tmp/WebKit/httpd.pid', '42')
return True
host = MockHost()
host.executive = MockExecutive(should_log=True)
test_port = test.TestPort(host)
host.filesystem.write_text_file(test_port.path_to_apache_config_file(),
'')
server = ApacheHTTP(test_port,
"/mock/output_dir",
additional_dirs=[],
number_of_servers=4)
server._check_that_all_ports_are_available = lambda: True
server._is_server_running_on_all_ports = lambda: True
server._wait_for_action = fake_pid
oc = OutputCapture()
try:
oc.capture_output()
server.start()
server.stop()
finally:
_, _, logs = oc.restore_output()
self.assertIn("StartServers 4", logs)
self.assertIn("MinSpareServers 4", logs)
self.assertIn("MaxSpareServers 4", logs)
def test_parse_output_with_subtests(self):
output = DriverOutput("""
Running 20 times
some test: [1, 2, 3, 4, 5]
other test = else: [6, 7, 8, 9, 10]
Ignoring warm-up run (1115)
Time:
values 1080, 1120, 1095, 1101, 1104 ms
avg 1100 ms
median 1101 ms
stdev 14.50862 ms
min 1080 ms
max 1120 ms
""", image=None, image_hash=None, audio=None)
output_capture = OutputCapture()
output_capture.capture_output()
try:
test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test')
self._assert_results_are_correct(test, output)
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
self.assertEqual(actual_stdout, '')
self.assertEqual(actual_stderr, '')
self.assertEqual(actual_logs, '')
def test_analyze_test_reftest_match_and_mismatch(self):
test_html = """<head>
<link rel="match" href="green-box-ref.xht" />
<link rel="match" href="blue-box-ref.xht" />
<link rel="mismatch" href="orange-box-notref.xht" />
</head>
"""
oc = OutputCapture()
oc.capture_output()
try:
test_path = '/some/madeup/path/'
parser = TestParser(test_path + 'somefile.html', MockHost())
test_info = parser.analyze_test(test_contents=test_html)
finally:
_, _, logs = oc.restore_output()
self.assertNotEqual(test_info, None, 'did not find a test')
self.assertTrue('test' in test_info.keys(), 'did not find a test file')
self.assertTrue('reference' in test_info.keys(),
'did not find a reference file')
self.assertTrue(test_info['reference'].startswith(test_path),
'reference path is not correct')
self.assertFalse('refsupport' in test_info.keys(),
'there should be no refsupport files for this test')
self.assertFalse('jstest' in test_info.keys(),
'test should not have been analyzed as a jstest')
self.assertEqual(
logs,
'Multiple references are not supported. Importing the first ref defined in somefile.html\n'
)
def test_convert_attributes_if_needed(self):
"""Tests convert_attributes_if_needed() using a reference file that has some relative src paths."""
test_html = """<html>
<head>
<script src="../../some-script.js"></script>
<style src="../../../some-style.css"></style>
</head>
<body>
<img src="../../../../some-image.jpg">
</body>
</html>
"""
test_reference_support_info = {
'reference_relpath': '../',
'files': ['../../some-script.js', '../../../some-style.css', '../../../../some-image.jpg'],
'elements': ['script', 'style', 'img']
}
converter = _W3CTestConverter(DUMMY_PATH, DUMMY_FILENAME, test_reference_support_info)
oc = OutputCapture()
oc.capture_output()
try:
converter.feed(test_html)
converter.close()
converted = converter.output()
finally:
oc.restore_output()
self.verify_conversion_happened(converted)
self.verify_reference_relative_paths(converted, test_reference_support_info)
def test_parse_output_with_failing_line(self):
output = DriverOutput("""
Running 20 times
Ignoring warm-up run (1115)
some-unrecognizable-line
Time:
values 1080, 1120, 1095, 1101, 1104 ms
avg 1100 ms
median 1101 ms
stdev 14.50862 ms
min 1080 ms
max 1120 ms
""", image=None, image_hash=None, audio=None)
output_capture = OutputCapture()
output_capture.capture_output()
try:
test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test')
test.run_single = lambda driver, path, time_out_ms: output
self.assertFalse(test._run_with_driver(None, None))
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
self.assertEqual(actual_stdout, '')
self.assertEqual(actual_stderr, '')
self.assertEqual(actual_logs, 'ERROR: some-unrecognizable-line\n')
def test_check_httpd_success(self):
port = self.make_port(executive=MockExecutive())
port.path_to_apache = lambda: '/usr/sbin/httpd'
capture = OutputCapture()
capture.capture_output()
self.assertTrue(port.check_httpd())
_, _, logs = capture.restore_output()
self.assertEqual('', logs)
def test_httpd_returns_error_code(self):
port = self.make_port(executive=MockExecutive(exit_code=1))
port.path_to_apache = lambda: '/usr/sbin/httpd'
capture = OutputCapture()
capture.capture_output()
self.assertFalse(port.check_httpd())
_, _, logs = capture.restore_output()
self.assertEqual('httpd seems broken. Cannot run http tests.\n', logs)
def test_help_command(self):
oc = OutputCapture()
oc.capture_output()
tool = WebKitPatch('path')
tool.main(['tool', 'help'])
out, err, logs = oc.restore_output()
self.assertTrue(out.startswith('Usage: '))
self.assertEqual('', err)
self.assertEqual('', logs)
def test_pretty_patch_os_error(self):
port = self.make_port(executive=MockExecutive(exception=OSError))
oc = OutputCapture()
oc.capture_output()
self.assertEqual(port.pretty_patch_text("patch.txt"),
port._pretty_patch_error_html)
# This tests repeated calls to make sure we cache the result.
self.assertEqual(port.pretty_patch_text("patch.txt"),
port._pretty_patch_error_html)
oc.restore_output()
def test_run_test_set_for_parser_tests(self):
runner, _ = self.create_runner()
tests = self._tests_for_runner(runner, ['Bindings/event-target-wrapper.html', 'Parser/some-parser.html'])
output = OutputCapture()
output.capture_output()
try:
unexpected_result_count = runner._run_tests_set(tests)
finally:
_, _, log = output.restore_output()
self.assertEqual(unexpected_result_count, 0)
self.assertEqual(self._normalize_output(log), EventTargetWrapperTestData.output + SomeParserTestData.output)
def test_rebaseline_expectations_noop(self):
self._zero_out_test_expectations()
oc = OutputCapture()
try:
oc.capture_output()
self.command.execute(self.options(), [], self.tool)
finally:
_, _, logs = oc.restore_output()
self.assertEqual(self.tool.filesystem.written_files, {})
self.assertEqual(logs, 'Did not find any tests marked Rebaseline.\n')
def test_run_test_set(self):
runner, _ = self.create_runner()
tests = self._tests_for_runner(runner, ['inspector/pass.html', 'inspector/silent.html', 'inspector/failed.html',
'inspector/tonguey.html', 'inspector/timeout.html', 'inspector/crash.html'])
output = OutputCapture()
output.capture_output()
try:
unexpected_result_count = runner._run_tests_set(tests)
finally:
_, _, log = output.restore_output()
self.assertEqual(unexpected_result_count, len(tests) - 1)
self.assertTrue('\nRESULT group_name: test_name= 42 ms\n' in log)
def test_help_argument(self):
oc = OutputCapture()
oc.capture_output()
tool = WebKitPatch('path')
try:
tool.main(['tool', '--help'])
except SystemExit:
pass # optparse calls sys.exit after showing help.
finally:
out, err, logs = oc.restore_output()
self.assertTrue(out.startswith('Usage: '))
self.assertEqual('', err)
self.assertEqual('', logs)
def _test_run_with_json_output(self,
runner,
filesystem,
upload_succeeds=False,
results_shown=True,
expected_exit_code=0,
repeat=1,
compare_logs=True):
filesystem.write_text_file(runner._base_path + '/inspector/pass.html',
'some content')
filesystem.write_text_file(
runner._base_path + '/Bindings/event-target-wrapper.html',
'some content')
uploaded = [False]
def mock_upload_json(hostname, json_path, host_path=None):
# FIXME: Get rid of the hard-coded perf.webkit.org once we've completed the transition.
self.assertIn(hostname, ['some.host'])
self.assertIn(json_path, ['/mock-checkout/output.json'])
self.assertIn(host_path, [None, '/api/report'])
uploaded[0] = upload_succeeds
return upload_succeeds
runner._upload_json = mock_upload_json
runner._timestamp = 123456789
runner._utc_timestamp = datetime.datetime(2013, 2, 8, 15, 19, 37,
460000)
output_capture = OutputCapture()
output_capture.capture_output()
try:
self.assertEqual(runner.run(), expected_exit_code)
finally:
_, _, logs = output_capture.restore_output()
if not expected_exit_code and compare_logs:
expected_logs = ''
for i in xrange(repeat):
runs = ' (Run %d of %d)' % (i + 1,
repeat) if repeat > 1 else ''
expected_logs += 'Running 2 tests%s\n' % runs + EventTargetWrapperTestData.output + InspectorPassTestData.output
if results_shown:
expected_logs += 'MOCK: user.open_url: file://...\n'
self.assertEqual(self._normalize_output(logs), expected_logs)
self.assertEqual(uploaded[0], upload_succeeds)
return logs
def test_run_memory_test(self):
runner, port = self.create_runner_and_setup_results_template()
runner._timestamp = 123456789
port.host.filesystem.write_text_file(runner._base_path + '/Parser/memory-test.html', 'some content')
output = OutputCapture()
output.capture_output()
try:
unexpected_result_count = runner.run()
finally:
_, _, log = output.restore_output()
self.assertEqual(unexpected_result_count, 0)
self.assertEqual(self._normalize_output(log), MemoryTestData.output + '\nMOCK: user.open_url: file://...\n')
parser_tests = self._load_output_json(runner)[0]['tests']['Parser']['tests']
self.assertEqual(parser_tests['memory-test']['metrics']['Time'], MemoryTestData.results)
self.assertEqual(parser_tests['memory-test']['metrics']['JSHeap'], MemoryTestData.js_heap_results)
self.assertEqual(parser_tests['memory-test']['metrics']['Malloc'], MemoryTestData.malloc_results)
def test_optimize_all_suffixes_by_default(self):
test_port = self.tool.port_factory.get('test')
self._write_test_file(test_port, 'another/test.html',
"Dummy test contents")
self._write_test_file(
test_port, 'platform/test-mac-mac10.10/another/test-expected.txt',
"result A")
self._write_test_file(
test_port, 'platform/test-mac-mac10.10/another/test-expected.png',
"result A png")
self._write_test_file(test_port, 'another/test-expected.txt',
"result A")
self._write_test_file(test_port, 'another/test-expected.png',
"result A png")
try:
oc = OutputCapture()
oc.capture_output()
self.command.execute(
optparse.Values({
'suffixes': 'txt,wav,png',
'no_modify_scm': True,
'platform': 'test-mac-mac10.10'
}), ['another/test.html'], self.tool)
finally:
oc.restore_output()
self.assertFalse(
self.tool.filesystem.exists(
self.tool.filesystem.join(
test_port.layout_tests_dir(),
'platform/mac/another/test-expected.txt')))
self.assertFalse(
self.tool.filesystem.exists(
self.tool.filesystem.join(
test_port.layout_tests_dir(),
'platform/mac/another/test-expected.png')))
self.assertTrue(
self.tool.filesystem.exists(
self.tool.filesystem.join(test_port.layout_tests_dir(),
'another/test-expected.txt')))
self.assertTrue(
self.tool.filesystem.exists(
self.tool.filesystem.join(test_port.layout_tests_dir(),
'another/test-expected.png')))
class PrintBaselinesTest(unittest.TestCase):
def setUp(self):
self.oc = None
self.tool = MockBlinkTool()
self.test_port = self.tool.port_factory.get('test-win-win7')
self.tool.port_factory.get = lambda port_name=None: self.test_port
self.tool.port_factory.all_port_names = lambda: [
'test-linux-trusty', 'test-linux-precise', 'test-mac-mac10.11',
'test-mac-mac10.10', 'test-win-win10', 'test-win-win7'
]
def tearDown(self):
if self.oc:
self.restore_output()
def capture_output(self):
self.oc = OutputCapture()
self.oc.capture_output()
def restore_output(self):
stdout, stderr, logs = self.oc.restore_output()
self.oc = None
return (stdout, stderr, logs)
def test_basic(self):
command = PrintBaselines()
self.capture_output()
options = optparse.Values({
'all': False,
'include_virtual_tests': False,
'csv': False,
'platform': None
})
command.execute(options, ['passes/text.html'], self.tool)
stdout, _, _ = self.restore_output()
self.assertMultiLineEqual(stdout, ('// For test-win-win7\n'
'passes/text-expected.png\n'
'passes/text-expected.txt\n'))
def test_multiple(self):
command = PrintBaselines()
self.capture_output()
options = optparse.Values({
'all': False,
'include_virtual_tests': False,
'csv': False,
'platform': 'test-win-*'
})
command.execute(options, ['passes/text.html'], self.tool)
stdout, _, _ = self.restore_output()
self.assertMultiLineEqual(stdout, ('// For test-win-win10\n'
'passes/text-expected.png\n'
'passes/text-expected.txt\n'
'\n'
'// For test-win-win7\n'
'passes/text-expected.png\n'
'passes/text-expected.txt\n'))
def test_csv(self):
command = PrintBaselines()
self.capture_output()
options = optparse.Values({
'all': False,
'platform': '*win7',
'csv': True,
'include_virtual_tests': False
})
command.execute(options, ['passes/text.html'], self.tool)
stdout, _, _ = self.restore_output()
self.assertMultiLineEqual(stdout, (
'test-win-win7,passes/text.html,None,png,passes/text-expected.png,None\n'
'test-win-win7,passes/text.html,None,txt,passes/text-expected.txt,None\n'
))
def test_upload_json(self):
runner, port = self.create_runner()
port.host.filesystem.files['/mock-checkout/some.json'] = 'some content'
class MockFileUploader:
called = []
upload_single_text_file_throws = False
upload_single_text_file_return_value = None
@classmethod
def reset(cls):
cls.called = []
cls.upload_single_text_file_throws = False
cls.upload_single_text_file_return_value = None
def __init__(mock, url, timeout):
self.assertEqual(url, 'https://some.host/some/path')
self.assertTrue(isinstance(timeout, int) and timeout)
mock.called.append('FileUploader')
def upload_single_text_file(mock, filesystem, content_type,
filename):
self.assertEqual(filesystem, port.host.filesystem)
self.assertEqual(content_type, 'application/json')
self.assertEqual(filename, 'some.json')
mock.called.append('upload_single_text_file')
if mock.upload_single_text_file_throws:
raise Exception
return mock.upload_single_text_file_return_value
MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO(
'OK')
self.assertTrue(
runner._upload_json('some.host', 'some.json', '/some/path',
MockFileUploader))
self.assertEqual(MockFileUploader.called,
['FileUploader', 'upload_single_text_file'])
MockFileUploader.reset()
MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO(
'Some error')
output = OutputCapture()
output.capture_output()
self.assertFalse(
runner._upload_json('some.host', 'some.json', '/some/path',
MockFileUploader))
_, _, logs = output.restore_output()
self.assertEqual(
logs,
'Uploaded JSON to https://some.host/some/path but got a bad response:\nSome error\n'
)
# Throwing an exception upload_single_text_file shouldn't blow up _upload_json
MockFileUploader.reset()
MockFileUploader.upload_single_text_file_throws = True
self.assertFalse(
runner._upload_json('some.host', 'some.json', '/some/path',
MockFileUploader))
self.assertEqual(MockFileUploader.called,
['FileUploader', 'upload_single_text_file'])
MockFileUploader.reset()
MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO(
'{"status": "OK"}')
self.assertTrue(
runner._upload_json('some.host', 'some.json', '/some/path',
MockFileUploader))
self.assertEqual(MockFileUploader.called,
['FileUploader', 'upload_single_text_file'])
MockFileUploader.reset()
MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO(
'{"status": "SomethingHasFailed", "failureStored": false}')
output = OutputCapture()
output.capture_output()
self.assertFalse(
runner._upload_json('some.host', 'some.json', '/some/path',
MockFileUploader))
_, _, logs = output.restore_output()
serialized_json = json.dumps(
{
'status': 'SomethingHasFailed',
'failureStored': False
}, indent=4)
self.assertEqual(
logs,
'Uploaded JSON to https://some.host/some/path but got an error:\n%s\n'
% serialized_json)
def test_convert_prefixed_properties(self):
"""Tests convert_prefixed_properties() file that has 20 properties requiring the -webkit- prefix.
The properties are:
10 in one style block + 5 in another style
block + 5 inline styles, including one with multiple prefixed properties.
2 when prefixed properties appear in comments without ending ';'.
The properties in the test content are in all sorts of wack formatting.
"""
test_html = """<html>
<style type="text/css"><![CDATA[
.block1 {
width: 300px;
height: 300px
}
.block2 {
@test0@: propvalue;
}
.block3{@test1@: propvalue;}
.block4 { @test2@:propvalue; }
.block5{ @test3@ :propvalue; }
#block6 { @test4@ : propvalue; }
#block7
{
@test5@: propvalue;
}
#block8 { @test6@: propvalue; }
#block9:pseudo
{
@test7@: propvalue;
@test8@: propvalue propvalue propvalue;
}
]]></style>
</head>
<body>
<div id="elem1" style="@test9@: propvalue;"></div>
<div id="elem2" style="propname: propvalue; @test10@ : propvalue; propname:propvalue;"></div>
<div id="elem2" style="@test11@: propvalue; @test12@ : propvalue; @test13@ :propvalue;"></div>
<div id="elem3" style="@test14@:propvalue"></div>
</body>
<style type="text/css"><![CDATA[
.block10{ @test15@: propvalue; }
.block11{ @test16@: propvalue; }
.block12{ @test17@: propvalue; }
#block13:pseudo
{
@test18@: propvalue;
@test19@: propvalue;
}
#missing-semicolon-in-comments {
/* @test20@: propvalue */
@test21@: propvalue;
}
]]></style>
</html>
"""
converter = _W3CTestConverter(DUMMY_PATH, DUMMY_FILENAME, None)
test_content = self.generate_test_content(converter.prefixed_properties, 22, test_html)
oc = OutputCapture()
oc.capture_output()
try:
converter.feed(test_content[1])
converter.close()
converted = converter.output()
finally:
oc.restore_output()
self.verify_conversion_happened(converted)
self.verify_prefixed_properties(converted, test_content[0])