def test_help_command(self):
oc = OutputCapture()
oc.capture_output()
tool = WebKitPatch('path')
tool.main(['tool', 'help'])
out, err, logs = oc.restore_output()
self.assertTrue(out.startswith('Usage: '))
self.assertEqual('', err)
self.assertEqual('', logs)
def test_run_test_set_for_parser_tests(self):
runner, _ = self.create_runner()
tests = self._tests_for_runner(runner, ['Bindings/event-target-wrapper.html', 'Parser/some-parser.html'])
output = OutputCapture()
output.capture_output()
try:
unexpected_result_count = runner._run_tests_set(tests)
finally:
_, _, log = output.restore_output()
self.assertEqual(unexpected_result_count, 0)
self.assertEqual(self._normalize_output(log), EventTargetWrapperTestData.output + SomeParserTestData.output)
def test_rebaseline_expectations_noop(self):
self._zero_out_test_expectations()
oc = OutputCapture()
try:
oc.capture_output()
self.command.execute(self.options(), [], self.tool)
finally:
_, _, logs = oc.restore_output()
self.assertEqual(self.tool.filesystem.written_files, {})
self.assertEqual(logs, 'Did not find any tests marked Rebaseline.\n')
def test_run_test_set(self):
runner, _ = self.create_runner()
tests = self._tests_for_runner(runner, ['inspector/pass.html', 'inspector/silent.html', 'inspector/failed.html',
'inspector/tonguey.html', 'inspector/timeout.html', 'inspector/crash.html'])
output = OutputCapture()
output.capture_output()
try:
unexpected_result_count = runner._run_tests_set(tests)
finally:
_, _, log = output.restore_output()
self.assertEqual(unexpected_result_count, len(tests) - 1)
self.assertTrue('\nRESULT group_name: test_name= 42 ms\n' in log)
def test_help_argument(self):
oc = OutputCapture()
oc.capture_output()
tool = WebKitPatch('path')
try:
tool.main(['tool', '--help'])
except SystemExit:
pass # optparse calls sys.exit after showing help.
finally:
out, err, logs = oc.restore_output()
self.assertTrue(out.startswith('Usage: '))
self.assertEqual('', err)
self.assertEqual('', logs)
def test_convert_attributes_if_needed(self):
"""Tests convert_attributes_if_needed() using a reference file that has some relative src paths."""
test_html = """<html>
<head>
<script src="../../some-script.js"></script>
<style src="../../../some-style.css"></style>
</head>
<body>
<img src="../../../../some-image.jpg">
</body>
</html>
"""
test_reference_support_info = {
'reference_relpath':
'../',
'files': [
'../../some-script.js', '../../../some-style.css',
'../../../../some-image.jpg'
],
'elements': ['script', 'style', 'img']
}
converter = _W3CTestConverter(DUMMY_PATH, DUMMY_FILENAME,
test_reference_support_info)
oc = OutputCapture()
oc.capture_output()
try:
converter.feed(test_html)
converter.close()
converted = converter.output()
finally:
oc.restore_output()
self.verify_conversion_happened(converted)
self.verify_reference_relative_paths(converted,
test_reference_support_info)
def _test_run_with_json_output(self,
runner,
filesystem,
upload_succeeds=False,
results_shown=True,
expected_exit_code=0,
repeat=1,
compare_logs=True):
filesystem.write_text_file(runner._base_path + '/inspector/pass.html',
'some content')
filesystem.write_text_file(
runner._base_path + '/Bindings/event-target-wrapper.html',
'some content')
uploaded = [False]
def mock_upload_json(hostname, json_path, host_path=None):
# FIXME: Get rid of the hard-coded perf.webkit.org once we've completed the transition.
self.assertIn(hostname, ['some.host'])
self.assertIn(json_path, ['/mock-checkout/output.json'])
self.assertIn(host_path, [None, '/api/report'])
uploaded[0] = upload_succeeds
return upload_succeeds
runner._upload_json = mock_upload_json
runner._timestamp = 123456789
runner._utc_timestamp = datetime.datetime(2013, 2, 8, 15, 19, 37,
460000)
output_capture = OutputCapture()
output_capture.capture_output()
try:
self.assertEqual(runner.run(), expected_exit_code)
finally:
_, _, logs = output_capture.restore_output()
if not expected_exit_code and compare_logs:
expected_logs = ''
for i in xrange(repeat):
runs = ' (Run %d of %d)' % (i + 1,
repeat) if repeat > 1 else ''
expected_logs += 'Running 2 tests%s\n' % runs + EventTargetWrapperTestData.output + InspectorPassTestData.output
if results_shown:
expected_logs += 'MOCK: user.open_url: file://...\n'
self.assertEqual(self._normalize_output(logs), expected_logs)
self.assertEqual(uploaded[0], upload_succeeds)
return logs
def test_run_memory_test(self):
runner, port = self.create_runner_and_setup_results_template()
runner._timestamp = 123456789
port.host.filesystem.write_text_file(runner._base_path + '/Parser/memory-test.html', 'some content')
output = OutputCapture()
output.capture_output()
try:
unexpected_result_count = runner.run()
finally:
_, _, log = output.restore_output()
self.assertEqual(unexpected_result_count, 0)
self.assertEqual(self._normalize_output(log), MemoryTestData.output + '\nMOCK: user.open_url: file://...\n')
parser_tests = self._load_output_json(runner)[0]['tests']['Parser']['tests']
self.assertEqual(parser_tests['memory-test']['metrics']['Time'], MemoryTestData.results)
self.assertEqual(parser_tests['memory-test']['metrics']['JSHeap'], MemoryTestData.js_heap_results)
self.assertEqual(parser_tests['memory-test']['metrics']['Malloc'], MemoryTestData.malloc_results)
def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self):
self.tool.executive = MockExecutive()
port = self.tool.port_factory.get('test-win-win7')
self._write(
port.host.filesystem.join(
port.layout_tests_dir(),
'platform/test-win-win10/failures/expected/image-expected.txt'
), 'original win10 result')
oc = OutputCapture()
try:
options = optparse.Values({
'optimize': True,
'builder': 'MOCK Win10',
'port_name': None,
'suffixes': 'txt',
'verbose': True,
'test': 'failures/expected/image.html',
'results_directory': None,
'build_number': None
})
oc.capture_output()
self.command.execute(options, [], self.tool)
finally:
out, _, _ = oc.restore_output()
self.assertMultiLineEqual(
self._read(
self.tool.filesystem.join(
port.layout_tests_dir(),
'platform/test-win-win10/failures/expected/image-expected.txt'
)), 'MOCK Web result, convert 404 to None=True')
self.assertFalse(
self.tool.filesystem.exists(
self.tool.filesystem.join(
port.layout_tests_dir(),
'platform/test-win-win7/failures/expected/image-expected.txt'
)))
self.assertMultiLineEqual(
out,
'{"remove-lines": [{"test": "failures/expected/image.html", "port_name": "test-win-win10"}]}\n'
)
def test_parse_output(self):
output = DriverOutput("""
Running 20 times
Ignoring warm-up run (1115)
Time:
values 1080, 1120, 1095, 1101, 1104 ms
avg 1100 ms
median 1101 ms
stdev 14.50862 ms
min 1080 ms
max 1120 ms
""", image=None, image_hash=None, audio=None)
output_capture = OutputCapture()
output_capture.capture_output()
try:
test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test')
self._assert_results_are_correct(test, output)
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
self.assertEqual(actual_stdout, '')
self.assertEqual(actual_stderr, '')
self.assertEqual(actual_logs, '')
class PrintBaselinesTest(unittest.TestCase):
def setUp(self):
self.oc = None
self.tool = MockBlinkTool()
self.test_port = self.tool.port_factory.get('test-win-win7')
self.tool.port_factory.get = lambda port_name=None: self.test_port
self.tool.port_factory.all_port_names = lambda: [
'test-linux-trusty', 'test-linux-precise', 'test-mac-mac10.11',
'test-mac-mac10.10', 'test-win-win10', 'test-win-win7'
]
def tearDown(self):
if self.oc:
self.restore_output()
def capture_output(self):
self.oc = OutputCapture()
self.oc.capture_output()
def restore_output(self):
stdout, stderr, logs = self.oc.restore_output()
self.oc = None
return (stdout, stderr, logs)
def test_basic(self):
command = PrintBaselines()
self.capture_output()
options = optparse.Values({
'all': False,
'include_virtual_tests': False,
'csv': False,
'platform': None
})
command.execute(options, ['passes/text.html'], self.tool)
stdout, _, _ = self.restore_output()
self.assertMultiLineEqual(stdout, ('// For test-win-win7\n'
'passes/text-expected.png\n'
'passes/text-expected.txt\n'))
def test_multiple(self):
command = PrintBaselines()
self.capture_output()
options = optparse.Values({
'all': False,
'include_virtual_tests': False,
'csv': False,
'platform': 'test-win-*'
})
command.execute(options, ['passes/text.html'], self.tool)
stdout, _, _ = self.restore_output()
self.assertMultiLineEqual(stdout, ('// For test-win-win10\n'
'passes/text-expected.png\n'
'passes/text-expected.txt\n'
'\n'
'// For test-win-win7\n'
'passes/text-expected.png\n'
'passes/text-expected.txt\n'))
def test_csv(self):
command = PrintBaselines()
self.capture_output()
options = optparse.Values({
'all': False,
'platform': '*win7',
'csv': True,
'include_virtual_tests': False
})
command.execute(options, ['passes/text.html'], self.tool)
stdout, _, _ = self.restore_output()
self.assertMultiLineEqual(stdout, (
'test-win-win7,passes/text.html,None,png,passes/text-expected.png,None\n'
'test-win-win7,passes/text.html,None,txt,passes/text-expected.txt,None\n'
))
def test_upload_json(self):
runner, port = self.create_runner()
port.host.filesystem.files['/mock-checkout/some.json'] = 'some content'
class MockFileUploader:
called = []
upload_single_text_file_throws = False
upload_single_text_file_return_value = None
@classmethod
def reset(cls):
cls.called = []
cls.upload_single_text_file_throws = False
cls.upload_single_text_file_return_value = None
def __init__(mock, url, timeout):
self.assertEqual(url, 'https://some.host/some/path')
self.assertTrue(isinstance(timeout, int) and timeout)
mock.called.append('FileUploader')
def upload_single_text_file(mock, filesystem, content_type,
filename):
self.assertEqual(filesystem, port.host.filesystem)
self.assertEqual(content_type, 'application/json')
self.assertEqual(filename, 'some.json')
mock.called.append('upload_single_text_file')
if mock.upload_single_text_file_throws:
raise Exception
return mock.upload_single_text_file_return_value
MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO(
'OK')
self.assertTrue(
runner._upload_json('some.host', 'some.json', '/some/path',
MockFileUploader))
self.assertEqual(MockFileUploader.called,
['FileUploader', 'upload_single_text_file'])
MockFileUploader.reset()
MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO(
'Some error')
output = OutputCapture()
output.capture_output()
self.assertFalse(
runner._upload_json('some.host', 'some.json', '/some/path',
MockFileUploader))
_, _, logs = output.restore_output()
self.assertEqual(
logs,
'Uploaded JSON to https://some.host/some/path but got a bad response:\nSome error\n'
)
# Throwing an exception upload_single_text_file shouldn't blow up _upload_json
MockFileUploader.reset()
MockFileUploader.upload_single_text_file_throws = True
self.assertFalse(
runner._upload_json('some.host', 'some.json', '/some/path',
MockFileUploader))
self.assertEqual(MockFileUploader.called,
['FileUploader', 'upload_single_text_file'])
MockFileUploader.reset()
MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO(
'{"status": "OK"}')
self.assertTrue(
runner._upload_json('some.host', 'some.json', '/some/path',
MockFileUploader))
self.assertEqual(MockFileUploader.called,
['FileUploader', 'upload_single_text_file'])
MockFileUploader.reset()
MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO(
'{"status": "SomethingHasFailed", "failureStored": false}')
output = OutputCapture()
output.capture_output()
self.assertFalse(
runner._upload_json('some.host', 'some.json', '/some/path',
MockFileUploader))
_, _, logs = output.restore_output()
serialized_json = json.dumps(
{
'status': 'SomethingHasFailed',
'failureStored': False
}, indent=4)
self.assertEqual(
logs,
'Uploaded JSON to https://some.host/some/path but got an error:\n%s\n'
% serialized_json)
def test_convert_prefixed_properties(self):
"""Tests convert_prefixed_properties() file that has 20 properties requiring the -webkit- prefix.
The properties are:
10 in one style block + 5 in another style
block + 5 inline styles, including one with multiple prefixed properties.
2 when prefixed properties appear in comments without ending ';'.
The properties in the test content are in all sorts of wack formatting.
"""
test_html = """<html>
<style type="text/css"><![CDATA[
.block1 {
width: 300px;
height: 300px
}
.block2 {
@test0@: propvalue;
}
.block3{@test1@: propvalue;}
.block4 { @test2@:propvalue; }
.block5{ @test3@ :propvalue; }
#block6 { @test4@ : propvalue; }
#block7
{
@test5@: propvalue;
}
#block8 { @test6@: propvalue; }
#block9:pseudo
{
@test7@: propvalue;
@test8@: propvalue propvalue propvalue;
}
]]></style>
</head>
<body>
<div id="elem1" style="@test9@: propvalue;"></div>
<div id="elem2" style="propname: propvalue; @test10@ : propvalue; propname:propvalue;"></div>
<div id="elem2" style="@test11@: propvalue; @test12@ : propvalue; @test13@ :propvalue;"></div>
<div id="elem3" style="@test14@:propvalue"></div>
</body>
<style type="text/css"><![CDATA[
.block10{ @test15@: propvalue; }
.block11{ @test16@: propvalue; }
.block12{ @test17@: propvalue; }
#block13:pseudo
{
@test18@: propvalue;
@test19@: propvalue;
}
#missing-semicolon-in-comments {
/* @test20@: propvalue */
@test21@: propvalue;
}
]]></style>
</html>
"""
converter = _W3CTestConverter(DUMMY_PATH, DUMMY_FILENAME, None)
test_content = self.generate_test_content(converter.prefixed_properties, 22, test_html)
oc = OutputCapture()
oc.capture_output()
try:
converter.feed(test_content[1])
converter.close()
converted = converter.output()
finally:
oc.restore_output()
self.verify_conversion_happened(converted)
self.verify_prefixed_properties(converted, test_content[0])