def test_run_test_pause_before_testing(self):
class TestDriverWithStartCount(MainTest.TestDriver):
start_count = 0
def start(self):
TestDriverWithStartCount.start_count += 1
runner, port = self.create_runner(
args=["--pause-before-testing"],
driver_class=TestDriverWithStartCount)
tests = self._tests_for_runner(runner, ['inspector/pass.html'])
output = OutputCapture()
output.capture_output()
try:
unexpected_result_count = runner._run_tests_set(tests, port)
self.assertEqual(TestDriverWithStartCount.start_count, 1)
finally:
stdout, stderr, log = output.restore_output()
self.assertEqual(stderr, "Ready to run test?\n")
self.assertEqual(
self.normalizeFinishedTime(log),
"Running inspector/pass.html (1 of 1)\nRESULT group_name: test_name= 42 ms\nFinished: 0.1 s\n\n"
)
def test_convert_test_harness_paths(self):
""" Tests convert_testharness_paths() with a test that uses all three testharness files """
test_html = """<head>
<link href="/resources/testharness.css" rel="stylesheet" type="text/css">
<script src="/resources/testharness.js"></script>
<script src="/resources/testharnessreport.js"></script>
</head>
"""
fake_dir_path = self.fake_dir_path('testharnesspaths')
converter = _W3CTestConverter(fake_dir_path, DUMMY_FILENAME, None)
oc = OutputCapture()
oc.capture_output()
try:
converter.feed(test_html)
converter.close()
converted = converter.output()
finally:
oc.restore_output()
self.verify_conversion_happened(converted)
self.verify_test_harness_paths(converter, converted[2], fake_dir_path,
2, 1)
def test_convert_for_webkit_properties_only(self):
""" Tests convert_for_webkit() using a test that has 2 prefixed properties: 1 in a style block + 1 inline style """
test_html = """<html>
<head>
<link href="/resources/testharness.css" rel="stylesheet" type="text/css">
<script src="/resources/testharness.js"></script>
<style type="text/css">
#block1 { @test0@: propvalue; }
</style>
</head>
<body>
<div id="elem1" style="@test1@: propvalue;"></div>
</body>
</html>
"""
fake_dir_path = self.fake_dir_path('harnessandprops')
converter = _W3CTestConverter(fake_dir_path, DUMMY_FILENAME)
test_content = self.generate_test_content(
converter.prefixed_properties, 1, test_html)
oc = OutputCapture()
oc.capture_output()
try:
converter.feed(test_content[1])
converter.close()
converted = converter.output()
finally:
oc.restore_output()
self.verify_conversion_happened(converted)
self.verify_test_harness_paths(converter, converted[1], fake_dir_path,
1, 1)
self.verify_prefixed_properties(converted, test_content[0])
def test_webkit_run_unit_tests(self):
tool = MockTool(log_executive=True)
tool._deprecated_port.run_python_unittests_command = lambda: None
tool._deprecated_port.run_perl_unittests_command = lambda: None
step = RunTests(tool, MockOptions(test=True, non_interactive=True, quiet=False))
if sys.platform != "cygwin":
expected_logs = """Running bindings generation tests
MOCK run_and_throw_if_fail: ['mock-run-bindings-tests'], cwd=/mock-checkout
Running WebKit unit tests
MOCK run_and_throw_if_fail: ['mock-run-webkit-unit-tests'], cwd=/mock-checkout
Running run-webkit-tests
MOCK run_and_throw_if_fail: ['mock-run-webkit-tests', '--no-new-test-results', '--no-show-results', '--exit-after-n-failures=30', '--quiet', '--skip-failing-tests'], cwd=/mock-checkout
"""
else:
expected_logs = """Running bindings generation tests
MOCK run_and_throw_if_fail: ['mock-run-bindings-tests'], cwd=/mock-checkout
Running WebKit unit tests
MOCK run_and_throw_if_fail: ['mock-run-webkit-unit-tests'], cwd=/mock-checkout
Running run-webkit-tests
MOCK run_and_throw_if_fail: ['mock-run-webkit-tests', '--no-new-test-results', '--no-show-results', '--exit-after-n-failures=30', '--no-build'], cwd=/mock-checkout
"""
OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
def test_convert_for_webkit_harness_and_properties(self):
""" Tests convert_for_webkit() using a basic JS test that uses testharness.js and testharness.css and has 4 prefixed properties: 3 in a style block + 1 inline style """
test_html = """<html>
<head>
<link href="/resources/testharness.css" rel="stylesheet" type="text/css">
<script src="/resources/testharness.js"></script>
<style type="text/css">
#block1 { @test0@: propvalue; }
#block2 { @test1@: propvalue; }
#block3 { @test2@: propvalue; }
</style>
</head>
<body>
<div id="elem1" style="@test3@: propvalue;"></div>
</body>
</html>
"""
converter = W3CTestConverter()
fake_dir_path = self.fake_dir_path(converter, 'harnessandprops')
oc = OutputCapture()
oc.capture_output()
try:
test_content = self.generate_test_content(
converter.prefixed_properties, 2, test_html)
converted = converter.convert_html(fake_dir_path, test_content[1])
finally:
oc.restore_output()
self.verify_conversion_happened(converted)
self.verify_test_harness_paths(converter, converted[1], fake_dir_path,
1, 1)
self.verify_prefixed_properties(converted, test_content[0])
def test_analyze_test_reftest_multiple_matches(self):
test_html = """<head>
<link rel="match" href="green-box-ref.xht" />
<link rel="match" href="blue-box-ref.xht" />
<link rel="match" href="orange-box-ref.xht" />
</head>
"""
oc = OutputCapture()
oc.capture_output()
try:
test_path = '/some/madeup/path/'
parser = TestParser(test_path + 'somefile.html', MockHost())
test_info = parser.analyze_test(test_contents=test_html)
finally:
_, _, logs = oc.restore_output()
self.assertNotEqual(test_info, None, 'did not find a test')
self.assertTrue('test' in test_info.keys(), 'did not find a test file')
self.assertTrue('reference' in test_info.keys(), 'did not find a reference file')
self.assertTrue(test_info['reference'].startswith(test_path), 'reference path is not correct')
self.assertFalse('refsupport' in test_info.keys(), 'there should be no refsupport files for this test')
self.assertFalse('jstest' in test_info.keys(), 'test should not have been analyzed as a jstest')
self.assertEqual(logs, 'Multiple references are not supported. Importing the first ref defined in somefile.html\n')
def _test_run_with_json_output(self, runner, filesystem, upload_succeeds=False, results_shown=True, expected_exit_code=0, repeat=1, compare_logs=True):
filesystem.write_text_file(runner._base_path + '/inspector/pass.html', 'some content')
filesystem.write_text_file(runner._base_path + '/Bindings/event-target-wrapper.html', 'some content')
uploaded = [False]
def mock_upload_json(hostname, json_path, host_path=None):
# FIXME: Get rid of the hard-coded perf.webkit.org once we've completed the transition.
self.assertIn(hostname, ['some.host'])
self.assertIn(json_path, ['/mock-checkout/output.json'])
self.assertIn(host_path, [None, '/api/report'])
uploaded[0] = upload_succeeds
return upload_succeeds
runner._upload_json = mock_upload_json
runner._timestamp = 123456789
runner._utc_timestamp = datetime.datetime(2013, 2, 8, 15, 19, 37, 460000)
output_capture = OutputCapture()
output_capture.capture_output()
try:
self.assertEqual(runner.run(), expected_exit_code)
finally:
stdout, stderr, logs = output_capture.restore_output()
if not expected_exit_code and compare_logs:
expected_logs = ''
for i in xrange(repeat):
runs = ' (Run %d of %d)' % (i + 1, repeat) if repeat > 1 else ''
expected_logs += 'Running 2 tests%s\n' % runs + EventTargetWrapperTestData.output + InspectorPassTestData.output
if results_shown:
expected_logs += 'MOCK: user.open_url: file://...\n'
self.assertEqual(self._normalize_output(logs), expected_logs)
self.assertEqual(uploaded[0], upload_succeeds)
return logs
def test_run_memory_test(self):
runner, port = self.create_runner_and_setup_results_template()
runner._timestamp = 123456789
port.host.filesystem.write_text_file(
runner._base_path + '/Parser/memory-test.html', 'some content')
output = OutputCapture()
output.capture_output()
try:
unexpected_result_count = runner.run()
finally:
stdout, stderr, log = output.restore_output()
self.assertEqual(unexpected_result_count, 0)
self.assertEqual(
self._normalize_output(log),
MemoryTestData.output + '\nMOCK: user.open_url: file://...\n')
parser_tests = self._load_output_json(
runner)[0]['tests']['Parser']['tests']
self.assertEqual(parser_tests['memory-test']['metrics']['Time'],
MemoryTestData.results)
self.assertEqual(parser_tests['memory-test']['metrics']['JSHeap'],
MemoryTestData.js_heap_results)
self.assertEqual(parser_tests['memory-test']['metrics']['Malloc'],
MemoryTestData.malloc_results)
def test_prepare_calls_run_single(self):
output_capture = OutputCapture()
output_capture.capture_output()
called = [False]
def run_single(driver, url, time_out_ms, record):
self.assertTrue(record)
self.assertEqual(url, 'http://some-test/')
called[0] = True
return False
test, port = self._setup_test()
test.run_single = run_single
try:
self.assertEqual(test.prepare(time_out_ms=100), False)
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
self.assertTrue(called[0])
self.assertEqual(test._archive_path, '/path/some-dir/some-test.wpr')
self.assertEqual(test._url, 'http://some-test/')
self.assertEqual(actual_stdout, '')
self.assertEqual(actual_stderr, '')
self.assertEqual(actual_logs, "Preparing replay for some-test.replay\nFailed to prepare a replay for some-test.replay\n")
def test_rebaseline(self):
self.command._builders_to_pull_from = lambda: [
MockBuilder('MOCK builder')
]
self.command._tests_to_update = lambda builder: [
'mock/path/to/test.html'
]
self._zero_out_test_expectations()
old_exact_matches = builders._exact_matches
oc = OutputCapture()
try:
builders._exact_matches = {
"MOCK builder": {
"port_name": "test-mac-leopard",
"specifiers": set(["mock-specifier"])
},
}
oc.capture_output()
self.command.execute(
MockOptions(optimize=False,
builders=None,
suffixes="txt,png",
verbose=True,
move_overwritten_baselines=False), [], self.tool)
finally:
oc.restore_output()
builders._exact_matches = old_exact_matches
calls = filter(lambda x: x[0] != 'perl', self.tool.executive.calls)
self.assertEqual(calls, [[[
'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png',
'--builder', 'MOCK builder', '--test', 'mock/path/to/test.html',
'--verbose'
]]])
def test_double_flaky_test_failure(self):
commit_queue = FailingTestCommitQueue([
None,
None,
None,
None,
ScriptError("MOCK test failure"),
ScriptError("MOCK test failure again"),
], [
"foo.html",
"bar.html",
"foo.html",
])
# The (subtle) point of this test is that report_flaky_tests does not appear
# in the expected_stderr for this run.
# Note also that there is no attempt to run the tests w/o the patch.
expected_stderr = """run_webkit_patch: ['clean']
command_passed: success_message='Cleaned working directory' patch='197'
run_webkit_patch: ['update']
command_passed: success_message='Updated working directory' patch='197'
run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197]
command_passed: success_message='Applied patch' patch='197'
run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
command_passed: success_message='Built patch' patch='197'
run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='197'
archive_last_layout_test_results: patch='197'
run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='197'
"""
tool = MockTool(log_executive=True)
patch = tool.bugs.fetch_attachment(197)
task = CommitQueueTask(commit_queue, patch)
success = OutputCapture().assert_outputs(
self, task.run, expected_stderr=expected_stderr)
self.assertEqual(success, False)
def test_convert_prefixed_properties(self):
""" Tests convert_prefixed_properties() file that has 20 properties requiring the -webkit- prefix:
10 in one style block + 5 in another style
block + 5 inline styles, including one with multiple prefixed properties.
The properties in the test content are in all sorts of wack formatting.
"""
test_html = """<html>
<style type="text/css"><![CDATA[
.block1 {
width: 300px;
height: 300px
}
.block2 {
@test0@: propvalue;
}
.block3{@test1@: propvalue;}
.block4 { @test2@:propvalue; }
.block5{ @test3@ :propvalue; }
#block6 { @test4@ : propvalue; }
#block7
{
@test5@: propvalue;
}
#block8 { @test6@: propvalue; }
#block9:pseudo
{
@test7@: propvalue;
@test8@: propvalue propvalue propvalue;;
}
]]></style>
</head>
<body>
<div id="elem1" style="@test9@: propvalue;"></div>
<div id="elem2" style="propname: propvalue; @test10@ : propvalue; propname:propvalue;"></div>
<div id="elem2" style="@test11@: propvalue; @test12@ : propvalue; @test13@ :propvalue;"></div>
<div id="elem3" style="@test14@:propvalue"></div>
</body>
<style type="text/css"><![CDATA[
.block10{ @test15@: propvalue; }
.block11{ @test16@: propvalue; }
.block12{ @test17@: propvalue; }
#block13:pseudo
{
@test18@: propvalue;
@test19@: propvalue;
}
]]></style>
</html>
"""
converter = _W3CTestConverter(DUMMY_PATH, DUMMY_FILENAME, None)
test_content = self.generate_test_content(
converter.prefixed_properties, 20, test_html)
oc = OutputCapture()
oc.capture_output()
try:
converter.feed(test_content[1])
converter.close()
converted = converter.output()
finally:
oc.restore_output()
self.verify_conversion_happened(converted)
self.verify_prefixed_properties(converted, test_content[0])
def test_rollout_no_reason(self):
expected_logs = "MOCK: irc.post: mock_nick: Usage: rollout SVN_REVISION [SVN_REVISIONS] REASON\n"
OutputCapture().assert_outputs(self,
run,
args=["rollout 21654"],
expected_logs=expected_logs)
def _assert_step_output_with_bug(self, step, bug_id, expected_stderr, options=None):
state = {'bug_id': bug_id}
OutputCapture().assert_outputs(self, self._run_step, [step, MockTool(), options, state], expected_stderr=expected_stderr)
def test_update_step(self):
tool = MockTool()
options = self._step_options()
options.update = True
expected_stderr = "Updating working directory\n"
OutputCapture().assert_outputs(self, self._run_step, [steps.Update, tool, options], expected_stderr=expected_stderr)
def test_show_results_html_file(self):
port = self.make_port()
port._executive = MockExecutive(should_log=True)
expected_stderr = "MOCK: user.open_url: test.html\n"
OutputCapture().assert_outputs(self, port.show_results_html_file, ["test.html"], expected_stderr=expected_stderr)
def test_guess_reviewer_from_bug(self):
capture = OutputCapture()
step = UpdateChangeLogsWithReviewer(MockTool(), MockOptions())
expected_stderr = "0 reviewed patches on bug 75, cannot infer reviewer.\n"
capture.assert_outputs(self, step._guess_reviewer_from_bug, [75], expected_stderr=expected_stderr)
def test_lgr(self):
expected_stderr = "MOCK: irc.post: mock_nick: http://trac.webkit.org/changeset/9479\n"
OutputCapture().assert_outputs(self,
run,
args=["last-green-revision"],
expected_stderr=expected_stderr)
def test_multi_rollout_no_reason(self):
expected_stderr = "MOCK: irc.post: mock_nick: Usage: SVN_REVISION [SVN_REVISIONS] REASON\n"
OutputCapture().assert_outputs(self,
run,
args=["rollout 21654 21655 r21656"],
expected_stderr=expected_stderr)
def test_add_cc_to_bug(self):
bugzilla = Bugzilla()
bugzilla.browser = MockBrowser()
bugzilla.authenticate = lambda: None
expected_stderr = "Adding ['*****@*****.**'] to the CC list for bug 42\n"
OutputCapture().assert_outputs(self, bugzilla.add_cc_to_bug, [42, ["*****@*****.**"]], expected_stderr=expected_stderr)
def test_rollout_bananas(self):
expected_stderr = "MOCK: irc.post: mock_nick: Usage: SVN_REVISION [SVN_REVISIONS] REASON\n"
OutputCapture().assert_outputs(self,
run,
args=["rollout bananas"],
expected_stderr=expected_stderr)
def _post_to_path(self, path, body=None, expected_stderr=None, expected_stdout=None, server=None):
handler = TestGardeningHTTPRequestHandler(server or MockServer())
handler.path = path
handler.body = body
OutputCapture().assert_outputs(self, handler.do_POST, expected_stderr=expected_stderr, expected_stdout=expected_stdout)
def test_help(self):
expected_stderr = "MOCK: irc.post: mock_nick: Available commands: whois, hi, last-green-revision, rollout, restart, help\n"
OutputCapture().assert_outputs(self,
run,
args=["help"],
expected_stderr=expected_stderr)
def test_show_results_html_file(self):
port = self.make_port()
# Delay setting a should_log executive to avoid logging from MacPort.__init__.
port._executive = MockExecutive(should_log=True)
expected_logs = "MOCK popen: ['Tools/Scripts/run-safari', '--release', '--no-saved-state', '-NSOpen', 'test.html'], cwd=/mock-checkout\n"
OutputCapture().assert_outputs(self, port.show_results_html_file, ["test.html"], expected_logs=expected_logs)
def test_disabled(self):
step = SuggestReviewers(MockTool(),
MockOptions(suggest_reviewers=False))
OutputCapture().assert_outputs(self, step.run, [{}])
def test_convert_style_multiple_url(self):
""" Tests convert_attributes_if_needed() using a reference file that has several relative URL paths in the style """
test_html = """<html>
<head>
<style type="text/css">
.redSquare {
position: absolute;
left:50px;
width: 100px;
height: 100px;
background-image:url(../support/yyy.png);
}
.greenSquare {
position: absolute;
left:50px;
width: 100px;
height: 100px;
background-image:url(../support/yy.png);
}
.yellowSquare {
position: absolute;
left:50px;
width: 100px;
height: 100px;
background-image:url(../../another/directory/x.png);
}
.container {
position: absolute;
}
</style>
</head>
<body>
</body>
</html>
"""
test_reference_support_info = {
'reference_relpath':
'../',
'files': [
'../support/yyy.png', '../support/yy.png',
'../../another/directory/x.png'
]
}
converter = _W3CTestConverter(DUMMY_PATH, DUMMY_FILENAME,
test_reference_support_info)
oc = OutputCapture()
oc.capture_output()
try:
converter.feed(test_html)
converter.close()
converted = converter.output()
finally:
oc.restore_output()
self.verify_conversion_happened(converted)
for path in test_reference_support_info['files']:
expected_path = re.sub(
test_reference_support_info['reference_relpath'], '', path, 1)
expected_url = 'background-image:url(' + expected_path + ');'
self.assertTrue(
expected_url in converted[2],
'relative path ' + path + ' was not converted correcty')
def begin_work_queue(self):
output_capture = OutputCapture()
output_capture.capture_output()
CommitQueue.begin_work_queue(self)
output_capture.restore_output()
def _assert_tool_main_outputs(self, tool, main_args, expected_stdout, expected_stderr = "", expected_exit_code=0):
exit_code = OutputCapture().assert_outputs(self, tool.main, [main_args], expected_stdout=expected_stdout, expected_stderr=expected_stderr)
self.assertEqual(exit_code, expected_exit_code)
def capture_output(self):
self.oc = OutputCapture()
self.oc.capture_output()
def test_required_arguments(self):
two_required_arguments = TrivialCommand(argument_names="ARG1 ARG2 [ARG3]")
expected_missing_args_error = "2 arguments required, 1 argument provided. Provided: 'foo' Required: ARG1 ARG2\nSee 'trivial-tool help trivial' for usage.\n"
exit_code = OutputCapture().assert_outputs(self, two_required_arguments.check_arguments_and_execute, [None, ["foo"], TrivialTool()], expected_stderr=expected_missing_args_error)
self.assertEqual(exit_code, 1)
