def test_results_html(self):
mock_port = Mock()
mock_port.relative_test_filename = lambda name: name
mock_port.filename_to_uri = lambda name: name
runner = run_webkit_tests.TestRunner(port=mock_port,
options=Mock(),
printer=Mock())
expected_html = u"""<html>
<head>
<title>Layout Test Results (time)</title>
</head>
<body>
<h2>Title (time)</h2>
<p><a href='test_path'>test_path</a><br />
</p>
</body></html>
"""
html = runner._results_html(["test_path"], {},
"Title",
override_time="time")
self.assertEqual(html, expected_html)
def _test_message(self, ews, results, message):
ews.bind_to_tool(MockTool())
ews.host = MockHost()
ews._options = MockOptions(port=None, confirm=False)
OutputCapture().assert_outputs(
self,
ews.begin_work_queue,
expected_logs=self._default_begin_work_queue_logs(ews.name))
task = Mock()
task.results_from_patch_test_run = results
patch = ews._tool.bugs.fetch_attachment(10000)
self.assertMultiLineEqual(ews._failing_tests_message(task, patch),
message)
def _test_message(self, ews, results, message):
ews.bind_to_tool(MockTool())
ews.host = MockHost()
ews._options = MockOptions(port=None, confirm=False)
with OutputCapture(level=logging.INFO) as captured:
ews.begin_work_queue()
self.assertEqual(captured.root.log.getvalue(), self._default_begin_work_queue_logs(ews.name))
task = Mock()
task.results_from_patch_test_run = results
patch = ews._tool.bugs.fetch_attachment(10000)
self.assertMultiLineEqual(ews._failing_tests_message(task, patch), message)
def test_land(self):
expected_logs = """Building WebKit
Running Python unit tests
Running Perl unit tests
Running JavaScriptCore tests
Running bindings generation tests
Running WebKit unit tests
Running run-webkit-tests
Committed r49824: <http://trac.webkit.org/changeset/49824>
Updating bug 50000
"""
mock_tool = MockTool()
mock_tool.scm().create_patch = Mock(return_value="Patch1\nMockPatch\n")
mock_tool.checkout().modified_changelogs = Mock(return_value=[])
self.assert_execute_outputs(Land(), [50000],
options=self._default_options(),
expected_logs=expected_logs,
tool=mock_tool)
# Make sure we're not calling expensive calls too often.
self.assertEqual(mock_tool.scm().create_patch.call_count, 0)
self.assertEqual(mock_tool.checkout().modified_changelogs.call_count,
1)
def test_can_build_and_test(self):
queue = CommitQueue()
tool = MockTool()
tool.executive = Mock()
queue.bind_to_tool(tool)
self.assertTrue(queue._can_build_and_test())
expected_run_args = [
"echo", "--status-host=example.com", "build-and-test",
"--force-clean", "--build", "--test", "--non-interactive",
"--no-update", "--build-style=both", "--quiet"
]
tool.executive.run_and_throw_if_fail.assert_called_with(
expected_run_args)
def test_failing_tests_message(self):
# Needed to define port_name, used in AbstractEarlyWarningSystem.__init__
class TestEWS(AbstractEarlyWarningSystem):
port_name = "win" # Needs to be a port which port/factory understands.
ews = TestEWS()
ews.bind_to_tool(MockTool())
ews._options = MockOptions(port=None, confirm=False)
OutputCapture().assert_outputs(self, ews.begin_work_queue, expected_stderr=self._default_begin_work_queue_stderr(ews.name))
ews._expected_failures.unexpected_failures_observed = lambda results: set(["foo.html", "bar.html"])
task = Mock()
patch = ews._tool.bugs.fetch_attachment(10000)
self.assertEqual(ews._failing_tests_message(task, patch), "New failing tests:\nbar.html\nfoo.html")
def test_next_patch(self):
queue = AbstractPatchQueue()
tool = MockTool()
queue.bind_to_tool(tool)
queue._options = Mock()
queue._options.port = None
self.assertEquals(queue._next_patch(), None)
tool.status_server = MockStatusServer(work_items=[2, 197])
expected_stdout = "MOCK: fetch_attachment: 2 is not a known attachment id\n" # A mock-only message to prevent us from making mistakes.
expected_stderr = "MOCK: release_work_item: None 2\n"
patch_id = OutputCapture().assert_outputs(self, queue._next_patch, [], expected_stdout=expected_stdout, expected_stderr=expected_stderr)
self.assertEquals(patch_id, None) # 2 is an invalid patch id
self.assertEquals(queue._next_patch().id(), 197)
def test_interrupt_if_at_failure_limits(self):
port = Mock() # FIXME: This should be a tighter mock.
port.TEST_PATH_SEPARATOR = '/'
port._filesystem = MockFileSystem()
manager = Manager(port=port, options=MockOptions(), printer=Mock())
manager._options = MockOptions(exit_after_n_failures=None,
exit_after_n_crashes_or_timeouts=None)
manager._test_files = ['foo/bar.html', 'baz.html']
result_summary = ResultSummary(expectations=Mock(),
test_files=manager._test_files)
result_summary.unexpected_failures = 100
result_summary.unexpected_crashes = 50
result_summary.unexpected_timeouts = 50
# No exception when the exit_after* options are None.
manager._interrupt_if_at_failure_limits(result_summary)
# No exception when we haven't hit the limit yet.
manager._options.exit_after_n_failures = 101
manager._options.exit_after_n_crashes_or_timeouts = 101
manager._interrupt_if_at_failure_limits(result_summary)
# Interrupt if we've exceeded either limit:
manager._options.exit_after_n_crashes_or_timeouts = 10
self.assertRaises(TestRunInterruptedException,
manager._interrupt_if_at_failure_limits,
result_summary)
self.assertEquals(result_summary.results['foo/bar.html'].type,
test_expectations.SKIP)
self.assertEquals(result_summary.results['baz.html'].type,
test_expectations.SKIP)
manager._options.exit_after_n_crashes_or_timeouts = None
manager._options.exit_after_n_failures = 10
exception = self.assertRaises(TestRunInterruptedException,
manager._interrupt_if_at_failure_limits,
result_summary)
def test_post_blame_comment_on_bug(self):
sheriff = Sheriff(MockTool(), MockSheriffBot())
builders = [
Builder("Foo", None),
Builder("Bar", None),
]
commit_info = Mock()
commit_info.bug_id = lambda: None
commit_info.revision = lambda: 4321
commit_info.committer = lambda: None
commit_info.committer_email = lambda: "*****@*****.**"
commit_info.reviewer = lambda: None
commit_info.author = lambda: None
sheriff.post_automatic_rollout_patch(commit_info, builders)
def test_upload_results_archive_for_patch(self):
queue = CommitQueue()
queue.bind_to_tool(MockTool())
patch = queue._tool.bugs.fetch_attachment(128)
expected_stderr = """MOCK add_attachment_to_bug: bug_id=42, description=Archive of layout-test-results from bot filename=layout-test-results.zip
-- Begin comment --
The attached test failures were seen while running run-webkit-tests on the commit-queue.
Port: MockPort Platform: MockPlatform 1.0
-- End comment --
"""
OutputCapture().assert_outputs(self,
queue._upload_results_archive_for_patch,
[patch, Mock()],
expected_stderr=expected_stderr)
def test_manual_reject_during_processing(self):
queue = SecondThoughtsCommitQueue()
queue.bind_to_tool(MockTool())
queue._options = Mock()
queue._options.port = None
expected_stderr = """MOCK: update_status: commit-queue Cleaned working directory
MOCK: update_status: commit-queue Updated working directory
MOCK: update_status: commit-queue Applied patch
MOCK: update_status: commit-queue Built patch
MOCK: update_status: commit-queue Passed tests
MOCK: update_status: commit-queue Retry
MOCK: release_work_item: commit-queue 197
"""
OutputCapture().assert_outputs(self, queue.process_work_item, [QueuesTest.mock_work_item], expected_stderr=expected_stderr)
def _test_json_generation(self, passed_tests_list, failed_tests_list):
tests_set = set(passed_tests_list) | set(failed_tests_list)
DISABLED_tests = set(
[t for t in tests_set if t.startswith('DISABLED_')])
FLAKY_tests = set([t for t in tests_set if t.startswith('FLAKY_')])
FAILS_tests = set([t for t in tests_set if t.startswith('FAILS_')])
PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests)
failed_tests = set(failed_tests_list) - DISABLED_tests
failed_count_map = dict([(t, 1) for t in failed_tests])
test_timings = {}
i = 0
for test in tests_set:
test_timings[test] = float(self._num_runs * 100 + i)
i += 1
test_results_map = dict()
for test in tests_set:
test_results_map[test] = json_results_generator.TestResult(
test,
failed=(test in failed_tests),
elapsed_time=test_timings[test])
host = MockHost()
port = Mock()
port._filesystem = host.filesystem
generator = json_results_generator.JSONResultsGeneratorBase(
port,
self.builder_name,
self.build_name,
self.build_number,
'',
None, # don't fetch past json results archive
test_results_map)
failed_count_map = dict([(t, 1) for t in failed_tests])
# Test incremental json results
incremental_json = generator.get_json()
self._verify_json_results(tests_set, test_timings, failed_count_map,
len(PASS_tests), len(DISABLED_tests),
len(FLAKY_tests),
len(DISABLED_tests | failed_tests),
incremental_json, 1)
# We don't verify the results here, but at least we make sure the code runs without errors.
generator.generate_json_output()
generator.generate_times_ms_file()
def test_auto_retry(self):
queue = CommitQueue()
options = Mock()
options.parent_command = "commit-queue"
tool = AlwaysCommitQueueTool()
sequence = NeedsUpdateSequence(None)
expected_stderr = "Commit failed because the checkout is out of date. Please update and try again.\nMOCK: update_status: commit-queue Tests passed, but commit failed (checkout out of date). Updating, then landing without building or re-running tests.\n"
state = {'patch': None}
OutputCapture().assert_outputs(self, sequence.run_and_handle_errors, [tool, options, state], expected_exception=TryAgain, expected_stderr=expected_stderr)
self.assertEquals(options.update, True)
self.assertEquals(options.build, False)
self.assertEquals(options.test, False)
def test_latest_entry_for_changelog_at_revision(self):
scm = Mock()
def mock_contents_at_revision(changelog_path, revision):
self.assertEqual(changelog_path, "foo")
self.assertEqual(revision, "bar")
# contents_at_revision is expected to return a byte array (str)
# so we encode our unicode ChangeLog down to a utf-8 stream.
return _changelog1.encode("utf-8")
scm.contents_at_revision = mock_contents_at_revision
checkout = Checkout(scm)
entry = checkout._latest_entry_for_changelog_at_revision("foo", "bar")
self.assertEqual(entry.contents(), _changelog1entry1)
def test_manual_reject_during_processing(self):
queue = SecondThoughtsCommitQueue(MockTool())
queue.begin_work_queue()
queue._tool.filesystem.write_text_file('/tmp/layout-test-results/full_results.json', '') # Otherwise the commit-queue will hit a KeyError trying to read the results from the MockFileSystem.
queue._tool.filesystem.write_text_file('/tmp/layout-test-results/webkit_unit_tests_output.xml', '')
queue._options = Mock()
queue._options.port = None
expected_logs = """Running: webkit-patch --status-host=example.com clean --port=mac
MOCK: update_status: commit-queue Cleaned working directory
MOCK: update_status: commit-queue Error: commit-queue did not process patch. Reason: Patch is obsolete.
MOCK: release_work_item: commit-queue 10000
"""
self.maxDiff = None
OutputCapture().assert_outputs(self, queue.process_work_item, [QueuesTest.mock_work_item], expected_logs=expected_logs)
def test_upload_results_archive_for_patch(self):
queue = PatchProcessingQueue()
queue.name = "mock-queue"
tool = MockTool()
queue.bind_to_tool(tool)
queue._options = Mock()
queue._options.port = None
patch = queue._tool.bugs.fetch_attachment(10001)
expected_logs = """MOCK add_attachment_to_bug: bug_id=50000, description=Archive of layout-test-results from bot for mac-snowleopard filename=layout-test-results.zip mimetype=None
-- Begin comment --
The attached test failures were seen while running run-webkit-tests on the mock-queue.
Port: mac-snowleopard Platform: MockPlatform 1.0
-- End comment --
"""
OutputCapture().assert_outputs(self, queue._upload_results_archive_for_patch, [patch, Mock()], expected_logs=expected_logs)
def test_next_patch(self):
queue = AbstractPatchQueue()
tool = MockTool()
queue.bind_to_tool(tool)
queue._options = Mock()
queue._options.port = None
self.assertIsNone(queue._next_patch())
tool.status_server = MockStatusServer(work_items=[2, 10000, 10001])
expected_stdout = "MOCK: fetch_attachment: 2 is not a known attachment id\n" # A mock-only message to prevent us from making mistakes.
expected_logs = "MOCK: release_work_item: None 2\n"
patch = OutputCapture().assert_outputs(self, queue._next_patch, expected_stdout=expected_stdout, expected_logs=expected_logs)
# The patch.id() == 2 is ignored because it doesn't exist.
self.assertEqual(patch.id(), 10000)
self.assertEqual(queue._next_patch().id(), 10001)
self.assertEqual(queue._next_patch(), None) # When the queue is empty
def get_shards(self,
num_workers,
fully_parallel,
test_list=None,
max_locked_shards=None):
test_list = test_list or self.test_list
host = MockHost()
port = host.port_factory.get(port_name='test')
port._filesystem = MockFileSystem()
options = MockOptions(max_locked_shards=max_locked_shards)
self.manager = ManagerWrapper(port=port,
options=options,
printer=Mock())
return self.manager._shard_tests(test_list, num_workers,
fully_parallel)
def test_manual_reject_during_processing(self):
queue = SecondThoughtsCommitQueue(MockTool())
queue.begin_work_queue()
queue._tool.filesystem.write_text_file('/mock-results/results.html', '') # Otherwise the commit-queue will hit a KeyError trying to read the results from the MockFileSystem.
queue._options = Mock()
queue._options.port = None
expected_stderr = """MOCK: update_status: commit-queue Cleaned working directory
MOCK: update_status: commit-queue Updated working directory
MOCK: update_status: commit-queue Applied patch
MOCK: update_status: commit-queue Built patch
MOCK: update_status: commit-queue Passed tests
MOCK: update_status: commit-queue Retry
MOCK: release_work_item: commit-queue 197
"""
OutputCapture().assert_outputs(self, queue.process_work_item, [QueuesTest.mock_work_item], expected_stderr=expected_stderr)
def test_shard_tests(self):
# Test that _shard_tests in test_runner.TestRunner really
# put the http tests first in the queue.
port = Mock()
port._filesystem = filesystem_mock.MockFileSystem()
runner = TestRunnerWrapper(port=port, options=Mock(), printer=Mock())
test_list = [
"LayoutTests/websocket/tests/unicode.htm",
"LayoutTests/animations/keyframes.html",
"LayoutTests/http/tests/security/view-source-no-refresh.html",
"LayoutTests/websocket/tests/websocket-protocol-ignored.html",
"LayoutTests/fast/css/display-none-inline-style-change-crash.html",
"LayoutTests/http/tests/xmlhttprequest/supported-xml-content-types.html",
"LayoutTests/dom/html/level2/html/HTMLAnchorElement03.html",
"LayoutTests/ietestcenter/Javascript/11.1.5_4-4-c-1.html",
"LayoutTests/dom/html/level2/html/HTMLAnchorElement06.html",
]
expected_tests_to_http_lock = set([
'LayoutTests/websocket/tests/unicode.htm',
'LayoutTests/http/tests/security/view-source-no-refresh.html',
'LayoutTests/websocket/tests/websocket-protocol-ignored.html',
'LayoutTests/http/tests/xmlhttprequest/supported-xml-content-types.html',
])
# FIXME: Ideally the HTTP tests don't have to all be in one shard.
single_thread_results = runner._shard_tests(test_list, False)
multi_thread_results = runner._shard_tests(test_list, True)
self.assertEqual("tests_to_http_lock", single_thread_results[0][0])
self.assertEqual(expected_tests_to_http_lock,
set(single_thread_results[0][1]))
self.assertEqual("tests_to_http_lock", multi_thread_results[0][0])
self.assertEqual(expected_tests_to_http_lock,
set(multi_thread_results[0][1]))
def test_commit_info(self):
command = AbstractRolloutPrepCommand()
tool = MockTool()
command.bind_to_tool(tool)
output = OutputCapture()
expected_logs = "Preparing rollout for bug 50000.\n"
commit_info = output.assert_outputs(self, command._commit_info, [1234], expected_logs=expected_logs)
self.assertTrue(commit_info)
mock_commit_info = Mock()
mock_commit_info.bug_id = lambda: None
tool._checkout.commit_info_for_revision = lambda revision: mock_commit_info
expected_logs = "Unable to parse bug number from diff.\n"
commit_info = output.assert_outputs(self, command._commit_info, [1234], expected_logs=expected_logs)
self.assertEqual(commit_info, mock_commit_info)
def test_changelog_contains_oops(self):
tool = MockTool()
tool._checkout.is_path_to_changelog = lambda path: True
step = ValidateChangeLogs(tool, MockOptions(git_commit=None, non_interactive=True, check_oops=True))
diff_file = Mock()
diff_file.filename = "mock/ChangeLog"
diff_file.lines = [(1, 1, "foo"), (2, 2, "bar OOPS! bar"), (3, 3, "foo")]
with OutputCapture(level=logging.INFO) as captured:
self.assertTrue(step._changelog_contains_oops(diff_file))
self.assertEqual(captured.root.log.getvalue(), '')
diff_file.lines = [(1, 1, "foo"), (2, 2, "bar OOPS bar"), (3, 3, "foo")]
with OutputCapture(level=logging.INFO) as captured:
self.assertFalse(step._changelog_contains_oops(diff_file))
self.assertEqual(captured.root.log.getvalue(), '')
def test_crashed_process_name(self):
self.driver._proc = Mock()
# Simulate a crash by having stdout close unexpectedly.
def mock_readline():
raise IOError
self.driver._proc.stdout.readline = mock_readline
self.driver._port.test_to_uri = lambda test: 'mocktesturi'
driver_output = self.driver.run_test(
DriverInput(test_name='some/test.html',
timeout=1,
image_hash=None,
is_reftest=False))
self.assertEqual(self.driver._port.driver_name(),
driver_output.crashed_process_name)
def test_crash_log(self):
self.driver._proc = Mock()
# Simulate a crash by having stdout close unexpectedly.
def mock_readline():
raise IOError
self.driver._proc.stdout.readline = mock_readline
self.driver._proc.pid = 1234
self.driver.test_to_uri = lambda test: 'mocktesturi'
self.driver._port.driver_name = lambda: 'mockdriver'
self.driver._port._get_crash_log = lambda name, pid, out, err, newer_than: 'mockcrashlog'
driver_output = self.driver.run_test(DriverInput(test_name='some/test.html', timeout=1, image_hash=None, should_run_pixel_test=False))
self.assertTrue(driver_output.crash)
self.assertEqual(driver_output.crashed_process_name, 'mockdriver')
self.assertEqual(driver_output.crashed_pid, 1234)
self.assertEqual(driver_output.crash_log, 'mockcrashlog')
def run():
sheriff = Sheriff(MockTool(), MockSheriffBot())
builders = [
Builder("Foo", None),
Builder("Bar", None),
]
commit_info = Mock()
commit_info.bug_id = lambda: None
commit_info.revision = lambda: 4321
# Should do nothing with no bug_id
sheriff.post_blame_comment_on_bug(commit_info, builders, [])
sheriff.post_blame_comment_on_bug(commit_info, builders, ["mock-test-1", "mock-test-2"])
# Should try to post a comment to the bug, but MockTool.bugs does nothing.
commit_info.bug_id = lambda: 1234
sheriff.post_blame_comment_on_bug(commit_info, builders, [])
sheriff.post_blame_comment_on_bug(commit_info, builders, ["mock-test-1"])
sheriff.post_blame_comment_on_bug(commit_info, builders, ["mock-test-1", "mock-test-2"])
def test_commit_info(self):
command = AbstractRevertPrepCommand()
tool = MockTool()
command.bind_to_tool(tool)
with OutputCapture(level=logging.INFO) as captured:
commit_info = command._commit_info(1234)
self.assertEqual(captured.root.log.getvalue(), 'Preparing revert for bug 50000.\n')
self.assertTrue(commit_info)
mock_commit_info = Mock()
mock_commit_info.bug_id = lambda: None
tool._checkout.commit_info_for_revision = lambda revision: mock_commit_info
with OutputCapture(level=logging.INFO) as captured:
commit_info = command._commit_info(1234)
self.assertEqual(captured.root.log.getvalue(), 'Unable to parse bug number from diff.\n')
self.assertEqual(commit_info, mock_commit_info)
def test_failing_tests_message(self):
# Needed to define port_name, used in AbstractEarlyWarningSystem.__init__
class TestEWS(AbstractEarlyWarningSystem):
port_name = "win" # Needs to be a port which port/factory understands.
_build_style = None
ews = TestEWS()
ews.bind_to_tool(MockTool())
ews.host = MockHost()
ews._options = MockOptions(port=None, confirm=False)
OutputCapture().assert_outputs(self, ews.begin_work_queue, expected_logs=self._default_begin_work_queue_logs(ews.name))
task = Mock()
task.results_from_patch_test_run = lambda a: LayoutTestResults([test_results.TestResult("foo.html", failures=[test_failures.FailureTextMismatch()]),
test_results.TestResult("bar.html", failures=[test_failures.FailureTextMismatch()])],
did_exceed_test_failure_limit=False)
patch = ews._tool.bugs.fetch_attachment(10000)
self.assertMultiLineEqual(ews._failing_tests_message(task, patch), "New failing tests:\nfoo.html\nbar.html")
def _assert_security_call(self, username=None):
executive_mock = Mock()
credentials = Credentials("example.com", executive=executive_mock)
expected_stderr = "Reading Keychain for example.com account and password. Click \"Allow\" to continue...\n"
OutputCapture().assert_outputs(self,
credentials._run_security_tool,
[username],
expected_stderr=expected_stderr)
security_args = [
"/usr/bin/security", "find-internet-password", "-g", "-s",
"example.com"
]
if username:
security_args += ["-a", username]
executive_mock.run_command.assert_called_with(security_args)
def test_patches_to_commit_queue(self):
expected_stdout = "http://example.com/10003&action=edit\n"
expected_logs = "10000 already has cq=+\n10001 already has cq=+\n10004 committer = \"Eric Seidel\" <*****@*****.**>\n"
options = Mock()
options.bugs = False
self.assert_execute_outputs(PatchesToCommitQueue(),
None,
expected_stdout,
expected_logs=expected_logs,
options=options)
expected_stdout = "http://example.com/50003\n"
options.bugs = True
self.assert_execute_outputs(PatchesToCommitQueue(),
None,
expected_stdout,
expected_logs=expected_logs,
options=options)
def test_upload(self):
options = Mock()
options.description = "MOCK description"
options.request_commit = False
options.review = True
# Rietveld upload code requires a real SCM checkout.
options.fancy_review = False
options.cc = None
expected_stderr = """Running check-webkit-style
MOCK: user.open_url: file://...
Obsoleting 2 old patches on bug 42
MOCK add_patch_to_bug: bug_id=42, description=MOCK description, mark_for_review=True, mark_for_commit_queue=False, mark_for_landing=False
-- Begin comment --
None
-- End comment --
MOCK: user.open_url: http://example.com/42
"""
self.assert_execute_outputs(Upload(), [42], options=options, expected_stderr=expected_stderr)