def test_filter_latest_jobs_higher_build_last(self):
self.assertEqual(
filter_latest_builds(
[Build('foo', 3),
Build('bar', 5),
Build('foo', 5)]),
[Build('bar', 5), Build('foo', 5)])
def test_latest_try_jobs(self):
git_cl = GitCL(MockHost())
git_cl.fetch_raw_try_job_results = lambda **_: [
{
'builder_name': 'builder-b',
'status': 'COMPLETED',
'result': 'SUCCESS',
'url': 'http://build.chromium.org/p/master/builders/builder-b/builds/100',
},
{
'builder_name': 'builder-b',
'status': 'COMPLETED',
'result': 'SUCCESS',
'url': 'http://build.chromium.org/p/master/builders/builder-b/builds/90',
},
{
'builder_name': 'builder-a',
'status': 'SCHEDULED',
'result': None,
'url': None,
},
{
'builder_name': 'builder-c',
'status': 'COMPLETED',
'result': 'SUCCESS',
'url': 'http://ci.chromium.org/master/builder-c/123',
},
]
self.assertEqual(
git_cl.latest_try_jobs(['builder-a', 'builder-b']),
{
Build('builder-a'): TryJobStatus('SCHEDULED'),
Build('builder-b', 100): TryJobStatus('COMPLETED', 'SUCCESS'),
})
def test_run_commit_queue_for_cl_closed_cl(self):
host = MockHost()
host.filesystem.write_text_file(
'/mock-checkout/third_party/WebKit/LayoutTests/W3CImportExpectations',
'')
importer = TestImporter(host)
importer.git_cl = MockGitCL(
host,
results=CLStatus(
status='closed',
try_job_results={
Build('cq-builder-a', 120):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('cq-builder-b', 200):
TryJobStatus('COMPLETED', 'SUCCESS'),
},
))
success = importer.run_commit_queue_for_cl()
self.assertFalse(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'ERROR: The CL was closed; aborting.\n',
])
self.assertEqual(importer.git_cl.calls, [
['git', 'cl', 'try'],
])
def test_rebaseline_test_passes_on_all_builders(self):
self._setup_mock_build_data()
self.tool.buildbot.set_results(
Build('MOCK Win7'),
LayoutTestResults({
'tests': {
'userscripts': {
'first-test.html': {
'expected': 'NEEDSREBASELINE',
'actual': 'PASS'
}
}
}
}))
self._write(self.mac_expectations_path,
'Bug(x) userscripts/first-test.html [ Failure ]\n')
self._write('userscripts/first-test.html', 'Dummy test contents')
self.command.rebaseline(self.options(), {
'userscripts/first-test.html': {
Build('MOCK Win7'): ['txt', 'png']
}
})
self.assertEqual(self.tool.executive.calls, [])
def test_get_failing_results_dict_some_failing_results(self):
host = self.mock_host()
host.buildbot.set_results(
Build('MOCK Try Mac10.10', 123),
LayoutTestResults({
'tests': {
'external': {
'wpt': {
'x': {
'failing-test.html': {
'expected': 'PASS',
'actual': 'IMAGE',
'is_unexpected': True,
},
},
},
},
},
}))
updater = WPTExpectationsUpdater(host)
results_dict = updater.get_failing_results_dict(
Build('MOCK Try Mac10.10', 123))
self.assertEqual(
results_dict, {
'external/wpt/x/failing-test.html': {
'test-mac-mac10.10':
SimpleTestResult(
actual='IMAGE',
expected='PASS',
bug='crbug.com/626703',
),
},
})
def test_execute_with_flaky_test_that_fails_on_retry(self):
# In this example, the --only-changed-tests flag is not given, but
# svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html
# failed both with and without the patch in the try job, so it is not
# rebaselined.
builds = [
Build('MOCK Try Win', 5000),
Build('MOCK Try Mac', 4000),
Build('MOCK Try Linux', 6000),
]
for build in builds:
self.tool.buildbot.set_retry_sumary_json(
build,
json.dumps({
'failures': [
'fast/dom/prototype-taco.html',
'fast/dom/prototype-inheritance.html',
],
'ignored': [
'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html'
],
}))
return_code = self.command.execute(self.command_options(), [],
self.tool)
self.assertEqual(return_code, 0)
self.assertLog([
'INFO: Rebaselining fast/dom/prototype-inheritance.html\n',
'INFO: Rebaselining fast/dom/prototype-taco.html\n',
])
def test_run_commit_queue_for_cl_fails(self):
host = MockHost()
host.filesystem.write_text_file(
'/mock-checkout/third_party/WebKit/LayoutTests/W3CImportExpectations',
'')
importer = TestImporter(host)
importer.git_cl = MockGitCL(host,
results={
Build('cq-builder-a', 120):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('cq-builder-a', 123):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('cq-builder-b', 200):
TryJobStatus('COMPLETED', 'SUCCESS'),
})
importer.fetch_new_expectations_and_baselines = lambda: None
success = importer.run_commit_queue_for_cl()
self.assertFalse(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'ERROR: CQ appears to have failed; aborting.\n',
])
self.assertEqual(importer.git_cl.calls, [
['git', 'cl', 'try'],
['git', 'cl', 'set-close'],
])
def test_get_failing_results_dict_some_failing_results(self):
self.host.buildbot.set_results(
Build('mac', 123),
LayoutTestResults({
'tests': {
'fake': {
'test.html': {
'failing-test.html': {
'expected': 'PASS',
'actual': 'IMAGE',
'is_unexpected': True,
},
},
},
},
}))
line_adder = W3CExpectationsLineAdder(self.host)
self.assertEqual(
line_adder.get_failing_results_dict(Build('mac', 123)), {
'fake/test.html/failing-test.html': {
'Mac': {
'actual': 'IMAGE',
'expected': 'PASS',
'bug': 'crbug.com/626703',
},
},
})
def test_run_commit_queue_for_cl_only_checks_non_blink_bots(self):
host = MockHost()
host.filesystem.write_text_file(
'/mock-checkout/third_party/WebKit/LayoutTests/W3CImportExpectations',
'')
host.builders = BuilderList({
'fakeos_blink_rel': {
'port_name': 'test-fakeos',
'specifiers': ['FakeOS', 'Release'],
'is_try_builder': True,
}
})
importer = TestImporter(host)
importer.git_cl = MockGitCL(host,
results={
Build('fakeos_blink_rel', 123):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('cq-builder-b', 200):
TryJobStatus('COMPLETED', 'SUCCESS'),
})
importer.fetch_new_expectations_and_baselines = lambda: None
success = importer.run_commit_queue_for_cl()
self.assertTrue(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'INFO: CQ appears to have passed; trying to commit.\n',
'INFO: Update completed.\n',
])
self.assertEqual(importer.git_cl.calls, [
['git', 'cl', 'try'],
['git', 'cl', 'upload', '-f', '--send-mail'],
['git', 'cl', 'set-commit'],
])
def test_run_commit_queue_for_cl_pass(self):
host = MockHost()
host.filesystem.write_text_file(
'/mock-checkout/third_party/WebKit/LayoutTests/W3CImportExpectations',
'')
importer = TestImporter(host)
# Only the latest job for each builder is counted.
importer.git_cl = MockGitCL(host,
results={
Build('cq-builder-a', 120):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('cq-builder-a', 123):
TryJobStatus('COMPLETED', 'SUCCESS'),
})
success = importer.run_commit_queue_for_cl()
self.assertTrue(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'INFO: CQ appears to have passed; trying to commit.\n',
'INFO: Update completed.\n',
])
self.assertEqual(importer.git_cl.calls, [
['git', 'cl', 'try'],
['git', 'cl', 'upload', '-f', '--send-mail'],
['git', 'cl', 'set-commit'],
])
def test_latest_try_builds_failures(self):
git_cl = GitCL(MockHost())
git_cl.fetch_raw_try_job_results = lambda: [
{
'builder_name':
'builder-a',
'status':
'COMPLETED',
'result':
'FAILURE',
'failure_reason':
'BUILD_FAILURE',
'url':
'http://build.chromium.org/p/master/builders/builder-a/builds/100',
},
{
'builder_name':
'builder-b',
'status':
'COMPLETED',
'result':
'FAILURE',
'failure_reason':
'INFRA_FAILURE',
'url':
'http://build.chromium.org/p/master/builders/builder-b/builds/200',
},
]
self.assertEqual(
git_cl.latest_try_jobs(['builder-a', 'builder-b']), {
Build('builder-a', 100): TryJobStatus('COMPLETED', 'FAILURE'),
Build('builder-b', 200): TryJobStatus('COMPLETED', 'FAILURE'),
})
def test_rebaseline_test_passes_on_all_builders(self):
self._setup_mock_build_data()
self.tool.buildbot.set_results(
Build('MOCK Win7'),
LayoutTestResults({
"tests": {
"userscripts": {
"first-test.html": {
"expected": "NEEDSREBASELINE",
"actual": "PASS"
}
}
}
}))
self._write(self.mac_expectations_path,
"Bug(x) userscripts/first-test.html [ Failure ]\n")
self._write("userscripts/first-test.html", "Dummy test contents")
self.command.rebaseline(self.options(), {
"userscripts/first-test.html": {
Build("MOCK Win7"): ["txt", "png"]
}
})
self.assertEqual(self.tool.executive.calls, [])
def test_execute_with_trigger_jobs_option(self):
builds = [
Build('MOCK Try Win', 5000),
Build('MOCK Try Mac', 4000),
]
git_cl = GitCL(self.tool)
git_cl.get_issue_number = lambda: '11112222'
git_cl.latest_try_jobs = lambda _: builds
self.command.git_cl = lambda: git_cl
return_code = self.command.execute(
self.command_options(trigger_jobs=True), [], self.tool)
self.assertEqual(return_code, 1)
self.assertLog([
'INFO: Triggering try jobs for:\n',
'INFO: MOCK Try Linux\n',
'INFO: Please re-run webkit-patch rebaseline-cl once all pending try jobs have finished.\n',
])
self.assertEqual(self.tool.executive.calls, [[
'python',
'/mock-checkout/third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/manifest',
'--work', '--tests-root',
'/mock-checkout/third_party/WebKit/LayoutTests/external/wpt'
], [
'git', 'cl', 'try', '-m', 'tryserver.blink', '-b', 'MOCK Try Linux'
]])
def test_execute_with_unfinished_jobs(self):
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('STARTED'),
Build('MOCK Try Linux', 6000): TryJobStatus('SCHEDULED'),
}
git_cl = GitCL(self.tool)
git_cl.get_issue_number = lambda: '11112222'
git_cl.latest_try_jobs = lambda _: builds
self.command.git_cl = lambda: git_cl
exit_code = self.command.execute(self.command_options(), [], self.tool)
self.assertEqual(exit_code, 1)
self.assertLog([
'INFO: Finished try jobs:\n',
'INFO: MOCK Try Win\n',
'INFO: Scheduled or started try jobs:\n',
'INFO: MOCK Try Linux\n',
'INFO: MOCK Try Mac\n',
'INFO: There are some builders with no results:\n',
'INFO: MOCK Try Linux\n',
'INFO: MOCK Try Mac\n',
'INFO: Would you like to try to fill in missing results with\n'
'available results? This assumes that layout test results\n'
'for the platforms with missing results are the same as\n'
'results on other platforms.\n',
'INFO: Aborting.\n',
])
def test_latest_try_builds(self):
git_cl = GitCL(MockHost())
git_cl.fetch_try_results = lambda: [
{
'builder_name': 'builder-b',
'status': 'COMPLETED',
'result': 'SUCCESS',
'url': 'http://build.chromium.org/p/master/builders/some-builder/builds/100',
},
{
'builder_name': 'builder-b',
'status': 'COMPLETED',
'result': 'SUCCESS',
'url': 'http://build.chromium.org/p/master/builders/some-builder/builds/90',
},
{
'builder_name': 'builder-a',
'status': 'SCHEDULED',
'result': None,
'url': None,
},
{
'builder_name': 'builder-c',
'status': 'COMPLETED',
'result': 'SUCCESS',
'url': 'http://build.chromium.org/p/master/builders/some-builder/builds/123',
},
]
self.assertEqual(
git_cl.latest_try_jobs(['builder-a', 'builder-b']),
[Build('builder-a'), Build('builder-b', 100)])
def test_get_failing_results_dict_no_results(self):
host = self.mock_host()
host.buildbot = MockBuildBot()
host.buildbot.set_results(Build('MOCK Try Mac10.10', 123), None)
updater = WPTExpectationsUpdater(host)
self.assertEqual(
updater.get_failing_results_dict(Build('MOCK Try Mac10.10', 123)), {})
def test_try_job_results_with_task_id_in_url(self):
git_cl = GitCL(MockHost())
git_cl.fetch_raw_try_job_results = lambda **_: [
{
'builder_name':
'builder-a',
'status':
'COMPLETED',
'result':
'FAILURE',
'failure_reason':
'BUILD_FAILURE',
'url': ('https://ci.chromium.org/swarming/task/'
'36a767f405d9ee10'),
},
{
'builder_name':
'builder-b',
'status':
'COMPLETED',
'result':
'SUCCESS',
'url': ('https://ci.chromium.org/swarming/task/'
'38740befcd9c0010?server=chromium-swarm.appspot.com'),
},
]
self.assertEqual(
git_cl.try_job_results(), {
Build('builder-a', '36a767f405d9ee10'):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('builder-b', '38740befcd9c0010'):
TryJobStatus('COMPLETED', 'SUCCESS'),
})
def test_has_failing_try_results_only_success_and_started(self):
self.assertFalse(
GitCL.some_failed({
Build('some-builder', 90):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('some-builder', 100):
TryJobStatus('STARTED'),
}))
def test_all_success_with_started_build(self):
self.assertFalse(
GitCL.all_success({
Build('some-builder', 1):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('some-builder', 2):
TryJobStatus('STARTED'),
}))
def test_builders_with_no_jobs(self):
# In this example, Linux has a scheduled but not started build, Win has
# a started or finished build, and Mac has no triggered build.
# MOCK Try Mac is listed because it's listed in the BuilderList in setUp.
# Test for protected method - pylint: disable=protected-access
builds = [Build('MOCK Try Linux', None), Build('MOCK Try Win', 123)]
self.assertEqual(self.command.builders_with_no_jobs(builds),
{'MOCK Try Mac'})
def test_filter_latest_jobs_higher_build_last(self):
rietveld = Rietveld(self.mock_web())
self.assertEqual(
rietveld._filter_latest_builds(
[Build('foo', 3),
Build('bar', 5),
Build('foo', 5)]),
[Build('bar', 5), Build('foo', 5)])
def test_str_basic(self):
test_baseline_set = TestBaselineSet(host=self.host)
test_baseline_set.add('a/x.html', Build('MOCK Mac10.12'))
test_baseline_set.add('a/x.html', Build('MOCK Win10'))
self.assertEqual(
str(test_baseline_set),
('<TestBaselineSet with:\n'
' a/x.html: Build(builder_name=\'MOCK Mac10.12\', build_number=None), test-mac-mac10.12\n'
' a/x.html: Build(builder_name=\'MOCK Win10\', build_number=None), test-win-win10>'))
def __init__(self, *args, **kwargs):
self.linux_build = Build('linux_chromium_rel_ng', 100)
self.mac_build = Build('mac_chromium_rel_ng', 101)
self.win_build = Build('win7_chromium_rel_ng', 102)
self.mock_try_results = {
self.linux_build: TryJobStatus('COMPLETED', 'SUCCESS'),
self.win_build: TryJobStatus('COMPLETED', 'SUCCESS'),
self.mac_build: TryJobStatus('COMPLETED', 'SUCCESS')
}
super(TryFlagTest, self).__init__(*args, **kwargs)
def _build(result_dict):
"""Converts a parsed try result dict to a Build object."""
builder_name = result_dict['builder_name']
url = result_dict['url']
if url is None:
return Build(builder_name, None)
match = re.match(r'.*/builds/(\d+)?$', url)
build_number = match.group(1)
assert build_number and build_number.isdigit()
return Build(builder_name, int(build_number))
def test_filter_latest(self):
try_job_results = {
Build('builder-a', 100): TryJobStatus('COMPLETED', 'FAILURE'),
Build('builder-a', 200): TryJobStatus('COMPLETED', 'SUCCESS'),
Build('builder-b', 50): TryJobStatus('SCHEDULED'),
}
self.assertEqual(
GitCL.filter_latest(try_job_results), {
Build('builder-a', 200): TryJobStatus('COMPLETED', 'SUCCESS'),
Build('builder-b', 50): TryJobStatus('SCHEDULED'),
})
def test_trigger_jobs_for_missing_builds_started_and_successful(self):
# A build number of None implies that a job has been started but not finished yet.
self.assertTrue(self.command.trigger_jobs_for_missing_builds([
Build('MOCK Try Linux', None),
Build('MOCK Try Win', 123),
]))
self.assertEqual(self.tool.executive.calls, [])
self.assertLog([
'INFO: There are existing pending builds for:\n',
'INFO: MOCK Try Linux\n',
])
def test_getters(self):
test_baseline_set = TestBaselineSet(host=self.host)
test_baseline_set.add('a/x.html', Build('MOCK Mac10.12'))
test_baseline_set.add('a/x.html', Build('MOCK Win10'))
self.assertEqual(test_baseline_set.test_prefixes(), ['a/x.html'])
self.assertEqual(
test_baseline_set.build_port_pairs('a/x.html'),
[
(Build(builder_name='MOCK Mac10.12'), 'test-mac-mac10.12'),
(Build(builder_name='MOCK Win10'), 'test-win-win10')
])
def test_remove_all_pass_testharness_baselines(self):
self.tool.filesystem.write_text_file(
'/test.checkout/LayoutTests/passes/text-expected.txt',
('This is a testharness.js-based test.\n'
'PASS: foo\n'
'Harness: the test ran to completion.\n'))
test_baseline_set = TestBaselineSet(self.tool)
test_baseline_set.add('passes/text.html', Build('MOCK Win7'))
test_baseline_set.add('passes/text.html', Build('MOCK Win10'))
self.command._remove_all_pass_testharness_baselines(test_baseline_set)
self.assertFalse(self.tool.filesystem.exists(
'/test.checkout/LayoutTests/passes/text-expected.txt'))
def test_run_single_platform_failure(self):
"""Tests the main run method in a case where one test fails on one platform."""
host = self.mock_host()
# Fill in an initial value for TestExpectations
expectations_path = host.port_factory.get(
).path_to_generic_test_expectations_file()
host.filesystem.write_text_file(expectations_path,
MARKER_COMMENT + '\n')
# Set up fake try job results.
updater = WPTExpectationsUpdater(host)
updater.git_cl = MockGitCL(
updater.host, {
Build('MOCK Try Mac10.10', 333):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac10.11', 111):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Trusty', 222):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Precise', 333):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Win10', 444):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Win7', 555):
TryJobStatus('COMPLETED', 'SUCCESS'),
})
# Set up failing results for one try bot. It shouldn't matter what
# results are for the other builders since we shouldn't need to even
# fetch results, since the try job status already tells us that all
# of the tests passed.
host.buildbot.set_results(
Build('MOCK Try Mac10.10', 333),
LayoutTestResults({
'tests': {
'external': {
'wpt': {
'test': {
'path.html': {
'expected': 'PASS',
'actual': 'TIMEOUT',
'is_unexpected': True,
}
}
}
}
}
}))
self.assertEqual(0, updater.run(args=[]))
# Results are only fetched for failing builds.
self.assertEqual(host.buildbot.fetched_builds,
[Build('MOCK Try Mac10.10', 333)])
self.assertEqual(
host.filesystem.read_text_file(expectations_path),
'# ====== New tests from wpt-importer added here ======\n'
'crbug.com/626703 [ Mac10.10 ] external/wpt/test/path.html [ Timeout ]\n'
)
class FailureMapTest(unittest.TestCase):
builder1 = MockBuilder("Builder1")
builder2 = MockBuilder("Builder2")
build1a = Build(builder1, build_number=22, revision=1233, is_green=True)
build1b = Build(builder1, build_number=23, revision=1234, is_green=False)
build2a = Build(builder2, build_number=89, revision=1233, is_green=True)
build2b = Build(builder2, build_number=90, revision=1235, is_green=False)
regression_window1 = RegressionWindow(build1a,
build1b,
failing_tests=[u'test1', u'test1'])
regression_window2 = RegressionWindow(build2a,
build2b,
failing_tests=[u'test1'])
def _make_failure_map(self):
failure_map = FailureMap()
failure_map.add_regression_window(self.builder1,
self.regression_window1)
failure_map.add_regression_window(self.builder2,
self.regression_window2)
return failure_map
def test_failing_revisions(self):
failure_map = self._make_failure_map()
self.assertEqual(failure_map.failing_revisions(), [1234, 1235])
def test_new_failures(self):
failure_map = self._make_failure_map()
failure_map.filter_out_old_failures(lambda revision: False)
self.assertEqual(failure_map.failing_revisions(), [1234, 1235])
def test_new_failures_with_old_revisions(self):
failure_map = self._make_failure_map()
failure_map.filter_out_old_failures(lambda revision: revision == 1234)
self.assertEqual(failure_map.failing_revisions(), [])
def test_new_failures_with_more_old_revisions(self):
failure_map = self._make_failure_map()
failure_map.filter_out_old_failures(lambda revision: revision == 1235)
self.assertEqual(failure_map.failing_revisions(), [1234])
def test_tests_failing_for(self):
failure_map = self._make_failure_map()
self.assertEqual(failure_map.tests_failing_for(1234), [u'test1'])
def test_failing_tests(self):
failure_map = self._make_failure_map()
self.assertEqual(failure_map.failing_tests(), set([u'test1']))
