def test_layout_test_results(self):
buildbot = BuildBot()
builder = Builder(u"Foo Builder (test)", buildbot)
builder._fetch_file_from_results = lambda results_url, file_name: None
build = Build(builder, None, None, None)
# Test that layout_test_results() returns None if the fetch fails.
self.assertEqual(build.layout_test_results(), None)
def _mock_fetch_build(build_number):
build = Build(
builder=self.builder, build_number=build_number, revision=build_number + 1000, is_green=build_number < 4
)
parsed_results = {LayoutTestResults.fail_key: failure(build_number)}
build._layout_test_results = LayoutTestResults(parsed_results)
return build
def _mock_fetch_build(build_number):
build = Build(
builder=self.builder, build_number=build_number, revision=build_number + 1000, is_green=build_number < 4
)
results = [self._mock_test_result(testname) for testname in failure(build_number)]
build._layout_test_results = LayoutTestResults(results)
return build
def _mock_fetch_build(build_number):
build = Build(
builder=self.builder, build_number=build_number, revision=build_number + 1000, is_green=build_number < 4
)
build._layout_test_results = LayoutTestResults(
"http://buildbot.example.com/foo", {LayoutTestResults.fail_key: failure(build_number)}
)
return build
def _build_from_fake(self, fake_builder, index):
if index >= len(fake_builder):
return None
fake_build = fake_builder[index]
build = Build(builder=fake_builder, build_number=index, revision=fake_build[0], is_green=fake_build[1])
def mock_previous_build():
return self._build_from_fake(fake_builder, index + 1)
build.previous_build = mock_previous_build
return build
def _mock_fetch_build(build_number):
build = Build(
builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4
)
results = [self._mock_test_result(testname) for testname in failure(build_number)]
layout_test_results = LayoutTestResults(test_results=results, did_exceed_test_failure_limit=False)
def mock_layout_test_results():
return layout_test_results
build.layout_test_results = mock_layout_test_results
return build
def latest_try_jobs(self, issue_number, builder_names=None, patchset_number=None):
"""Returns a list of Build objects for builds on the latest patchset.
Args:
issue_number: A Rietveld issue number.
builder_names: A collection of builder names. If specified, only results
from the given list of builders will be kept.
patchset_number: If given, a specific patchset will be used instead of the latest one.
Returns:
A list of Build objects, where Build objects for completed jobs have a build number,
and Build objects for pending jobs have no build number.
"""
try:
if patchset_number:
url = self._patchset_url(issue_number, patchset_number)
else:
url = self._latest_patchset_url(issue_number)
patchset_data = self._get_json(url)
except (urllib2.URLError, ValueError):
return []
builds = []
for result_dict in patchset_data["try_job_results"]:
build = Build(result_dict["builder"], result_dict["buildnumber"])
# Normally, a value of -1 or 6 in the "result" field indicates the job is
# started or pending, and the "buildnumber" field is null.
if build.build_number and result_dict["result"] in (-1, 6):
_log.warning(
"Build %s has result %d, but unexpectedly has a build number.", build, result_dict["result"]
)
build.build_number = None
builds.append(build)
if builder_names is not None:
builds = [b for b in builds if b.builder_name in builder_names]
return self._filter_latest_builds(builds)
def test_rebaseline_handles_smoke_tests(self):
# This test is just like test_rebaseline_handles_platform_skips, except that we check for
# a test not being in the SmokeTests file, instead of using overrides files.
# If a test is not part of the smoke tests, we count that as passing on ports that only
# run smoke tests, and do not think that we still need to rebaseline it.
self._write(self.mac_expectations_path,
'Bug(x) userscripts/first-test.html [ Failure ]\n')
self._write('SmokeTests', 'fast/html/article-element.html')
self._write('userscripts/first-test.html', 'Dummy test contents')
self._setup_mock_build_data()
self.command.rebaseline(
self.options(), {
'userscripts/first-test.html': {
Build('MOCK Mac10.11'): ['txt', 'png']
}
})
new_expectations = self._read(self.mac_expectations_path)
self.assertMultiLineEqual(
new_expectations,
'Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failure ]\n'
)
def test_rebaseline_handles_skips_in_file(self):
# This test is like test_Rebaseline_handles_platform_skips, except that the
# Skip is in the same (generic) file rather than a platform file. In this case,
# the Skip line should be left unmodified. Note that the first line is now
# qualified as "[Linux Mac Win]"; if it was unqualified, it would conflict with
# the second line.
self._write(self.mac_expectations_path, (
'Bug(x) [ Linux Mac Win ] userscripts/first-test.html [ Failure ]\n'
'Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n'))
self._write('userscripts/first-test.html', 'Dummy test contents')
self._setup_mock_build_data()
self.command.rebaseline(
self.options(), {
'userscripts/first-test.html': {
Build('MOCK Mac10.11'): ['txt', 'png']
}
})
new_expectations = self._read(self.mac_expectations_path)
self.assertMultiLineEqual(new_expectations, (
'Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failure ]\n'
'Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n'))
def test_rebaseline_handles_platform_skips(self):
# This test is just like test_rebaseline_updates_expectations_file_all_platforms(),
# except that if a particular port happens to SKIP a test in an overrides file,
# we count that as passing, and do not think that we still need to rebaseline it.
self._write(self.mac_expectations_path,
'Bug(x) userscripts/first-test.html [ Failure ]\n')
self._write('NeverFixTests',
'Bug(y) [ Android ] userscripts [ WontFix ]\n')
self._write('userscripts/first-test.html', 'Dummy test contents')
self._setup_mock_build_data()
self.command.rebaseline(
self.options(), {
'userscripts/first-test.html': {
Build('MOCK Mac10.11'): ['txt', 'png']
}
})
new_expectations = self._read(self.mac_expectations_path)
self.assertMultiLineEqual(
new_expectations,
'Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failure ]\n'
)
def test_fill_in_missing_results_prefers_build_with_same_os_type(self):
self.tool.builders = BuilderList({
'MOCK Foo12': {
'port_name': 'foo-foo12',
'specifiers': ['Foo12', 'Release'],
'is_try_builder': True,
},
'MOCK Foo45': {
'port_name': 'foo-foo45',
'specifiers': ['Foo45', 'Release'],
'is_try_builder': True,
},
'MOCK Bar3': {
'port_name': 'bar-bar3',
'specifiers': ['Bar3', 'Release'],
'is_try_builder': True,
},
'MOCK Bar4': {
'port_name': 'bar-bar4',
'specifiers': ['Bar4', 'Release'],
'is_try_builder': True,
},
})
test_baseline_set = TestBaselineSet(self.tool)
test_baseline_set.add('one/flaky-fail.html', Build('MOCK Foo12', 100))
test_baseline_set.add('one/flaky-fail.html', Build('MOCK Bar4', 200))
self.command.fill_in_missing_results(test_baseline_set)
self.assertEqual(
test_baseline_set.build_port_pairs('one/flaky-fail.html'), [
(Build(builder_name='MOCK Foo12',
build_number=100), 'foo-foo12'),
(Build(builder_name='MOCK Bar4',
build_number=200), 'bar-bar4'),
(Build(builder_name='MOCK Foo12',
build_number=100), 'foo-foo45'),
(Build(builder_name='MOCK Bar4',
build_number=200), 'bar-bar3'),
])
self.assertLog([
'INFO: For one/flaky-fail.html:\n',
'INFO: Using Build(builder_name=\'MOCK Foo12\', build_number=100) '
'to supply results for foo-foo45.\n',
'INFO: Using Build(builder_name=\'MOCK Bar4\', build_number=200) to supply results for bar-bar3.\n',
])
def test_latest_try_jobs_ignores_swarming_task(self):
git_cl = GitCL(MockHost())
git_cl.fetch_raw_try_job_results = lambda **_: [
{
'builder_name': 'builder-b',
'status': 'COMPLETED',
'result': 'SUCCESS',
'url':
'https://ci.chromium.org/buildbot/mymaster/builder-b/10',
}, {
'builder_name':
'builder-b',
'status':
'COMPLETED',
'result':
'SUCCESS',
'url': ('https://ci.chromium.org/swarming/task/'
'1234abcd1234abcd?server=chromium-swarm.appspot.com'),
}
]
self.assertEqual(
git_cl.latest_try_jobs(['builder-b']), {
Build('builder-b', 10): TryJobStatus('COMPLETED', 'SUCCESS'),
})
def test_rebaseline_calls(self):
"""Tests the list of commands that are invoked when rebaseline is called."""
# First write test contents to the mock filesystem so that
# fast/dom/prototype-taco.html is considered a real test to rebaseline.
port = self.tool.port_factory.get('test-win-win7')
self._write(
port.host.filesystem.join(port.layout_tests_dir(), 'fast/dom/prototype-taco.html'),
'test contents')
test_baseline_set = TestBaselineSet(self.tool)
test_baseline_set.add('fast/dom/prototype-taco.html', Build('MOCK Try Win', 5000))
self.command.rebaseline(self.command_options(), test_baseline_set)
self.assertEqual(
self.tool.executive.calls,
[
[[
'python', 'echo', 'copy-existing-baselines-internal',
'--test', 'fast/dom/prototype-taco.html',
'--suffixes', 'txt',
'--port-name', 'test-win-win7',
]],
[[
'python', 'echo', 'rebaseline-test-internal',
'--test', 'fast/dom/prototype-taco.html',
'--suffixes', 'txt',
'--port-name', 'test-win-win7',
'--builder', 'MOCK Try Win',
'--build-number', '5000',
]],
[[
'python', 'echo', 'optimize-baselines',
'--suffixes', 'txt',
'fast/dom/prototype-taco.html',
]]
])
def test_wait_for_try_jobs_cl_closed(self):
host = MockHost()
host.executive = MockExecutive(output='closed')
git_cl = GitCL(host)
git_cl.fetch_raw_try_job_results = lambda **_: [
{
'builder_name': 'some-builder',
'status': 'STARTED',
'result': None,
'url': None,
},
]
self.assertEqual(
git_cl.wait_for_try_jobs(),
CLStatus(
status='closed',
try_job_results={
Build('some-builder', None): TryJobStatus('STARTED', None),
},
)
)
self.assertEqual(
host.stdout.getvalue(),
'Waiting for try jobs, timeout: 7200 seconds.\n')
def test_wait_for_try_jobs_done(self):
host = MockHost()
host.executive = MockExecutive(output='lgtm')
git_cl = GitCL(host)
git_cl.fetch_raw_try_job_results = lambda **_: [
{
'builder_name': 'some-builder',
'status': 'COMPLETED',
'result': 'FAILURE',
'url': 'http://ci.chromium.org/master/some-builder/100',
},
]
self.assertEqual(
git_cl.wait_for_try_jobs(),
CLStatus(
status='lgtm',
try_job_results={
Build('some-builder', 100): TryJobStatus('COMPLETED', 'FAILURE'),
}
)
)
self.assertEqual(
host.stdout.getvalue(),
'Waiting for try jobs, timeout: 7200 seconds.\n')
def test_results_directory(self):
self._setup_mock_build_data()
self._write("userscripts/first-test.html", "Dummy test contents")
self.command.rebaseline(
self.options(optimize=False, results_directory='/tmp'), {
"userscripts/first-test.html": {
Build("MOCK Win7"): ["txt", "png"]
}
})
# Note that we have only one run_in_parallel() call
self.assertEqual(
self.tool.executive.calls,
[[[
'python', 'echo', 'copy-existing-baselines-internal',
'--suffixes', 'txt,png', '--builder', 'MOCK Win7', '--test',
'userscripts/first-test.html', '--verbose'
]],
[[
'python', 'echo', 'rebaseline-test-internal', '--suffixes',
'txt,png', '--builder', 'MOCK Win7', '--test',
'userscripts/first-test.html', '--verbose',
'--results-directory', '/tmp'
]]])
def test_has_failing_try_results_with_failing_results(self):
self.assertTrue(
GitCL.some_failed({
Build('some-builder', 1):
TryJobStatus('COMPLETED', 'FAILURE'),
}))
def test_results_zip_url(self):
b = Build(None, 123, 123, False)
b.results_url = self.results_url
self.assertEqual("some-url.zip", b.results_zip_url())
def test_rebaseline_expectations_reftests(self):
self._zero_out_test_expectations()
self.tool.executive = MockExecutive()
for builder in ['MOCK Mac10.10', 'MOCK Mac10.11']:
self.tool.buildbot.set_results(
Build(builder),
LayoutTestResults({
"tests": {
"userscripts": {
"reftest-text.html": {
"expected": "PASS",
"actual": "TEXT"
},
"reftest-image.html": {
"expected": "FAIL",
"actual": "IMAGE"
},
"reftest-image-text.html": {
"expected": "FAIL",
"actual": "IMAGE+TEXT"
}
}
}
}))
self._write("userscripts/reftest-text.html", "Dummy test contents")
self._write("userscripts/reftest-text-expected.html",
"Dummy test contents")
self._write("userscripts/reftest-text-expected.html",
"Dummy test contents")
self.command._tests_to_rebaseline = lambda port: {
'userscripts/reftest-text.html': set(['txt']),
'userscripts/reftest-image.html': set(['png']),
'userscripts/reftest-image-text.html': set(['png', 'txt']),
}
self.command.execute(self.options(), [], self.tool)
self.assertEqual(self.tool.executive.calls, [
[
[
'python', 'echo', 'copy-existing-baselines-internal',
'--suffixes', 'txt', '--builder', 'MOCK Mac10.10',
'--test', 'userscripts/reftest-text.html'
],
[
'python', 'echo', 'copy-existing-baselines-internal',
'--suffixes', 'txt', '--builder', 'MOCK Mac10.11',
'--test', 'userscripts/reftest-text.html'
],
],
[
[
'python', 'echo', 'rebaseline-test-internal', '--suffixes',
'txt', '--builder', 'MOCK Mac10.10', '--test',
'userscripts/reftest-text.html'
],
[
'python', 'echo', 'rebaseline-test-internal', '--suffixes',
'txt', '--builder', 'MOCK Mac10.11', '--test',
'userscripts/reftest-text.html'
],
],
])
def test_rebaseline_missing(self):
self.tool.buildbot.set_results(
Build('MOCK Mac10.10'),
LayoutTestResults({
"tests": {
"fast": {
"dom": {
"missing-text.html": {
"expected": "PASS",
"actual": "MISSING",
"is_unexpected": True,
"is_missing_text": True
},
"missing-text-and-image.html": {
"expected": "PASS",
"actual": "MISSING",
"is_unexpected": True,
"is_missing_text": True,
"is_missing_image": True
},
"missing-image.html": {
"expected": "PASS",
"actual": "MISSING",
"is_unexpected": True,
"is_missing_image": True
}
}
}
}
}))
self._write('fast/dom/missing-text.html', "Dummy test contents")
self._write('fast/dom/missing-text-and-image.html',
"Dummy test contents")
self._write('fast/dom/missing-image.html', "Dummy test contents")
self.command._tests_to_rebaseline = lambda port: {
'fast/dom/missing-text.html': set(['txt', 'png']),
'fast/dom/missing-text-and-image.html': set(['txt', 'png']),
'fast/dom/missing-image.html': set(['txt', 'png']),
}
self.command.execute(self.options(), [], self.tool)
self.assertEqual(
self.tool.executive.calls,
[[
[
'python', 'echo', 'copy-existing-baselines-internal',
'--suffixes', 'txt', '--builder', 'MOCK Mac10.10',
'--test', 'fast/dom/missing-text.html'
],
[
'python', 'echo', 'copy-existing-baselines-internal',
'--suffixes', 'txt,png', '--builder', 'MOCK Mac10.10',
'--test', 'fast/dom/missing-text-and-image.html'
],
[
'python', 'echo', 'copy-existing-baselines-internal',
'--suffixes', 'png', '--builder', 'MOCK Mac10.10',
'--test', 'fast/dom/missing-image.html'
],
],
[
[
'python', 'echo', 'rebaseline-test-internal',
'--suffixes', 'txt', '--builder', 'MOCK Mac10.10',
'--test', 'fast/dom/missing-text.html'
],
[
'python', 'echo', 'rebaseline-test-internal',
'--suffixes', 'txt,png', '--builder', 'MOCK Mac10.10',
'--test', 'fast/dom/missing-text-and-image.html'
],
[
'python', 'echo', 'rebaseline-test-internal',
'--suffixes', 'png', '--builder', 'MOCK Mac10.10',
'--test', 'fast/dom/missing-image.html'
],
]])
def test_layout_test_results(self):
build = Build(None, None, None, None)
build._fetch_results_html = lambda: None
# Test that layout_test_results() returns None if the fetch fails.
self.assertEqual(build.layout_test_results(), None)
def test_execute_git_cl_hangs(self):
def blame(_):
return """
6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
"""
self.tool.scm().blame = blame
test_port = self.tool.port_factory.get('test')
# Have prototype-chocolate only fail on "MOCK Mac10.11".
self.tool.buildbot.set_results(
Build('MOCK Mac10.11'),
LayoutTestResults({
"tests": {
"fast": {
"dom": {
"prototype-taco.html": {
"expected": "PASS",
"actual": "PASS TEXT",
"is_unexpected": True
}
}
}
}
}))
self.tool.filesystem.write_text_file(
test_port.path_to_generic_test_expectations_file(), """
Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
""")
self._write_test_file(test_port, 'fast/dom/prototype-taco.html',
"Dummy test contents")
self.tool.builders = BuilderList({
"MOCK Mac10.11": {
"port_name": "test-mac-mac10.11",
"specifiers": ["Mac10.11", "Release"]
},
})
self.command.SECONDS_BEFORE_GIVING_UP = 0
self.command.tree_status = lambda: 'open'
self.tool.executive = MockExecutive()
self.tool.executive.calls = []
self._execute_with_mock_options()
self.assertEqual(self.tool.executive.calls, [
[[
'python', 'echo', 'copy-existing-baselines-internal',
'--suffixes', 'txt', '--builder', 'MOCK Mac10.11', '--test',
'fast/dom/prototype-taco.html'
]],
[[
'python', 'echo', 'rebaseline-test-internal', '--suffixes',
'txt', '--builder', 'MOCK Mac10.11', '--test',
'fast/dom/prototype-taco.html'
]],
[[
'python', 'echo', 'optimize-baselines', '--suffixes', 'txt',
'fast/dom/prototype-taco.html'
]],
['git', 'cl', 'upload', '-f'],
])
def test_has_failing_try_results_only_success_and_started(self):
self.assertFalse(GitCL.has_failing_try_results({
Build('some-builder', 90): TryJobStatus('COMPLETED', 'SUCCESS'),
Build('some-builder', 100): TryJobStatus('STARTED'),
}))
def test_get_failing_results_dict_no_results(self):
self.host.buildbot = MockBuildBot()
self.host.buildbot.set_results(Build('mac', 123), None)
line_adder = W3CExpectationsLineAdder(self.host)
self.assertEqual(
line_adder.get_failing_results_dict(Build('mac', 123)), {})
def _mock_fetch_build(build_number):
build = Build(builder=self.builder,
build_number=build_number,
revision=build_number + 1000,
is_green=build_number < 4)
return build
def test_layout_test_results(self):
build = Build(None, None, None, None)
build._fetch_results_html = lambda: None
# Test that layout_test_results() returns None if the fetch fails.
self.assertEqual(build.layout_test_results(), None)
def test_execute_stuck_on_alternate_rebaseline_branch(self):
def blame(_):
return """
6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
"""
self.tool.scm().blame = blame
test_port = self.tool.port_factory.get('test')
self.tool.buildbot.set_results(
Build('MOCK Win7'),
LayoutTestResults({
"tests": {
"fast": {
"dom": {
"prototype-taco.html": {
"expected": "FAIL",
"actual": "PASS",
"is_unexpected": True
}
}
}
}
}))
self.tool.filesystem.write_text_file(
test_port.path_to_generic_test_expectations_file(), """
Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
""")
self._write_test_file(test_port, 'fast/dom/prototype-taco.html',
"Dummy test contents")
self.tool.executive = MockLineRemovingExecutive()
self.tool.builders = BuilderList({
"MOCK Win7": {
"port_name": "test-win-win7",
"specifiers": ["Win7", "Release"]
},
})
old_branch_name = self.tool.scm().current_branch_or_ref
try:
self.command.tree_status = lambda: 'open'
self.tool.scm(
).current_branch_or_ref = lambda: 'auto-rebaseline-alt-temporary-branch'
self._execute_with_mock_options()
self.assertEqual(self.tool.executive.calls, [
['git', 'cl', 'upload', '-f'],
['git', 'pull'],
['git', 'cl', 'land', '-f', '-v'],
[
'git', 'config',
'branch.auto-rebaseline-temporary-branch.rietveldissue'
],
])
self.assertEqual(
self.tool.filesystem.read_text_file(
test_port.path_to_generic_test_expectations_file()), """
Bug(foo) [ Linux Mac Win10 ] fast/dom/prototype-taco.html [ NeedsRebaseline ]
""")
finally:
self.tool.scm().current_branch_or_ref = old_branch_name
def test_builders_with_scheduled_jobs(self):
# Test for protected method - pylint: disable=protected-access
builds = [Build('MOCK Try Linux', None), Build('MOCK Try Win', 123)]
self.assertEqual(self.command._builders_with_scheduled_jobs(builds),
{'MOCK Try Linux'})
def test_all_success_true(self):
self.assertTrue(
GitCL.all_success({
Build('some-builder', 1):
TryJobStatus('COMPLETED', 'SUCCESS'),
}))
def setUp(self):
BaseTestCase.setUp(self)
LoggingTestCase.setUp(self)
builds = [
Build('MOCK Try Win', 5000),
Build('MOCK Try Mac', 4000),
Build('MOCK Try Linux', 6000),
]
git_cl = GitCL(self.tool)
git_cl.get_issue_number = lambda: '11112222'
git_cl.latest_try_jobs = lambda _: builds
self.command.git_cl = lambda: git_cl
git = MockGit(filesystem=self.tool.filesystem,
executive=self.tool.executive)
git.changed_files = lambda **_: [
'third_party/WebKit/LayoutTests/fast/dom/prototype-inheritance.html',
'third_party/WebKit/LayoutTests/fast/dom/prototype-taco.html',
]
self.tool.git = lambda: git
self.tool.builders = BuilderList({
'MOCK Try Win': {
'port_name': 'test-win-win7',
'specifiers': ['Win7', 'Release'],
'is_try_builder': True,
},
'MOCK Try Linux': {
'port_name': 'test-linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
'MOCK Try Mac': {
'port_name': 'test-mac-mac10.11',
'specifiers': ['Mac10.11', 'Release'],
'is_try_builder': True,
},
})
layout_test_results = LayoutTestResults({
'tests': {
'fast': {
'dom': {
'prototype-inheritance.html': {
'expected': 'PASS',
'actual': 'TEXT',
'is_unexpected': True,
},
'prototype-banana.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True,
},
'prototype-taco.html': {
'expected': 'PASS',
'actual': 'PASS TEXT',
'is_unexpected': True,
},
'prototype-chocolate.html': {
'expected': 'FAIL',
'actual': 'IMAGE+TEXT'
},
'prototype-crashy.html': {
'expected': 'PASS',
'actual': 'CRASH',
'is_unexpected': True,
},
'prototype-newtest.html': {
'expected': 'PASS',
'actual': 'MISSING',
'is_unexpected': True,
'is_missing_text': True,
},
'prototype-slowtest.html': {
'expected': 'SLOW',
'actual': 'TEXT',
'is_unexpected': True,
},
}
},
'svg': {
'dynamic-updates': {
'SVGFEDropShadowElement-dom-stdDeviation-attr.html': {
'expected': 'PASS',
'actual': 'IMAGE',
'has_stderr': True,
'is_unexpected': True,
}
}
}
}
})
for build in builds:
self.tool.buildbot.set_results(build, layout_test_results)
self.tool.buildbot.set_retry_sumary_json(
build,
json.dumps({
'failures': [
'fast/dom/prototype-inheritance.html',
'fast/dom/prototype-newtest.html',
'fast/dom/prototype-slowtest.html',
'fast/dom/prototype-taco.html',
'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html',
],
'ignored': [],
}))
# Write to the mock filesystem so that these tests are considered to exist.
tests = [
'fast/dom/prototype-taco.html',
'fast/dom/prototype-inheritance.html',
'fast/dom/prototype-newtest.html',
'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html',
]
for test in tests:
self._write(
self.mac_port.host.filesystem.join(
self.mac_port.layout_tests_dir(), test), 'contents')
def test_execute(self):
def blame(_):
return """
6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-06-14 20:18:46 +0000 11) # Test NeedsRebaseline being in a comment doesn't bork parsing.
6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Debug ] path/to/norebaseline.html [ Failure ]
6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Mac10.11 ] fast/dom/prototype-strawberry.html [ NeedsRebaseline ]
6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 12) crbug.com/24182 fast/dom/prototype-chocolate.html [ NeedsRebaseline ]
624caaaaaa path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ]
0000000000 path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ]
"""
self.tool.scm().blame = blame
test_port = self.tool.port_factory.get('test')
# Have prototype-chocolate only fail on "MOCK Mac10.11",
# and pass on "Mock Mac10.10".
self.tool.buildbot.set_results(
Build('MOCK Mac10.11'),
LayoutTestResults({
"tests": {
"fast": {
"dom": {
"prototype-taco.html": {
"expected": "PASS",
"actual": "PASS TEXT",
"is_unexpected": True
},
"prototype-chocolate.html": {
"expected": "FAIL",
"actual": "PASS"
},
"prototype-strawberry.html": {
"expected": "PASS",
"actual": "IMAGE PASS",
"is_unexpected": True
}
}
}
}
}))
self.tool.buildbot.set_results(
Build('MOCK Mac10.10'),
LayoutTestResults({
"tests": {
"fast": {
"dom": {
"prototype-taco.html": {
"expected": "PASS",
"actual": "PASS",
},
"prototype-chocolate.html": {
"expected": "FAIL",
"actual": "FAIL"
},
"prototype-strawberry.html": {
"expected": "PASS",
"actual": "PASS",
}
}
}
}
}))
self.tool.filesystem.write_text_file(
test_port.path_to_generic_test_expectations_file(), """
crbug.com/24182 [ Debug ] path/to/norebaseline.html [ Rebaseline ]
Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
crbug.com/24182 [ Mac10.11 ] fast/dom/prototype-strawberry.html [ NeedsRebaseline ]
crbug.com/24182 fast/dom/prototype-chocolate.html [ NeedsRebaseline ]
crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ]
crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ]
""")
self._write_test_file(test_port, 'fast/dom/prototype-taco.html',
"Dummy test contents")
self._write_test_file(test_port, 'fast/dom/prototype-strawberry.html',
"Dummy test contents")
self._write_test_file(test_port, 'fast/dom/prototype-chocolate.html',
"Dummy test contents")
self.tool.executive = MockLineRemovingExecutive()
self.tool.builders = BuilderList({
"MOCK Mac10.10": {
"port_name": "test-mac-mac10.10",
"specifiers": ["Mac10.10", "Release"]
},
"MOCK Mac10.11": {
"port_name": "test-mac-mac10.11",
"specifiers": ["Mac10.11", "Release"]
},
})
self.command.tree_status = lambda: 'closed'
self._execute_with_mock_options()
self.assertEqual(self.tool.executive.calls, [])
self.command.tree_status = lambda: 'open'
self.tool.executive.calls = []
self._execute_with_mock_options()
self.assertEqual(self.tool.executive.calls, [
[
[
'python', 'echo', 'copy-existing-baselines-internal',
'--suffixes', 'png', '--builder', 'MOCK Mac10.11',
'--test', 'fast/dom/prototype-strawberry.html'
],
[
'python', 'echo', 'copy-existing-baselines-internal',
'--suffixes', 'txt', '--builder', 'MOCK Mac10.11',
'--test', 'fast/dom/prototype-taco.html'
],
],
[
[
'python', 'echo', 'rebaseline-test-internal', '--suffixes',
'png', '--builder', 'MOCK Mac10.11', '--test',
'fast/dom/prototype-strawberry.html'
],
[
'python', 'echo', 'rebaseline-test-internal', '--suffixes',
'txt', '--builder', 'MOCK Mac10.11', '--test',
'fast/dom/prototype-taco.html'
],
],
[
[
'python', 'echo', 'optimize-baselines', '--suffixes',
'png', 'fast/dom/prototype-strawberry.html'
],
[
'python', 'echo', 'optimize-baselines', '--suffixes',
'txt', 'fast/dom/prototype-taco.html'
],
],
['git', 'cl', 'upload', '-f'],
['git', 'pull'],
['git', 'cl', 'land', '-f', '-v'],
[
'git', 'config',
'branch.auto-rebaseline-temporary-branch.rietveldissue'
],
])
# The mac ports should both be removed since they're the only ones in builders._exact_matches.
self.assertEqual(
self.tool.filesystem.read_text_file(
test_port.path_to_generic_test_expectations_file()), """
crbug.com/24182 [ Debug ] path/to/norebaseline.html [ Rebaseline ]
Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ NeedsRebaseline ]
crbug.com/24182 [ Linux Win ] fast/dom/prototype-chocolate.html [ NeedsRebaseline ]
crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ]
crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ]
""")
def test_builders_with_pending_builds(self):
# A build number of None implies that a job has been started but not finished yet.
self.assertEqual(
self.command.builders_with_pending_builds(
[Build('MOCK Try Linux', None),
Build('MOCK Try Win', 123)]), {'MOCK Try Linux'})
def test_execute_test_passes_everywhere(self):
def blame(_):
return """
6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
"""
self.tool.scm().blame = blame
test_port = self.tool.port_factory.get('test')
for builder in ['MOCK Mac10.10', 'MOCK Mac10.11']:
self.tool.buildbot.set_results(
Build(builder),
LayoutTestResults({
"tests": {
"fast": {
"dom": {
"prototype-taco.html": {
"expected": "FAIL",
"actual": "PASS",
"is_unexpected": True
}
}
}
}
}))
self.tool.filesystem.write_text_file(
test_port.path_to_generic_test_expectations_file(), """
Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
""")
self._write_test_file(test_port, 'fast/dom/prototype-taco.html',
"Dummy test contents")
self.tool.executive = MockLineRemovingExecutive()
self.tool.builders = BuilderList({
"MOCK Mac10.10": {
"port_name": "test-mac-mac10.10",
"specifiers": ["Mac10.10", "Release"]
},
"MOCK Mac10.11": {
"port_name": "test-mac-mac10.11",
"specifiers": ["Mac10.11", "Release"]
},
})
self.command.tree_status = lambda: 'open'
self._execute_with_mock_options()
self.assertEqual(self.tool.executive.calls, [
['git', 'cl', 'upload', '-f'],
['git', 'pull'],
['git', 'cl', 'land', '-f', '-v'],
[
'git', 'config',
'branch.auto-rebaseline-temporary-branch.rietveldissue'
],
])
# The mac ports should both be removed since they're the only ones in builders._exact_matches.
self.assertEqual(
self.tool.filesystem.read_text_file(
test_port.path_to_generic_test_expectations_file()), """
Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ NeedsRebaseline ]
""")
def setUp(self):
BaseTestCase.setUp(self)
LoggingTestCase.setUp(self)
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Linux', 6000):
TryJobStatus('COMPLETED', 'FAILURE'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
git = MockGit(filesystem=self.tool.filesystem,
executive=self.tool.executive)
git.changed_files = lambda **_: [
'third_party/WebKit/LayoutTests/one/text-fail.html',
'third_party/WebKit/LayoutTests/one/flaky-fail.html',
]
self.tool.git = lambda: git
self.tool.builders = BuilderList({
'MOCK Try Win': {
'port_name': 'test-win-win7',
'specifiers': ['Win7', 'Release'],
'is_try_builder': True,
},
'MOCK Try Linux': {
'port_name': 'test-linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
'MOCK Try Mac': {
'port_name': 'test-mac-mac10.11',
'specifiers': ['Mac10.11', 'Release'],
'is_try_builder': True,
},
})
layout_test_results = LayoutTestResults({
'tests': {
'one': {
'crash.html': {
'expected': 'PASS',
'actual': 'CRASH',
'is_unexpected': True
},
'expected-fail.html': {
'expected': 'FAIL',
'actual': 'IMAGE+TEXT'
},
'flaky-fail.html': {
'expected': 'PASS',
'actual': 'PASS TEXT',
'is_unexpected': True
},
'missing.html': {
'expected': 'PASS',
'actual': 'MISSING',
'is_unexpected': True
},
'slow-fail.html': {
'expected': 'SLOW',
'actual': 'TEXT',
'is_unexpected': True
},
'text-fail.html': {
'expected': 'PASS',
'actual': 'TEXT',
'is_unexpected': True
},
'unexpected-pass.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True
},
},
'two': {
'image-fail.html': {
'expected': 'PASS',
'actual': 'IMAGE',
'is_unexpected': True
}
},
},
})
for build in builds:
self.tool.buildbot.set_results(build, layout_test_results)
self.tool.buildbot.set_retry_sumary_json(
build,
json.dumps({
'failures': [
'one/flaky-fail.html',
'one/missing.html',
'one/slow-fail.html',
'one/text-fail.html',
'two/image-fail.html',
],
'ignored': [],
}))
# Write to the mock filesystem so that these tests are considered to exist.
tests = [
'one/flaky-fail.html',
'one/missing.html',
'one/slow-fail.html',
'one/text-fail.html',
'two/image-fail.html',
]
for test in tests:
path = self.mac_port.host.filesystem.join(
self.mac_port.layout_tests_dir(), test)
self._write(path, 'contents')
self.mac_port.host.filesystem.write_text_file(
'/test.checkout/LayoutTests/external/wpt/MANIFEST.json', '{}')
def _basic_execute_test(self,
expected_executive_calls,
auth_refresh_token_json=None,
commit_author=None,
dry_run=False):
def blame(_):
return """
6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
"""
self.tool.scm().blame = blame
test_port = self.tool.port_factory.get('test')
for builder in ['MOCK Mac10.10', 'MOCK Mac10.11']:
self.tool.buildbot.set_results(
Build(builder),
LayoutTestResults({
"tests": {
"fast": {
"dom": {
"prototype-taco.html": {
"expected": "FAIL",
"actual": "PASS",
"is_unexpected": True
}
}
}
}
}))
self.tool.filesystem.write_text_file(
test_port.path_to_generic_test_expectations_file(), """
Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
""")
self._write_test_file(test_port, 'fast/dom/prototype-taco.html',
"Dummy test contents")
self.tool.executive = MockLineRemovingExecutive()
self.tool.builders = BuilderList({
"MOCK Mac10.10": {
"port_name": "test-mac-mac10.10",
"specifiers": ["Mac10.10", "Release"]
},
"MOCK Mac10.11": {
"port_name": "test-mac-mac10.11",
"specifiers": ["Mac10.11", "Release"]
},
})
self.command.tree_status = lambda: 'open'
self._execute_with_mock_options(
auth_refresh_token_json=auth_refresh_token_json,
commit_author=commit_author,
dry_run=dry_run)
self.assertEqual(self.tool.executive.calls, expected_executive_calls)
# The mac ports should both be removed since they're the only ones in builders._exact_matches.
self.assertEqual(
self.tool.filesystem.read_text_file(
test_port.path_to_generic_test_expectations_file()), """
Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ NeedsRebaseline ]
""")
def test_results(self):
builder = Builder("builder", BuildBot())
b = Build(builder, 123, 123, True)
self.assertTrue(b.results())
def test_results(self):
builder = Builder('builder', BuildBot())
b = Build(builder, 123, 123, True)
self.assertTrue(b.results())
def setUp(self):
BaseTestCase.setUp(self)
LoggingTestCase.setUp(self)
web = MockWeb(urls={
'https://codereview.chromium.org/api/11112222': json.dumps({
'patchsets': [1, 2],
}),
'https://codereview.chromium.org/api/11112222/2': json.dumps({
'try_job_results': [
{
'builder': 'MOCK Try Win',
'buildnumber': 5000,
'result': 0,
},
{
'builder': 'MOCK Try Mac',
'buildnumber': 4000,
'result': 0,
},
],
'files': {
'third_party/WebKit/LayoutTests/fast/dom/prototype-inheritance.html': {'status': 'M'},
'third_party/WebKit/LayoutTests/fast/dom/prototype-taco.html': {'status': 'M'},
},
}),
})
self.tool.builders = BuilderList({
"MOCK Try Win": {
"port_name": "test-win-win7",
"specifiers": ["Win7", "Release"],
"is_try_builder": True,
},
"MOCK Try Linux": {
"port_name": "test-linux-trusty",
"specifiers": ["Trusty", "Release"],
"is_try_builder": True,
},
})
self.command.rietveld = Rietveld(web)
layout_test_results = LayoutTestResults({
'tests': {
'fast': {
'dom': {
'prototype-inheritance.html': {
'expected': 'PASS',
'actual': 'TEXT',
'is_unexpected': True,
},
'prototype-banana.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True,
},
'prototype-taco.html': {
'expected': 'PASS',
'actual': 'PASS TEXT',
'is_unexpected': True,
},
'prototype-chocolate.html': {
'expected': 'FAIL',
'actual': 'IMAGE+TEXT'
},
'prototype-crashy.html': {
'expected': 'PASS',
'actual': 'CRASH',
'is_unexpected': True,
},
'prototype-newtest.html': {
'expected': 'PASS',
'actual': 'MISSING',
'is_unexpected': True,
'is_missing_text': True,
},
'prototype-slowtest.html': {
'expected': 'SLOW',
'actual': 'TEXT',
'is_unexpected': True,
},
}
},
'svg': {
'dynamic-updates': {
'SVGFEDropShadowElement-dom-stdDeviation-attr.html': {
'expected': 'PASS',
'actual': 'IMAGE',
'has_stderr': True,
'is_unexpected': True,
}
}
}
}
})
for build in [Build('MOCK Try Win', 5000), Build('MOCK Try Mac', 4000)]:
self.tool.buildbot.set_results(build, layout_test_results)
self.tool.buildbot.set_retry_sumary_json(Build('MOCK Try Win', 5000), json.dumps({
'failures': [
'fast/dom/prototype-inheritance.html',
'fast/dom/prototype-newtest.html',
'fast/dom/prototype-slowtest.html',
'fast/dom/prototype-taco.html',
'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html',
],
'ignored': [],
}))
# Write to the mock filesystem so that these tests are considered to exist.
port = self.mac_port
tests = [
'fast/dom/prototype-taco.html',
'fast/dom/prototype-inheritance.html',
'fast/dom/prototype-newtest.html',
'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html',
]
for test in tests:
self._write(port.host.filesystem.join(port.layout_tests_dir(), test), 'contents')
def test_results_zip_url(self):
b = Build(None, 123, 123, False)
b.results_url = self.results_url
self.assertEqual("some-url.zip", b.results_zip_url())
def test_filter_latest_jobs_no_build_number(self):
self.assertEqual(
filter_latest_builds([Build('foo', 3),
Build('bar'),
Build('bar')]),
[Build('bar'), Build('foo', 3)])
