def test_wait_for_try_jobs_cl_closed(self):
host = MockHost()
host.executive = MockExecutive(output='closed')
git_cl = GitCL(host)
git_cl.fetch_raw_try_job_results = lambda **_: [
{
'builder_name': 'some-builder',
'status': 'STARTED',
'result': None,
'url': None,
},
]
self.assertEqual(
git_cl.wait_for_try_jobs(),
CLStatus(
status='closed',
try_job_results={
Build('some-builder', None): TryJobStatus('STARTED', None),
},
))
self.assertEqual(host.stdout.getvalue(),
'Waiting for try jobs, timeout: 7200 seconds.\n')
def test_latest_try_jobs_ignores_swarming_task(self):
git_cl = GitCL(MockHost())
git_cl.fetch_raw_try_job_results = lambda **_: [
{
'builder_name': 'builder-b',
'status': 'COMPLETED',
'result': 'SUCCESS',
'url': 'https://ci.chromium.org/buildbot/mymaster/builder-b/10',
},
{
'builder_name': 'builder-b',
'status': 'COMPLETED',
'result': 'SUCCESS',
'url': ('https://ci.chromium.org/swarming/task/'
'1234abcd1234abcd?server=chromium-swarm.appspot.com'),
}
]
self.assertEqual(
git_cl.latest_try_jobs(['builder-b']),
{
Build('builder-b', 10): TryJobStatus('COMPLETED', 'SUCCESS'),
})
def execute(self, options, args, tool):
self._tool = tool
if not args:
_log.error('Must list tests to rebaseline.')
return
if options.builders:
builders_to_check = []
for builder_names in options.builders:
builders_to_check += builder_names.split(',')
else:
builders_to_check = self._builders_to_pull_from()
test_baseline_set = TestBaselineSet(tool)
for builder in builders_to_check:
for test_prefix in args:
test_baseline_set.add(test_prefix, Build(builder))
_log.debug('Rebaselining: %s', test_baseline_set)
self.rebaseline(options, test_baseline_set)
def test_results_directory(self):
test_baseline_set = TestBaselineSet(self.tool)
test_baseline_set.add('userscripts/first-test.html',
Build('MOCK Win7'))
self.command.rebaseline(
self.options(optimize=False, results_directory='/tmp'),
test_baseline_set)
self.assertEqual(self.tool.executive.calls, [
[[
'python',
'echo',
'copy-existing-baselines-internal',
'--verbose',
'--test',
'userscripts/first-test.html',
'--suffixes',
'txt,png',
'--port-name',
'test-win-win7',
]],
[[
'python',
'echo',
'rebaseline-test-internal',
'--verbose',
'--test',
'userscripts/first-test.html',
'--suffixes',
'txt,png',
'--port-name',
'test-win-win7',
'--builder',
'MOCK Win7',
'--results-directory',
'/tmp',
]],
])
def test_rebaseline_command_invocations(self):
"""Tests the list of commands that are called for rebaselining."""
# First write test contents to the mock filesystem so that
# one/flaky-fail.html is considered a real test to rebaseline.
port = self.tool.port_factory.get('test-win-win7')
path = port.host.filesystem.join(
port.web_tests_dir(), 'one/flaky-fail.html')
self._write(path, 'contents')
test_baseline_set = TestBaselineSet(self.tool)
test_baseline_set.add(
'one/flaky-fail.html', Build('MOCK Try Win', 5000))
self.command.rebaseline(self.command_options(), test_baseline_set)
self.assertEqual(
self.tool.executive.calls,
[
[[
'python', 'echo', 'copy-existing-baselines-internal',
'--test', 'one/flaky-fail.html',
'--suffixes', 'wav',
'--port-name', 'test-win-win7',
]],
[[
'python', 'echo', 'rebaseline-test-internal',
'--test', 'one/flaky-fail.html',
'--suffixes', 'wav',
'--port-name', 'test-win-win7',
'--builder', 'MOCK Try Win',
'--build-number', '5000',
'--step-name', 'webkit_layout_tests (with patch)',
]],
[[
'python', 'echo', 'optimize-baselines',
'--no-manifest-update',
'--suffixes', 'wav',
'one/flaky-fail.html',
]]
])
def test_wait_for_try_jobs_done(self):
host = MockHost()
host.executive = MockExecutive(output='lgtm')
git_cl = GitCL(host)
git_cl.fetch_raw_try_job_results = lambda **_: [
{
'builder_name': 'some-builder',
'status': 'COMPLETED',
'result': 'FAILURE',
'tags': [
'build_address:luci.chromium.try/chromium_presubmit/100',
],
'url': 'http://ci.chromium.org/b/8931586523737389552',
},
]
self.assertEqual(
git_cl.wait_for_try_jobs(),
CLStatus(status='lgtm',
try_job_results={
Build('some-builder', 100):
TryJobStatus('COMPLETED', 'FAILURE'),
}))
self.assertEqual(host.stdout.getvalue(),
'Waiting for try jobs, timeout: 7200 seconds.\n')
def test_fetch_layout_test_results_weird_step_name(self):
buildbot = BuildBot()
def fetch_file(url):
if '/testfile' in url:
return ('ADD_RESULTS(%s);' %
(json.dumps([{
"TestType":
"webkit_layout_tests on Intel GPU (with patch)"
}, {
"TestType": "base_unittests (with patch)"
}])))
return json.dumps({
'passed': True
}) if url.endswith('failing_results.json') else 'deadbeef'
buildbot.fetch_file = fetch_file
results = buildbot.fetch_results(Build('builder', 123))
self.assertEqual(
results._results,
{ # pylint: disable=protected-access
'passed': True
})
self.assertLog([])
def test_no_optimize(self):
test_baseline_set = TestBaselineSet(self.tool)
test_baseline_set.add('userscripts/first-test.html',
Build('MOCK Win7'))
self.command.rebaseline(self.options(optimize=False),
test_baseline_set)
self.assertEqual(self.tool.executive.calls,
[[[
'python',
'echo',
'copy-existing-baselines-internal',
'--verbose',
'--test',
'userscripts/first-test.html',
'--suffixes',
'txt,png',
'--port-name',
'test-win-win7',
]],
[[
'python',
'echo',
'rebaseline-test-internal',
'--verbose',
'--test',
'userscripts/first-test.html',
'--suffixes',
'txt,png',
'--port-name',
'test-win-win7',
'--builder',
'MOCK Win7',
'--step-name',
'webkit_layout_tests (with patch)',
]]])
def test_fill_in_missing_results_prefers_build_with_same_os_type(self):
self.tool.builders = BuilderList({
'MOCK Foo12': {
'port_name': 'foo-foo12',
'specifiers': ['Foo12', 'Release'],
'is_try_builder': True,
},
'MOCK Foo45': {
'port_name': 'foo-foo45',
'specifiers': ['Foo45', 'Release'],
'is_try_builder': True,
},
'MOCK Bar3': {
'port_name': 'bar-bar3',
'specifiers': ['Bar3', 'Release'],
'is_try_builder': True,
},
'MOCK Bar4': {
'port_name': 'bar-bar4',
'specifiers': ['Bar4', 'Release'],
'is_try_builder': True,
},
})
test_baseline_set = TestBaselineSet(self.tool)
test_baseline_set.add('one/flaky-fail.html', Build('MOCK Foo12', 100))
test_baseline_set.add('one/flaky-fail.html', Build('MOCK Bar4', 200))
self.command.fill_in_missing_results(test_baseline_set)
self.assertEqual(
test_baseline_set.build_port_pairs('one/flaky-fail.html'),
[
(Build('MOCK Foo12', 100), 'foo-foo12'),
(Build('MOCK Bar4', 200), 'bar-bar4'),
(Build('MOCK Foo12', 100), 'foo-foo45'),
(Build('MOCK Bar4', 200), 'bar-bar3'),
])
self.assertLog([
'INFO: For one/flaky-fail.html:\n',
'INFO: Using "MOCK Foo12" build 100 for foo-foo45.\n',
'INFO: Using "MOCK Bar4" build 200 for bar-bar3.\n',
])
def setUp(self):
BaseTestCase.setUp(self)
LoggingTestCase.setUp(self)
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Linux', 6000):
TryJobStatus('COMPLETED', 'FAILURE'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
git = MockGit(filesystem=self.tool.filesystem,
executive=self.tool.executive)
git.changed_files = lambda **_: [
'third_party/WebKit/LayoutTests/one/text-fail.html',
'third_party/WebKit/LayoutTests/one/flaky-fail.html',
]
self.tool.git = lambda: git
self.tool.builders = BuilderList({
'MOCK Try Win': {
'port_name': 'test-win-win7',
'specifiers': ['Win7', 'Release'],
'is_try_builder': True,
},
'MOCK Try Linux': {
'port_name': 'test-linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
'MOCK Try Mac': {
'port_name': 'test-mac-mac10.11',
'specifiers': ['Mac10.11', 'Release'],
'is_try_builder': True,
},
})
layout_test_results = LayoutTestResults({
'tests': {
'one': {
'crash.html': {
'expected': 'PASS',
'actual': 'CRASH',
'is_unexpected': True
},
'expected-fail.html': {
'expected': 'FAIL',
'actual': 'IMAGE+TEXT'
},
'flaky-fail.html': {
'expected': 'PASS',
'actual': 'PASS TEXT',
'is_unexpected': True
},
'missing.html': {
'expected': 'PASS',
'actual': 'MISSING',
'is_unexpected': True
},
'slow-fail.html': {
'expected': 'SLOW',
'actual': 'TEXT',
'is_unexpected': True
},
'text-fail.html': {
'expected': 'PASS',
'actual': 'TEXT',
'is_unexpected': True
},
'unexpected-pass.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True
},
},
'two': {
'image-fail.html': {
'expected': 'PASS',
'actual': 'IMAGE',
'is_unexpected': True
}
},
},
})
for build in builds:
self.tool.buildbot.set_results(build, layout_test_results)
self.tool.buildbot.set_retry_sumary_json(
build,
json.dumps({
'failures': [
'one/flaky-fail.html',
'one/missing.html',
'one/slow-fail.html',
'one/text-fail.html',
'two/image-fail.html',
],
'ignored': [],
}))
# Write to the mock filesystem so that these tests are considered to exist.
tests = [
'one/flaky-fail.html',
'one/missing.html',
'one/slow-fail.html',
'one/text-fail.html',
'two/image-fail.html',
]
for test in tests:
path = self.mac_port.host.filesystem.join(
self.mac_port.layout_tests_dir(), test)
self._write(path, 'contents')
self.mac_port.host.filesystem.write_text_file(
'/test.checkout/LayoutTests/external/wpt/MANIFEST.json', '{}')
def test_all_success_with_started_build(self):
self.assertFalse(GitCL.all_success({
Build('some-builder', 1): TryJobStatus('COMPLETED', 'SUCCESS'),
Build('some-builder', 2): TryJobStatus('STARTED'),
}))
def test_has_failing_try_results_only_success_and_started(self):
self.assertFalse(GitCL.some_failed({
Build('some-builder', 90): TryJobStatus('COMPLETED', 'SUCCESS'),
Build('some-builder', 100): TryJobStatus('STARTED'),
}))
def test_get_failing_results_dict_webdriver_failing_results_(self):
host = self.mock_host()
host.buildbot.set_results(
Build('MOCK Try Trusty', 123),
WebTestResults({
'tests': {
'external': {
'wpt': {
'x': {
'failing-test.html': {
'expected': 'PASS',
'actual': 'IMAGE',
'is_unexpected': True,
},
},
},
},
},
}))
host.buildbot.set_webdriver_test_results(
Build('MOCK Try Trusty', 123), "tryserver.blink",
WebTestResults({
'tests': {
'external': {
'wpt': {
'y': {
'webdriver-fail.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
},
},
},
},
},
}))
updater = WPTExpectationsUpdater(host)
results_dict = updater.get_failing_results_dict(
Build('MOCK Try Trusty', 123))
self.assertEqual(len(results_dict.keys()), 2)
self.assertEqual(
results_dict, {
'external/wpt/x/failing-test.html': {
'test-linux-trusty':
SimpleTestResult(
actual='IMAGE',
expected='PASS',
bug='crbug.com/626703',
),
},
'external/wpt/y/webdriver-fail.html': {
'test-linux-trusty':
SimpleTestResult(
actual='FAIL',
expected='PASS',
bug='crbug.com/626703',
),
},
})
def test_filter_latest_jobs_no_build_number(self):
self.assertEqual(
filter_latest_builds([Build('foo', 3),
Build('bar'),
Build('bar')]),
[Build('bar'), Build('foo', 3)])
def test_fetch_webdriver_results_without_master(self):
buildbot = BuildBot()
self.assertIsNone(
buildbot.fetch_webdriver_test_results(Build('builder', 1), ''))
self.assertLog(
['DEBUG: Builder name or build number or master is None\n'])
def test_all_success_true(self):
self.assertTrue(
GitCL.all_success({
Build('some-builder', 1):
TryJobStatus('COMPLETED', 'SUCCESS'),
}))
def test_filter_latest_jobs_higher_build_last(self):
self.assertEqual(
filter_latest_builds([Build('foo', 3), Build('bar', 5), Build('foo', 5)]),
[Build('bar', 5), Build('foo', 5)])
def test_has_failing_try_results_with_failing_results(self):
self.assertTrue(
GitCL.some_failed({
Build('some-builder', 1):
TryJobStatus('COMPLETED', 'FAILURE'),
}))
def test_fetch_results_without_build_number(self):
buildbot = BuildBot()
self.assertIsNone(buildbot.fetch_results(Build('builder', None)))
def test_latest_try_jobs_cq_only(self):
git_cl = GitCL(MockHost())
git_cl.fetch_raw_try_job_results = lambda **_: [
{
'builder_name': 'cq-a',
'experimental': False,
'result': None,
'status': 'SCHEDULED',
'tags': ['user_agent:cq'],
'url': None,
},
{
'builder_name': 'cq-b',
'experimental': False,
'result': None,
'status': 'SCHEDULED',
'tags': ['cq_experimental:false', 'user_agent:cq'],
'url': None,
},
{
'builder_name': 'cq-c',
'experimental': True,
'result': None,
'status': 'SCHEDULED',
'tags': ['cq_experimental:false', 'user_agent:cq'],
'url': None,
},
{
'builder_name': 'cq-a-experimental',
'experimental': True,
'result': None,
'status': 'SCHEDULED',
'tags': ['cq_experimental:true', 'user_agent:cq'],
'url': None,
},
{
'builder_name': 'cq-b-experimental',
'experimental': False,
'result': None,
'status': 'SCHEDULED',
'tags': ['cq_experimental:true', 'user_agent:cq'],
'url': None,
},
{
'builder_name': 'other-a',
'experimental': False,
'status': 'SCHEDULED',
'result': None,
'tags': ['user_agent:git_cl_try'],
'url': None,
},
{
'builder_name': 'other-b',
'experimental': False,
'status': 'SCHEDULED',
'result': None,
'tags': ['is_experimental:false', 'user_agent:git_cl_try'],
'url': None,
},
]
self.assertEqual(
git_cl.latest_try_jobs(cq_only=True), {
Build('cq-a'): TryJobStatus('SCHEDULED'),
Build('cq-b'): TryJobStatus('SCHEDULED'),
Build('cq-c'): TryJobStatus('SCHEDULED'),
})
def test_get_step_name_without_build_number(self):
buildbot = BuildBot()
self.assertIsNone(
buildbot.get_layout_test_step_name(Build('builder', None)))