def test_layout_test_results(self): buildbot = BuildBot() builder = Builder(u"Foo Builder (test)", buildbot) builder._fetch_file_from_results = lambda results_url, file_name: None build = Build(builder, None, None, None) # Test that layout_test_results() returns None if the fetch fails. self.assertEqual(build.layout_test_results(), None)
def _mock_fetch_build(build_number): build = Build( builder=self.builder, build_number=build_number, revision=build_number + 1000, is_green=build_number < 4 ) parsed_results = {LayoutTestResults.fail_key: failure(build_number)} build._layout_test_results = LayoutTestResults(parsed_results) return build
def _mock_fetch_build(build_number): build = Build( builder=self.builder, build_number=build_number, revision=build_number + 1000, is_green=build_number < 4 ) results = [self._mock_test_result(testname) for testname in failure(build_number)] build._layout_test_results = LayoutTestResults(results) return build
def _mock_fetch_build(build_number): build = Build( builder=self.builder, build_number=build_number, revision=build_number + 1000, is_green=build_number < 4 ) build._layout_test_results = LayoutTestResults( "http://buildbot.example.com/foo", {LayoutTestResults.fail_key: failure(build_number)} ) return build
def _build_from_fake(self, fake_builder, index): if index >= len(fake_builder): return None fake_build = fake_builder[index] build = Build(builder=fake_builder, build_number=index, revision=fake_build[0], is_green=fake_build[1]) def mock_previous_build(): return self._build_from_fake(fake_builder, index + 1) build.previous_build = mock_previous_build return build
def _mock_fetch_build(build_number): build = Build( builder=self.builder, build_number=build_number, revision=build_number + 1000, is_green=build_number < 4 ) results = [self._mock_test_result(testname) for testname in failure(build_number)] layout_test_results = LayoutTestResults(test_results=results, did_exceed_test_failure_limit=False) def mock_layout_test_results(): return layout_test_results build.layout_test_results = mock_layout_test_results return build
def latest_try_jobs(self, issue_number, builder_names=None, patchset_number=None): """Returns a list of Build objects for builds on the latest patchset. Args: issue_number: A Rietveld issue number. builder_names: A collection of builder names. If specified, only results from the given list of builders will be kept. patchset_number: If given, a specific patchset will be used instead of the latest one. Returns: A list of Build objects, where Build objects for completed jobs have a build number, and Build objects for pending jobs have no build number. """ try: if patchset_number: url = self._patchset_url(issue_number, patchset_number) else: url = self._latest_patchset_url(issue_number) patchset_data = self._get_json(url) except (urllib2.URLError, ValueError): return [] builds = [] for result_dict in patchset_data["try_job_results"]: build = Build(result_dict["builder"], result_dict["buildnumber"]) # Normally, a value of -1 or 6 in the "result" field indicates the job is # started or pending, and the "buildnumber" field is null. if build.build_number and result_dict["result"] in (-1, 6): _log.warning( "Build %s has result %d, but unexpectedly has a build number.", build, result_dict["result"] ) build.build_number = None builds.append(build) if builder_names is not None: builds = [b for b in builds if b.builder_name in builder_names] return self._filter_latest_builds(builds)
def test_rebaseline_handles_smoke_tests(self): # This test is just like test_rebaseline_handles_platform_skips, except that we check for # a test not being in the SmokeTests file, instead of using overrides files. # If a test is not part of the smoke tests, we count that as passing on ports that only # run smoke tests, and do not think that we still need to rebaseline it. self._write(self.mac_expectations_path, 'Bug(x) userscripts/first-test.html [ Failure ]\n') self._write('SmokeTests', 'fast/html/article-element.html') self._write('userscripts/first-test.html', 'Dummy test contents') self._setup_mock_build_data() self.command.rebaseline( self.options(), { 'userscripts/first-test.html': { Build('MOCK Mac10.11'): ['txt', 'png'] } }) new_expectations = self._read(self.mac_expectations_path) self.assertMultiLineEqual( new_expectations, 'Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failure ]\n' )
def test_rebaseline_handles_skips_in_file(self): # This test is like test_Rebaseline_handles_platform_skips, except that the # Skip is in the same (generic) file rather than a platform file. In this case, # the Skip line should be left unmodified. Note that the first line is now # qualified as "[Linux Mac Win]"; if it was unqualified, it would conflict with # the second line. self._write(self.mac_expectations_path, ( 'Bug(x) [ Linux Mac Win ] userscripts/first-test.html [ Failure ]\n' 'Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n')) self._write('userscripts/first-test.html', 'Dummy test contents') self._setup_mock_build_data() self.command.rebaseline( self.options(), { 'userscripts/first-test.html': { Build('MOCK Mac10.11'): ['txt', 'png'] } }) new_expectations = self._read(self.mac_expectations_path) self.assertMultiLineEqual(new_expectations, ( 'Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failure ]\n' 'Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n'))
def test_rebaseline_handles_platform_skips(self): # This test is just like test_rebaseline_updates_expectations_file_all_platforms(), # except that if a particular port happens to SKIP a test in an overrides file, # we count that as passing, and do not think that we still need to rebaseline it. self._write(self.mac_expectations_path, 'Bug(x) userscripts/first-test.html [ Failure ]\n') self._write('NeverFixTests', 'Bug(y) [ Android ] userscripts [ WontFix ]\n') self._write('userscripts/first-test.html', 'Dummy test contents') self._setup_mock_build_data() self.command.rebaseline( self.options(), { 'userscripts/first-test.html': { Build('MOCK Mac10.11'): ['txt', 'png'] } }) new_expectations = self._read(self.mac_expectations_path) self.assertMultiLineEqual( new_expectations, 'Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failure ]\n' )
def test_fill_in_missing_results_prefers_build_with_same_os_type(self): self.tool.builders = BuilderList({ 'MOCK Foo12': { 'port_name': 'foo-foo12', 'specifiers': ['Foo12', 'Release'], 'is_try_builder': True, }, 'MOCK Foo45': { 'port_name': 'foo-foo45', 'specifiers': ['Foo45', 'Release'], 'is_try_builder': True, }, 'MOCK Bar3': { 'port_name': 'bar-bar3', 'specifiers': ['Bar3', 'Release'], 'is_try_builder': True, }, 'MOCK Bar4': { 'port_name': 'bar-bar4', 'specifiers': ['Bar4', 'Release'], 'is_try_builder': True, }, }) test_baseline_set = TestBaselineSet(self.tool) test_baseline_set.add('one/flaky-fail.html', Build('MOCK Foo12', 100)) test_baseline_set.add('one/flaky-fail.html', Build('MOCK Bar4', 200)) self.command.fill_in_missing_results(test_baseline_set) self.assertEqual( test_baseline_set.build_port_pairs('one/flaky-fail.html'), [ (Build(builder_name='MOCK Foo12', build_number=100), 'foo-foo12'), (Build(builder_name='MOCK Bar4', build_number=200), 'bar-bar4'), (Build(builder_name='MOCK Foo12', build_number=100), 'foo-foo45'), (Build(builder_name='MOCK Bar4', build_number=200), 'bar-bar3'), ]) self.assertLog([ 'INFO: For one/flaky-fail.html:\n', 'INFO: Using Build(builder_name=\'MOCK Foo12\', build_number=100) ' 'to supply results for foo-foo45.\n', 'INFO: Using Build(builder_name=\'MOCK Bar4\', build_number=200) to supply results for bar-bar3.\n', ])
def test_latest_try_jobs_ignores_swarming_task(self): git_cl = GitCL(MockHost()) git_cl.fetch_raw_try_job_results = lambda **_: [ { 'builder_name': 'builder-b', 'status': 'COMPLETED', 'result': 'SUCCESS', 'url': 'https://ci.chromium.org/buildbot/mymaster/builder-b/10', }, { 'builder_name': 'builder-b', 'status': 'COMPLETED', 'result': 'SUCCESS', 'url': ('https://ci.chromium.org/swarming/task/' '1234abcd1234abcd?server=chromium-swarm.appspot.com'), } ] self.assertEqual( git_cl.latest_try_jobs(['builder-b']), { Build('builder-b', 10): TryJobStatus('COMPLETED', 'SUCCESS'), })
def test_rebaseline_calls(self): """Tests the list of commands that are invoked when rebaseline is called.""" # First write test contents to the mock filesystem so that # fast/dom/prototype-taco.html is considered a real test to rebaseline. port = self.tool.port_factory.get('test-win-win7') self._write( port.host.filesystem.join(port.layout_tests_dir(), 'fast/dom/prototype-taco.html'), 'test contents') test_baseline_set = TestBaselineSet(self.tool) test_baseline_set.add('fast/dom/prototype-taco.html', Build('MOCK Try Win', 5000)) self.command.rebaseline(self.command_options(), test_baseline_set) self.assertEqual( self.tool.executive.calls, [ [[ 'python', 'echo', 'copy-existing-baselines-internal', '--test', 'fast/dom/prototype-taco.html', '--suffixes', 'txt', '--port-name', 'test-win-win7', ]], [[ 'python', 'echo', 'rebaseline-test-internal', '--test', 'fast/dom/prototype-taco.html', '--suffixes', 'txt', '--port-name', 'test-win-win7', '--builder', 'MOCK Try Win', '--build-number', '5000', ]], [[ 'python', 'echo', 'optimize-baselines', '--suffixes', 'txt', 'fast/dom/prototype-taco.html', ]] ])
def test_wait_for_try_jobs_cl_closed(self): host = MockHost() host.executive = MockExecutive(output='closed') git_cl = GitCL(host) git_cl.fetch_raw_try_job_results = lambda **_: [ { 'builder_name': 'some-builder', 'status': 'STARTED', 'result': None, 'url': None, }, ] self.assertEqual( git_cl.wait_for_try_jobs(), CLStatus( status='closed', try_job_results={ Build('some-builder', None): TryJobStatus('STARTED', None), }, ) ) self.assertEqual( host.stdout.getvalue(), 'Waiting for try jobs, timeout: 7200 seconds.\n')
def test_wait_for_try_jobs_done(self): host = MockHost() host.executive = MockExecutive(output='lgtm') git_cl = GitCL(host) git_cl.fetch_raw_try_job_results = lambda **_: [ { 'builder_name': 'some-builder', 'status': 'COMPLETED', 'result': 'FAILURE', 'url': 'http://ci.chromium.org/master/some-builder/100', }, ] self.assertEqual( git_cl.wait_for_try_jobs(), CLStatus( status='lgtm', try_job_results={ Build('some-builder', 100): TryJobStatus('COMPLETED', 'FAILURE'), } ) ) self.assertEqual( host.stdout.getvalue(), 'Waiting for try jobs, timeout: 7200 seconds.\n')
def test_results_directory(self): self._setup_mock_build_data() self._write("userscripts/first-test.html", "Dummy test contents") self.command.rebaseline( self.options(optimize=False, results_directory='/tmp'), { "userscripts/first-test.html": { Build("MOCK Win7"): ["txt", "png"] } }) # Note that we have only one run_in_parallel() call self.assertEqual( self.tool.executive.calls, [[[ 'python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose' ]], [[ 'python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.html', '--verbose', '--results-directory', '/tmp' ]]])
def test_has_failing_try_results_with_failing_results(self): self.assertTrue( GitCL.some_failed({ Build('some-builder', 1): TryJobStatus('COMPLETED', 'FAILURE'), }))
def test_results_zip_url(self): b = Build(None, 123, 123, False) b.results_url = self.results_url self.assertEqual("some-url.zip", b.results_zip_url())
def test_rebaseline_expectations_reftests(self): self._zero_out_test_expectations() self.tool.executive = MockExecutive() for builder in ['MOCK Mac10.10', 'MOCK Mac10.11']: self.tool.buildbot.set_results( Build(builder), LayoutTestResults({ "tests": { "userscripts": { "reftest-text.html": { "expected": "PASS", "actual": "TEXT" }, "reftest-image.html": { "expected": "FAIL", "actual": "IMAGE" }, "reftest-image-text.html": { "expected": "FAIL", "actual": "IMAGE+TEXT" } } } })) self._write("userscripts/reftest-text.html", "Dummy test contents") self._write("userscripts/reftest-text-expected.html", "Dummy test contents") self._write("userscripts/reftest-text-expected.html", "Dummy test contents") self.command._tests_to_rebaseline = lambda port: { 'userscripts/reftest-text.html': set(['txt']), 'userscripts/reftest-image.html': set(['png']), 'userscripts/reftest-image-text.html': set(['png', 'txt']), } self.command.execute(self.options(), [], self.tool) self.assertEqual(self.tool.executive.calls, [ [ [ 'python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK Mac10.10', '--test', 'userscripts/reftest-text.html' ], [ 'python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK Mac10.11', '--test', 'userscripts/reftest-text.html' ], ], [ [ 'python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK Mac10.10', '--test', 'userscripts/reftest-text.html' ], [ 'python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK Mac10.11', '--test', 'userscripts/reftest-text.html' ], ], ])
def test_rebaseline_missing(self): self.tool.buildbot.set_results( Build('MOCK Mac10.10'), LayoutTestResults({ "tests": { "fast": { "dom": { "missing-text.html": { "expected": "PASS", "actual": "MISSING", "is_unexpected": True, "is_missing_text": True }, "missing-text-and-image.html": { "expected": "PASS", "actual": "MISSING", "is_unexpected": True, "is_missing_text": True, "is_missing_image": True }, "missing-image.html": { "expected": "PASS", "actual": "MISSING", "is_unexpected": True, "is_missing_image": True } } } } })) self._write('fast/dom/missing-text.html', "Dummy test contents") self._write('fast/dom/missing-text-and-image.html', "Dummy test contents") self._write('fast/dom/missing-image.html', "Dummy test contents") self.command._tests_to_rebaseline = lambda port: { 'fast/dom/missing-text.html': set(['txt', 'png']), 'fast/dom/missing-text-and-image.html': set(['txt', 'png']), 'fast/dom/missing-image.html': set(['txt', 'png']), } self.command.execute(self.options(), [], self.tool) self.assertEqual( self.tool.executive.calls, [[ [ 'python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-text.html' ], [ 'python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-text-and-image.html' ], [ 'python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'png', '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-image.html' ], ], [ [ 'python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-text.html' ], [ 'python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-text-and-image.html' ], [ 'python', 'echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'MOCK Mac10.10', '--test', 'fast/dom/missing-image.html' ], ]])
def test_layout_test_results(self): build = Build(None, None, None, None) build._fetch_results_html = lambda: None # Test that layout_test_results() returns None if the fetch fails. self.assertEqual(build.layout_test_results(), None)
def test_execute_git_cl_hangs(self): def blame(_): return """ 6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] """ self.tool.scm().blame = blame test_port = self.tool.port_factory.get('test') # Have prototype-chocolate only fail on "MOCK Mac10.11". self.tool.buildbot.set_results( Build('MOCK Mac10.11'), LayoutTestResults({ "tests": { "fast": { "dom": { "prototype-taco.html": { "expected": "PASS", "actual": "PASS TEXT", "is_unexpected": True } } } } })) self.tool.filesystem.write_text_file( test_port.path_to_generic_test_expectations_file(), """ Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] """) self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents") self.tool.builders = BuilderList({ "MOCK Mac10.11": { "port_name": "test-mac-mac10.11", "specifiers": ["Mac10.11", "Release"] }, }) self.command.SECONDS_BEFORE_GIVING_UP = 0 self.command.tree_status = lambda: 'open' self.tool.executive = MockExecutive() self.tool.executive.calls = [] self._execute_with_mock_options() self.assertEqual(self.tool.executive.calls, [ [[ 'python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK Mac10.11', '--test', 'fast/dom/prototype-taco.html' ]], [[ 'python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK Mac10.11', '--test', 'fast/dom/prototype-taco.html' ]], [[ 'python', 'echo', 'optimize-baselines', '--suffixes', 'txt', 'fast/dom/prototype-taco.html' ]], ['git', 'cl', 'upload', '-f'], ])
def test_has_failing_try_results_only_success_and_started(self): self.assertFalse(GitCL.has_failing_try_results({ Build('some-builder', 90): TryJobStatus('COMPLETED', 'SUCCESS'), Build('some-builder', 100): TryJobStatus('STARTED'), }))
def test_get_failing_results_dict_no_results(self): self.host.buildbot = MockBuildBot() self.host.buildbot.set_results(Build('mac', 123), None) line_adder = W3CExpectationsLineAdder(self.host) self.assertEqual( line_adder.get_failing_results_dict(Build('mac', 123)), {})
def _mock_fetch_build(build_number): build = Build(builder=self.builder, build_number=build_number, revision=build_number + 1000, is_green=build_number < 4) return build
def test_execute_stuck_on_alternate_rebaseline_branch(self): def blame(_): return """ 6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] """ self.tool.scm().blame = blame test_port = self.tool.port_factory.get('test') self.tool.buildbot.set_results( Build('MOCK Win7'), LayoutTestResults({ "tests": { "fast": { "dom": { "prototype-taco.html": { "expected": "FAIL", "actual": "PASS", "is_unexpected": True } } } } })) self.tool.filesystem.write_text_file( test_port.path_to_generic_test_expectations_file(), """ Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] """) self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents") self.tool.executive = MockLineRemovingExecutive() self.tool.builders = BuilderList({ "MOCK Win7": { "port_name": "test-win-win7", "specifiers": ["Win7", "Release"] }, }) old_branch_name = self.tool.scm().current_branch_or_ref try: self.command.tree_status = lambda: 'open' self.tool.scm( ).current_branch_or_ref = lambda: 'auto-rebaseline-alt-temporary-branch' self._execute_with_mock_options() self.assertEqual(self.tool.executive.calls, [ ['git', 'cl', 'upload', '-f'], ['git', 'pull'], ['git', 'cl', 'land', '-f', '-v'], [ 'git', 'config', 'branch.auto-rebaseline-temporary-branch.rietveldissue' ], ]) self.assertEqual( self.tool.filesystem.read_text_file( test_port.path_to_generic_test_expectations_file()), """ Bug(foo) [ Linux Mac Win10 ] fast/dom/prototype-taco.html [ NeedsRebaseline ] """) finally: self.tool.scm().current_branch_or_ref = old_branch_name
def test_builders_with_scheduled_jobs(self): # Test for protected method - pylint: disable=protected-access builds = [Build('MOCK Try Linux', None), Build('MOCK Try Win', 123)] self.assertEqual(self.command._builders_with_scheduled_jobs(builds), {'MOCK Try Linux'})
def test_all_success_true(self): self.assertTrue( GitCL.all_success({ Build('some-builder', 1): TryJobStatus('COMPLETED', 'SUCCESS'), }))
def setUp(self): BaseTestCase.setUp(self) LoggingTestCase.setUp(self) builds = [ Build('MOCK Try Win', 5000), Build('MOCK Try Mac', 4000), Build('MOCK Try Linux', 6000), ] git_cl = GitCL(self.tool) git_cl.get_issue_number = lambda: '11112222' git_cl.latest_try_jobs = lambda _: builds self.command.git_cl = lambda: git_cl git = MockGit(filesystem=self.tool.filesystem, executive=self.tool.executive) git.changed_files = lambda **_: [ 'third_party/WebKit/LayoutTests/fast/dom/prototype-inheritance.html', 'third_party/WebKit/LayoutTests/fast/dom/prototype-taco.html', ] self.tool.git = lambda: git self.tool.builders = BuilderList({ 'MOCK Try Win': { 'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Release'], 'is_try_builder': True, }, 'MOCK Try Linux': { 'port_name': 'test-linux-trusty', 'specifiers': ['Trusty', 'Release'], 'is_try_builder': True, }, 'MOCK Try Mac': { 'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release'], 'is_try_builder': True, }, }) layout_test_results = LayoutTestResults({ 'tests': { 'fast': { 'dom': { 'prototype-inheritance.html': { 'expected': 'PASS', 'actual': 'TEXT', 'is_unexpected': True, }, 'prototype-banana.html': { 'expected': 'FAIL', 'actual': 'PASS', 'is_unexpected': True, }, 'prototype-taco.html': { 'expected': 'PASS', 'actual': 'PASS TEXT', 'is_unexpected': True, }, 'prototype-chocolate.html': { 'expected': 'FAIL', 'actual': 'IMAGE+TEXT' }, 'prototype-crashy.html': { 'expected': 'PASS', 'actual': 'CRASH', 'is_unexpected': True, }, 'prototype-newtest.html': { 'expected': 'PASS', 'actual': 'MISSING', 'is_unexpected': True, 'is_missing_text': True, }, 'prototype-slowtest.html': { 'expected': 'SLOW', 'actual': 'TEXT', 'is_unexpected': True, }, } }, 'svg': { 'dynamic-updates': { 'SVGFEDropShadowElement-dom-stdDeviation-attr.html': { 'expected': 'PASS', 'actual': 'IMAGE', 'has_stderr': True, 'is_unexpected': True, } } } } }) for build in builds: self.tool.buildbot.set_results(build, layout_test_results) self.tool.buildbot.set_retry_sumary_json( build, json.dumps({ 'failures': [ 'fast/dom/prototype-inheritance.html', 'fast/dom/prototype-newtest.html', 'fast/dom/prototype-slowtest.html', 'fast/dom/prototype-taco.html', 'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html', ], 'ignored': [], })) # Write to the mock filesystem so that these tests are considered to exist. tests = [ 'fast/dom/prototype-taco.html', 'fast/dom/prototype-inheritance.html', 'fast/dom/prototype-newtest.html', 'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html', ] for test in tests: self._write( self.mac_port.host.filesystem.join( self.mac_port.layout_tests_dir(), test), 'contents')
def test_execute(self): def blame(_): return """ 6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-06-14 20:18:46 +0000 11) # Test NeedsRebaseline being in a comment doesn't bork parsing. 6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Debug ] path/to/norebaseline.html [ Failure ] 6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] 6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Mac10.11 ] fast/dom/prototype-strawberry.html [ NeedsRebaseline ] 6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 12) crbug.com/24182 fast/dom/prototype-chocolate.html [ NeedsRebaseline ] 624caaaaaa path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ] 0000000000 path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ] """ self.tool.scm().blame = blame test_port = self.tool.port_factory.get('test') # Have prototype-chocolate only fail on "MOCK Mac10.11", # and pass on "Mock Mac10.10". self.tool.buildbot.set_results( Build('MOCK Mac10.11'), LayoutTestResults({ "tests": { "fast": { "dom": { "prototype-taco.html": { "expected": "PASS", "actual": "PASS TEXT", "is_unexpected": True }, "prototype-chocolate.html": { "expected": "FAIL", "actual": "PASS" }, "prototype-strawberry.html": { "expected": "PASS", "actual": "IMAGE PASS", "is_unexpected": True } } } } })) self.tool.buildbot.set_results( Build('MOCK Mac10.10'), LayoutTestResults({ "tests": { "fast": { "dom": { "prototype-taco.html": { "expected": "PASS", "actual": "PASS", }, "prototype-chocolate.html": { "expected": "FAIL", "actual": "FAIL" }, "prototype-strawberry.html": { "expected": "PASS", "actual": "PASS", } } } } })) self.tool.filesystem.write_text_file( test_port.path_to_generic_test_expectations_file(), """ crbug.com/24182 [ Debug ] path/to/norebaseline.html [ Rebaseline ] Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] crbug.com/24182 [ Mac10.11 ] fast/dom/prototype-strawberry.html [ NeedsRebaseline ] crbug.com/24182 fast/dom/prototype-chocolate.html [ NeedsRebaseline ] crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ] crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ] """) self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents") self._write_test_file(test_port, 'fast/dom/prototype-strawberry.html', "Dummy test contents") self._write_test_file(test_port, 'fast/dom/prototype-chocolate.html', "Dummy test contents") self.tool.executive = MockLineRemovingExecutive() self.tool.builders = BuilderList({ "MOCK Mac10.10": { "port_name": "test-mac-mac10.10", "specifiers": ["Mac10.10", "Release"] }, "MOCK Mac10.11": { "port_name": "test-mac-mac10.11", "specifiers": ["Mac10.11", "Release"] }, }) self.command.tree_status = lambda: 'closed' self._execute_with_mock_options() self.assertEqual(self.tool.executive.calls, []) self.command.tree_status = lambda: 'open' self.tool.executive.calls = [] self._execute_with_mock_options() self.assertEqual(self.tool.executive.calls, [ [ [ 'python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'png', '--builder', 'MOCK Mac10.11', '--test', 'fast/dom/prototype-strawberry.html' ], [ 'python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK Mac10.11', '--test', 'fast/dom/prototype-taco.html' ], ], [ [ 'python', 'echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'MOCK Mac10.11', '--test', 'fast/dom/prototype-strawberry.html' ], [ 'python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK Mac10.11', '--test', 'fast/dom/prototype-taco.html' ], ], [ [ 'python', 'echo', 'optimize-baselines', '--suffixes', 'png', 'fast/dom/prototype-strawberry.html' ], [ 'python', 'echo', 'optimize-baselines', '--suffixes', 'txt', 'fast/dom/prototype-taco.html' ], ], ['git', 'cl', 'upload', '-f'], ['git', 'pull'], ['git', 'cl', 'land', '-f', '-v'], [ 'git', 'config', 'branch.auto-rebaseline-temporary-branch.rietveldissue' ], ]) # The mac ports should both be removed since they're the only ones in builders._exact_matches. self.assertEqual( self.tool.filesystem.read_text_file( test_port.path_to_generic_test_expectations_file()), """ crbug.com/24182 [ Debug ] path/to/norebaseline.html [ Rebaseline ] Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ NeedsRebaseline ] crbug.com/24182 [ Linux Win ] fast/dom/prototype-chocolate.html [ NeedsRebaseline ] crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ] crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ] """)
def test_builders_with_pending_builds(self): # A build number of None implies that a job has been started but not finished yet. self.assertEqual( self.command.builders_with_pending_builds( [Build('MOCK Try Linux', None), Build('MOCK Try Win', 123)]), {'MOCK Try Linux'})
def test_execute_test_passes_everywhere(self): def blame(_): return """ 6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] """ self.tool.scm().blame = blame test_port = self.tool.port_factory.get('test') for builder in ['MOCK Mac10.10', 'MOCK Mac10.11']: self.tool.buildbot.set_results( Build(builder), LayoutTestResults({ "tests": { "fast": { "dom": { "prototype-taco.html": { "expected": "FAIL", "actual": "PASS", "is_unexpected": True } } } } })) self.tool.filesystem.write_text_file( test_port.path_to_generic_test_expectations_file(), """ Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] """) self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents") self.tool.executive = MockLineRemovingExecutive() self.tool.builders = BuilderList({ "MOCK Mac10.10": { "port_name": "test-mac-mac10.10", "specifiers": ["Mac10.10", "Release"] }, "MOCK Mac10.11": { "port_name": "test-mac-mac10.11", "specifiers": ["Mac10.11", "Release"] }, }) self.command.tree_status = lambda: 'open' self._execute_with_mock_options() self.assertEqual(self.tool.executive.calls, [ ['git', 'cl', 'upload', '-f'], ['git', 'pull'], ['git', 'cl', 'land', '-f', '-v'], [ 'git', 'config', 'branch.auto-rebaseline-temporary-branch.rietveldissue' ], ]) # The mac ports should both be removed since they're the only ones in builders._exact_matches. self.assertEqual( self.tool.filesystem.read_text_file( test_port.path_to_generic_test_expectations_file()), """ Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ NeedsRebaseline ] """)
def setUp(self): BaseTestCase.setUp(self) LoggingTestCase.setUp(self) builds = { Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'), Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'), Build('MOCK Try Linux', 6000): TryJobStatus('COMPLETED', 'FAILURE'), } self.command.git_cl = MockGitCL(self.tool, builds) git = MockGit(filesystem=self.tool.filesystem, executive=self.tool.executive) git.changed_files = lambda **_: [ 'third_party/WebKit/LayoutTests/one/text-fail.html', 'third_party/WebKit/LayoutTests/one/flaky-fail.html', ] self.tool.git = lambda: git self.tool.builders = BuilderList({ 'MOCK Try Win': { 'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Release'], 'is_try_builder': True, }, 'MOCK Try Linux': { 'port_name': 'test-linux-trusty', 'specifiers': ['Trusty', 'Release'], 'is_try_builder': True, }, 'MOCK Try Mac': { 'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release'], 'is_try_builder': True, }, }) layout_test_results = LayoutTestResults({ 'tests': { 'one': { 'crash.html': { 'expected': 'PASS', 'actual': 'CRASH', 'is_unexpected': True }, 'expected-fail.html': { 'expected': 'FAIL', 'actual': 'IMAGE+TEXT' }, 'flaky-fail.html': { 'expected': 'PASS', 'actual': 'PASS TEXT', 'is_unexpected': True }, 'missing.html': { 'expected': 'PASS', 'actual': 'MISSING', 'is_unexpected': True }, 'slow-fail.html': { 'expected': 'SLOW', 'actual': 'TEXT', 'is_unexpected': True }, 'text-fail.html': { 'expected': 'PASS', 'actual': 'TEXT', 'is_unexpected': True }, 'unexpected-pass.html': { 'expected': 'FAIL', 'actual': 'PASS', 'is_unexpected': True }, }, 'two': { 'image-fail.html': { 'expected': 'PASS', 'actual': 'IMAGE', 'is_unexpected': True } }, }, }) for build in builds: self.tool.buildbot.set_results(build, layout_test_results) self.tool.buildbot.set_retry_sumary_json( build, json.dumps({ 'failures': [ 'one/flaky-fail.html', 'one/missing.html', 'one/slow-fail.html', 'one/text-fail.html', 'two/image-fail.html', ], 'ignored': [], })) # Write to the mock filesystem so that these tests are considered to exist. tests = [ 'one/flaky-fail.html', 'one/missing.html', 'one/slow-fail.html', 'one/text-fail.html', 'two/image-fail.html', ] for test in tests: path = self.mac_port.host.filesystem.join( self.mac_port.layout_tests_dir(), test) self._write(path, 'contents') self.mac_port.host.filesystem.write_text_file( '/test.checkout/LayoutTests/external/wpt/MANIFEST.json', '{}')
def _basic_execute_test(self, expected_executive_calls, auth_refresh_token_json=None, commit_author=None, dry_run=False): def blame(_): return """ 6469e754a1 path/to/TestExpectations (<*****@*****.**> 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] """ self.tool.scm().blame = blame test_port = self.tool.port_factory.get('test') for builder in ['MOCK Mac10.10', 'MOCK Mac10.11']: self.tool.buildbot.set_results( Build(builder), LayoutTestResults({ "tests": { "fast": { "dom": { "prototype-taco.html": { "expected": "FAIL", "actual": "PASS", "is_unexpected": True } } } } })) self.tool.filesystem.write_text_file( test_port.path_to_generic_test_expectations_file(), """ Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] """) self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents") self.tool.executive = MockLineRemovingExecutive() self.tool.builders = BuilderList({ "MOCK Mac10.10": { "port_name": "test-mac-mac10.10", "specifiers": ["Mac10.10", "Release"] }, "MOCK Mac10.11": { "port_name": "test-mac-mac10.11", "specifiers": ["Mac10.11", "Release"] }, }) self.command.tree_status = lambda: 'open' self._execute_with_mock_options( auth_refresh_token_json=auth_refresh_token_json, commit_author=commit_author, dry_run=dry_run) self.assertEqual(self.tool.executive.calls, expected_executive_calls) # The mac ports should both be removed since they're the only ones in builders._exact_matches. self.assertEqual( self.tool.filesystem.read_text_file( test_port.path_to_generic_test_expectations_file()), """ Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ NeedsRebaseline ] """)
def test_results(self): builder = Builder("builder", BuildBot()) b = Build(builder, 123, 123, True) self.assertTrue(b.results())
def test_results(self): builder = Builder('builder', BuildBot()) b = Build(builder, 123, 123, True) self.assertTrue(b.results())
def setUp(self): BaseTestCase.setUp(self) LoggingTestCase.setUp(self) web = MockWeb(urls={ 'https://codereview.chromium.org/api/11112222': json.dumps({ 'patchsets': [1, 2], }), 'https://codereview.chromium.org/api/11112222/2': json.dumps({ 'try_job_results': [ { 'builder': 'MOCK Try Win', 'buildnumber': 5000, 'result': 0, }, { 'builder': 'MOCK Try Mac', 'buildnumber': 4000, 'result': 0, }, ], 'files': { 'third_party/WebKit/LayoutTests/fast/dom/prototype-inheritance.html': {'status': 'M'}, 'third_party/WebKit/LayoutTests/fast/dom/prototype-taco.html': {'status': 'M'}, }, }), }) self.tool.builders = BuilderList({ "MOCK Try Win": { "port_name": "test-win-win7", "specifiers": ["Win7", "Release"], "is_try_builder": True, }, "MOCK Try Linux": { "port_name": "test-linux-trusty", "specifiers": ["Trusty", "Release"], "is_try_builder": True, }, }) self.command.rietveld = Rietveld(web) layout_test_results = LayoutTestResults({ 'tests': { 'fast': { 'dom': { 'prototype-inheritance.html': { 'expected': 'PASS', 'actual': 'TEXT', 'is_unexpected': True, }, 'prototype-banana.html': { 'expected': 'FAIL', 'actual': 'PASS', 'is_unexpected': True, }, 'prototype-taco.html': { 'expected': 'PASS', 'actual': 'PASS TEXT', 'is_unexpected': True, }, 'prototype-chocolate.html': { 'expected': 'FAIL', 'actual': 'IMAGE+TEXT' }, 'prototype-crashy.html': { 'expected': 'PASS', 'actual': 'CRASH', 'is_unexpected': True, }, 'prototype-newtest.html': { 'expected': 'PASS', 'actual': 'MISSING', 'is_unexpected': True, 'is_missing_text': True, }, 'prototype-slowtest.html': { 'expected': 'SLOW', 'actual': 'TEXT', 'is_unexpected': True, }, } }, 'svg': { 'dynamic-updates': { 'SVGFEDropShadowElement-dom-stdDeviation-attr.html': { 'expected': 'PASS', 'actual': 'IMAGE', 'has_stderr': True, 'is_unexpected': True, } } } } }) for build in [Build('MOCK Try Win', 5000), Build('MOCK Try Mac', 4000)]: self.tool.buildbot.set_results(build, layout_test_results) self.tool.buildbot.set_retry_sumary_json(Build('MOCK Try Win', 5000), json.dumps({ 'failures': [ 'fast/dom/prototype-inheritance.html', 'fast/dom/prototype-newtest.html', 'fast/dom/prototype-slowtest.html', 'fast/dom/prototype-taco.html', 'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html', ], 'ignored': [], })) # Write to the mock filesystem so that these tests are considered to exist. port = self.mac_port tests = [ 'fast/dom/prototype-taco.html', 'fast/dom/prototype-inheritance.html', 'fast/dom/prototype-newtest.html', 'svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html', ] for test in tests: self._write(port.host.filesystem.join(port.layout_tests_dir(), test), 'contents')
def test_filter_latest_jobs_no_build_number(self): self.assertEqual( filter_latest_builds([Build('foo', 3), Build('bar'), Build('bar')]), [Build('bar'), Build('foo', 3)])