def setUp(self): BaseTestCase.setUp(self) LoggingTestCase.setUp(self) builds = { Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'), Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'), Build('MOCK Try Linux', 6000): TryJobStatus('COMPLETED', 'FAILURE'), } self.command.git_cl = MockGitCL(self.tool, builds) git = MockGit(filesystem=self.tool.filesystem, executive=self.tool.executive) git.changed_files = lambda **_: [ RELATIVE_WEB_TESTS + 'one/text-fail.html', RELATIVE_WEB_TESTS + 'one/flaky-fail.html', ] self.tool.git = lambda: git self.tool.builders = BuilderList({ 'MOCK Try Win': { 'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Release'], 'is_try_builder': True, }, 'MOCK Try Linux': { 'port_name': 'test-linux-trusty', 'specifiers': ['Trusty', 'Release'], 'is_try_builder': True, }, 'MOCK Try Mac': { 'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release'], 'is_try_builder': True, }, }) web_test_results = WebTestResults({ 'tests': { 'one': { 'crash.html': { 'expected': 'PASS', 'actual': 'CRASH', 'is_unexpected': True, 'artifacts': { 'crash_log': ['crash.log'] } }, 'expected-fail.html': { 'expected': 'FAIL', 'actual': 'FAIL', 'artifacts': { 'expected_text': ['expected-fail-expected.txt'], 'actual_text': ['expected-fail-actual.txt'] } }, 'flaky-fail.html': { 'expected': 'PASS', 'actual': 'PASS FAIL', 'is_unexpected': True, 'artifacts': { 'expected_audio': ['flaky-fail-expected.wav'], 'actual_audio': ['flaky-fail-actual.wav'] } }, 'missing.html': { 'expected': 'PASS', 'actual': 'FAIL', 'is_unexpected': True, 'artifacts': { 'actual_image': ['missing-actual.png'] }, 'is_missing_image': True }, 'slow-fail.html': { 'expected': 'SLOW', 'actual': 'FAIL', 'is_unexpected': True, 'artifacts': { 'actual_text': ['slow-fail-actual.txt'], 'expected_text': ['slow-fail-expected.txt'] } }, 'text-fail.html': { 'expected': 'PASS', 'actual': 'FAIL', 'is_unexpected': True, 'artifacts': { 'actual_text': ['text-fail-actual.txt'], 'expected_text': ['text-fail-expected.txt'] } }, 'unexpected-pass.html': { 'expected': 'FAIL', 'actual': 'PASS', 'is_unexpected': True }, }, 'two': { 'image-fail.html': { 'expected': 'PASS', 'actual': 'FAIL', 'is_unexpected': True, 'artifacts': { 'actual_image': ['image-fail-actual.png'], 'expected_image': ['image-fail-expected.png'] } } }, }, }) for build in builds: self.tool.results_fetcher.set_results(build, web_test_results) self.tool.results_fetcher.set_retry_sumary_json( build, json.dumps({ 'failures': [ 'one/flaky-fail.html', 'one/missing.html', 'one/slow-fail.html', 'one/text-fail.html', 'two/image-fail.html', ], 'ignored': [], })) # Write to the mock filesystem so that these tests are considered to exist. tests = [ 'one/flaky-fail.html', 'one/missing.html', 'one/slow-fail.html', 'one/text-fail.html', 'two/image-fail.html', ] for test in tests: path = self.mac_port.host.filesystem.join( self.mac_port.web_tests_dir(), test) self._write(path, 'contents') self.mac_port.host.filesystem.write_text_file( '/test.checkout/web_tests/external/wpt/MANIFEST.json', '{}')
def setUp(self): BaseTestCase.setUp(self) LoggingTestCase.setUp(self) builds = { Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'), Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'), Build('MOCK Try Linux', 6000): TryJobStatus('COMPLETED', 'FAILURE'), } self.command.git_cl = MockGitCL(self.tool, builds) git = MockGit(filesystem=self.tool.filesystem, executive=self.tool.executive) git.changed_files = lambda **_: [ 'third_party/WebKit/LayoutTests/one/text-fail.html', 'third_party/WebKit/LayoutTests/one/flaky-fail.html', ] self.tool.git = lambda: git self.tool.builders = BuilderList({ 'MOCK Try Win': { 'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Release'], 'is_try_builder': True, }, 'MOCK Try Linux': { 'port_name': 'test-linux-trusty', 'specifiers': ['Trusty', 'Release'], 'is_try_builder': True, }, 'MOCK Try Mac': { 'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release'], 'is_try_builder': True, }, }) layout_test_results = LayoutTestResults({ 'tests': { 'one': { 'crash.html': { 'expected': 'PASS', 'actual': 'CRASH', 'is_unexpected': True }, 'expected-fail.html': { 'expected': 'FAIL', 'actual': 'IMAGE+TEXT' }, 'flaky-fail.html': { 'expected': 'PASS', 'actual': 'PASS TEXT', 'is_unexpected': True }, 'missing.html': { 'expected': 'PASS', 'actual': 'MISSING', 'is_unexpected': True }, 'slow-fail.html': { 'expected': 'SLOW', 'actual': 'TEXT', 'is_unexpected': True }, 'text-fail.html': { 'expected': 'PASS', 'actual': 'TEXT', 'is_unexpected': True }, 'unexpected-pass.html': { 'expected': 'FAIL', 'actual': 'PASS', 'is_unexpected': True }, }, 'two': { 'image-fail.html': { 'expected': 'PASS', 'actual': 'IMAGE', 'is_unexpected': True } }, }, }) for build in builds: self.tool.buildbot.set_results(build, layout_test_results) self.tool.buildbot.set_retry_sumary_json( build, json.dumps({ 'failures': [ 'one/flaky-fail.html', 'one/missing.html', 'one/slow-fail.html', 'one/text-fail.html', 'two/image-fail.html', ], 'ignored': [], })) # Write to the mock filesystem so that these tests are considered to exist. tests = [ 'one/flaky-fail.html', 'one/missing.html', 'one/slow-fail.html', 'one/text-fail.html', 'two/image-fail.html', ] for test in tests: path = self.mac_port.host.filesystem.join( self.mac_port.layout_tests_dir(), test) self._write(path, 'contents') self.mac_port.host.filesystem.write_text_file( '/test.checkout/LayoutTests/external/wpt/MANIFEST.json', '{}')