Exemplo n.º 1
0
    def test_rebaseline_updates_expectations_file(self):
        self._write(self.test_expectations_path, (
            '# tags: [ Mac10.10 Mac Linux ]\n'
            '# tags: [ Debug ]\n'
            '# results: [ Failure ]\n'
            'crbug.com/123 [ Debug Mac ] userscripts/first-test.html [ Failure ]\n'
            '[ Linux ] userscripts/first-test.html [ Failure ]\n'))
        self._setup_mock_build_data()
        test_baseline_set = TestBaselineSet(self.tool)
        test_baseline_set.add('userscripts/first-test.html',
                              Build('MOCK Mac10.11'))

        self.command.rebaseline(self.options(), test_baseline_set)

        new_expectations = self._read(self.test_expectations_path)
        self.assertMultiLineEqual(new_expectations, (
            '# tags: [ Mac10.10 Mac Linux ]\n'
            '# tags: [ Debug ]\n'
            '# results: [ Failure ]\n'
            'crbug.com/123 [ Debug Mac10.10 ] userscripts/first-test.html [ Failure ]\n'
            '[ Linux ] userscripts/first-test.html [ Failure ]\n'))
Exemplo n.º 2
0
 def test_get_step_name(self):
     fetcher = TestResultsFetcher()
     fetcher.web = MockWeb(
         urls={
             'https://test-results.appspot.com/testfile?buildnumber=5&'
             'callback=ADD_RESULTS&builder=foo&name=full_results.json':
             'ADD_RESULTS(%s);' %
             (json.dumps([{
                 "TestType": "blink_web_tests (with patch)"
             }, {
                 "TestType":
                 "not_site_per_process_blink_web_tests (with patch)"
             }, {
                 "TestType": "blink_web_tests (retry with patch)"
             }, {
                 "TestType": "base_unittests (with patch)"
             }]))
         })
     step_name = fetcher.get_layout_test_step_name(Build('foo', 5))
     self.assertEqual(step_name, 'blink_web_tests (with patch)')
     self.assertLog([])
Exemplo n.º 3
0
    def test_results_directory(self):
        test_baseline_set = TestBaselineSet(self.tool)
        test_baseline_set.add('userscripts/first-test.html',
                              Build('MOCK Win7'))
        self.command.rebaseline(
            self.options(optimize=False, results_directory='/tmp'),
            test_baseline_set)

        self.assertEqual(self.tool.executive.calls, [
            [[
                'python',
                'echo',
                'copy-existing-baselines-internal',
                '--verbose',
                '--test',
                'userscripts/first-test.html',
                '--suffixes',
                'png,txt',
                '--port-name',
                'test-win-win7',
            ]],
            [[
                'python',
                'echo',
                'rebaseline-test-internal',
                '--verbose',
                '--test',
                'userscripts/first-test.html',
                '--suffixes',
                'png,txt',
                '--port-name',
                'test-win-win7',
                '--builder',
                'MOCK Win7',
                '--results-directory',
                '/tmp',
                '--step-name',
                'blink_web_tests (with patch)',
            ]],
        ])
Exemplo n.º 4
0
    def test_rebaseline_handles_slow_in_file(self):
        self._write(self.test_expectations_path,
                    ('# tags: [ Linux Mac Mac10.10 Win ]\n'
                     '# results: [ Failure Slow ]\n'
                     '[ Linux ] userscripts/first-test.html [ Failure ]\n'
                     '[ Mac ] userscripts/first-test.html [ Failure ]\n'
                     '[ Win ] userscripts/first-test.html [ Failure Slow ]\n'))
        self._setup_mock_build_data()
        test_baseline_set = TestBaselineSet(self.tool)
        test_baseline_set.add('userscripts/first-test.html',
                              Build('MOCK Mac10.11'))

        self.command.rebaseline(self.options(), test_baseline_set)

        new_expectations = self._read(self.test_expectations_path)
        self.assertMultiLineEqual(
            new_expectations,
            ('# tags: [ Linux Mac Mac10.10 Win ]\n'
             '# results: [ Failure Slow ]\n'
             '[ Linux ] userscripts/first-test.html [ Failure ]\n'
             '[ Mac10.10 ] userscripts/first-test.html [ Failure ]\n'
             '[ Win ] userscripts/first-test.html [ Failure Slow ]\n'))
Exemplo n.º 5
0
 def test_get_step_name(self):
     fetcher = TestResultsFetcher()
     fetcher.builders.step_name_for_builder = lambda builder: None
     fetcher.web = MockWeb(
         urls={
             'https://test-results.appspot.com/testfile?buildnumber=5&'
             'callback=ADD_RESULTS&builder=foo&name=full_results.json':
             b'ADD_RESULTS(' +
             json.dumps([{
                 "TestType": "blink_web_tests (with patch)"
             }, {
                 "TestType":
                 "not_site_per_process_blink_web_tests (with patch)"
             }, {
                 "TestType": "blink_web_tests (retry with patch)"
             }, {
                 "TestType": "base_unittests (with patch)"
             }]).encode('utf8', 'replace') + b');'
         })
     step_name = fetcher.get_layout_test_step_name(Build('foo', 5))
     self.assertEqual(step_name, 'blink_web_tests (with patch)')
     self.assertLog([])
Exemplo n.º 6
0
 def test_try_job_results_skip_experimental_cq(self):
     git_cl = GitCL(MockHost())
     git_cl._host.web = MockWeb(responses=[{
         'status_code':
         200,
         'body':
         SEARCHBUILDS_RESPONSE_PREFIX + b"""{
                 "builds": [
                     {
                         "status": "SUCCESS",
                         "builder": {
                             "builder": "builder-a"
                         },
                         "number": 111,
                         "tags": [
                             {"key": "user_agent", "value": "cq"}
                         ]
                     },
                     {
                         "status": "SUCCESS",
                         "builder": {
                             "builder": "builder-b"
                         },
                         "number": 222,
                         "tags": [
                             {"key": "user_agent", "value": "cq"},
                             {"key": "cq_experimental", "value": "true"}
                         ]
                     }
                 ]
             }"""
     }])
     self.assertEqual(
         # Only one build appears - builder-b is ignored because it is
         # experimental.
         git_cl.try_job_results(issue_number=None, cq_only=True),
         {
             Build('builder-a', 111): TryJobStatus('COMPLETED', 'SUCCESS'),
         })
Exemplo n.º 7
0
 def test_fetch_results_with_weird_step_name(self):
     fetcher = TestResultsFetcher()
     fetcher.web = MockWeb(
         urls={
             'https://test-results.appspot.com/testfile?buildnumber=123&'
             'callback=ADD_RESULTS&builder=builder&name=full_results.json':
             'ADD_RESULTS(%s);' % (json.dumps(
                 [{
                     "TestType": "blink_web_tests on Intel GPU (with patch)"
                 }, {
                     "TestType": "base_unittests (with patch)"
                 }])),
             'https://test-results.appspot.com/data/layout_results/builder/123/'
             'blink_web_tests%20on%20Intel%20GPU%20%28with%20patch%29/'
             'layout-test-results/failing_results.json':
             json.dumps({'passed': True}),
         })
     results = fetcher.fetch_results(Build('builder', 123))
     self.assertEqual(
         results._results,  # pylint: disable=protected-access
         {'passed': True})
     self.assertLog([])
Exemplo n.º 8
0
    def execute(self, options, args, tool):
        self._tool = tool
        if not args:
            _log.error('Must list tests to rebaseline.')
            return

        if options.builders:
            builders_to_check = []
            for builder_names in options.builders:
                builders_to_check += builder_names.split(',')
        else:
            builders_to_check = self._builders_to_pull_from()

        test_baseline_set = TestBaselineSet(tool)

        for builder in builders_to_check:
            for test_prefix in args:
                test_baseline_set.add(test_prefix, Build(builder))

        _log.debug('Rebaselining: %s', test_baseline_set)

        self.rebaseline(options, test_baseline_set)
Exemplo n.º 9
0
    def test_rebaseline_handles_skips_in_file(self):
        # This test is like test_rebaseline_handles_platform_skips, except that the
        # Skip is in the same (generic) file rather than a platform file. In this case,
        # the Skip line should be left unmodified. Note that the first line is now
        # qualified as "[Linux Mac Win]"; if it was unqualified, it would conflict with
        # the second line.
        self._write(
            self.test_expectations_path,
            ('Bug(x) [ Linux Mac ] userscripts/first-test.html [ Failure ]\n'
             'Bug(y) [ Win ] userscripts/first-test.html [ Skip ]\n'))
        self._setup_mock_build_data()
        test_baseline_set = TestBaselineSet(self.tool)
        test_baseline_set.add('userscripts/first-test.html',
                              Build('MOCK Mac10.11'))

        self.command.rebaseline(self.options(), test_baseline_set)

        new_expectations = self._read(self.test_expectations_path)
        self.assertMultiLineEqual(
            new_expectations,
            ('Bug(x) [ Linux ] userscripts/first-test.html [ Failure ]\n'
             'Bug(x) [ Mac10.10 ] userscripts/first-test.html [ Failure ]\n'
             'Bug(y) [ Win ] userscripts/first-test.html [ Skip ]\n'))
Exemplo n.º 10
0
    def test_no_optimize(self):
        test_baseline_set = TestBaselineSet(self.tool)
        test_baseline_set.add('userscripts/first-test.html',
                              Build('MOCK Win7'))
        self.command.rebaseline(self.options(optimize=False),
                                test_baseline_set)

        self.assertEqual(self.tool.executive.calls,
                         [[[
                             'python',
                             'echo',
                             'copy-existing-baselines-internal',
                             '--verbose',
                             '--test',
                             'userscripts/first-test.html',
                             '--suffixes',
                             'txt,png',
                             '--port-name',
                             'test-win-win7',
                         ]],
                          [[
                              'python',
                              'echo',
                              'rebaseline-test-internal',
                              '--verbose',
                              '--test',
                              'userscripts/first-test.html',
                              '--suffixes',
                              'txt,png',
                              '--port-name',
                              'test-win-win7',
                              '--builder',
                              'MOCK Win7',
                              '--step-name',
                              'webkit_layout_tests (with patch)',
                          ]]])
Exemplo n.º 11
0
 def test_fill_in_missing_results_prefers_build_with_same_os_type(self):
     self.tool.builders = BuilderList({
         'MOCK Foo12': {
             'port_name': 'foo-foo12',
             'specifiers': ['Foo12', 'Release'],
             'is_try_builder': True,
         },
         'MOCK Foo45': {
             'port_name': 'foo-foo45',
             'specifiers': ['Foo45', 'Release'],
             'is_try_builder': True,
         },
         'MOCK Bar3': {
             'port_name': 'bar-bar3',
             'specifiers': ['Bar3', 'Release'],
             'is_try_builder': True,
         },
         'MOCK Bar4': {
             'port_name': 'bar-bar4',
             'specifiers': ['Bar4', 'Release'],
             'is_try_builder': True,
         },
     })
     test_baseline_set = TestBaselineSet(self.tool)
     test_baseline_set.add('one/flaky-fail.html', Build('MOCK Foo12', 100))
     test_baseline_set.add('one/flaky-fail.html', Build('MOCK Bar4', 200))
     self.command.fill_in_missing_results(test_baseline_set)
     self.assertEqual(
         sorted(test_baseline_set.build_port_pairs('one/flaky-fail.html')),
         [
             (Build('MOCK Bar4', 200), 'bar-bar3'),
             (Build('MOCK Bar4', 200), 'bar-bar4'),
             (Build('MOCK Foo12', 100), 'foo-foo12'),
             (Build('MOCK Foo12', 100), 'foo-foo45'),
         ])
     self.assertLog([
         'INFO: For one/flaky-fail.html:\n',
         'INFO: Using "MOCK Foo12" build 100 for foo-foo45.\n',
         'INFO: Using "MOCK Bar4" build 200 for bar-bar3.\n',
     ])
Exemplo n.º 12
0
 def test_get_step_name_without_build_number(self):
     fetcher = TestResultsFetcher()
     self.assertIsNone(
         fetcher.get_layout_test_step_name(Build('builder', None)))
Exemplo n.º 13
0
 def test_fetch_results_without_build_number(self):
     fetcher = TestResultsFetcher()
     self.assertIsNone(fetcher.fetch_results(Build('builder', None)))
Exemplo n.º 14
0
 def test_all_success_true(self):
     self.assertTrue(
         GitCL.all_success({
             Build('some-builder', 1):
             TryJobStatus('COMPLETED', 'SUCCESS'),
         }))
Exemplo n.º 15
0
    def setUp(self):
        BaseTestCase.setUp(self)
        LoggingTestCase.setUp(self)

        builds = {
            Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
            Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'),
            Build('MOCK Try Linux', 6000):
            TryJobStatus('COMPLETED', 'FAILURE'),
        }

        self.command.git_cl = MockGitCL(self.tool, builds)

        git = MockGit(filesystem=self.tool.filesystem,
                      executive=self.tool.executive)
        git.changed_files = lambda **_: [
            RELATIVE_WEB_TESTS + 'one/text-fail.html',
            RELATIVE_WEB_TESTS + 'one/flaky-fail.html',
        ]
        self.tool.git = lambda: git

        self.tool.builders = BuilderList({
            'MOCK Try Win': {
                'port_name': 'test-win-win7',
                'specifiers': ['Win7', 'Release'],
                'is_try_builder': True,
            },
            'MOCK Try Linux': {
                'port_name': 'test-linux-trusty',
                'specifiers': ['Trusty', 'Release'],
                'is_try_builder': True,
            },
            'MOCK Try Mac': {
                'port_name': 'test-mac-mac10.11',
                'specifiers': ['Mac10.11', 'Release'],
                'is_try_builder': True,
            },
        })
        web_test_results = WebTestResults({
            'tests': {
                'one': {
                    'crash.html': {
                        'expected': 'PASS',
                        'actual': 'CRASH',
                        'is_unexpected': True,
                        'artifacts': {
                            'crash_log': ['crash.log']
                        }
                    },
                    'expected-fail.html': {
                        'expected': 'FAIL',
                        'actual': 'FAIL',
                        'artifacts': {
                            'expected_text': ['expected-fail-expected.txt'],
                            'actual_text': ['expected-fail-actual.txt']
                        }
                    },
                    'flaky-fail.html': {
                        'expected': 'PASS',
                        'actual': 'PASS FAIL',
                        'is_unexpected': True,
                        'artifacts': {
                            'expected_audio': ['flaky-fail-expected.wav'],
                            'actual_audio': ['flaky-fail-actual.wav']
                        }
                    },
                    'missing.html': {
                        'expected': 'PASS',
                        'actual': 'FAIL',
                        'is_unexpected': True,
                        'artifacts': {
                            'actual_image': ['missing-actual.png']
                        },
                        'is_missing_image': True
                    },
                    'slow-fail.html': {
                        'expected': 'SLOW',
                        'actual': 'FAIL',
                        'is_unexpected': True,
                        'artifacts': {
                            'actual_text': ['slow-fail-actual.txt'],
                            'expected_text': ['slow-fail-expected.txt']
                        }
                    },
                    'text-fail.html': {
                        'expected': 'PASS',
                        'actual': 'FAIL',
                        'is_unexpected': True,
                        'artifacts': {
                            'actual_text': ['text-fail-actual.txt'],
                            'expected_text': ['text-fail-expected.txt']
                        }
                    },
                    'unexpected-pass.html': {
                        'expected': 'FAIL',
                        'actual': 'PASS',
                        'is_unexpected': True
                    },
                },
                'two': {
                    'image-fail.html': {
                        'expected': 'PASS',
                        'actual': 'FAIL',
                        'is_unexpected': True,
                        'artifacts': {
                            'actual_image': ['image-fail-actual.png'],
                            'expected_image': ['image-fail-expected.png']
                        }
                    }
                },
            },
        })

        for build in builds:
            self.tool.results_fetcher.set_results(build, web_test_results)
            self.tool.results_fetcher.set_retry_sumary_json(
                build,
                json.dumps({
                    'failures': [
                        'one/flaky-fail.html',
                        'one/missing.html',
                        'one/slow-fail.html',
                        'one/text-fail.html',
                        'two/image-fail.html',
                    ],
                    'ignored': [],
                }))

        # Write to the mock filesystem so that these tests are considered to exist.
        tests = [
            'one/flaky-fail.html',
            'one/missing.html',
            'one/slow-fail.html',
            'one/text-fail.html',
            'two/image-fail.html',
        ]
        for test in tests:
            path = self.mac_port.host.filesystem.join(
                self.mac_port.web_tests_dir(), test)
            self._write(path, 'contents')

        self.mac_port.host.filesystem.write_text_file(
            '/test.checkout/web_tests/external/wpt/MANIFEST.json', '{}')
Exemplo n.º 16
0
 def testUpdateTestExpectationsForWebview(self):
     host = self._setup_host()
     host.results_fetcher.set_results(
         Build('MOCK Android Pie', 123),
         WebTestResults({
             'tests': {
                 'abc.html': {
                     'expected': 'PASS',
                     'actual': 'CRASH TIMEOUT',
                     'is_unexpected': True,
                 },
                 'jkl.html': {
                     'expected': 'PASS',
                     'actual': 'FAIL',
                     'is_unexpected': True,
                 },
                 'cat.html': {
                     'expected': 'PASS',
                     'actual': 'CRASH CRASH TIMEOUT',
                     'is_unexpected': True,
                 },
             },
         }, step_name='system_webview_wpt (with patch)'),
         step_name='system_webview_wpt (with patch)')
     updater = AndroidWPTExpectationsUpdater(
         host, ['-vvv',  '--android-product', 'android_webview'])
     updater.git_cl = MockGitCL(host, {
         Build('MOCK Android Pie', 123):
         TryJobStatus('COMPLETED', 'FAILURE')})
     # Run command
     updater.run()
     # Get new expectations
     content = host.filesystem.read_text_file(
         PRODUCTS_TO_EXPECTATION_FILE_PATHS['android_webview'])
     self.assertEqual(
         content,
         ('# results: [ Failure Crash Timeout]\n'
          '\n'
          '# Add untriaged failures in this block\n'
          'crbug.com/1050754 external/wpt/abc.html [ Crash Failure Timeout ]\n'
          'crbug.com/1050754 external/wpt/cat.html [ Crash Failure Timeout ]\n'
          'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
          'external/wpt/dog.html [ Crash Timeout ]\n'
          'crbug.com/1050754 external/wpt/ghi.html [ Timeout ]\n'
          'crbug.com/1111111 crbug.com/1050754'
          ' external/wpt/jkl.html [ Failure ]\n'
          'crbug.com/6789043 external/wpt/van.html [ Failure ]\n'
          'external/wpt/www.html [ Crash Failure ]\n'
          '\n'
          '# This comment will not be deleted\n'
          'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n'))
     # check that chrome android's expectation file was not modified
     # since the same bot is used to update chrome android & webview
     # expectations
     self.assertEqual(
         host.filesystem.read_text_file(
             PRODUCTS_TO_EXPECTATION_FILE_PATHS['chrome_android']),
         self._raw_android_expectations)
     # Check logs
     logs = ''.join(self.logMessages()).lower()
     self.assertNotIn('weblayer', logs)
     self.assertNotIn('chrome', logs)
Exemplo n.º 17
0
 def testUpdateTestExpectationsForAll(self):
     host = self._setup_host()
     # Add results for Weblayer
     host.results_fetcher.set_results(
         Build('MOCK Android Weblayer - Pie', 123),
         WebTestResults({
             'tests': {
                 'abc.html': {
                     'expected': 'PASS',
                     'actual': 'CRASH TIMEOUT',
                     'is_unexpected': True,
                 },
                 'weblayer_only.html': {
                     'expected': 'PASS',
                     'actual': 'CRASH CRASH FAIL',
                     'is_unexpected': True,
                 },
             },
         }, step_name='weblayer_shell_wpt (with patch)'),
         step_name='weblayer_shell_wpt (with patch)')
     # Add Results for Webview
     host.results_fetcher.set_results(
         Build('MOCK Android Pie', 101),
         WebTestResults({
             'tests': {
                 'cat.html': {
                     'expected': 'PASS',
                     'actual': 'CRASH FAIL TIMEOUT',
                     'is_unexpected': True,
                 },
                 'webview_only.html': {
                     'expected': 'PASS',
                     'actual': 'TIMEOUT',
                     'is_unexpected': True,
                 },
             },
         }, step_name='system_webview_wpt (with patch)'),
         step_name='system_webview_wpt (with patch)')
     # Add Results for Chrome
     host.results_fetcher.set_results(
         Build('MOCK Android Pie', 101),
         WebTestResults({
             'tests': {
                 'jkl.html': {
                     'expected': 'PASS',
                     'actual': 'FAIL',
                     'is_unexpected': True,
                 },
                 'chrome_only.html': {
                     'expected': 'PASS',
                     'actual': 'CRASH CRASH TIMEOUT',
                     'is_unexpected': True,
                 },
             },
         }, step_name='chrome_public_wpt (with patch)'),
         step_name='chrome_public_wpt (with patch)')
     updater = AndroidWPTExpectationsUpdater(
         host, ['-vvv',
                '--android-product', 'android_weblayer',
                '--android-product', 'chrome_android',
                '--android-product', 'android_webview'])
     updater.git_cl = MockGitCL(host, {
         Build('MOCK Android Weblayer - Pie', 123):
         TryJobStatus('COMPLETED', 'FAILURE'),
         Build('MOCK Android Pie', 101):
         TryJobStatus('COMPLETED', 'FAILURE')})
     # Run command
     updater.run()
     # Check expectations for weblayer
     content = host.filesystem.read_text_file(
         PRODUCTS_TO_EXPECTATION_FILE_PATHS['android_weblayer'])
     self.assertEqual(
         content,
         ('# results: [ Failure Crash Timeout]\n'
          '\n'
          '# Add untriaged failures in this block\n'
          'crbug.com/1050754 external/wpt/abc.html [ Crash Failure Timeout ]\n'
          'crbug.com/1050754 external/wpt/cat.html [ Failure ]\n'
          'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
          'external/wpt/dog.html [ Crash Timeout ]\n'
          'crbug.com/1050754 external/wpt/ghi.html [ Timeout ]\n'
          'crbug.com/1111111 external/wpt/jkl.html [ Failure ]\n'
          'crbug.com/6789043 external/wpt/van.html [ Failure ]\n'
          'crbug.com/1050754 external/wpt/weblayer_only.html [ Failure Crash ]\n'
          'external/wpt/www.html [ Crash Failure ]\n'
          '\n'
          '# This comment will not be deleted\n'
          'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n'))
     # Check expectations for webview
     content = host.filesystem.read_text_file(
         PRODUCTS_TO_EXPECTATION_FILE_PATHS['android_webview'])
     self.assertEqual(
         content,
         ('# results: [ Failure Crash Timeout]\n'
          '\n'
          '# Add untriaged failures in this block\n'
          'external/wpt/abc.html [ Failure ]\n'
          'crbug.com/1050754 external/wpt/cat.html [ Crash Failure Timeout ]\n'
          'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
          'external/wpt/dog.html [ Crash Timeout ]\n'
          'crbug.com/1050754 external/wpt/ghi.html [ Timeout ]\n'
          'crbug.com/1111111 external/wpt/jkl.html [ Failure ]\n'
          'crbug.com/6789043 external/wpt/van.html [ Failure ]\n'
          'crbug.com/1050754 external/wpt/webview_only.html [ Timeout ]\n'
          'external/wpt/www.html [ Crash Failure ]\n'
          '\n'
          '# This comment will not be deleted\n'
          'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n'))
     # Check expectations chrome
     content = host.filesystem.read_text_file(
         PRODUCTS_TO_EXPECTATION_FILE_PATHS['chrome_android'])
     self.assertEqual(
         content,
         ('# results: [ Failure Crash Timeout]\n'
          '\n'
          '# Add untriaged failures in this block\n'
          'external/wpt/abc.html [ Failure ]\n'
          'crbug.com/1050754 external/wpt/cat.html [ Failure ]\n'
          'crbug.com/1050754 external/wpt/chrome_only.html [ Crash Timeout ]\n'
          'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
          'external/wpt/dog.html [ Crash Timeout ]\n'
          'crbug.com/1050754 external/wpt/ghi.html [ Timeout ]\n'
          'crbug.com/1111111 crbug.com/1050754'
          ' external/wpt/jkl.html [ Failure ]\n'
          'crbug.com/6789043 external/wpt/van.html [ Failure ]\n'
          'external/wpt/www.html [ Crash Failure ]\n'
          '\n'
          '# This comment will not be deleted\n'
          'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n'))
Exemplo n.º 18
0
 def test_filter_latest_jobs_higher_build_last(self):
     self.assertEqual(
         filter_latest_builds(
             [Build('foo', 3), Build('bar', 5), Build('foo', 5)]),
         [Build('bar', 5), Build('foo', 5)])
Exemplo n.º 19
0
 def test_latest_try_jobs_cq_only(self):
     git_cl = GitCL(MockHost())
     git_cl._host.web = MockWeb(responses=[{
         'status_code':
         200,
         'body':
         SEARCHBUILDS_RESPONSE_PREFIX + b"""{
                 "builds": [
                     {
                         "status": "SCHEDULED",
                         "builder": {
                             "builder": "cq-a"
                         },
                         "tags": [
                             {"key": "user_agent", "value": "cq"}
                         ]
                     },
                     {
                         "status": "SCHEDULED",
                         "builder": {
                             "builder": "cq-b"
                         },
                         "tags": [
                             {"key": "user_agent", "value": "cq"},
                             {"key": "cq_experimental", "value": "false"}
                         ]
                     },
                     {
                         "status": "SCHEDULED",
                         "builder": {
                             "builder": "cq-c"
                         },
                         "tags": [
                             {"key": "user_agent", "value": "cq"},
                             {"key": "cq_experimental", "value": "false"}
                         ]
                     },
                     {
                         "status": "SCHEDULED",
                         "builder": {
                             "builder": "cq-a-experimental"
                         },
                         "tags": [
                             {"key": "user_agent", "value": "cq"},
                             {"key": "cq_experimental", "value": "true"}
                         ]
                     },
                     {
                         "status": "SCHEDULED",
                         "builder": {
                             "builder": "cq-b-experimental"
                         },
                         "tags": [
                             {"key": "user_agent", "value": "cq"},
                             {"key": "cq_experimental", "value": "true"}
                         ]
                     },
                     {
                         "status": "SCHEDULED",
                         "builder": {
                             "builder": "other-a"
                         },
                         "tags": [
                             {"key": "user_agent", "value": "git_cl_try"}
                         ]
                     },
                     {
                         "status": "SCHEDULED",
                         "builder": {
                             "builder": "other-b"
                         },
                         "tags": [
                             {"key": "user_agent", "value": "git_cl_try"},
                             {"key": "cq_experimental", "value": "false"}
                         ]
                     }
                 ]
             }"""
     }])
     self.assertEqual(
         git_cl.latest_try_jobs(cq_only=True), {
             Build('cq-a'): TryJobStatus('SCHEDULED'),
             Build('cq-b'): TryJobStatus('SCHEDULED'),
             Build('cq-c'): TryJobStatus('SCHEDULED'),
         })
Exemplo n.º 20
0
 def test_fetch_webdriver_results_without_master(self):
     fetcher = TestResultsFetcher()
     self.assertIsNone(
         fetcher.fetch_webdriver_test_results(Build('builder', 1), ''))
     self.assertLog(
         ['DEBUG: Builder name or build number or master is None\n'])
Exemplo n.º 21
0
 def test_filter_latest_jobs_no_build_number(self):
     self.assertEqual(
         filter_latest_builds([Build('foo', 3),
                               Build('bar'),
                               Build('bar')]),
         [Build('bar'), Build('foo', 3)])
    def testCleanupAndUpdateTestExpectationsForAll(self):
        # Full integration test for expectations cleanup and update
        # using builder results.
        host = self._setup_host()
        # Add results for Weblayer
        host.results_fetcher.set_results(
            Build('MOCK Android Weblayer - Pie', 123),
            WebTestResults(
                {
                    'tests': {
                        'abc.html': {
                            'expected': 'PASS',
                            'actual': 'CRASH TIMEOUT',
                            'is_unexpected': True,
                        },
                        'weblayer_only.html': {
                            'expected': 'PASS',
                            'actual': 'CRASH CRASH FAIL',
                            'is_unexpected': True,
                        },
                        'disabled_weblayer_only.html': {
                            'expected': 'SKIP',
                            'actual': 'SKIP',
                            'is_unexpected': True,
                        },
                        'unexpected_pass.html': {
                            'expected': 'FAIL',
                            'actual': 'PASS',
                            'is_unexpected': True
                        },
                    },
                },
                step_name=WEBLAYER_WPT_STEP + ' (with patch)'),
            step_name=WEBLAYER_WPT_STEP + ' (with patch)')
        # Add Results for Webview
        host.results_fetcher.set_results(
            Build('MOCK Android Pie', 101),
            WebTestResults(
                {
                    'tests': {
                        'cat.html': {
                            'expected': 'PASS',
                            'actual': 'CRASH FAIL TIMEOUT',
                            'is_unexpected': True,
                        },
                        'webview_only.html': {
                            'expected': 'PASS',
                            'actual': 'TIMEOUT',
                            'is_unexpected': True,
                        },
                        'disabled.html': {
                            'expected': 'SKIP',
                            'actual': 'SKIP',
                        },
                        'unexpected_pass.html': {
                            'expected': 'FAIL',
                            'actual': 'PASS',
                            'is_unexpected': True
                        },
                    },
                },
                step_name=WEBVIEW_WPT_STEP + ' (with patch)'),
            step_name=WEBVIEW_WPT_STEP + ' (with patch)')
        # Add Results for Chrome
        host.results_fetcher.set_results(
            Build('MOCK Android Pie', 101),
            WebTestResults(
                {
                    'tests': {
                        'jkl.html': {
                            'expected': 'PASS',
                            'actual': 'FAIL',
                            'is_unexpected': True,
                        },
                        'chrome_only.html': {
                            'expected': 'PASS',
                            'actual': 'CRASH CRASH TIMEOUT',
                            'is_unexpected': True,
                        },
                        'disabled.html': {
                            'expected': 'SKIP',
                            'actual': 'SKIP',
                            'is_unexpected': True,
                        },
                        'unexpected_pass.html': {
                            'expected': 'FAIL',
                            'actual': 'PASS',
                            'is_unexpected': True
                        },
                    },
                },
                step_name=CHROME_ANDROID_WPT_STEP + ' (with patch)'),
            step_name=CHROME_ANDROID_WPT_STEP + ' (with patch)')
        updater = AndroidWPTExpectationsUpdater(host, [
            '-vvv', '--clean-up-test-expectations',
            '--clean-up-affected-tests-only', '--include-unexpected-pass',
            '--android-product', ANDROID_WEBLAYER, '--android-product',
            CHROME_ANDROID, '--android-product', ANDROID_WEBVIEW
        ])

        def _git_command_return_val(cmd):
            if '--diff-filter=D' in cmd:
                return 'external/wpt/ghi.html'
            if '--diff-filter=R' in cmd:
                return 'C external/wpt/van.html external/wpt/wagon.html'
            return ''

        updater.git_cl = MockGitCL(
            host, {
                Build('MOCK Android Weblayer - Pie', 123):
                TryJobStatus('COMPLETED', 'FAILURE'),
                Build('MOCK Android Pie', 101):
                TryJobStatus('COMPLETED', 'FAILURE')
            })

        updater.git.run = _git_command_return_val
        updater._relative_to_web_test_dir = lambda test_path: test_path

        # Run command
        updater.run()
        # Check expectations for weblayer
        content = host.filesystem.read_text_file(
            PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBLAYER])
        self.assertEqual(content, (
            '# results: [ Failure Crash Timeout]\n'
            '\n'
            '# Add untriaged failures in this block\n'
            'crbug.com/1050754 external/wpt/abc.html [ Crash Failure Timeout ]\n'
            'crbug.com/1050754 external/wpt/cat.html [ Failure ]\n'
            'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
            'external/wpt/dog.html [ Crash Timeout ]\n'
            'crbug.com/1111111 external/wpt/jkl.html [ Failure ]\n'
            'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n'
            'crbug.com/6789043 external/wpt/wagon.html [ Failure ]\n'
            'crbug.com/1050754 external/wpt/weblayer_only.html [ Failure Crash ]\n'
            'external/wpt/www.html [ Crash Failure ]\n'
            '\n'
            '# This comment will not be deleted\n'
            'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n'))
        # Check expectations for webview
        content = host.filesystem.read_text_file(
            PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBVIEW])
        self.assertEqual(content, (
            '# results: [ Failure Crash Timeout]\n'
            '\n'
            '# Add untriaged failures in this block\n'
            'external/wpt/abc.html [ Failure ]\n'
            'crbug.com/1050754 external/wpt/cat.html [ Crash Failure Timeout ]\n'
            'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
            'external/wpt/dog.html [ Crash Timeout ]\n'
            'crbug.com/1111111 external/wpt/jkl.html [ Failure ]\n'
            'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n'
            'crbug.com/6789043 external/wpt/wagon.html [ Failure ]\n'
            'crbug.com/1050754 external/wpt/webview_only.html [ Timeout ]\n'
            'external/wpt/www.html [ Crash Failure ]\n'
            '\n'
            '# This comment will not be deleted\n'
            'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n'))
        # Check expectations chrome
        content = host.filesystem.read_text_file(
            PRODUCTS_TO_EXPECTATION_FILE_PATHS[CHROME_ANDROID])
        self.assertEqual(content, (
            '# results: [ Failure Crash Timeout]\n'
            '\n'
            '# Add untriaged failures in this block\n'
            'external/wpt/abc.html [ Failure ]\n'
            'crbug.com/1050754 external/wpt/cat.html [ Failure ]\n'
            'crbug.com/1050754 external/wpt/chrome_only.html [ Crash Timeout ]\n'
            'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
            'external/wpt/dog.html [ Crash Timeout ]\n'
            'crbug.com/1111111 crbug.com/1050754'
            ' external/wpt/jkl.html [ Failure ]\n'
            'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n'
            'crbug.com/6789043 external/wpt/wagon.html [ Failure ]\n'
            'external/wpt/www.html [ Crash Failure ]\n'
            '\n'
            '# This comment will not be deleted\n'
            'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n'))
        # Check disabled test file
        neverfix_content = host.filesystem.read_text_file(
            ANDROID_DISABLED_TESTS)
        self.assertEqual(neverfix_content, (
            '# tags: [ android-weblayer android-webview chrome-android ]\n'
            '# results: [ Skip ]\n'
            '\n'
            '# Add untriaged disabled tests in this block\n'
            'crbug.com/1050754 [ android-webview ] external/wpt/disabled.html [ Skip ]\n'
            'crbug.com/1050754 [ chrome-android ] external/wpt/disabled.html [ Skip ]\n'
            'crbug.com/1050754 [ android-weblayer ] external/wpt/disabled_weblayer_only.html [ Skip ]\n'
        ))
Exemplo n.º 23
0
    def test_run_single_platform_failure(self):
        """Tests the main run method in a case where one test fails on one platform."""
        host = self.mock_host()

        # Fill in an initial value for TestExpectations
        expectations_path = \
            host.port_factory.get().path_to_generic_test_expectations_file()
        host.filesystem.write_text_file(
            expectations_path, WPTExpectationsUpdater.MARKER_COMMENT + '\n')

        # Set up fake try job results.
        updater = WPTExpectationsUpdater(host)
        updater.git_cl = MockGitCL(
            updater.host, {
                Build('MOCK Try Mac10.10', 333):
                TryJobStatus('COMPLETED', 'FAILURE'),
                Build('MOCK Try Mac10.11', 111):
                TryJobStatus('COMPLETED', 'SUCCESS'),
                Build('MOCK Try Trusty', 222):
                TryJobStatus('COMPLETED', 'SUCCESS'),
                Build('MOCK Try Precise', 333):
                TryJobStatus('COMPLETED', 'SUCCESS'),
                Build('MOCK Try Win10', 444):
                TryJobStatus('COMPLETED', 'SUCCESS'),
                Build('MOCK Try Win7', 555):
                TryJobStatus('COMPLETED', 'SUCCESS'),
            })

        # Set up failing results for one try bot. It shouldn't matter what
        # results are for the other builders since we shouldn't need to even
        # fetch results, since the try job status already tells us that all
        # of the tests passed.
        host.results_fetcher.set_results(
            Build('MOCK Try Mac10.10', 333),
            WebTestResults({
                'tests': {
                    'external': {
                        'wpt': {
                            'test': {
                                'path.html': {
                                    'expected': 'PASS',
                                    'actual': 'TIMEOUT',
                                    'is_unexpected': True,
                                }
                            }
                        }
                    }
                }
            }))
        self.assertEqual(0, updater.run())

        # Results are only fetched for failing builds.
        self.assertEqual(host.results_fetcher.fetched_builds, [
            BuilderStep(Build('MOCK Try Mac10.10', 333),
                        'blink_web_tests (with patch)')
        ])

        self.assertEqual(
            host.filesystem.read_text_file(expectations_path),
            '# ====== New tests from wpt-importer added here ======\n'
            'crbug.com/626703 [ Mac10.10 ] external/wpt/test/path.html [ Timeout ]\n'
        )
    def testCleanupAndUpdateTestExpectationsForAll(self):
        # Full integration test for expectations cleanup and update
        # using builder results.
        raw_android_expectations = (
            '# results: [ Failure Crash Timeout]\n'
            '\n'
            'crbug.com/1000754 external/wpt/foo1.html [ Failure ]\n'
            'crbug.com/1000754 external/wpt/foo2.html [ Failure ]\n'
            'crbug.com/1000754 external/wpt/bar.html [ Failure ]\n'
            '\n'
            '# Add untriaged failures in this block\n'
            '\n'
            '# This comment will not be deleted\n')
        host = self._setup_host(raw_android_expectations)
        # Add results for Weblayer
        host.results_fetcher.set_results(
            Build('MOCK Android Weblayer - Pie', 123),
            WebTestResults({
                'tests': {
                    'disabled_weblayer_only.html': {
                        'expected': 'SKIP',
                        'actual': 'SKIP',
                        'is_unexpected': True,
                    },
                    'bar.html': {
                        'expected': 'FAIL',
                        'actual': 'CRASH',
                        'is_unexpected': True
                    },
                },
            }, step_name=WEBLAYER_WPT_STEP + ' (with patch)'),
            step_name=WEBLAYER_WPT_STEP + ' (with patch)')
        updater = AndroidWPTExpectationsUpdater(
            host, ['-vvv',
                   '--clean-up-test-expectations',
                   '--clean-up-affected-tests-only',
                   '--include-unexpected-pass',
                   '--android-product', ANDROID_WEBLAYER])

        def _git_command_return_val(cmd):
            if '--diff-filter=D' in cmd:
                return 'external/wpt/foo2.html'
            if '--diff-filter=R' in cmd:
                return 'C\texternal/wpt/foo1.html\texternal/wpt/foo3.html'
            if '--diff-filter=M' in cmd:
                return 'external/wpt/bar.html'
            return ''

        updater.git_cl = MockGitCL(host, {
            Build('MOCK Android Weblayer - Pie', 123):
            TryJobStatus('COMPLETED', 'FAILURE')})

        updater.git.run = _git_command_return_val
        updater._relative_to_web_test_dir = lambda test_path: test_path

        # Run command
        updater.run()

        # Check expectations for weblayer
        content = host.filesystem.read_text_file(
            PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBLAYER])
        _new_expectations = (
            '# results: [ Failure Crash Timeout]\n'
            '\n'
            'crbug.com/1000754 external/wpt/foo3.html [ Failure ]\n'
            '\n'
            '# Add untriaged failures in this block\n'
            'crbug.com/1050754 external/wpt/bar.html [ Crash ]\n'
            '\n'
            '# This comment will not be deleted\n')
        self.assertEqual(content, _new_expectations)

        # Check disabled test file
        neverfix_content = host.filesystem.read_text_file(ANDROID_DISABLED_TESTS)
        self.assertEqual(
            neverfix_content,
            ('# tags: [ android-weblayer android-webview chrome-android ]\n'
             '# results: [ Skip ]\n'
             '\n'
             '# Add untriaged disabled tests in this block\n'
             'crbug.com/1050754 [ android-webview ] external/wpt/disabled.html [ Skip ]\n'
             'crbug.com/1050754 [ android-weblayer ] external/wpt/disabled_weblayer_only.html [ Skip ]\n'))
Exemplo n.º 25
0
    def test_get_failing_results_dict_webdriver_failing_results_(self):
        host = self.mock_host()
        host.results_fetcher.set_results(
            Build('MOCK Try Trusty', 123),
            WebTestResults({
                'tests': {
                    'external': {
                        'wpt': {
                            'x': {
                                'failing-test.html': {
                                    'expected': 'PASS',
                                    'actual': 'IMAGE',
                                    'is_unexpected': True,
                                },
                            },
                        },
                    },
                },
            }))

        host.results_fetcher.set_webdriver_test_results(
            Build('MOCK Try Trusty', 123), "tryserver.blink",
            WebTestResults({
                'tests': {
                    'external': {
                        'wpt': {
                            'y': {
                                'webdriver-fail.html': {
                                    'expected': 'PASS',
                                    'actual': 'FAIL',
                                    'is_unexpected': True,
                                },
                            },
                        },
                    },
                },
            }))
        updater = WPTExpectationsUpdater(host)
        results_dict = updater.get_failing_results_dict(
            Build('MOCK Try Trusty', 123))

        self.assertEqual(len(results_dict.keys()), 2)
        self.assertEqual(
            results_dict, {
                'external/wpt/x/failing-test.html': {
                    'test-linux-trusty':
                    SimpleTestResult(
                        actual='IMAGE',
                        expected='PASS',
                        bug='crbug.com/626703',
                    ),
                },
                'external/wpt/y/webdriver-fail.html': {
                    'test-linux-trusty':
                    SimpleTestResult(
                        actual='FAIL',
                        expected='PASS',
                        bug='crbug.com/626703',
                    ),
                },
            })
    def testUpdateTestExpectationsForWeblayer(self):
        raw_android_expectations = (
            '# results: [ Failure Crash Timeout]\n'
            '\n'
            'crbug.com/1000754 external/wpt/foo.html [ Failure ]\n'
            '\n'
            '# Add untriaged failures in this block\n'
            'crbug.com/1050754 external/wpt/bar.html [ Failure ]\n'
            '\n'
            '# This comment will not be deleted\n')
        host = self._setup_host(raw_android_expectations)
        host.results_fetcher.set_results(
            Build('MOCK Android Weblayer - Pie', 123),
            WebTestResults({
                'tests': {
                    # A test result covered by default expectation
                    'new1.html': {
                        'expected': 'PASS',
                        'actual': 'FAIL',
                        'is_unexpected': True,
                    },
                    # A test result covered by baseline
                    'new2.html': {
                        'expected': 'PASS',
                        'actual': 'FAIL',
                        'is_unexpected': True,
                    },
                    # A new test case
                    'new3.html': {
                        'expected': 'PASS',
                        'actual': 'CRASH CRASH FAIL',
                        'is_unexpected': True,
                    },
                },
            }, step_name=WEBLAYER_WPT_STEP + ' (with patch)'),
            step_name=WEBLAYER_WPT_STEP + ' (with patch)')
        updater = AndroidWPTExpectationsUpdater(
            host, ['-vvv', '--android-product', ANDROID_WEBLAYER,
                   '--include-unexpected-pass'])
        updater.git_cl = MockGitCL(host, {
            Build('MOCK Android Weblayer - Pie', 123):
            TryJobStatus('COMPLETED', 'FAILURE')})
        # Run command
        updater.run()
        # Get new expectations
        content = host.filesystem.read_text_file(
            PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBLAYER])
        _new_expectations = (
            '# results: [ Failure Crash Timeout]\n'
            '\n'
            'crbug.com/1000754 external/wpt/foo.html [ Failure ]\n'
            '\n'
            '# Add untriaged failures in this block\n'
            'crbug.com/1050754 external/wpt/bar.html [ Failure ]\n'
            'crbug.com/1050754 external/wpt/new3.html [ Crash Failure ]\n'
            '\n'
            '# This comment will not be deleted\n')
        self.assertEqual(content, _new_expectations)

        # Check that ANDROID_DISABLED_TESTS expectation files were not changed
        self.assertEqual(
            self._raw_android_never_fix_tests,
            host.filesystem.read_text_file(ANDROID_DISABLED_TESTS))
 def testUpdateTestExpectationsForWebview(self):
     host = self._setup_host()
     host.results_fetcher.set_results(
         Build('MOCK Android Pie', 123),
         WebTestResults(
             {
                 'tests': {
                     'abc.html': {
                         'expected': 'PASS',
                         'actual': 'CRASH TIMEOUT',
                         'is_unexpected': True,
                     },
                     'jkl.html': {
                         'expected': 'PASS',
                         'actual': 'FAIL',
                         'is_unexpected': True,
                     },
                     'cat.html': {
                         'expected': 'PASS',
                         'actual': 'CRASH CRASH TIMEOUT',
                         'is_unexpected': True,
                     },
                     'unexpected_pass.html': {
                         'expected': 'FAIL',
                         'actual': 'PASS',
                         'is_unexpected': True
                     },
                     'dog.html': {
                         'expected': 'SKIP',
                         'actual': 'SKIP',
                         'is_unexpected': True,
                     },
                 },
             },
             step_name=WEBVIEW_WPT_STEP + ' (with patch)'),
         step_name=WEBVIEW_WPT_STEP + ' (with patch)')
     updater = AndroidWPTExpectationsUpdater(host, [
         '-vvv', '--android-product', ANDROID_WEBVIEW,
         '--clean-up-test-expectations', '--clean-up-affected-tests-only',
         '--include-unexpected-pass'
     ])
     updater.git_cl = MockGitCL(host, {
         Build('MOCK Android Pie', 123):
         TryJobStatus('COMPLETED', 'FAILURE')
     })
     # Run command
     updater.run()
     # Get new expectations
     content = host.filesystem.read_text_file(
         PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBVIEW])
     self.assertEqual(content, (
         '# results: [ Failure Crash Timeout]\n'
         '\n'
         '# Add untriaged failures in this block\n'
         'crbug.com/1050754 external/wpt/abc.html [ Crash Failure Timeout ]\n'
         'crbug.com/1050754 external/wpt/cat.html [ Crash Failure Timeout ]\n'
         'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
         'external/wpt/dog.html [ Crash Timeout ]\n'
         'crbug.com/1050754 external/wpt/ghi.html [ Timeout ]\n'
         'crbug.com/1111111 crbug.com/1050754'
         ' external/wpt/jkl.html [ Failure ]\n'
         'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n'
         'crbug.com/6789043 external/wpt/van.html [ Failure ]\n'
         'external/wpt/www.html [ Crash Failure ]\n'
         '\n'
         '# This comment will not be deleted\n'
         'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n'))
     neverfix_content = host.filesystem.read_text_file(
         ANDROID_DISABLED_TESTS)
     self.assertEqual(neverfix_content, (
         '# tags: [ android-weblayer android-webview chrome-android ]\n'
         '# results: [ Skip ]\n'
         '\n'
         '# Add untriaged disabled tests in this block\n'
         'crbug.com/1050754 [ android-webview ] external/wpt/disabled.html [ Skip ]\n'
         'crbug.com/1050754 [ android-webview ] external/wpt/dog.html [ Skip ]\n'
     ))
     # check that chrome android's expectation file was not modified
     # since the same bot is used to update chrome android & webview
     # expectations
     self.assertEqual(
         host.filesystem.read_text_file(
             PRODUCTS_TO_EXPECTATION_FILE_PATHS[CHROME_ANDROID]),
         self._raw_android_expectations)
     # Check logs
     logs = ''.join(self.logMessages()).lower()
     self.assertNotIn(WEBLAYER_WPT_STEP, logs)
     self.assertNotIn(CHROME_ANDROID_WPT_STEP, logs)
     # Check that weblayer and chrome expectation files were not changed
     self.assertEqual(
         self._raw_android_expectations,
         host.filesystem.read_text_file(
             PRODUCTS_TO_EXPECTATION_FILE_PATHS[CHROME_ANDROID]))
     self.assertEqual(
         self._raw_android_expectations,
         host.filesystem.read_text_file(
             PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBLAYER]))
Exemplo n.º 28
0
 def test_has_failing_try_results_with_failing_results(self):
     self.assertTrue(
         GitCL.some_failed({
             Build('some-builder', 1):
             TryJobStatus('COMPLETED', 'FAILURE'),
         }))