def _gather_baselines(self, results_json):
        # Rebaseline server and it's associated JavaScript expected the tests subtree to
        # be key-value pairs instead of hierarchical.
        # FIXME: make the rebaseline server use the hierarchical tree.
        new_tests_subtree = {}

        def gather_baselines_for_test(test_name, result_dict):
            result = JSONTestResult(test_name, result_dict)
            if result.did_pass_or_run_as_expected():
                return
            result_dict['state'] = STATE_NEEDS_REBASELINE
            result_dict['baselines'] = get_test_baselines(test_name, self._test_config)
            new_tests_subtree[test_name] = result_dict

        for_each_test(results_json['tests'], gather_baselines_for_test)
        results_json['tests'] = new_tests_subtree
    def _gather_baselines(self, results_json):
        # Rebaseline server and it's associated JavaScript expected the tests subtree to
        # be key-value pairs instead of hierarchical.
        # FIXME: make the rebaseline server use the hierarchical tree.
        new_tests_subtree = {}

        def gather_baselines_for_test(test_name, result_dict):
            result = JSONTestResult(test_name, result_dict)
            if result.did_pass_or_run_as_expected():
                return
            result_dict["state"] = STATE_NEEDS_REBASELINE
            result_dict["baselines"] = get_test_baselines(test_name, self._test_config)
            new_tests_subtree[test_name] = result_dict

        for_each_test(results_json["tests"], gather_baselines_for_test)
        results_json["tests"] = new_tests_subtree
Esempio n. 3
0
    def print_unexpected_results(self, summarized_results, enabled_pixel_tests_in_retry=False):
        passes = {}
        flaky = {}
        regressions = {}

        def add_to_dict_of_lists(dict, key, value):
            dict.setdefault(key, []).append(value)

        def add_result(test, results, passes=passes, flaky=flaky, regressions=regressions):
            actual = results['actual'].split(" ")
            expected = results['expected'].split(" ")

            if 'is_unexpected' not in results or not results['is_unexpected']:
                # Don't print anything for tests that ran as expected.
                return

            if actual == ['PASS']:
                if 'CRASH' in expected:
                    add_to_dict_of_lists(passes, 'Expected to crash, but passed', test)
                elif 'TIMEOUT' in expected:
                    add_to_dict_of_lists(passes, 'Expected to timeout, but passed', test)
                else:
                    add_to_dict_of_lists(passes, 'Expected to fail, but passed', test)
            elif enabled_pixel_tests_in_retry and actual == ['TEXT', 'IMAGE+TEXT']:
                add_to_dict_of_lists(regressions, actual[0], test)
            elif len(actual) > 1 and actual[-1] in expected:
                # We group flaky tests by the first actual result we got.
                add_to_dict_of_lists(flaky, actual[0], test)
            else:
                add_to_dict_of_lists(regressions, actual[0], test)

        layouttestresults.for_each_test(summarized_results['tests'], add_result)

        if len(passes) or len(flaky) or len(regressions):
            self._print("")
        if len(passes):
            for key, tests in passes.iteritems():
                self._print("%s: (%d)" % (key, len(tests)))
                tests.sort()
                for test in tests:
                    self._print("  %s" % test)
                self._print("")
            self._print("")

        if len(flaky):
            descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS
            for key, tests in flaky.iteritems():
                result_type = TestExpectations.EXPECTATIONS[key.lower()]
                self._print("Unexpected flakiness: %s (%d)" % (descriptions[result_type], len(tests)))
                tests.sort()

                for test in tests:
                    result = layouttestresults.result_for_test(summarized_results['tests'], test)
                    actual = result['actual'].split(" ")
                    expected = result['expected'].split(" ")
                    # FIXME: clean this up once the old syntax is gone
                    new_expectations_list = [TestExpectationParser._inverted_expectation_tokens[exp] for exp in list(set(actual) | set(expected))]
                    self._print("  %s [ %s ]" % (test, " ".join(new_expectations_list)))
                self._print("")
            self._print("")

        if len(regressions):
            descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS
            for key, tests in regressions.iteritems():
                result_type = TestExpectations.EXPECTATIONS[key.lower()]
                self._print("Regressions: Unexpected %s (%d)" % (descriptions[result_type], len(tests)))
                tests.sort()
                for test in tests:
                    result = layouttestresults.result_for_test(summarized_results['tests'], test)
                    actual = result['actual'].split(" ")
                    expected = result['expected'].split(" ")
                    new_expectations_list = [TestExpectationParser._inverted_expectation_tokens[exp] for exp in actual]
                    self._print("  %s [ %s ]" % (test, " ".join(new_expectations_list)))
                self._print("")

        if len(summarized_results['tests']) and self.debug_logging:
            self._print("%s" % ("-" * 78))
    def print_unexpected_results(self, summarized_results, enabled_pixel_tests_in_retry=False):
        passes = {}
        flaky = {}
        regressions = {}

        def add_to_dict_of_lists(dict, key, value):
            dict.setdefault(key, []).append(value)

        def add_result(test, results, passes=passes, flaky=flaky, regressions=regressions):
            actual = results['actual'].split(" ")
            expected = results['expected'].split(" ")

            if 'is_unexpected' not in results or not results['is_unexpected']:
                # Don't print anything for tests that ran as expected.
                return

            if actual == ['PASS']:
                if 'CRASH' in expected:
                    add_to_dict_of_lists(passes, 'Expected to crash, but passed', test)
                elif 'TIMEOUT' in expected:
                    add_to_dict_of_lists(passes, 'Expected to timeout, but passed', test)
                else:
                    add_to_dict_of_lists(passes, 'Expected to fail, but passed', test)
            elif enabled_pixel_tests_in_retry and actual == ['TEXT', 'IMAGE+TEXT']:
                add_to_dict_of_lists(regressions, actual[0], test)
            elif len(actual) > 1:
                # We group flaky tests by the first actual result we got.
                add_to_dict_of_lists(flaky, actual[0], test)
            else:
                add_to_dict_of_lists(regressions, results['actual'], test)

        layouttestresults.for_each_test(summarized_results['tests'], add_result)

        if len(passes) or len(flaky) or len(regressions):
            self._print("")
        if len(passes):
            for key, tests in passes.iteritems():
                self._print("%s: (%d)" % (key, len(tests)))
                tests.sort()
                for test in tests:
                    self._print("  %s" % test)
                self._print("")
            self._print("")

        if len(flaky):
            descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS
            for key, tests in flaky.iteritems():
                result = TestExpectations.EXPECTATIONS[key.lower()]
                self._print("Unexpected flakiness: %s (%d)" % (descriptions[result], len(tests)))
                tests.sort()

                for test in tests:
                    result = layouttestresults.result_for_test(summarized_results['tests'], test)
                    actual = result['actual'].split(" ")
                    expected = result['expected'].split(" ")
                    result = TestExpectations.EXPECTATIONS[key.lower()]
                    # FIXME: clean this up once the old syntax is gone
                    new_expectations_list = [TestExpectationParser._inverted_expectation_tokens[exp] for exp in list(set(actual) | set(expected))]
                    self._print("  %s [ %s ]" % (test, " ".join(new_expectations_list)))
                self._print("")
            self._print("")

        if len(regressions):
            descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS
            for key, tests in regressions.iteritems():
                result = TestExpectations.EXPECTATIONS[key.lower()]
                self._print("Regressions: Unexpected %s (%d)" % (descriptions[result], len(tests)))
                tests.sort()
                for test in tests:
                    self._print("  %s [ %s ]" % (test, TestExpectationParser._inverted_expectation_tokens[key]))
                self._print("")

        if len(summarized_results['tests']) and self.debug_logging:
            self._print("%s" % ("-" * 78))