def _gather_unexpected_results(port): """Returns the unexpected results from the previous run, if any.""" filesystem = port._filesystem results_directory = port.results_directory() options = port._options last_unexpected_results = [] if options.print_last_failures or options.retest_last_failures: unexpected_results_filename = filesystem.join(results_directory, "unexpected_results.json") if filesystem.exists(unexpected_results_filename): results = json_results_generator.load_json(filesystem, unexpected_results_filename) resultsjsonparser.for_each_test(results['tests'], lambda test, result: last_unexpected_results.append(test)) return last_unexpected_results
def _gather_baselines(self, results_json): # Rebaseline server and it's associated JavaScript expected the tests subtree to # be key-value pairs instead of hierarchical. # FIXME: make the rebaseline server use the hierarchical tree. new_tests_subtree = {} def gather_baselines_for_test(test_name, result_dict): result = JSONTestResult(test_name, result_dict) if result.did_pass_or_run_as_expected(): return result_dict['state'] = STATE_NEEDS_REBASELINE result_dict['baselines'] = get_test_baselines(test_name, self._test_config) new_tests_subtree[test_name] = result_dict for_each_test(results_json['tests'], gather_baselines_for_test) results_json['tests'] = new_tests_subtree
def _gather_baselines(self, results_json): # Rebaseline server and it's associated JavaScript expected the tests subtree to # be key-value pairs instead of hierarchical. # FIXME: make the rebaseline server use the hierarchical tree. new_tests_subtree = {} def gather_baselines_for_test(test_name, result_dict): result = JSONTestResult(test_name, result_dict) if result.did_pass_or_run_as_expected(): return result_dict["state"] = STATE_NEEDS_REBASELINE result_dict["baselines"] = get_test_baselines(test_name, self._test_config) new_tests_subtree[test_name] = result_dict for_each_test(results_json["tests"], gather_baselines_for_test) results_json["tests"] = new_tests_subtree
def print_unexpected_results(self, summarized_results, enabled_pixel_tests_in_retry=False): passes = {} flaky = {} regressions = {} def add_to_dict_of_lists(dict, key, value): dict.setdefault(key, []).append(value) def add_result(test, results, passes=passes, flaky=flaky, regressions=regressions): actual = results['actual'].split(" ") expected = results['expected'].split(" ") if 'is_unexpected' not in results or not results['is_unexpected']: # Don't print anything for tests that ran as expected. return if actual == ['PASS']: if 'CRASH' in expected: add_to_dict_of_lists(passes, 'Expected to crash, but passed', test) elif 'TIMEOUT' in expected: add_to_dict_of_lists(passes, 'Expected to timeout, but passed', test) else: add_to_dict_of_lists(passes, 'Expected to fail, but passed', test) elif enabled_pixel_tests_in_retry and actual == [ 'TEXT', 'IMAGE+TEXT' ]: add_to_dict_of_lists(regressions, actual[0], test) elif len(actual) > 1: # We group flaky tests by the first actual result we got. add_to_dict_of_lists(flaky, actual[0], test) else: add_to_dict_of_lists(regressions, results['actual'], test) resultsjsonparser.for_each_test(summarized_results['tests'], add_result) if len(passes) or len(flaky) or len(regressions): self._print("") if len(passes): for key, tests in passes.iteritems(): self._print("%s: (%d)" % (key, len(tests))) tests.sort() for test in tests: self._print(" %s" % test) self._print("") self._print("") if len(flaky): descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS for key, tests in flaky.iteritems(): result = TestExpectations.EXPECTATIONS[key.lower()] self._print("Unexpected flakiness: %s (%d)" % (descriptions[result], len(tests))) tests.sort() for test in tests: result = resultsjsonparser.result_for_test( summarized_results['tests'], test) actual = result['actual'].split(" ") expected = result['expected'].split(" ") result = TestExpectations.EXPECTATIONS[key.lower()] # FIXME: clean this up once the old syntax is gone new_expectations_list = [ TestExpectationParser._inverted_expectation_tokens[exp] for exp in list(set(actual) | set(expected)) ] self._print(" %s [ %s ]" % (test, " ".join(new_expectations_list))) self._print("") self._print("") if len(regressions): descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS for key, tests in regressions.iteritems(): result = TestExpectations.EXPECTATIONS[key.lower()] self._print("Regressions: Unexpected %s (%d)" % (descriptions[result], len(tests))) tests.sort() for test in tests: self._print(" %s [ %s ]" % (test, TestExpectationParser. _inverted_expectation_tokens[key])) self._print("") if len(summarized_results['tests']) and self.debug_logging: self._print("%s" % ("-" * 78))
def _print_unexpected_results(self, unexpected_results): # Prints to the buildbot stream passes = {} flaky = {} regressions = {} def add_to_dict_of_lists(dict, key, value): dict.setdefault(key, []).append(value) def add_result(test, results, passes=passes, flaky=flaky, regressions=regressions): actual = results["actual"].split(" ") expected = results["expected"].split(" ") if actual == ["PASS"]: if "CRASH" in expected: add_to_dict_of_lists(passes, "Expected to crash, but passed", test) elif "TIMEOUT" in expected: add_to_dict_of_lists(passes, "Expected to timeout, but passed", test) else: add_to_dict_of_lists(passes, "Expected to fail, but passed", test) elif len(actual) > 1: # We group flaky tests by the first actual result we got. add_to_dict_of_lists(flaky, actual[0], test) else: add_to_dict_of_lists(regressions, results["actual"], test) resultsjsonparser.for_each_test(unexpected_results["tests"], add_result) if len(passes) or len(flaky) or len(regressions): self._print_for_bot("") if len(passes): for key, tests in passes.iteritems(): self._print_for_bot("%s: (%d)" % (key, len(tests))) tests.sort() for test in tests: self._print_for_bot(" %s" % test) self._print_for_bot("") self._print_for_bot("") if len(flaky): descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS for key, tests in flaky.iteritems(): result = TestExpectations.EXPECTATIONS[key.lower()] self._print_for_bot("Unexpected flakiness: %s (%d)" % (descriptions[result][0], len(tests))) tests.sort() for test in tests: result = resultsjsonparser.result_for_test(unexpected_results["tests"], test) actual = result["actual"].split(" ") expected = result["expected"].split(" ") result = TestExpectations.EXPECTATIONS[key.lower()] new_expectations_list = list(set(actual) | set(expected)) self._print_for_bot(" %s = %s" % (test, " ".join(new_expectations_list))) self._print_for_bot("") self._print_for_bot("") if len(regressions): descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS for key, tests in regressions.iteritems(): result = TestExpectations.EXPECTATIONS[key.lower()] self._print_for_bot("Regressions: Unexpected %s : (%d)" % (descriptions[result][0], len(tests))) tests.sort() for test in tests: self._print_for_bot(" %s = %s" % (test, key)) self._print_for_bot("") if len(unexpected_results["tests"]) and self._options.debug_rwt_logging: self._print_for_bot("%s" % ("-" * 78))
def _print_unexpected_results(self, unexpected_results): # Prints to the buildbot stream passes = {} flaky = {} regressions = {} def add_to_dict_of_lists(dict, key, value): dict.setdefault(key, []).append(value) def add_result(test, results, passes=passes, flaky=flaky, regressions=regressions): actual = results['actual'].split(" ") expected = results['expected'].split(" ") if actual == ['PASS']: if 'CRASH' in expected: add_to_dict_of_lists(passes, 'Expected to crash, but passed', test) elif 'TIMEOUT' in expected: add_to_dict_of_lists(passes, 'Expected to timeout, but passed', test) else: add_to_dict_of_lists(passes, 'Expected to fail, but passed', test) elif len(actual) > 1: # We group flaky tests by the first actual result we got. add_to_dict_of_lists(flaky, actual[0], test) else: add_to_dict_of_lists(regressions, results['actual'], test) resultsjsonparser.for_each_test(unexpected_results['tests'], add_result) if len(passes) or len(flaky) or len(regressions): self._print_for_bot("") if len(passes): for key, tests in passes.iteritems(): self._print_for_bot("%s: (%d)" % (key, len(tests))) tests.sort() for test in tests: self._print_for_bot(" %s" % test) self._print_for_bot("") self._print_for_bot("") if len(flaky): descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS for key, tests in flaky.iteritems(): result = TestExpectations.EXPECTATIONS[key.lower()] self._print_for_bot("Unexpected flakiness: %s (%d)" % (descriptions[result], len(tests))) tests.sort() for test in tests: result = resultsjsonparser.result_for_test(unexpected_results['tests'], test) actual = result['actual'].split(" ") expected = result['expected'].split(" ") result = TestExpectations.EXPECTATIONS[key.lower()] # FIXME: clean this up once the old syntax is gone new_expectations_list = [TestExpectationParser._inverted_expectation_tokens[exp] for exp in list(set(actual) | set(expected))] self._print_for_bot(" %s [ %s ]" % (test, " ".join(new_expectations_list))) self._print_for_bot("") self._print_for_bot("") if len(regressions): descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS for key, tests in regressions.iteritems(): result = TestExpectations.EXPECTATIONS[key.lower()] self._print_for_bot("Regressions: Unexpected %s (%d)" % (descriptions[result], len(tests))) tests.sort() for test in tests: self._print_for_bot(" %s [ %s ]" % (test, TestExpectationParser._inverted_expectation_tokens[key])) self._print_for_bot("") if len(unexpected_results['tests']) and self._options.debug_rwt_logging: self._print_for_bot("%s" % ("-" * 78))
def print_unexpected_results(self, summarized_results, enabled_pixel_tests_in_retry=False): passes = {} flaky = {} regressions = {} def add_to_dict_of_lists(dict, key, value): dict.setdefault(key, []).append(value) def add_result(test, results, passes=passes, flaky=flaky, regressions=regressions): actual = results['actual'].split(" ") expected = results['expected'].split(" ") def is_expected(result): return (result in expected) or (result in ('AUDIO', 'TEXT', 'IMAGE+TEXT') and 'FAIL' in expected) if all(is_expected(actual_result) for actual_result in actual): # Don't print anything for tests that ran as expected. return if actual == ['PASS']: if 'CRASH' in expected: add_to_dict_of_lists(passes, 'Expected to crash, but passed', test) elif 'TIMEOUT' in expected: add_to_dict_of_lists(passes, 'Expected to timeout, but passed', test) else: add_to_dict_of_lists(passes, 'Expected to fail, but passed', test) elif enabled_pixel_tests_in_retry and actual == ['TEXT', 'IMAGE+TEXT']: add_to_dict_of_lists(regressions, actual[0], test) elif len(actual) > 1: # We group flaky tests by the first actual result we got. add_to_dict_of_lists(flaky, actual[0], test) else: add_to_dict_of_lists(regressions, results['actual'], test) resultsjsonparser.for_each_test(summarized_results['tests'], add_result) if len(passes) or len(flaky) or len(regressions): self._print("") if len(passes): for key, tests in passes.iteritems(): self._print("%s: (%d)" % (key, len(tests))) tests.sort() for test in tests: self._print(" %s" % test) self._print("") self._print("") if len(flaky): descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS for key, tests in flaky.iteritems(): result = TestExpectations.EXPECTATIONS[key.lower()] self._print("Unexpected flakiness: %s (%d)" % (descriptions[result], len(tests))) tests.sort() for test in tests: result = resultsjsonparser.result_for_test(summarized_results['tests'], test) actual = result['actual'].split(" ") expected = result['expected'].split(" ") result = TestExpectations.EXPECTATIONS[key.lower()] # FIXME: clean this up once the old syntax is gone new_expectations_list = [TestExpectationParser._inverted_expectation_tokens[exp] for exp in list(set(actual) | set(expected))] self._print(" %s [ %s ]" % (test, " ".join(new_expectations_list))) self._print("") self._print("") if len(regressions): descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS for key, tests in regressions.iteritems(): result = TestExpectations.EXPECTATIONS[key.lower()] self._print("Regressions: Unexpected %s (%d)" % (descriptions[result], len(tests))) tests.sort() for test in tests: self._print(" %s [ %s ]" % (test, TestExpectationParser._inverted_expectation_tokens[key])) self._print("") if len(summarized_results['tests']) and self.debug_logging: self._print("%s" % ("-" * 78))
def print_unexpected_results(self, unexpected_results): """Prints a list of the unexpected results to the buildbot stream.""" if self.disabled('unexpected-results'): return passes = {} flaky = {} regressions = {} def add_to_dict_of_lists(dict, key, value): dict.setdefault(key, []).append(value) def add_result(test, results, passes=passes, flaky=flaky, regressions=regressions): actual = results['actual'].split(" ") expected = results['expected'].split(" ") if actual == ['PASS']: if 'CRASH' in expected: add_to_dict_of_lists(passes, 'Expected to crash, but passed', test) elif 'TIMEOUT' in expected: add_to_dict_of_lists(passes, 'Expected to timeout, but passed', test) else: add_to_dict_of_lists(passes, 'Expected to fail, but passed', test) elif len(actual) > 1: # We group flaky tests by the first actual result we got. add_to_dict_of_lists(flaky, actual[0], test) else: add_to_dict_of_lists(regressions, results['actual'], test) resultsjsonparser.for_each_test(unexpected_results['tests'], add_result) if len(passes) or len(flaky) or len(regressions): self._buildbot_stream.write("\n") if len(passes): for key, tests in passes.iteritems(): self._buildbot_stream.write("%s: (%d)\n" % (key, len(tests))) tests.sort() for test in tests: self._buildbot_stream.write(" %s\n" % test) self._buildbot_stream.write("\n") self._buildbot_stream.write("\n") if len(flaky): descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS for key, tests in flaky.iteritems(): result = TestExpectations.EXPECTATIONS[key.lower()] self._buildbot_stream.write("Unexpected flakiness: %s (%d)\n" % (descriptions[result][1], len(tests))) tests.sort() for test in tests: result = resultsjsonparser.result_for_test(unexpected_results['tests'], test) actual = result['actual'].split(" ") expected = result['expected'].split(" ") result = TestExpectations.EXPECTATIONS[key.lower()] new_expectations_list = list(set(actual) | set(expected)) self._buildbot_stream.write(" %s = %s\n" % (test, " ".join(new_expectations_list))) self._buildbot_stream.write("\n") self._buildbot_stream.write("\n") if len(regressions): descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS for key, tests in regressions.iteritems(): result = TestExpectations.EXPECTATIONS[key.lower()] self._buildbot_stream.write( "Regressions: Unexpected %s : (%d)\n" % ( descriptions[result][1], len(tests))) tests.sort() for test in tests: self._buildbot_stream.write(" %s = %s\n" % (test, key)) self._buildbot_stream.write("\n") self._buildbot_stream.write("\n") if len(unexpected_results['tests']) and self._options.verbose: self._buildbot_stream.write("%s\n" % ("-" * 78))
def print_unexpected_results(self, unexpected_results): """Prints a list of the unexpected results to the buildbot stream.""" if self.disabled('unexpected-results'): return passes = {} flaky = {} regressions = {} def add_to_dict_of_lists(dict, key, value): dict.setdefault(key, []).append(value) def add_result(test, results, passes=passes, flaky=flaky, regressions=regressions): actual = results['actual'].split(" ") expected = results['expected'].split(" ") if actual == ['PASS']: if 'CRASH' in expected: add_to_dict_of_lists(passes, 'Expected to crash, but passed', test) elif 'TIMEOUT' in expected: add_to_dict_of_lists(passes, 'Expected to timeout, but passed', test) else: add_to_dict_of_lists(passes, 'Expected to fail, but passed', test) elif len(actual) > 1: # We group flaky tests by the first actual result we got. add_to_dict_of_lists(flaky, actual[0], test) else: add_to_dict_of_lists(regressions, results['actual'], test) resultsjsonparser.for_each_test(unexpected_results['tests'], add_result) if len(passes) or len(flaky) or len(regressions): self._buildbot_stream.write("\n") if len(passes): for key, tests in passes.iteritems(): self._buildbot_stream.write("%s: (%d)\n" % (key, len(tests))) tests.sort() for test in tests: self._buildbot_stream.write(" %s\n" % test) self._buildbot_stream.write("\n") self._buildbot_stream.write("\n") if len(flaky): descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS for key, tests in flaky.iteritems(): result = TestExpectations.EXPECTATIONS[key.lower()] self._buildbot_stream.write("Unexpected flakiness: %s (%d)\n" % (descriptions[result][1], len(tests))) tests.sort() for test in tests: result = resultsjsonparser.result_for_test(unexpected_results['tests'], test) actual = result['actual'].split(" ") expected = result['expected'].split(" ") result = TestExpectations.EXPECTATIONS[key.lower()] new_expectations_list = list(set(actual) | set(expected)) self._buildbot_stream.write(" %s = %s\n" % (test, " ".join(new_expectations_list))) self._buildbot_stream.write("\n") self._buildbot_stream.write("\n") if len(regressions): descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS for key, tests in regressions.iteritems(): result = TestExpectations.EXPECTATIONS[key.lower()] self._buildbot_stream.write( "Regressions: Unexpected %s : (%d)\n" % ( descriptions[result][1], len(tests))) tests.sort() for test in tests: self._buildbot_stream.write(" %s = %s\n" % (test, key)) self._buildbot_stream.write("\n") self._buildbot_stream.write("\n") if len(unexpected_results['tests']) and self._options.verbose: self._buildbot_stream.write("%s\n" % ("-" * 78))
def execute(self, options, args, tool): results_directory = args[0] filesystem = system.filesystem.FileSystem() scm = self._tool.scm() if options.dry_run: def no_op_copyfile(src, dest): pass def no_op_add(path, return_exit_code=False): if return_exit_code: return 0 filesystem.copyfile = no_op_copyfile scm.add = no_op_add print 'Parsing unexpected_results.json...' results_json_path = filesystem.join(results_directory, 'unexpected_results.json') results_json = json_results_generator.load_json(filesystem, results_json_path) port = factory.get() layout_tests_directory = port.layout_tests_dir() platforms = filesystem.listdir( filesystem.join(layout_tests_directory, 'platform')) test_config = TestConfig( port, layout_tests_directory, results_directory, platforms, filesystem, scm) print 'Gathering current baselines...' # Rebaseline server and it's associated JavaScript expected the tests subtree to # be key-value pairs instead of hierarchical. # FIXME: make the rebaseline server use the hierarchical tree. new_tests_subtree = {} def gather_baselines(test, result): result['state'] = STATE_NEEDS_REBASELINE result['baselines'] = _get_test_baselines(test, test_config) new_tests_subtree[test] = result resultsjsonparser.for_each_test(results_json['tests'], gather_baselines) results_json['tests'] = new_tests_subtree server_url = "http://localhost:%d/" % options.httpd_port print "Starting server at %s" % server_url print ("Use the 'Exit' link in the UI, %squitquitquit " "or Ctrl-C to stop") % server_url threading.Timer( .1, lambda: self._tool.user.open_url(server_url)).start() httpd = RebaselineHTTPServer( httpd_port=options.httpd_port, test_config=test_config, results_json=results_json, platforms_json={ 'platforms': platforms, 'defaultPlatform': port.name(), }) httpd.serve_forever()