class FailureMapTest(unittest.TestCase):
builder1 = MockBuilder("Builder1")
builder2 = MockBuilder("Builder2")
build1a = Build(builder1, build_number=22, revision=1233, is_green=True)
build1b = Build(builder1, build_number=23, revision=1234, is_green=False)
build2a = Build(builder2, build_number=89, revision=1233, is_green=True)
build2b = Build(builder2, build_number=90, revision=1235, is_green=False)
regression_window1 = RegressionWindow(build1a,
build1b,
failing_tests=[u'test1', u'test1'])
regression_window2 = RegressionWindow(build2a,
build2b,
failing_tests=[u'test1'])
def _make_failure_map(self):
failure_map = FailureMap()
failure_map.add_regression_window(self.builder1,
self.regression_window1)
failure_map.add_regression_window(self.builder2,
self.regression_window2)
return failure_map
def test_failing_revisions(self):
failure_map = self._make_failure_map()
self.assertEqual(failure_map.failing_revisions(), [1234, 1235])
def test_new_failures(self):
failure_map = self._make_failure_map()
failure_map.filter_out_old_failures(lambda revision: False)
self.assertEqual(failure_map.failing_revisions(), [1234, 1235])
def test_new_failures_with_old_revisions(self):
failure_map = self._make_failure_map()
failure_map.filter_out_old_failures(lambda revision: revision == 1234)
self.assertEqual(failure_map.failing_revisions(), [])
def test_new_failures_with_more_old_revisions(self):
failure_map = self._make_failure_map()
failure_map.filter_out_old_failures(lambda revision: revision == 1235)
self.assertEqual(failure_map.failing_revisions(), [1234])
def test_tests_failing_for(self):
failure_map = self._make_failure_map()
self.assertEqual(failure_map.tests_failing_for(1234), [u'test1'])
def test_failing_tests(self):
failure_map = self._make_failure_map()
self.assertEqual(failure_map.failing_tests(), set([u'test1']))
def _explain_failures_for_builder(self, builder, start_revision):
print("Examining failures for \"%s\", starting at r%s" %
(builder.name(), start_revision))
revision_to_test = start_revision
build = builder.build_for_revision(revision_to_test,
allow_failed_lookups=True)
layout_test_results = build.layout_test_results()
if not layout_test_results:
# FIXME: This could be made more user friendly.
print(
"Failed to load layout test results from %s; can't continue. (start revision = r%s)"
% (build.results_url(), start_revision))
return 1
results_to_explain = set(layout_test_results.failing_tests())
last_build_with_results = build
print("Starting at %s" % revision_to_test)
while results_to_explain and not self._done_explaining():
revision_to_test -= 1
new_build = builder.build_for_revision(revision_to_test,
allow_failed_lookups=True)
if not new_build:
print("No build for %s" % revision_to_test)
continue
build = new_build
latest_results = build.layout_test_results()
if not latest_results:
print("No results build %s (r%s)" %
(build._number, build.revision()))
continue
failures = set(latest_results.failing_tests())
if len(failures) >= 500:
# FIXME: We may need to move this logic into the LayoutTestResults class.
# The buildbot stops runs after 500 failures so we don't have full results to work with here.
print("Too many failures in build %s (r%s), ignoring." %
(build._number, build.revision()))
continue
fixed_results = results_to_explain - failures
if not fixed_results:
print(
"No change in build %s (r%s), %s unexplained failures (%s in this build)"
% (build._number, build.revision(),
len(results_to_explain), len(failures)))
last_build_with_results = build
continue
self.explained_failures.update(fixed_results)
regression_window = RegressionWindow(build,
last_build_with_results)
self._print_blame_information_for_transition(
regression_window, fixed_results)
last_build_with_results = build
results_to_explain -= fixed_results
if results_to_explain:
print("Failed to explain failures: %s" % results_to_explain)
return 1
print("Explained all results for %s" % builder.name())
return 0
def find_regression_window(self, red_build, look_back_limit=30):
if not red_build or red_build.is_green():
return RegressionWindow(None, None)
common_failures = None
current_build = red_build
build_after_current_build = None
look_back_count = 0
while current_build:
if current_build.is_green():
# current_build can't possibly have any failures in common
# with red_build because it's green.
break
results = current_build.layout_test_results()
# We treat a lack of results as if all the test failed.
# This occurs, for example, when we can't compile at all.
if results:
failures = set(results.failing_tests())
if common_failures == None:
common_failures = failures
else:
common_failures = common_failures.intersection(failures)
if not common_failures:
# current_build doesn't have any failures in common with
# the red build we're worried about. We assume that any
# failures in current_build were due to flakiness.
break
look_back_count += 1
if look_back_count > look_back_limit:
return RegressionWindow(None,
current_build,
failing_tests=common_failures)
build_after_current_build = current_build
current_build = current_build.previous_build()
# We must iterate at least once because red_build is red.
assert (build_after_current_build)
# Current build must either be green or have no failures in common
# with red build, so we've found our failure transition.
return RegressionWindow(current_build,
build_after_current_build,
failing_tests=common_failures)