def test_unexpected_failures_observed_when_tree_is_hosed(self):
failures = ExpectedFailures()
failures.grow_expected_failures(MockResults(['foo.html', 'banana.html'], failure_limit=2))
self.assertEquals(failures.unexpected_failures_observed(MockResults(['foo.html', 'bar.html'])), None)
self.assertEquals(failures.unexpected_failures_observed(MockResults(['baz.html'])), None)
unbounded_results = MockResults(['baz.html', 'qux.html', 'taco.html'], failure_limit=3)
self.assertEquals(failures.unexpected_failures_observed(unbounded_results), None)
unbounded_results_with_existing_failure = MockResults(['foo.html', 'baz.html', 'qux.html', 'taco.html'], failure_limit=4)
self.assertEquals(failures.unexpected_failures_observed(unbounded_results_with_existing_failure), None)
def test_unexpected_failures_observed(self):
failures = ExpectedFailures()
failures.update(MockResults(['foo.html']))
self.assertEqual(failures.unexpected_failures_observed(MockResults(['foo.html', 'bar.html'])), set(['bar.html']))
self.assertEqual(failures.unexpected_failures_observed(MockResults(['baz.html'])), set(['baz.html']))
unbounded_results = MockResults(['baz.html', 'qux.html', 'taco.html'], failure_limit=3)
self.assertEqual(failures.unexpected_failures_observed(unbounded_results), set(['baz.html', 'qux.html', 'taco.html']))
unbounded_results_with_existing_failure = MockResults(['foo.html', 'baz.html', 'qux.html', 'taco.html'], failure_limit=4)
self.assertEqual(failures.unexpected_failures_observed(unbounded_results_with_existing_failure), set(['baz.html', 'qux.html', 'taco.html']))
class AbstractEarlyWarningSystem(AbstractReviewQueue, EarlyWarningSystemTaskDelegate):
_build_style = "release"
# FIXME: Switch _default_run_tests from opt-in to opt-out once more bots are ready to run tests.
_default_run_tests = False
def __init__(self):
options = [make_option("--run-tests", action="store_true", dest="run_tests", default=self._default_run_tests, help="Run the Layout tests for each patch")]
AbstractReviewQueue.__init__(self, options=options)
self.port = DeprecatedPort.port(self.port_name)
def should_proceed_with_work_item(self, patch):
return True
def begin_work_queue(self):
# FIXME: This violates abstraction
self._tool._deprecated_port = self.port
AbstractReviewQueue.begin_work_queue(self)
self._expected_failures = ExpectedFailures()
self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory())
def _failing_tests_message(self, task, patch):
results = task.results_from_patch_test_run(patch)
unexpected_failures = self._expected_failures.unexpected_failures_observed(results)
if not unexpected_failures:
return None
return "New failing tests:\n%s" % "\n".join(unexpected_failures)
def _post_reject_message_on_bug(self, tool, patch, status_id, extra_message_text=None):
results_link = tool.status_server.results_url_for_status(status_id)
message = "Attachment %s did not pass %s (%s):\nOutput: %s" % (patch.id(), self.name, self.port_name, results_link)
# FIXME: We might want to add some text about rejecting from the commit-queue in
# the case where patch.commit_queue() isn't already set to '-'.
if self.watchers:
tool.bugs.add_cc_to_bug(patch.bug_id(), self.watchers)
tool.bugs.set_flag_on_attachment(patch.id(), "commit-queue", "-", message, extra_message_text)
def review_patch(self, patch):
task = EarlyWarningSystemTask(self, patch, self._options.run_tests)
if not task.validate():
self._did_error(patch, "%s did not process patch." % self.name)
return False
try:
return task.run()
except UnableToApplyPatch, e:
self._did_error(patch, "%s unable to apply patch." % self.name)
return False
except ScriptError, e:
self._post_reject_message_on_bug(self._tool, patch, task.failure_status_id, self._failing_tests_message(task, patch))
results_archive = task.results_archive_from_patch_test_run(patch)
if results_archive:
self._upload_results_archive_for_patch(patch, results_archive)
self._did_fail(patch)
# FIXME: We're supposed to be able to raise e again here and have
# one of our base classes mark the patch as fail, but there seems
# to be an issue with the exit_code.
return False
class AbstractEarlyWarningSystem(AbstractReviewQueue, EarlyWarningSystemTaskDelegate):
_build_style = "release"
# FIXME: Switch _default_run_tests from opt-in to opt-out once more bots are ready to run tests.
run_tests = False
def __init__(self):
options = [make_option("--run-tests", action="store_true", dest="run_tests", default=self.run_tests, help="Run the Layout tests for each patch")]
AbstractReviewQueue.__init__(self, options=options)
def begin_work_queue(self):
AbstractReviewQueue.begin_work_queue(self)
self._expected_failures = ExpectedFailures()
self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._port.results_directory(), self._log_directory())
def _failing_tests_message(self, task, patch):
results = task.results_from_patch_test_run(patch)
unexpected_failures = self._expected_failures.unexpected_failures_observed(results)
if not unexpected_failures:
return None
return "New failing tests:\n%s" % "\n".join(unexpected_failures)
def _post_reject_message_on_bug(self, tool, patch, status_id, extra_message_text=None):
if not extra_message_text:
return # Don't comment on Bugzilla if we don't have failing tests.
results_link = tool.status_server.results_url_for_status(status_id)
message = "Attachment %s did not pass %s (%s):\nOutput: %s" % (patch.id(), self.name, self.port_name, results_link)
if extra_message_text:
message += "\n\n%s" % extra_message_text
# FIXME: We might want to add some text about rejecting from the commit-queue in
# the case where patch.commit_queue() isn't already set to '-'.
if self.watchers:
tool.bugs.add_cc_to_bug(patch.bug_id(), self.watchers)
tool.bugs.set_flag_on_attachment(patch.id(), "commit-queue", "-", message)
def review_patch(self, patch):
task = EarlyWarningSystemTask(self, patch, self._options.run_tests)
if not task.validate():
self._did_error(patch, "%s did not process patch." % self.name)
return False
try:
succeeded = task.run()
if not succeeded:
# Caller unlocks when review_patch returns True, so we only need to unlock on transient failure.
self._unlock_patch(patch)
return succeeded
except UnableToApplyPatch, e:
self._did_error(patch, "%s unable to apply patch." % self.name)
return False
except ScriptError, e:
self._post_reject_message_on_bug(self._tool, patch, task.failure_status_id, self._failing_tests_message(task, patch))
results_archive = task.results_archive_from_patch_test_run(patch)
if results_archive:
self._upload_results_archive_for_patch(patch, results_archive)
self._did_fail(patch)
raise e
def test_unexpected_failures_observed_when_tree_is_hosed(self):
failures = ExpectedFailures()
failures.update(
MockResults(['foo.html', 'banana.html'], failure_limit=2))
self.assertEquals(
failures.unexpected_failures_observed(
MockResults(['foo.html', 'bar.html'])), None)
self.assertEquals(
failures.unexpected_failures_observed(MockResults(['baz.html'])),
None)
unbounded_results = MockResults(['baz.html', 'qux.html', 'taco.html'],
failure_limit=3)
self.assertEquals(
failures.unexpected_failures_observed(unbounded_results), None)
unbounded_results_with_existing_failure = MockResults(
['foo.html', 'baz.html', 'qux.html', 'taco.html'], failure_limit=4)
self.assertEquals(
failures.unexpected_failures_observed(
unbounded_results_with_existing_failure), None)
def test_unexpected_failures_observed(self):
failures = ExpectedFailures()
failures.grow_expected_failures(MockResults(['foo.html']))
self.assertEquals(
failures.unexpected_failures_observed(
MockResults(['foo.html', 'bar.html'])), set(['bar.html']))
self.assertEquals(
failures.unexpected_failures_observed(MockResults(['baz.html'])),
set(['baz.html']))
unbounded_results = MockResults(['baz.html', 'qux.html', 'taco.html'],
failure_limit=3)
self.assertEquals(
failures.unexpected_failures_observed(unbounded_results),
set(['baz.html', 'qux.html', 'taco.html']))
unbounded_results_with_existing_failure = MockResults(
['foo.html', 'baz.html', 'qux.html', 'taco.html'], failure_limit=4)
self.assertEquals(
failures.unexpected_failures_observed(
unbounded_results_with_existing_failure),
set(['baz.html', 'qux.html', 'taco.html']))
def test_unexpected_failures_observed(self):
failures = ExpectedFailures()
failures.update(MockResults(['foo.html']))
self.assertEqual(
failures.unexpected_failures_observed(
MockResults(['foo.html', 'bar.html'])), set(['bar.html']))
self.assertEqual(
failures.unexpected_failures_observed(MockResults(['baz.html'])),
set(['baz.html']))
unbounded_results = MockResults(['baz.html', 'qux.html', 'taco.html'],
did_exceed_test_failure_limit=True)
self.assertEqual(
failures.unexpected_failures_observed(unbounded_results),
set(['baz.html', 'qux.html', 'taco.html']))
unbounded_results_with_existing_failure = MockResults(
['foo.html', 'baz.html', 'qux.html', 'taco.html'],
did_exceed_test_failure_limit=True)
self.assertEqual(
failures.unexpected_failures_observed(
unbounded_results_with_existing_failure),
set(['baz.html', 'qux.html', 'taco.html']))
class AbstractEarlyWarningSystem(AbstractReviewQueue, EarlyWarningSystemTaskDelegate):
_build_style = "release"
# FIXME: Switch _default_run_tests from opt-in to opt-out once more bots are ready to run tests.
_default_run_tests = False
def __init__(self):
options = [make_option("--run-tests", action="store_true", dest="run_tests", default=self._default_run_tests, help="Run the Layout tests for each patch")]
AbstractReviewQueue.__init__(self, options=options)
self.port = DeprecatedPort.port(self.port_name)
def begin_work_queue(self):
# FIXME: This violates abstraction
self._tool._deprecated_port = self.port
AbstractReviewQueue.begin_work_queue(self)
self._expected_failures = ExpectedFailures()
self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory())
def _failing_tests_message(self, task, patch):
results = task.results_from_patch_test_run(patch)
unexpected_failures = self._expected_failures.unexpected_failures_observed(results)
if not unexpected_failures:
return None
return "New failing tests:\n%s" % "\n".join(unexpected_failures)
def _post_reject_message_on_bug(self, tool, patch, status_id, extra_message_text=None):
results_link = tool.status_server.results_url_for_status(status_id)
message = "Attachment %s did not pass %s (%s):\nOutput: %s" % (patch.id(), self.name, self.port_name, results_link)
# FIXME: We might want to add some text about rejecting from the commit-queue in
# the case where patch.commit_queue() isn't already set to '-'.
if self.watchers:
tool.bugs.add_cc_to_bug(patch.bug_id(), self.watchers)
tool.bugs.set_flag_on_attachment(patch.id(), "commit-queue", "-", message, extra_message_text)
def review_patch(self, patch):
task = EarlyWarningSystemTask(self, patch, self._options.run_tests)
if not task.validate():
self._did_error(patch, "%s did not process patch." % self.name)
return False
try:
return task.run()
except UnableToApplyPatch, e:
self._did_error(patch, "%s unable to apply patch." % self.name)
return False
except ScriptError, e:
self._post_reject_message_on_bug(self._tool, patch, task.failure_status_id, self._failing_tests_message(task, patch))
results_archive = task.results_archive_from_patch_test_run(patch)
if results_archive:
self._upload_results_archive_for_patch(patch, results_archive)
self._did_fail(patch)
# FIXME: We're supposed to be able to raise e again here and have
# one of our base classes mark the patch as fail, but there seems
# to be an issue with the exit_code.
return False