def test_unexpected_failures_observed_when_tree_is_hosed(self): failures = ExpectedFailures() failures.grow_expected_failures(MockResults(['foo.html', 'banana.html'], failure_limit=2)) self.assertEquals(failures.unexpected_failures_observed(MockResults(['foo.html', 'bar.html'])), None) self.assertEquals(failures.unexpected_failures_observed(MockResults(['baz.html'])), None) unbounded_results = MockResults(['baz.html', 'qux.html', 'taco.html'], failure_limit=3) self.assertEquals(failures.unexpected_failures_observed(unbounded_results), None) unbounded_results_with_existing_failure = MockResults(['foo.html', 'baz.html', 'qux.html', 'taco.html'], failure_limit=4) self.assertEquals(failures.unexpected_failures_observed(unbounded_results_with_existing_failure), None)
def test_unexpected_failures_observed(self): failures = ExpectedFailures() failures.update(MockResults(['foo.html'])) self.assertEqual(failures.unexpected_failures_observed(MockResults(['foo.html', 'bar.html'])), set(['bar.html'])) self.assertEqual(failures.unexpected_failures_observed(MockResults(['baz.html'])), set(['baz.html'])) unbounded_results = MockResults(['baz.html', 'qux.html', 'taco.html'], failure_limit=3) self.assertEqual(failures.unexpected_failures_observed(unbounded_results), set(['baz.html', 'qux.html', 'taco.html'])) unbounded_results_with_existing_failure = MockResults(['foo.html', 'baz.html', 'qux.html', 'taco.html'], failure_limit=4) self.assertEqual(failures.unexpected_failures_observed(unbounded_results_with_existing_failure), set(['baz.html', 'qux.html', 'taco.html']))
class AbstractEarlyWarningSystem(AbstractReviewQueue, EarlyWarningSystemTaskDelegate): _build_style = "release" # FIXME: Switch _default_run_tests from opt-in to opt-out once more bots are ready to run tests. _default_run_tests = False def __init__(self): options = [make_option("--run-tests", action="store_true", dest="run_tests", default=self._default_run_tests, help="Run the Layout tests for each patch")] AbstractReviewQueue.__init__(self, options=options) self.port = DeprecatedPort.port(self.port_name) def should_proceed_with_work_item(self, patch): return True def begin_work_queue(self): # FIXME: This violates abstraction self._tool._deprecated_port = self.port AbstractReviewQueue.begin_work_queue(self) self._expected_failures = ExpectedFailures() self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory()) def _failing_tests_message(self, task, patch): results = task.results_from_patch_test_run(patch) unexpected_failures = self._expected_failures.unexpected_failures_observed(results) if not unexpected_failures: return None return "New failing tests:\n%s" % "\n".join(unexpected_failures) def _post_reject_message_on_bug(self, tool, patch, status_id, extra_message_text=None): results_link = tool.status_server.results_url_for_status(status_id) message = "Attachment %s did not pass %s (%s):\nOutput: %s" % (patch.id(), self.name, self.port_name, results_link) # FIXME: We might want to add some text about rejecting from the commit-queue in # the case where patch.commit_queue() isn't already set to '-'. if self.watchers: tool.bugs.add_cc_to_bug(patch.bug_id(), self.watchers) tool.bugs.set_flag_on_attachment(patch.id(), "commit-queue", "-", message, extra_message_text) def review_patch(self, patch): task = EarlyWarningSystemTask(self, patch, self._options.run_tests) if not task.validate(): self._did_error(patch, "%s did not process patch." % self.name) return False try: return task.run() except UnableToApplyPatch, e: self._did_error(patch, "%s unable to apply patch." % self.name) return False except ScriptError, e: self._post_reject_message_on_bug(self._tool, patch, task.failure_status_id, self._failing_tests_message(task, patch)) results_archive = task.results_archive_from_patch_test_run(patch) if results_archive: self._upload_results_archive_for_patch(patch, results_archive) self._did_fail(patch) # FIXME: We're supposed to be able to raise e again here and have # one of our base classes mark the patch as fail, but there seems # to be an issue with the exit_code. return False
class AbstractEarlyWarningSystem(AbstractReviewQueue, EarlyWarningSystemTaskDelegate): _build_style = "release" # FIXME: Switch _default_run_tests from opt-in to opt-out once more bots are ready to run tests. run_tests = False def __init__(self): options = [make_option("--run-tests", action="store_true", dest="run_tests", default=self.run_tests, help="Run the Layout tests for each patch")] AbstractReviewQueue.__init__(self, options=options) def begin_work_queue(self): AbstractReviewQueue.begin_work_queue(self) self._expected_failures = ExpectedFailures() self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._port.results_directory(), self._log_directory()) def _failing_tests_message(self, task, patch): results = task.results_from_patch_test_run(patch) unexpected_failures = self._expected_failures.unexpected_failures_observed(results) if not unexpected_failures: return None return "New failing tests:\n%s" % "\n".join(unexpected_failures) def _post_reject_message_on_bug(self, tool, patch, status_id, extra_message_text=None): if not extra_message_text: return # Don't comment on Bugzilla if we don't have failing tests. results_link = tool.status_server.results_url_for_status(status_id) message = "Attachment %s did not pass %s (%s):\nOutput: %s" % (patch.id(), self.name, self.port_name, results_link) if extra_message_text: message += "\n\n%s" % extra_message_text # FIXME: We might want to add some text about rejecting from the commit-queue in # the case where patch.commit_queue() isn't already set to '-'. if self.watchers: tool.bugs.add_cc_to_bug(patch.bug_id(), self.watchers) tool.bugs.set_flag_on_attachment(patch.id(), "commit-queue", "-", message) def review_patch(self, patch): task = EarlyWarningSystemTask(self, patch, self._options.run_tests) if not task.validate(): self._did_error(patch, "%s did not process patch." % self.name) return False try: succeeded = task.run() if not succeeded: # Caller unlocks when review_patch returns True, so we only need to unlock on transient failure. self._unlock_patch(patch) return succeeded except UnableToApplyPatch, e: self._did_error(patch, "%s unable to apply patch." % self.name) return False except ScriptError, e: self._post_reject_message_on_bug(self._tool, patch, task.failure_status_id, self._failing_tests_message(task, patch)) results_archive = task.results_archive_from_patch_test_run(patch) if results_archive: self._upload_results_archive_for_patch(patch, results_archive) self._did_fail(patch) raise e
def test_unexpected_failures_observed_when_tree_is_hosed(self): failures = ExpectedFailures() failures.update( MockResults(['foo.html', 'banana.html'], failure_limit=2)) self.assertEquals( failures.unexpected_failures_observed( MockResults(['foo.html', 'bar.html'])), None) self.assertEquals( failures.unexpected_failures_observed(MockResults(['baz.html'])), None) unbounded_results = MockResults(['baz.html', 'qux.html', 'taco.html'], failure_limit=3) self.assertEquals( failures.unexpected_failures_observed(unbounded_results), None) unbounded_results_with_existing_failure = MockResults( ['foo.html', 'baz.html', 'qux.html', 'taco.html'], failure_limit=4) self.assertEquals( failures.unexpected_failures_observed( unbounded_results_with_existing_failure), None)
def test_unexpected_failures_observed(self): failures = ExpectedFailures() failures.grow_expected_failures(MockResults(['foo.html'])) self.assertEquals( failures.unexpected_failures_observed( MockResults(['foo.html', 'bar.html'])), set(['bar.html'])) self.assertEquals( failures.unexpected_failures_observed(MockResults(['baz.html'])), set(['baz.html'])) unbounded_results = MockResults(['baz.html', 'qux.html', 'taco.html'], failure_limit=3) self.assertEquals( failures.unexpected_failures_observed(unbounded_results), set(['baz.html', 'qux.html', 'taco.html'])) unbounded_results_with_existing_failure = MockResults( ['foo.html', 'baz.html', 'qux.html', 'taco.html'], failure_limit=4) self.assertEquals( failures.unexpected_failures_observed( unbounded_results_with_existing_failure), set(['baz.html', 'qux.html', 'taco.html']))
def test_unexpected_failures_observed(self): failures = ExpectedFailures() failures.update(MockResults(['foo.html'])) self.assertEqual( failures.unexpected_failures_observed( MockResults(['foo.html', 'bar.html'])), set(['bar.html'])) self.assertEqual( failures.unexpected_failures_observed(MockResults(['baz.html'])), set(['baz.html'])) unbounded_results = MockResults(['baz.html', 'qux.html', 'taco.html'], did_exceed_test_failure_limit=True) self.assertEqual( failures.unexpected_failures_observed(unbounded_results), set(['baz.html', 'qux.html', 'taco.html'])) unbounded_results_with_existing_failure = MockResults( ['foo.html', 'baz.html', 'qux.html', 'taco.html'], did_exceed_test_failure_limit=True) self.assertEqual( failures.unexpected_failures_observed( unbounded_results_with_existing_failure), set(['baz.html', 'qux.html', 'taco.html']))
class AbstractEarlyWarningSystem(AbstractReviewQueue, EarlyWarningSystemTaskDelegate): _build_style = "release" # FIXME: Switch _default_run_tests from opt-in to opt-out once more bots are ready to run tests. _default_run_tests = False def __init__(self): options = [make_option("--run-tests", action="store_true", dest="run_tests", default=self._default_run_tests, help="Run the Layout tests for each patch")] AbstractReviewQueue.__init__(self, options=options) self.port = DeprecatedPort.port(self.port_name) def begin_work_queue(self): # FIXME: This violates abstraction self._tool._deprecated_port = self.port AbstractReviewQueue.begin_work_queue(self) self._expected_failures = ExpectedFailures() self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory()) def _failing_tests_message(self, task, patch): results = task.results_from_patch_test_run(patch) unexpected_failures = self._expected_failures.unexpected_failures_observed(results) if not unexpected_failures: return None return "New failing tests:\n%s" % "\n".join(unexpected_failures) def _post_reject_message_on_bug(self, tool, patch, status_id, extra_message_text=None): results_link = tool.status_server.results_url_for_status(status_id) message = "Attachment %s did not pass %s (%s):\nOutput: %s" % (patch.id(), self.name, self.port_name, results_link) # FIXME: We might want to add some text about rejecting from the commit-queue in # the case where patch.commit_queue() isn't already set to '-'. if self.watchers: tool.bugs.add_cc_to_bug(patch.bug_id(), self.watchers) tool.bugs.set_flag_on_attachment(patch.id(), "commit-queue", "-", message, extra_message_text) def review_patch(self, patch): task = EarlyWarningSystemTask(self, patch, self._options.run_tests) if not task.validate(): self._did_error(patch, "%s did not process patch." % self.name) return False try: return task.run() except UnableToApplyPatch, e: self._did_error(patch, "%s unable to apply patch." % self.name) return False except ScriptError, e: self._post_reject_message_on_bug(self._tool, patch, task.failure_status_id, self._failing_tests_message(task, patch)) results_archive = task.results_archive_from_patch_test_run(patch) if results_archive: self._upload_results_archive_for_patch(patch, results_archive) self._did_fail(patch) # FIXME: We're supposed to be able to raise e again here and have # one of our base classes mark the patch as fail, but there seems # to be an issue with the exit_code. return False