def test_failures_were_expected(self): failures = ExpectedFailures() failures.grow_expected_failures(MockResults(['foo.html'])) self._assert_expected(failures, ['foo.html'], True) self._assert_expected(failures, ['bar.html'], False) failures.shrink_expected_failures(MockResults(['baz.html']), False) self._assert_expected(failures, ['foo.html'], False) self._assert_expected(failures, ['baz.html'], False) failures.grow_expected_failures(MockResults(['baz.html'])) self._assert_expected(failures, ['baz.html'], True) failures.shrink_expected_failures(MockResults(), True) self._assert_expected(failures, ['baz.html'], False)
class AbstractEarlyWarningSystem(AbstractReviewQueue, EarlyWarningSystemTaskDelegate): _build_style = "release" # FIXME: Switch _default_run_tests from opt-in to opt-out once more bots are ready to run tests. _default_run_tests = False def __init__(self): options = [make_option("--run-tests", action="store_true", dest="run_tests", default=self._default_run_tests, help="Run the Layout tests for each patch")] AbstractReviewQueue.__init__(self, options=options) self.port = DeprecatedPort.port(self.port_name) def should_proceed_with_work_item(self, patch): return True def begin_work_queue(self): # FIXME: This violates abstraction self._tool._deprecated_port = self.port AbstractReviewQueue.begin_work_queue(self) self._expected_failures = ExpectedFailures() self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory()) def _failing_tests_message(self, task, patch): results = task.results_from_patch_test_run(patch) unexpected_failures = self._expected_failures.unexpected_failures_observed(results) if not unexpected_failures: return None return "New failing tests:\n%s" % "\n".join(unexpected_failures) def _post_reject_message_on_bug(self, tool, patch, status_id, extra_message_text=None): results_link = tool.status_server.results_url_for_status(status_id) message = "Attachment %s did not pass %s (%s):\nOutput: %s" % (patch.id(), self.name, self.port_name, results_link) # FIXME: We might want to add some text about rejecting from the commit-queue in # the case where patch.commit_queue() isn't already set to '-'. if self.watchers: tool.bugs.add_cc_to_bug(patch.bug_id(), self.watchers) tool.bugs.set_flag_on_attachment(patch.id(), "commit-queue", "-", message, extra_message_text) def review_patch(self, patch): task = EarlyWarningSystemTask(self, patch, self._options.run_tests) if not task.validate(): self._did_error(patch, "%s did not process patch." % self.name) return False try: return task.run() except UnableToApplyPatch, e: self._did_error(patch, "%s unable to apply patch." % self.name) return False except ScriptError, e: self._post_reject_message_on_bug(self._tool, patch, task.failure_status_id, self._failing_tests_message(task, patch)) results_archive = task.results_archive_from_patch_test_run(patch) if results_archive: self._upload_results_archive_for_patch(patch, results_archive) self._did_fail(patch) # FIXME: We're supposed to be able to raise e again here and have # one of our base classes mark the patch as fail, but there seems # to be an issue with the exit_code. return False
def test_unexpected_failures_observed_when_tree_is_hosed(self): failures = ExpectedFailures() failures.grow_expected_failures(MockResults(['foo.html', 'banana.html'], failure_limit=2)) self.assertEquals(failures.unexpected_failures_observed(MockResults(['foo.html', 'bar.html'])), None) self.assertEquals(failures.unexpected_failures_observed(MockResults(['baz.html'])), None) unbounded_results = MockResults(['baz.html', 'qux.html', 'taco.html'], failure_limit=3) self.assertEquals(failures.unexpected_failures_observed(unbounded_results), None) unbounded_results_with_existing_failure = MockResults(['foo.html', 'baz.html', 'qux.html', 'taco.html'], failure_limit=4) self.assertEquals(failures.unexpected_failures_observed(unbounded_results_with_existing_failure), None)
def test_unexpected_failures_observed(self): failures = ExpectedFailures() failures.update(MockResults(['foo.html'])) self.assertEqual(failures.unexpected_failures_observed(MockResults(['foo.html', 'bar.html'])), set(['bar.html'])) self.assertEqual(failures.unexpected_failures_observed(MockResults(['baz.html'])), set(['baz.html'])) unbounded_results = MockResults(['baz.html', 'qux.html', 'taco.html'], failure_limit=3) self.assertEqual(failures.unexpected_failures_observed(unbounded_results), set(['baz.html', 'qux.html', 'taco.html'])) unbounded_results_with_existing_failure = MockResults(['foo.html', 'baz.html', 'qux.html', 'taco.html'], failure_limit=4) self.assertEqual(failures.unexpected_failures_observed(unbounded_results_with_existing_failure), set(['baz.html', 'qux.html', 'taco.html']))
def test_failures_were_expected(self): failures = ExpectedFailures() failures.update(MockResults(['foo.html'])) self._assert_expected(failures, ['foo.html'], True) self._assert_expected(failures, ['bar.html'], False) self._assert_expected(failures, ['bar.html', 'foo.html'], False) failures.update(MockResults(['baz.html'])) self._assert_expected(failures, ['baz.html'], True) self._assert_expected(failures, ['foo.html'], False) failures.update(MockResults([])) self._assert_expected(failures, ['baz.html'], False) self._assert_expected(failures, ['foo.html'], False)
class AbstractTestingEWS(AbstractEarlyWarningSystem, EarlyWarningSystemTaskDelegate): def begin_work_queue(self): # FIXME: This violates abstraction self._tool._port = self.port AbstractEarlyWarningSystem.begin_work_queue(self) self._expected_failures = ExpectedFailures() self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory()) def _post_reject_message_on_bug(self, task, patch): results_link = self._tool.status_server.results_url_for_status(task.failure_status_id) message = "Attachment %s did not pass %s:\nOutput: %s" % (patch.id(), self.name, results_link) results = task.results_from_patch_test_run(patch) unexpected_failures = self._expected_failures.unexpected_failures(results) if unexpected_failures: message += "\nNew failing tests:\n%s" % "\n".join(unexpected_failures) self._tool.bugs.post_comment_to_bug(patch.bug_id(), message, cc=self.watchers) def review_patch(self, patch): task = EarlyWarningSystemTask(self, patch) if not task.validate(): self._did_error(patch, "%s did not process patch." % self.name) return False try: return task.run() except UnableToApplyPatch, e: self._did_error(patch, "%s unable to apply patch." % self.name) return False except ScriptError, e: # FIXME: This should just use CommitterValidator.reject_patch_from_commit_queue self._post_reject_message_on_bug(task, patch) results_archive = task.results_archive_from_patch_test_run(patch) if results_archive: self._upload_results_archive_for_patch(patch, results_archive) self._did_fail(patch) # FIXME: We're supposed to be able to raise e again here and have # one of our base classes mark the patch as fail, but there seems # to be an issue with the exit_code. return False
def _assert_can_trust(self, results, can_trust): self.assertEquals(ExpectedFailures()._can_trust_results(results), can_trust)
def begin_work_queue(self): AbstractReviewQueue.begin_work_queue(self) self._expected_failures = ExpectedFailures() self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._port.results_directory(), self._log_directory())
def begin_work_queue(self): # FIXME: This violates abstraction self._tool._deprecated_port = self.port AbstractReviewQueue.begin_work_queue(self) self._expected_failures = ExpectedFailures() self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory())
def begin_work_queue(self): AbstractPatchQueue.begin_work_queue(self) self.committer_validator = CommitterValidator(self._tool.bugs) self._expected_failures = ExpectedFailures() self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory())
def expected_failures(self): return ExpectedFailures()
def begin_work_queue(self): PatchProcessingQueue.begin_work_queue(self) self.committer_validator = CommitterValidator(self._tool) self._expected_failures = ExpectedFailures() self._layout_test_results_reader = LayoutTestResultsReader( self._tool, self._port.results_directory(), self._log_directory())
class AbstractEarlyWarningSystem(AbstractReviewQueue, EarlyWarningSystemTaskDelegate): _build_style = "release" # FIXME: Switch _default_run_tests from opt-in to opt-out once more bots are ready to run tests. _default_run_tests = False def __init__(self): options = [ make_option("--run-tests", action="store_true", dest="run_tests", default=self._default_run_tests, help="Run the Layout tests for each patch") ] AbstractReviewQueue.__init__(self, options=options) self.port = DeprecatedPort.port(self.port_name) def begin_work_queue(self): # FIXME: This violates abstraction self._tool._deprecated_port = self.port AbstractReviewQueue.begin_work_queue(self) self._expected_failures = ExpectedFailures() self._layout_test_results_reader = LayoutTestResultsReader( self._tool, self._log_directory()) def _failing_tests_message(self, task, patch): results = task.results_from_patch_test_run(patch) unexpected_failures = self._expected_failures.unexpected_failures_observed( results) if not unexpected_failures: return None return "New failing tests:\n%s" % "\n".join(unexpected_failures) def _post_reject_message_on_bug(self, tool, patch, status_id, extra_message_text=None): results_link = tool.status_server.results_url_for_status(status_id) message = "Attachment %s did not pass %s (%s):\nOutput: %s" % ( patch.id(), self.name, self.port_name, results_link) # FIXME: We might want to add some text about rejecting from the commit-queue in # the case where patch.commit_queue() isn't already set to '-'. if self.watchers: tool.bugs.add_cc_to_bug(patch.bug_id(), self.watchers) tool.bugs.set_flag_on_attachment(patch.id(), "commit-queue", "-", message, extra_message_text) def review_patch(self, patch): task = EarlyWarningSystemTask(self, patch, self._options.run_tests) if not task.validate(): self._did_error(patch, "%s did not process patch." % self.name) return False try: return task.run() except UnableToApplyPatch, e: self._did_error(patch, "%s unable to apply patch." % self.name) return False except ScriptError, e: self._post_reject_message_on_bug( self._tool, patch, task.failure_status_id, self._failing_tests_message(task, patch)) results_archive = task.results_archive_from_patch_test_run(patch) if results_archive: self._upload_results_archive_for_patch(patch, results_archive) self._did_fail(patch) # FIXME: We're supposed to be able to raise e again here and have # one of our base classes mark the patch as fail, but there seems # to be an issue with the exit_code. return False
def _assert_can_trust(self, results, can_trust): self.assertEqual(ExpectedFailures._should_trust(results), can_trust)
def test_unexpected_failures(self): failures = ExpectedFailures() failures.grow_expected_failures(MockResults(['foo.html'])) self.assertEquals(failures.unexpected_failures(MockResults(['foo.html', 'bar.html'])), set(['bar.html'])) self.assertEquals(failures.unexpected_failures(MockResults(['baz.html'])), set(['baz.html']))
def begin_work_queue(self): AbstractReviewQueue.begin_work_queue(self) self._expected_failures = ExpectedFailures() self._layout_test_results_reader = LayoutTestResultsReader( self._tool, self._port.results_directory(), self._log_directory())