def test_failures_were_expected(self):
        failures = ExpectedFailures()
        failures.grow_expected_failures(MockResults(['foo.html']))
        self._assert_expected(failures, ['foo.html'], True)
        self._assert_expected(failures, ['bar.html'], False)
        failures.shrink_expected_failures(MockResults(['baz.html']), False)
        self._assert_expected(failures, ['foo.html'], False)
        self._assert_expected(failures, ['baz.html'], False)

        failures.grow_expected_failures(MockResults(['baz.html']))
        self._assert_expected(failures, ['baz.html'], True)
        failures.shrink_expected_failures(MockResults(), True)
        self._assert_expected(failures, ['baz.html'], False)
Beispiel #2
0
class AbstractEarlyWarningSystem(AbstractReviewQueue, EarlyWarningSystemTaskDelegate):
    _build_style = "release"
    # FIXME: Switch _default_run_tests from opt-in to opt-out once more bots are ready to run tests.
    _default_run_tests = False

    def __init__(self):
        options = [make_option("--run-tests", action="store_true", dest="run_tests", default=self._default_run_tests, help="Run the Layout tests for each patch")]
        AbstractReviewQueue.__init__(self, options=options)
        self.port = DeprecatedPort.port(self.port_name)

    def should_proceed_with_work_item(self, patch):
        return True

    def begin_work_queue(self):
        # FIXME: This violates abstraction
        self._tool._deprecated_port = self.port
        AbstractReviewQueue.begin_work_queue(self)
        self._expected_failures = ExpectedFailures()
        self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory())

    def _failing_tests_message(self, task, patch):
        results = task.results_from_patch_test_run(patch)
        unexpected_failures = self._expected_failures.unexpected_failures_observed(results)
        if not unexpected_failures:
            return None
        return "New failing tests:\n%s" % "\n".join(unexpected_failures)

    def _post_reject_message_on_bug(self, tool, patch, status_id, extra_message_text=None):
        results_link = tool.status_server.results_url_for_status(status_id)
        message = "Attachment %s did not pass %s (%s):\nOutput: %s" % (patch.id(), self.name, self.port_name, results_link)
        # FIXME: We might want to add some text about rejecting from the commit-queue in
        # the case where patch.commit_queue() isn't already set to '-'.
        if self.watchers:
            tool.bugs.add_cc_to_bug(patch.bug_id(), self.watchers)
        tool.bugs.set_flag_on_attachment(patch.id(), "commit-queue", "-", message, extra_message_text)

    def review_patch(self, patch):
        task = EarlyWarningSystemTask(self, patch, self._options.run_tests)
        if not task.validate():
            self._did_error(patch, "%s did not process patch." % self.name)
            return False
        try:
            return task.run()
        except UnableToApplyPatch, e:
            self._did_error(patch, "%s unable to apply patch." % self.name)
            return False
        except ScriptError, e:
            self._post_reject_message_on_bug(self._tool, patch, task.failure_status_id, self._failing_tests_message(task, patch))
            results_archive = task.results_archive_from_patch_test_run(patch)
            if results_archive:
                self._upload_results_archive_for_patch(patch, results_archive)
            self._did_fail(patch)
            # FIXME: We're supposed to be able to raise e again here and have
            # one of our base classes mark the patch as fail, but there seems
            # to be an issue with the exit_code.
            return False
 def test_unexpected_failures_observed_when_tree_is_hosed(self):
     failures = ExpectedFailures()
     failures.grow_expected_failures(MockResults(['foo.html', 'banana.html'], failure_limit=2))
     self.assertEquals(failures.unexpected_failures_observed(MockResults(['foo.html', 'bar.html'])), None)
     self.assertEquals(failures.unexpected_failures_observed(MockResults(['baz.html'])), None)
     unbounded_results = MockResults(['baz.html', 'qux.html', 'taco.html'], failure_limit=3)
     self.assertEquals(failures.unexpected_failures_observed(unbounded_results), None)
     unbounded_results_with_existing_failure = MockResults(['foo.html', 'baz.html', 'qux.html', 'taco.html'], failure_limit=4)
     self.assertEquals(failures.unexpected_failures_observed(unbounded_results_with_existing_failure), None)
 def test_unexpected_failures_observed(self):
     failures = ExpectedFailures()
     failures.update(MockResults(['foo.html']))
     self.assertEqual(failures.unexpected_failures_observed(MockResults(['foo.html', 'bar.html'])), set(['bar.html']))
     self.assertEqual(failures.unexpected_failures_observed(MockResults(['baz.html'])), set(['baz.html']))
     unbounded_results = MockResults(['baz.html', 'qux.html', 'taco.html'], failure_limit=3)
     self.assertEqual(failures.unexpected_failures_observed(unbounded_results), set(['baz.html', 'qux.html', 'taco.html']))
     unbounded_results_with_existing_failure = MockResults(['foo.html', 'baz.html', 'qux.html', 'taco.html'], failure_limit=4)
     self.assertEqual(failures.unexpected_failures_observed(unbounded_results_with_existing_failure), set(['baz.html', 'qux.html', 'taco.html']))
    def test_failures_were_expected(self):
        failures = ExpectedFailures()
        failures.update(MockResults(['foo.html']))
        self._assert_expected(failures, ['foo.html'], True)
        self._assert_expected(failures, ['bar.html'], False)
        self._assert_expected(failures, ['bar.html', 'foo.html'], False)

        failures.update(MockResults(['baz.html']))
        self._assert_expected(failures, ['baz.html'], True)
        self._assert_expected(failures, ['foo.html'], False)

        failures.update(MockResults([]))
        self._assert_expected(failures, ['baz.html'], False)
        self._assert_expected(failures, ['foo.html'], False)
class AbstractTestingEWS(AbstractEarlyWarningSystem, EarlyWarningSystemTaskDelegate):
    def begin_work_queue(self):
        # FIXME: This violates abstraction
        self._tool._port = self.port
        AbstractEarlyWarningSystem.begin_work_queue(self)
        self._expected_failures = ExpectedFailures()
        self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory())

    def _post_reject_message_on_bug(self, task, patch):
        results_link = self._tool.status_server.results_url_for_status(task.failure_status_id)
        message = "Attachment %s did not pass %s:\nOutput: %s" % (patch.id(), self.name, results_link)
        results = task.results_from_patch_test_run(patch)
        unexpected_failures = self._expected_failures.unexpected_failures(results)
        if unexpected_failures:
            message += "\nNew failing tests:\n%s" % "\n".join(unexpected_failures)
        self._tool.bugs.post_comment_to_bug(patch.bug_id(), message, cc=self.watchers)

    def review_patch(self, patch):
        task = EarlyWarningSystemTask(self, patch)
        if not task.validate():
            self._did_error(patch, "%s did not process patch." % self.name)
            return False
        try:
            return task.run()
        except UnableToApplyPatch, e:
            self._did_error(patch, "%s unable to apply patch." % self.name)
            return False
        except ScriptError, e:
            # FIXME: This should just use CommitterValidator.reject_patch_from_commit_queue
            self._post_reject_message_on_bug(task, patch)
            results_archive = task.results_archive_from_patch_test_run(patch)
            if results_archive:
                self._upload_results_archive_for_patch(patch, results_archive)
            self._did_fail(patch)
            # FIXME: We're supposed to be able to raise e again here and have
            # one of our base classes mark the patch as fail, but there seems
            # to be an issue with the exit_code.
            return False
Beispiel #7
0
 def _assert_can_trust(self, results, can_trust):
     self.assertEquals(ExpectedFailures()._can_trust_results(results),
                       can_trust)
Beispiel #8
0
 def begin_work_queue(self):
     AbstractReviewQueue.begin_work_queue(self)
     self._expected_failures = ExpectedFailures()
     self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._port.results_directory(), self._log_directory())
Beispiel #9
0
 def begin_work_queue(self):
     # FIXME: This violates abstraction
     self._tool._deprecated_port = self.port
     AbstractReviewQueue.begin_work_queue(self)
     self._expected_failures = ExpectedFailures()
     self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory())
Beispiel #10
0
 def begin_work_queue(self):
     AbstractPatchQueue.begin_work_queue(self)
     self.committer_validator = CommitterValidator(self._tool.bugs)
     self._expected_failures = ExpectedFailures()
     self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory())
Beispiel #11
0
 def expected_failures(self):
     return ExpectedFailures()
Beispiel #12
0
 def begin_work_queue(self):
     PatchProcessingQueue.begin_work_queue(self)
     self.committer_validator = CommitterValidator(self._tool)
     self._expected_failures = ExpectedFailures()
     self._layout_test_results_reader = LayoutTestResultsReader(
         self._tool, self._port.results_directory(), self._log_directory())
Beispiel #13
0
class AbstractEarlyWarningSystem(AbstractReviewQueue,
                                 EarlyWarningSystemTaskDelegate):
    _build_style = "release"
    # FIXME: Switch _default_run_tests from opt-in to opt-out once more bots are ready to run tests.
    _default_run_tests = False

    def __init__(self):
        options = [
            make_option("--run-tests",
                        action="store_true",
                        dest="run_tests",
                        default=self._default_run_tests,
                        help="Run the Layout tests for each patch")
        ]
        AbstractReviewQueue.__init__(self, options=options)
        self.port = DeprecatedPort.port(self.port_name)

    def begin_work_queue(self):
        # FIXME: This violates abstraction
        self._tool._deprecated_port = self.port
        AbstractReviewQueue.begin_work_queue(self)
        self._expected_failures = ExpectedFailures()
        self._layout_test_results_reader = LayoutTestResultsReader(
            self._tool, self._log_directory())

    def _failing_tests_message(self, task, patch):
        results = task.results_from_patch_test_run(patch)
        unexpected_failures = self._expected_failures.unexpected_failures_observed(
            results)
        if not unexpected_failures:
            return None
        return "New failing tests:\n%s" % "\n".join(unexpected_failures)

    def _post_reject_message_on_bug(self,
                                    tool,
                                    patch,
                                    status_id,
                                    extra_message_text=None):
        results_link = tool.status_server.results_url_for_status(status_id)
        message = "Attachment %s did not pass %s (%s):\nOutput: %s" % (
            patch.id(), self.name, self.port_name, results_link)
        # FIXME: We might want to add some text about rejecting from the commit-queue in
        # the case where patch.commit_queue() isn't already set to '-'.
        if self.watchers:
            tool.bugs.add_cc_to_bug(patch.bug_id(), self.watchers)
        tool.bugs.set_flag_on_attachment(patch.id(), "commit-queue", "-",
                                         message, extra_message_text)

    def review_patch(self, patch):
        task = EarlyWarningSystemTask(self, patch, self._options.run_tests)
        if not task.validate():
            self._did_error(patch, "%s did not process patch." % self.name)
            return False
        try:
            return task.run()
        except UnableToApplyPatch, e:
            self._did_error(patch, "%s unable to apply patch." % self.name)
            return False
        except ScriptError, e:
            self._post_reject_message_on_bug(
                self._tool, patch, task.failure_status_id,
                self._failing_tests_message(task, patch))
            results_archive = task.results_archive_from_patch_test_run(patch)
            if results_archive:
                self._upload_results_archive_for_patch(patch, results_archive)
            self._did_fail(patch)
            # FIXME: We're supposed to be able to raise e again here and have
            # one of our base classes mark the patch as fail, but there seems
            # to be an issue with the exit_code.
            return False
 def _assert_can_trust(self, results, can_trust):
     self.assertEqual(ExpectedFailures._should_trust(results), can_trust)
 def test_unexpected_failures(self):
     failures = ExpectedFailures()
     failures.grow_expected_failures(MockResults(['foo.html']))
     self.assertEquals(failures.unexpected_failures(MockResults(['foo.html', 'bar.html'])), set(['bar.html']))
     self.assertEquals(failures.unexpected_failures(MockResults(['baz.html'])), set(['baz.html']))
 def _assert_can_trust(self, results, can_trust):
     self.assertEqual(ExpectedFailures._should_trust(results), can_trust)
Beispiel #17
0
 def begin_work_queue(self):
     AbstractReviewQueue.begin_work_queue(self)
     self._expected_failures = ExpectedFailures()
     self._layout_test_results_reader = LayoutTestResultsReader(
         self._tool, self._port.results_directory(), self._log_directory())
Beispiel #18
0
 def begin_work_queue(self):
     # FIXME: This violates abstraction
     self._tool._deprecated_port = self.port
     AbstractReviewQueue.begin_work_queue(self)
     self._expected_failures = ExpectedFailures()
     self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory())