def test_failures_were_expected(self):
failures = ExpectedFailures()
failures.grow_expected_failures(MockResults(['foo.html']))
self._assert_expected(failures, ['foo.html'], True)
self._assert_expected(failures, ['bar.html'], False)
failures.shrink_expected_failures(MockResults(['baz.html']), False)
self._assert_expected(failures, ['foo.html'], False)
self._assert_expected(failures, ['baz.html'], False)
failures.grow_expected_failures(MockResults(['baz.html']))
self._assert_expected(failures, ['baz.html'], True)
failures.shrink_expected_failures(MockResults(), True)
self._assert_expected(failures, ['baz.html'], False)
class AbstractEarlyWarningSystem(AbstractReviewQueue, EarlyWarningSystemTaskDelegate):
_build_style = "release"
# FIXME: Switch _default_run_tests from opt-in to opt-out once more bots are ready to run tests.
_default_run_tests = False
def __init__(self):
options = [make_option("--run-tests", action="store_true", dest="run_tests", default=self._default_run_tests, help="Run the Layout tests for each patch")]
AbstractReviewQueue.__init__(self, options=options)
self.port = DeprecatedPort.port(self.port_name)
def should_proceed_with_work_item(self, patch):
return True
def begin_work_queue(self):
# FIXME: This violates abstraction
self._tool._deprecated_port = self.port
AbstractReviewQueue.begin_work_queue(self)
self._expected_failures = ExpectedFailures()
self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory())
def _failing_tests_message(self, task, patch):
results = task.results_from_patch_test_run(patch)
unexpected_failures = self._expected_failures.unexpected_failures_observed(results)
if not unexpected_failures:
return None
return "New failing tests:\n%s" % "\n".join(unexpected_failures)
def _post_reject_message_on_bug(self, tool, patch, status_id, extra_message_text=None):
results_link = tool.status_server.results_url_for_status(status_id)
message = "Attachment %s did not pass %s (%s):\nOutput: %s" % (patch.id(), self.name, self.port_name, results_link)
# FIXME: We might want to add some text about rejecting from the commit-queue in
# the case where patch.commit_queue() isn't already set to '-'.
if self.watchers:
tool.bugs.add_cc_to_bug(patch.bug_id(), self.watchers)
tool.bugs.set_flag_on_attachment(patch.id(), "commit-queue", "-", message, extra_message_text)
def review_patch(self, patch):
task = EarlyWarningSystemTask(self, patch, self._options.run_tests)
if not task.validate():
self._did_error(patch, "%s did not process patch." % self.name)
return False
try:
return task.run()
except UnableToApplyPatch, e:
self._did_error(patch, "%s unable to apply patch." % self.name)
return False
except ScriptError, e:
self._post_reject_message_on_bug(self._tool, patch, task.failure_status_id, self._failing_tests_message(task, patch))
results_archive = task.results_archive_from_patch_test_run(patch)
if results_archive:
self._upload_results_archive_for_patch(patch, results_archive)
self._did_fail(patch)
# FIXME: We're supposed to be able to raise e again here and have
# one of our base classes mark the patch as fail, but there seems
# to be an issue with the exit_code.
return False
def test_unexpected_failures_observed_when_tree_is_hosed(self):
failures = ExpectedFailures()
failures.grow_expected_failures(MockResults(['foo.html', 'banana.html'], failure_limit=2))
self.assertEquals(failures.unexpected_failures_observed(MockResults(['foo.html', 'bar.html'])), None)
self.assertEquals(failures.unexpected_failures_observed(MockResults(['baz.html'])), None)
unbounded_results = MockResults(['baz.html', 'qux.html', 'taco.html'], failure_limit=3)
self.assertEquals(failures.unexpected_failures_observed(unbounded_results), None)
unbounded_results_with_existing_failure = MockResults(['foo.html', 'baz.html', 'qux.html', 'taco.html'], failure_limit=4)
self.assertEquals(failures.unexpected_failures_observed(unbounded_results_with_existing_failure), None)
def test_unexpected_failures_observed(self):
failures = ExpectedFailures()
failures.update(MockResults(['foo.html']))
self.assertEqual(failures.unexpected_failures_observed(MockResults(['foo.html', 'bar.html'])), set(['bar.html']))
self.assertEqual(failures.unexpected_failures_observed(MockResults(['baz.html'])), set(['baz.html']))
unbounded_results = MockResults(['baz.html', 'qux.html', 'taco.html'], failure_limit=3)
self.assertEqual(failures.unexpected_failures_observed(unbounded_results), set(['baz.html', 'qux.html', 'taco.html']))
unbounded_results_with_existing_failure = MockResults(['foo.html', 'baz.html', 'qux.html', 'taco.html'], failure_limit=4)
self.assertEqual(failures.unexpected_failures_observed(unbounded_results_with_existing_failure), set(['baz.html', 'qux.html', 'taco.html']))
def test_failures_were_expected(self):
failures = ExpectedFailures()
failures.update(MockResults(['foo.html']))
self._assert_expected(failures, ['foo.html'], True)
self._assert_expected(failures, ['bar.html'], False)
self._assert_expected(failures, ['bar.html', 'foo.html'], False)
failures.update(MockResults(['baz.html']))
self._assert_expected(failures, ['baz.html'], True)
self._assert_expected(failures, ['foo.html'], False)
failures.update(MockResults([]))
self._assert_expected(failures, ['baz.html'], False)
self._assert_expected(failures, ['foo.html'], False)
class AbstractTestingEWS(AbstractEarlyWarningSystem, EarlyWarningSystemTaskDelegate):
def begin_work_queue(self):
# FIXME: This violates abstraction
self._tool._port = self.port
AbstractEarlyWarningSystem.begin_work_queue(self)
self._expected_failures = ExpectedFailures()
self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory())
def _post_reject_message_on_bug(self, task, patch):
results_link = self._tool.status_server.results_url_for_status(task.failure_status_id)
message = "Attachment %s did not pass %s:\nOutput: %s" % (patch.id(), self.name, results_link)
results = task.results_from_patch_test_run(patch)
unexpected_failures = self._expected_failures.unexpected_failures(results)
if unexpected_failures:
message += "\nNew failing tests:\n%s" % "\n".join(unexpected_failures)
self._tool.bugs.post_comment_to_bug(patch.bug_id(), message, cc=self.watchers)
def review_patch(self, patch):
task = EarlyWarningSystemTask(self, patch)
if not task.validate():
self._did_error(patch, "%s did not process patch." % self.name)
return False
try:
return task.run()
except UnableToApplyPatch, e:
self._did_error(patch, "%s unable to apply patch." % self.name)
return False
except ScriptError, e:
# FIXME: This should just use CommitterValidator.reject_patch_from_commit_queue
self._post_reject_message_on_bug(task, patch)
results_archive = task.results_archive_from_patch_test_run(patch)
if results_archive:
self._upload_results_archive_for_patch(patch, results_archive)
self._did_fail(patch)
# FIXME: We're supposed to be able to raise e again here and have
# one of our base classes mark the patch as fail, but there seems
# to be an issue with the exit_code.
return False
def _assert_can_trust(self, results, can_trust):
self.assertEquals(ExpectedFailures()._can_trust_results(results),
can_trust)
def begin_work_queue(self):
AbstractReviewQueue.begin_work_queue(self)
self._expected_failures = ExpectedFailures()
self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._port.results_directory(), self._log_directory())
def begin_work_queue(self):
# FIXME: This violates abstraction
self._tool._deprecated_port = self.port
AbstractReviewQueue.begin_work_queue(self)
self._expected_failures = ExpectedFailures()
self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory())
def begin_work_queue(self):
AbstractPatchQueue.begin_work_queue(self)
self.committer_validator = CommitterValidator(self._tool.bugs)
self._expected_failures = ExpectedFailures()
self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory())
def expected_failures(self):
return ExpectedFailures()
def begin_work_queue(self):
PatchProcessingQueue.begin_work_queue(self)
self.committer_validator = CommitterValidator(self._tool)
self._expected_failures = ExpectedFailures()
self._layout_test_results_reader = LayoutTestResultsReader(
self._tool, self._port.results_directory(), self._log_directory())
class AbstractEarlyWarningSystem(AbstractReviewQueue,
EarlyWarningSystemTaskDelegate):
_build_style = "release"
# FIXME: Switch _default_run_tests from opt-in to opt-out once more bots are ready to run tests.
_default_run_tests = False
def __init__(self):
options = [
make_option("--run-tests",
action="store_true",
dest="run_tests",
default=self._default_run_tests,
help="Run the Layout tests for each patch")
]
AbstractReviewQueue.__init__(self, options=options)
self.port = DeprecatedPort.port(self.port_name)
def begin_work_queue(self):
# FIXME: This violates abstraction
self._tool._deprecated_port = self.port
AbstractReviewQueue.begin_work_queue(self)
self._expected_failures = ExpectedFailures()
self._layout_test_results_reader = LayoutTestResultsReader(
self._tool, self._log_directory())
def _failing_tests_message(self, task, patch):
results = task.results_from_patch_test_run(patch)
unexpected_failures = self._expected_failures.unexpected_failures_observed(
results)
if not unexpected_failures:
return None
return "New failing tests:\n%s" % "\n".join(unexpected_failures)
def _post_reject_message_on_bug(self,
tool,
patch,
status_id,
extra_message_text=None):
results_link = tool.status_server.results_url_for_status(status_id)
message = "Attachment %s did not pass %s (%s):\nOutput: %s" % (
patch.id(), self.name, self.port_name, results_link)
# FIXME: We might want to add some text about rejecting from the commit-queue in
# the case where patch.commit_queue() isn't already set to '-'.
if self.watchers:
tool.bugs.add_cc_to_bug(patch.bug_id(), self.watchers)
tool.bugs.set_flag_on_attachment(patch.id(), "commit-queue", "-",
message, extra_message_text)
def review_patch(self, patch):
task = EarlyWarningSystemTask(self, patch, self._options.run_tests)
if not task.validate():
self._did_error(patch, "%s did not process patch." % self.name)
return False
try:
return task.run()
except UnableToApplyPatch, e:
self._did_error(patch, "%s unable to apply patch." % self.name)
return False
except ScriptError, e:
self._post_reject_message_on_bug(
self._tool, patch, task.failure_status_id,
self._failing_tests_message(task, patch))
results_archive = task.results_archive_from_patch_test_run(patch)
if results_archive:
self._upload_results_archive_for_patch(patch, results_archive)
self._did_fail(patch)
# FIXME: We're supposed to be able to raise e again here and have
# one of our base classes mark the patch as fail, but there seems
# to be an issue with the exit_code.
return False
def _assert_can_trust(self, results, can_trust):
self.assertEqual(ExpectedFailures._should_trust(results), can_trust)
def test_unexpected_failures(self):
failures = ExpectedFailures()
failures.grow_expected_failures(MockResults(['foo.html']))
self.assertEquals(failures.unexpected_failures(MockResults(['foo.html', 'bar.html'])), set(['bar.html']))
self.assertEquals(failures.unexpected_failures(MockResults(['baz.html'])), set(['baz.html']))
def _assert_can_trust(self, results, can_trust):
self.assertEqual(ExpectedFailures._should_trust(results), can_trust)
def begin_work_queue(self):
AbstractReviewQueue.begin_work_queue(self)
self._expected_failures = ExpectedFailures()
self._layout_test_results_reader = LayoutTestResultsReader(
self._tool, self._port.results_directory(), self._log_directory())
def begin_work_queue(self):
# FIXME: This violates abstraction
self._tool._deprecated_port = self.port
AbstractReviewQueue.begin_work_queue(self)
self._expected_failures = ExpectedFailures()
self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory())