def process_work_item(self, patch):
self._cc_watchers(patch.bug_id())
task = CommitQueueTask(self, patch)
try:
if task.run():
self._did_pass(patch)
return True
self._did_retry(patch)
except ScriptError, e:
validator = CommitterValidator(self._tool.bugs)
validator.reject_patch_from_commit_queue(patch.id(), self._error_message_for_bug(task.failure_status_id, e))
self._did_fail(patch)
def process_work_item(self, patch):
self._cc_watchers(patch.bug_id())
task = CommitQueueTask(self, patch)
try:
if task.run():
self._did_pass(patch)
return True
self._did_retry(patch)
except ScriptError, e:
validator = CommitterValidator(self._tool.bugs)
validator.reject_patch_from_commit_queue(
patch.id(),
self._error_message_for_bug(task.failure_status_id, e))
self._did_fail(patch)
def process_work_item(self, patch):
self._cc_watchers(patch.bug_id())
task = CommitQueueTask(self, patch)
try:
if task.run():
self._did_pass(patch)
return True
self._did_retry(patch)
except ScriptError, e:
validator = CommitterValidator(self._tool)
validator.reject_patch_from_commit_queue(patch.id(), self._error_message_for_bug(task, patch, e))
results_archive = task.results_archive_from_patch_test_run(patch)
if results_archive:
self._upload_results_archive_for_patch(patch, results_archive)
self._did_fail(patch)
def process_work_item(self, patch):
self._cc_watchers(patch.bug_id())
task = CommitQueueTask(self, patch)
try:
if task.run():
self._did_pass(patch)
return True
self._did_retry(patch)
except ScriptError, e:
validator = CommitterValidator(self._tool)
validator.reject_patch_from_commit_queue(patch.id(), self._error_message_for_bug(task, patch, e))
results_archive = task.results_archive_from_patch_test_run(patch)
if results_archive:
self._upload_results_archive_for_patch(patch, results_archive)
self._did_fail(patch)
class CommitQueueFeeder(AbstractFeeder):
queue_name = "commit-queue"
def __init__(self, tool):
AbstractFeeder.__init__(self, tool)
self.committer_validator = CommitterValidator(self._tool.bugs)
def _update_work_items(self, item_ids):
# FIXME: This is the last use of update_work_items, the commit-queue
# should move to feeding patches one at a time like the EWS does.
self._tool.status_server.update_work_items(self.queue_name, item_ids)
log("Feeding %s items %s" % (self.queue_name, item_ids))
def feed(self):
patches = self._validate_patches()
patches = sorted(patches, self._patch_cmp)
patch_ids = [patch.id() for patch in patches]
self._update_work_items(patch_ids)
def _patches_for_bug(self, bug_id):
return self._tool.bugs.fetch_bug(bug_id).commit_queued_patches(include_invalid=True)
def _validate_patches(self):
# Not using BugzillaQueries.fetch_patches_from_commit_queue() so we can reject patches with invalid committers/reviewers.
bug_ids = self._tool.bugs.queries.fetch_bug_ids_from_commit_queue()
all_patches = sum([self._patches_for_bug(bug_id) for bug_id in bug_ids], [])
return self.committer_validator.patches_after_rejecting_invalid_commiters_and_reviewers(all_patches)
def _patch_cmp(self, a, b):
# Sort first by is_rollout, then by attach_date.
# Reversing the order so that is_rollout is first.
rollout_cmp = cmp(b.is_rollout(), a.is_rollout())
if rollout_cmp != 0:
return rollout_cmp
return cmp(a.attach_date(), b.attach_date())
class CommitQueueFeeder(AbstractFeeder):
queue_name = "commit-queue"
def __init__(self, tool):
AbstractFeeder.__init__(self, tool)
self.committer_validator = CommitterValidator(self._tool)
def feed(self):
patches = self._validate_patches()
patches = self._patches_with_acceptable_review_flag(patches)
patches = sorted(patches, self._patch_cmp)
high_priority_item_ids = [patch.id() for patch in patches if patch.is_rollout()]
item_ids = [patch.id() for patch in patches if not patch.is_rollout()]
_log.info("Feeding %s high priority items %s, regular items %s" % (self.queue_name, high_priority_item_ids, item_ids))
self._tool.status_server.update_work_items(self.queue_name, high_priority_item_ids, item_ids)
def _patches_for_bug(self, bug_id):
return self._tool.bugs.fetch_bug(bug_id).commit_queued_patches(include_invalid=True)
# Filters out patches with r? or r-, only r+ or no review are OK to land.
def _patches_with_acceptable_review_flag(self, patches):
return [patch for patch in patches if patch.review() in [None, '+']]
def _validate_patches(self):
# Not using BugzillaQueries.fetch_patches_from_commit_queue() so we can reject patches with invalid committers/reviewers.
bug_ids = self._tool.bugs.queries.fetch_bug_ids_from_commit_queue()
all_patches = sum([self._patches_for_bug(bug_id) for bug_id in bug_ids], [])
return self.committer_validator.patches_after_rejecting_invalid_commiters_and_reviewers(all_patches)
def _patch_cmp(self, a, b):
return cmp(a.attach_date(), b.attach_date())
class CommitQueueFeeder(AbstractFeeder):
queue_name = "commit-queue"
def __init__(self, tool):
AbstractFeeder.__init__(self, tool)
self.committer_validator = CommitterValidator(self._tool)
def feed(self):
patches = self._validate_patches()
patches = self._patches_with_acceptable_review_flag(patches)
patches = sorted(patches, key=lambda patch: patch.attach_date() or 0)
high_priority_item_ids = [patch.id() for patch in patches if patch.is_rollout()]
item_ids = [patch.id() for patch in patches if not patch.is_rollout()]
_log.info("Feeding %s high priority items %s, regular items %s" % (self.queue_name, high_priority_item_ids, item_ids))
self._tool.status_server.update_work_items(self.queue_name, high_priority_item_ids, item_ids)
def _patches_for_bug(self, bug_id):
return self._tool.bugs.fetch_bug(bug_id).commit_queued_patches(include_invalid=True)
# Filters out patches with r? or r-, only r+ or no review are OK to land.
def _patches_with_acceptable_review_flag(self, patches):
return [patch for patch in patches if patch.review() in [None, '+']]
def _validate_patches(self):
# Not using BugzillaQueries.fetch_patches_from_commit_queue() so we can reject patches with invalid committers/reviewers.
bug_ids = self._tool.bugs.queries.fetch_bug_ids_from_commit_queue()
all_patches = sum([self._patches_for_bug(bug_id) for bug_id in bug_ids], [])
return self.committer_validator.patches_after_rejecting_invalid_commiters_and_reviewers(all_patches)
def begin_work_queue(self):
# FIXME: This violates abstraction
self._tool._deprecated_port = self.port
AbstractPatchQueue.begin_work_queue(self)
self.committer_validator = CommitterValidator(self._tool)
self._expected_failures = ExpectedFailures()
self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory())
def process_work_item(self, patch):
self._cc_watchers(patch.bug_id())
task = self._commit_queue_task_class(self, patch)
try:
if task.run():
self._did_pass(patch)
return True
self._unlock_patch(patch)
return False
except PatchIsNotValid as error:
self._did_error(patch, "%s did not process patch. Reason: %s" % (self.name, error.failure_message))
return False
except ScriptError, e:
validator = CommitterValidator(self._tool)
validator.reject_patch_from_commit_queue(patch.id(), self._error_message_for_bug(task, patch, e))
results_archive = task.results_archive_from_patch_test_run(patch)
if results_archive:
self._upload_results_archive_for_patch(patch, results_archive)
self._did_fail(patch)
return False
class CommitQueueFeeder(AbstractFeeder):
queue_name = "commit-queue"
def __init__(self, tool):
AbstractFeeder.__init__(self, tool)
self.committer_validator = CommitterValidator(self._tool)
def _update_work_items(self, item_ids):
# FIXME: This is the last use of update_work_items, the commit-queue
# should move to feeding patches one at a time like the EWS does.
self._tool.status_server.update_work_items(self.queue_name, item_ids)
_log.info("Feeding %s items %s" % (self.queue_name, item_ids))
def feed(self):
patches = self._validate_patches()
patches = self._patches_with_acceptable_review_flag(patches)
patches = sorted(patches, self._patch_cmp)
patch_ids = [patch.id() for patch in patches]
self._update_work_items(patch_ids)
def _patches_for_bug(self, bug_id):
return self._tool.bugs.fetch_bug(bug_id).commit_queued_patches(
include_invalid=True)
# Filters out patches with r? or r-, only r+ or no review are OK to land.
def _patches_with_acceptable_review_flag(self, patches):
return [patch for patch in patches if patch.review() in [None, '+']]
def _validate_patches(self):
# Not using BugzillaQueries.fetch_patches_from_commit_queue() so we can reject patches with invalid committers/reviewers.
bug_ids = self._tool.bugs.queries.fetch_bug_ids_from_commit_queue()
all_patches = sum(
[self._patches_for_bug(bug_id) for bug_id in bug_ids], [])
return self.committer_validator.patches_after_rejecting_invalid_commiters_and_reviewers(
all_patches)
def _patch_cmp(self, a, b):
# Sort first by is_rollout, then by attach_date.
# Reversing the order so that is_rollout is first.
rollout_cmp = cmp(b.is_rollout(), a.is_rollout())
if rollout_cmp != 0:
return rollout_cmp
return cmp(a.attach_date(), b.attach_date())
def begin_work_queue(self):
PatchProcessingQueue.begin_work_queue(self)
self.committer_validator = CommitterValidator(self._tool)
self._layout_test_results_reader = LayoutTestResultsReader(
self._tool, self._port.results_directory(), self._log_directory())
def begin_work_queue(self):
AbstractPatchQueue.begin_work_queue(self)
self.committer_validator = CommitterValidator(self._tool.bugs)
class CommitQueue(PatchProcessingQueue, StepSequenceErrorHandler,
CommitQueueTaskDelegate):
def __init__(self, commit_queue_task_class=CommitQueueTask):
self._commit_queue_task_class = commit_queue_task_class
PatchProcessingQueue.__init__(self)
name = "commit-queue"
port_name = "mac"
# AbstractPatchQueue methods
def begin_work_queue(self):
PatchProcessingQueue.begin_work_queue(self)
self.committer_validator = CommitterValidator(self._tool)
self._layout_test_results_reader = LayoutTestResultsReader(
self._tool, self._port.results_directory(), self._log_directory())
def next_work_item(self):
return self._next_patch()
def process_work_item(self, patch):
self._cc_watchers(patch.bug_id())
task = self._commit_queue_task_class(self, patch)
try:
if task.run():
self._did_pass(patch)
return True
self._unlock_patch(patch)
return False
except PatchIsNotValid as error:
self._did_error(
patch, "%s did not process patch. Reason: %s" %
(self.name, error.failure_message))
return False
except ScriptError as e:
if self._can_access_bug(patch.bug_id()):
validator = CommitterValidator(self._tool)
validator.reject_patch_from_commit_queue(
patch.id(), self._error_message_for_bug(task, patch, e))
results_archive = task.results_archive_from_patch_test_run(patch)
if results_archive:
self._upload_results_archive_for_patch(patch, results_archive)
self._did_fail(patch)
return False
def _failing_tests_message(self, task, patch):
results = task.results_from_patch_test_run(patch)
if not results:
return None
if results.did_exceed_test_failure_limit():
return "Number of test failures exceeded the failure limit."
return "New failing tests:\n%s" % "\n".join(results.failing_tests())
def _error_message_for_bug(self, task, patch, script_error):
message = self._failing_tests_message(task, patch)
if not message:
message = script_error.message_with_output(output_limit=5000)
results_link = self._tool.status_server.results_url_for_status(
task.failure_status_id)
return "%s\nFull output: %s" % (message, results_link)
def handle_unexpected_error(self, patch, message):
self.committer_validator.reject_patch_from_commit_queue(
patch.id(), message)
# CommitQueueTaskDelegate methods
def run_command(self, command):
self.run_webkit_patch(command + [self._deprecated_port.flag()])
def command_passed(self, message, patch):
self._update_status(message, patch=patch)
def command_failed(self, message, script_error, patch):
failure_log = self._log_from_script_error_for_upload(script_error)
return self._update_status(message,
patch=patch,
results_file=failure_log)
def expected_failures(self):
return self._expected_failures
def test_results(self):
return self._layout_test_results_reader.results()
def archive_last_test_results(self, patch):
return self._layout_test_results_reader.archive(patch)
def refetch_patch(self, patch):
return self._tool.bugs.fetch_attachment(patch.id())
def report_flaky_tests(self,
patch,
flaky_test_results,
results_archive=None):
reporter = FlakyTestReporter(self._tool, self.name)
reporter.report_flaky_tests(patch, flaky_test_results, results_archive)
def did_pass_testing_ews(self, patch):
# Only Mac and Mac WK2 run tests
# FIXME: We shouldn't have to hard-code it here.
patch_status = self._tool.status_server.patch_status
return patch_status("mac-ews",
patch.id()) == self._pass_status or patch_status(
"mac-wk2-ews", patch.id()) == self._pass_status
# StepSequenceErrorHandler methods
@classmethod
def handle_script_error(cls, tool, state, script_error):
# Hitting this error handler should be pretty rare. It does occur,
# however, when a patch no longer applies to top-of-tree in the final
# land step.
_log.error(script_error.message_with_output(output_limit=5000))
@classmethod
def handle_checkout_needs_update(cls, tool, state, options, error):
message = "Tests passed, but commit failed (checkout out of date). Updating, then landing without building or re-running tests."
tool.status_server.update_status(cls.name, message, state["patch"])
# The only time when we find out that out checkout needs update is
# when we were ready to actually pull the trigger and land the patch.
# Rather than spinning in the master process, we retry without
# building or testing, which is much faster.
options.build = False
options.test = False
options.update = True
raise TryAgain()
def __init__(self, tool):
AbstractFeeder.__init__(self, tool)
self.committer_validator = CommitterValidator(self._tool)
def begin_work_queue(self):
AbstractPatchQueue.begin_work_queue(self)
self.committer_validator = CommitterValidator(self._tool)
self._expected_failures = ExpectedFailures()
self._layout_test_results_reader = LayoutTestResultsReader(
self._tool, self._log_directory())