def process_work_item(self, patch):
     self._cc_watchers(patch.bug_id())
     task = CommitQueueTask(self, patch)
     try:
         if task.run():
             self._did_pass(patch)
             return True
         self._did_retry(patch)
     except ScriptError, e:
         validator = CommitterValidator(self._tool.bugs)
         validator.reject_patch_from_commit_queue(patch.id(), self._error_message_for_bug(task.failure_status_id, e))
         self._did_fail(patch)
Beispiel #2
0
 def process_work_item(self, patch):
     self._cc_watchers(patch.bug_id())
     task = CommitQueueTask(self, patch)
     try:
         if task.run():
             self._did_pass(patch)
             return True
         self._did_retry(patch)
     except ScriptError, e:
         validator = CommitterValidator(self._tool.bugs)
         validator.reject_patch_from_commit_queue(
             patch.id(),
             self._error_message_for_bug(task.failure_status_id, e))
         self._did_fail(patch)
Beispiel #3
0
 def process_work_item(self, patch):
     self._cc_watchers(patch.bug_id())
     task = CommitQueueTask(self, patch)
     try:
         if task.run():
             self._did_pass(patch)
             return True
         self._did_retry(patch)
     except ScriptError, e:
         validator = CommitterValidator(self._tool)
         validator.reject_patch_from_commit_queue(patch.id(), self._error_message_for_bug(task, patch, e))
         results_archive = task.results_archive_from_patch_test_run(patch)
         if results_archive:
             self._upload_results_archive_for_patch(patch, results_archive)
         self._did_fail(patch)
Beispiel #4
0
 def process_work_item(self, patch):
     self._cc_watchers(patch.bug_id())
     task = CommitQueueTask(self, patch)
     try:
         if task.run():
             self._did_pass(patch)
             return True
         self._did_retry(patch)
     except ScriptError, e:
         validator = CommitterValidator(self._tool)
         validator.reject_patch_from_commit_queue(patch.id(), self._error_message_for_bug(task, patch, e))
         results_archive = task.results_archive_from_patch_test_run(patch)
         if results_archive:
             self._upload_results_archive_for_patch(patch, results_archive)
         self._did_fail(patch)
class CommitQueueFeeder(AbstractFeeder):
    queue_name = "commit-queue"

    def __init__(self, tool):
        AbstractFeeder.__init__(self, tool)
        self.committer_validator = CommitterValidator(self._tool.bugs)

    def _update_work_items(self, item_ids):
        # FIXME: This is the last use of update_work_items, the commit-queue
        # should move to feeding patches one at a time like the EWS does.
        self._tool.status_server.update_work_items(self.queue_name, item_ids)
        log("Feeding %s items %s" % (self.queue_name, item_ids))

    def feed(self):
        patches = self._validate_patches()
        patches = sorted(patches, self._patch_cmp)
        patch_ids = [patch.id() for patch in patches]
        self._update_work_items(patch_ids)

    def _patches_for_bug(self, bug_id):
        return self._tool.bugs.fetch_bug(bug_id).commit_queued_patches(include_invalid=True)

    def _validate_patches(self):
        # Not using BugzillaQueries.fetch_patches_from_commit_queue() so we can reject patches with invalid committers/reviewers.
        bug_ids = self._tool.bugs.queries.fetch_bug_ids_from_commit_queue()
        all_patches = sum([self._patches_for_bug(bug_id) for bug_id in bug_ids], [])
        return self.committer_validator.patches_after_rejecting_invalid_commiters_and_reviewers(all_patches)

    def _patch_cmp(self, a, b):
        # Sort first by is_rollout, then by attach_date.
        # Reversing the order so that is_rollout is first.
        rollout_cmp = cmp(b.is_rollout(), a.is_rollout())
        if rollout_cmp != 0:
            return rollout_cmp
        return cmp(a.attach_date(), b.attach_date())
Beispiel #6
0
class CommitQueueFeeder(AbstractFeeder):
    queue_name = "commit-queue"

    def __init__(self, tool):
        AbstractFeeder.__init__(self, tool)
        self.committer_validator = CommitterValidator(self._tool)

    def feed(self):
        patches = self._validate_patches()
        patches = self._patches_with_acceptable_review_flag(patches)
        patches = sorted(patches, self._patch_cmp)

        high_priority_item_ids = [patch.id() for patch in patches if patch.is_rollout()]
        item_ids = [patch.id() for patch in patches if not patch.is_rollout()]

        _log.info("Feeding %s high priority items %s, regular items %s" % (self.queue_name, high_priority_item_ids, item_ids))
        self._tool.status_server.update_work_items(self.queue_name, high_priority_item_ids, item_ids)

    def _patches_for_bug(self, bug_id):
        return self._tool.bugs.fetch_bug(bug_id).commit_queued_patches(include_invalid=True)

    # Filters out patches with r? or r-, only r+ or no review are OK to land.
    def _patches_with_acceptable_review_flag(self, patches):
        return [patch for patch in patches if patch.review() in [None, '+']]

    def _validate_patches(self):
        # Not using BugzillaQueries.fetch_patches_from_commit_queue() so we can reject patches with invalid committers/reviewers.
        bug_ids = self._tool.bugs.queries.fetch_bug_ids_from_commit_queue()
        all_patches = sum([self._patches_for_bug(bug_id) for bug_id in bug_ids], [])
        return self.committer_validator.patches_after_rejecting_invalid_commiters_and_reviewers(all_patches)

    def _patch_cmp(self, a, b):
        return cmp(a.attach_date(), b.attach_date())
Beispiel #7
0
class CommitQueueFeeder(AbstractFeeder):
    queue_name = "commit-queue"

    def __init__(self, tool):
        AbstractFeeder.__init__(self, tool)
        self.committer_validator = CommitterValidator(self._tool)

    def feed(self):
        patches = self._validate_patches()
        patches = self._patches_with_acceptable_review_flag(patches)
        patches = sorted(patches, key=lambda patch: patch.attach_date() or 0)

        high_priority_item_ids = [patch.id() for patch in patches if patch.is_rollout()]
        item_ids = [patch.id() for patch in patches if not patch.is_rollout()]

        _log.info("Feeding %s high priority items %s, regular items %s" % (self.queue_name, high_priority_item_ids, item_ids))
        self._tool.status_server.update_work_items(self.queue_name, high_priority_item_ids, item_ids)

    def _patches_for_bug(self, bug_id):
        return self._tool.bugs.fetch_bug(bug_id).commit_queued_patches(include_invalid=True)

    # Filters out patches with r? or r-, only r+ or no review are OK to land.
    def _patches_with_acceptable_review_flag(self, patches):
        return [patch for patch in patches if patch.review() in [None, '+']]

    def _validate_patches(self):
        # Not using BugzillaQueries.fetch_patches_from_commit_queue() so we can reject patches with invalid committers/reviewers.
        bug_ids = self._tool.bugs.queries.fetch_bug_ids_from_commit_queue()
        all_patches = sum([self._patches_for_bug(bug_id) for bug_id in bug_ids], [])
        return self.committer_validator.patches_after_rejecting_invalid_commiters_and_reviewers(all_patches)
Beispiel #8
0
 def begin_work_queue(self):
     # FIXME: This violates abstraction
     self._tool._deprecated_port = self.port
     AbstractPatchQueue.begin_work_queue(self)
     self.committer_validator = CommitterValidator(self._tool)
     self._expected_failures = ExpectedFailures()
     self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._log_directory())
Beispiel #9
0
 def process_work_item(self, patch):
     self._cc_watchers(patch.bug_id())
     task = self._commit_queue_task_class(self, patch)
     try:
         if task.run():
             self._did_pass(patch)
             return True
         self._unlock_patch(patch)
         return False
     except PatchIsNotValid as error:
         self._did_error(patch, "%s did not process patch. Reason: %s" % (self.name, error.failure_message))
         return False
     except ScriptError, e:
         validator = CommitterValidator(self._tool)
         validator.reject_patch_from_commit_queue(patch.id(), self._error_message_for_bug(task, patch, e))
         results_archive = task.results_archive_from_patch_test_run(patch)
         if results_archive:
             self._upload_results_archive_for_patch(patch, results_archive)
         self._did_fail(patch)
         return False
Beispiel #10
0
class CommitQueueFeeder(AbstractFeeder):
    queue_name = "commit-queue"

    def __init__(self, tool):
        AbstractFeeder.__init__(self, tool)
        self.committer_validator = CommitterValidator(self._tool)

    def _update_work_items(self, item_ids):
        # FIXME: This is the last use of update_work_items, the commit-queue
        # should move to feeding patches one at a time like the EWS does.
        self._tool.status_server.update_work_items(self.queue_name, item_ids)
        _log.info("Feeding %s items %s" % (self.queue_name, item_ids))

    def feed(self):
        patches = self._validate_patches()
        patches = self._patches_with_acceptable_review_flag(patches)
        patches = sorted(patches, self._patch_cmp)
        patch_ids = [patch.id() for patch in patches]
        self._update_work_items(patch_ids)

    def _patches_for_bug(self, bug_id):
        return self._tool.bugs.fetch_bug(bug_id).commit_queued_patches(
            include_invalid=True)

    # Filters out patches with r? or r-, only r+ or no review are OK to land.
    def _patches_with_acceptable_review_flag(self, patches):
        return [patch for patch in patches if patch.review() in [None, '+']]

    def _validate_patches(self):
        # Not using BugzillaQueries.fetch_patches_from_commit_queue() so we can reject patches with invalid committers/reviewers.
        bug_ids = self._tool.bugs.queries.fetch_bug_ids_from_commit_queue()
        all_patches = sum(
            [self._patches_for_bug(bug_id) for bug_id in bug_ids], [])
        return self.committer_validator.patches_after_rejecting_invalid_commiters_and_reviewers(
            all_patches)

    def _patch_cmp(self, a, b):
        # Sort first by is_rollout, then by attach_date.
        # Reversing the order so that is_rollout is first.
        rollout_cmp = cmp(b.is_rollout(), a.is_rollout())
        if rollout_cmp != 0:
            return rollout_cmp
        return cmp(a.attach_date(), b.attach_date())
Beispiel #11
0
 def begin_work_queue(self):
     PatchProcessingQueue.begin_work_queue(self)
     self.committer_validator = CommitterValidator(self._tool)
     self._layout_test_results_reader = LayoutTestResultsReader(
         self._tool, self._port.results_directory(), self._log_directory())
Beispiel #12
0
 def begin_work_queue(self):
     AbstractPatchQueue.begin_work_queue(self)
     self.committer_validator = CommitterValidator(self._tool.bugs)
Beispiel #13
0
class CommitQueue(PatchProcessingQueue, StepSequenceErrorHandler,
                  CommitQueueTaskDelegate):
    def __init__(self, commit_queue_task_class=CommitQueueTask):
        self._commit_queue_task_class = commit_queue_task_class
        PatchProcessingQueue.__init__(self)

    name = "commit-queue"
    port_name = "mac"

    # AbstractPatchQueue methods

    def begin_work_queue(self):
        PatchProcessingQueue.begin_work_queue(self)
        self.committer_validator = CommitterValidator(self._tool)
        self._layout_test_results_reader = LayoutTestResultsReader(
            self._tool, self._port.results_directory(), self._log_directory())

    def next_work_item(self):
        return self._next_patch()

    def process_work_item(self, patch):
        self._cc_watchers(patch.bug_id())
        task = self._commit_queue_task_class(self, patch)
        try:
            if task.run():
                self._did_pass(patch)
                return True
            self._unlock_patch(patch)
            return False
        except PatchIsNotValid as error:
            self._did_error(
                patch, "%s did not process patch. Reason: %s" %
                (self.name, error.failure_message))
            return False
        except ScriptError as e:
            if self._can_access_bug(patch.bug_id()):
                validator = CommitterValidator(self._tool)
                validator.reject_patch_from_commit_queue(
                    patch.id(), self._error_message_for_bug(task, patch, e))
            results_archive = task.results_archive_from_patch_test_run(patch)
            if results_archive:
                self._upload_results_archive_for_patch(patch, results_archive)
            self._did_fail(patch)
            return False

    def _failing_tests_message(self, task, patch):
        results = task.results_from_patch_test_run(patch)

        if not results:
            return None

        if results.did_exceed_test_failure_limit():
            return "Number of test failures exceeded the failure limit."
        return "New failing tests:\n%s" % "\n".join(results.failing_tests())

    def _error_message_for_bug(self, task, patch, script_error):
        message = self._failing_tests_message(task, patch)
        if not message:
            message = script_error.message_with_output(output_limit=5000)
        results_link = self._tool.status_server.results_url_for_status(
            task.failure_status_id)
        return "%s\nFull output: %s" % (message, results_link)

    def handle_unexpected_error(self, patch, message):
        self.committer_validator.reject_patch_from_commit_queue(
            patch.id(), message)

    # CommitQueueTaskDelegate methods

    def run_command(self, command):
        self.run_webkit_patch(command + [self._deprecated_port.flag()])

    def command_passed(self, message, patch):
        self._update_status(message, patch=patch)

    def command_failed(self, message, script_error, patch):
        failure_log = self._log_from_script_error_for_upload(script_error)
        return self._update_status(message,
                                   patch=patch,
                                   results_file=failure_log)

    def expected_failures(self):
        return self._expected_failures

    def test_results(self):
        return self._layout_test_results_reader.results()

    def archive_last_test_results(self, patch):
        return self._layout_test_results_reader.archive(patch)

    def refetch_patch(self, patch):
        return self._tool.bugs.fetch_attachment(patch.id())

    def report_flaky_tests(self,
                           patch,
                           flaky_test_results,
                           results_archive=None):
        reporter = FlakyTestReporter(self._tool, self.name)
        reporter.report_flaky_tests(patch, flaky_test_results, results_archive)

    def did_pass_testing_ews(self, patch):
        # Only Mac and Mac WK2 run tests
        # FIXME: We shouldn't have to hard-code it here.
        patch_status = self._tool.status_server.patch_status
        return patch_status("mac-ews",
                            patch.id()) == self._pass_status or patch_status(
                                "mac-wk2-ews", patch.id()) == self._pass_status

    # StepSequenceErrorHandler methods

    @classmethod
    def handle_script_error(cls, tool, state, script_error):
        # Hitting this error handler should be pretty rare.  It does occur,
        # however, when a patch no longer applies to top-of-tree in the final
        # land step.
        _log.error(script_error.message_with_output(output_limit=5000))

    @classmethod
    def handle_checkout_needs_update(cls, tool, state, options, error):
        message = "Tests passed, but commit failed (checkout out of date).  Updating, then landing without building or re-running tests."
        tool.status_server.update_status(cls.name, message, state["patch"])
        # The only time when we find out that out checkout needs update is
        # when we were ready to actually pull the trigger and land the patch.
        # Rather than spinning in the master process, we retry without
        # building or testing, which is much faster.
        options.build = False
        options.test = False
        options.update = True
        raise TryAgain()
Beispiel #14
0
 def __init__(self, tool):
     AbstractFeeder.__init__(self, tool)
     self.committer_validator = CommitterValidator(self._tool)
Beispiel #15
0
 def begin_work_queue(self):
     AbstractPatchQueue.begin_work_queue(self)
     self.committer_validator = CommitterValidator(self._tool)
     self._expected_failures = ExpectedFailures()
     self._layout_test_results_reader = LayoutTestResultsReader(
         self._tool, self._log_directory())