def ensure_clean_working_directory(self, force_clean): if not force_clean and not self.working_directory_is_clean(): print run_command(self.status_command(), error_handler=Executive.ignore_error) raise ScriptError(message="Working directory has modifications, pass --force-clean or --no-clean to continue.") log("Cleaning working directory") self.clean_working_directory()
def bug_id_for_attachment_id(self, attachment_id): self.authenticate() attachment_url = self.attachment_url_for_id(attachment_id, 'edit') log("Fetching: %s" % attachment_url) page = self.browser.open(attachment_url) return self._parse_bug_id_from_attachment_page(page)
def begin_work_queue(self): log("CAUTION: %s will discard all local changes in \"%s\"" % (self.name, self.tool.scm().checkout_root)) if self.options.confirm: response = self.tool.user.prompt("Are you sure? Type \"yes\" to continue: ") if (response != "yes"): error("User declined.") log("Running WebKit %s." % self.name)
def _fetch_list_of_patches_to_process(self, options, args, tool): all_patches = [] for bug_id in args: patches = tool.bugs.fetch_bug(bug_id).reviewed_patches() log("%s found on bug %s." % (pluralize("reviewed patch", len(patches)), bug_id)) all_patches += patches return all_patches
def run(self): self._begin_logging() self._delegate.begin_work_queue() while (self._delegate.should_continue_work_queue()): try: self._ensure_work_log_closed() work_item = self._delegate.next_work_item() if not work_item: self._sleep("No work item.") continue if not self._delegate.should_proceed_with_work_item(work_item): self._sleep("Not proceeding with work item.") continue # FIXME: Work logs should not depend on bug_id specificaly. # This looks fixed, no? self._open_work_log(work_item) try: self._delegate.process_work_item(work_item) except ScriptError, e: # Use a special exit code to indicate that the error was already # handled in the child process and we should just keep looping. if e.exit_code == self.handled_error_code: continue message = "Unexpected failure when landing patch! Please file a bug against webkit-patch.\n%s" % e.message_with_output() self._delegate.handle_unexpected_error(work_item, message) except KeyboardInterrupt, e: log("\nUser terminated queue.") return 1 except Exception, e: traceback.print_exc() # Don't try tell the status bot, in case telling it causes an exception. self._sleep("Exception while preparing queue")
def update_status(self, queue_name, status, patch=None, results_file=None): # During unit testing, host is None if not self.host: return log(status) return NetworkTransaction().run(lambda: self._post_to_server(queue_name, status, patch, results_file))
def __init__(self, bot_class=IRCBot, password=None): log("Connecting to IRC") self._message_queue = ThreadedMessageQueue() self._child_thread = _IRCThread(self._message_queue, bot_class, password=password) self._child_thread.start()
def update_status(self, queue_name, status, patch=None, results_file=None): # During unit testing, host is None if not self.host: return log(status) return NetworkTransaction().run(lambda: self._post_to_server( queue_name, status, patch, results_file))
def prompt_for_component(self, components): log("Please pick a component:") i = 0 for name in components: i += 1 log("%2d. %s" % (i, name)) result = int(User.prompt("Enter a number: ")) - 1 return components[result]
def _guess_reviewer_from_bug(self, bug_id): patches = self._tool.bugs.fetch_bug(bug_id).reviewed_patches() if len(patches) != 1: log("%s on bug %s, cannot infer reviewer." % (pluralize("reviewed patch", len(patches)), bug_id)) return None patch = patches[0] log("Guessing \"%s\" as reviewer from attachment %s on bug %s." % (patch.reviewer().full_name, patch.id(), bug_id)) return patch.reviewer().full_name
def check_arguments_and_execute(self, options, args, tool=None): if len(args) < len(self.required_arguments): log("%s required, %s provided. Provided: %s Required: %s\nSee '%s help %s' for usage." % (pluralize("argument", len(self.required_arguments)), pluralize("argument", len(args)), "'%s'" % " ".join(args), " ".join(self.required_arguments), tool.name(), self.name)) return 1 return self.execute(options, args, tool) or 0
def run(self, state): if not self._options.build: return log("Building WebKit") if self._options.build_style == "both": self.build("debug") self.build("release") else: self.build(self._options.build_style)
def run_and_handle_errors(self, tool, options, state=None): if not state: state = {} try: self._run(tool, options, state) except CheckoutNeedsUpdate, e: log("Commit failed because the checkout is out of date. Please update and try again.") log("You can pass --no-build to skip building/testing after update if you believe the new commits did not affect the results.") QueueEngine.exit_after_handled_error(e)
def run(self, state): commit_comment = bug_comment_from_commit_text(self._tool.scm(), state["commit_text"]) comment_text = "Reverted r%s for reason:\n\n%s\n\n%s" % (state["revision"], state["reason"], commit_comment) bug_id = state["bug_id"] if not bug_id: log(comment_text) log("No bugs were updated.") return self._tool.bugs.reopen_bug(bug_id, comment_text)
def execute(self, options, args, tool): if args: bug_ids = self._find_bugs_in_iterable(args) else: # This won't open bugs until stdin is closed but could be made to easily. That would just make unit testing slightly harder. bug_ids = self._find_bugs_in_iterable(sys.stdin) log("%s bugs found in input." % len(bug_ids)) self._open_bugs(bug_ids)
def run(self, request): self._total_sleep = 0 self._backoff_seconds = self._initial_backoff_seconds while True: try: return request() except HTTPError, e: self._check_for_timeout() log("Received HTTP status %s from server. Retrying in %s seconds..." % (e.code, self._backoff_seconds)) self._sleep()
def execute(self, options, args, tool): self._prepare_to_process(options, args, tool) patches = self._fetch_list_of_patches_to_process(options, args, tool) # It's nice to print out total statistics. bugs_to_patches = self._collect_patches_by_bug(patches) log("Processing %s from %s." % (pluralize("patch", len(patches)), pluralize("bug", len(bugs_to_patches)))) for patch in patches: self._process_patch(patch, options, args, tool)
def run(self, state): if not self._options.obsolete_patches: return bug_id = state["bug_id"] patches = self._tool.bugs.fetch_bug(bug_id).patches() if not patches: return log("Obsoleting %s on bug %s" % (pluralize("old patch", len(patches)), bug_id)) for patch in patches: self._tool.bugs.obsolete_attachment(patch.id())
def _needs_commit_queue(patch): if patch.commit_queue() == "+": # If it's already cq+, ignore the patch. log("%s already has cq=%s" % (patch.id(), patch.commit_queue())) return False # We only need to worry about patches from contributers who are not yet committers. committer_record = CommitterList().committer_by_email(patch.attacher_email()) if committer_record: log("%s committer = %s" % (patch.id(), committer_record)) return not committer_record
def apply_reverse_diff(self, revision): # '-c -revision' applies the inverse diff of 'revision' svn_merge_args = [ 'svn', 'merge', '--non-interactive', '-c', '-%s' % revision, self._repository_url() ] log("WARNING: svn merge has been known to take more than 10 minutes to complete. It is recommended you use git for rollouts." ) log("Running '%s'" % " ".join(svn_merge_args)) run_command(svn_merge_args)
def _validate_flag_value(self, flag): email = self._attachment_dictionary.get("%s_email" % flag) if not email: return None committer = getattr(self._bugzilla().committers, "%s_by_email" % flag)(email) if committer: return committer log("Warning, attachment %s on bug %s has invalid %s (%s)" % ( self._attachment_dictionary['id'], self._attachment_dictionary['bug_id'], flag, email))
def _parse_entry(self): match = re.match(self.date_line_regexp, self._contents, re.MULTILINE) if not match: log("WARNING: Creating invalid ChangeLogEntry:\n%s" % contents) # FIXME: group("name") does not seem to be Unicode? Probably due to self._contents not being unicode. self._author_name = match.group("name") if match else None self._author_email = match.group("email") if match else None match = re.search("^\s+Reviewed by (?P<reviewer>.*?)[\.,]?\s*$", self._contents, re.MULTILINE) # Discard everything after the first period self._reviewer_text = match.group("reviewer") if match else None
def check_arguments_and_execute(self, options, args, tool=None): if len(args) < len(self.required_arguments): log("%s required, %s provided. Provided: %s Required: %s\nSee '%s help %s' for usage." % ( pluralize("argument", len(self.required_arguments)), pluralize("argument", len(args)), "'%s'" % " ".join(args), " ".join(self.required_arguments), tool.name(), self.name)) return 1 return self.execute(options, args, tool) or 0
def ensure_clean_working_directory(self, force_clean): if not force_clean and not self.working_directory_is_clean(): print run_command(self.status_command(), error_handler=Executive.ignore_error) raise ScriptError( message= "Working directory has modifications, pass --force-clean or --no-clean to continue." ) log("Cleaning working directory") self.clean_working_directory()
def run(self, state): if not self._options.close_bug: return # Check to make sure there are no r? or r+ patches on the bug before closing. # Assume that r- patches are just previous patches someone forgot to obsolete. patches = self._tool.bugs.fetch_bug(state["patch"].bug_id()).patches() for patch in patches: if patch.review() == "?" or patch.review() == "+": log("Not closing bug %s as attachment %s has review=%s. Assuming there are more patches to land from this bug." % (patch.bug_id(), patch.id(), patch.review())) return self._tool.bugs.close_bug_as_fixed(state["patch"].bug_id(), "All reviewed patches have been landed. Closing bug.")
def add_cc_to_bug(self, bug_id, email_address_list): self.authenticate() log("Adding %s to the CC list for bug %s" % (email_address_list, bug_id)) if self.dryrun: return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") self.browser["newcc"] = ", ".join(email_address_list) self.browser.submit()
def run(self, state): commit_comment = bug_comment_from_commit_text(self._tool.scm(), state["commit_text"]) comment_text = "Reverted r%s for reason:\n\n%s\n\n%s" % ( state["revision"], state["reason"], commit_comment) bug_id = state["bug_id"] if not bug_id: log(comment_text) log("No bugs were updated.") return self._tool.bugs.reopen_bug(bug_id, comment_text)
def run(self, state): if not self._options.build: return if not self._options.test: return # Run the scripting unit tests first because they're quickest. log("Running Python unit tests") self._tool.executive.run_and_throw_if_fail( self.port().run_python_unittests_command()) log("Running Perl unit tests") self._tool.executive.run_and_throw_if_fail( self.port().run_perl_unittests_command()) log("Running JavaScriptCore tests") self._tool.executive.run_and_throw_if_fail( self.port().run_javascriptcore_tests_command(), quiet=True) log("Running run-webkit-tests") args = self.port().run_webkit_tests_command() if self._options.non_interactive: args.append("--no-launch-safari") args.append("--exit-after-n-failures=1") if self._options.quiet: args.append("--quiet") self._tool.executive.run_and_throw_if_fail(args)
def post_comment_to_bug(self, bug_id, comment_text, cc=None): self.authenticate() log("Adding comment to bug %s" % bug_id) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") self.browser["comment"] = comment_text if cc: self.browser["newcc"] = ", ".join(cc) self.browser.submit()
def _assign_bug_to_last_patch_attacher(self, bug_id): committers = CommitterList() bug = self.tool.bugs.fetch_bug(bug_id) assigned_to_email = bug.assigned_to_email() if assigned_to_email != self.tool.bugs.unassigned_email: log("Bug %s is already assigned to %s (%s)." % (bug_id, assigned_to_email, committers.committer_by_email(assigned_to_email))) return reviewed_patches = bug.reviewed_patches() if not reviewed_patches: log("Bug %s has no non-obsolete patches, ignoring." % bug_id) return # We only need to do anything with this bug if one of the r+'d patches does not have a valid committer (cq+ set). if self._patches_have_commiters(reviewed_patches): log("All reviewed patches on bug %s already have commit-queue+, ignoring." % bug_id) return latest_patch = reviewed_patches[-1] attacher_email = latest_patch.attacher_email() committer = committers.committer_by_email(attacher_email) if not committer: log("Attacher %s is not a committer. Bug %s likely needs commit-queue+." % (attacher_email, bug_id)) return reassign_message = "Attachment %s was posted by a committer and has review+, assigning to %s for commit." % ( latest_patch.id(), committer.full_name) self.tool.bugs.reassign_bug(bug_id, committer.bugzilla_email(), reassign_message)
def _assign_bug_to_last_patch_attacher(self, bug_id): committers = CommitterList() bug = self.tool.bugs.fetch_bug(bug_id) assigned_to_email = bug.assigned_to_email() if assigned_to_email != self.tool.bugs.unassigned_email: log("Bug %s is already assigned to %s (%s)." % (bug_id, assigned_to_email, committers.committer_by_email(assigned_to_email))) return reviewed_patches = bug.reviewed_patches() if not reviewed_patches: log("Bug %s has no non-obsolete patches, ignoring." % bug_id) return # We only need to do anything with this bug if one of the r+'d patches does not have a valid committer (cq+ set). if self._patches_have_commiters(reviewed_patches): log("All reviewed patches on bug %s already have commit-queue+, ignoring." % bug_id) return latest_patch = reviewed_patches[-1] attacher_email = latest_patch.attacher_email() committer = committers.committer_by_email(attacher_email) if not committer: log("Attacher %s is not a committer. Bug %s likely needs commit-queue+." % (attacher_email, bug_id)) return reassign_message = "Attachment %s was posted by a committer and has review+, assigning to %s for commit." % (latest_patch.id(), committer.full_name) self.tool.bugs.reassign_bug(bug_id, committer.bugzilla_email(), reassign_message)
def execute(self, options, args, tool): bug_id = options.bug_id svn_revision = args and args[0] if svn_revision: if re.match("^r[0-9]+$", svn_revision, re.IGNORECASE): svn_revision = svn_revision[1:] if not re.match("^[0-9]+$", svn_revision): error("Invalid svn revision: '%s'" % svn_revision) needs_prompt = False if not bug_id or not svn_revision: needs_prompt = True (bug_id, svn_revision) = self._determine_bug_id_and_svn_revision(tool, bug_id, svn_revision) log("Bug: <%s> %s" % (tool.bugs.bug_url_for_bug_id(bug_id), tool.bugs.fetch_bug_dictionary(bug_id)["title"])) log("Revision: %s" % svn_revision) if options.open_bug: tool.user.open_url(tool.bugs.bug_url_for_bug_id(bug_id)) if needs_prompt: if not tool.user.confirm("Is this correct?"): exit(1) bug_comment = bug_comment_from_svn_revision(svn_revision) if options.comment: bug_comment = "%s\n\n%s" % (options.comment, bug_comment) if options.update_only: log("Adding comment to Bug %s." % bug_id) tool.bugs.post_comment_to_bug(bug_id, bug_comment) else: log("Adding comment to Bug %s and marking as Resolved/Fixed." % bug_id) tool.bugs.close_bug_as_fixed(bug_id, bug_comment)
def scm(self): # Lazily initialize SCM to not error-out before command line parsing (or when running non-scm commands). original_cwd = os.path.abspath(".") if not self._scm: self._scm = detect_scm_system(original_cwd) if not self._scm: script_directory = os.path.abspath(sys.path[0]) self._scm = detect_scm_system(script_directory) if self._scm: log("The current directory (%s) is not a WebKit checkout, using %s" % (original_cwd, self._scm.checkout_root)) else: error("FATAL: Failed to determine the SCM system for either %s or %s" % (original_cwd, script_directory)) return self._scm
class StepSequence(object): def __init__(self, steps): self._steps = steps or [] def options(self): collected_options = [ steps.Options.parent_command, steps.Options.quiet, ] for step in self._steps: collected_options = collected_options + step.options() # Remove duplicates. collected_options = sorted(set(collected_options)) return collected_options def _run(self, tool, options, state): for step in self._steps: step(tool, options).run(state) def run_and_handle_errors(self, tool, options, state=None): if not state: state = {} try: self._run(tool, options, state) except CheckoutNeedsUpdate, e: log("Commit failed because the checkout is out of date. Please update and try again.") log("You can pass --no-build to skip building/testing after update if you believe the new commits did not affect the results.") QueueEngine.exit_after_handled_error(e) except ScriptError, e: if not options.quiet: log(e.message_with_output()) if options.parent_command: command = tool.command_by_name(options.parent_command) command.handle_script_error(tool, state, e) QueueEngine.exit_after_handled_error(e)
def execute(self, options, args, tool): revision = args[0] reason = args[1] bug_id = self._parse_bug_id_from_revision_diff(tool, revision) if options.complete_rollout: if bug_id: log("Will re-open bug %s after rollout." % bug_id) else: log("Failed to parse bug number from diff. No bugs will be updated/reopened after the rollout.") state = { "revision" : revision, "bug_id" : bug_id, "reason" : reason, } self._sequence.run_and_handle_errors(tool, options, state)
def commit_message_for_this_commit(self): changelog_paths = self.modified_changelogs() if not len(changelog_paths): raise ScriptError(message="Found no modified ChangeLogs, cannot create a commit message.\n" "All changes require a ChangeLog. See:\n" "http://webkit.org/coding/contributing.html") changelog_messages = [] for changelog_path in changelog_paths: log("Parsing ChangeLog: %s" % changelog_path) changelog_entry = ChangeLog(changelog_path).latest_entry() if not changelog_entry: raise ScriptError(message="Failed to parse ChangeLog: " + os.path.abspath(changelog_path)) changelog_messages.append(changelog_entry) # FIXME: We should sort and label the ChangeLog messages like commit-log-editor does. return CommitMessage("".join(changelog_messages).splitlines())
def scm(self): # Lazily initialize SCM to not error-out before command line parsing (or when running non-scm commands). original_cwd = os.path.abspath(".") if not self._scm: self._scm = detect_scm_system(original_cwd) if not self._scm: script_directory = os.path.abspath(sys.path[0]) self._scm = detect_scm_system(script_directory) if self._scm: log("The current directory (%s) is not a WebKit checkout, using %s" % (original_cwd, self._scm.checkout_root)) else: error( "FATAL: Failed to determine the SCM system for either %s or %s" % (original_cwd, script_directory)) return self._scm
def clear_attachment_flags(self, attachment_id, additional_comment_text=None): self.authenticate() comment_text = "Clearing flags on attachment: %s" % attachment_id if additional_comment_text: comment_text += "\n\n%s" % additional_comment_text log(comment_text) if self.dryrun: return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) self.browser.set_value(comment_text, name='comment', nr=0) self._find_select_element_for_flag('review').value = ("X",) self._find_select_element_for_flag('commit-queue').value = ("X",) self.browser.submit()
def run(self, state): bug_id = state.get("bug_id") if not bug_id and state.get("patch"): bug_id = state.get("patch").bug_id() reviewer = self._options.reviewer if not reviewer: if not bug_id: log("No bug id provided and --reviewer= not provided. Not updating ChangeLogs with reviewer.") return reviewer = self._guess_reviewer_from_bug(bug_id) if not reviewer: log("Failed to guess reviewer from bug %s and --reviewer= not provided. Not updating ChangeLogs with reviewer." % bug_id) return os.chdir(self._tool.scm().checkout_root) for changelog_path in self._tool.scm().modified_changelogs(): ChangeLog(changelog_path).set_reviewer(reviewer)
def commit_message_for_this_commit(self): changelog_paths = self.modified_changelogs() if not len(changelog_paths): raise ScriptError( message= "Found no modified ChangeLogs, cannot create a commit message.\n" "All changes require a ChangeLog. See:\n" "http://webkit.org/coding/contributing.html") changelog_messages = [] for changelog_path in changelog_paths: log("Parsing ChangeLog: %s" % changelog_path) changelog_entry = ChangeLog(changelog_path).latest_entry() if not changelog_entry: raise ScriptError(message="Failed to parse ChangeLog: " + os.path.abspath(changelog_path)) changelog_messages.append(changelog_entry) # FIXME: We should sort and label the ChangeLog messages like commit-log-editor does. return CommitMessage("".join(changelog_messages).splitlines())
def main(self, argv=sys.argv): (command_name, args) = self._split_command_name_from_args(argv[1:]) option_parser = self._create_option_parser() self._add_global_options(option_parser) command = self.command_by_name(command_name) or self.help_command if not command: option_parser.error("%s is not a recognized command" % command_name) command.set_option_parser(option_parser) (options, args) = command.parse_args(args) self.handle_global_options(options) (should_execute, failure_reason) = self.should_execute_command(command) if not should_execute: log(failure_reason) return 0 # FIXME: Should this really be 0? return command.check_arguments_and_execute(options, args, self)