class TestResultsUploader: def __init__(self, host): self._host = host self._browser = Browser() def _upload_files(self, attrs, file_objs): self._browser.open("http://%s/testfile/uploadform" % self._host) self._browser.select_form("test_result_upload") for (name, data) in attrs: self._browser[name] = str(data) for (filename, handle) in file_objs: self._browser.add_file(handle, get_mime_type(filename), filename, "file") self._browser.submit() def upload(self, params, files, timeout_seconds): orig_timeout = socket.getdefaulttimeout() file_objs = [] try: file_objs = [(filename, open(path, "rb")) for (filename, path) in files] socket.setdefaulttimeout(timeout_seconds) NetworkTransaction(timeout_seconds=timeout_seconds).run( lambda: self._upload_files(params, file_objs)) finally: socket.setdefaulttimeout(orig_timeout) for (filename, handle) in file_objs: handle.close()
class Bugzilla(object): def __init__(self, dryrun=False, committers=committers.CommitterList()): self.dryrun = dryrun self.authenticated = False self.queries = BugzillaQueries(self) self.committers = committers # FIXME: We should use some sort of Browser mock object when in dryrun # mode (to prevent any mistakes). self.browser = Browser() # Ignore bugs.webkit.org/robots.txt until we fix it to allow this # script. self.browser.set_handle_robots(False) # FIXME: Much of this should go into some sort of config module: bug_server_host = "bugs.webkit.org" bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host) bug_server_url = "https://%s/" % bug_server_host def bug_url_for_bug_id(self, bug_id, xml=False): if not bug_id: return None content_type = "&ctype=xml" if xml else "" return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type) def short_bug_url_for_bug_id(self, bug_id): if not bug_id: return None return "http://webkit.org/b/%s" % bug_id def attachment_url_for_id(self, attachment_id, action="view"): if not attachment_id: return None action_param = "" if action and action != "view": action_param = "&action=%s" % action return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, attachment_id, action_param) def _parse_attachment_flag(self, element, flag_name, attachment, result_key): flag = element.find('flag', attrs={'name': flag_name}) if flag: attachment[flag_name] = flag['status'] if flag['status'] == '+': attachment[result_key] = flag['setter'] def _parse_attachment_element(self, element, bug_id): attachment = {} attachment['bug_id'] = bug_id attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1") attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1") attachment['id'] = int(element.find('attachid').string) # FIXME: No need to parse out the url here. attachment['url'] = self.attachment_url_for_id(attachment['id']) attachment['name'] = unicode(element.find('desc').string) attachment['attacher_email'] = str(element.find('attacher').string) attachment['type'] = str(element.find('type').string) self._parse_attachment_flag( element, 'review', attachment, 'reviewer_email') self._parse_attachment_flag( element, 'commit-queue', attachment, 'committer_email') return attachment def _parse_bug_page(self, page): soup = BeautifulSoup(page) bug = {} bug["id"] = int(soup.find("bug_id").string) bug["title"] = unicode(soup.find("short_desc").string) bug["reporter_email"] = str(soup.find("reporter").string) bug["assigned_to_email"] = str(soup.find("assigned_to").string) bug["cc_emails"] = [str(element.string) for element in soup.findAll('cc')] bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')] return bug # Makes testing fetch_*_from_bug() possible until we have a better # BugzillaNetwork abstration. def _fetch_bug_page(self, bug_id): bug_url = self.bug_url_for_bug_id(bug_id, xml=True) log("Fetching: %s" % bug_url) return self.browser.open(bug_url) def fetch_bug_dictionary(self, bug_id): try: return self._parse_bug_page(self._fetch_bug_page(bug_id)) except: self.authenticate() return self._parse_bug_page(self._fetch_bug_page(bug_id)) # FIXME: A BugzillaCache object should provide all these fetch_ methods. def fetch_bug(self, bug_id): return Bug(self.fetch_bug_dictionary(bug_id), self) def _parse_bug_id_from_attachment_page(self, page): # The "Up" relation happens to point to the bug. up_link = BeautifulSoup(page).find('link', rel='Up') if not up_link: # This attachment does not exist (or you don't have permissions to # view it). return None match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href']) return int(match.group('bug_id')) def bug_id_for_attachment_id(self, attachment_id): self.authenticate() attachment_url = self.attachment_url_for_id(attachment_id, 'edit') log("Fetching: %s" % attachment_url) page = self.browser.open(attachment_url) return self._parse_bug_id_from_attachment_page(page) # FIXME: This should just return Attachment(id), which should be able to # lazily fetch needed data. def fetch_attachment(self, attachment_id): # We could grab all the attachment details off of the attachment edit # page but we already have working code to do so off of the bugs page, # so re-use that. bug_id = self.bug_id_for_attachment_id(attachment_id) if not bug_id: return None attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True) for attachment in attachments: if attachment.id() == int(attachment_id): return attachment return None # This should never be hit. def authenticate(self): if self.authenticated: return if self.dryrun: log("Skipping log in for dry run...") self.authenticated = True return attempts = 0 while not self.authenticated: attempts += 1 (username, password) = Credentials( self.bug_server_host, git_prefix="bugzilla").read_credentials() log("Logging in as %s..." % username) self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1") self.browser.select_form(name="login") self.browser['Bugzilla_login'] = username self.browser['Bugzilla_password'] = password response = self.browser.submit() match = re.search("<title>(.+?)</title>", response.read()) # If the resulting page has a title, and it contains the word # "invalid" assume it's the login failure page. if match and re.search("Invalid", match.group(1), re.IGNORECASE): errorMessage = "Bugzilla login failed: %s" % match.group(1) # raise an exception only if this was the last attempt if attempts < 5: log(errorMessage) else: raise Exception(errorMessage) else: self.authenticated = True def _fill_attachment_form(self, description, patch_file_object, comment_text=None, mark_for_review=False, mark_for_commit_queue=False, mark_for_landing=False, bug_id=None): self.browser['description'] = description self.browser['ispatch'] = ("1",) self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',) if mark_for_landing: self.browser['flag_type-3'] = ('+',) elif mark_for_commit_queue: self.browser['flag_type-3'] = ('?',) else: self.browser['flag_type-3'] = ('X',) if bug_id: patch_name = "bug-%s-%s.patch" % (bug_id, timestamp()) else: patch_name ="%s.patch" % timestamp() self.browser.add_file(patch_file_object, "text/plain", patch_name, 'data') def add_patch_to_bug(self, bug_id, patch_file_object, description, comment_text=None, mark_for_review=False, mark_for_commit_queue=False, mark_for_landing=False): self.authenticate() log('Adding patch "%s" to %sshow_bug.cgi?id=%s' % (description, self.bug_server_url, bug_id)) if self.dryrun: log(comment_text) return self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % ( self.bug_server_url, bug_id)) self.browser.select_form(name="entryform") self._fill_attachment_form(description, patch_file_object, mark_for_review=mark_for_review, mark_for_commit_queue=mark_for_commit_queue, mark_for_landing=mark_for_landing, bug_id=bug_id) if comment_text: log(comment_text) self.browser['comment'] = comment_text self.browser.submit() def _check_create_bug_response(self, response_html): match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>", response_html) if match: return match.group('bug_id') match = re.search( '<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">', response_html, re.DOTALL) error_message = "FAIL" if match: text_lines = BeautifulSoup( match.group('error_message')).findAll(text=True) error_message = "\n" + '\n'.join( [" " + line.strip() for line in text_lines if line.strip()]) raise Exception("Bug not created: %s" % error_message) def create_bug(self, bug_title, bug_description, component=None, patch_file_object=None, patch_description=None, cc=None, blocked=None, mark_for_review=False, mark_for_commit_queue=False): self.authenticate() log('Creating bug with title "%s"' % bug_title) if self.dryrun: log(bug_description) return self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit") self.browser.select_form(name="Create") component_items = self.browser.find_control('component').items component_names = map(lambda item: item.name, component_items) if not component: component = "New Bugs" if component not in component_names: component = User.prompt_with_list("Please pick a component:", component_names) self.browser["component"] = [component] if cc: self.browser["cc"] = cc if blocked: self.browser["blocked"] = str(blocked) self.browser["short_desc"] = bug_title self.browser["comment"] = bug_description if patch_file_object: self._fill_attachment_form( patch_description, patch_file_object, mark_for_review=mark_for_review, mark_for_commit_queue=mark_for_commit_queue) response = self.browser.submit() bug_id = self._check_create_bug_response(response.read()) log("Bug %s created." % bug_id) log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id)) return bug_id def _find_select_element_for_flag(self, flag_name): # FIXME: This will break if we ever re-order attachment flags if flag_name == "review": return self.browser.find_control(type='select', nr=0) if flag_name == "commit-queue": return self.browser.find_control(type='select', nr=1) raise Exception("Don't know how to find flag named \"%s\"" % flag_name) def clear_attachment_flags(self, attachment_id, additional_comment_text=None): self.authenticate() comment_text = "Clearing flags on attachment: %s" % attachment_id if additional_comment_text: comment_text += "\n\n%s" % additional_comment_text log(comment_text) if self.dryrun: return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) self.browser.set_value(comment_text, name='comment', nr=0) self._find_select_element_for_flag('review').value = ("X",) self._find_select_element_for_flag('commit-queue').value = ("X",) self.browser.submit() def set_flag_on_attachment(self, attachment_id, flag_name, flag_value, comment_text, additional_comment_text): # FIXME: We need a way to test this function on a live bugzilla # instance. self.authenticate() if additional_comment_text: comment_text += "\n\n%s" % additional_comment_text log(comment_text) if self.dryrun: return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) self.browser.set_value(comment_text, name='comment', nr=0) self._find_select_element_for_flag(flag_name).value = (flag_value,) self.browser.submit() # FIXME: All of these bug editing methods have a ridiculous amount of # copy/paste code. def obsolete_attachment(self, attachment_id, comment_text=None): self.authenticate() log("Obsoleting attachment: %s" % attachment_id) if self.dryrun: log(comment_text) return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) self.browser.find_control('isobsolete').items[0].selected = True # Also clear any review flag (to remove it from review/commit queues) self._find_select_element_for_flag('review').value = ("X",) self._find_select_element_for_flag('commit-queue').value = ("X",) if comment_text: log(comment_text) # Bugzilla has two textareas named 'comment', one is somehow # hidden. We want the first. self.browser.set_value(comment_text, name='comment', nr=0) self.browser.submit() def add_cc_to_bug(self, bug_id, email_address_list): self.authenticate() log("Adding %s to the CC list for bug %s" % (email_address_list, bug_id)) if self.dryrun: return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") self.browser["newcc"] = ", ".join(email_address_list) self.browser.submit() def post_comment_to_bug(self, bug_id, comment_text, cc=None): self.authenticate() log("Adding comment to bug %s" % bug_id) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") self.browser["comment"] = comment_text if cc: self.browser["newcc"] = ", ".join(cc) self.browser.submit() def close_bug_as_fixed(self, bug_id, comment_text=None): self.authenticate() log("Closing bug %s as fixed" % bug_id) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") if comment_text: log(comment_text) self.browser['comment'] = comment_text self.browser['bug_status'] = ['RESOLVED'] self.browser['resolution'] = ['FIXED'] self.browser.submit() def reassign_bug(self, bug_id, assignee, comment_text=None): self.authenticate() log("Assigning bug %s to %s" % (bug_id, assignee)) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") if comment_text: log(comment_text) self.browser["comment"] = comment_text self.browser["assigned_to"] = assignee self.browser.submit() def reopen_bug(self, bug_id, comment_text): self.authenticate() log("Re-opening bug %s" % bug_id) # Bugzilla requires a comment when re-opening a bug, so we know it will # never be None. log(comment_text) if self.dryrun: return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") bug_status = self.browser.find_control("bug_status", type="select") # This is a hack around the fact that ClientForm.ListControl seems to # have no simpler way to ask if a control has an item named "REOPENED" # without using exceptions for control flow. possible_bug_statuses = map(lambda item: item.name, bug_status.items) if "REOPENED" in possible_bug_statuses: bug_status.value = ["REOPENED"] else: log("Did not reopen bug %s. " + "It appears to already be open with status %s." % ( bug_id, bug_status.value)) self.browser['comment'] = comment_text self.browser.submit()
class Bugzilla(object): def __init__(self, dryrun=False, committers=committers.CommitterList()): self.dryrun = dryrun self.authenticated = False self.queries = BugzillaQueries(self) self.committers = committers self.cached_quips = [] # FIXME: We should use some sort of Browser mock object when in dryrun # mode (to prevent any mistakes). self.browser = Browser() # Ignore bugs.webkit.org/robots.txt until we fix it to allow this # script. self.browser.set_handle_robots(False) # FIXME: Much of this should go into some sort of config module: bug_server_host = "bugs.webkit.org" bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host) bug_server_url = "https://%s/" % bug_server_host def quips(self): # We only fetch and parse the list of quips once per instantiation # so that we do not burden bugs.webkit.org. if not self.cached_quips and not self.dryrun: self.cached_quips = self.queries.fetch_quips() return self.cached_quips def bug_url_for_bug_id(self, bug_id, xml=False): if not bug_id: return None content_type = "&ctype=xml" if xml else "" return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type) def short_bug_url_for_bug_id(self, bug_id): if not bug_id: return None return "http://webkit.org/b/%s" % bug_id def add_attachment_url(self, bug_id): return "%sattachment.cgi?action=enter&bugid=%s" % (self.bug_server_url, bug_id) def attachment_url_for_id(self, attachment_id, action="view"): if not attachment_id: return None action_param = "" if action and action != "view": action_param = "&action=%s" % action return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, attachment_id, action_param) def _parse_attachment_flag(self, element, flag_name, attachment, result_key): flag = element.find('flag', attrs={'name': flag_name}) if flag: attachment[flag_name] = flag['status'] if flag['status'] == '+': attachment[result_key] = flag['setter'] # Sadly show_bug.cgi?ctype=xml does not expose the flag modification date. def _string_contents(self, soup): # WebKit's bugzilla instance uses UTF-8. # BeautifulSoup always returns Unicode strings, however # the .string method returns a (unicode) NavigableString. # NavigableString can confuse other parts of the code, so we # convert from NavigableString to a real unicode() object using unicode(). return unicode(soup.string) # Example: 2010-01-20 14:31 PST # FIXME: Some bugzilla dates seem to have seconds in them? # Python does not support timezones out of the box. # Assume that bugzilla always uses PST (which is true for bugs.webkit.org) _bugzilla_date_format = "%Y-%m-%d %H:%M" @classmethod def _parse_date(cls, date_string): (date, time, time_zone) = date_string.split(" ") # Ignore the timezone because python doesn't understand timezones out of the box. date_string = "%s %s" % (date, time) return datetime.strptime(date_string, cls._bugzilla_date_format) def _date_contents(self, soup): return self._parse_date(self._string_contents(soup)) def _parse_attachment_element(self, element, bug_id): attachment = {} attachment['bug_id'] = bug_id attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1") attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1") attachment['id'] = int(element.find('attachid').string) # FIXME: No need to parse out the url here. attachment['url'] = self.attachment_url_for_id(attachment['id']) attachment["attach_date"] = self._date_contents(element.find("date")) attachment['name'] = self._string_contents(element.find('desc')) attachment['attacher_email'] = self._string_contents(element.find('attacher')) attachment['type'] = self._string_contents(element.find('type')) self._parse_attachment_flag( element, 'review', attachment, 'reviewer_email') self._parse_attachment_flag( element, 'commit-queue', attachment, 'committer_email') return attachment def _parse_bugs_from_xml(self, page): soup = BeautifulSoup(page) # Without the unicode() call, BeautifulSoup occasionally complains of being # passed None for no apparent reason. return [Bug(self._parse_bug_dictionary_from_xml(unicode(bug_xml)), self) for bug_xml in soup('bug')] def _parse_bug_dictionary_from_xml(self, page): soup = BeautifulSoup(page) bug = {} bug["id"] = int(soup.find("bug_id").string) bug["title"] = self._string_contents(soup.find("short_desc")) bug["bug_status"] = self._string_contents(soup.find("bug_status")) dup_id = soup.find("dup_id") if dup_id: bug["dup_id"] = self._string_contents(dup_id) bug["reporter_email"] = self._string_contents(soup.find("reporter")) bug["assigned_to_email"] = self._string_contents(soup.find("assigned_to")) bug["cc_emails"] = [self._string_contents(element) for element in soup.findAll('cc')] bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')] return bug # Makes testing fetch_*_from_bug() possible until we have a better # BugzillaNetwork abstration. def _fetch_bug_page(self, bug_id): bug_url = self.bug_url_for_bug_id(bug_id, xml=True) log("Fetching: %s" % bug_url) return self.browser.open(bug_url) def fetch_bug_dictionary(self, bug_id): try: return self._parse_bug_dictionary_from_xml(self._fetch_bug_page(bug_id)) except KeyboardInterrupt: raise except: self.authenticate() return self._parse_bug_dictionary_from_xml(self._fetch_bug_page(bug_id)) # FIXME: A BugzillaCache object should provide all these fetch_ methods. def fetch_bug(self, bug_id): return Bug(self.fetch_bug_dictionary(bug_id), self) def fetch_attachment_contents(self, attachment_id): attachment_url = self.attachment_url_for_id(attachment_id) # We need to authenticate to download patches from security bugs. self.authenticate() return self.browser.open(attachment_url).read() def _parse_bug_id_from_attachment_page(self, page): # The "Up" relation happens to point to the bug. up_link = BeautifulSoup(page).find('link', rel='Up') if not up_link: # This attachment does not exist (or you don't have permissions to # view it). return None match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href']) return int(match.group('bug_id')) def bug_id_for_attachment_id(self, attachment_id): self.authenticate() attachment_url = self.attachment_url_for_id(attachment_id, 'edit') log("Fetching: %s" % attachment_url) page = self.browser.open(attachment_url) return self._parse_bug_id_from_attachment_page(page) # FIXME: This should just return Attachment(id), which should be able to # lazily fetch needed data. def fetch_attachment(self, attachment_id): # We could grab all the attachment details off of the attachment edit # page but we already have working code to do so off of the bugs page, # so re-use that. bug_id = self.bug_id_for_attachment_id(attachment_id) if not bug_id: return None attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True) for attachment in attachments: if attachment.id() == int(attachment_id): return attachment return None # This should never be hit. def authenticate(self): if self.authenticated: return if self.dryrun: log("Skipping log in for dry run...") self.authenticated = True return credentials = Credentials(self.bug_server_host, git_prefix="bugzilla") attempts = 0 while not self.authenticated: attempts += 1 username, password = credentials.read_credentials() log("Logging in as %s..." % username) self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1") self.browser.select_form(name="login") self.browser['Bugzilla_login'] = username self.browser['Bugzilla_password'] = password response = self.browser.submit() match = re.search("<title>(.+?)</title>", response.read()) # If the resulting page has a title, and it contains the word # "invalid" assume it's the login failure page. if match and re.search("Invalid", match.group(1), re.IGNORECASE): errorMessage = "Bugzilla login failed: %s" % match.group(1) # raise an exception only if this was the last attempt if attempts < 5: log(errorMessage) else: raise Exception(errorMessage) else: self.authenticated = True self.username = username def _commit_queue_flag(self, mark_for_landing, mark_for_commit_queue): if mark_for_landing: return '+' elif mark_for_commit_queue: return '?' return 'X' # FIXME: mark_for_commit_queue and mark_for_landing should be joined into a single commit_flag argument. def _fill_attachment_form(self, description, file_object, mark_for_review=False, mark_for_commit_queue=False, mark_for_landing=False, is_patch=False, filename=None, mimetype=None): self.browser['description'] = description if is_patch: self.browser['ispatch'] = ("1",) # FIXME: Should this use self._find_select_element_for_flag? self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',) self.browser['flag_type-3'] = (self._commit_queue_flag(mark_for_landing, mark_for_commit_queue),) filename = filename or "%s.patch" % timestamp() mimetype = mimetype or "text/plain" self.browser.add_file(file_object, mimetype, filename, 'data') def _file_object_for_upload(self, file_or_string): if hasattr(file_or_string, 'read'): return file_or_string # Only if file_or_string is not already encoded do we want to encode it. if isinstance(file_or_string, unicode): file_or_string = file_or_string.encode('utf-8') return StringIO.StringIO(file_or_string) # timestamp argument is just for unittests. def _filename_for_upload(self, file_object, bug_id, extension="txt", timestamp=timestamp): if hasattr(file_object, "name"): return file_object.name return "bug-%s-%s.%s" % (bug_id, timestamp(), extension) def add_attachment_to_bug(self, bug_id, file_or_string, description, filename=None, comment_text=None): self.authenticate() log('Adding attachment "%s" to %s' % (description, self.bug_url_for_bug_id(bug_id))) if self.dryrun: log(comment_text) return self.browser.open(self.add_attachment_url(bug_id)) self.browser.select_form(name="entryform") file_object = self._file_object_for_upload(file_or_string) filename = filename or self._filename_for_upload(file_object, bug_id) self._fill_attachment_form(description, file_object, filename=filename) if comment_text: log(comment_text) self.browser['comment'] = comment_text self.browser.submit() # FIXME: The arguments to this function should be simplified and then # this should be merged into add_attachment_to_bug def add_patch_to_bug(self, bug_id, file_or_string, description, comment_text=None, mark_for_review=False, mark_for_commit_queue=False, mark_for_landing=False): self.authenticate() log('Adding patch "%s" to %s' % (description, self.bug_url_for_bug_id(bug_id))) if self.dryrun: log(comment_text) return self.browser.open(self.add_attachment_url(bug_id)) self.browser.select_form(name="entryform") file_object = self._file_object_for_upload(file_or_string) filename = self._filename_for_upload(file_object, bug_id, extension="patch") self._fill_attachment_form(description, file_object, mark_for_review=mark_for_review, mark_for_commit_queue=mark_for_commit_queue, mark_for_landing=mark_for_landing, is_patch=True, filename=filename) if comment_text: log(comment_text) self.browser['comment'] = comment_text self.browser.submit() # FIXME: There has to be a more concise way to write this method. def _check_create_bug_response(self, response_html): match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>", response_html) if match: return match.group('bug_id') match = re.search( '<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">', response_html, re.DOTALL) error_message = "FAIL" if match: text_lines = BeautifulSoup( match.group('error_message')).findAll(text=True) error_message = "\n" + '\n'.join( [" " + line.strip() for line in text_lines if line.strip()]) raise Exception("Bug not created: %s" % error_message) def create_bug(self, bug_title, bug_description, component=None, diff=None, patch_description=None, cc=None, blocked=None, assignee=None, mark_for_review=False, mark_for_commit_queue=False): self.authenticate() log('Creating bug with title "%s"' % bug_title) if self.dryrun: log(bug_description) # FIXME: This will make some paths fail, as they assume this returns an id. return self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit") self.browser.select_form(name="Create") component_items = self.browser.find_control('component').items component_names = map(lambda item: item.name, component_items) if not component: component = "New Bugs" if component not in component_names: component = User.prompt_with_list("Please pick a component:", component_names) self.browser["component"] = [component] if cc: self.browser["cc"] = cc if blocked: self.browser["blocked"] = unicode(blocked) if not assignee: assignee = self.username if assignee and not self.browser.find_control("assigned_to").disabled: self.browser["assigned_to"] = assignee self.browser["short_desc"] = bug_title self.browser["comment"] = bug_description if diff: # _fill_attachment_form expects a file-like object # Patch files are already binary, so no encoding needed. assert(isinstance(diff, str)) patch_file_object = StringIO.StringIO(diff) self._fill_attachment_form( patch_description, patch_file_object, mark_for_review=mark_for_review, mark_for_commit_queue=mark_for_commit_queue, is_patch=True) response = self.browser.submit() bug_id = self._check_create_bug_response(response.read()) log("Bug %s created." % bug_id) log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id)) return bug_id def _find_select_element_for_flag(self, flag_name): # FIXME: This will break if we ever re-order attachment flags if flag_name == "review": return self.browser.find_control(type='select', nr=0) elif flag_name == "commit-queue": return self.browser.find_control(type='select', nr=1) raise Exception("Don't know how to find flag named \"%s\"" % flag_name) def clear_attachment_flags(self, attachment_id, additional_comment_text=None): self.authenticate() comment_text = "Clearing flags on attachment: %s" % attachment_id if additional_comment_text: comment_text += "\n\n%s" % additional_comment_text log(comment_text) if self.dryrun: return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) self.browser.set_value(comment_text, name='comment', nr=0) self._find_select_element_for_flag('review').value = ("X",) self._find_select_element_for_flag('commit-queue').value = ("X",) self.browser.submit() def set_flag_on_attachment(self, attachment_id, flag_name, flag_value, comment_text=None, additional_comment_text=None): # FIXME: We need a way to test this function on a live bugzilla # instance. self.authenticate() if additional_comment_text: comment_text += "\n\n%s" % additional_comment_text log(comment_text) if self.dryrun: return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) if comment_text: self.browser.set_value(comment_text, name='comment', nr=0) self._find_select_element_for_flag(flag_name).value = (flag_value,) self.browser.submit() # FIXME: All of these bug editing methods have a ridiculous amount of # copy/paste code. def obsolete_attachment(self, attachment_id, comment_text=None): self.authenticate() log("Obsoleting attachment: %s" % attachment_id) if self.dryrun: log(comment_text) return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) self.browser.find_control('isobsolete').items[0].selected = True # Also clear any review flag (to remove it from review/commit queues) self._find_select_element_for_flag('review').value = ("X",) self._find_select_element_for_flag('commit-queue').value = ("X",) if comment_text: log(comment_text) # Bugzilla has two textareas named 'comment', one is somehow # hidden. We want the first. self.browser.set_value(comment_text, name='comment', nr=0) self.browser.submit() def add_cc_to_bug(self, bug_id, email_address_list): self.authenticate() log("Adding %s to the CC list for bug %s" % (email_address_list, bug_id)) if self.dryrun: return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") self.browser["newcc"] = ", ".join(email_address_list) self.browser.submit() def post_comment_to_bug(self, bug_id, comment_text, cc=None): self.authenticate() log("Adding comment to bug %s" % bug_id) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") self.browser["comment"] = comment_text if cc: self.browser["newcc"] = ", ".join(cc) self.browser.submit() def close_bug_as_fixed(self, bug_id, comment_text=None): self.authenticate() log("Closing bug %s as fixed" % bug_id) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") if comment_text: self.browser['comment'] = comment_text self.browser['bug_status'] = ['RESOLVED'] self.browser['resolution'] = ['FIXED'] self.browser.submit() def reassign_bug(self, bug_id, assignee, comment_text=None): self.authenticate() log("Assigning bug %s to %s" % (bug_id, assignee)) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") if comment_text: log(comment_text) self.browser["comment"] = comment_text self.browser["assigned_to"] = assignee self.browser.submit() def reopen_bug(self, bug_id, comment_text): self.authenticate() log("Re-opening bug %s" % bug_id) # Bugzilla requires a comment when re-opening a bug, so we know it will # never be None. log(comment_text) if self.dryrun: return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") bug_status = self.browser.find_control("bug_status", type="select") # This is a hack around the fact that ClientForm.ListControl seems to # have no simpler way to ask if a control has an item named "REOPENED" # without using exceptions for control flow. possible_bug_statuses = map(lambda item: item.name, bug_status.items) if "REOPENED" in possible_bug_statuses: bug_status.value = ["REOPENED"] # If the bug was never confirmed it will not have a "REOPENED" # state, but only an "UNCONFIRMED" state. elif "UNCONFIRMED" in possible_bug_statuses: bug_status.value = ["UNCONFIRMED"] else: # FIXME: This logic is slightly backwards. We won't print this # message if the bug is already open with state "UNCONFIRMED". log("Did not reopen bug %s, it appears to already be open with status %s." % (bug_id, bug_status.value)) self.browser['comment'] = comment_text self.browser.submit()
class StatusServer: default_host = "webkit-commit-queue.appspot.com" def __init__(self, host=default_host): self.set_host(host) self.browser = Browser() def set_host(self, host): self.host = host self.url = "http://%s" % self.host def results_url_for_status(self, status_id): return "%s/results/%s" % (self.url, status_id) def _add_patch(self, patch): if not patch: return if patch.bug_id(): self.browser["bug_id"] = unicode(patch.bug_id()) if patch.id(): self.browser["patch_id"] = unicode(patch.id()) def _add_results_file(self, results_file): if not results_file: return self.browser.add_file(results_file, "text/plain", "results.txt", "results_file") def _post_status_to_server(self, queue_name, status, patch, results_file): if results_file: # We might need to re-wind the file if we've already tried to post it. results_file.seek(0) update_status_url = "%s/update-status" % self.url self.browser.open(update_status_url) self.browser.select_form(name="update_status") self.browser["queue_name"] = queue_name self._add_patch(patch) self.browser["status"] = status self._add_results_file(results_file) return self.browser.submit().read() # This is the id of the newly created status object. def _post_svn_revision_to_server(self, svn_revision_number, broken_bot): update_svn_revision_url = "%s/update-svn-revision" % self.url self.browser.open(update_svn_revision_url) self.browser.select_form(name="update_svn_revision") self.browser["number"] = unicode(svn_revision_number) self.browser["broken_bot"] = broken_bot return self.browser.submit().read() def _post_work_items_to_server(self, queue_name, work_items): update_work_items_url = "%s/update-work-items" % self.url self.browser.open(update_work_items_url) self.browser.select_form(name="update_work_items") self.browser["queue_name"] = queue_name work_items = map(unicode, work_items) # .join expects strings self.browser["work_items"] = " ".join(work_items) return self.browser.submit().read() def update_work_items(self, queue_name, work_items): _log.debug("Recording work items: %s for %s" % (work_items, queue_name)) return NetworkTransaction().run(lambda: self._post_work_items_to_server(queue_name, work_items)) def update_status(self, queue_name, status, patch=None, results_file=None): log(status) return NetworkTransaction().run(lambda: self._post_status_to_server(queue_name, status, patch, results_file)) def update_svn_revision(self, svn_revision_number, broken_bot): log("SVN revision: %s broke %s" % (svn_revision_number, broken_bot)) return NetworkTransaction().run(lambda: self._post_svn_revision_to_server(svn_revision_number, broken_bot)) def _fetch_url(self, url): try: return urllib2.urlopen(url).read() except urllib2.HTTPError, e: if e.code == 404: return None raise e
class Bugzilla(object): def __init__(self, dryrun=False, committers=committers.CommitterList()): self.dryrun = dryrun self.authenticated = False self.queries = BugzillaQueries(self) self.committers = committers self.cached_quips = [] # FIXME: We should use some sort of Browser mock object when in dryrun # mode (to prevent any mistakes). self.browser = Browser() # Ignore bugs.webkit.org/robots.txt until we fix it to allow this # script. self.browser.set_handle_robots(False) # FIXME: Much of this should go into some sort of config module: bug_server_host = "bugs.webkit.org" bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host) bug_server_url = "https://%s/" % bug_server_host def quips(self): # We only fetch and parse the list of quips once per instantiation # so that we do not burden bugs.webkit.org. if not self.cached_quips and not self.dryrun: self.cached_quips = self.queries.fetch_quips() return self.cached_quips def bug_url_for_bug_id(self, bug_id, xml=False): if not bug_id: return None content_type = "&ctype=xml" if xml else "" return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type) def short_bug_url_for_bug_id(self, bug_id): if not bug_id: return None return "http://webkit.org/b/%s" % bug_id def attachment_url_for_id(self, attachment_id, action="view"): if not attachment_id: return None action_param = "" if action and action != "view": action_param = "&action=%s" % action return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, attachment_id, action_param) def _parse_attachment_flag(self, element, flag_name, attachment, result_key): flag = element.find('flag', attrs={'name': flag_name}) if flag: attachment[flag_name] = flag['status'] if flag['status'] == '+': attachment[result_key] = flag['setter'] # Sadly show_bug.cgi?ctype=xml does not expose the flag modification date. def _string_contents(self, soup): # WebKit's bugzilla instance uses UTF-8. # BeautifulSoup always returns Unicode strings, however # the .string method returns a (unicode) NavigableString. # NavigableString can confuse other parts of the code, so we # convert from NavigableString to a real unicode() object using unicode(). return unicode(soup.string) # Example: 2010-01-20 14:31 PST # FIXME: Some bugzilla dates seem to have seconds in them? # Python does not support timezones out of the box. # Assume that bugzilla always uses PST (which is true for bugs.webkit.org) _bugzilla_date_format = "%Y-%m-%d %H:%M" @classmethod def _parse_date(cls, date_string): (date, time, time_zone) = date_string.split(" ") # Ignore the timezone because python doesn't understand timezones out of the box. date_string = "%s %s" % (date, time) return datetime.strptime(date_string, cls._bugzilla_date_format) def _date_contents(self, soup): return self._parse_date(self._string_contents(soup)) def _parse_attachment_element(self, element, bug_id): attachment = {} attachment['bug_id'] = bug_id attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1") attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1") attachment['id'] = int(element.find('attachid').string) # FIXME: No need to parse out the url here. attachment['url'] = self.attachment_url_for_id(attachment['id']) attachment["attach_date"] = self._date_contents(element.find("date")) attachment['name'] = self._string_contents(element.find('desc')) attachment['attacher_email'] = self._string_contents(element.find('attacher')) attachment['type'] = self._string_contents(element.find('type')) self._parse_attachment_flag( element, 'review', attachment, 'reviewer_email') self._parse_attachment_flag( element, 'in-rietveld', attachment, 'rietveld_uploader_email') self._parse_attachment_flag( element, 'commit-queue', attachment, 'committer_email') return attachment def _parse_bug_page(self, page): soup = BeautifulSoup(page) bug = {} bug["id"] = int(soup.find("bug_id").string) bug["title"] = self._string_contents(soup.find("short_desc")) bug["reporter_email"] = self._string_contents(soup.find("reporter")) bug["assigned_to_email"] = self._string_contents(soup.find("assigned_to")) bug["cc_emails"] = [self._string_contents(element) for element in soup.findAll('cc')] bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')] return bug # Makes testing fetch_*_from_bug() possible until we have a better # BugzillaNetwork abstration. def _fetch_bug_page(self, bug_id): bug_url = self.bug_url_for_bug_id(bug_id, xml=True) log("Fetching: %s" % bug_url) return self.browser.open(bug_url) def fetch_bug_dictionary(self, bug_id): try: return self._parse_bug_page(self._fetch_bug_page(bug_id)) except KeyboardInterrupt: raise except: self.authenticate() return self._parse_bug_page(self._fetch_bug_page(bug_id)) # FIXME: A BugzillaCache object should provide all these fetch_ methods. def fetch_bug(self, bug_id): return Bug(self.fetch_bug_dictionary(bug_id), self) def fetch_attachment_contents(self, attachment_id): attachment_url = self.attachment_url_for_id(attachment_id) # We need to authenticate to download patches from security bugs. self.authenticate() return self.browser.open(attachment_url).read() def _parse_bug_id_from_attachment_page(self, page): # The "Up" relation happens to point to the bug. up_link = BeautifulSoup(page).find('link', rel='Up') if not up_link: # This attachment does not exist (or you don't have permissions to # view it). return None match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href']) return int(match.group('bug_id')) def bug_id_for_attachment_id(self, attachment_id): self.authenticate() attachment_url = self.attachment_url_for_id(attachment_id, 'edit') log("Fetching: %s" % attachment_url) page = self.browser.open(attachment_url) return self._parse_bug_id_from_attachment_page(page) # FIXME: This should just return Attachment(id), which should be able to # lazily fetch needed data. def fetch_attachment(self, attachment_id): # We could grab all the attachment details off of the attachment edit # page but we already have working code to do so off of the bugs page, # so re-use that. bug_id = self.bug_id_for_attachment_id(attachment_id) if not bug_id: return None attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True) for attachment in attachments: if attachment.id() == int(attachment_id): return attachment return None # This should never be hit. def authenticate(self): if self.authenticated: return if self.dryrun: log("Skipping log in for dry run...") self.authenticated = True return attempts = 0 while not self.authenticated: attempts += 1 (username, password) = Credentials( self.bug_server_host, git_prefix="bugzilla").read_credentials() log("Logging in as %s..." % username) self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1") self.browser.select_form(name="login") self.browser['Bugzilla_login'] = username self.browser['Bugzilla_password'] = password response = self.browser.submit() match = re.search("<title>(.+?)</title>", response.read()) # If the resulting page has a title, and it contains the word # "invalid" assume it's the login failure page. if match and re.search("Invalid", match.group(1), re.IGNORECASE): errorMessage = "Bugzilla login failed: %s" % match.group(1) # raise an exception only if this was the last attempt if attempts < 5: log(errorMessage) else: raise Exception(errorMessage) else: self.authenticated = True self.username = username def _fill_attachment_form(self, description, patch_file_object, comment_text=None, mark_for_review=False, mark_for_commit_queue=False, mark_for_landing=False, bug_id=None): self.browser['description'] = description self.browser['ispatch'] = ("1",) self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',) if mark_for_landing: self.browser['flag_type-3'] = ('+',) elif mark_for_commit_queue: self.browser['flag_type-3'] = ('?',) else: self.browser['flag_type-3'] = ('X',) if bug_id: patch_name = "bug-%s-%s.patch" % (bug_id, timestamp()) else: patch_name ="%s.patch" % timestamp() self.browser.add_file(patch_file_object, "text/plain", patch_name, 'data') def add_patch_to_bug(self, bug_id, diff, description, comment_text=None, mark_for_review=False, mark_for_commit_queue=False, mark_for_landing=False): self.authenticate() log('Adding patch "%s" to %sshow_bug.cgi?id=%s' % (description, self.bug_server_url, bug_id)) if self.dryrun: log(comment_text) return self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % ( self.bug_server_url, bug_id)) self.browser.select_form(name="entryform") # _fill_attachment_form expects a file-like object # Patch files are already binary, so no encoding needed. assert(isinstance(diff, str)) patch_file_object = StringIO.StringIO(diff) self._fill_attachment_form(description, patch_file_object, mark_for_review=mark_for_review, mark_for_commit_queue=mark_for_commit_queue, mark_for_landing=mark_for_landing, bug_id=bug_id) if comment_text: log(comment_text) self.browser['comment'] = comment_text self.browser.submit() def _check_create_bug_response(self, response_html): match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>", response_html) if match: return match.group('bug_id') match = re.search( '<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">', response_html, re.DOTALL) error_message = "FAIL" if match: text_lines = BeautifulSoup( match.group('error_message')).findAll(text=True) error_message = "\n" + '\n'.join( [" " + line.strip() for line in text_lines if line.strip()]) raise Exception("Bug not created: %s" % error_message) def create_bug(self, bug_title, bug_description, component=None, diff=None, patch_description=None, cc=None, blocked=None, assignee=None, mark_for_review=False, mark_for_commit_queue=False): self.authenticate() log('Creating bug with title "%s"' % bug_title) if self.dryrun: log(bug_description) return self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit") self.browser.select_form(name="Create") component_items = self.browser.find_control('component').items component_names = map(lambda item: item.name, component_items) if not component: component = "New Bugs" if component not in component_names: component = User.prompt_with_list("Please pick a component:", component_names) self.browser["component"] = [component] if cc: self.browser["cc"] = cc if blocked: self.browser["blocked"] = unicode(blocked) if assignee == None: assignee = self.username if assignee and not self.browser.find_control("assigned_to").disabled: self.browser["assigned_to"] = assignee self.browser["short_desc"] = bug_title self.browser["comment"] = bug_description if diff: # _fill_attachment_form expects a file-like object # Patch files are already binary, so no encoding needed. assert(isinstance(diff, str)) patch_file_object = StringIO.StringIO(diff) self._fill_attachment_form( patch_description, patch_file_object, mark_for_review=mark_for_review, mark_for_commit_queue=mark_for_commit_queue) response = self.browser.submit() bug_id = self._check_create_bug_response(response.read()) log("Bug %s created." % bug_id) log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id)) return bug_id def _find_select_element_for_flag(self, flag_name): # FIXME: This will break if we ever re-order attachment flags if flag_name == "review": return self.browser.find_control(type='select', nr=0) elif flag_name == "commit-queue": return self.browser.find_control(type='select', nr=1) elif flag_name == "in-rietveld": return self.browser.find_control(type='select', nr=2) raise Exception("Don't know how to find flag named \"%s\"" % flag_name) def clear_attachment_flags(self, attachment_id, additional_comment_text=None): self.authenticate() comment_text = "Clearing flags on attachment: %s" % attachment_id if additional_comment_text: comment_text += "\n\n%s" % additional_comment_text log(comment_text) if self.dryrun: return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) self.browser.set_value(comment_text, name='comment', nr=0) self._find_select_element_for_flag('review').value = ("X",) self._find_select_element_for_flag('commit-queue').value = ("X",) self.browser.submit() def set_flag_on_attachment(self, attachment_id, flag_name, flag_value, comment_text=None, additional_comment_text=None): # FIXME: We need a way to test this function on a live bugzilla # instance. self.authenticate() if additional_comment_text: comment_text += "\n\n%s" % additional_comment_text log(comment_text) if self.dryrun: return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) if comment_text: self.browser.set_value(comment_text, name='comment', nr=0) self._find_select_element_for_flag(flag_name).value = (flag_value,) self.browser.submit() # FIXME: All of these bug editing methods have a ridiculous amount of # copy/paste code. def obsolete_attachment(self, attachment_id, comment_text=None): self.authenticate() log("Obsoleting attachment: %s" % attachment_id) if self.dryrun: log(comment_text) return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) self.browser.find_control('isobsolete').items[0].selected = True # Also clear any review flag (to remove it from review/commit queues) self._find_select_element_for_flag('review').value = ("X",) self._find_select_element_for_flag('commit-queue').value = ("X",) if comment_text: log(comment_text) # Bugzilla has two textareas named 'comment', one is somehow # hidden. We want the first. self.browser.set_value(comment_text, name='comment', nr=0) self.browser.submit() def add_cc_to_bug(self, bug_id, email_address_list): self.authenticate() log("Adding %s to the CC list for bug %s" % (email_address_list, bug_id)) if self.dryrun: return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") self.browser["newcc"] = ", ".join(email_address_list) self.browser.submit() def post_comment_to_bug(self, bug_id, comment_text, cc=None): self.authenticate() log("Adding comment to bug %s" % bug_id) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") self.browser["comment"] = comment_text if cc: self.browser["newcc"] = ", ".join(cc) self.browser.submit() def close_bug_as_fixed(self, bug_id, comment_text=None): self.authenticate() log("Closing bug %s as fixed" % bug_id) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") if comment_text: log(comment_text) self.browser['comment'] = comment_text self.browser['bug_status'] = ['RESOLVED'] self.browser['resolution'] = ['FIXED'] self.browser.submit() def reassign_bug(self, bug_id, assignee, comment_text=None): self.authenticate() log("Assigning bug %s to %s" % (bug_id, assignee)) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") if comment_text: log(comment_text) self.browser["comment"] = comment_text self.browser["assigned_to"] = assignee self.browser.submit() def reopen_bug(self, bug_id, comment_text): self.authenticate() log("Re-opening bug %s" % bug_id) # Bugzilla requires a comment when re-opening a bug, so we know it will # never be None. log(comment_text) if self.dryrun: return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") bug_status = self.browser.find_control("bug_status", type="select") # This is a hack around the fact that ClientForm.ListControl seems to # have no simpler way to ask if a control has an item named "REOPENED" # without using exceptions for control flow. possible_bug_statuses = map(lambda item: item.name, bug_status.items) if "REOPENED" in possible_bug_statuses: bug_status.value = ["REOPENED"] # If the bug was never confirmed it will not have a "REOPENED" # state, but only an "UNCONFIRMED" state. elif "UNCONFIRMED" in possible_bug_statuses: bug_status.value = ["UNCONFIRMED"] else: # FIXME: This logic is slightly backwards. We won't print this # message if the bug is already open with state "UNCONFIRMED". log("Did not reopen bug %s, it appears to already be open with status %s." % (bug_id, bug_status.value)) self.browser['comment'] = comment_text self.browser.submit()
class Builder(object): def __init__(self, name, buildbot): self._name = name self._buildbot = buildbot self._builds_cache = {} self._revision_to_build_number = None self._browser = None def name(self): return self._name def results_url(self): return "%s/results/%s" % (self._buildbot.buildbot_url, self.url_encoded_name()) # In addition to per-build results, the build.chromium.org builders also # keep a directory that accumulates test results over many runs. def accumulated_results_url(self): return None def latest_layout_test_results_url(self): return self.accumulated_results_url() or self.latest_cached_build( ).results_url() @memoized def latest_layout_test_results(self): return self.fetch_layout_test_results( self.latest_layout_test_results_url()) def _fetch_file_from_results(self, results_url, file_name): # It seems this can return None if the url redirects and then returns 404. result = urllib2.urlopen("%s/%s" % (results_url, file_name)) if not result: return None # urlopen returns a file-like object which sometimes works fine with str() # but sometimes is a addinfourl object. In either case calling read() is correct. return result.read() def fetch_layout_test_results(self, results_url): # FIXME: This should cache that the result was a 404 and stop hitting the network. results_file = NetworkTransaction(convert_404_to_None=True).run( lambda: self._fetch_file_from_results(results_url, "full_results.json")) return LayoutTestResults.results_from_string(results_file) def url_encoded_name(self): return urllib.quote(self._name) def url(self): return "%s/builders/%s" % (self._buildbot.buildbot_url, self.url_encoded_name()) # This provides a single place to mock def _fetch_build(self, build_number): build_dictionary = self._buildbot._fetch_build_dictionary( self, build_number) if not build_dictionary: return None revision_string = build_dictionary['sourceStamp']['revision'] return Build( self, build_number=int(build_dictionary['number']), # 'revision' may be None if a trunk build was started by the force-build button on the web page. revision=(int(revision_string) if revision_string else None), # Buildbot uses any nubmer other than 0 to mean fail. Since we fetch with # filter=1, passing builds may contain no 'results' value. is_green=(not build_dictionary.get('results')), ) def build(self, build_number): if not build_number: return None cached_build = self._builds_cache.get(build_number) if cached_build: return cached_build build = self._fetch_build(build_number) self._builds_cache[build_number] = build return build def latest_cached_build(self): revision_build_pairs = self.revision_build_pairs_with_results() revision_build_pairs.sort(key=lambda i: i[1]) latest_build_number = revision_build_pairs[-1][1] return self.build(latest_build_number) def force_build(self, username="******", comments=None): def predicate(form): try: return form.find_control("username") except Exception as e: return False if not self._browser: self._browser = Browser() self._browser.set_handle_robots( False) # The builder pages are excluded by robots.txt # ignore false positives for missing Browser methods - pylint: disable=E1102 self._browser.open(self.url()) self._browser.select_form(predicate=predicate) self._browser["username"] = username if comments: self._browser["comments"] = comments return self._browser.submit() file_name_regexp = re.compile( r"r(?P<revision>\d+) \((?P<build_number>\d+)\)") def _revision_and_build_for_filename(self, filename): # Example: "r47483 (1)/" or "r47483 (1).zip" match = self.file_name_regexp.match(filename) if not match: return None return (int(match.group("revision")), int(match.group("build_number"))) def _fetch_revision_to_build_map(self): # All _fetch requests go through _buildbot for easier mocking # FIXME: This should use NetworkTransaction's 404 handling instead. try: # FIXME: This method is horribly slow due to the huge network load. # FIXME: This is a poor way to do revision -> build mapping. # Better would be to ask buildbot through some sort of API. print("Loading revision/build list from %s." % self.results_url()) print("This may take a while...") result_files = self._buildbot._fetch_twisted_directory_listing( self.results_url()) except urllib2.HTTPError as error: if error.code != 404: raise _log.debug("Revision/build list failed to load.") result_files = [] return dict( self._file_info_list_to_revision_to_build_list(result_files)) def _file_info_list_to_revision_to_build_list(self, file_info_list): # This assumes there was only one build per revision, which is false but we don't care for now. revisions_and_builds = [] for file_info in file_info_list: revision_and_build = self._revision_and_build_for_filename( file_info["filename"]) if revision_and_build: revisions_and_builds.append(revision_and_build) return revisions_and_builds def _revision_to_build_map(self): if not self._revision_to_build_number: self._revision_to_build_number = self._fetch_revision_to_build_map( ) return self._revision_to_build_number def revision_build_pairs_with_results(self): return self._revision_to_build_map().items() # This assumes there can be only one build per revision, which is false, but we don't care for now. def build_for_revision(self, revision, allow_failed_lookups=False): # NOTE: This lookup will fail if that exact revision was never built. build_number = self._revision_to_build_map().get(int(revision)) if not build_number: return None build = self.build(build_number) if not build and allow_failed_lookups: # Builds for old revisions with fail to lookup via buildbot's json api. build = Build( self, build_number=build_number, revision=revision, is_green=False, ) return build def find_regression_window(self, red_build, look_back_limit=30): if not red_build or red_build.is_green(): return RegressionWindow(None, None) common_failures = None current_build = red_build build_after_current_build = None look_back_count = 0 while current_build: if current_build.is_green(): # current_build can't possibly have any failures in common # with red_build because it's green. break results = current_build.layout_test_results() # We treat a lack of results as if all the test failed. # This occurs, for example, when we can't compile at all. if results: failures = set(results.failing_tests()) if common_failures == None: common_failures = failures else: common_failures = common_failures.intersection(failures) if not common_failures: # current_build doesn't have any failures in common with # the red build we're worried about. We assume that any # failures in current_build were due to flakiness. break look_back_count += 1 if look_back_count > look_back_limit: return RegressionWindow(None, current_build, failing_tests=common_failures) build_after_current_build = current_build current_build = current_build.previous_build() # We must iterate at least once because red_build is red. assert (build_after_current_build) # Current build must either be green or have no failures in common # with red build, so we've found our failure transition. return RegressionWindow(current_build, build_after_current_build, failing_tests=common_failures) def find_blameworthy_regression_window(self, red_build_number, look_back_limit=30, avoid_flakey_tests=True): red_build = self.build(red_build_number) regression_window = self.find_regression_window( red_build, look_back_limit) if not regression_window.build_before_failure(): return None # We ran off the limit of our search # If avoid_flakey_tests, require at least 2 bad builds before we # suspect a real failure transition. if avoid_flakey_tests and regression_window.failing_build( ) == red_build: return None return regression_window
class StatusServer: default_host = "webkit-commit-queue.appspot.com" def __init__(self, host=default_host): self.set_host(host) self.browser = Browser() def set_host(self, host): self.host = host self.url = "http://%s" % self.host def results_url_for_status(self, status_id): return "%s/results/%s" % (self.url, status_id) def _add_patch(self, patch): if not patch: return if patch.bug_id(): self.browser["bug_id"] = unicode(patch.bug_id()) if patch.id(): self.browser["patch_id"] = unicode(patch.id()) def _add_results_file(self, results_file): if not results_file: return self.browser.add_file(results_file, "text/plain", "results.txt", 'results_file') def _post_status_to_server(self, queue_name, status, patch, results_file): if results_file: # We might need to re-wind the file if we've already tried to post it. results_file.seek(0) update_status_url = "%s/update-status" % self.url self.browser.open(update_status_url) self.browser.select_form(name="update_status") self.browser["queue_name"] = queue_name self._add_patch(patch) self.browser["status"] = status self._add_results_file(results_file) return self.browser.submit().read( ) # This is the id of the newly created status object. def _post_svn_revision_to_server(self, svn_revision_number, broken_bot): update_svn_revision_url = "%s/update-svn-revision" % self.url self.browser.open(update_svn_revision_url) self.browser.select_form(name="update_svn_revision") self.browser["number"] = unicode(svn_revision_number) self.browser["broken_bot"] = broken_bot return self.browser.submit().read() def update_status(self, queue_name, status, patch=None, results_file=None): log(status) return NetworkTransaction().run(lambda: self._post_status_to_server( queue_name, status, patch, results_file)) def update_svn_revision(self, svn_revision_number, broken_bot): log("SVN revision: %s broke %s" % (svn_revision_number, broken_bot)) return NetworkTransaction().run( lambda: self._post_svn_revision_to_server(svn_revision_number, broken_bot)) def _fetch_url(self, url): try: return urllib2.urlopen(url).read() except urllib2.HTTPError, e: if e.code == 404: return None raise e