예제 #1
0
class TestResultsUploader:
    def __init__(self, host):
        self._host = host
        self._browser = Browser()

    def _upload_files(self, attrs, file_objs):
        self._browser.open("http://%s/testfile/uploadform" % self._host)
        self._browser.select_form("test_result_upload")
        for (name, data) in attrs:
            self._browser[name] = str(data)

        for (filename, handle) in file_objs:
            self._browser.add_file(handle, get_mime_type(filename), filename,
                                   "file")

        self._browser.submit()

    def upload(self, params, files, timeout_seconds):
        orig_timeout = socket.getdefaulttimeout()
        file_objs = []
        try:
            file_objs = [(filename, open(path, "rb"))
                         for (filename, path) in files]

            socket.setdefaulttimeout(timeout_seconds)
            NetworkTransaction(timeout_seconds=timeout_seconds).run(
                lambda: self._upload_files(params, file_objs))
        finally:
            socket.setdefaulttimeout(orig_timeout)
            for (filename, handle) in file_objs:
                handle.close()
예제 #2
0
class TestResultsUploader:
    def __init__(self, host):
        self._host = host
        self._browser = Browser()

    def _upload_files(self, attrs, file_objs):
        self._browser.open("http://%s/testfile/uploadform" % self._host)
        self._browser.select_form("test_result_upload")
        for (name, data) in attrs:
            self._browser[name] = str(data)

        for (filename, handle) in file_objs:
            self._browser.add_file(handle, get_mime_type(filename), filename,
                "file")

        self._browser.submit()

    def upload(self, params, files, timeout_seconds):
        orig_timeout = socket.getdefaulttimeout()
        file_objs = []
        try:
            file_objs = [(filename, open(path, "rb")) for (filename, path)
                in files]

            socket.setdefaulttimeout(timeout_seconds)
            NetworkTransaction(timeout_seconds=timeout_seconds).run(
                lambda: self._upload_files(params, file_objs))
        finally:
            socket.setdefaulttimeout(orig_timeout)
            for (filename, handle) in file_objs:
                handle.close()
예제 #3
0
class Bugzilla(object):

    def __init__(self, dryrun=False, committers=committers.CommitterList()):
        self.dryrun = dryrun
        self.authenticated = False
        self.queries = BugzillaQueries(self)
        self.committers = committers

        # FIXME: We should use some sort of Browser mock object when in dryrun
        # mode (to prevent any mistakes).
        self.browser = Browser()
        # Ignore bugs.webkit.org/robots.txt until we fix it to allow this
        # script.
        self.browser.set_handle_robots(False)

    # FIXME: Much of this should go into some sort of config module:
    bug_server_host = "bugs.webkit.org"
    bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
    bug_server_url = "https://%s/" % bug_server_host

    def bug_url_for_bug_id(self, bug_id, xml=False):
        if not bug_id:
            return None
        content_type = "&ctype=xml" if xml else ""
        return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url,
                                           bug_id,
                                           content_type)

    def short_bug_url_for_bug_id(self, bug_id):
        if not bug_id:
            return None
        return "http://webkit.org/b/%s" % bug_id

    def attachment_url_for_id(self, attachment_id, action="view"):
        if not attachment_id:
            return None
        action_param = ""
        if action and action != "view":
            action_param = "&action=%s" % action
        return "%sattachment.cgi?id=%s%s" % (self.bug_server_url,
                                             attachment_id,
                                             action_param)

    def _parse_attachment_flag(self,
                               element,
                               flag_name,
                               attachment,
                               result_key):
        flag = element.find('flag', attrs={'name': flag_name})
        if flag:
            attachment[flag_name] = flag['status']
            if flag['status'] == '+':
                attachment[result_key] = flag['setter']

    def _parse_attachment_element(self, element, bug_id):
        attachment = {}
        attachment['bug_id'] = bug_id
        attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
        attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
        attachment['id'] = int(element.find('attachid').string)
        # FIXME: No need to parse out the url here.
        attachment['url'] = self.attachment_url_for_id(attachment['id'])
        attachment['name'] = unicode(element.find('desc').string)
        attachment['attacher_email'] = str(element.find('attacher').string)
        attachment['type'] = str(element.find('type').string)
        self._parse_attachment_flag(
                element, 'review', attachment, 'reviewer_email')
        self._parse_attachment_flag(
                element, 'commit-queue', attachment, 'committer_email')
        return attachment

    def _parse_bug_page(self, page):
        soup = BeautifulSoup(page)
        bug = {}
        bug["id"] = int(soup.find("bug_id").string)
        bug["title"] = unicode(soup.find("short_desc").string)
        bug["reporter_email"] = str(soup.find("reporter").string)
        bug["assigned_to_email"] = str(soup.find("assigned_to").string)
        bug["cc_emails"] = [str(element.string)
                            for element in soup.findAll('cc')]
        bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')]
        return bug

    # Makes testing fetch_*_from_bug() possible until we have a better
    # BugzillaNetwork abstration.

    def _fetch_bug_page(self, bug_id):
        bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
        log("Fetching: %s" % bug_url)
        return self.browser.open(bug_url)

    def fetch_bug_dictionary(self, bug_id):
        try:
            return self._parse_bug_page(self._fetch_bug_page(bug_id))
        except:
            self.authenticate()
            return self._parse_bug_page(self._fetch_bug_page(bug_id))

    # FIXME: A BugzillaCache object should provide all these fetch_ methods.

    def fetch_bug(self, bug_id):
        return Bug(self.fetch_bug_dictionary(bug_id), self)

    def _parse_bug_id_from_attachment_page(self, page):
        # The "Up" relation happens to point to the bug.
        up_link = BeautifulSoup(page).find('link', rel='Up')
        if not up_link:
            # This attachment does not exist (or you don't have permissions to
            # view it).
            return None
        match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href'])
        return int(match.group('bug_id'))

    def bug_id_for_attachment_id(self, attachment_id):
        self.authenticate()

        attachment_url = self.attachment_url_for_id(attachment_id, 'edit')
        log("Fetching: %s" % attachment_url)
        page = self.browser.open(attachment_url)
        return self._parse_bug_id_from_attachment_page(page)

    # FIXME: This should just return Attachment(id), which should be able to
    # lazily fetch needed data.

    def fetch_attachment(self, attachment_id):
        # We could grab all the attachment details off of the attachment edit
        # page but we already have working code to do so off of the bugs page,
        # so re-use that.
        bug_id = self.bug_id_for_attachment_id(attachment_id)
        if not bug_id:
            return None
        attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True)
        for attachment in attachments:
            if attachment.id() == int(attachment_id):
                return attachment
        return None # This should never be hit.

    def authenticate(self):
        if self.authenticated:
            return

        if self.dryrun:
            log("Skipping log in for dry run...")
            self.authenticated = True
            return

        attempts = 0
        while not self.authenticated:
            attempts += 1
            (username, password) = Credentials(
                self.bug_server_host, git_prefix="bugzilla").read_credentials()

            log("Logging in as %s..." % username)
            self.browser.open(self.bug_server_url +
                              "index.cgi?GoAheadAndLogIn=1")
            self.browser.select_form(name="login")
            self.browser['Bugzilla_login'] = username
            self.browser['Bugzilla_password'] = password
            response = self.browser.submit()

            match = re.search("<title>(.+?)</title>", response.read())
            # If the resulting page has a title, and it contains the word
            # "invalid" assume it's the login failure page.
            if match and re.search("Invalid", match.group(1), re.IGNORECASE):
                errorMessage = "Bugzilla login failed: %s" % match.group(1)
                # raise an exception only if this was the last attempt
                if attempts < 5:
                    log(errorMessage)
                else:
                    raise Exception(errorMessage)
            else:
                self.authenticated = True

    def _fill_attachment_form(self,
                              description,
                              patch_file_object,
                              comment_text=None,
                              mark_for_review=False,
                              mark_for_commit_queue=False,
                              mark_for_landing=False, bug_id=None):
        self.browser['description'] = description
        self.browser['ispatch'] = ("1",)
        self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)

        if mark_for_landing:
            self.browser['flag_type-3'] = ('+',)
        elif mark_for_commit_queue:
            self.browser['flag_type-3'] = ('?',)
        else:
            self.browser['flag_type-3'] = ('X',)

        if bug_id:
            patch_name = "bug-%s-%s.patch" % (bug_id, timestamp())
        else:
            patch_name ="%s.patch" % timestamp()
        self.browser.add_file(patch_file_object,
                              "text/plain",
                              patch_name,
                              'data')

    def add_patch_to_bug(self,
                         bug_id,
                         patch_file_object,
                         description,
                         comment_text=None,
                         mark_for_review=False,
                         mark_for_commit_queue=False,
                         mark_for_landing=False):
        self.authenticate()

        log('Adding patch "%s" to %sshow_bug.cgi?id=%s' % (description,
                                                           self.bug_server_url,
                                                           bug_id))

        if self.dryrun:
            log(comment_text)
            return

        self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % (
                          self.bug_server_url, bug_id))
        self.browser.select_form(name="entryform")
        self._fill_attachment_form(description,
                                   patch_file_object,
                                   mark_for_review=mark_for_review,
                                   mark_for_commit_queue=mark_for_commit_queue,
                                   mark_for_landing=mark_for_landing,
                                   bug_id=bug_id)
        if comment_text:
            log(comment_text)
            self.browser['comment'] = comment_text
        self.browser.submit()

    def _check_create_bug_response(self, response_html):
        match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>",
                          response_html)
        if match:
            return match.group('bug_id')

        match = re.search(
            '<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">',
            response_html,
            re.DOTALL)
        error_message = "FAIL"
        if match:
            text_lines = BeautifulSoup(
                    match.group('error_message')).findAll(text=True)
            error_message = "\n" + '\n'.join(
                    ["  " + line.strip()
                     for line in text_lines if line.strip()])
        raise Exception("Bug not created: %s" % error_message)

    def create_bug(self,
                   bug_title,
                   bug_description,
                   component=None,
                   patch_file_object=None,
                   patch_description=None,
                   cc=None,
                   blocked=None,
                   mark_for_review=False,
                   mark_for_commit_queue=False):
        self.authenticate()

        log('Creating bug with title "%s"' % bug_title)
        if self.dryrun:
            log(bug_description)
            return

        self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit")
        self.browser.select_form(name="Create")
        component_items = self.browser.find_control('component').items
        component_names = map(lambda item: item.name, component_items)
        if not component:
            component = "New Bugs"
        if component not in component_names:
            component = User.prompt_with_list("Please pick a component:", component_names)
        self.browser["component"] = [component]
        if cc:
            self.browser["cc"] = cc
        if blocked:
            self.browser["blocked"] = str(blocked)
        self.browser["short_desc"] = bug_title
        self.browser["comment"] = bug_description

        if patch_file_object:
            self._fill_attachment_form(
                    patch_description,
                    patch_file_object,
                    mark_for_review=mark_for_review,
                    mark_for_commit_queue=mark_for_commit_queue)

        response = self.browser.submit()

        bug_id = self._check_create_bug_response(response.read())
        log("Bug %s created." % bug_id)
        log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id))
        return bug_id

    def _find_select_element_for_flag(self, flag_name):
        # FIXME: This will break if we ever re-order attachment flags
        if flag_name == "review":
            return self.browser.find_control(type='select', nr=0)
        if flag_name == "commit-queue":
            return self.browser.find_control(type='select', nr=1)
        raise Exception("Don't know how to find flag named \"%s\"" % flag_name)

    def clear_attachment_flags(self,
                               attachment_id,
                               additional_comment_text=None):
        self.authenticate()

        comment_text = "Clearing flags on attachment: %s" % attachment_id
        if additional_comment_text:
            comment_text += "\n\n%s" % additional_comment_text
        log(comment_text)

        if self.dryrun:
            return

        self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
        self.browser.select_form(nr=1)
        self.browser.set_value(comment_text, name='comment', nr=0)
        self._find_select_element_for_flag('review').value = ("X",)
        self._find_select_element_for_flag('commit-queue').value = ("X",)
        self.browser.submit()

    def set_flag_on_attachment(self,
                               attachment_id,
                               flag_name,
                               flag_value,
                               comment_text,
                               additional_comment_text):
        # FIXME: We need a way to test this function on a live bugzilla
        # instance.

        self.authenticate()

        if additional_comment_text:
            comment_text += "\n\n%s" % additional_comment_text
        log(comment_text)

        if self.dryrun:
            return

        self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
        self.browser.select_form(nr=1)
        self.browser.set_value(comment_text, name='comment', nr=0)
        self._find_select_element_for_flag(flag_name).value = (flag_value,)
        self.browser.submit()

    # FIXME: All of these bug editing methods have a ridiculous amount of
    # copy/paste code.

    def obsolete_attachment(self, attachment_id, comment_text=None):
        self.authenticate()

        log("Obsoleting attachment: %s" % attachment_id)
        if self.dryrun:
            log(comment_text)
            return

        self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
        self.browser.select_form(nr=1)
        self.browser.find_control('isobsolete').items[0].selected = True
        # Also clear any review flag (to remove it from review/commit queues)
        self._find_select_element_for_flag('review').value = ("X",)
        self._find_select_element_for_flag('commit-queue').value = ("X",)
        if comment_text:
            log(comment_text)
            # Bugzilla has two textareas named 'comment', one is somehow
            # hidden.  We want the first.
            self.browser.set_value(comment_text, name='comment', nr=0)
        self.browser.submit()

    def add_cc_to_bug(self, bug_id, email_address_list):
        self.authenticate()

        log("Adding %s to the CC list for bug %s" % (email_address_list,
                                                     bug_id))
        if self.dryrun:
            return

        self.browser.open(self.bug_url_for_bug_id(bug_id))
        self.browser.select_form(name="changeform")
        self.browser["newcc"] = ", ".join(email_address_list)
        self.browser.submit()

    def post_comment_to_bug(self, bug_id, comment_text, cc=None):
        self.authenticate()

        log("Adding comment to bug %s" % bug_id)
        if self.dryrun:
            log(comment_text)
            return

        self.browser.open(self.bug_url_for_bug_id(bug_id))
        self.browser.select_form(name="changeform")
        self.browser["comment"] = comment_text
        if cc:
            self.browser["newcc"] = ", ".join(cc)
        self.browser.submit()

    def close_bug_as_fixed(self, bug_id, comment_text=None):
        self.authenticate()

        log("Closing bug %s as fixed" % bug_id)
        if self.dryrun:
            log(comment_text)
            return

        self.browser.open(self.bug_url_for_bug_id(bug_id))
        self.browser.select_form(name="changeform")
        if comment_text:
            log(comment_text)
            self.browser['comment'] = comment_text
        self.browser['bug_status'] = ['RESOLVED']
        self.browser['resolution'] = ['FIXED']
        self.browser.submit()

    def reassign_bug(self, bug_id, assignee, comment_text=None):
        self.authenticate()

        log("Assigning bug %s to %s" % (bug_id, assignee))
        if self.dryrun:
            log(comment_text)
            return

        self.browser.open(self.bug_url_for_bug_id(bug_id))
        self.browser.select_form(name="changeform")
        if comment_text:
            log(comment_text)
            self.browser["comment"] = comment_text
        self.browser["assigned_to"] = assignee
        self.browser.submit()

    def reopen_bug(self, bug_id, comment_text):
        self.authenticate()

        log("Re-opening bug %s" % bug_id)
        # Bugzilla requires a comment when re-opening a bug, so we know it will
        # never be None.
        log(comment_text)
        if self.dryrun:
            return

        self.browser.open(self.bug_url_for_bug_id(bug_id))
        self.browser.select_form(name="changeform")
        bug_status = self.browser.find_control("bug_status", type="select")
        # This is a hack around the fact that ClientForm.ListControl seems to
        # have no simpler way to ask if a control has an item named "REOPENED"
        # without using exceptions for control flow.
        possible_bug_statuses = map(lambda item: item.name, bug_status.items)
        if "REOPENED" in possible_bug_statuses:
            bug_status.value = ["REOPENED"]
        else:
            log("Did not reopen bug %s.  " +
                "It appears to already be open with status %s." % (
                        bug_id, bug_status.value))
        self.browser['comment'] = comment_text
        self.browser.submit()
예제 #4
0
class Bugzilla(object):

    def __init__(self, dryrun=False, committers=committers.CommitterList()):
        self.dryrun = dryrun
        self.authenticated = False
        self.queries = BugzillaQueries(self)
        self.committers = committers
        self.cached_quips = []

        # FIXME: We should use some sort of Browser mock object when in dryrun
        # mode (to prevent any mistakes).
        self.browser = Browser()
        # Ignore bugs.webkit.org/robots.txt until we fix it to allow this
        # script.
        self.browser.set_handle_robots(False)

    # FIXME: Much of this should go into some sort of config module:
    bug_server_host = "bugs.webkit.org"
    bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
    bug_server_url = "https://%s/" % bug_server_host

    def quips(self):
        # We only fetch and parse the list of quips once per instantiation
        # so that we do not burden bugs.webkit.org.
        if not self.cached_quips and not self.dryrun:
            self.cached_quips = self.queries.fetch_quips()
        return self.cached_quips

    def bug_url_for_bug_id(self, bug_id, xml=False):
        if not bug_id:
            return None
        content_type = "&ctype=xml" if xml else ""
        return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type)

    def short_bug_url_for_bug_id(self, bug_id):
        if not bug_id:
            return None
        return "http://webkit.org/b/%s" % bug_id

    def add_attachment_url(self, bug_id):
        return "%sattachment.cgi?action=enter&bugid=%s" % (self.bug_server_url, bug_id)

    def attachment_url_for_id(self, attachment_id, action="view"):
        if not attachment_id:
            return None
        action_param = ""
        if action and action != "view":
            action_param = "&action=%s" % action
        return "%sattachment.cgi?id=%s%s" % (self.bug_server_url,
                                             attachment_id,
                                             action_param)

    def _parse_attachment_flag(self,
                               element,
                               flag_name,
                               attachment,
                               result_key):
        flag = element.find('flag', attrs={'name': flag_name})
        if flag:
            attachment[flag_name] = flag['status']
            if flag['status'] == '+':
                attachment[result_key] = flag['setter']
        # Sadly show_bug.cgi?ctype=xml does not expose the flag modification date.

    def _string_contents(self, soup):
        # WebKit's bugzilla instance uses UTF-8.
        # BeautifulSoup always returns Unicode strings, however
        # the .string method returns a (unicode) NavigableString.
        # NavigableString can confuse other parts of the code, so we
        # convert from NavigableString to a real unicode() object using unicode().
        return unicode(soup.string)

    # Example: 2010-01-20 14:31 PST
    # FIXME: Some bugzilla dates seem to have seconds in them?
    # Python does not support timezones out of the box.
    # Assume that bugzilla always uses PST (which is true for bugs.webkit.org)
    _bugzilla_date_format = "%Y-%m-%d %H:%M"

    @classmethod
    def _parse_date(cls, date_string):
        (date, time, time_zone) = date_string.split(" ")
        # Ignore the timezone because python doesn't understand timezones out of the box.
        date_string = "%s %s" % (date, time)
        return datetime.strptime(date_string, cls._bugzilla_date_format)

    def _date_contents(self, soup):
        return self._parse_date(self._string_contents(soup))

    def _parse_attachment_element(self, element, bug_id):
        attachment = {}
        attachment['bug_id'] = bug_id
        attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
        attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
        attachment['id'] = int(element.find('attachid').string)
        # FIXME: No need to parse out the url here.
        attachment['url'] = self.attachment_url_for_id(attachment['id'])
        attachment["attach_date"] = self._date_contents(element.find("date"))
        attachment['name'] = self._string_contents(element.find('desc'))
        attachment['attacher_email'] = self._string_contents(element.find('attacher'))
        attachment['type'] = self._string_contents(element.find('type'))
        self._parse_attachment_flag(
                element, 'review', attachment, 'reviewer_email')
        self._parse_attachment_flag(
                element, 'commit-queue', attachment, 'committer_email')
        return attachment

    def _parse_bugs_from_xml(self, page):
        soup = BeautifulSoup(page)
        # Without the unicode() call, BeautifulSoup occasionally complains of being
        # passed None for no apparent reason.
        return [Bug(self._parse_bug_dictionary_from_xml(unicode(bug_xml)), self) for bug_xml in soup('bug')]

    def _parse_bug_dictionary_from_xml(self, page):
        soup = BeautifulSoup(page)
        bug = {}
        bug["id"] = int(soup.find("bug_id").string)
        bug["title"] = self._string_contents(soup.find("short_desc"))
        bug["bug_status"] = self._string_contents(soup.find("bug_status"))
        dup_id = soup.find("dup_id")
        if dup_id:
            bug["dup_id"] = self._string_contents(dup_id)
        bug["reporter_email"] = self._string_contents(soup.find("reporter"))
        bug["assigned_to_email"] = self._string_contents(soup.find("assigned_to"))
        bug["cc_emails"] = [self._string_contents(element) for element in soup.findAll('cc')]
        bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')]
        return bug

    # Makes testing fetch_*_from_bug() possible until we have a better
    # BugzillaNetwork abstration.

    def _fetch_bug_page(self, bug_id):
        bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
        log("Fetching: %s" % bug_url)
        return self.browser.open(bug_url)

    def fetch_bug_dictionary(self, bug_id):
        try:
            return self._parse_bug_dictionary_from_xml(self._fetch_bug_page(bug_id))
        except KeyboardInterrupt:
            raise
        except:
            self.authenticate()
            return self._parse_bug_dictionary_from_xml(self._fetch_bug_page(bug_id))

    # FIXME: A BugzillaCache object should provide all these fetch_ methods.

    def fetch_bug(self, bug_id):
        return Bug(self.fetch_bug_dictionary(bug_id), self)

    def fetch_attachment_contents(self, attachment_id):
        attachment_url = self.attachment_url_for_id(attachment_id)
        # We need to authenticate to download patches from security bugs.
        self.authenticate()
        return self.browser.open(attachment_url).read()

    def _parse_bug_id_from_attachment_page(self, page):
        # The "Up" relation happens to point to the bug.
        up_link = BeautifulSoup(page).find('link', rel='Up')
        if not up_link:
            # This attachment does not exist (or you don't have permissions to
            # view it).
            return None
        match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href'])
        return int(match.group('bug_id'))

    def bug_id_for_attachment_id(self, attachment_id):
        self.authenticate()

        attachment_url = self.attachment_url_for_id(attachment_id, 'edit')
        log("Fetching: %s" % attachment_url)
        page = self.browser.open(attachment_url)
        return self._parse_bug_id_from_attachment_page(page)

    # FIXME: This should just return Attachment(id), which should be able to
    # lazily fetch needed data.

    def fetch_attachment(self, attachment_id):
        # We could grab all the attachment details off of the attachment edit
        # page but we already have working code to do so off of the bugs page,
        # so re-use that.
        bug_id = self.bug_id_for_attachment_id(attachment_id)
        if not bug_id:
            return None
        attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True)
        for attachment in attachments:
            if attachment.id() == int(attachment_id):
                return attachment
        return None # This should never be hit.

    def authenticate(self):
        if self.authenticated:
            return

        if self.dryrun:
            log("Skipping log in for dry run...")
            self.authenticated = True
            return

        credentials = Credentials(self.bug_server_host, git_prefix="bugzilla")

        attempts = 0
        while not self.authenticated:
            attempts += 1
            username, password = credentials.read_credentials()

            log("Logging in as %s..." % username)
            self.browser.open(self.bug_server_url +
                              "index.cgi?GoAheadAndLogIn=1")
            self.browser.select_form(name="login")
            self.browser['Bugzilla_login'] = username
            self.browser['Bugzilla_password'] = password
            response = self.browser.submit()

            match = re.search("<title>(.+?)</title>", response.read())
            # If the resulting page has a title, and it contains the word
            # "invalid" assume it's the login failure page.
            if match and re.search("Invalid", match.group(1), re.IGNORECASE):
                errorMessage = "Bugzilla login failed: %s" % match.group(1)
                # raise an exception only if this was the last attempt
                if attempts < 5:
                    log(errorMessage)
                else:
                    raise Exception(errorMessage)
            else:
                self.authenticated = True
                self.username = username

    def _commit_queue_flag(self, mark_for_landing, mark_for_commit_queue):
        if mark_for_landing:
            return '+'
        elif mark_for_commit_queue:
            return '?'
        return 'X'

    # FIXME: mark_for_commit_queue and mark_for_landing should be joined into a single commit_flag argument.
    def _fill_attachment_form(self,
                              description,
                              file_object,
                              mark_for_review=False,
                              mark_for_commit_queue=False,
                              mark_for_landing=False,
                              is_patch=False,
                              filename=None,
                              mimetype=None):
        self.browser['description'] = description
        if is_patch:
            self.browser['ispatch'] = ("1",)
        # FIXME: Should this use self._find_select_element_for_flag?
        self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
        self.browser['flag_type-3'] = (self._commit_queue_flag(mark_for_landing, mark_for_commit_queue),)

        filename = filename or "%s.patch" % timestamp()
        mimetype = mimetype or "text/plain"
        self.browser.add_file(file_object, mimetype, filename, 'data')

    def _file_object_for_upload(self, file_or_string):
        if hasattr(file_or_string, 'read'):
            return file_or_string
        # Only if file_or_string is not already encoded do we want to encode it.
        if isinstance(file_or_string, unicode):
            file_or_string = file_or_string.encode('utf-8')
        return StringIO.StringIO(file_or_string)

    # timestamp argument is just for unittests.
    def _filename_for_upload(self, file_object, bug_id, extension="txt", timestamp=timestamp):
        if hasattr(file_object, "name"):
            return file_object.name
        return "bug-%s-%s.%s" % (bug_id, timestamp(), extension)

    def add_attachment_to_bug(self,
                              bug_id,
                              file_or_string,
                              description,
                              filename=None,
                              comment_text=None):
        self.authenticate()
        log('Adding attachment "%s" to %s' % (description, self.bug_url_for_bug_id(bug_id)))
        if self.dryrun:
            log(comment_text)
            return

        self.browser.open(self.add_attachment_url(bug_id))
        self.browser.select_form(name="entryform")
        file_object = self._file_object_for_upload(file_or_string)
        filename = filename or self._filename_for_upload(file_object, bug_id)
        self._fill_attachment_form(description, file_object, filename=filename)
        if comment_text:
            log(comment_text)
            self.browser['comment'] = comment_text
        self.browser.submit()

    # FIXME: The arguments to this function should be simplified and then
    # this should be merged into add_attachment_to_bug
    def add_patch_to_bug(self,
                         bug_id,
                         file_or_string,
                         description,
                         comment_text=None,
                         mark_for_review=False,
                         mark_for_commit_queue=False,
                         mark_for_landing=False):
        self.authenticate()
        log('Adding patch "%s" to %s' % (description, self.bug_url_for_bug_id(bug_id)))

        if self.dryrun:
            log(comment_text)
            return

        self.browser.open(self.add_attachment_url(bug_id))
        self.browser.select_form(name="entryform")
        file_object = self._file_object_for_upload(file_or_string)
        filename = self._filename_for_upload(file_object, bug_id, extension="patch")
        self._fill_attachment_form(description,
                                   file_object,
                                   mark_for_review=mark_for_review,
                                   mark_for_commit_queue=mark_for_commit_queue,
                                   mark_for_landing=mark_for_landing,
                                   is_patch=True,
                                   filename=filename)
        if comment_text:
            log(comment_text)
            self.browser['comment'] = comment_text
        self.browser.submit()

    # FIXME: There has to be a more concise way to write this method.
    def _check_create_bug_response(self, response_html):
        match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>",
                          response_html)
        if match:
            return match.group('bug_id')

        match = re.search(
            '<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">',
            response_html,
            re.DOTALL)
        error_message = "FAIL"
        if match:
            text_lines = BeautifulSoup(
                    match.group('error_message')).findAll(text=True)
            error_message = "\n" + '\n'.join(
                    ["  " + line.strip()
                     for line in text_lines if line.strip()])
        raise Exception("Bug not created: %s" % error_message)

    def create_bug(self,
                   bug_title,
                   bug_description,
                   component=None,
                   diff=None,
                   patch_description=None,
                   cc=None,
                   blocked=None,
                   assignee=None,
                   mark_for_review=False,
                   mark_for_commit_queue=False):
        self.authenticate()

        log('Creating bug with title "%s"' % bug_title)
        if self.dryrun:
            log(bug_description)
            # FIXME: This will make some paths fail, as they assume this returns an id.
            return

        self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit")
        self.browser.select_form(name="Create")
        component_items = self.browser.find_control('component').items
        component_names = map(lambda item: item.name, component_items)
        if not component:
            component = "New Bugs"
        if component not in component_names:
            component = User.prompt_with_list("Please pick a component:", component_names)
        self.browser["component"] = [component]
        if cc:
            self.browser["cc"] = cc
        if blocked:
            self.browser["blocked"] = unicode(blocked)
        if not assignee:
            assignee = self.username
        if assignee and not self.browser.find_control("assigned_to").disabled:
            self.browser["assigned_to"] = assignee
        self.browser["short_desc"] = bug_title
        self.browser["comment"] = bug_description

        if diff:
            # _fill_attachment_form expects a file-like object
            # Patch files are already binary, so no encoding needed.
            assert(isinstance(diff, str))
            patch_file_object = StringIO.StringIO(diff)
            self._fill_attachment_form(
                    patch_description,
                    patch_file_object,
                    mark_for_review=mark_for_review,
                    mark_for_commit_queue=mark_for_commit_queue,
                    is_patch=True)

        response = self.browser.submit()

        bug_id = self._check_create_bug_response(response.read())
        log("Bug %s created." % bug_id)
        log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id))
        return bug_id

    def _find_select_element_for_flag(self, flag_name):
        # FIXME: This will break if we ever re-order attachment flags
        if flag_name == "review":
            return self.browser.find_control(type='select', nr=0)
        elif flag_name == "commit-queue":
            return self.browser.find_control(type='select', nr=1)
        raise Exception("Don't know how to find flag named \"%s\"" % flag_name)

    def clear_attachment_flags(self,
                               attachment_id,
                               additional_comment_text=None):
        self.authenticate()

        comment_text = "Clearing flags on attachment: %s" % attachment_id
        if additional_comment_text:
            comment_text += "\n\n%s" % additional_comment_text
        log(comment_text)

        if self.dryrun:
            return

        self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
        self.browser.select_form(nr=1)
        self.browser.set_value(comment_text, name='comment', nr=0)
        self._find_select_element_for_flag('review').value = ("X",)
        self._find_select_element_for_flag('commit-queue').value = ("X",)
        self.browser.submit()

    def set_flag_on_attachment(self,
                               attachment_id,
                               flag_name,
                               flag_value,
                               comment_text=None,
                               additional_comment_text=None):
        # FIXME: We need a way to test this function on a live bugzilla
        # instance.

        self.authenticate()

        if additional_comment_text:
            comment_text += "\n\n%s" % additional_comment_text
        log(comment_text)

        if self.dryrun:
            return

        self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
        self.browser.select_form(nr=1)

        if comment_text:
            self.browser.set_value(comment_text, name='comment', nr=0)

        self._find_select_element_for_flag(flag_name).value = (flag_value,)
        self.browser.submit()

    # FIXME: All of these bug editing methods have a ridiculous amount of
    # copy/paste code.

    def obsolete_attachment(self, attachment_id, comment_text=None):
        self.authenticate()

        log("Obsoleting attachment: %s" % attachment_id)
        if self.dryrun:
            log(comment_text)
            return

        self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
        self.browser.select_form(nr=1)
        self.browser.find_control('isobsolete').items[0].selected = True
        # Also clear any review flag (to remove it from review/commit queues)
        self._find_select_element_for_flag('review').value = ("X",)
        self._find_select_element_for_flag('commit-queue').value = ("X",)
        if comment_text:
            log(comment_text)
            # Bugzilla has two textareas named 'comment', one is somehow
            # hidden.  We want the first.
            self.browser.set_value(comment_text, name='comment', nr=0)
        self.browser.submit()

    def add_cc_to_bug(self, bug_id, email_address_list):
        self.authenticate()

        log("Adding %s to the CC list for bug %s" % (email_address_list,
                                                     bug_id))
        if self.dryrun:
            return

        self.browser.open(self.bug_url_for_bug_id(bug_id))
        self.browser.select_form(name="changeform")
        self.browser["newcc"] = ", ".join(email_address_list)
        self.browser.submit()

    def post_comment_to_bug(self, bug_id, comment_text, cc=None):
        self.authenticate()

        log("Adding comment to bug %s" % bug_id)
        if self.dryrun:
            log(comment_text)
            return

        self.browser.open(self.bug_url_for_bug_id(bug_id))
        self.browser.select_form(name="changeform")
        self.browser["comment"] = comment_text
        if cc:
            self.browser["newcc"] = ", ".join(cc)
        self.browser.submit()

    def close_bug_as_fixed(self, bug_id, comment_text=None):
        self.authenticate()

        log("Closing bug %s as fixed" % bug_id)
        if self.dryrun:
            log(comment_text)
            return

        self.browser.open(self.bug_url_for_bug_id(bug_id))
        self.browser.select_form(name="changeform")
        if comment_text:
            self.browser['comment'] = comment_text
        self.browser['bug_status'] = ['RESOLVED']
        self.browser['resolution'] = ['FIXED']
        self.browser.submit()

    def reassign_bug(self, bug_id, assignee, comment_text=None):
        self.authenticate()

        log("Assigning bug %s to %s" % (bug_id, assignee))
        if self.dryrun:
            log(comment_text)
            return

        self.browser.open(self.bug_url_for_bug_id(bug_id))
        self.browser.select_form(name="changeform")
        if comment_text:
            log(comment_text)
            self.browser["comment"] = comment_text
        self.browser["assigned_to"] = assignee
        self.browser.submit()

    def reopen_bug(self, bug_id, comment_text):
        self.authenticate()

        log("Re-opening bug %s" % bug_id)
        # Bugzilla requires a comment when re-opening a bug, so we know it will
        # never be None.
        log(comment_text)
        if self.dryrun:
            return

        self.browser.open(self.bug_url_for_bug_id(bug_id))
        self.browser.select_form(name="changeform")
        bug_status = self.browser.find_control("bug_status", type="select")
        # This is a hack around the fact that ClientForm.ListControl seems to
        # have no simpler way to ask if a control has an item named "REOPENED"
        # without using exceptions for control flow.
        possible_bug_statuses = map(lambda item: item.name, bug_status.items)
        if "REOPENED" in possible_bug_statuses:
            bug_status.value = ["REOPENED"]
        # If the bug was never confirmed it will not have a "REOPENED"
        # state, but only an "UNCONFIRMED" state.
        elif "UNCONFIRMED" in possible_bug_statuses:
            bug_status.value = ["UNCONFIRMED"]
        else:
            # FIXME: This logic is slightly backwards.  We won't print this
            # message if the bug is already open with state "UNCONFIRMED".
            log("Did not reopen bug %s, it appears to already be open with status %s." % (bug_id, bug_status.value))
        self.browser['comment'] = comment_text
        self.browser.submit()
예제 #5
0
class StatusServer:
    default_host = "webkit-commit-queue.appspot.com"

    def __init__(self, host=default_host):
        self.set_host(host)
        self.browser = Browser()

    def set_host(self, host):
        self.host = host
        self.url = "http://%s" % self.host

    def results_url_for_status(self, status_id):
        return "%s/results/%s" % (self.url, status_id)

    def _add_patch(self, patch):
        if not patch:
            return
        if patch.bug_id():
            self.browser["bug_id"] = unicode(patch.bug_id())
        if patch.id():
            self.browser["patch_id"] = unicode(patch.id())

    def _add_results_file(self, results_file):
        if not results_file:
            return
        self.browser.add_file(results_file, "text/plain", "results.txt", "results_file")

    def _post_status_to_server(self, queue_name, status, patch, results_file):
        if results_file:
            # We might need to re-wind the file if we've already tried to post it.
            results_file.seek(0)

        update_status_url = "%s/update-status" % self.url
        self.browser.open(update_status_url)
        self.browser.select_form(name="update_status")
        self.browser["queue_name"] = queue_name
        self._add_patch(patch)
        self.browser["status"] = status
        self._add_results_file(results_file)
        return self.browser.submit().read()  # This is the id of the newly created status object.

    def _post_svn_revision_to_server(self, svn_revision_number, broken_bot):
        update_svn_revision_url = "%s/update-svn-revision" % self.url
        self.browser.open(update_svn_revision_url)
        self.browser.select_form(name="update_svn_revision")
        self.browser["number"] = unicode(svn_revision_number)
        self.browser["broken_bot"] = broken_bot
        return self.browser.submit().read()

    def _post_work_items_to_server(self, queue_name, work_items):
        update_work_items_url = "%s/update-work-items" % self.url
        self.browser.open(update_work_items_url)
        self.browser.select_form(name="update_work_items")
        self.browser["queue_name"] = queue_name
        work_items = map(unicode, work_items)  # .join expects strings
        self.browser["work_items"] = " ".join(work_items)
        return self.browser.submit().read()

    def update_work_items(self, queue_name, work_items):
        _log.debug("Recording work items: %s for %s" % (work_items, queue_name))
        return NetworkTransaction().run(lambda: self._post_work_items_to_server(queue_name, work_items))

    def update_status(self, queue_name, status, patch=None, results_file=None):
        log(status)
        return NetworkTransaction().run(lambda: self._post_status_to_server(queue_name, status, patch, results_file))

    def update_svn_revision(self, svn_revision_number, broken_bot):
        log("SVN revision: %s broke %s" % (svn_revision_number, broken_bot))
        return NetworkTransaction().run(lambda: self._post_svn_revision_to_server(svn_revision_number, broken_bot))

    def _fetch_url(self, url):
        try:
            return urllib2.urlopen(url).read()
        except urllib2.HTTPError, e:
            if e.code == 404:
                return None
            raise e
예제 #6
0
class Bugzilla(object):

    def __init__(self, dryrun=False, committers=committers.CommitterList()):
        self.dryrun = dryrun
        self.authenticated = False
        self.queries = BugzillaQueries(self)
        self.committers = committers
        self.cached_quips = []

        # FIXME: We should use some sort of Browser mock object when in dryrun
        # mode (to prevent any mistakes).
        self.browser = Browser()
        # Ignore bugs.webkit.org/robots.txt until we fix it to allow this
        # script.
        self.browser.set_handle_robots(False)

    # FIXME: Much of this should go into some sort of config module:
    bug_server_host = "bugs.webkit.org"
    bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
    bug_server_url = "https://%s/" % bug_server_host

    def quips(self):
        # We only fetch and parse the list of quips once per instantiation
        # so that we do not burden bugs.webkit.org.
        if not self.cached_quips and not self.dryrun:
            self.cached_quips = self.queries.fetch_quips()
        return self.cached_quips

    def bug_url_for_bug_id(self, bug_id, xml=False):
        if not bug_id:
            return None
        content_type = "&ctype=xml" if xml else ""
        return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url,
                                           bug_id,
                                           content_type)

    def short_bug_url_for_bug_id(self, bug_id):
        if not bug_id:
            return None
        return "http://webkit.org/b/%s" % bug_id

    def attachment_url_for_id(self, attachment_id, action="view"):
        if not attachment_id:
            return None
        action_param = ""
        if action and action != "view":
            action_param = "&action=%s" % action
        return "%sattachment.cgi?id=%s%s" % (self.bug_server_url,
                                             attachment_id,
                                             action_param)

    def _parse_attachment_flag(self,
                               element,
                               flag_name,
                               attachment,
                               result_key):
        flag = element.find('flag', attrs={'name': flag_name})
        if flag:
            attachment[flag_name] = flag['status']
            if flag['status'] == '+':
                attachment[result_key] = flag['setter']
        # Sadly show_bug.cgi?ctype=xml does not expose the flag modification date.

    def _string_contents(self, soup):
        # WebKit's bugzilla instance uses UTF-8.
        # BeautifulSoup always returns Unicode strings, however
        # the .string method returns a (unicode) NavigableString.
        # NavigableString can confuse other parts of the code, so we
        # convert from NavigableString to a real unicode() object using unicode().
        return unicode(soup.string)

    # Example: 2010-01-20 14:31 PST
    # FIXME: Some bugzilla dates seem to have seconds in them?
    # Python does not support timezones out of the box.
    # Assume that bugzilla always uses PST (which is true for bugs.webkit.org)
    _bugzilla_date_format = "%Y-%m-%d %H:%M"

    @classmethod
    def _parse_date(cls, date_string):
        (date, time, time_zone) = date_string.split(" ")
        # Ignore the timezone because python doesn't understand timezones out of the box.
        date_string = "%s %s" % (date, time)
        return datetime.strptime(date_string, cls._bugzilla_date_format)

    def _date_contents(self, soup):
        return self._parse_date(self._string_contents(soup))

    def _parse_attachment_element(self, element, bug_id):
        attachment = {}
        attachment['bug_id'] = bug_id
        attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
        attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
        attachment['id'] = int(element.find('attachid').string)
        # FIXME: No need to parse out the url here.
        attachment['url'] = self.attachment_url_for_id(attachment['id'])
        attachment["attach_date"] = self._date_contents(element.find("date"))
        attachment['name'] = self._string_contents(element.find('desc'))
        attachment['attacher_email'] = self._string_contents(element.find('attacher'))
        attachment['type'] = self._string_contents(element.find('type'))
        self._parse_attachment_flag(
                element, 'review', attachment, 'reviewer_email')
        self._parse_attachment_flag(
                element, 'in-rietveld', attachment, 'rietveld_uploader_email')
        self._parse_attachment_flag(
                element, 'commit-queue', attachment, 'committer_email')
        return attachment

    def _parse_bug_page(self, page):
        soup = BeautifulSoup(page)
        bug = {}
        bug["id"] = int(soup.find("bug_id").string)
        bug["title"] = self._string_contents(soup.find("short_desc"))
        bug["reporter_email"] = self._string_contents(soup.find("reporter"))
        bug["assigned_to_email"] = self._string_contents(soup.find("assigned_to"))
        bug["cc_emails"] = [self._string_contents(element)
                            for element in soup.findAll('cc')]
        bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')]
        return bug

    # Makes testing fetch_*_from_bug() possible until we have a better
    # BugzillaNetwork abstration.

    def _fetch_bug_page(self, bug_id):
        bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
        log("Fetching: %s" % bug_url)
        return self.browser.open(bug_url)

    def fetch_bug_dictionary(self, bug_id):
        try:
            return self._parse_bug_page(self._fetch_bug_page(bug_id))
        except KeyboardInterrupt:
            raise
        except:
            self.authenticate()
            return self._parse_bug_page(self._fetch_bug_page(bug_id))

    # FIXME: A BugzillaCache object should provide all these fetch_ methods.

    def fetch_bug(self, bug_id):
        return Bug(self.fetch_bug_dictionary(bug_id), self)

    def fetch_attachment_contents(self, attachment_id):
        attachment_url = self.attachment_url_for_id(attachment_id)
        # We need to authenticate to download patches from security bugs.
        self.authenticate()
        return self.browser.open(attachment_url).read()

    def _parse_bug_id_from_attachment_page(self, page):
        # The "Up" relation happens to point to the bug.
        up_link = BeautifulSoup(page).find('link', rel='Up')
        if not up_link:
            # This attachment does not exist (or you don't have permissions to
            # view it).
            return None
        match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href'])
        return int(match.group('bug_id'))

    def bug_id_for_attachment_id(self, attachment_id):
        self.authenticate()

        attachment_url = self.attachment_url_for_id(attachment_id, 'edit')
        log("Fetching: %s" % attachment_url)
        page = self.browser.open(attachment_url)
        return self._parse_bug_id_from_attachment_page(page)

    # FIXME: This should just return Attachment(id), which should be able to
    # lazily fetch needed data.

    def fetch_attachment(self, attachment_id):
        # We could grab all the attachment details off of the attachment edit
        # page but we already have working code to do so off of the bugs page,
        # so re-use that.
        bug_id = self.bug_id_for_attachment_id(attachment_id)
        if not bug_id:
            return None
        attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True)
        for attachment in attachments:
            if attachment.id() == int(attachment_id):
                return attachment
        return None # This should never be hit.

    def authenticate(self):
        if self.authenticated:
            return

        if self.dryrun:
            log("Skipping log in for dry run...")
            self.authenticated = True
            return

        attempts = 0
        while not self.authenticated:
            attempts += 1
            (username, password) = Credentials(
                self.bug_server_host, git_prefix="bugzilla").read_credentials()

            log("Logging in as %s..." % username)
            self.browser.open(self.bug_server_url +
                              "index.cgi?GoAheadAndLogIn=1")
            self.browser.select_form(name="login")
            self.browser['Bugzilla_login'] = username
            self.browser['Bugzilla_password'] = password
            response = self.browser.submit()

            match = re.search("<title>(.+?)</title>", response.read())
            # If the resulting page has a title, and it contains the word
            # "invalid" assume it's the login failure page.
            if match and re.search("Invalid", match.group(1), re.IGNORECASE):
                errorMessage = "Bugzilla login failed: %s" % match.group(1)
                # raise an exception only if this was the last attempt
                if attempts < 5:
                    log(errorMessage)
                else:
                    raise Exception(errorMessage)
            else:
                self.authenticated = True
                self.username = username

    def _fill_attachment_form(self,
                              description,
                              patch_file_object,
                              comment_text=None,
                              mark_for_review=False,
                              mark_for_commit_queue=False,
                              mark_for_landing=False,
                              bug_id=None):
        self.browser['description'] = description
        self.browser['ispatch'] = ("1",)
        self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)

        if mark_for_landing:
            self.browser['flag_type-3'] = ('+',)
        elif mark_for_commit_queue:
            self.browser['flag_type-3'] = ('?',)
        else:
            self.browser['flag_type-3'] = ('X',)

        if bug_id:
            patch_name = "bug-%s-%s.patch" % (bug_id, timestamp())
        else:
            patch_name ="%s.patch" % timestamp()

        self.browser.add_file(patch_file_object,
                              "text/plain",
                              patch_name,
                              'data')

    def add_patch_to_bug(self,
                         bug_id,
                         diff,
                         description,
                         comment_text=None,
                         mark_for_review=False,
                         mark_for_commit_queue=False,
                         mark_for_landing=False):
        self.authenticate()

        log('Adding patch "%s" to %sshow_bug.cgi?id=%s' % (description,
                                                           self.bug_server_url,
                                                           bug_id))

        if self.dryrun:
            log(comment_text)
            return

        self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % (
                          self.bug_server_url, bug_id))
        self.browser.select_form(name="entryform")

        # _fill_attachment_form expects a file-like object
        # Patch files are already binary, so no encoding needed.
        assert(isinstance(diff, str))
        patch_file_object = StringIO.StringIO(diff)
        self._fill_attachment_form(description,
                                   patch_file_object,
                                   mark_for_review=mark_for_review,
                                   mark_for_commit_queue=mark_for_commit_queue,
                                   mark_for_landing=mark_for_landing,
                                   bug_id=bug_id)
        if comment_text:
            log(comment_text)
            self.browser['comment'] = comment_text
        self.browser.submit()

    def _check_create_bug_response(self, response_html):
        match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>",
                          response_html)
        if match:
            return match.group('bug_id')

        match = re.search(
            '<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">',
            response_html,
            re.DOTALL)
        error_message = "FAIL"
        if match:
            text_lines = BeautifulSoup(
                    match.group('error_message')).findAll(text=True)
            error_message = "\n" + '\n'.join(
                    ["  " + line.strip()
                     for line in text_lines if line.strip()])
        raise Exception("Bug not created: %s" % error_message)

    def create_bug(self,
                   bug_title,
                   bug_description,
                   component=None,
                   diff=None,
                   patch_description=None,
                   cc=None,
                   blocked=None,
                   assignee=None,
                   mark_for_review=False,
                   mark_for_commit_queue=False):
        self.authenticate()

        log('Creating bug with title "%s"' % bug_title)
        if self.dryrun:
            log(bug_description)
            return

        self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit")
        self.browser.select_form(name="Create")
        component_items = self.browser.find_control('component').items
        component_names = map(lambda item: item.name, component_items)
        if not component:
            component = "New Bugs"
        if component not in component_names:
            component = User.prompt_with_list("Please pick a component:", component_names)
        self.browser["component"] = [component]
        if cc:
            self.browser["cc"] = cc
        if blocked:
            self.browser["blocked"] = unicode(blocked)
        if assignee == None:
            assignee = self.username
        if assignee and not self.browser.find_control("assigned_to").disabled:
            self.browser["assigned_to"] = assignee
        self.browser["short_desc"] = bug_title
        self.browser["comment"] = bug_description

        if diff:
            # _fill_attachment_form expects a file-like object
            # Patch files are already binary, so no encoding needed.
            assert(isinstance(diff, str))
            patch_file_object = StringIO.StringIO(diff)
            self._fill_attachment_form(
                    patch_description,
                    patch_file_object,
                    mark_for_review=mark_for_review,
                    mark_for_commit_queue=mark_for_commit_queue)

        response = self.browser.submit()

        bug_id = self._check_create_bug_response(response.read())
        log("Bug %s created." % bug_id)
        log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id))
        return bug_id

    def _find_select_element_for_flag(self, flag_name):
        # FIXME: This will break if we ever re-order attachment flags
        if flag_name == "review":
            return self.browser.find_control(type='select', nr=0)
        elif flag_name == "commit-queue":
            return self.browser.find_control(type='select', nr=1)
        elif flag_name == "in-rietveld":
            return self.browser.find_control(type='select', nr=2)
        raise Exception("Don't know how to find flag named \"%s\"" % flag_name)

    def clear_attachment_flags(self,
                               attachment_id,
                               additional_comment_text=None):
        self.authenticate()

        comment_text = "Clearing flags on attachment: %s" % attachment_id
        if additional_comment_text:
            comment_text += "\n\n%s" % additional_comment_text
        log(comment_text)

        if self.dryrun:
            return

        self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
        self.browser.select_form(nr=1)
        self.browser.set_value(comment_text, name='comment', nr=0)
        self._find_select_element_for_flag('review').value = ("X",)
        self._find_select_element_for_flag('commit-queue').value = ("X",)
        self.browser.submit()

    def set_flag_on_attachment(self,
                               attachment_id,
                               flag_name,
                               flag_value,
                               comment_text=None,
                               additional_comment_text=None):
        # FIXME: We need a way to test this function on a live bugzilla
        # instance.

        self.authenticate()

        if additional_comment_text:
            comment_text += "\n\n%s" % additional_comment_text
        log(comment_text)

        if self.dryrun:
            return

        self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
        self.browser.select_form(nr=1)

        if comment_text:
            self.browser.set_value(comment_text, name='comment', nr=0)

        self._find_select_element_for_flag(flag_name).value = (flag_value,)
        self.browser.submit()

    # FIXME: All of these bug editing methods have a ridiculous amount of
    # copy/paste code.

    def obsolete_attachment(self, attachment_id, comment_text=None):
        self.authenticate()

        log("Obsoleting attachment: %s" % attachment_id)
        if self.dryrun:
            log(comment_text)
            return

        self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
        self.browser.select_form(nr=1)
        self.browser.find_control('isobsolete').items[0].selected = True
        # Also clear any review flag (to remove it from review/commit queues)
        self._find_select_element_for_flag('review').value = ("X",)
        self._find_select_element_for_flag('commit-queue').value = ("X",)
        if comment_text:
            log(comment_text)
            # Bugzilla has two textareas named 'comment', one is somehow
            # hidden.  We want the first.
            self.browser.set_value(comment_text, name='comment', nr=0)
        self.browser.submit()

    def add_cc_to_bug(self, bug_id, email_address_list):
        self.authenticate()

        log("Adding %s to the CC list for bug %s" % (email_address_list,
                                                     bug_id))
        if self.dryrun:
            return

        self.browser.open(self.bug_url_for_bug_id(bug_id))
        self.browser.select_form(name="changeform")
        self.browser["newcc"] = ", ".join(email_address_list)
        self.browser.submit()

    def post_comment_to_bug(self, bug_id, comment_text, cc=None):
        self.authenticate()

        log("Adding comment to bug %s" % bug_id)
        if self.dryrun:
            log(comment_text)
            return

        self.browser.open(self.bug_url_for_bug_id(bug_id))
        self.browser.select_form(name="changeform")
        self.browser["comment"] = comment_text
        if cc:
            self.browser["newcc"] = ", ".join(cc)
        self.browser.submit()

    def close_bug_as_fixed(self, bug_id, comment_text=None):
        self.authenticate()

        log("Closing bug %s as fixed" % bug_id)
        if self.dryrun:
            log(comment_text)
            return

        self.browser.open(self.bug_url_for_bug_id(bug_id))
        self.browser.select_form(name="changeform")
        if comment_text:
            log(comment_text)
            self.browser['comment'] = comment_text
        self.browser['bug_status'] = ['RESOLVED']
        self.browser['resolution'] = ['FIXED']
        self.browser.submit()

    def reassign_bug(self, bug_id, assignee, comment_text=None):
        self.authenticate()

        log("Assigning bug %s to %s" % (bug_id, assignee))
        if self.dryrun:
            log(comment_text)
            return

        self.browser.open(self.bug_url_for_bug_id(bug_id))
        self.browser.select_form(name="changeform")
        if comment_text:
            log(comment_text)
            self.browser["comment"] = comment_text
        self.browser["assigned_to"] = assignee
        self.browser.submit()

    def reopen_bug(self, bug_id, comment_text):
        self.authenticate()

        log("Re-opening bug %s" % bug_id)
        # Bugzilla requires a comment when re-opening a bug, so we know it will
        # never be None.
        log(comment_text)
        if self.dryrun:
            return

        self.browser.open(self.bug_url_for_bug_id(bug_id))
        self.browser.select_form(name="changeform")
        bug_status = self.browser.find_control("bug_status", type="select")
        # This is a hack around the fact that ClientForm.ListControl seems to
        # have no simpler way to ask if a control has an item named "REOPENED"
        # without using exceptions for control flow.
        possible_bug_statuses = map(lambda item: item.name, bug_status.items)
        if "REOPENED" in possible_bug_statuses:
            bug_status.value = ["REOPENED"]
        # If the bug was never confirmed it will not have a "REOPENED"
        # state, but only an "UNCONFIRMED" state.
        elif "UNCONFIRMED" in possible_bug_statuses:
            bug_status.value = ["UNCONFIRMED"]
        else:
            # FIXME: This logic is slightly backwards.  We won't print this
            # message if the bug is already open with state "UNCONFIRMED".
            log("Did not reopen bug %s, it appears to already be open with status %s." % (bug_id, bug_status.value))
        self.browser['comment'] = comment_text
        self.browser.submit()
예제 #7
0
class Builder(object):
    def __init__(self, name, buildbot):
        self._name = name
        self._buildbot = buildbot
        self._builds_cache = {}
        self._revision_to_build_number = None
        self._browser = None

    def name(self):
        return self._name

    def results_url(self):
        return "%s/results/%s" % (self._buildbot.buildbot_url,
                                  self.url_encoded_name())

    # In addition to per-build results, the build.chromium.org builders also
    # keep a directory that accumulates test results over many runs.
    def accumulated_results_url(self):
        return None

    def latest_layout_test_results_url(self):
        return self.accumulated_results_url() or self.latest_cached_build(
        ).results_url()

    @memoized
    def latest_layout_test_results(self):
        return self.fetch_layout_test_results(
            self.latest_layout_test_results_url())

    def _fetch_file_from_results(self, results_url, file_name):
        # It seems this can return None if the url redirects and then returns 404.
        result = urllib2.urlopen("%s/%s" % (results_url, file_name))
        if not result:
            return None
        # urlopen returns a file-like object which sometimes works fine with str()
        # but sometimes is a addinfourl object.  In either case calling read() is correct.
        return result.read()

    def fetch_layout_test_results(self, results_url):
        # FIXME: This should cache that the result was a 404 and stop hitting the network.
        results_file = NetworkTransaction(convert_404_to_None=True).run(
            lambda: self._fetch_file_from_results(results_url,
                                                  "full_results.json"))
        return LayoutTestResults.results_from_string(results_file)

    def url_encoded_name(self):
        return urllib.quote(self._name)

    def url(self):
        return "%s/builders/%s" % (self._buildbot.buildbot_url,
                                   self.url_encoded_name())

    # This provides a single place to mock
    def _fetch_build(self, build_number):
        build_dictionary = self._buildbot._fetch_build_dictionary(
            self, build_number)
        if not build_dictionary:
            return None
        revision_string = build_dictionary['sourceStamp']['revision']
        return Build(
            self,
            build_number=int(build_dictionary['number']),
            # 'revision' may be None if a trunk build was started by the force-build button on the web page.
            revision=(int(revision_string) if revision_string else None),
            # Buildbot uses any nubmer other than 0 to mean fail.  Since we fetch with
            # filter=1, passing builds may contain no 'results' value.
            is_green=(not build_dictionary.get('results')),
        )

    def build(self, build_number):
        if not build_number:
            return None
        cached_build = self._builds_cache.get(build_number)
        if cached_build:
            return cached_build

        build = self._fetch_build(build_number)
        self._builds_cache[build_number] = build
        return build

    def latest_cached_build(self):
        revision_build_pairs = self.revision_build_pairs_with_results()
        revision_build_pairs.sort(key=lambda i: i[1])
        latest_build_number = revision_build_pairs[-1][1]
        return self.build(latest_build_number)

    def force_build(self, username="******", comments=None):
        def predicate(form):
            try:
                return form.find_control("username")
            except Exception as e:
                return False

        if not self._browser:
            self._browser = Browser()
            self._browser.set_handle_robots(
                False)  # The builder pages are excluded by robots.txt

        # ignore false positives for missing Browser methods - pylint: disable=E1102
        self._browser.open(self.url())
        self._browser.select_form(predicate=predicate)
        self._browser["username"] = username
        if comments:
            self._browser["comments"] = comments
        return self._browser.submit()

    file_name_regexp = re.compile(
        r"r(?P<revision>\d+) \((?P<build_number>\d+)\)")

    def _revision_and_build_for_filename(self, filename):
        # Example: "r47483 (1)/" or "r47483 (1).zip"
        match = self.file_name_regexp.match(filename)
        if not match:
            return None
        return (int(match.group("revision")), int(match.group("build_number")))

    def _fetch_revision_to_build_map(self):
        # All _fetch requests go through _buildbot for easier mocking
        # FIXME: This should use NetworkTransaction's 404 handling instead.
        try:
            # FIXME: This method is horribly slow due to the huge network load.
            # FIXME: This is a poor way to do revision -> build mapping.
            # Better would be to ask buildbot through some sort of API.
            print("Loading revision/build list from %s." % self.results_url())
            print("This may take a while...")
            result_files = self._buildbot._fetch_twisted_directory_listing(
                self.results_url())
        except urllib2.HTTPError as error:
            if error.code != 404:
                raise
            _log.debug("Revision/build list failed to load.")
            result_files = []
        return dict(
            self._file_info_list_to_revision_to_build_list(result_files))

    def _file_info_list_to_revision_to_build_list(self, file_info_list):
        # This assumes there was only one build per revision, which is false but we don't care for now.
        revisions_and_builds = []
        for file_info in file_info_list:
            revision_and_build = self._revision_and_build_for_filename(
                file_info["filename"])
            if revision_and_build:
                revisions_and_builds.append(revision_and_build)
        return revisions_and_builds

    def _revision_to_build_map(self):
        if not self._revision_to_build_number:
            self._revision_to_build_number = self._fetch_revision_to_build_map(
            )
        return self._revision_to_build_number

    def revision_build_pairs_with_results(self):
        return self._revision_to_build_map().items()

    # This assumes there can be only one build per revision, which is false, but we don't care for now.
    def build_for_revision(self, revision, allow_failed_lookups=False):
        # NOTE: This lookup will fail if that exact revision was never built.
        build_number = self._revision_to_build_map().get(int(revision))
        if not build_number:
            return None
        build = self.build(build_number)
        if not build and allow_failed_lookups:
            # Builds for old revisions with fail to lookup via buildbot's json api.
            build = Build(
                self,
                build_number=build_number,
                revision=revision,
                is_green=False,
            )
        return build

    def find_regression_window(self, red_build, look_back_limit=30):
        if not red_build or red_build.is_green():
            return RegressionWindow(None, None)
        common_failures = None
        current_build = red_build
        build_after_current_build = None
        look_back_count = 0
        while current_build:
            if current_build.is_green():
                # current_build can't possibly have any failures in common
                # with red_build because it's green.
                break
            results = current_build.layout_test_results()
            # We treat a lack of results as if all the test failed.
            # This occurs, for example, when we can't compile at all.
            if results:
                failures = set(results.failing_tests())
                if common_failures == None:
                    common_failures = failures
                else:
                    common_failures = common_failures.intersection(failures)
                    if not common_failures:
                        # current_build doesn't have any failures in common with
                        # the red build we're worried about.  We assume that any
                        # failures in current_build were due to flakiness.
                        break
            look_back_count += 1
            if look_back_count > look_back_limit:
                return RegressionWindow(None,
                                        current_build,
                                        failing_tests=common_failures)
            build_after_current_build = current_build
            current_build = current_build.previous_build()
        # We must iterate at least once because red_build is red.
        assert (build_after_current_build)
        # Current build must either be green or have no failures in common
        # with red build, so we've found our failure transition.
        return RegressionWindow(current_build,
                                build_after_current_build,
                                failing_tests=common_failures)

    def find_blameworthy_regression_window(self,
                                           red_build_number,
                                           look_back_limit=30,
                                           avoid_flakey_tests=True):
        red_build = self.build(red_build_number)
        regression_window = self.find_regression_window(
            red_build, look_back_limit)
        if not regression_window.build_before_failure():
            return None  # We ran off the limit of our search
        # If avoid_flakey_tests, require at least 2 bad builds before we
        # suspect a real failure transition.
        if avoid_flakey_tests and regression_window.failing_build(
        ) == red_build:
            return None
        return regression_window
예제 #8
0
class StatusServer:
    default_host = "webkit-commit-queue.appspot.com"

    def __init__(self, host=default_host):
        self.set_host(host)
        self.browser = Browser()

    def set_host(self, host):
        self.host = host
        self.url = "http://%s" % self.host

    def results_url_for_status(self, status_id):
        return "%s/results/%s" % (self.url, status_id)

    def _add_patch(self, patch):
        if not patch:
            return
        if patch.bug_id():
            self.browser["bug_id"] = unicode(patch.bug_id())
        if patch.id():
            self.browser["patch_id"] = unicode(patch.id())

    def _add_results_file(self, results_file):
        if not results_file:
            return
        self.browser.add_file(results_file, "text/plain", "results.txt",
                              'results_file')

    def _post_status_to_server(self, queue_name, status, patch, results_file):
        if results_file:
            # We might need to re-wind the file if we've already tried to post it.
            results_file.seek(0)

        update_status_url = "%s/update-status" % self.url
        self.browser.open(update_status_url)
        self.browser.select_form(name="update_status")
        self.browser["queue_name"] = queue_name
        self._add_patch(patch)
        self.browser["status"] = status
        self._add_results_file(results_file)
        return self.browser.submit().read(
        )  # This is the id of the newly created status object.

    def _post_svn_revision_to_server(self, svn_revision_number, broken_bot):
        update_svn_revision_url = "%s/update-svn-revision" % self.url
        self.browser.open(update_svn_revision_url)
        self.browser.select_form(name="update_svn_revision")
        self.browser["number"] = unicode(svn_revision_number)
        self.browser["broken_bot"] = broken_bot
        return self.browser.submit().read()

    def update_status(self, queue_name, status, patch=None, results_file=None):
        log(status)
        return NetworkTransaction().run(lambda: self._post_status_to_server(
            queue_name, status, patch, results_file))

    def update_svn_revision(self, svn_revision_number, broken_bot):
        log("SVN revision: %s broke %s" % (svn_revision_number, broken_bot))
        return NetworkTransaction().run(
            lambda: self._post_svn_revision_to_server(svn_revision_number,
                                                      broken_bot))

    def _fetch_url(self, url):
        try:
            return urllib2.urlopen(url).read()
        except urllib2.HTTPError, e:
            if e.code == 404:
                return None
            raise e