class TestResultsUploader:
def __init__(self, host):
self._host = host
self._browser = Browser()
def _upload_files(self, attrs, file_objs):
self._browser.open("http://%s/testfile/uploadform" % self._host)
self._browser.select_form("test_result_upload")
for (name, data) in attrs:
self._browser[name] = str(data)
for (filename, handle) in file_objs:
self._browser.add_file(handle, get_mime_type(filename), filename,
"file")
self._browser.submit()
def upload(self, params, files, timeout_seconds):
orig_timeout = socket.getdefaulttimeout()
file_objs = []
try:
file_objs = [(filename, open(path, "rb"))
for (filename, path) in files]
socket.setdefaulttimeout(timeout_seconds)
NetworkTransaction(timeout_seconds=timeout_seconds).run(
lambda: self._upload_files(params, file_objs))
finally:
socket.setdefaulttimeout(orig_timeout)
for (filename, handle) in file_objs:
handle.close()
class TestResultsUploader:
def __init__(self, host):
self._host = host
self._browser = Browser()
def _upload_files(self, attrs, file_objs):
self._browser.open("http://%s/testfile/uploadform" % self._host)
self._browser.select_form("test_result_upload")
for (name, data) in attrs:
self._browser[name] = str(data)
for (filename, handle) in file_objs:
self._browser.add_file(handle, get_mime_type(filename), filename,
"file")
self._browser.submit()
def upload(self, params, files, timeout_seconds):
orig_timeout = socket.getdefaulttimeout()
file_objs = []
try:
file_objs = [(filename, open(path, "rb")) for (filename, path)
in files]
socket.setdefaulttimeout(timeout_seconds)
NetworkTransaction(timeout_seconds=timeout_seconds).run(
lambda: self._upload_files(params, file_objs))
finally:
socket.setdefaulttimeout(orig_timeout)
for (filename, handle) in file_objs:
handle.close()
class Bugzilla(object):
def __init__(self, dryrun=False, committers=committers.CommitterList()):
self.dryrun = dryrun
self.authenticated = False
self.queries = BugzillaQueries(self)
self.committers = committers
# FIXME: We should use some sort of Browser mock object when in dryrun
# mode (to prevent any mistakes).
self.browser = Browser()
# Ignore bugs.webkit.org/robots.txt until we fix it to allow this
# script.
self.browser.set_handle_robots(False)
# FIXME: Much of this should go into some sort of config module:
bug_server_host = "bugs.webkit.org"
bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
bug_server_url = "https://%s/" % bug_server_host
def bug_url_for_bug_id(self, bug_id, xml=False):
if not bug_id:
return None
content_type = "&ctype=xml" if xml else ""
return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url,
bug_id,
content_type)
def short_bug_url_for_bug_id(self, bug_id):
if not bug_id:
return None
return "http://webkit.org/b/%s" % bug_id
def attachment_url_for_id(self, attachment_id, action="view"):
if not attachment_id:
return None
action_param = ""
if action and action != "view":
action_param = "&action=%s" % action
return "%sattachment.cgi?id=%s%s" % (self.bug_server_url,
attachment_id,
action_param)
def _parse_attachment_flag(self,
element,
flag_name,
attachment,
result_key):
flag = element.find('flag', attrs={'name': flag_name})
if flag:
attachment[flag_name] = flag['status']
if flag['status'] == '+':
attachment[result_key] = flag['setter']
def _parse_attachment_element(self, element, bug_id):
attachment = {}
attachment['bug_id'] = bug_id
attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
attachment['id'] = int(element.find('attachid').string)
# FIXME: No need to parse out the url here.
attachment['url'] = self.attachment_url_for_id(attachment['id'])
attachment['name'] = unicode(element.find('desc').string)
attachment['attacher_email'] = str(element.find('attacher').string)
attachment['type'] = str(element.find('type').string)
self._parse_attachment_flag(
element, 'review', attachment, 'reviewer_email')
self._parse_attachment_flag(
element, 'commit-queue', attachment, 'committer_email')
return attachment
def _parse_bug_page(self, page):
soup = BeautifulSoup(page)
bug = {}
bug["id"] = int(soup.find("bug_id").string)
bug["title"] = unicode(soup.find("short_desc").string)
bug["reporter_email"] = str(soup.find("reporter").string)
bug["assigned_to_email"] = str(soup.find("assigned_to").string)
bug["cc_emails"] = [str(element.string)
for element in soup.findAll('cc')]
bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')]
return bug
# Makes testing fetch_*_from_bug() possible until we have a better
# BugzillaNetwork abstration.
def _fetch_bug_page(self, bug_id):
bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
log("Fetching: %s" % bug_url)
return self.browser.open(bug_url)
def fetch_bug_dictionary(self, bug_id):
try:
return self._parse_bug_page(self._fetch_bug_page(bug_id))
except:
self.authenticate()
return self._parse_bug_page(self._fetch_bug_page(bug_id))
# FIXME: A BugzillaCache object should provide all these fetch_ methods.
def fetch_bug(self, bug_id):
return Bug(self.fetch_bug_dictionary(bug_id), self)
def _parse_bug_id_from_attachment_page(self, page):
# The "Up" relation happens to point to the bug.
up_link = BeautifulSoup(page).find('link', rel='Up')
if not up_link:
# This attachment does not exist (or you don't have permissions to
# view it).
return None
match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href'])
return int(match.group('bug_id'))
def bug_id_for_attachment_id(self, attachment_id):
self.authenticate()
attachment_url = self.attachment_url_for_id(attachment_id, 'edit')
log("Fetching: %s" % attachment_url)
page = self.browser.open(attachment_url)
return self._parse_bug_id_from_attachment_page(page)
# FIXME: This should just return Attachment(id), which should be able to
# lazily fetch needed data.
def fetch_attachment(self, attachment_id):
# We could grab all the attachment details off of the attachment edit
# page but we already have working code to do so off of the bugs page,
# so re-use that.
bug_id = self.bug_id_for_attachment_id(attachment_id)
if not bug_id:
return None
attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True)
for attachment in attachments:
if attachment.id() == int(attachment_id):
return attachment
return None # This should never be hit.
def authenticate(self):
if self.authenticated:
return
if self.dryrun:
log("Skipping log in for dry run...")
self.authenticated = True
return
attempts = 0
while not self.authenticated:
attempts += 1
(username, password) = Credentials(
self.bug_server_host, git_prefix="bugzilla").read_credentials()
log("Logging in as %s..." % username)
self.browser.open(self.bug_server_url +
"index.cgi?GoAheadAndLogIn=1")
self.browser.select_form(name="login")
self.browser['Bugzilla_login'] = username
self.browser['Bugzilla_password'] = password
response = self.browser.submit()
match = re.search("<title>(.+?)</title>", response.read())
# If the resulting page has a title, and it contains the word
# "invalid" assume it's the login failure page.
if match and re.search("Invalid", match.group(1), re.IGNORECASE):
errorMessage = "Bugzilla login failed: %s" % match.group(1)
# raise an exception only if this was the last attempt
if attempts < 5:
log(errorMessage)
else:
raise Exception(errorMessage)
else:
self.authenticated = True
def _fill_attachment_form(self,
description,
patch_file_object,
comment_text=None,
mark_for_review=False,
mark_for_commit_queue=False,
mark_for_landing=False, bug_id=None):
self.browser['description'] = description
self.browser['ispatch'] = ("1",)
self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
if mark_for_landing:
self.browser['flag_type-3'] = ('+',)
elif mark_for_commit_queue:
self.browser['flag_type-3'] = ('?',)
else:
self.browser['flag_type-3'] = ('X',)
if bug_id:
patch_name = "bug-%s-%s.patch" % (bug_id, timestamp())
else:
patch_name ="%s.patch" % timestamp()
self.browser.add_file(patch_file_object,
"text/plain",
patch_name,
'data')
def add_patch_to_bug(self,
bug_id,
patch_file_object,
description,
comment_text=None,
mark_for_review=False,
mark_for_commit_queue=False,
mark_for_landing=False):
self.authenticate()
log('Adding patch "%s" to %sshow_bug.cgi?id=%s' % (description,
self.bug_server_url,
bug_id))
if self.dryrun:
log(comment_text)
return
self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % (
self.bug_server_url, bug_id))
self.browser.select_form(name="entryform")
self._fill_attachment_form(description,
patch_file_object,
mark_for_review=mark_for_review,
mark_for_commit_queue=mark_for_commit_queue,
mark_for_landing=mark_for_landing,
bug_id=bug_id)
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser.submit()
def _check_create_bug_response(self, response_html):
match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>",
response_html)
if match:
return match.group('bug_id')
match = re.search(
'<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">',
response_html,
re.DOTALL)
error_message = "FAIL"
if match:
text_lines = BeautifulSoup(
match.group('error_message')).findAll(text=True)
error_message = "\n" + '\n'.join(
[" " + line.strip()
for line in text_lines if line.strip()])
raise Exception("Bug not created: %s" % error_message)
def create_bug(self,
bug_title,
bug_description,
component=None,
patch_file_object=None,
patch_description=None,
cc=None,
blocked=None,
mark_for_review=False,
mark_for_commit_queue=False):
self.authenticate()
log('Creating bug with title "%s"' % bug_title)
if self.dryrun:
log(bug_description)
return
self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit")
self.browser.select_form(name="Create")
component_items = self.browser.find_control('component').items
component_names = map(lambda item: item.name, component_items)
if not component:
component = "New Bugs"
if component not in component_names:
component = User.prompt_with_list("Please pick a component:", component_names)
self.browser["component"] = [component]
if cc:
self.browser["cc"] = cc
if blocked:
self.browser["blocked"] = str(blocked)
self.browser["short_desc"] = bug_title
self.browser["comment"] = bug_description
if patch_file_object:
self._fill_attachment_form(
patch_description,
patch_file_object,
mark_for_review=mark_for_review,
mark_for_commit_queue=mark_for_commit_queue)
response = self.browser.submit()
bug_id = self._check_create_bug_response(response.read())
log("Bug %s created." % bug_id)
log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id))
return bug_id
def _find_select_element_for_flag(self, flag_name):
# FIXME: This will break if we ever re-order attachment flags
if flag_name == "review":
return self.browser.find_control(type='select', nr=0)
if flag_name == "commit-queue":
return self.browser.find_control(type='select', nr=1)
raise Exception("Don't know how to find flag named \"%s\"" % flag_name)
def clear_attachment_flags(self,
attachment_id,
additional_comment_text=None):
self.authenticate()
comment_text = "Clearing flags on attachment: %s" % attachment_id
if additional_comment_text:
comment_text += "\n\n%s" % additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.set_value(comment_text, name='comment', nr=0)
self._find_select_element_for_flag('review').value = ("X",)
self._find_select_element_for_flag('commit-queue').value = ("X",)
self.browser.submit()
def set_flag_on_attachment(self,
attachment_id,
flag_name,
flag_value,
comment_text,
additional_comment_text):
# FIXME: We need a way to test this function on a live bugzilla
# instance.
self.authenticate()
if additional_comment_text:
comment_text += "\n\n%s" % additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.set_value(comment_text, name='comment', nr=0)
self._find_select_element_for_flag(flag_name).value = (flag_value,)
self.browser.submit()
# FIXME: All of these bug editing methods have a ridiculous amount of
# copy/paste code.
def obsolete_attachment(self, attachment_id, comment_text=None):
self.authenticate()
log("Obsoleting attachment: %s" % attachment_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.find_control('isobsolete').items[0].selected = True
# Also clear any review flag (to remove it from review/commit queues)
self._find_select_element_for_flag('review').value = ("X",)
self._find_select_element_for_flag('commit-queue').value = ("X",)
if comment_text:
log(comment_text)
# Bugzilla has two textareas named 'comment', one is somehow
# hidden. We want the first.
self.browser.set_value(comment_text, name='comment', nr=0)
self.browser.submit()
def add_cc_to_bug(self, bug_id, email_address_list):
self.authenticate()
log("Adding %s to the CC list for bug %s" % (email_address_list,
bug_id))
if self.dryrun:
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser["newcc"] = ", ".join(email_address_list)
self.browser.submit()
def post_comment_to_bug(self, bug_id, comment_text, cc=None):
self.authenticate()
log("Adding comment to bug %s" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser["comment"] = comment_text
if cc:
self.browser["newcc"] = ", ".join(cc)
self.browser.submit()
def close_bug_as_fixed(self, bug_id, comment_text=None):
self.authenticate()
log("Closing bug %s as fixed" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser['bug_status'] = ['RESOLVED']
self.browser['resolution'] = ['FIXED']
self.browser.submit()
def reassign_bug(self, bug_id, assignee, comment_text=None):
self.authenticate()
log("Assigning bug %s to %s" % (bug_id, assignee))
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser["comment"] = comment_text
self.browser["assigned_to"] = assignee
self.browser.submit()
def reopen_bug(self, bug_id, comment_text):
self.authenticate()
log("Re-opening bug %s" % bug_id)
# Bugzilla requires a comment when re-opening a bug, so we know it will
# never be None.
log(comment_text)
if self.dryrun:
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
bug_status = self.browser.find_control("bug_status", type="select")
# This is a hack around the fact that ClientForm.ListControl seems to
# have no simpler way to ask if a control has an item named "REOPENED"
# without using exceptions for control flow.
possible_bug_statuses = map(lambda item: item.name, bug_status.items)
if "REOPENED" in possible_bug_statuses:
bug_status.value = ["REOPENED"]
else:
log("Did not reopen bug %s. " +
"It appears to already be open with status %s." % (
bug_id, bug_status.value))
self.browser['comment'] = comment_text
self.browser.submit()
class Bugzilla(object):
def __init__(self, dryrun=False, committers=committers.CommitterList()):
self.dryrun = dryrun
self.authenticated = False
self.queries = BugzillaQueries(self)
self.committers = committers
self.cached_quips = []
# FIXME: We should use some sort of Browser mock object when in dryrun
# mode (to prevent any mistakes).
self.browser = Browser()
# Ignore bugs.webkit.org/robots.txt until we fix it to allow this
# script.
self.browser.set_handle_robots(False)
# FIXME: Much of this should go into some sort of config module:
bug_server_host = "bugs.webkit.org"
bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
bug_server_url = "https://%s/" % bug_server_host
def quips(self):
# We only fetch and parse the list of quips once per instantiation
# so that we do not burden bugs.webkit.org.
if not self.cached_quips and not self.dryrun:
self.cached_quips = self.queries.fetch_quips()
return self.cached_quips
def bug_url_for_bug_id(self, bug_id, xml=False):
if not bug_id:
return None
content_type = "&ctype=xml" if xml else ""
return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type)
def short_bug_url_for_bug_id(self, bug_id):
if not bug_id:
return None
return "http://webkit.org/b/%s" % bug_id
def add_attachment_url(self, bug_id):
return "%sattachment.cgi?action=enter&bugid=%s" % (self.bug_server_url, bug_id)
def attachment_url_for_id(self, attachment_id, action="view"):
if not attachment_id:
return None
action_param = ""
if action and action != "view":
action_param = "&action=%s" % action
return "%sattachment.cgi?id=%s%s" % (self.bug_server_url,
attachment_id,
action_param)
def _parse_attachment_flag(self,
element,
flag_name,
attachment,
result_key):
flag = element.find('flag', attrs={'name': flag_name})
if flag:
attachment[flag_name] = flag['status']
if flag['status'] == '+':
attachment[result_key] = flag['setter']
# Sadly show_bug.cgi?ctype=xml does not expose the flag modification date.
def _string_contents(self, soup):
# WebKit's bugzilla instance uses UTF-8.
# BeautifulSoup always returns Unicode strings, however
# the .string method returns a (unicode) NavigableString.
# NavigableString can confuse other parts of the code, so we
# convert from NavigableString to a real unicode() object using unicode().
return unicode(soup.string)
# Example: 2010-01-20 14:31 PST
# FIXME: Some bugzilla dates seem to have seconds in them?
# Python does not support timezones out of the box.
# Assume that bugzilla always uses PST (which is true for bugs.webkit.org)
_bugzilla_date_format = "%Y-%m-%d %H:%M"
@classmethod
def _parse_date(cls, date_string):
(date, time, time_zone) = date_string.split(" ")
# Ignore the timezone because python doesn't understand timezones out of the box.
date_string = "%s %s" % (date, time)
return datetime.strptime(date_string, cls._bugzilla_date_format)
def _date_contents(self, soup):
return self._parse_date(self._string_contents(soup))
def _parse_attachment_element(self, element, bug_id):
attachment = {}
attachment['bug_id'] = bug_id
attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
attachment['id'] = int(element.find('attachid').string)
# FIXME: No need to parse out the url here.
attachment['url'] = self.attachment_url_for_id(attachment['id'])
attachment["attach_date"] = self._date_contents(element.find("date"))
attachment['name'] = self._string_contents(element.find('desc'))
attachment['attacher_email'] = self._string_contents(element.find('attacher'))
attachment['type'] = self._string_contents(element.find('type'))
self._parse_attachment_flag(
element, 'review', attachment, 'reviewer_email')
self._parse_attachment_flag(
element, 'commit-queue', attachment, 'committer_email')
return attachment
def _parse_bugs_from_xml(self, page):
soup = BeautifulSoup(page)
# Without the unicode() call, BeautifulSoup occasionally complains of being
# passed None for no apparent reason.
return [Bug(self._parse_bug_dictionary_from_xml(unicode(bug_xml)), self) for bug_xml in soup('bug')]
def _parse_bug_dictionary_from_xml(self, page):
soup = BeautifulSoup(page)
bug = {}
bug["id"] = int(soup.find("bug_id").string)
bug["title"] = self._string_contents(soup.find("short_desc"))
bug["bug_status"] = self._string_contents(soup.find("bug_status"))
dup_id = soup.find("dup_id")
if dup_id:
bug["dup_id"] = self._string_contents(dup_id)
bug["reporter_email"] = self._string_contents(soup.find("reporter"))
bug["assigned_to_email"] = self._string_contents(soup.find("assigned_to"))
bug["cc_emails"] = [self._string_contents(element) for element in soup.findAll('cc')]
bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')]
return bug
# Makes testing fetch_*_from_bug() possible until we have a better
# BugzillaNetwork abstration.
def _fetch_bug_page(self, bug_id):
bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
log("Fetching: %s" % bug_url)
return self.browser.open(bug_url)
def fetch_bug_dictionary(self, bug_id):
try:
return self._parse_bug_dictionary_from_xml(self._fetch_bug_page(bug_id))
except KeyboardInterrupt:
raise
except:
self.authenticate()
return self._parse_bug_dictionary_from_xml(self._fetch_bug_page(bug_id))
# FIXME: A BugzillaCache object should provide all these fetch_ methods.
def fetch_bug(self, bug_id):
return Bug(self.fetch_bug_dictionary(bug_id), self)
def fetch_attachment_contents(self, attachment_id):
attachment_url = self.attachment_url_for_id(attachment_id)
# We need to authenticate to download patches from security bugs.
self.authenticate()
return self.browser.open(attachment_url).read()
def _parse_bug_id_from_attachment_page(self, page):
# The "Up" relation happens to point to the bug.
up_link = BeautifulSoup(page).find('link', rel='Up')
if not up_link:
# This attachment does not exist (or you don't have permissions to
# view it).
return None
match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href'])
return int(match.group('bug_id'))
def bug_id_for_attachment_id(self, attachment_id):
self.authenticate()
attachment_url = self.attachment_url_for_id(attachment_id, 'edit')
log("Fetching: %s" % attachment_url)
page = self.browser.open(attachment_url)
return self._parse_bug_id_from_attachment_page(page)
# FIXME: This should just return Attachment(id), which should be able to
# lazily fetch needed data.
def fetch_attachment(self, attachment_id):
# We could grab all the attachment details off of the attachment edit
# page but we already have working code to do so off of the bugs page,
# so re-use that.
bug_id = self.bug_id_for_attachment_id(attachment_id)
if not bug_id:
return None
attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True)
for attachment in attachments:
if attachment.id() == int(attachment_id):
return attachment
return None # This should never be hit.
def authenticate(self):
if self.authenticated:
return
if self.dryrun:
log("Skipping log in for dry run...")
self.authenticated = True
return
credentials = Credentials(self.bug_server_host, git_prefix="bugzilla")
attempts = 0
while not self.authenticated:
attempts += 1
username, password = credentials.read_credentials()
log("Logging in as %s..." % username)
self.browser.open(self.bug_server_url +
"index.cgi?GoAheadAndLogIn=1")
self.browser.select_form(name="login")
self.browser['Bugzilla_login'] = username
self.browser['Bugzilla_password'] = password
response = self.browser.submit()
match = re.search("<title>(.+?)</title>", response.read())
# If the resulting page has a title, and it contains the word
# "invalid" assume it's the login failure page.
if match and re.search("Invalid", match.group(1), re.IGNORECASE):
errorMessage = "Bugzilla login failed: %s" % match.group(1)
# raise an exception only if this was the last attempt
if attempts < 5:
log(errorMessage)
else:
raise Exception(errorMessage)
else:
self.authenticated = True
self.username = username
def _commit_queue_flag(self, mark_for_landing, mark_for_commit_queue):
if mark_for_landing:
return '+'
elif mark_for_commit_queue:
return '?'
return 'X'
# FIXME: mark_for_commit_queue and mark_for_landing should be joined into a single commit_flag argument.
def _fill_attachment_form(self,
description,
file_object,
mark_for_review=False,
mark_for_commit_queue=False,
mark_for_landing=False,
is_patch=False,
filename=None,
mimetype=None):
self.browser['description'] = description
if is_patch:
self.browser['ispatch'] = ("1",)
# FIXME: Should this use self._find_select_element_for_flag?
self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
self.browser['flag_type-3'] = (self._commit_queue_flag(mark_for_landing, mark_for_commit_queue),)
filename = filename or "%s.patch" % timestamp()
mimetype = mimetype or "text/plain"
self.browser.add_file(file_object, mimetype, filename, 'data')
def _file_object_for_upload(self, file_or_string):
if hasattr(file_or_string, 'read'):
return file_or_string
# Only if file_or_string is not already encoded do we want to encode it.
if isinstance(file_or_string, unicode):
file_or_string = file_or_string.encode('utf-8')
return StringIO.StringIO(file_or_string)
# timestamp argument is just for unittests.
def _filename_for_upload(self, file_object, bug_id, extension="txt", timestamp=timestamp):
if hasattr(file_object, "name"):
return file_object.name
return "bug-%s-%s.%s" % (bug_id, timestamp(), extension)
def add_attachment_to_bug(self,
bug_id,
file_or_string,
description,
filename=None,
comment_text=None):
self.authenticate()
log('Adding attachment "%s" to %s' % (description, self.bug_url_for_bug_id(bug_id)))
if self.dryrun:
log(comment_text)
return
self.browser.open(self.add_attachment_url(bug_id))
self.browser.select_form(name="entryform")
file_object = self._file_object_for_upload(file_or_string)
filename = filename or self._filename_for_upload(file_object, bug_id)
self._fill_attachment_form(description, file_object, filename=filename)
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser.submit()
# FIXME: The arguments to this function should be simplified and then
# this should be merged into add_attachment_to_bug
def add_patch_to_bug(self,
bug_id,
file_or_string,
description,
comment_text=None,
mark_for_review=False,
mark_for_commit_queue=False,
mark_for_landing=False):
self.authenticate()
log('Adding patch "%s" to %s' % (description, self.bug_url_for_bug_id(bug_id)))
if self.dryrun:
log(comment_text)
return
self.browser.open(self.add_attachment_url(bug_id))
self.browser.select_form(name="entryform")
file_object = self._file_object_for_upload(file_or_string)
filename = self._filename_for_upload(file_object, bug_id, extension="patch")
self._fill_attachment_form(description,
file_object,
mark_for_review=mark_for_review,
mark_for_commit_queue=mark_for_commit_queue,
mark_for_landing=mark_for_landing,
is_patch=True,
filename=filename)
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser.submit()
# FIXME: There has to be a more concise way to write this method.
def _check_create_bug_response(self, response_html):
match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>",
response_html)
if match:
return match.group('bug_id')
match = re.search(
'<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">',
response_html,
re.DOTALL)
error_message = "FAIL"
if match:
text_lines = BeautifulSoup(
match.group('error_message')).findAll(text=True)
error_message = "\n" + '\n'.join(
[" " + line.strip()
for line in text_lines if line.strip()])
raise Exception("Bug not created: %s" % error_message)
def create_bug(self,
bug_title,
bug_description,
component=None,
diff=None,
patch_description=None,
cc=None,
blocked=None,
assignee=None,
mark_for_review=False,
mark_for_commit_queue=False):
self.authenticate()
log('Creating bug with title "%s"' % bug_title)
if self.dryrun:
log(bug_description)
# FIXME: This will make some paths fail, as they assume this returns an id.
return
self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit")
self.browser.select_form(name="Create")
component_items = self.browser.find_control('component').items
component_names = map(lambda item: item.name, component_items)
if not component:
component = "New Bugs"
if component not in component_names:
component = User.prompt_with_list("Please pick a component:", component_names)
self.browser["component"] = [component]
if cc:
self.browser["cc"] = cc
if blocked:
self.browser["blocked"] = unicode(blocked)
if not assignee:
assignee = self.username
if assignee and not self.browser.find_control("assigned_to").disabled:
self.browser["assigned_to"] = assignee
self.browser["short_desc"] = bug_title
self.browser["comment"] = bug_description
if diff:
# _fill_attachment_form expects a file-like object
# Patch files are already binary, so no encoding needed.
assert(isinstance(diff, str))
patch_file_object = StringIO.StringIO(diff)
self._fill_attachment_form(
patch_description,
patch_file_object,
mark_for_review=mark_for_review,
mark_for_commit_queue=mark_for_commit_queue,
is_patch=True)
response = self.browser.submit()
bug_id = self._check_create_bug_response(response.read())
log("Bug %s created." % bug_id)
log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id))
return bug_id
def _find_select_element_for_flag(self, flag_name):
# FIXME: This will break if we ever re-order attachment flags
if flag_name == "review":
return self.browser.find_control(type='select', nr=0)
elif flag_name == "commit-queue":
return self.browser.find_control(type='select', nr=1)
raise Exception("Don't know how to find flag named \"%s\"" % flag_name)
def clear_attachment_flags(self,
attachment_id,
additional_comment_text=None):
self.authenticate()
comment_text = "Clearing flags on attachment: %s" % attachment_id
if additional_comment_text:
comment_text += "\n\n%s" % additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.set_value(comment_text, name='comment', nr=0)
self._find_select_element_for_flag('review').value = ("X",)
self._find_select_element_for_flag('commit-queue').value = ("X",)
self.browser.submit()
def set_flag_on_attachment(self,
attachment_id,
flag_name,
flag_value,
comment_text=None,
additional_comment_text=None):
# FIXME: We need a way to test this function on a live bugzilla
# instance.
self.authenticate()
if additional_comment_text:
comment_text += "\n\n%s" % additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
if comment_text:
self.browser.set_value(comment_text, name='comment', nr=0)
self._find_select_element_for_flag(flag_name).value = (flag_value,)
self.browser.submit()
# FIXME: All of these bug editing methods have a ridiculous amount of
# copy/paste code.
def obsolete_attachment(self, attachment_id, comment_text=None):
self.authenticate()
log("Obsoleting attachment: %s" % attachment_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.find_control('isobsolete').items[0].selected = True
# Also clear any review flag (to remove it from review/commit queues)
self._find_select_element_for_flag('review').value = ("X",)
self._find_select_element_for_flag('commit-queue').value = ("X",)
if comment_text:
log(comment_text)
# Bugzilla has two textareas named 'comment', one is somehow
# hidden. We want the first.
self.browser.set_value(comment_text, name='comment', nr=0)
self.browser.submit()
def add_cc_to_bug(self, bug_id, email_address_list):
self.authenticate()
log("Adding %s to the CC list for bug %s" % (email_address_list,
bug_id))
if self.dryrun:
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser["newcc"] = ", ".join(email_address_list)
self.browser.submit()
def post_comment_to_bug(self, bug_id, comment_text, cc=None):
self.authenticate()
log("Adding comment to bug %s" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser["comment"] = comment_text
if cc:
self.browser["newcc"] = ", ".join(cc)
self.browser.submit()
def close_bug_as_fixed(self, bug_id, comment_text=None):
self.authenticate()
log("Closing bug %s as fixed" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
self.browser['comment'] = comment_text
self.browser['bug_status'] = ['RESOLVED']
self.browser['resolution'] = ['FIXED']
self.browser.submit()
def reassign_bug(self, bug_id, assignee, comment_text=None):
self.authenticate()
log("Assigning bug %s to %s" % (bug_id, assignee))
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser["comment"] = comment_text
self.browser["assigned_to"] = assignee
self.browser.submit()
def reopen_bug(self, bug_id, comment_text):
self.authenticate()
log("Re-opening bug %s" % bug_id)
# Bugzilla requires a comment when re-opening a bug, so we know it will
# never be None.
log(comment_text)
if self.dryrun:
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
bug_status = self.browser.find_control("bug_status", type="select")
# This is a hack around the fact that ClientForm.ListControl seems to
# have no simpler way to ask if a control has an item named "REOPENED"
# without using exceptions for control flow.
possible_bug_statuses = map(lambda item: item.name, bug_status.items)
if "REOPENED" in possible_bug_statuses:
bug_status.value = ["REOPENED"]
# If the bug was never confirmed it will not have a "REOPENED"
# state, but only an "UNCONFIRMED" state.
elif "UNCONFIRMED" in possible_bug_statuses:
bug_status.value = ["UNCONFIRMED"]
else:
# FIXME: This logic is slightly backwards. We won't print this
# message if the bug is already open with state "UNCONFIRMED".
log("Did not reopen bug %s, it appears to already be open with status %s." % (bug_id, bug_status.value))
self.browser['comment'] = comment_text
self.browser.submit()
class StatusServer:
default_host = "webkit-commit-queue.appspot.com"
def __init__(self, host=default_host):
self.set_host(host)
self.browser = Browser()
def set_host(self, host):
self.host = host
self.url = "http://%s" % self.host
def results_url_for_status(self, status_id):
return "%s/results/%s" % (self.url, status_id)
def _add_patch(self, patch):
if not patch:
return
if patch.bug_id():
self.browser["bug_id"] = unicode(patch.bug_id())
if patch.id():
self.browser["patch_id"] = unicode(patch.id())
def _add_results_file(self, results_file):
if not results_file:
return
self.browser.add_file(results_file, "text/plain", "results.txt", "results_file")
def _post_status_to_server(self, queue_name, status, patch, results_file):
if results_file:
# We might need to re-wind the file if we've already tried to post it.
results_file.seek(0)
update_status_url = "%s/update-status" % self.url
self.browser.open(update_status_url)
self.browser.select_form(name="update_status")
self.browser["queue_name"] = queue_name
self._add_patch(patch)
self.browser["status"] = status
self._add_results_file(results_file)
return self.browser.submit().read() # This is the id of the newly created status object.
def _post_svn_revision_to_server(self, svn_revision_number, broken_bot):
update_svn_revision_url = "%s/update-svn-revision" % self.url
self.browser.open(update_svn_revision_url)
self.browser.select_form(name="update_svn_revision")
self.browser["number"] = unicode(svn_revision_number)
self.browser["broken_bot"] = broken_bot
return self.browser.submit().read()
def _post_work_items_to_server(self, queue_name, work_items):
update_work_items_url = "%s/update-work-items" % self.url
self.browser.open(update_work_items_url)
self.browser.select_form(name="update_work_items")
self.browser["queue_name"] = queue_name
work_items = map(unicode, work_items) # .join expects strings
self.browser["work_items"] = " ".join(work_items)
return self.browser.submit().read()
def update_work_items(self, queue_name, work_items):
_log.debug("Recording work items: %s for %s" % (work_items, queue_name))
return NetworkTransaction().run(lambda: self._post_work_items_to_server(queue_name, work_items))
def update_status(self, queue_name, status, patch=None, results_file=None):
log(status)
return NetworkTransaction().run(lambda: self._post_status_to_server(queue_name, status, patch, results_file))
def update_svn_revision(self, svn_revision_number, broken_bot):
log("SVN revision: %s broke %s" % (svn_revision_number, broken_bot))
return NetworkTransaction().run(lambda: self._post_svn_revision_to_server(svn_revision_number, broken_bot))
def _fetch_url(self, url):
try:
return urllib2.urlopen(url).read()
except urllib2.HTTPError, e:
if e.code == 404:
return None
raise e
class Bugzilla(object):
def __init__(self, dryrun=False, committers=committers.CommitterList()):
self.dryrun = dryrun
self.authenticated = False
self.queries = BugzillaQueries(self)
self.committers = committers
self.cached_quips = []
# FIXME: We should use some sort of Browser mock object when in dryrun
# mode (to prevent any mistakes).
self.browser = Browser()
# Ignore bugs.webkit.org/robots.txt until we fix it to allow this
# script.
self.browser.set_handle_robots(False)
# FIXME: Much of this should go into some sort of config module:
bug_server_host = "bugs.webkit.org"
bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
bug_server_url = "https://%s/" % bug_server_host
def quips(self):
# We only fetch and parse the list of quips once per instantiation
# so that we do not burden bugs.webkit.org.
if not self.cached_quips and not self.dryrun:
self.cached_quips = self.queries.fetch_quips()
return self.cached_quips
def bug_url_for_bug_id(self, bug_id, xml=False):
if not bug_id:
return None
content_type = "&ctype=xml" if xml else ""
return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url,
bug_id,
content_type)
def short_bug_url_for_bug_id(self, bug_id):
if not bug_id:
return None
return "http://webkit.org/b/%s" % bug_id
def attachment_url_for_id(self, attachment_id, action="view"):
if not attachment_id:
return None
action_param = ""
if action and action != "view":
action_param = "&action=%s" % action
return "%sattachment.cgi?id=%s%s" % (self.bug_server_url,
attachment_id,
action_param)
def _parse_attachment_flag(self,
element,
flag_name,
attachment,
result_key):
flag = element.find('flag', attrs={'name': flag_name})
if flag:
attachment[flag_name] = flag['status']
if flag['status'] == '+':
attachment[result_key] = flag['setter']
# Sadly show_bug.cgi?ctype=xml does not expose the flag modification date.
def _string_contents(self, soup):
# WebKit's bugzilla instance uses UTF-8.
# BeautifulSoup always returns Unicode strings, however
# the .string method returns a (unicode) NavigableString.
# NavigableString can confuse other parts of the code, so we
# convert from NavigableString to a real unicode() object using unicode().
return unicode(soup.string)
# Example: 2010-01-20 14:31 PST
# FIXME: Some bugzilla dates seem to have seconds in them?
# Python does not support timezones out of the box.
# Assume that bugzilla always uses PST (which is true for bugs.webkit.org)
_bugzilla_date_format = "%Y-%m-%d %H:%M"
@classmethod
def _parse_date(cls, date_string):
(date, time, time_zone) = date_string.split(" ")
# Ignore the timezone because python doesn't understand timezones out of the box.
date_string = "%s %s" % (date, time)
return datetime.strptime(date_string, cls._bugzilla_date_format)
def _date_contents(self, soup):
return self._parse_date(self._string_contents(soup))
def _parse_attachment_element(self, element, bug_id):
attachment = {}
attachment['bug_id'] = bug_id
attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
attachment['id'] = int(element.find('attachid').string)
# FIXME: No need to parse out the url here.
attachment['url'] = self.attachment_url_for_id(attachment['id'])
attachment["attach_date"] = self._date_contents(element.find("date"))
attachment['name'] = self._string_contents(element.find('desc'))
attachment['attacher_email'] = self._string_contents(element.find('attacher'))
attachment['type'] = self._string_contents(element.find('type'))
self._parse_attachment_flag(
element, 'review', attachment, 'reviewer_email')
self._parse_attachment_flag(
element, 'in-rietveld', attachment, 'rietveld_uploader_email')
self._parse_attachment_flag(
element, 'commit-queue', attachment, 'committer_email')
return attachment
def _parse_bug_page(self, page):
soup = BeautifulSoup(page)
bug = {}
bug["id"] = int(soup.find("bug_id").string)
bug["title"] = self._string_contents(soup.find("short_desc"))
bug["reporter_email"] = self._string_contents(soup.find("reporter"))
bug["assigned_to_email"] = self._string_contents(soup.find("assigned_to"))
bug["cc_emails"] = [self._string_contents(element)
for element in soup.findAll('cc')]
bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')]
return bug
# Makes testing fetch_*_from_bug() possible until we have a better
# BugzillaNetwork abstration.
def _fetch_bug_page(self, bug_id):
bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
log("Fetching: %s" % bug_url)
return self.browser.open(bug_url)
def fetch_bug_dictionary(self, bug_id):
try:
return self._parse_bug_page(self._fetch_bug_page(bug_id))
except KeyboardInterrupt:
raise
except:
self.authenticate()
return self._parse_bug_page(self._fetch_bug_page(bug_id))
# FIXME: A BugzillaCache object should provide all these fetch_ methods.
def fetch_bug(self, bug_id):
return Bug(self.fetch_bug_dictionary(bug_id), self)
def fetch_attachment_contents(self, attachment_id):
attachment_url = self.attachment_url_for_id(attachment_id)
# We need to authenticate to download patches from security bugs.
self.authenticate()
return self.browser.open(attachment_url).read()
def _parse_bug_id_from_attachment_page(self, page):
# The "Up" relation happens to point to the bug.
up_link = BeautifulSoup(page).find('link', rel='Up')
if not up_link:
# This attachment does not exist (or you don't have permissions to
# view it).
return None
match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href'])
return int(match.group('bug_id'))
def bug_id_for_attachment_id(self, attachment_id):
self.authenticate()
attachment_url = self.attachment_url_for_id(attachment_id, 'edit')
log("Fetching: %s" % attachment_url)
page = self.browser.open(attachment_url)
return self._parse_bug_id_from_attachment_page(page)
# FIXME: This should just return Attachment(id), which should be able to
# lazily fetch needed data.
def fetch_attachment(self, attachment_id):
# We could grab all the attachment details off of the attachment edit
# page but we already have working code to do so off of the bugs page,
# so re-use that.
bug_id = self.bug_id_for_attachment_id(attachment_id)
if not bug_id:
return None
attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True)
for attachment in attachments:
if attachment.id() == int(attachment_id):
return attachment
return None # This should never be hit.
def authenticate(self):
if self.authenticated:
return
if self.dryrun:
log("Skipping log in for dry run...")
self.authenticated = True
return
attempts = 0
while not self.authenticated:
attempts += 1
(username, password) = Credentials(
self.bug_server_host, git_prefix="bugzilla").read_credentials()
log("Logging in as %s..." % username)
self.browser.open(self.bug_server_url +
"index.cgi?GoAheadAndLogIn=1")
self.browser.select_form(name="login")
self.browser['Bugzilla_login'] = username
self.browser['Bugzilla_password'] = password
response = self.browser.submit()
match = re.search("<title>(.+?)</title>", response.read())
# If the resulting page has a title, and it contains the word
# "invalid" assume it's the login failure page.
if match and re.search("Invalid", match.group(1), re.IGNORECASE):
errorMessage = "Bugzilla login failed: %s" % match.group(1)
# raise an exception only if this was the last attempt
if attempts < 5:
log(errorMessage)
else:
raise Exception(errorMessage)
else:
self.authenticated = True
self.username = username
def _fill_attachment_form(self,
description,
patch_file_object,
comment_text=None,
mark_for_review=False,
mark_for_commit_queue=False,
mark_for_landing=False,
bug_id=None):
self.browser['description'] = description
self.browser['ispatch'] = ("1",)
self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
if mark_for_landing:
self.browser['flag_type-3'] = ('+',)
elif mark_for_commit_queue:
self.browser['flag_type-3'] = ('?',)
else:
self.browser['flag_type-3'] = ('X',)
if bug_id:
patch_name = "bug-%s-%s.patch" % (bug_id, timestamp())
else:
patch_name ="%s.patch" % timestamp()
self.browser.add_file(patch_file_object,
"text/plain",
patch_name,
'data')
def add_patch_to_bug(self,
bug_id,
diff,
description,
comment_text=None,
mark_for_review=False,
mark_for_commit_queue=False,
mark_for_landing=False):
self.authenticate()
log('Adding patch "%s" to %sshow_bug.cgi?id=%s' % (description,
self.bug_server_url,
bug_id))
if self.dryrun:
log(comment_text)
return
self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % (
self.bug_server_url, bug_id))
self.browser.select_form(name="entryform")
# _fill_attachment_form expects a file-like object
# Patch files are already binary, so no encoding needed.
assert(isinstance(diff, str))
patch_file_object = StringIO.StringIO(diff)
self._fill_attachment_form(description,
patch_file_object,
mark_for_review=mark_for_review,
mark_for_commit_queue=mark_for_commit_queue,
mark_for_landing=mark_for_landing,
bug_id=bug_id)
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser.submit()
def _check_create_bug_response(self, response_html):
match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>",
response_html)
if match:
return match.group('bug_id')
match = re.search(
'<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">',
response_html,
re.DOTALL)
error_message = "FAIL"
if match:
text_lines = BeautifulSoup(
match.group('error_message')).findAll(text=True)
error_message = "\n" + '\n'.join(
[" " + line.strip()
for line in text_lines if line.strip()])
raise Exception("Bug not created: %s" % error_message)
def create_bug(self,
bug_title,
bug_description,
component=None,
diff=None,
patch_description=None,
cc=None,
blocked=None,
assignee=None,
mark_for_review=False,
mark_for_commit_queue=False):
self.authenticate()
log('Creating bug with title "%s"' % bug_title)
if self.dryrun:
log(bug_description)
return
self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit")
self.browser.select_form(name="Create")
component_items = self.browser.find_control('component').items
component_names = map(lambda item: item.name, component_items)
if not component:
component = "New Bugs"
if component not in component_names:
component = User.prompt_with_list("Please pick a component:", component_names)
self.browser["component"] = [component]
if cc:
self.browser["cc"] = cc
if blocked:
self.browser["blocked"] = unicode(blocked)
if assignee == None:
assignee = self.username
if assignee and not self.browser.find_control("assigned_to").disabled:
self.browser["assigned_to"] = assignee
self.browser["short_desc"] = bug_title
self.browser["comment"] = bug_description
if diff:
# _fill_attachment_form expects a file-like object
# Patch files are already binary, so no encoding needed.
assert(isinstance(diff, str))
patch_file_object = StringIO.StringIO(diff)
self._fill_attachment_form(
patch_description,
patch_file_object,
mark_for_review=mark_for_review,
mark_for_commit_queue=mark_for_commit_queue)
response = self.browser.submit()
bug_id = self._check_create_bug_response(response.read())
log("Bug %s created." % bug_id)
log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id))
return bug_id
def _find_select_element_for_flag(self, flag_name):
# FIXME: This will break if we ever re-order attachment flags
if flag_name == "review":
return self.browser.find_control(type='select', nr=0)
elif flag_name == "commit-queue":
return self.browser.find_control(type='select', nr=1)
elif flag_name == "in-rietveld":
return self.browser.find_control(type='select', nr=2)
raise Exception("Don't know how to find flag named \"%s\"" % flag_name)
def clear_attachment_flags(self,
attachment_id,
additional_comment_text=None):
self.authenticate()
comment_text = "Clearing flags on attachment: %s" % attachment_id
if additional_comment_text:
comment_text += "\n\n%s" % additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.set_value(comment_text, name='comment', nr=0)
self._find_select_element_for_flag('review').value = ("X",)
self._find_select_element_for_flag('commit-queue').value = ("X",)
self.browser.submit()
def set_flag_on_attachment(self,
attachment_id,
flag_name,
flag_value,
comment_text=None,
additional_comment_text=None):
# FIXME: We need a way to test this function on a live bugzilla
# instance.
self.authenticate()
if additional_comment_text:
comment_text += "\n\n%s" % additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
if comment_text:
self.browser.set_value(comment_text, name='comment', nr=0)
self._find_select_element_for_flag(flag_name).value = (flag_value,)
self.browser.submit()
# FIXME: All of these bug editing methods have a ridiculous amount of
# copy/paste code.
def obsolete_attachment(self, attachment_id, comment_text=None):
self.authenticate()
log("Obsoleting attachment: %s" % attachment_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.find_control('isobsolete').items[0].selected = True
# Also clear any review flag (to remove it from review/commit queues)
self._find_select_element_for_flag('review').value = ("X",)
self._find_select_element_for_flag('commit-queue').value = ("X",)
if comment_text:
log(comment_text)
# Bugzilla has two textareas named 'comment', one is somehow
# hidden. We want the first.
self.browser.set_value(comment_text, name='comment', nr=0)
self.browser.submit()
def add_cc_to_bug(self, bug_id, email_address_list):
self.authenticate()
log("Adding %s to the CC list for bug %s" % (email_address_list,
bug_id))
if self.dryrun:
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser["newcc"] = ", ".join(email_address_list)
self.browser.submit()
def post_comment_to_bug(self, bug_id, comment_text, cc=None):
self.authenticate()
log("Adding comment to bug %s" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser["comment"] = comment_text
if cc:
self.browser["newcc"] = ", ".join(cc)
self.browser.submit()
def close_bug_as_fixed(self, bug_id, comment_text=None):
self.authenticate()
log("Closing bug %s as fixed" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser['bug_status'] = ['RESOLVED']
self.browser['resolution'] = ['FIXED']
self.browser.submit()
def reassign_bug(self, bug_id, assignee, comment_text=None):
self.authenticate()
log("Assigning bug %s to %s" % (bug_id, assignee))
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser["comment"] = comment_text
self.browser["assigned_to"] = assignee
self.browser.submit()
def reopen_bug(self, bug_id, comment_text):
self.authenticate()
log("Re-opening bug %s" % bug_id)
# Bugzilla requires a comment when re-opening a bug, so we know it will
# never be None.
log(comment_text)
if self.dryrun:
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
bug_status = self.browser.find_control("bug_status", type="select")
# This is a hack around the fact that ClientForm.ListControl seems to
# have no simpler way to ask if a control has an item named "REOPENED"
# without using exceptions for control flow.
possible_bug_statuses = map(lambda item: item.name, bug_status.items)
if "REOPENED" in possible_bug_statuses:
bug_status.value = ["REOPENED"]
# If the bug was never confirmed it will not have a "REOPENED"
# state, but only an "UNCONFIRMED" state.
elif "UNCONFIRMED" in possible_bug_statuses:
bug_status.value = ["UNCONFIRMED"]
else:
# FIXME: This logic is slightly backwards. We won't print this
# message if the bug is already open with state "UNCONFIRMED".
log("Did not reopen bug %s, it appears to already be open with status %s." % (bug_id, bug_status.value))
self.browser['comment'] = comment_text
self.browser.submit()
class Builder(object):
def __init__(self, name, buildbot):
self._name = name
self._buildbot = buildbot
self._builds_cache = {}
self._revision_to_build_number = None
self._browser = None
def name(self):
return self._name
def results_url(self):
return "%s/results/%s" % (self._buildbot.buildbot_url,
self.url_encoded_name())
# In addition to per-build results, the build.chromium.org builders also
# keep a directory that accumulates test results over many runs.
def accumulated_results_url(self):
return None
def latest_layout_test_results_url(self):
return self.accumulated_results_url() or self.latest_cached_build(
).results_url()
@memoized
def latest_layout_test_results(self):
return self.fetch_layout_test_results(
self.latest_layout_test_results_url())
def _fetch_file_from_results(self, results_url, file_name):
# It seems this can return None if the url redirects and then returns 404.
result = urllib2.urlopen("%s/%s" % (results_url, file_name))
if not result:
return None
# urlopen returns a file-like object which sometimes works fine with str()
# but sometimes is a addinfourl object. In either case calling read() is correct.
return result.read()
def fetch_layout_test_results(self, results_url):
# FIXME: This should cache that the result was a 404 and stop hitting the network.
results_file = NetworkTransaction(convert_404_to_None=True).run(
lambda: self._fetch_file_from_results(results_url,
"full_results.json"))
return LayoutTestResults.results_from_string(results_file)
def url_encoded_name(self):
return urllib.quote(self._name)
def url(self):
return "%s/builders/%s" % (self._buildbot.buildbot_url,
self.url_encoded_name())
# This provides a single place to mock
def _fetch_build(self, build_number):
build_dictionary = self._buildbot._fetch_build_dictionary(
self, build_number)
if not build_dictionary:
return None
revision_string = build_dictionary['sourceStamp']['revision']
return Build(
self,
build_number=int(build_dictionary['number']),
# 'revision' may be None if a trunk build was started by the force-build button on the web page.
revision=(int(revision_string) if revision_string else None),
# Buildbot uses any nubmer other than 0 to mean fail. Since we fetch with
# filter=1, passing builds may contain no 'results' value.
is_green=(not build_dictionary.get('results')),
)
def build(self, build_number):
if not build_number:
return None
cached_build = self._builds_cache.get(build_number)
if cached_build:
return cached_build
build = self._fetch_build(build_number)
self._builds_cache[build_number] = build
return build
def latest_cached_build(self):
revision_build_pairs = self.revision_build_pairs_with_results()
revision_build_pairs.sort(key=lambda i: i[1])
latest_build_number = revision_build_pairs[-1][1]
return self.build(latest_build_number)
def force_build(self, username="******", comments=None):
def predicate(form):
try:
return form.find_control("username")
except Exception as e:
return False
if not self._browser:
self._browser = Browser()
self._browser.set_handle_robots(
False) # The builder pages are excluded by robots.txt
# ignore false positives for missing Browser methods - pylint: disable=E1102
self._browser.open(self.url())
self._browser.select_form(predicate=predicate)
self._browser["username"] = username
if comments:
self._browser["comments"] = comments
return self._browser.submit()
file_name_regexp = re.compile(
r"r(?P<revision>\d+) \((?P<build_number>\d+)\)")
def _revision_and_build_for_filename(self, filename):
# Example: "r47483 (1)/" or "r47483 (1).zip"
match = self.file_name_regexp.match(filename)
if not match:
return None
return (int(match.group("revision")), int(match.group("build_number")))
def _fetch_revision_to_build_map(self):
# All _fetch requests go through _buildbot for easier mocking
# FIXME: This should use NetworkTransaction's 404 handling instead.
try:
# FIXME: This method is horribly slow due to the huge network load.
# FIXME: This is a poor way to do revision -> build mapping.
# Better would be to ask buildbot through some sort of API.
print("Loading revision/build list from %s." % self.results_url())
print("This may take a while...")
result_files = self._buildbot._fetch_twisted_directory_listing(
self.results_url())
except urllib2.HTTPError as error:
if error.code != 404:
raise
_log.debug("Revision/build list failed to load.")
result_files = []
return dict(
self._file_info_list_to_revision_to_build_list(result_files))
def _file_info_list_to_revision_to_build_list(self, file_info_list):
# This assumes there was only one build per revision, which is false but we don't care for now.
revisions_and_builds = []
for file_info in file_info_list:
revision_and_build = self._revision_and_build_for_filename(
file_info["filename"])
if revision_and_build:
revisions_and_builds.append(revision_and_build)
return revisions_and_builds
def _revision_to_build_map(self):
if not self._revision_to_build_number:
self._revision_to_build_number = self._fetch_revision_to_build_map(
)
return self._revision_to_build_number
def revision_build_pairs_with_results(self):
return self._revision_to_build_map().items()
# This assumes there can be only one build per revision, which is false, but we don't care for now.
def build_for_revision(self, revision, allow_failed_lookups=False):
# NOTE: This lookup will fail if that exact revision was never built.
build_number = self._revision_to_build_map().get(int(revision))
if not build_number:
return None
build = self.build(build_number)
if not build and allow_failed_lookups:
# Builds for old revisions with fail to lookup via buildbot's json api.
build = Build(
self,
build_number=build_number,
revision=revision,
is_green=False,
)
return build
def find_regression_window(self, red_build, look_back_limit=30):
if not red_build or red_build.is_green():
return RegressionWindow(None, None)
common_failures = None
current_build = red_build
build_after_current_build = None
look_back_count = 0
while current_build:
if current_build.is_green():
# current_build can't possibly have any failures in common
# with red_build because it's green.
break
results = current_build.layout_test_results()
# We treat a lack of results as if all the test failed.
# This occurs, for example, when we can't compile at all.
if results:
failures = set(results.failing_tests())
if common_failures == None:
common_failures = failures
else:
common_failures = common_failures.intersection(failures)
if not common_failures:
# current_build doesn't have any failures in common with
# the red build we're worried about. We assume that any
# failures in current_build were due to flakiness.
break
look_back_count += 1
if look_back_count > look_back_limit:
return RegressionWindow(None,
current_build,
failing_tests=common_failures)
build_after_current_build = current_build
current_build = current_build.previous_build()
# We must iterate at least once because red_build is red.
assert (build_after_current_build)
# Current build must either be green or have no failures in common
# with red build, so we've found our failure transition.
return RegressionWindow(current_build,
build_after_current_build,
failing_tests=common_failures)
def find_blameworthy_regression_window(self,
red_build_number,
look_back_limit=30,
avoid_flakey_tests=True):
red_build = self.build(red_build_number)
regression_window = self.find_regression_window(
red_build, look_back_limit)
if not regression_window.build_before_failure():
return None # We ran off the limit of our search
# If avoid_flakey_tests, require at least 2 bad builds before we
# suspect a real failure transition.
if avoid_flakey_tests and regression_window.failing_build(
) == red_build:
return None
return regression_window
class StatusServer:
default_host = "webkit-commit-queue.appspot.com"
def __init__(self, host=default_host):
self.set_host(host)
self.browser = Browser()
def set_host(self, host):
self.host = host
self.url = "http://%s" % self.host
def results_url_for_status(self, status_id):
return "%s/results/%s" % (self.url, status_id)
def _add_patch(self, patch):
if not patch:
return
if patch.bug_id():
self.browser["bug_id"] = unicode(patch.bug_id())
if patch.id():
self.browser["patch_id"] = unicode(patch.id())
def _add_results_file(self, results_file):
if not results_file:
return
self.browser.add_file(results_file, "text/plain", "results.txt",
'results_file')
def _post_status_to_server(self, queue_name, status, patch, results_file):
if results_file:
# We might need to re-wind the file if we've already tried to post it.
results_file.seek(0)
update_status_url = "%s/update-status" % self.url
self.browser.open(update_status_url)
self.browser.select_form(name="update_status")
self.browser["queue_name"] = queue_name
self._add_patch(patch)
self.browser["status"] = status
self._add_results_file(results_file)
return self.browser.submit().read(
) # This is the id of the newly created status object.
def _post_svn_revision_to_server(self, svn_revision_number, broken_bot):
update_svn_revision_url = "%s/update-svn-revision" % self.url
self.browser.open(update_svn_revision_url)
self.browser.select_form(name="update_svn_revision")
self.browser["number"] = unicode(svn_revision_number)
self.browser["broken_bot"] = broken_bot
return self.browser.submit().read()
def update_status(self, queue_name, status, patch=None, results_file=None):
log(status)
return NetworkTransaction().run(lambda: self._post_status_to_server(
queue_name, status, patch, results_file))
def update_svn_revision(self, svn_revision_number, broken_bot):
log("SVN revision: %s broke %s" % (svn_revision_number, broken_bot))
return NetworkTransaction().run(
lambda: self._post_svn_revision_to_server(svn_revision_number,
broken_bot))
def _fetch_url(self, url):
try:
return urllib2.urlopen(url).read()
except urllib2.HTTPError, e:
if e.code == 404:
return None
raise e