class StatusServer:
default_host = "webkit-commit-queue.appspot.com"
def __init__(self, host=default_host):
self.set_host(host)
self.browser = Browser()
def set_host(self, host):
self.host = host
self.url = "http://%s" % self.host
def results_url_for_status(self, status_id):
return "%s/results/%s" % (self.url, status_id)
def _add_patch(self, patch):
if not patch:
return
if patch.bug_id():
self.browser["bug_id"] = str(patch.bug_id())
if patch.id():
self.browser["patch_id"] = str(patch.id())
def _add_results_file(self, results_file):
if not results_file:
return
self.browser.add_file(results_file, "text/plain", "results.txt", 'results_file')
def _post_to_server(self, queue_name, status, patch, results_file):
if results_file:
# We might need to re-wind the file if we've already tried to post it.
results_file.seek(0)
update_status_url = "%s/update-status" % self.url
self.browser.open(update_status_url)
self.browser.select_form(name="update_status")
self.browser['queue_name'] = queue_name
self._add_patch(patch)
self.browser['status'] = status
self._add_results_file(results_file)
return self.browser.submit().read() # This is the id of the newly created status object.
def update_status(self, queue_name, status, patch=None, results_file=None):
# During unit testing, host is None
if not self.host:
return
log(status)
return NetworkTransaction().run(lambda: self._post_to_server(queue_name, status, patch, results_file))
def patch_status(self, queue_name, patch_id):
update_status_url = "%s/patch-status/%s/%s" % (self.url, queue_name, patch_id)
try:
return urllib2.urlopen(update_status_url).read()
except urllib2.HTTPError, e:
if e.code == 404:
return None
raise e
def get_products(sequence, file_in):
# open site and get form
br = Browser()
br.open("http://peptide.alexmijalis.com/")
br.select_form(nr=0) # there's only a single form
# get response
# TODO make action statement for input arg
br['text'] = sequence
br.add_file(open(file_in, 'r'), 'text/plain/', file_in, name='file')
response = br.submit()
return response.readlines()
def _mapgen_speed_fired(self):
test_dir(join(self.dirname, 'gps_vis'))
br = Browser()
# Ignore robots.txt
br.set_handle_robots( False )
# Google demands a user-agent that isn't a robot
br.addheaders = [('User-agent', 'Firefox')]
resp1 = br.open( "http://www.gpsvisualizer.com/map_input" )
# Select the search box and search for 'foo'
br.select_form( name='main' )
br.form['width'] = '870'
br.form['height'] = '600'
br.set_value(['google_openstreetmap'], name='bg_map')
br.set_value(['speed'], name='trk_colorize')
br.form['legend_steps'] = '10'
br.add_file(open(self.filename_converted), "text/plain", self.filename_converted, name='uploaded_file_1')
# Get the search results
resp2 = br.submit()
resp = None
for link in br.links():
siteMatch = re.compile( 'download/' ).search( link.url )
if siteMatch:
resp = br.follow_link( link )
break
# Print the site
content = resp.get_data()
ofile = open(join(self.dirname, 'gps_vis', 'map_speed.html'),'w')
ofile.write(content)
ofile.close()
br.close()
print 'map generated (speed color)'
def _profilegen_fired(self):
test_dir(join(self.dirname, 'gps_vis'))
br = Browser()
# Ignore robots.txt
br.set_handle_robots( False )
# Google demands a user-agent that isn't a robot
br.addheaders = [('User-agent', 'Firefox')]
# Retrieve the Google home page, saving the response
resp1 = br.open( "http://www.gpsvisualizer.com/profile_input" )
# Select the search box and search for 'foo'
br.select_form( name='main' )
br.form['width'] = '870'
br.form['height'] = '250'
br.form['legend_steps'] = '10'
br.add_file(open(self.filename_converted), "text/plain", self.filename_converted, name='uploaded_file_1')
# Get the search results
resp2 = br.submit()
resp = None
for link in br.links():
siteMatch = re.compile( 'download/' ).search( link.url )
if siteMatch:
resp = br.follow_link( link )
break
# Print the site
content = resp.get_data()
ofile = open(join(self.dirname, 'gps_vis', 'profile.png'),'wb')
ofile.write(content)
ofile.close()
br.close()
print 'profile generated'
class LEAMsite:
def __init__(self, site, user, passwd):
self.site = site
self.error = False
self.b = Browser()
self.b.set_handle_robots(False)
try:
self.b.open(site)
except urllib2.URLError:
self.error = True
return
try:
# try and log in from the portlet
self.b.select_form('loginform')
except:
# try logging in from the main login page
self.b.open('/'.join((site,"login_form")))
self.b.select_form(nr=1)
self.b['__ac_name'] = user
self.b['__ac_password'] = passwd
r = self.b.open(self.b.click())
# plone changes have rendered this inoperable
# capture the response and look in the content
# body tag has class with login failure
def checkURL(self, url):
"""Tries to open a URL and return true if successful (object exists)
or false if error occurs.
"""
try:
rsp = self.b.open(url)
except:
return False
return True
def getURL(self, url, data=None, filename=None):
"""Simple interface for retrieving the contents of a URL
and writing it to a file or returning it as stringIO.
"""
#sys.stderr.write('getURL %s\n' % url)
rsp = self.b.open(url, data)
if filename:
f = file("./Inputs/"+filename, 'wb') # always write to Input folder
f.write(rsp.read())
f.close()
return None
else:
return StringIO(rsp.read())
def putFileURL(self, filename, url,
fileobj=None, title=None, type='text/plain'):
"""Simple interface for uploading a file to a plone site.
<URL> should be an existing folder on the site and
<filename> should be a readable file.
"""
#sys.stderr.write('putFileURL %s to %s\n' % (filename,url))
if not title: title = path.basename(filename)
if not fileobj: fileobj = open(filename, 'rb')
self.b.open('/'.join((url,'createObject?type_name=File')))
self.b.select_form("edit_form")
self.b['title'] = title
self.b.add_file(fileobj, type, path.basename(filename))
# form = self.b.find_control('file_delete')
# form.value = ["",]
self.b.submit("form.button.save")
# r = self.b.open(self.b.click())
# should check that it worked
def getFile(self, url, filename=None):
""" getFile -- gets a file using standard download from Plone site
url: The URL is pointer to the file on the Plone site
filename: optional filename where data will be written
"""
rsp = self.b.open(url_join(url,'at_download/file'))
if filename:
f = open(filename, 'wb')
f.write(rsp.read())
f.close()
return None
else:
return rsp.read()
def saveFile(self, url, dir=".", at_download=False):
"""Reads the response from a URL and saves it to a local
file based on the name provided in the Content-Disposition
header.
The dir field specifies to the directory where the file will
be stored.
If the at_download flag is True then 'at_download/file' will
be appended the URL.
"""
if at_download:
rsp = self.b.open(url_join(url,'at_download/file'))
else:
rsp = self.b.open(url)
fname = get_filename(rsp)
if fname:
f = open('/'.join([dir, fname]), 'wb')
f.write(rsp.read())
f.close()
return fname
def putImageURL_old(self, imgfile, url, title=None, type='image/jpg'):
sys.stderr.write('putImageURL %s to %s\n' % (imgfile,url))
self.b.open('/'.join((url,'createObject?type_name=Image')))
self.b.select_form("edit_form")
if not title: title = path.basename(imgfile)
self.b['title'] = title
self.b.add_file(open(imgfile), type, path.basename(imgfile))
try:
# doesn't seem necessary but it is required for file upload
form = self.b.find_control('image_delete')
form.value = ["",]
# print "This really does need to happen.."
except:
# print "That delete stuff never happens..."
pass
self.b.submit("form.button.save")
def putImageURL(self, filename, url,
fileobj=None, title=None, type='image/jpg'):
"""Simple interface for uploading a file to a plone site.
<URL> should be an existing folder on the site and
<filename> should be a readable file.
"""
#sys.stderr.write('putFileURL %s to %s\n' % (filename,url))
if not title: title = path.basename(filename)
if not fileobj: fileobj = open(filename, 'rb')
self.b.open('/'.join((url,'createObject?type_name=Image')))
self.b.select_form("edit_form")
self.b['title'] = title
self.b.add_file(fileobj, type, path.basename(filename))
# form = self.b.find_control('file_delete')
# form.value = ["",]
self.b.submit("form.button.save")
# r = self.b.open(self.b.click())
# should check that it worked
def putDocument(self, doc, url, title):
"""Creates a new document and add the doc (file-like object) to it."""
self.b.open('/'.join((url,'createObject?type_name=Document')))
self.b.select_form("edit_form")
self.b['title'] = title
doc.seek(0)
self.b['text'] = doc.read()
self.b.submit("form.button.save")
return self.b.geturl()
def getDocument(self, url):
"""Returns a string with the text of the current document."""
self.b.open('/'.join((url,'edit')))
self.b.select_form("edit_form")
s = self.b['text']
self.b.submit("form.button.cancel")
return s
def editDocument(self, doc, url, title=None):
"""Replaces the contents of a document"""
self.b.open('/'.join((url,'edit')))
self.b.select_form("edit_form")
# update the title if given
if title: self.b['title'] = title
# slightly dangerous approach where we seek first
# to test for a file-like object. If exception is
# thrown then assume doc is a string.
try:
doc.seek(0)
self.b['text'] = doc.read()
except:
self.b['text'] = doc
self.b.submit("form.button.save")
return self.b.geturl()
def createFolder(self, folder, url):
"""Creates a folder titled <folder> at the location <url> if
it doesn't already exist. Returns the full path of new folder.
"""
pathurl = '/'.join((url,folder.lower().replace(' ','-')))
print pathurl
try:
self.b.open(pathurl)
except:
self.b.open('/'.join((url,'createObject?type_name=Folder')))
self.b.select_form("edit_form")
self.b['title'] = folder
self.b.submit("form.button.save")
return self.b.geturl()
def editFolder(self, url, title="", description=""):
"""Edit the basic fields of the Folder. Mostly useful for
setting the title AFTER creating the folder with a reasonable
short name.
"""
try:
self.b.open(url)
except:
self.error = True
return None
self.b.open(url+'/edit')
self.b.select_form("edit_form")
if title:
self.b['title'] = title
if description:
self.b['description'] = description
self.b.submit("form.button.save")
return self.b.geturl()
def deleteFolder(self, url):
"Deletes folder and all of its contents"
sys.stderr.write('DELETING folder %s\n' % url)
return
# Puts SimMaps on to the site
def putSimMap(self, simmap, mapfile, url,
simmap_file=None, mapfile_file=None,
title=None, description=None,
trans=.7, details=None, zoom=8):
""" putSimMap
Required Input: simmap, mapfile, url
simmap is a file that contains the desired GIS layer
mapfile is the standard .map file that maps GIS layer to image
url - is the full URL to the folder where the SimMap will be stored
Optional Inputs: simmap_file, mapfile_file, title, trans, details
simmap_file - file-like object of the simmap if None simmap
will be openned and read.
mapfile_file - file-like object of the mapfile. If None mapfile
will be openned and read.
title - title sting of the SimMap defaults to basename(simmap)
trans - transparency level
details - description of SimMap as stored in the details field
"""
self.b.open('/'.join((url,'createObject?type_name=SimMap')))
self.b.select_form("edit_form")
if not simmap_file: simmap_file = open(simmap, 'rb')
if not mapfile_file: mapfile_file = open(mapfile, 'rb')
if not title: title = path.splitext(path.basename(simmap))[0]
self.b['title'] = str(title)
self.b.add_file(simmap_file, 'application/octet-stream',
path.basename(simmap), "simImage_file")
self.b.add_file(mapfile_file, 'application/octet-stream',
path.basename(mapfile), "mapFile_file")
if description: self.b['description'] = str(description)
self.b['transparency'] = str(trans)
self.b['zoom'] = str(zoom)
if details: self.b['details'] = str(details)
self.b.submit("form.button.save")
return self.b.geturl()
def getSimMap(self, url):
""" return the SimMap metadata
Gets the metadata associated with SimMap including title,
description, location, transparency, and zoom.
= """
self.b.open(url+'/edit')
self.b.select_form('edit_form')
d = dict(
title = self.b['title'],
description = self.b['description'],
details = self.b['details'],
location = self.b['location'],
transparency = self.b['Transparency'],
zoom = self.b['zoom'],
)
self.b.submit("form.button.cancel")
return d
def updateSimMap(self, url, **kwargs):
""" update the SimMap metadata
Keywords must match the field names from the edit form exactly,
extra keywords (or mispelled ones) will be ignored.
"""
self.b.open(url+'/edit')
self.b.select_form('edit_form')
for k in kwargs:
if k in self.b:
self.b[k] = str(kwargs[k])
self.b.submit("form.button.save")
def getSimMapData(self, url, filename=None):
""" getSimMapData -- gets the data component of the the SimMap
url: The URL is pointer to the SimMap on the Plone site
filename: optional filename where data will be written
"""
bufsize = 15 * 1024 * 1024
rsp = self.b.open(url_join(url,'at_download/simImage'))
if filename:
f = file(filename, 'wb')
while 1:
b = rsp.read(bufsize)
f.write(b)
if len(b) < bufsize: break
f.close()
return None
else:
return StringIO(rsp.read())
def getSimMapMapfile(self, url, filename=None):
""" getSimMapData -- gets the mapfile component of the the SimMap
url: The URL is pointer to the SimMap on the Plone site
filename: optional filename where data will be written
"""
rsp = self.b.open(url_join(url,'at_download/mapFile'))
if filename:
f = file(filename, 'wb')
f.write(rsp.read())
f.close()
return None
else:
data = StringIO()
data.write(rsp.read())
return data
def putProbmapCache(self, url, filename):
"""store precomputed probmaps as part of the Driver Set
This is a temporary method until the REST interface
is ready. It's really depricated before it written!
"""
self.b.open(url+'/edit')
self.b.select_form('edit_form')
with open(filename, 'rb') as f:
self.b.add_file(f, 'application/octet-stream',
path.basename(filename), name='probfile_file')
self.b.submit("form.button.save")
# def putAttrmapCache(self, url, filepathbasename, filename):
# """store precomputed maps as part of the Driver Set
# This is a temporary method until the REST interface
# is ready. It's really depricated before it written!
# """
# self.b.open(url+'/edit')
# self.b.select_form('edit_form')
# with open(filename, 'w') as f:
# self.b.add_file(f, 'application/octet-stream',
# path.basename(filepathbasename), name=filename)
# self.b.submit("form.button.save")
# DELETE FUNCTIONS -----------------------------------------
# These functions delete items from the LEAM Plone sites
# ----------------------------------------------------------
def deleteItem(self,fname,url):
""" deleteItem
Deletes <fname> from the folder <url>
"""
print '/'.join((url,fname,'delete_confirmation'))
print self.b.open('/'.join((url,fname,'delete_confirmation')))
self.b.select_form(None,None,1)
print self.b.submit()
class LEAMsite:
def __init__(self, site, user, passwd):
self.site = site
self.error = False
self.b = Browser()
self.b.set_handle_robots(False)
try:
self.b.open(site)
except urllib2.URLError:
self.error = True
return
try:
# try and log in from the portlet
self.b.select_form('loginform')
except:
# try logging in from the main login page
self.b.open('/'.join((site, "login_form")))
self.b.select_form(nr=1)
if not user or not passwd:
raise ValueError('user and password are required')
self.b['__ac_name'] = user
self.b['__ac_password'] = passwd
r = self.b.open(self.b.click())
# plone changes have rendered this inoperable
# capture the response and look in the content
#
#if 'logged_in' in path.split(self.b.geturl()):
# sys.stderr.write("Error: unable to login to LEAM Plone site\n")
# sys.exit(1)
def getURL(self, url, data=None, filename=None):
"""Simple interface for retrieving the contents of a URL
and writing it to a file or returning it as stringIO.
"""
#sys.stderr.write('getURL %s\n' % url)
rsp = self.b.open(url, data)
if filename:
f = file(filename, 'wb')
f.write(rsp.read())
f.close()
return None
else:
return StringIO(rsp.read())
def putFileURL(self,
filename,
url,
fileobj=None,
title=None,
type='text/plain'):
"""Simple interface for uploading a file to a plone site.
<URL> should be an existing folder on the site and
<filename> should be a readable file.
"""
#sys.stderr.write('putFileURL %s to %s\n' % (filename,url))
if not title: title = path.basename(filename)
if not fileobj: fileobj = open(filename, 'rb')
self.b.open('/'.join((url, 'createObject?type_name=File')))
self.b.select_form("edit_form")
self.b['title'] = title
self.b.add_file(fileobj, type, path.basename(filename))
# form = self.b.find_control('file_delete')
# form.value = ["",]
self.b.submit("form.button.save")
# r = self.b.open(self.b.click())
# should check that it worked
def getFile(self, url, filename=None):
""" getFile -- gets a file using standard download from Plone site
url: The URL is pointer to the file on the Plone site
filename: optional filename where data will be written
"""
rsp = self.b.open(url_join(url, 'at_download/file'))
if filename:
f = open(filename, 'wb')
f.write(rsp.read())
f.close()
return None
else:
return rsp.read()
def saveFile(self, url, dir=".", at_download=False):
"""Reads the response from a URL and saves it to a local
file based on the name provided in the Content-Disposition
header.
The dir field specifies to the directory where the file will be stored.
If the at_download flag is True then 'at_download/file' will
be appended the URL.
"""
if at_download:
rsp = self.b.open(url_join(url, 'at_download/file'))
else:
rsp = self.b.open(url)
fname = get_filename(rsp)
if fname:
f = open('/'.join([dir, fname]), 'wb')
f.write(rsp.read())
f.close()
return fname
def putImageURL_old(self, imgfile, url, title=None, type='image/jpg'):
sys.stderr.write('putImageURL %s to %s\n' % (imgfile, url))
self.b.open('/'.join((url, 'createObject?type_name=Image')))
self.b.select_form("edit_form")
if not title: title = path.basename(imgfile)
self.b['title'] = title
self.b.add_file(open(imgfile), type, path.basename(imgfile))
try:
# doesn't seem necessary but it is required for file upload
form = self.b.find_control('image_delete')
form.value = [
"",
]
# print "This really does need to happen.."
except:
# print "That delete stuff never happens..."
pass
self.b.submit("form.button.save")
def putImageURL(self,
filename,
url,
fileobj=None,
title=None,
type='image/jpg'):
"""Simple interface for uploading a file to a plone site.
<URL> should be an existing folder on the site and
<filename> should be a readable file.
"""
#sys.stderr.write('putFileURL %s to %s\n' % (filename,url))
if not title: title = path.basename(filename)
if not fileobj: fileobj = open(filename, 'rb')
self.b.open('/'.join((url, 'createObject?type_name=Image')))
self.b.select_form("edit_form")
self.b['title'] = title
self.b.add_file(fileobj, type, path.basename(filename))
# form = self.b.find_control('file_delete')
# form.value = ["",]
self.b.submit("form.button.save")
# r = self.b.open(self.b.click())
# should check that it worked
def putDocument(self, doc, url, title):
"""Creates a new document and add the doc (file-like object) to it."""
self.b.open('/'.join((url, 'createObject?type_name=Document')))
self.b.select_form("edit_form")
self.b['title'] = title
doc.seek(0)
self.b['text'] = doc.read()
self.b.submit("form.button.save")
return self.b.geturl()
def getDocument(self, url):
"""Returns a string with the text of the current document."""
self.b.open('/'.join((url, 'edit')))
self.b.select_form("edit_form")
s = self.b['text']
self.b.submit("form.button.cancel")
return s
def editDocument(self, doc, url, title=None):
"""Replaces the contents of a document"""
self.b.open('/'.join((url, 'edit')))
self.b.select_form("edit_form")
# update the title if given
if title: self.b['title'] = title
# slightly dangerous approach where we seek first
# to test for a file-like object. If exception is
# thrown then assume doc is a string.
try:
doc.seek(0)
self.b['text'] = doc.read()
except:
self.b['text'] = doc
self.b.submit("form.button.save")
return self.b.geturl()
def createFolder(self, folder, url):
"""Creates a folder titled <folder> at the location <url> if
it doesn't already exist. Returns the full path of new folder.
"""
pathurl = '/'.join((url, folder.lower().replace(' ', '-')))
try:
self.b.open(pathurl)
except:
self.b.open('/'.join((url, 'createObject?type_name=Folder')))
self.b.select_form("edit_form")
self.b['title'] = folder
self.b.submit("form.button.save")
return self.b.geturl()
def editFolder(self, url, title="", description=""):
"""Edit the basic fields of the Folder. Mostly useful for
setting the title AFTER creating the folder with a reasonable
short name.
"""
try:
self.b.open(url)
except:
self.error = True
return None
self.b.open(url + '/edit')
self.b.select_form("edit_form")
if title:
self.b['title'] = title
if description:
self.b['description'] = description
self.b.submit("form.button.save")
return self.b.geturl()
def deleteFolder(self, url):
"Deletes folder and all of its contents"
sys.stderr.write('DELETING folder %s\n' % url)
return
# Puts SimMaps on to the site
def putSimMap(self,
simmap,
mapfile,
url,
simmap_file=None,
mapfile_file=None,
title=None,
description=None,
trans=.7,
details=None,
zoom=11):
""" putSimMap
Required Input: simmap, mapfile, url
simmap is a file that contains the desired GIS layer
mapfile is the standard .map file that maps GIS layer to image
url - is the full URL to the folder where the SimMap will be stored
Optional Inputs: simmap_file, mapfile_file, title, trans, details
simmap_file - file-like object of the simmap if None simmap
will be openned and read.
mapfile_file - file-like object of the mapfile. If None mapfile
will be openned and read.
title - title sting of the SimMap defaults to basename(simmap)
trans - transparency level
details - description of SimMap as stored in the details field
"""
self.b.open('/'.join((url, 'createObject?type_name=SimMap')))
self.b.select_form("edit_form")
if not simmap_file: simmap_file = open(simmap, 'rb')
if not mapfile_file: mapfile_file = open(mapfile, 'rb')
if not title: title = path.splitext(path.basename(simmap))[0]
self.b['title'] = str(title)
self.b.add_file(simmap_file, 'application/octet-stream',
path.basename(simmap), "simImage_file")
self.b.add_file(mapfile_file, 'application/octet-stream',
path.basename(mapfile), "mapFile_file")
if description: self.b['description'] = str(description)
self.b['transparency'] = str(trans)
self.b['zoom'] = str(zoom)
if details: self.b['details'] = str(details)
self.b.submit("form.button.save")
return self.b.geturl()
def getSimMap(self, url):
""" getSimMap
Gets the metadata associated with SimMap including title,
description, location, transparency, and zoom.
"""
self.b.open(url + '/edit')
self.b.select_form('edit_form')
d = dict(
title=self.b['title'],
description=self.b['description'],
details=self.b['details'],
location=self.b['location'],
transparency=self.b['Transparency'],
zoom=self.b['zoom'],
)
self.b.submit("form.button.cancel")
return d
def getSimMapData(self, url, filename=None):
""" getSimMapData -- gets the data component of the the SimMap
url: The URL is pointer to the SimMap on the Plone site
filename: optional filename where data will be written
"""
bufsize = 15 * 1024 * 1024
rsp = self.b.open(url_join(url, 'get_layer'))
if filename:
f = file(filename, 'wb')
while 1:
b = rsp.read(bufsize)
f.write(b)
if len(b) < bufsize: break
f.close()
return None
else:
return StringIO(rsp.read())
def getSimMapMapfile(self, url, filename=None):
""" getSimMapData -- gets the mapfile component of the the SimMap
url: The URL is pointer to the SimMap on the Plone site
filename: optional filename where data will be written
"""
rsp = self.b.open(url_join(url, 'get_mapfile'))
if filename:
f = file(filename, 'wb')
f.write(rsp.read())
f.close()
return None
else:
data = StringIO()
data.write(rsp.read())
return data
# DELETE FUNCTIONS -----------------------------------------
# These functions delete items from the LEAM Plone sites
# ----------------------------------------------------------
def deleteItem(self, fname, url):
""" deleteItem
Deletes <fname> from the folder <url>
"""
print '/'.join((url, fname, 'delete_confirmation'))
print self.b.open('/'.join((url, fname, 'delete_confirmation')))
self.b.select_form(None, None, 1)
print self.b.submit()
class Bugzilla:
def __init__(self, dryrun=False, committers=CommitterList()):
self.dryrun = dryrun
self.authenticated = False
self.browser = Browser()
# Ignore bugs.webkit.org/robots.txt until we fix it to allow this script
self.browser.set_handle_robots(False)
self.committers = committers
# Defaults (until we support better option parsing):
bug_server_host = "bugs.webkit.org"
bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
bug_server_url = "https://%s/" % bug_server_host
def bug_url_for_bug_id(self, bug_id, xml=False):
content_type = "&ctype=xml" if xml else ""
return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type)
def attachment_url_for_id(self, attachment_id, action="view"):
action_param = ""
if action and action != "view":
action_param = "&action=" + action
return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, attachment_id, action_param)
def _parse_attachment_element(self, element, bug_id):
attachment = {}
attachment['bug_id'] = bug_id
attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
attachment['id'] = str(element.find('attachid').string)
attachment['url'] = self.attachment_url_for_id(attachment['id'])
attachment['name'] = unicode(element.find('desc').string)
attachment['type'] = str(element.find('type').string)
review_flag = element.find('flag', attrs={"name" : "review"})
if review_flag and review_flag['status'] == '+':
reviewer_email = review_flag['setter']
reviewer = self.committers.reviewer_by_bugzilla_email(reviewer_email)
attachment['reviewer'] = reviewer.full_name
commit_queue_flag = element.find('flag', attrs={"name" : "commit-queue"})
if commit_queue_flag and commit_queue_flag['status'] == '+':
committer_email = commit_queue_flag['setter']
committer = self.committers.committer_by_bugzilla_email(committer_email)
attachment['commit-queue'] = committer.full_name
return attachment
def fetch_attachments_from_bug(self, bug_id):
bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
log("Fetching: " + bug_url)
page = urllib2.urlopen(bug_url)
soup = BeautifulSoup(page)
attachments = []
for element in soup.findAll('attachment'):
attachment = self._parse_attachment_element(element, bug_id)
attachments.append(attachment)
return attachments
def fetch_patches_from_bug(self, bug_id):
patches = []
for attachment in self.fetch_attachments_from_bug(bug_id):
if attachment['is_patch'] and not attachment['is_obsolete']:
patches.append(attachment)
return patches
def fetch_reviewed_patches_from_bug(self, bug_id):
reviewed_patches = []
for attachment in self.fetch_attachments_from_bug(bug_id):
if 'reviewer' in attachment and not attachment['is_obsolete']:
reviewed_patches.append(attachment)
return reviewed_patches
def fetch_commit_queue_patches_from_bug(self, bug_id):
commit_queue_patches = []
for attachment in self.fetch_reviewed_patches_from_bug(bug_id):
if 'commit-queue' in attachment and not attachment['is_obsolete']:
commit_queue_patches.append(attachment)
return commit_queue_patches
def fetch_bug_ids_from_commit_queue(self):
commit_queue_url = self.bug_server_url + "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=commit-queue%2B"
page = urllib2.urlopen(commit_queue_url)
soup = BeautifulSoup(page)
bug_ids = []
# Grab the cells in the first column (which happens to be the bug ids)
for bug_link_cell in soup('td', "first-child"): # tds with the class "first-child"
bug_link = bug_link_cell.find("a")
bug_ids.append(bug_link.string) # the contents happen to be the bug id
return bug_ids
def fetch_patches_from_commit_queue(self):
patches_to_land = []
for bug_id in self.fetch_bug_ids_from_commit_queue():
patches = self.fetch_commit_queue_patches_from_bug(bug_id)
patches_to_land += patches
return patches_to_land
def authenticate(self):
if self.authenticated:
return
if self.dryrun:
log("Skipping log in for dry run...")
self.authenticated = True
return
(username, password) = read_credentials()
log("Logging in as %s..." % username)
self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1")
self.browser.select_form(name="login")
self.browser['Bugzilla_login'] = username
self.browser['Bugzilla_password'] = password
response = self.browser.submit()
match = re.search("<title>(.+?)</title>", response.read())
# If the resulting page has a title, and it contains the word "invalid" assume it's the login failure page.
if match and re.search("Invalid", match.group(1), re.IGNORECASE):
# FIXME: We could add the ability to try again on failure.
raise ScriptError("Bugzilla login failed: %s" % match.group(1))
self.authenticated = True
def add_patch_to_bug(self, bug_id, patch_file_object, description, comment_text=None, mark_for_review=False):
self.authenticate()
log('Adding patch "%s" to bug %s' % (description, bug_id))
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_server_url + "attachment.cgi?action=enter&bugid=" + bug_id)
self.browser.select_form(name="entryform")
self.browser['description'] = description
self.browser['ispatch'] = ("1",)
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
self.browser.add_file(patch_file_object, "text/plain", "bug-%s-%s.patch" % (bug_id, timestamp()))
self.browser.submit()
def prompt_for_component(self, components):
log("Please pick a component:")
i = 0
for name in components:
i += 1
log("%2d. %s" % (i, name))
result = int(raw_input("Enter a number: ")) - 1
return components[result]
def _check_create_bug_response(self, response_html):
match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>", response_html)
if match:
return match.group('bug_id')
match = re.search('<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">', response_html, re.DOTALL)
error_message = "FAIL"
if match:
text_lines = BeautifulSoup(match.group('error_message')).findAll(text=True)
error_message = "\n" + '\n'.join([" " + line.strip() for line in text_lines if line.strip()])
raise ScriptError("Bug not created: %s" % error_message)
def create_bug_with_patch(self, bug_title, bug_description, component, patch_file_object, patch_description, cc, mark_for_review=False):
self.authenticate()
log('Creating bug with patch description "%s"' % patch_description)
if self.dryrun:
log(bug_description)
return
self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit")
self.browser.select_form(name="Create")
component_items = self.browser.find_control('component').items
component_names = map(lambda item: item.name, component_items)
if not component or component not in component_names:
component = self.prompt_for_component(component_names)
self.browser['component'] = [component]
self.browser['cc'] = cc
self.browser['short_desc'] = bug_title
if bug_description:
log(bug_description)
self.browser['comment'] = bug_description
self.browser['description'] = patch_description
self.browser['ispatch'] = ("1",)
self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
self.browser.add_file(patch_file_object, "text/plain", "%s.patch" % timestamp(), 'data')
response = self.browser.submit()
bug_id = self._check_create_bug_response(response.read())
log("Bug %s created." % bug_id)
log(self.bug_server_url + "show_bug.cgi?id=" + bug_id)
return bug_id
def clear_attachment_review_flag(self, attachment_id, additional_comment_text=None):
self.authenticate()
comment_text = "Clearing review flag on attachment: %s" % attachment_id
if additional_comment_text:
comment_text += "\n\n" + additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.set_value(comment_text, name='comment', nr=0)
self.browser.find_control(type='select', nr=0).value = ("X",)
self.browser.submit()
def obsolete_attachment(self, attachment_id, comment_text = None):
self.authenticate()
log("Obsoleting attachment: %s" % attachment_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.find_control('isobsolete').items[0].selected = True
# Also clear any review flag (to remove it from review/commit queues)
self.browser.find_control(type='select', nr=0).value = ("X",)
if comment_text:
log(comment_text)
# Bugzilla has two textareas named 'comment', one is somehow hidden. We want the first.
self.browser.set_value(comment_text, name='comment', nr=0)
self.browser.submit()
def post_comment_to_bug(self, bug_id, comment_text):
self.authenticate()
log("Adding comment to bug %s" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser['comment'] = comment_text
self.browser.submit()
def close_bug_as_fixed(self, bug_id, comment_text=None):
self.authenticate()
log("Closing bug %s as fixed" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser['bug_status'] = ['RESOLVED']
self.browser['resolution'] = ['FIXED']
self.browser.submit()
class StatusServer:
default_host = "webkit-commit-queue.appspot.com"
def __init__(self, host=default_host):
self.set_host(host)
self.browser = Browser()
def set_host(self, host):
self.host = host
self.url = "http://%s" % self.host
def results_url_for_status(self, status_id):
return "%s/results/%s" % (self.url, status_id)
def _add_patch(self, patch):
if not patch:
return
if patch.bug_id():
self.browser["bug_id"] = str(patch.bug_id())
if patch.id():
self.browser["patch_id"] = str(patch.id())
def _add_results_file(self, results_file):
if not results_file:
return
self.browser.add_file(results_file, "text/plain", "results.txt",
'results_file')
def _post_to_server(self, queue_name, status, patch, results_file):
if results_file:
# We might need to re-wind the file if we've already tried to post it.
results_file.seek(0)
update_status_url = "%s/update-status" % self.url
self.browser.open(update_status_url)
self.browser.select_form(name="update_status")
self.browser['queue_name'] = queue_name
self._add_patch(patch)
self.browser['status'] = status
self._add_results_file(results_file)
return self.browser.submit().read(
) # This is the id of the newly created status object.
def update_status(self, queue_name, status, patch=None, results_file=None):
# During unit testing, host is None
if not self.host:
return
log(status)
return NetworkTransaction().run(lambda: self._post_to_server(
queue_name, status, patch, results_file))
def patch_status(self, queue_name, patch_id):
update_status_url = "%s/patch-status/%s/%s" % (self.url, queue_name,
patch_id)
try:
return urllib2.urlopen(update_status_url).read()
except urllib2.HTTPError, e:
if e.code == 404:
return None
raise e
class Bugzilla:
def __init__(self, dryrun=False):
self.dryrun = dryrun
self.authenticated = False
self.browser = Browser()
# Ignore bugs.webkit.org/robots.txt until we fix it to allow this script
self.browser.set_handle_robots(False)
# Defaults (until we support better option parsing):
bug_server_host = "bugs.webkit.org"
bug_server_regex = "https?://%s/" % re.sub("\.", "\\.", bug_server_host)
bug_server_url = "https://%s/" % bug_server_host
# This could eventually be a text file
reviewer_usernames_to_full_names = {
"abarth": "Adam Barth",
"adele": "Adele Peterson",
"aroben": "Adam Roben",
"ap": "Alexey Proskuryakov",
"ariya.hidayat": "Ariya Hidayat",
"barraclough": "Gavin Barraclough",
"beidson": "Brady Eidson",
"darin": "Darin Adler",
"ddkilzer": "David Kilzer",
"dglazkov": "Dimitri Glazkov",
"eric": "Eric Seidel",
"fishd": "Darin Fisher",
"gns": "Gustavo Noronha",
"hausmann": "Simon Hausmann",
"hyatt": "David Hyatt",
"jmalonzo": "Jan Alonzo",
"justin.garcia": "Justin Garcia",
"kevino": "Kevin Ollivier",
"koivisto": "Antti Koivisto",
"levin": "David Levin",
"mitz": "Dan Bernstein",
"mjs": "Maciej Stachowiak",
"mrowe": "Mark Rowe",
"oliver": "Oliver Hunt",
"sam": "Sam Weinig",
"simon.fraser": "Simon Fraser",
"staikos": "George Staikos",
"timothy": "Timothy Hatcher",
"treat": "Adam Treat",
"vestbo": u"Tor Arne Vestb\xf8",
"xan.lopez": "Xan Lopez",
"zecke": "Holger Freyther",
"zimmermann": "Nikolas Zimmermann",
}
def full_name_from_bugzilla_name(self, bugzilla_name):
if not bugzilla_name in self.reviewer_usernames_to_full_names:
raise Exception("ERROR: Unknown reviewer! " + bugzilla_name)
return self.reviewer_usernames_to_full_names[bugzilla_name]
def bug_url_for_bug_id(self, bug_id, xml=False):
content_type = "&ctype=xml" if xml else ""
return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type)
def attachment_url_for_id(self, attachment_id, action="view"):
action_param = ""
if action and action != "view":
action_param = "&action=" + action
return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, attachment_id, action_param)
def fetch_attachments_from_bug(self, bug_id):
bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
log("Fetching: " + bug_url)
page = urllib2.urlopen(bug_url)
soup = BeautifulSoup(page)
attachments = []
for element in soup.findAll("attachment"):
attachment = {}
attachment["bug_id"] = bug_id
attachment["is_obsolete"] = element.has_key("isobsolete") and element["isobsolete"] == "1"
attachment["is_patch"] = element.has_key("ispatch") and element["ispatch"] == "1"
attachment["id"] = str(element.find("attachid").string)
attachment["url"] = self.attachment_url_for_id(attachment["id"])
attachment["name"] = unicode(element.find("desc").string)
attachment["type"] = str(element.find("type").string)
review_flag = element.find("flag", attrs={"name": "review"})
if review_flag and review_flag["status"] == "+":
reviewer_email = review_flag["setter"]
# We could lookup the full email address instead once we update full_name_from_bugzilla_name
bugzilla_name = reviewer_email.split("@")[0]
attachment["reviewer"] = self.full_name_from_bugzilla_name(bugzilla_name)
attachments.append(attachment)
return attachments
def fetch_patches_from_bug(self, bug_id):
patches = []
for attachment in self.fetch_attachments_from_bug(bug_id):
if attachment["is_patch"] and not attachment["is_obsolete"]:
patches.append(attachment)
return patches
def fetch_reviewed_patches_from_bug(self, bug_id):
reviewed_patches = []
for attachment in self.fetch_attachments_from_bug(bug_id):
if "reviewer" in attachment and not attachment["is_obsolete"]:
reviewed_patches.append(attachment)
return reviewed_patches
def fetch_bug_ids_from_commit_queue(self):
# FIXME: We should have an option for restricting the search by email. Example:
# unassigned_only = "&emailassigned_to1=1&emailtype1=substring&email1=unassigned"
commit_queue_url = (
self.bug_server_url
+ "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review%2B"
)
log("Loading commit queue")
page = urllib2.urlopen(commit_queue_url)
soup = BeautifulSoup(page)
bug_ids = []
# Grab the cells in the first column (which happens to be the bug ids)
for bug_link_cell in soup("td", "first-child"): # tds with the class "first-child"
bug_link = bug_link_cell.find("a")
bug_ids.append(bug_link.string) # the contents happen to be the bug id
return bug_ids
def fetch_patches_from_commit_queue(self):
patches_to_land = []
for bug_id in self.fetch_bug_ids_from_commit_queue():
patches = self.fetch_reviewed_patches_from_bug(bug_id)
patches_to_land += patches
return patches_to_land
def authenticate(self):
if self.authenticated:
return
if self.dryrun:
log("Skipping log in for dry run...")
self.authenticated = True
return
(username, password) = read_credentials()
log("Logging in as %s..." % username)
self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1")
self.browser.select_form(name="login")
self.browser["Bugzilla_login"] = username
self.browser["Bugzilla_password"] = password
response = self.browser.submit()
match = re.search("<title>(.+?)</title>", response.read())
# If the resulting page has a title, and it contains the word "invalid" assume it's the login failure page.
if match and re.search("Invalid", match.group(1), re.IGNORECASE):
# FIXME: We could add the ability to try again on failure.
error("Bugzilla login failed: %s" % match.group(1))
self.authenticated = True
def add_patch_to_bug(self, bug_id, patch_file_object, description, comment_text=None, mark_for_review=False):
self.authenticate()
log('Adding patch "%s" to bug %s' % (description, bug_id))
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_server_url + "attachment.cgi?action=enter&bugid=" + bug_id)
self.browser.select_form(name="entryform")
self.browser["description"] = description
self.browser["ispatch"] = ("1",)
if comment_text:
log(comment_text)
self.browser["comment"] = comment_text
self.browser["flag_type-1"] = ("?",) if mark_for_review else ("X",)
self.browser.add_file(patch_file_object, "text/plain", "bug-%s-%s.patch" % (bug_id, timestamp()))
self.browser.submit()
def obsolete_attachment(self, attachment_id, comment_text=None):
self.authenticate()
log("Obsoleting attachment: %s" % attachment_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.attachment_url_for_id(attachment_id, "edit"))
self.browser.select_form(nr=1)
self.browser.find_control("isobsolete").items[0].selected = True
# Also clear any review flag (to remove it from review/commit queues)
self.browser.find_control(type="select", nr=0).value = ("X",)
if comment_text:
log(comment_text)
# Bugzilla has two textareas named 'comment', one is somehow hidden. We want the first.
self.browser.set_value(comment_text, name="comment", nr=0)
self.browser.submit()
def post_comment_to_bug(self, bug_id, comment_text):
self.authenticate()
log("Adding comment to bug %s" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser["comment"] = comment_text
self.browser.submit()
def close_bug_as_fixed(self, bug_id, comment_text=None):
self.authenticate()
log("Closing bug %s as fixed" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser["comment"] = comment_text
self.browser["bug_status"] = ["RESOLVED"]
self.browser["resolution"] = ["FIXED"]
self.browser.submit()
class Bugzilla:
def __init__(self, dryrun=False):
self.dryrun = dryrun
self.authenticated = False
self.browser = Browser()
# Ignore bugs.webkit.org/robots.txt until we fix it to allow this script
self.browser.set_handle_robots(False)
# Defaults (until we support better option parsing):
bug_server_host = "bugs.webkit.org"
bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
bug_server_url = "https://%s/" % bug_server_host
# This could eventually be a text file
reviewer_usernames_to_full_names = {
"abarth": "Adam Barth",
"adele": "Adele Peterson",
"aroben": "Adam Roben",
"ap": "Alexey Proskuryakov",
"ariya.hidayat": "Ariya Hidayat",
"barraclough": "Gavin Barraclough",
"beidson": "Brady Eidson",
"darin": "Darin Adler",
"ddkilzer": "David Kilzer",
"dglazkov": "Dimitri Glazkov",
"eric": "Eric Seidel",
"fishd": "Darin Fisher",
"gns": "Gustavo Noronha",
"hausmann": "Simon Hausmann",
"hyatt": "David Hyatt",
"jmalonzo": "Jan Alonzo",
"justin.garcia": "Justin Garcia",
"kevino": "Kevin Ollivier",
"koivisto": "Antti Koivisto",
"levin": "David Levin",
"mitz": "Dan Bernstein",
"mjs": "Maciej Stachowiak",
"mrowe": "Mark Rowe",
"oliver": "Oliver Hunt",
"sam": "Sam Weinig",
"simon.fraser": "Simon Fraser",
"staikos": "George Staikos",
"timothy": "Timothy Hatcher",
"treat": "Adam Treat",
"vestbo": u'Tor Arne Vestb\xf8',
"xan.lopez": "Xan Lopez",
"zecke": "Holger Freyther",
"zimmermann": "Nikolas Zimmermann",
}
def full_name_from_bugzilla_name(self, bugzilla_name):
if not bugzilla_name in self.reviewer_usernames_to_full_names:
raise Exception("ERROR: Unknown reviewer! " + bugzilla_name)
return self.reviewer_usernames_to_full_names[bugzilla_name]
def bug_url_for_bug_id(self, bug_id, xml=False):
content_type = "&ctype=xml" if xml else ""
return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id,
content_type)
def attachment_url_for_id(self, attachment_id, action="view"):
action_param = ""
if action and action != "view":
action_param = "&action=" + action
return "%sattachment.cgi?id=%s%s" % (self.bug_server_url,
attachment_id, action_param)
def fetch_attachments_from_bug(self, bug_id):
bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
log("Fetching: " + bug_url)
page = urllib2.urlopen(bug_url)
soup = BeautifulSoup(page)
attachments = []
for element in soup.findAll('attachment'):
attachment = {}
attachment['bug_id'] = bug_id
attachment['is_obsolete'] = (element.has_key('isobsolete')
and element['isobsolete'] == "1")
attachment['is_patch'] = (element.has_key('ispatch')
and element['ispatch'] == "1")
attachment['id'] = str(element.find('attachid').string)
attachment['url'] = self.attachment_url_for_id(attachment['id'])
attachment['name'] = unicode(element.find('desc').string)
attachment['type'] = str(element.find('type').string)
review_flag = element.find('flag', attrs={"name": "review"})
if review_flag and review_flag['status'] == '+':
reviewer_email = review_flag['setter']
# We could lookup the full email address instead once we update full_name_from_bugzilla_name
bugzilla_name = reviewer_email.split('@')[0]
attachment['reviewer'] = self.full_name_from_bugzilla_name(
bugzilla_name)
attachments.append(attachment)
return attachments
def fetch_patches_from_bug(self, bug_id):
patches = []
for attachment in self.fetch_attachments_from_bug(bug_id):
if attachment['is_patch'] and not attachment['is_obsolete']:
patches.append(attachment)
return patches
def fetch_reviewed_patches_from_bug(self, bug_id):
reviewed_patches = []
for attachment in self.fetch_attachments_from_bug(bug_id):
if 'reviewer' in attachment and not attachment['is_obsolete']:
reviewed_patches.append(attachment)
return reviewed_patches
def fetch_bug_ids_from_commit_queue(self):
# FIXME: We should have an option for restricting the search by email. Example:
# unassigned_only = "&emailassigned_to1=1&emailtype1=substring&email1=unassigned"
commit_queue_url = self.bug_server_url + "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review%2B"
log("Loading commit queue")
page = urllib2.urlopen(commit_queue_url)
soup = BeautifulSoup(page)
bug_ids = []
# Grab the cells in the first column (which happens to be the bug ids)
for bug_link_cell in soup(
'td', "first-child"): # tds with the class "first-child"
bug_link = bug_link_cell.find("a")
bug_ids.append(
bug_link.string) # the contents happen to be the bug id
return bug_ids
def fetch_patches_from_commit_queue(self):
patches_to_land = []
for bug_id in self.fetch_bug_ids_from_commit_queue():
patches = self.fetch_reviewed_patches_from_bug(bug_id)
patches_to_land += patches
return patches_to_land
def authenticate(self):
if self.authenticated:
return
if self.dryrun:
log("Skipping log in for dry run...")
self.authenticated = True
return
(username, password) = read_credentials()
log("Logging in as %s..." % username)
self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1")
self.browser.select_form(name="login")
self.browser['Bugzilla_login'] = username
self.browser['Bugzilla_password'] = password
response = self.browser.submit()
match = re.search("<title>(.+?)</title>", response.read())
# If the resulting page has a title, and it contains the word "invalid" assume it's the login failure page.
if match and re.search("Invalid", match.group(1), re.IGNORECASE):
# FIXME: We could add the ability to try again on failure.
error("Bugzilla login failed: %s" % match.group(1))
self.authenticated = True
def add_patch_to_bug(self,
bug_id,
patch_file_object,
description,
comment_text=None,
mark_for_review=False):
self.authenticate()
log('Adding patch "%s" to bug %s' % (description, bug_id))
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_server_url +
"attachment.cgi?action=enter&bugid=" + bug_id)
self.browser.select_form(name="entryform")
self.browser['description'] = description
self.browser['ispatch'] = ("1", )
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser['flag_type-1'] = ('?', ) if mark_for_review else ('X', )
self.browser.add_file(patch_file_object, "text/plain",
"bug-%s-%s.patch" % (bug_id, timestamp()))
self.browser.submit()
def obsolete_attachment(self, attachment_id, comment_text=None):
self.authenticate()
log("Obsoleting attachment: %s" % attachment_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.find_control('isobsolete').items[0].selected = True
# Also clear any review flag (to remove it from review/commit queues)
self.browser.find_control(type='select', nr=0).value = ("X", )
if comment_text:
log(comment_text)
# Bugzilla has two textareas named 'comment', one is somehow hidden. We want the first.
self.browser.set_value(comment_text, name='comment', nr=0)
self.browser.submit()
def post_comment_to_bug(self, bug_id, comment_text):
self.authenticate()
log("Adding comment to bug %s" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser['comment'] = comment_text
self.browser.submit()
def close_bug_as_fixed(self, bug_id, comment_text=None):
self.authenticate()
log("Closing bug %s as fixed" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser['bug_status'] = ['RESOLVED']
self.browser['resolution'] = ['FIXED']
self.browser.submit()
password = None
username = args.username if args.username is not None else username
password = args.password if args.password is not None else password
if any(
(args.file is None, username is None, password is None, args.url is None)):
parser.print_help()
exit()
br = Browser()
cj = cookielib.LWPCookieJar()
br.set_cookiejar(cj)
br.open('https://www.kaggle.com/account/login')
br.select_form(nr=0)
br['UserName'] = username
br['Password'] = password
br.submit(nr=0)
br.open(args.url)
br.select_form(nr=0)
br.add_file(open(args.file),
'application/octet-stream',
os.path.basename(args.file),
name='SubmissionUpload')
if args.description is not None:
br['SubmissionDescription'] = args.description
br.submit(nr=0)
class Bugzilla:
def __init__(self, dryrun=False, committers=CommitterList()):
self.dryrun = dryrun
self.authenticated = False
self.browser = Browser()
# Ignore bugs.webkit.org/robots.txt until we fix it to allow this script
self.browser.set_handle_robots(False)
self.committers = committers
# Defaults (until we support better option parsing):
bug_server_host = "bugs.webkit.org"
bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
bug_server_url = "https://%s/" % bug_server_host
def bug_url_for_bug_id(self, bug_id, xml=False):
content_type = "&ctype=xml" if xml else ""
return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type)
def short_bug_url_for_bug_id(self, bug_id):
return "http://webkit.org/b/%s" % bug_id
def attachment_url_for_id(self, attachment_id, action="view"):
action_param = ""
if action and action != "view":
action_param = "&action=%s" % action
return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, attachment_id, action_param)
def _parse_attachment_flag(self, element, flag_name, attachment, result_key):
flag = element.find('flag', attrs={'name' : flag_name})
if flag and flag['status'] == '+':
attachment[result_key] = flag['setter']
def _parse_attachment_element(self, element, bug_id):
attachment = {}
attachment['bug_id'] = bug_id
attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
attachment['id'] = str(element.find('attachid').string)
attachment['url'] = self.attachment_url_for_id(attachment['id'])
attachment['name'] = unicode(element.find('desc').string)
attachment['type'] = str(element.find('type').string)
self._parse_attachment_flag(element, 'review', attachment, 'reviewer_email')
self._parse_attachment_flag(element, 'commit-queue', attachment, 'committer_email')
return attachment
def fetch_attachments_from_bug(self, bug_id):
bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
log("Fetching: %s" % bug_url)
page = urllib2.urlopen(bug_url)
soup = BeautifulSoup(page)
attachments = []
for element in soup.findAll('attachment'):
attachment = self._parse_attachment_element(element, bug_id)
attachments.append(attachment)
return attachments
def fetch_title_from_bug(self, bug_id):
bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
page = urllib2.urlopen(bug_url)
soup = BeautifulSoup(page)
return soup.find('short_desc').string
def fetch_patches_from_bug(self, bug_id):
patches = []
for attachment in self.fetch_attachments_from_bug(bug_id):
if attachment['is_patch'] and not attachment['is_obsolete']:
patches.append(attachment)
return patches
# _view_source_link belongs in some sort of webkit_config.py module.
def _view_source_link(self, local_path):
return "http://trac.webkit.org/browser/trunk/%s" % local_path
def _validate_setter_email(self, patch, result_key, lookup_function, rejection_function, reject_invalid_patches):
setter_email = patch.get(result_key + '_email')
if not setter_email:
return None
committer = lookup_function(setter_email)
if committer:
patch[result_key] = committer.full_name
return patch[result_key]
if reject_invalid_patches:
committer_list = "WebKitTools/Scripts/modules/committers.py"
failure_message = "%s does not have %s permissions according to %s." % (setter_email, result_key, self._view_source_link(committer_list))
rejection_function(patch['id'], failure_message)
else:
log("Warning, attachment %s on bug %s has invalid %s (%s)", (patch['id'], patch['bug_id'], result_key, setter_email))
return None
def _validate_reviewer(self, patch, reject_invalid_patches):
return self._validate_setter_email(patch, 'reviewer', self.committers.reviewer_by_bugzilla_email, self.reject_patch_from_review_queue, reject_invalid_patches)
def _validate_committer(self, patch, reject_invalid_patches):
return self._validate_setter_email(patch, 'committer', self.committers.committer_by_bugzilla_email, self.reject_patch_from_commit_queue, reject_invalid_patches)
def fetch_reviewed_patches_from_bug(self, bug_id, reject_invalid_patches=False):
reviewed_patches = []
for attachment in self.fetch_attachments_from_bug(bug_id):
if self._validate_reviewer(attachment, reject_invalid_patches) and not attachment['is_obsolete']:
reviewed_patches.append(attachment)
return reviewed_patches
def fetch_commit_queue_patches_from_bug(self, bug_id, reject_invalid_patches=False):
commit_queue_patches = []
for attachment in self.fetch_reviewed_patches_from_bug(bug_id, reject_invalid_patches):
if self._validate_committer(attachment, reject_invalid_patches) and not attachment['is_obsolete']:
commit_queue_patches.append(attachment)
return commit_queue_patches
def fetch_bug_ids_from_commit_queue(self):
commit_queue_url = self.bug_server_url + "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=commit-queue%2B"
page = urllib2.urlopen(commit_queue_url)
soup = BeautifulSoup(page)
bug_ids = []
# Grab the cells in the first column (which happens to be the bug ids)
for bug_link_cell in soup('td', "first-child"): # tds with the class "first-child"
bug_link = bug_link_cell.find("a")
bug_ids.append(bug_link.string) # the contents happen to be the bug id
return bug_ids
def fetch_patches_from_commit_queue(self, reject_invalid_patches=False):
patches_to_land = []
for bug_id in self.fetch_bug_ids_from_commit_queue():
patches = self.fetch_commit_queue_patches_from_bug(bug_id, reject_invalid_patches)
patches_to_land += patches
return patches_to_land
def authenticate(self):
if self.authenticated:
return
if self.dryrun:
log("Skipping log in for dry run...")
self.authenticated = True
return
(username, password) = read_credentials()
log("Logging in as %s..." % username)
self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1")
self.browser.select_form(name="login")
self.browser['Bugzilla_login'] = username
self.browser['Bugzilla_password'] = password
response = self.browser.submit()
match = re.search("<title>(.+?)</title>", response.read())
# If the resulting page has a title, and it contains the word "invalid" assume it's the login failure page.
if match and re.search("Invalid", match.group(1), re.IGNORECASE):
# FIXME: We could add the ability to try again on failure.
raise BugzillaError("Bugzilla login failed: %s" % match.group(1))
self.authenticated = True
def add_patch_to_bug(self, bug_id, patch_file_object, description, comment_text=None, mark_for_review=False):
self.authenticate()
log('Adding patch "%s" to bug %s' % (description, bug_id))
if self.dryrun:
log(comment_text)
return
self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % (self.bug_server_url, bug_id))
self.browser.select_form(name="entryform")
self.browser['description'] = description
self.browser['ispatch'] = ("1",)
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
self.browser.add_file(patch_file_object, "text/plain", "bug-%s-%s.patch" % (bug_id, timestamp()))
self.browser.submit()
def prompt_for_component(self, components):
log("Please pick a component:")
i = 0
for name in components:
i += 1
log("%2d. %s" % (i, name))
result = int(raw_input("Enter a number: ")) - 1
return components[result]
def _check_create_bug_response(self, response_html):
match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>", response_html)
if match:
return match.group('bug_id')
match = re.search('<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">', response_html, re.DOTALL)
error_message = "FAIL"
if match:
text_lines = BeautifulSoup(match.group('error_message')).findAll(text=True)
error_message = "\n" + '\n'.join([" " + line.strip() for line in text_lines if line.strip()])
raise BugzillaError("Bug not created: %s" % error_message)
def create_bug_with_patch(self, bug_title, bug_description, component, patch_file_object, patch_description, cc, mark_for_review=False):
self.authenticate()
log('Creating bug with patch description "%s"' % patch_description)
if self.dryrun:
log(bug_description)
return
self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit")
self.browser.select_form(name="Create")
component_items = self.browser.find_control('component').items
component_names = map(lambda item: item.name, component_items)
if not component or component not in component_names:
component = self.prompt_for_component(component_names)
self.browser['component'] = [component]
if cc:
self.browser['cc'] = cc
self.browser['short_desc'] = bug_title
if bug_description:
log(bug_description)
self.browser['comment'] = bug_description
self.browser['description'] = patch_description
self.browser['ispatch'] = ("1",)
self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
self.browser.add_file(patch_file_object, "text/plain", "%s.patch" % timestamp(), 'data')
response = self.browser.submit()
bug_id = self._check_create_bug_response(response.read())
log("Bug %s created." % bug_id)
log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id))
return bug_id
def _find_select_element_for_flag(self, flag_name):
# FIXME: This will break if we ever re-order attachment flags
if flag_name == "review":
return self.browser.find_control(type='select', nr=0)
if flag_name == "commit-queue":
return self.browser.find_control(type='select', nr=1)
raise Exception("Don't know how to find flag named \"%s\"" % flag_name)
def clear_attachment_flags(self, attachment_id, additional_comment_text=None):
self.authenticate()
comment_text = "Clearing flags on attachment: %s" % attachment_id
if additional_comment_text:
comment_text += "\n\n%s" % additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.set_value(comment_text, name='comment', nr=0)
self._find_select_element_for_flag('review').value = ("X",)
self._find_select_element_for_flag('commit-queue').value = ("X",)
self.browser.submit()
# FIXME: We need a way to test this on a live bugzilla instance.
def _set_flag_on_attachment(self, attachment_id, flag_name, flag_value, comment_text, additional_comment_text):
self.authenticate()
if additional_comment_text:
comment_text += "\n\n%s" % additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.set_value(comment_text, name='comment', nr=0)
self._find_select_element_for_flag(flag_name).value = (flag_value,)
self.browser.submit()
def reject_patch_from_commit_queue(self, attachment_id, additional_comment_text=None):
comment_text = "Rejecting patch %s from commit-queue." % attachment_id
self._set_flag_on_attachment(attachment_id, 'commit-queue', '-', comment_text, additional_comment_text)
def reject_patch_from_review_queue(self, attachment_id, additional_comment_text=None):
comment_text = "Rejecting patch %s from review queue." % attachment_id
self._set_flag_on_attachment(attachment_id, 'review', '-', comment_text, additional_comment_text)
def obsolete_attachment(self, attachment_id, comment_text = None):
self.authenticate()
log("Obsoleting attachment: %s" % attachment_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.find_control('isobsolete').items[0].selected = True
# Also clear any review flag (to remove it from review/commit queues)
self._find_select_element_for_flag('review').value = ("X",)
self._find_select_element_for_flag('commit-queue').value = ("X",)
if comment_text:
log(comment_text)
# Bugzilla has two textareas named 'comment', one is somehow hidden. We want the first.
self.browser.set_value(comment_text, name='comment', nr=0)
self.browser.submit()
def post_comment_to_bug(self, bug_id, comment_text):
self.authenticate()
log("Adding comment to bug %s" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser['comment'] = comment_text
self.browser.submit()
def close_bug_as_fixed(self, bug_id, comment_text=None):
self.authenticate()
log("Closing bug %s as fixed" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser['bug_status'] = ['RESOLVED']
self.browser['resolution'] = ['FIXED']
self.browser.submit()
def reopen_bug(self, bug_id, comment_text):
self.authenticate()
log("Re-opening bug %s" % bug_id)
log(comment_text) # Bugzilla requires a comment when re-opening a bug, so we know it will never be None.
if self.dryrun:
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser['bug_status'] = ['REOPENED']
self.browser['comment'] = comment_text
self.browser.submit()
class Bugzilla(object):
def __init__(self, dryrun=False, committers=CommitterList()):
self.dryrun = dryrun
self.authenticated = False
self.queries = BugzillaQueries(self)
self.committers = committers
# FIXME: We should use some sort of Browser mock object when in dryrun
# mode (to prevent any mistakes).
self.browser = Browser()
# Ignore bugs.webkit.org/robots.txt until we fix it to allow this
# script.
self.browser.set_handle_robots(False)
# FIXME: Much of this should go into some sort of config module:
bug_server_host = "bugs.webkit.org"
bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
bug_server_url = "https://%s/" % bug_server_host
unassigned_email = "*****@*****.**"
def bug_url_for_bug_id(self, bug_id, xml=False):
content_type = "&ctype=xml" if xml else ""
return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url,
bug_id,
content_type)
def short_bug_url_for_bug_id(self, bug_id):
return "http://webkit.org/b/%s" % bug_id
def attachment_url_for_id(self, attachment_id, action="view"):
action_param = ""
if action and action != "view":
action_param = "&action=%s" % action
return "%sattachment.cgi?id=%s%s" % (self.bug_server_url,
attachment_id,
action_param)
def _parse_attachment_flag(self,
element,
flag_name,
attachment,
result_key):
flag = element.find('flag', attrs={'name': flag_name})
if flag:
attachment[flag_name] = flag['status']
if flag['status'] == '+':
attachment[result_key] = flag['setter']
def _parse_attachment_element(self, element, bug_id):
attachment = {}
attachment['bug_id'] = bug_id
attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
attachment['id'] = int(element.find('attachid').string)
# FIXME: No need to parse out the url here.
attachment['url'] = self.attachment_url_for_id(attachment['id'])
attachment['name'] = unicode(element.find('desc').string)
attachment['attacher_email'] = str(element.find('attacher').string)
attachment['type'] = str(element.find('type').string)
self._parse_attachment_flag(
element, 'review', attachment, 'reviewer_email')
self._parse_attachment_flag(
element, 'commit-queue', attachment, 'committer_email')
return attachment
def _parse_bug_page(self, page):
soup = BeautifulSoup(page)
bug = {}
bug["id"] = int(soup.find("bug_id").string)
bug["title"] = unicode(soup.find("short_desc").string)
bug["reporter_email"] = str(soup.find("reporter").string)
bug["assigned_to_email"] = str(soup.find("assigned_to").string)
bug["cc_emails"] = [str(element.string)
for element in soup.findAll('cc')]
bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')]
return bug
# Makes testing fetch_*_from_bug() possible until we have a better
# BugzillaNetwork abstration.
def _fetch_bug_page(self, bug_id):
bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
log("Fetching: %s" % bug_url)
return self.browser.open(bug_url)
def fetch_bug_dictionary(self, bug_id):
return self._parse_bug_page(self._fetch_bug_page(bug_id))
# FIXME: A BugzillaCache object should provide all these fetch_ methods.
def fetch_bug(self, bug_id):
return Bug(self.fetch_bug_dictionary(bug_id), self)
def _parse_bug_id_from_attachment_page(self, page):
# The "Up" relation happens to point to the bug.
up_link = BeautifulSoup(page).find('link', rel='Up')
if not up_link:
# This attachment does not exist (or you don't have permissions to
# view it).
return None
match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href'])
return int(match.group('bug_id'))
def bug_id_for_attachment_id(self, attachment_id):
self.authenticate()
attachment_url = self.attachment_url_for_id(attachment_id, 'edit')
log("Fetching: %s" % attachment_url)
page = self.browser.open(attachment_url)
return self._parse_bug_id_from_attachment_page(page)
# FIXME: This should just return Attachment(id), which should be able to
# lazily fetch needed data.
def fetch_attachment(self, attachment_id):
# We could grab all the attachment details off of the attachment edit
# page but we already have working code to do so off of the bugs page,
# so re-use that.
bug_id = self.bug_id_for_attachment_id(attachment_id)
if not bug_id:
return None
attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True)
for attachment in attachments:
if attachment.id() == int(attachment_id):
return attachment
return None # This should never be hit.
def authenticate(self):
if self.authenticated:
return
if self.dryrun:
log("Skipping log in for dry run...")
self.authenticated = True
return
attempts = 0
while not self.authenticated:
attempts += 1
(username, password) = Credentials(
self.bug_server_host, git_prefix="bugzilla").read_credentials()
log("Logging in as %s..." % username)
self.browser.open(self.bug_server_url +
"index.cgi?GoAheadAndLogIn=1")
self.browser.select_form(name="login")
self.browser['Bugzilla_login'] = username
self.browser['Bugzilla_password'] = password
response = self.browser.submit()
match = re.search("<title>(.+?)</title>", response.read())
# If the resulting page has a title, and it contains the word
# "invalid" assume it's the login failure page.
if match and re.search("Invalid", match.group(1), re.IGNORECASE):
errorMessage = "Bugzilla login failed: %s" % match.group(1)
# raise an exception only if this was the last attempt
if attempts < 5:
log(errorMessage)
else:
raise Exception(errorMessage)
else:
self.authenticated = True
def _fill_attachment_form(self,
description,
patch_file_object,
comment_text=None,
mark_for_review=False,
mark_for_commit_queue=False,
mark_for_landing=False, bug_id=None):
self.browser['description'] = description
self.browser['ispatch'] = ("1",)
self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
if mark_for_landing:
self.browser['flag_type-3'] = ('+',)
elif mark_for_commit_queue:
self.browser['flag_type-3'] = ('?',)
else:
self.browser['flag_type-3'] = ('X',)
if bug_id:
patch_name = "bug-%s-%s.patch" % (bug_id, timestamp())
else:
patch_name ="%s.patch" % timestamp()
self.browser.add_file(patch_file_object,
"text/plain",
patch_name,
'data')
def add_patch_to_bug(self,
bug_id,
patch_file_object,
description,
comment_text=None,
mark_for_review=False,
mark_for_commit_queue=False,
mark_for_landing=False):
self.authenticate()
log('Adding patch "%s" to %sshow_bug.cgi?id=%s' % (description,
self.bug_server_url,
bug_id))
if self.dryrun:
log(comment_text)
return
self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % (
self.bug_server_url, bug_id))
self.browser.select_form(name="entryform")
self._fill_attachment_form(description,
patch_file_object,
mark_for_review=mark_for_review,
mark_for_commit_queue=mark_for_commit_queue,
mark_for_landing=mark_for_landing,
bug_id=bug_id)
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser.submit()
def prompt_for_component(self, components):
log("Please pick a component:")
i = 0
for name in components:
i += 1
log("%2d. %s" % (i, name))
result = int(User.prompt("Enter a number: ")) - 1
return components[result]
def _check_create_bug_response(self, response_html):
match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>",
response_html)
if match:
return match.group('bug_id')
match = re.search(
'<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">',
response_html,
re.DOTALL)
error_message = "FAIL"
if match:
text_lines = BeautifulSoup(
match.group('error_message')).findAll(text=True)
error_message = "\n" + '\n'.join(
[" " + line.strip()
for line in text_lines if line.strip()])
raise Exception("Bug not created: %s" % error_message)
def create_bug(self,
bug_title,
bug_description,
component=None,
patch_file_object=None,
patch_description=None,
cc=None,
mark_for_review=False,
mark_for_commit_queue=False):
self.authenticate()
log('Creating bug with title "%s"' % bug_title)
if self.dryrun:
log(bug_description)
return
self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit")
self.browser.select_form(name="Create")
component_items = self.browser.find_control('component').items
component_names = map(lambda item: item.name, component_items)
if not component:
component = "New Bugs"
if component not in component_names:
component = self.prompt_for_component(component_names)
self.browser['component'] = [component]
if cc:
self.browser['cc'] = cc
self.browser['short_desc'] = bug_title
self.browser['comment'] = bug_description
if patch_file_object:
self._fill_attachment_form(
patch_description,
patch_file_object,
mark_for_review=mark_for_review,
mark_for_commit_queue=mark_for_commit_queue)
response = self.browser.submit()
bug_id = self._check_create_bug_response(response.read())
log("Bug %s created." % bug_id)
log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id))
return bug_id
def _find_select_element_for_flag(self, flag_name):
# FIXME: This will break if we ever re-order attachment flags
if flag_name == "review":
return self.browser.find_control(type='select', nr=0)
if flag_name == "commit-queue":
return self.browser.find_control(type='select', nr=1)
raise Exception("Don't know how to find flag named \"%s\"" % flag_name)
def clear_attachment_flags(self,
attachment_id,
additional_comment_text=None):
self.authenticate()
comment_text = "Clearing flags on attachment: %s" % attachment_id
if additional_comment_text:
comment_text += "\n\n%s" % additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.set_value(comment_text, name='comment', nr=0)
self._find_select_element_for_flag('review').value = ("X",)
self._find_select_element_for_flag('commit-queue').value = ("X",)
self.browser.submit()
def set_flag_on_attachment(self,
attachment_id,
flag_name,
flag_value,
comment_text,
additional_comment_text):
# FIXME: We need a way to test this function on a live bugzilla
# instance.
self.authenticate()
if additional_comment_text:
comment_text += "\n\n%s" % additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.set_value(comment_text, name='comment', nr=0)
self._find_select_element_for_flag(flag_name).value = (flag_value,)
self.browser.submit()
# FIXME: All of these bug editing methods have a ridiculous amount of
# copy/paste code.
def obsolete_attachment(self, attachment_id, comment_text=None):
self.authenticate()
log("Obsoleting attachment: %s" % attachment_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.find_control('isobsolete').items[0].selected = True
# Also clear any review flag (to remove it from review/commit queues)
self._find_select_element_for_flag('review').value = ("X",)
self._find_select_element_for_flag('commit-queue').value = ("X",)
if comment_text:
log(comment_text)
# Bugzilla has two textareas named 'comment', one is somehow
# hidden. We want the first.
self.browser.set_value(comment_text, name='comment', nr=0)
self.browser.submit()
def add_cc_to_bug(self, bug_id, email_address_list):
self.authenticate()
log("Adding %s to the CC list for bug %s" % (email_address_list,
bug_id))
if self.dryrun:
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser["newcc"] = ", ".join(email_address_list)
self.browser.submit()
def post_comment_to_bug(self, bug_id, comment_text, cc=None):
self.authenticate()
log("Adding comment to bug %s" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser["comment"] = comment_text
if cc:
self.browser["newcc"] = ", ".join(cc)
self.browser.submit()
def close_bug_as_fixed(self, bug_id, comment_text=None):
self.authenticate()
log("Closing bug %s as fixed" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser['bug_status'] = ['RESOLVED']
self.browser['resolution'] = ['FIXED']
self.browser.submit()
def reassign_bug(self, bug_id, assignee, comment_text=None):
self.authenticate()
log("Assigning bug %s to %s" % (bug_id, assignee))
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser["comment"] = comment_text
self.browser["assigned_to"] = assignee
self.browser.submit()
def reopen_bug(self, bug_id, comment_text):
self.authenticate()
log("Re-opening bug %s" % bug_id)
# Bugzilla requires a comment when re-opening a bug, so we know it will
# never be None.
log(comment_text)
if self.dryrun:
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
bug_status = self.browser.find_control("bug_status", type="select")
# This is a hack around the fact that ClientForm.ListControl seems to
# have no simpler way to ask if a control has an item named "REOPENED"
# without using exceptions for control flow.
possible_bug_statuses = map(lambda item: item.name, bug_status.items)
if "REOPENED" in possible_bug_statuses:
bug_status.value = ["REOPENED"]
else:
log("Did not reopen bug %s. " +
"It appears to already be open with status %s." % (
bug_id, bug_status.value))
self.browser['comment'] = comment_text
self.browser.submit()
class Bugzilla(object):
def __init__(self, dryrun=False, committers=CommitterList()):
self.dryrun = dryrun
self.authenticated = False
self.queries = BugzillaQueries(self)
self.committers = committers
# FIXME: We should use some sort of Browser mock object when in dryrun
# mode (to prevent any mistakes).
self.browser = Browser()
# Ignore bugs.webkit.org/robots.txt until we fix it to allow this
# script.
self.browser.set_handle_robots(False)
# FIXME: Much of this should go into some sort of config module:
bug_server_host = "bugs.webkit.org"
bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
bug_server_url = "https://%s/" % bug_server_host
unassigned_email = "*****@*****.**"
def bug_url_for_bug_id(self, bug_id, xml=False):
content_type = "&ctype=xml" if xml else ""
return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url,
bug_id,
content_type)
def short_bug_url_for_bug_id(self, bug_id):
return "http://webkit.org/b/%s" % bug_id
def attachment_url_for_id(self, attachment_id, action="view"):
action_param = ""
if action and action != "view":
action_param = "&action=%s" % action
return "%sattachment.cgi?id=%s%s" % (self.bug_server_url,
attachment_id,
action_param)
def _parse_attachment_flag(self,
element,
flag_name,
attachment,
result_key):
flag = element.find('flag', attrs={'name': flag_name})
if flag:
attachment[flag_name] = flag['status']
if flag['status'] == '+':
attachment[result_key] = flag['setter']
def _parse_attachment_element(self, element, bug_id):
attachment = {}
attachment['bug_id'] = bug_id
attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
attachment['id'] = int(element.find('attachid').string)
# FIXME: No need to parse out the url here.
attachment['url'] = self.attachment_url_for_id(attachment['id'])
attachment['name'] = unicode(element.find('desc').string)
attachment['attacher_email'] = str(element.find('attacher').string)
attachment['type'] = str(element.find('type').string)
self._parse_attachment_flag(
element, 'review', attachment, 'reviewer_email')
self._parse_attachment_flag(
element, 'commit-queue', attachment, 'committer_email')
return attachment
def _parse_bug_page(self, page):
soup = BeautifulSoup(page)
bug = {}
bug["id"] = int(soup.find("bug_id").string)
bug["title"] = unicode(soup.find("short_desc").string)
bug["reporter_email"] = str(soup.find("reporter").string)
bug["assigned_to_email"] = str(soup.find("assigned_to").string)
bug["cc_emails"] = [str(element.string)
for element in soup.findAll('cc')]
bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')]
return bug
# Makes testing fetch_*_from_bug() possible until we have a better
# BugzillaNetwork abstration.
def _fetch_bug_page(self, bug_id):
bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
log("Fetching: %s" % bug_url)
return self.browser.open(bug_url)
def fetch_bug_dictionary(self, bug_id):
return self._parse_bug_page(self._fetch_bug_page(bug_id))
# FIXME: A BugzillaCache object should provide all these fetch_ methods.
def fetch_bug(self, bug_id):
return Bug(self.fetch_bug_dictionary(bug_id), self)
def _parse_bug_id_from_attachment_page(self, page):
# The "Up" relation happens to point to the bug.
up_link = BeautifulSoup(page).find('link', rel='Up')
if not up_link:
# This attachment does not exist (or you don't have permissions to
# view it).
return None
match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href'])
return int(match.group('bug_id'))
def bug_id_for_attachment_id(self, attachment_id):
self.authenticate()
attachment_url = self.attachment_url_for_id(attachment_id, 'edit')
log("Fetching: %s" % attachment_url)
page = self.browser.open(attachment_url)
return self._parse_bug_id_from_attachment_page(page)
# FIXME: This should just return Attachment(id), which should be able to
# lazily fetch needed data.
def fetch_attachment(self, attachment_id):
# We could grab all the attachment details off of the attachment edit
# page but we already have working code to do so off of the bugs page,
# so re-use that.
bug_id = self.bug_id_for_attachment_id(attachment_id)
if not bug_id:
return None
attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True)
for attachment in attachments:
if attachment.id() == int(attachment_id):
return attachment
return None # This should never be hit.
def authenticate(self):
if self.authenticated:
return
if self.dryrun:
log("Skipping log in for dry run...")
self.authenticated = True
return
attempts = 0
while not self.authenticated:
attempts += 1
(username, password) = Credentials(
self.bug_server_host, git_prefix="bugzilla").read_credentials()
log("Logging in as %s..." % username)
self.browser.open(self.bug_server_url +
"index.cgi?GoAheadAndLogIn=1")
self.browser.select_form(name="login")
self.browser['Bugzilla_login'] = username
self.browser['Bugzilla_password'] = password
response = self.browser.submit()
match = re.search("<title>(.+?)</title>", response.read())
# If the resulting page has a title, and it contains the word
# "invalid" assume it's the login failure page.
if match and re.search("Invalid", match.group(1), re.IGNORECASE):
errorMessage = "Bugzilla login failed: %s" % match.group(1)
# raise an exception only if this was the last attempt
if attempts < 5:
log(errorMessage)
else:
raise Exception(errorMessage)
else:
self.authenticated = True
def _fill_attachment_form(self,
description,
patch_file_object,
comment_text=None,
mark_for_review=False,
mark_for_commit_queue=False,
mark_for_landing=False, bug_id=None):
self.browser['description'] = description
self.browser['ispatch'] = ("1",)
self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
if mark_for_landing:
self.browser['flag_type-3'] = ('+',)
elif mark_for_commit_queue:
self.browser['flag_type-3'] = ('?',)
else:
self.browser['flag_type-3'] = ('X',)
if bug_id:
patch_name = "bug-%s-%s.patch" % (bug_id, timestamp())
else:
patch_name ="%s.patch" % timestamp()
self.browser.add_file(patch_file_object,
"text/plain",
patch_name,
'data')
def add_patch_to_bug(self,
bug_id,
patch_file_object,
description,
comment_text=None,
mark_for_review=False,
mark_for_commit_queue=False,
mark_for_landing=False):
self.authenticate()
log('Adding patch "%s" to %sshow_bug.cgi?id=%s' % (description,
self.bug_server_url,
bug_id))
if self.dryrun:
log(comment_text)
return
self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % (
self.bug_server_url, bug_id))
self.browser.select_form(name="entryform")
self._fill_attachment_form(description,
patch_file_object,
mark_for_review=mark_for_review,
mark_for_commit_queue=mark_for_commit_queue,
mark_for_landing=mark_for_landing,
bug_id=bug_id)
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser.submit()
def prompt_for_component(self, components):
log("Please pick a component:")
i = 0
for name in components:
i += 1
log("%2d. %s" % (i, name))
result = int(User.prompt("Enter a number: ")) - 1
return components[result]
def _check_create_bug_response(self, response_html):
match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>",
response_html)
if match:
return match.group('bug_id')
match = re.search(
'<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">',
response_html,
re.DOTALL)
error_message = "FAIL"
if match:
text_lines = BeautifulSoup(
match.group('error_message')).findAll(text=True)
error_message = "\n" + '\n'.join(
[" " + line.strip()
for line in text_lines if line.strip()])
raise Exception("Bug not created: %s" % error_message)
def create_bug(self,
bug_title,
bug_description,
component=None,
patch_file_object=None,
patch_description=None,
cc=None,
mark_for_review=False,
mark_for_commit_queue=False):
self.authenticate()
log('Creating bug with title "%s"' % bug_title)
if self.dryrun:
log(bug_description)
return
self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit")
self.browser.select_form(name="Create")
component_items = self.browser.find_control('component').items
component_names = map(lambda item: item.name, component_items)
if not component:
component = "New Bugs"
if component not in component_names:
component = self.prompt_for_component(component_names)
self.browser['component'] = [component]
if cc:
self.browser['cc'] = cc
self.browser['short_desc'] = bug_title
self.browser['comment'] = bug_description
if patch_file_object:
self._fill_attachment_form(
patch_description,
patch_file_object,
mark_for_review=mark_for_review,
mark_for_commit_queue=mark_for_commit_queue)
response = self.browser.submit()
bug_id = self._check_create_bug_response(response.read())
log("Bug %s created." % bug_id)
log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id))
return bug_id
def _find_select_element_for_flag(self, flag_name):
# FIXME: This will break if we ever re-order attachment flags
if flag_name == "review":
return self.browser.find_control(type='select', nr=0)
if flag_name == "commit-queue":
return self.browser.find_control(type='select', nr=1)
raise Exception("Don't know how to find flag named \"%s\"" % flag_name)
def clear_attachment_flags(self,
attachment_id,
additional_comment_text=None):
self.authenticate()
comment_text = "Clearing flags on attachment: %s" % attachment_id
if additional_comment_text:
comment_text += "\n\n%s" % additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.set_value(comment_text, name='comment', nr=0)
self._find_select_element_for_flag('review').value = ("X",)
self._find_select_element_for_flag('commit-queue').value = ("X",)
self.browser.submit()
def set_flag_on_attachment(self,
attachment_id,
flag_name,
flag_value,
comment_text,
additional_comment_text):
# FIXME: We need a way to test this function on a live bugzilla
# instance.
self.authenticate()
if additional_comment_text:
comment_text += "\n\n%s" % additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.set_value(comment_text, name='comment', nr=0)
self._find_select_element_for_flag(flag_name).value = (flag_value,)
self.browser.submit()
# FIXME: All of these bug editing methods have a ridiculous amount of
# copy/paste code.
def obsolete_attachment(self, attachment_id, comment_text=None):
self.authenticate()
log("Obsoleting attachment: %s" % attachment_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.find_control('isobsolete').items[0].selected = True
# Also clear any review flag (to remove it from review/commit queues)
self._find_select_element_for_flag('review').value = ("X",)
self._find_select_element_for_flag('commit-queue').value = ("X",)
if comment_text:
log(comment_text)
# Bugzilla has two textareas named 'comment', one is somehow
# hidden. We want the first.
self.browser.set_value(comment_text, name='comment', nr=0)
self.browser.submit()
def add_cc_to_bug(self, bug_id, email_address_list):
self.authenticate()
log("Adding %s to the CC list for bug %s" % (email_address_list,
bug_id))
if self.dryrun:
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser["newcc"] = ", ".join(email_address_list)
self.browser.submit()
def post_comment_to_bug(self, bug_id, comment_text, cc=None):
self.authenticate()
log("Adding comment to bug %s" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser["comment"] = comment_text
if cc:
self.browser["newcc"] = ", ".join(cc)
self.browser.submit()
def close_bug_as_fixed(self, bug_id, comment_text=None):
self.authenticate()
log("Closing bug %s as fixed" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser['bug_status'] = ['RESOLVED']
self.browser['resolution'] = ['FIXED']
self.browser.submit()
def reassign_bug(self, bug_id, assignee, comment_text=None):
self.authenticate()
log("Assigning bug %s to %s" % (bug_id, assignee))
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser["comment"] = comment_text
self.browser["assigned_to"] = assignee
self.browser.submit()
def reopen_bug(self, bug_id, comment_text):
self.authenticate()
log("Re-opening bug %s" % bug_id)
# Bugzilla requires a comment when re-opening a bug, so we know it will
# never be None.
log(comment_text)
if self.dryrun:
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
bug_status = self.browser.find_control("bug_status", type="select")
# This is a hack around the fact that ClientForm.ListControl seems to
# have no simpler way to ask if a control has an item named "REOPENED"
# without using exceptions for control flow.
possible_bug_statuses = map(lambda item: item.name, bug_status.items)
if "REOPENED" in possible_bug_statuses:
bug_status.value = ["REOPENED"]
else:
log("Did not reopen bug %s. " +
"It appears to already be open with status %s." % (
bug_id, bug_status.value))
self.browser['comment'] = comment_text
self.browser.submit()
