class Bugz:
""" Converts sane method calls to Bugzilla HTTP requests.
@ivar base: base url of bugzilla.
@ivar user: username for authenticated operations.
@ivar password: password for authenticated operations
@ivar cookiejar: for authenticated sessions so we only auth once.
@ivar forget: forget user/password after session.
@ivar authenticated: is this session authenticated already
"""
def __init__(self, base, user = None, password = None, forget = False,
skip_auth = False, httpuser = None, httppassword = None ):
"""
{user} and {password} will be prompted if an action needs them
and they are not supplied.
if {forget} is set, the login cookie will be destroyed on quit.
@param base: base url of the bugzilla
@type base: string
@keyword user: username for authenticated actions.
@type user: string
@keyword password: password for authenticated actions.
@type password: string
@keyword forget: forget login session after termination.
@type forget: bool
@keyword skip_auth: do not authenticate
@type skip_auth: bool
"""
self.base = base
scheme, self.host, self.path, query, frag = urlsplit(self.base)
self.authenticated = False
self.forget = forget
if not self.forget:
try:
cookie_file = os.path.join(os.environ['HOME'], COOKIE_FILE)
self.cookiejar = LWPCookieJar(cookie_file)
if forget:
try:
self.cookiejar.load()
self.cookiejar.clear()
self.cookiejar.save()
os.chmod(self.cookiejar.filename, 0700)
except IOError:
pass
except KeyError:
self.warn('Unable to save session cookies in %s' % cookie_file)
self.cookiejar = CookieJar(cookie_file)
else:
self.cookiejar = CookieJar()
self.opener = build_opener(HTTPCookieProcessor(self.cookiejar))
self.user = user
self.password = password
self.httpuser = httpuser
self.httppassword = httppassword
self.skip_auth = skip_auth
def log(self, status_msg):
"""Default logging handler. Expected to be overridden by
the UI implementing subclass.
@param status_msg: status message to print
@type status_msg: string
"""
return
def warn(self, warn_msg):
"""Default logging handler. Expected to be overridden by
the UI implementing subclass.
@param status_msg: status message to print
@type status_msg: string
"""
return
def get_input(self, prompt):
"""Default input handler. Expected to be override by the
UI implementing subclass.
@param prompt: Prompt message
@type prompt: string
"""
return ''
def auth(self):
"""Authenticate a session.
"""
# check if we need to authenticate
if self.authenticated:
return
# try seeing if we really need to request login
if not self.forget:
try:
self.cookiejar.load()
except IOError:
pass
req_url = urljoin(self.base, config.urls['auth'])
req_url += '?GoAheadAndLogIn=1'
req = Request(req_url, None, config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
re_request_login = re.compile(r'<title>.*Log in to Bugzilla</title>')
if not re_request_login.search(resp.read()):
self.log('Already logged in.')
self.authenticated = True
return
# prompt for username if we were not supplied with it
if not self.user:
self.log('No username given.')
self.user = self.get_input('Username: '******'No password given.')
self.password = getpass.getpass()
# perform login
qparams = config.params['auth'].copy()
qparams['Bugzilla_login'] = self.user
qparams['Bugzilla_password'] = self.password
if not self.forget:
qparams['Bugzilla_remember'] = 'on'
req_url = urljoin(self.base, config.urls['auth'])
req = Request(req_url, urlencode(qparams), config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
if resp.info().has_key('Set-Cookie'):
self.authenticated = True
if not self.forget:
self.cookiejar.save()
os.chmod(self.cookiejar.filename, 0700)
return True
else:
raise RuntimeError("Failed to login")
def extractResults(self, resp):
# parse the results into dicts.
results = []
columns = []
rows = []
for r in csv.reader(resp): rows.append(r)
for field in rows[0]:
if config.choices['column_alias'].has_key(field):
columns.append(config.choices['column_alias'][field])
else:
self.log('Unknown field: ' + field)
columns.append(field)
for row in rows[1:]:
if row[0].find("Missing Search") != -1:
self.log('Bugzilla error (Missing search found)')
return None
fields = {}
for i in range(min(len(row), len(columns))):
fields[columns[i]] = row[i]
results.append(fields)
return results
def search(self, query, comments = False, order = 'number',
assigned_to = None, reporter = None, cc = None,
commenter = None, whiteboard = None, keywords = None,
status = [], severity = [], priority = [], product = [],
component = []):
"""Search bugzilla for a bug.
@param query: query string to search in title or {comments}.
@type query: string
@param order: what order to returns bugs in.
@type order: string
@keyword assigned_to: email address which the bug is assigned to.
@type assigned_to: string
@keyword reporter: email address matching the bug reporter.
@type reporter: string
@keyword cc: email that is contained in the CC list
@type cc: string
@keyword commenter: email of a commenter.
@type commenter: string
@keyword whiteboard: string to search in status whiteboard (gentoo?)
@type whiteboard: string
@keyword keywords: keyword to search for
@type keywords: string
@keyword status: bug status to match. default is ['NEW', 'ASSIGNED',
'REOPENED'].
@type status: list
@keyword severity: severity to match, empty means all.
@type severity: list
@keyword priority: priority levels to patch, empty means all.
@type priority: list
@keyword comments: search comments instead of just bug title.
@type comments: bool
@keyword product: search within products. empty means all.
@type product: list
@keyword component: search within components. empty means all.
@type component: list
@return: list of bugs, each bug represented as a dict
@rtype: list of dicts
"""
if not self.authenticated and not self.skip_auth:
self.auth()
qparams = config.params['list'].copy()
if comments:
qparams['long_desc'] = query
else:
qparams['short_desc'] = query
qparams['order'] = config.choices['order'].get(order, 'Bug Number')
qparams['bug_severity'] = severity or []
qparams['priority'] = priority or []
if status == None:
qparams['bug_status'] = ['NEW', 'ASSIGNED', 'REOPENED']
elif [s.upper() for s in status] == ['ALL']:
qparams['bug_status'] = config.choices['status']
else:
qparams['bug_status'] = [s.upper() for s in status]
qparams['product'] = product or ''
qparams['component'] = component or ''
qparams['status_whiteboard'] = whiteboard or ''
qparams['keywords'] = keywords or ''
# hoops to jump through for emails, since there are
# only two fields, we have to figure out what combinations
# to use if all three are set.
unique = list(set([assigned_to, cc, reporter, commenter]))
unique = [u for u in unique if u]
if len(unique) < 3:
for i in range(len(unique)):
e = unique[i]
n = i + 1
qparams['email%d' % n] = e
qparams['emailassigned_to%d' % n] = int(e == assigned_to)
qparams['emailreporter%d' % n] = int(e == reporter)
qparams['emailcc%d' % n] = int(e == cc)
qparams['emaillongdesc%d' % n] = int(e == commenter)
else:
raise AssertionError('Cannot set assigned_to, cc, and '
'reporter in the same query')
req_params = urlencode(qparams, True)
req_url = urljoin(self.base, config.urls['list'])
req_url += '?' + req_params
req = Request(req_url, None, config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
return self.extractResults(resp)
def namedcmd(self, cmd):
"""Run command stored in Bugzilla by name.
@return: Result from the stored command.
@rtype: list of dicts
"""
if not self.authenticated and not self.skip_auth:
self.auth()
qparams = config.params['namedcmd'].copy()
# Is there a better way of getting a command with a space in its name
# to be encoded as foo%20bar instead of foo+bar or foo%2520bar?
qparams['namedcmd'] = quote(cmd)
req_params = urlencode(qparams, True)
req_params = req_params.replace('%25','%')
req_url = urljoin(self.base, config.urls['list'])
req_url += '?' + req_params
req = Request(req_url, None, config.headers)
if self.user and self.hpassword:
base64string = base64.encodestring('%s:%s' % (self.user, self.hpassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
return self.extractResults(resp)
def get(self, bugid):
"""Get an ElementTree representation of a bug.
@param bugid: bug id
@type bugid: int
@rtype: ElementTree
"""
if not self.authenticated and not self.skip_auth:
self.auth()
qparams = config.params['show'].copy()
qparams['id'] = bugid
req_params = urlencode(qparams, True)
req_url = urljoin(self.base, config.urls['show'])
req_url += '?' + req_params
req = Request(req_url, None, config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
fd = StringIO(resp.read())
# workaround for ill-defined XML templates in bugzilla 2.20.2
parser = ForcedEncodingXMLTreeBuilder(encoding = 'utf-8')
etree = ElementTree.parse(fd, parser)
bug = etree.find('.//bug')
if bug and bug.attrib.has_key('error'):
return None
else:
return etree
def modify(self, bugid, title = None, comment = None, url = None,
status = None, resolution = None,
assigned_to = None, duplicate = 0,
priority = None, severity = None,
add_cc = [], remove_cc = [],
add_dependson = [], remove_dependson = [],
add_blocked = [], remove_blocked = [],
whiteboard = None, keywords = None):
"""Modify an existing bug
@param bugid: bug id
@type bugid: int
@keyword title: new title for bug
@type title: string
@keyword comment: comment to add
@type comment: string
@keyword url: new url
@type url: string
@keyword status: new status (note, if you are changing it to RESOLVED, you need to set {resolution} as well.
@type status: string
@keyword resolution: new resolution (if status=RESOLVED)
@type resolution: string
@keyword assigned_to: email (needs to exist in bugzilla)
@type assigned_to: string
@keyword duplicate: bug id to duplicate against (if resolution = DUPLICATE)
@type duplicate: int
@keyword priority: new priority for bug
@type priority: string
@keyword severity: new severity for bug
@type severity: string
@keyword add_cc: list of emails to add to the cc list
@type add_cc: list of strings
@keyword remove_cc: list of emails to remove from cc list
@type remove_cc: list of string.
@keyword add_dependson: list of bug ids to add to the depend list
@type add_dependson: list of strings
@keyword remove_dependson: list of bug ids to remove from depend list
@type remove_dependson: list of strings
@keyword add_blocked: list of bug ids to add to the blocked list
@type add_blocked: list of strings
@keyword remove_blocked: list of bug ids to remove from blocked list
@type remove_blocked: list of strings
@keyword whiteboard: set status whiteboard
@type whiteboard: string
@keyword keywords: set keywords
@type keywords: string
@return: list of fields modified.
@rtype: list of strings
"""
if not self.authenticated and not self.skip_auth:
self.auth()
buginfo = Bugz.get(self, bugid)
if not buginfo:
return False
modified = []
qparams = config.params['modify'].copy()
qparams['id'] = bugid
qparams['knob'] = 'none'
# copy existing fields
FIELDS = ('bug_file_loc', 'bug_severity', 'short_desc', 'bug_status',
'status_whiteboard', 'keywords',
'op_sys', 'priority', 'version', 'target_milestone',
'assigned_to', 'rep_platform', 'product', 'component')
FIELDS_MULTI = ('blocked', 'dependson')
for field in FIELDS:
try:
qparams[field] = buginfo.find('.//%s' % field).text
except:
pass
for field in FIELDS_MULTI:
qparams[field] = [d.text for d in buginfo.findall('.//%s' % field)]
# set 'knob' if we are change the status/resolution
# or trying to reassign bug.
if status:
status = status.upper()
if resolution:
resolution = resolution.upper()
if status == 'RESOLVED' and status != qparams['bug_status']:
qparams['knob'] = 'resolve'
if resolution:
qparams['resolution'] = resolution
else:
qparams['resolution'] = 'FIXED'
modified.append(('status', status))
modified.append(('resolution', qparams['resolution']))
elif status == 'ASSIGNED' and status != qparams['bug_status']:
qparams['knob'] = 'accept'
modified.append(('status', status))
elif status == 'REOPENED' and status != qparams['bug_status']:
qparams['knob'] = 'reopen'
modified.append(('status', status))
elif status == 'VERIFIED' and status != qparams['bug_status']:
qparams['knob'] = 'verified'
modified.append(('status', status))
elif status == 'CLOSED' and status != qparams['bug_status']:
qparams['knob'] = 'closed'
modified.append(('status', status))
elif duplicate:
qparams['knob'] = 'duplicate'
qparams['dup_id'] = duplicate
modified.append(('status', 'RESOLVED'))
modified.append(('resolution', 'DUPLICATE'))
elif assigned_to:
qparams['knob'] = 'reassign'
qparams['assigned_to'] = assigned_to
modified.append(('assigned_to', assigned_to))
# setup modification of other bits
if comment:
qparams['comment'] = comment
modified.append(('comment', ellipsis(comment, 60)))
if title:
qparams['short_desc'] = title or ''
modified.append(('title', title))
if url != None:
qparams['bug_file_loc'] = url
modified.append(('url', url))
if severity != None:
qparams['bug_severity'] = severity
modified.append(('severity', severity))
if priority != None:
qparams['priority'] = priority
modified.append(('priority', priority))
# cc manipulation
if add_cc != None:
qparams['newcc'] = ', '.join(add_cc)
modified.append(('newcc', qparams['newcc']))
if remove_cc != None:
qparams['cc'] = remove_cc
qparams['removecc'] = 'on'
modified.append(('cc', remove_cc))
# bug depend/blocked manipulation
changed_dependson = False
changed_blocked = False
if remove_dependson:
for bug_id in remove_dependson:
qparams['dependson'].remove(str(bug_id))
changed_dependson = True
if remove_blocked:
for bug_id in remove_blocked:
qparams['blocked'].remove(str(bug_id))
changed_blocked = True
if add_dependson:
for bug_id in add_dependson:
qparams['dependson'].append(str(bug_id))
changed_dependson = True
if add_blocked:
for bug_id in add_blocked:
qparams['blocked'].append(str(bug_id))
changed_blocked = True
qparams['dependson'] = ','.join(qparams['dependson'])
qparams['blocked'] = ','.join(qparams['blocked'])
if changed_dependson:
modified.append(('dependson', qparams['dependson']))
if changed_blocked:
modified.append(('blocked', qparams['blocked']))
if whiteboard != None:
qparams['status_whiteboard'] = whiteboard
modified.append(('status_whiteboard', whiteboard))
if keywords != None:
qparams['keywords'] = keywords
modified.append(('keywords', keywords))
req_params = urlencode(qparams, True)
req_url = urljoin(self.base, config.urls['modify'])
req = Request(req_url, req_params, config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
try:
resp = self.opener.open(req)
return modified
except:
return []
def attachment(self, attachid):
"""Get an attachment by attachment_id
@param attachid: attachment id
@type attachid: int
@return: dict with three keys, 'filename', 'size', 'fd'
@rtype: dict
"""
if not self.authenticated and not self.skip_auth:
self.auth()
qparams = config.params['attach'].copy()
qparams['id'] = attachid
req_params = urlencode(qparams, True)
req_url = urljoin(self.base, config.urls['attach'])
req_url += '?' + req_params
req = Request(req_url, None, config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
try:
content_type = resp.info()['Content-type']
namefield = content_type.split(';')[1]
filename = re.search(r'name=\"(.*)\"', namefield).group(1)
content_length = int(resp.info()['Content-length'], 0)
return {'filename': filename, 'size': content_length, 'fd': resp}
except:
return {}
def post(self, product, component, title, description, url = '', assigned_to = '', cc = '', keywords = '', version = '', dependson = '', blocked = '', priority = '', severity = ''):
"""Post a bug
@param product: product where the bug should be placed
@type product: string
@param component: component where the bug should be placed
@type component: string
@param title: title of the bug.
@type title: string
@param description: description of the bug
@type description: string
@keyword url: optional url to submit with bug
@type url: string
@keyword assigned_to: optional email to assign bug to
@type assigned_to: string.
@keyword cc: option list of CC'd emails
@type: string
@keyword keywords: option list of bugzilla keywords
@type: string
@keyword version: version of the component
@type: string
@keyword dependson: bugs this one depends on
@type: string
@keyword blocked: bugs this one blocks
@type: string
@keyword priority: priority of this bug
@type: string
@keyword severity: severity of this bug
@type: string
@rtype: int
@return: the bug number, or 0 if submission failed.
"""
if not self.authenticated and not self.skip_auth:
self.auth()
qparams = config.params['post'].copy()
qparams['product'] = product
qparams['component'] = component
qparams['short_desc'] = title
qparams['comment'] = description
qparams['assigned_to'] = assigned_to
qparams['cc'] = cc
qparams['bug_file_loc'] = url
qparams['dependson'] = dependson
qparams['blocked'] = blocked
qparams['keywords'] = keywords
#XXX: default version is 'unspecified'
if version != '':
qparams['version'] = version
#XXX: default priority is 'P2'
if priority != '':
qparams['priority'] = priority
#XXX: default severity is 'normal'
if severity != '':
qparams['bug_severity'] = severity
req_params = urlencode(qparams, True)
req_url = urljoin(self.base, config.urls['post'])
req = Request(req_url, req_params, config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
try:
re_bug = re.compile(r'<title>.*Bug ([0-9]+) Submitted</title>')
bug_match = re_bug.search(resp.read())
if bug_match:
return int(bug_match.group(1))
except:
pass
return 0
def attach(self, bugid, title, description, filename,
content_type = 'text/plain'):
"""Attach a file to a bug.
@param bugid: bug id
@type bugid: int
@param title: short description of attachment
@type title: string
@param description: long description of the attachment
@type description: string
@param filename: filename of the attachment
@type filename: string
@keywords content_type: mime-type of the attachment
@type content_type: string
@rtype: bool
@return: True if successful, False if not successful.
"""
if not self.authenticated and not self.skip_auth:
self.auth()
qparams = config.params['attach_post'].copy()
qparams['bugid'] = bugid
qparams['description'] = title
qparams['comment'] = description
qparams['contenttypeentry'] = content_type
filedata = [('data', filename, open(filename).read())]
content_type, body = encode_multipart_formdata(qparams.items(),
filedata)
req_headers = config.headers.copy()
req_headers['Content-type'] = content_type
req_headers['Content-length'] = len(body)
req_url = urljoin(self.base, config.urls['attach_post'])
req = Request(req_url, body, req_headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
# TODO: return attachment id and success?
try:
re_success = re.compile(r'<title>Changes Submitted</title>')
if re_success.search(resp.read()):
return True
except:
pass
return False
class CookieTransport(xmlrpclib.Transport):
'''A subclass of xmlrpclib.Transport that supports cookies.'''
cookiejar = None
scheme = 'http'
# Cribbed from xmlrpclib.Transport.send_user_agent
def send_cookies(self, connection, cookie_request):
if self.cookiejar is None:
self.cookiejar = CookieJar()
elif self.cookiejar:
# Let the cookiejar figure out what cookies are appropriate
self.cookiejar.add_cookie_header(cookie_request)
# Pull the cookie headers out of the request object...
cookielist=list()
for h,v in cookie_request.header_items():
if h.startswith('Cookie'):
cookielist.append([h,v])
# ...and put them over the connection
for h,v in cookielist:
connection.putheader(h,v)
# This is the same request() method from xmlrpclib.Transport,
# with a couple additions noted below
def request_with_cookies(self, host, handler, request_body, verbose=0):
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
# ADDED: construct the URL and Request object for proper cookie handling
request_url = "%s://%s%s" % (self.scheme,host,handler)
#log.debug("request_url is %s" % request_url)
cookie_request = urllib2.Request(request_url)
self.send_request(h,handler,request_body)
self.send_host(h,host)
self.send_cookies(h,cookie_request) # ADDED. creates cookiejar if None.
self.send_user_agent(h)
self.send_content(h,request_body)
errcode, errmsg, headers = h.getreply()
# ADDED: parse headers and get cookies here
cookie_response = CookieResponse(headers)
# Okay, extract the cookies from the headers
self.cookiejar.extract_cookies(cookie_response,cookie_request)
#log.debug("cookiejar now contains: %s" % self.cookiejar._cookies)
# And write back any changes
if hasattr(self.cookiejar,'save'):
try:
self.cookiejar.save(self.cookiejar.filename)
except Exception as e:
raise
#log.error("Couldn't write cookiefile %s: %s" % \
# (self.cookiejar.filename,str(e)))
if errcode != 200:
# When runs here, the HTTPS connection isn't useful any more
# before raising an exception to caller
h.close()
raise xmlrpclib.ProtocolError(
host + handler,
errcode, errmsg,
headers
)
self.verbose = verbose
try:
sock = h._conn.sock
except AttributeError:
sock = None
try:
return self._parse_response(h.getfile(), sock)
finally:
h.close()
# This is just python 2.7's xmlrpclib.Transport.single_request, with
# send additions noted below to send cookies along with the request
def single_request_with_cookies(self, host, handler, request_body, verbose=0):
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
# ADDED: construct the URL and Request object for proper cookie handling
request_url = "%s://%s%s" % (self.scheme,host,handler)
#log.debug("request_url is %s" % request_url)
cookie_request = urllib2.Request(request_url)
try:
self.send_request(h,handler,request_body)
self.send_host(h,host)
self.send_cookies(h,cookie_request) # ADDED. creates cookiejar if None.
self.send_user_agent(h)
self.send_content(h,request_body)
response = h.getresponse(buffering=True)
# ADDED: parse headers and get cookies here
cookie_response = CookieResponse(response.msg)
# Okay, extract the cookies from the headers
self.cookiejar.extract_cookies(cookie_response,cookie_request)
#log.debug("cookiejar now contains: %s" % self.cookiejar._cookies)
# And write back any changes
if hasattr(self.cookiejar,'save'):
try:
self.cookiejar.save(self.cookiejar.filename)
except Exception as e:
raise
#log.error("Couldn't write cookiefile %s: %s" % \
# (self.cookiejar.filename,str(e)))
if response.status == 200:
self.verbose = verbose
return self.parse_response(response)
if (response.getheader("content-length", 0)):
response.read()
raise xmlrpclib.ProtocolError(
host + handler,
response.status, response.reason,
response.msg,
)
except xmlrpclib.Fault:
raise
finally:
h.close()
# Override the appropriate request method
if hasattr(xmlrpclib.Transport, 'single_request'):
single_request = single_request_with_cookies # python 2.7+
else:
request = request_with_cookies # python 2.6 and earlier
class CookieTransport(TimeoutTransport):
'''A subclass of xmlrpclib.Transport that supports cookies.'''
cookiejar = None
scheme = 'http'
# Cribbed from xmlrpclib.Transport.send_user_agent
def send_cookies(self, connection, cookie_request):
if self.cookiejar is None:
self.cookiejar = CookieJar()
elif self.cookiejar:
# Let the cookiejar figure out what cookies are appropriate
self.cookiejar.add_cookie_header(cookie_request)
# Pull the cookie headers out of the request object...
cookielist = list()
for h, v in cookie_request.header_items():
if h.startswith('Cookie'):
cookielist.append([h, v])
# ...and put them over the connection
for h, v in cookielist:
connection.putheader(h, v)
# This is the same request() method from xmlrpclib.Transport,
# with a couple additions noted below
def request(self, host, handler, request_body, verbose=0):
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
request_url = "%s://%s/" % (self.scheme, host)
cookie_request = urllib2.Request(request_url)
self.send_request(h, handler, request_body)
self.send_host(h, host)
self.send_cookies(h, cookie_request) # ADDED. creates cookiejar if None.
self.send_user_agent(h)
self.send_content(h, request_body)
errcode, errmsg, headers = h.getreply()
# ADDED: parse headers and get cookies here
# fake a response object that we can fill with the headers above
class CookieResponse:
def __init__(self, headers):
self.headers = headers
def info(self):
return self.headers
cookie_response = CookieResponse(headers)
# Okay, extract the cookies from the headers
self.cookiejar.extract_cookies(cookie_response, cookie_request)
# And write back any changes
if hasattr(self.cookiejar, 'save'):
self.cookiejar.save(self.cookiejar.filename)
if errcode != 200:
raise xmlrpclib.ProtocolError(
host + handler,
errcode, errmsg,
headers
)
self.verbose = verbose
try:
sock = h._conn.sock
except AttributeError:
sock = None
return self._parse_response(h.getfile(), sock)
class CookieTransport(xmlrpclib.Transport):
"""A subclass of xmlrpclib.Transport that supports cookies."""
cookiejar = None
scheme = "http"
# Cribbed from xmlrpclib.Transport.send_user_agent
def send_cookies(self, connection, cookie_request):
if self.cookiejar is None:
self.cookiejar = CookieJar()
elif self.cookiejar:
# Let the cookiejar figure out what cookies are appropriate
self.cookiejar.add_cookie_header(cookie_request)
# Pull the cookie headers out of the request object...
cookielist = list()
for h, v in cookie_request.header_items():
if h.startswith("Cookie"):
cookielist.append([h, v])
# ...and put them over the connection
for h, v in cookielist:
connection.putheader(h, v)
# This is the same request() method from xmlrpclib.Transport,
# with a couple additions noted below
def request_with_cookies(self, host, handler, request_body, verbose=0):
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
# ADDED: construct the URL and Request object for proper cookie handling
request_url = "%s://%s%s" % (self.scheme, host, handler)
# log.debug("request_url is %s" % request_url)
cookie_request = urllib2.Request(request_url)
self.send_request(h, handler, request_body)
self.send_host(h, host)
self.send_cookies(h, cookie_request) # ADDED. creates cookiejar if None.
self.send_user_agent(h)
self.send_content(h, request_body)
errcode, errmsg, headers = h.getreply()
# ADDED: parse headers and get cookies here
cookie_response = CookieResponse(headers)
# Okay, extract the cookies from the headers
self.cookiejar.extract_cookies(cookie_response, cookie_request)
# log.debug("cookiejar now contains: %s" % self.cookiejar._cookies)
# And write back any changes
if hasattr(self.cookiejar, "save"):
try:
self.cookiejar.save(self.cookiejar.filename)
except Exception, e:
raise
# log.error("Couldn't write cookiefile %s: %s" % \
# (self.cookiejar.filename,str(e)))
if errcode != 200:
raise xmlrpclib.ProtocolError(host + handler, errcode, errmsg, headers)
self.verbose = verbose
try:
sock = h._conn.sock
except AttributeError:
sock = None
return self._parse_response(h.getfile(), sock)
class _RequestsHandler(object):
def __init__(self,
cache_dir=None,
web_time_out=30,
cookie_jar=None,
ignore_ssl_errors=False):
""" Initialises the UriHandler class
Keyword Arguments:
:param str cache_dir: A path for http caching. If specified, caching will be used.
:param int web_time_out: Timeout for requests in seconds
:param str cookie_jar: The path to the cookie jar (in case of file storage)
:param ignore_ssl_errors: Ignore any SSL certificate errors.
"""
self.id = int(time.time())
if cookie_jar:
self.cookieJar = MozillaCookieJar(cookie_jar)
if not os.path.isfile(cookie_jar):
self.cookieJar.save()
self.cookieJar.load()
self.cookieJarFile = True
else:
self.cookieJar = CookieJar()
self.cookieJarFile = False
self.cacheDir = cache_dir
self.cacheStore = None
if cache_dir:
self.cacheStore = StreamCache(cache_dir)
Logger.debug("Opened %s", self.cacheStore)
else:
Logger.debug("No cache-store provided. Cached disabled.")
self.userAgent = "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13 (.NET CLR 3.5.30729)"
self.webTimeOut = web_time_out # max duration of request
self.ignoreSslErrors = ignore_ssl_errors # ignore SSL errors
if self.ignoreSslErrors:
Logger.warning("Ignoring all SSL errors in Python")
# status of the most recent call
self.status = UriStatus(code=0, url=None, error=False, reason=None)
# for download animation
self.__animationIndex = -1
def download(self,
uri,
filename,
folder,
progress_callback=None,
proxy=None,
params="",
data="",
json="",
referer=None,
additional_headers=None):
""" Downloads a remote file
:param str filename: The filename that should be used to store the file.
:param str folder: The folder to save the file in.
:param str params: Data to send with the request (open(uri, params)).
:param str uri: The URI to download.
:param dict[str, any]|str data: Data to send with the request (open(uri, data)).
:param dict[str, any] json: Json to send with the request (open(uri, params)).
:param ProxyInfo proxy: The address and port (proxy.address.ext:port) of a
proxy server that should be used.
:param str referer: The http referer to use.
:param dict additional_headers: The optional headers.
:param function progress_callback: The callback for progress update. The format is
function(retrievedSize, totalSize, perc, completed, status)
:return: The full path of the location to which it was downloaded.
:rtype: str
"""
if not folder or not filename:
raise ValueError(
"Destination folder and filename should be specified")
if not os.path.isdir(folder):
raise ValueError("Destination folder is not a valid location")
if not progress_callback:
raise ValueError("A callback must be specified")
download_path = os.path.join(folder, filename)
if os.path.isfile(download_path):
Logger.info("Url already downloaded to: %s", download_path)
return download_path
Logger.info("Creating Downloader for url '%s' to filename '%s'", uri,
download_path)
r = self.__requests(uri,
proxy=proxy,
params=params,
data=data,
json=json,
referer=referer,
additional_headers=additional_headers,
no_cache=True,
stream=True)
if r is None:
return ""
retrieved_bytes = 0
total_size = int(r.headers.get('Content-Length', '0').strip())
# There is an issue with the way Requests checks for input and it does not like the newInt.
if PY2:
chunk_size = 10 * 1024
else:
chunk_size = 1024 if total_size == 0 else total_size // 100
cancel = False
with open(download_path, 'wb') as fd:
for chunk in r.iter_content(chunk_size=chunk_size):
fd.write(chunk)
retrieved_bytes += len(chunk)
if progress_callback:
cancel = self.__do_progress_callback(
progress_callback, retrieved_bytes, total_size, False)
if cancel:
Logger.warning("Download of %s aborted", uri)
break
if cancel:
if os.path.isfile(download_path):
Logger.info("Removing partial download: %s", download_path)
os.remove(download_path)
return ""
if progress_callback:
self.__do_progress_callback(progress_callback, retrieved_bytes,
total_size, True)
return download_path
def open(self,
uri,
proxy=None,
params=None,
data=None,
json=None,
referer=None,
additional_headers=None,
no_cache=False,
force_text=False):
""" Open an URL Async using a thread
:param str uri: The URI to download.
:param str params: Data to send with the request (open(uri, params)).
:param dict[str, any]|str|bytes data: Data to send with the request (open(uri, data)).
:param dict[str, any] json: Json to send with the request (open(uri, params)).
:param ProxyInfo proxy: The address and port (proxy.address.ext:port) of a
proxy server that should be used.
:param str referer: The http referer to use.
:param dict|None additional_headers: The optional headers.
:param bool no_cache: Should cache be disabled.
:param bool force_text: In case no content type is specified, force text.
:return: The data that was retrieved from the URI.
:rtype: str|unicode
"""
r = self.__requests(uri,
proxy=proxy,
params=params,
data=data,
json=json,
referer=referer,
additional_headers=additional_headers,
no_cache=no_cache,
stream=False)
if r is None:
return ""
content_type = r.headers.get("content-type", "")
if r.encoding == 'ISO-8859-1' and "text" in content_type:
# Requests defaults to ISO-8859-1 for all text content that does not specify an encoding
Logger.debug(
"Found 'ISO-8859-1' for 'text' content-type. Using UTF-8 instead."
)
r.encoding = 'utf-8'
elif r.encoding is None and force_text:
Logger.debug(
"Found missing encoding and 'force_text' was specified. Using UTF-8."
)
r.encoding = 'utf-8'
elif r.encoding is None and self.__is_text_content_type(content_type):
Logger.debug(
"Found missing encoding for content type '%s' is considered text. Using UTF-8 instead.",
content_type)
r.encoding = 'utf-8'
# We might need a better mechanism here.
if not r.encoding and content_type.lower() in [
"application/json", "application/javascript"
]:
return r.text
return r.text if r.encoding else r.content
def header(self, uri, proxy=None, referer=None, additional_headers=None):
""" Retrieves header information only.
:param str uri: The URI to fetch the header from.
:param ProxyInfo|none proxy: The address and port (proxy.address.ext:port) of a
proxy server that should be used.
:param str|none referer: The http referer to use.
:param dict|none additional_headers: The optional headers.
:return: Content-type and the URL to which a redirect could have occurred.
:rtype: tuple[str,str]
"""
with requests.session() as s:
s.cookies = self.cookieJar
s.verify = not self.ignoreSslErrors
proxies = self.__get_proxies(proxy, uri)
headers = self.__get_headers(referer, additional_headers)
Logger.info("Performing a HEAD for %s", uri)
r = s.head(uri,
proxies=proxies,
headers=headers,
allow_redirects=True,
timeout=self.webTimeOut)
content_type = r.headers.get("Content-Type", "")
real_url = r.url
self.status = UriStatus(code=r.status_code,
url=uri,
error=not r.ok,
reason=r.reason)
if self.cookieJarFile:
# noinspection PyUnresolvedReferences
self.cookieJar.save()
if r.ok:
Logger.info("%s resulted in '%s %s' (%s) for %s",
r.request.method, r.status_code, r.reason,
r.elapsed, r.url)
return content_type, real_url
else:
Logger.error("%s failed with in '%s %s' (%s) for %s",
r.request.method, r.status_code, r.reason,
r.elapsed, r.url)
return "", ""
# noinspection PyUnusedLocal
def __requests(self, uri, proxy, params, data, json, referer,
additional_headers, no_cache, stream):
with requests.session() as s:
s.cookies = self.cookieJar
s.verify = not self.ignoreSslErrors
if self.cacheStore and not no_cache:
Logger.trace("Adding the %s to the request", self.cacheStore)
s.mount("https://", CacheHTTPAdapter(self.cacheStore))
s.mount("http://", CacheHTTPAdapter(self.cacheStore))
proxies = self.__get_proxies(proxy, uri)
headers = self.__get_headers(referer, additional_headers)
if params is not None:
# Old UriHandler behaviour. Set form header to keep compatible
if "content-type" not in headers:
headers[
"content-type"] = "application/x-www-form-urlencoded"
Logger.info("Performing a POST with '%s' for %s",
headers["content-type"], uri)
r = s.post(uri,
data=params,
proxies=proxies,
headers=headers,
stream=stream,
timeout=self.webTimeOut)
elif data is not None:
# Normal Requests compatible data object
Logger.info("Performing a POST with '%s' for %s",
headers.get("content-type", "<No Content-Type>"),
uri)
r = s.post(uri,
data=data,
proxies=proxies,
headers=headers,
stream=stream,
timeout=self.webTimeOut)
elif json is not None:
Logger.info("Performing a json POST with '%s' for %s",
headers.get("content-type", "<No Content-Type>"),
uri)
r = s.post(uri,
json=json,
proxies=proxies,
headers=headers,
stream=stream,
timeout=self.webTimeOut)
else:
Logger.info("Performing a GET for %s", uri)
r = s.get(uri,
proxies=proxies,
headers=headers,
stream=stream,
timeout=self.webTimeOut)
if r.ok:
Logger.info("%s resulted in '%s %s' (%s) for %s",
r.request.method, r.status_code, r.reason,
r.elapsed, r.url)
else:
Logger.error("%s failed with '%s %s' (%s) for %s",
r.request.method, r.status_code, r.reason,
r.elapsed, r.url)
self.status = UriStatus(code=r.status_code,
url=r.url,
error=not r.ok,
reason=r.reason)
if self.cookieJarFile:
# noinspection PyUnresolvedReferences
self.cookieJar.save()
return r
def __get_headers(self, referer, additional_headers):
headers = {}
if additional_headers:
for k, v in additional_headers.items():
headers[k.lower()] = v
if "user-agent" not in headers:
headers["user-agent"] = self.userAgent
if referer and "referer" not in headers:
headers["referer"] = referer
return headers
def __get_proxies(self, proxy, url):
"""
:param ProxyInfo proxy:
:param url:
:return:
:rtype: dict[str, str]
"""
if proxy is None:
return None
elif not proxy.use_proxy_for_url(url):
Logger.debug("Not using proxy due to filter mismatch")
elif proxy.Scheme == "http":
Logger.debug("Using a http(s) %s", proxy)
proxy_address = proxy.get_proxy_address()
return {"http": proxy_address, "https": proxy_address}
elif proxy.Scheme == "dns":
Logger.debug("Using a DNS %s", proxy)
return {"dns": proxy.Proxy}
Logger.warning("Unsupported Proxy Scheme: %s", proxy.Scheme)
return None
def __do_progress_callback(self, progress_callback, retrieved_size,
total_size, completed):
""" Performs a callback, if the progressCallback was specified.
:param progress_callback: The callback method
:param retrieved_size: Number of bytes retrieved
:param total_size: Total number of bytes
:param completed: Are we done?
@rtype : Boolean Should we cancel the download?
"""
if progress_callback is None:
# no callback so it was not cancelled
return False
# calculated some stuff
self.__animationIndex = (self.__animationIndex + 1) % 4
bytes_to_mb = 1048576
animation_frames = ["-", "\\", "|", "/"]
animation = animation_frames[self.__animationIndex]
retrievedsize_mb = 1.0 * retrieved_size / bytes_to_mb
totalsize_mb = 1.0 * total_size / bytes_to_mb
if total_size > 0:
percentage = 100.0 * retrieved_size / total_size
else:
percentage = 0
status = '%s - %i%% (%.1f of %.1f MB)' % \
(animation, percentage, retrievedsize_mb, totalsize_mb)
try:
return progress_callback(retrieved_size, total_size, percentage,
completed, status)
except:
Logger.error("Error in Progress Callback", exc_info=True)
# cancel the download
return True
def __is_text_content_type(self, content_type):
return content_type.lower() in [
"application/vnd.apple.mpegurl", "application/x-mpegurl"
]
def __str__(self):
return "UriHandler [id={0}, useCaching={1}, ignoreSslErrors={2}]"\
.format(self.id, self.cacheStore, self.ignoreSslErrors)
class Bugz:
""" Converts sane method calls to Bugzilla HTTP requests.
@ivar base: base url of bugzilla.
@ivar user: username for authenticated operations.
@ivar password: password for authenticated operations
@ivar cookiejar: for authenticated sessions so we only auth once.
@ivar forget: forget user/password after session.
@ivar authenticated: is this session authenticated already
"""
def __init__(self, base, user=None, password=None, forget=False, skip_auth=False, httpuser=None, httppassword=None):
"""
{user} and {password} will be prompted if an action needs them
and they are not supplied.
if {forget} is set, the login cookie will be destroyed on quit.
@param base: base url of the bugzilla
@type base: string
@keyword user: username for authenticated actions.
@type user: string
@keyword password: password for authenticated actions.
@type password: string
@keyword forget: forget login session after termination.
@type forget: bool
@keyword skip_auth: do not authenticate
@type skip_auth: bool
"""
self.base = base
scheme, self.host, self.path, query, frag = urlsplit(self.base)
self.authenticated = False
self.forget = forget
if not self.forget:
try:
cookie_file = os.path.join(os.environ["HOME"], COOKIE_FILE)
self.cookiejar = LWPCookieJar(cookie_file)
if forget:
try:
self.cookiejar.load()
self.cookiejar.clear()
self.cookiejar.save()
os.chmod(self.cookiejar.filename, 0600)
except IOError:
pass
except KeyError:
self.warn("Unable to save session cookies in %s" % cookie_file)
self.cookiejar = CookieJar(cookie_file)
else:
self.cookiejar = CookieJar()
self.opener = build_opener(HTTPCookieProcessor(self.cookiejar))
self.user = user
self.password = password
self.httpuser = httpuser
self.httppassword = httppassword
self.skip_auth = skip_auth
def log(self, status_msg):
"""Default logging handler. Expected to be overridden by
the UI implementing subclass.
@param status_msg: status message to print
@type status_msg: string
"""
return
def warn(self, warn_msg):
"""Default logging handler. Expected to be overridden by
the UI implementing subclass.
@param status_msg: status message to print
@type status_msg: string
"""
return
def get_input(self, prompt):
"""Default input handler. Expected to be override by the
UI implementing subclass.
@param prompt: Prompt message
@type prompt: string
"""
return ""
def auth(self):
"""Authenticate a session.
"""
# check if we need to authenticate
if self.authenticated:
return
# try seeing if we really need to request login
if not self.forget:
try:
self.cookiejar.load()
except IOError:
pass
req_url = urljoin(self.base, config.urls["auth"])
req_url += "?GoAheadAndLogIn=1"
req = Request(req_url, None, config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring("%s:%s" % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
re_request_login = re.compile(r"<title>.*Log in to .*</title>")
if not re_request_login.search(resp.read()):
self.log("Already logged in.")
self.authenticated = True
return
# prompt for username if we were not supplied with it
if not self.user:
self.log("No username given.")
self.user = self.get_input("Username: "******"No password given.")
self.password = getpass.getpass()
# perform login
qparams = config.params["auth"].copy()
qparams["Bugzilla_login"] = self.user
qparams["Bugzilla_password"] = self.password
if not self.forget:
qparams["Bugzilla_remember"] = "on"
req_url = urljoin(self.base, config.urls["auth"])
req = Request(req_url, urlencode(qparams), config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring("%s:%s" % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
if resp.info().has_key("Set-Cookie"):
self.authenticated = True
if not self.forget:
self.cookiejar.save()
os.chmod(self.cookiejar.filename, 0600)
return True
else:
raise RuntimeError("Failed to login")
def extractResults(self, resp):
# parse the results into dicts.
results = []
columns = []
rows = []
for r in csv.reader(resp):
rows.append(r)
for field in rows[0]:
if config.choices["column_alias"].has_key(field):
columns.append(config.choices["column_alias"][field])
else:
self.log("Unknown field: " + field)
columns.append(field)
for row in rows[1:]:
if "Missing Search" in row[0]:
self.log("Bugzilla error (Missing search found)")
return None
fields = {}
for i in range(min(len(row), len(columns))):
fields[columns[i]] = row[i]
results.append(fields)
return results
def search(
self,
query,
comments=False,
order="number",
assigned_to=None,
reporter=None,
cc=None,
commenter=None,
whiteboard=None,
keywords=None,
status=[],
severity=[],
priority=[],
product=[],
component=[],
):
"""Search bugzilla for a bug.
@param query: query string to search in title or {comments}.
@type query: string
@param order: what order to returns bugs in.
@type order: string
@keyword assigned_to: email address which the bug is assigned to.
@type assigned_to: string
@keyword reporter: email address matching the bug reporter.
@type reporter: string
@keyword cc: email that is contained in the CC list
@type cc: string
@keyword commenter: email of a commenter.
@type commenter: string
@keyword whiteboard: string to search in status whiteboard (gentoo?)
@type whiteboard: string
@keyword keywords: keyword to search for
@type keywords: string
@keyword status: bug status to match. default is ['NEW', 'ASSIGNED',
'REOPENED'].
@type status: list
@keyword severity: severity to match, empty means all.
@type severity: list
@keyword priority: priority levels to patch, empty means all.
@type priority: list
@keyword comments: search comments instead of just bug title.
@type comments: bool
@keyword product: search within products. empty means all.
@type product: list
@keyword component: search within components. empty means all.
@type component: list
@return: list of bugs, each bug represented as a dict
@rtype: list of dicts
"""
if not self.authenticated and not self.skip_auth:
self.auth()
qparams = config.params["list"].copy()
qparams["value0-0-0"] = query
if comments:
qparams["type0-0-1"] = qparams["type0-0-0"]
qparams["value0-0-1"] = query
qparams["order"] = config.choices["order"].get(order, "Bug Number")
qparams["bug_severity"] = severity or []
qparams["priority"] = priority or []
if status is None:
# NEW, ASSIGNED and REOPENED is obsolete as of bugzilla 3.x and has
# been removed from bugs.gentoo.org on 2011/05/01
qparams["bug_status"] = ["NEW", "ASSIGNED", "REOPENED", "UNCONFIRMED", "CONFIRMED", "IN_PROGRESS"]
elif [s.upper() for s in status] == ["ALL"]:
qparams["bug_status"] = config.choices["status"]
else:
qparams["bug_status"] = [s.upper() for s in status]
qparams["product"] = product or ""
qparams["component"] = component or ""
qparams["status_whiteboard"] = whiteboard or ""
qparams["keywords"] = keywords or ""
# hoops to jump through for emails, since there are
# only two fields, we have to figure out what combinations
# to use if all three are set.
unique = list(set([assigned_to, cc, reporter, commenter]))
unique = [u for u in unique if u]
if len(unique) < 3:
for i in range(len(unique)):
e = unique[i]
n = i + 1
qparams["email%d" % n] = e
qparams["emailassigned_to%d" % n] = int(e == assigned_to)
qparams["emailreporter%d" % n] = int(e == reporter)
qparams["emailcc%d" % n] = int(e == cc)
qparams["emaillongdesc%d" % n] = int(e == commenter)
else:
raise AssertionError("Cannot set assigned_to, cc, and " "reporter in the same query")
req_params = urlencode(qparams, True)
req_url = urljoin(self.base, config.urls["list"])
req_url += "?" + req_params
req = Request(req_url, None, config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring("%s:%s" % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
return self.extractResults(resp)
def namedcmd(self, cmd):
"""Run command stored in Bugzilla by name.
@return: Result from the stored command.
@rtype: list of dicts
"""
if not self.authenticated and not self.skip_auth:
self.auth()
qparams = config.params["namedcmd"].copy()
# Is there a better way of getting a command with a space in its name
# to be encoded as foo%20bar instead of foo+bar or foo%2520bar?
qparams["namedcmd"] = quote(cmd)
req_params = urlencode(qparams, True)
req_params = req_params.replace("%25", "%")
req_url = urljoin(self.base, config.urls["list"])
req_url += "?" + req_params
req = Request(req_url, None, config.headers)
if self.user and self.password:
base64string = base64.encodestring("%s:%s" % (self.user, self.password))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
return self.extractResults(resp)
def get(self, bugid):
"""Get an ElementTree representation of a bug.
@param bugid: bug id
@type bugid: int
@rtype: ElementTree
"""
if not self.authenticated and not self.skip_auth:
self.auth()
qparams = config.params["show"].copy()
qparams["id"] = bugid
req_params = urlencode(qparams, True)
req_url = urljoin(self.base, config.urls["show"])
req_url += "?" + req_params
req = Request(req_url, None, config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring("%s:%s" % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
data = resp.read()
# Get rid of control characters.
data = re.sub("[\x00-\x08\x0e-\x1f\x0b\x0c]", "", data)
fd = StringIO(data)
# workaround for ill-defined XML templates in bugzilla 2.20.2
(major_version, minor_version) = (sys.version_info[0], sys.version_info[1])
if major_version > 2 or (major_version == 2 and minor_version >= 7):
# If this is 2.7 or greater, then XMLTreeBuilder
# does what we want.
parser = ElementTree.XMLParser()
else:
# Running under Python 2.6, so we need to use our
# subclass of XMLTreeBuilder instead.
parser = ForcedEncodingXMLTreeBuilder(encoding="utf-8")
etree = ElementTree.parse(fd, parser)
bug = etree.find(".//bug")
if bug is not None and bug.attrib.has_key("error"):
return None
else:
return etree
def modify(
self,
bugid,
title=None,
comment=None,
url=None,
status=None,
resolution=None,
assigned_to=None,
duplicate=0,
priority=None,
severity=None,
add_cc=[],
remove_cc=[],
add_dependson=[],
remove_dependson=[],
add_blocked=[],
remove_blocked=[],
whiteboard=None,
keywords=None,
component=None,
):
"""Modify an existing bug
@param bugid: bug id
@type bugid: int
@keyword title: new title for bug
@type title: string
@keyword comment: comment to add
@type comment: string
@keyword url: new url
@type url: string
@keyword status: new status (note, if you are changing it to RESOLVED, you need to set {resolution} as well.
@type status: string
@keyword resolution: new resolution (if status=RESOLVED)
@type resolution: string
@keyword assigned_to: email (needs to exist in bugzilla)
@type assigned_to: string
@keyword duplicate: bug id to duplicate against (if resolution = DUPLICATE)
@type duplicate: int
@keyword priority: new priority for bug
@type priority: string
@keyword severity: new severity for bug
@type severity: string
@keyword add_cc: list of emails to add to the cc list
@type add_cc: list of strings
@keyword remove_cc: list of emails to remove from cc list
@type remove_cc: list of string.
@keyword add_dependson: list of bug ids to add to the depend list
@type add_dependson: list of strings
@keyword remove_dependson: list of bug ids to remove from depend list
@type remove_dependson: list of strings
@keyword add_blocked: list of bug ids to add to the blocked list
@type add_blocked: list of strings
@keyword remove_blocked: list of bug ids to remove from blocked list
@type remove_blocked: list of strings
@keyword whiteboard: set status whiteboard
@type whiteboard: string
@keyword keywords: set keywords
@type keywords: string
@keyword component: set component
@type component: string
@return: list of fields modified.
@rtype: list of strings
"""
if not self.authenticated and not self.skip_auth:
self.auth()
buginfo = Bugz.get(self, bugid)
if not buginfo:
return False
modified = []
qparams = config.params["modify"].copy()
qparams["id"] = bugid
# NOTE: knob has been removed in bugzilla 4 and 3?
qparams["knob"] = "none"
# copy existing fields
FIELDS = (
"bug_file_loc",
"bug_severity",
"short_desc",
"bug_status",
"status_whiteboard",
"keywords",
"resolution",
"op_sys",
"priority",
"version",
"target_milestone",
"assigned_to",
"rep_platform",
"product",
"component",
"token",
)
FIELDS_MULTI = ("blocked", "dependson")
for field in FIELDS:
try:
qparams[field] = buginfo.find(".//%s" % field).text
if qparams[field] is None:
del qparams[field]
except:
pass
for field in FIELDS_MULTI:
qparams[field] = [d.text for d in buginfo.findall(".//%s" % field) if d is not None and d.text is not None]
# set 'knob' if we are change the status/resolution
# or trying to reassign bug.
if status:
status = status.upper()
if resolution:
resolution = resolution.upper()
if status and status != qparams["bug_status"]:
# Bugzilla >= 3.x
qparams["bug_status"] = status
if status == "RESOLVED":
qparams["knob"] = "resolve"
if resolution:
qparams["resolution"] = resolution
else:
qparams["resolution"] = "FIXED"
modified.append(("status", status))
modified.append(("resolution", qparams["resolution"]))
elif status == "ASSIGNED" or status == "IN_PROGRESS":
qparams["knob"] = "accept"
modified.append(("status", status))
elif status == "REOPENED":
qparams["knob"] = "reopen"
modified.append(("status", status))
elif status == "VERIFIED":
qparams["knob"] = "verified"
modified.append(("status", status))
elif status == "CLOSED":
qparams["knob"] = "closed"
modified.append(("status", status))
elif duplicate:
# Bugzilla >= 3.x
qparams["bug_status"] = "RESOLVED"
qparams["resolution"] = "DUPLICATE"
qparams["knob"] = "duplicate"
qparams["dup_id"] = duplicate
modified.append(("status", "RESOLVED"))
modified.append(("resolution", "DUPLICATE"))
elif assigned_to:
qparams["knob"] = "reassign"
qparams["assigned_to"] = assigned_to
modified.append(("assigned_to", assigned_to))
# setup modification of other bits
if comment:
qparams["comment"] = comment
modified.append(("comment", ellipsis(comment, 60)))
if title:
qparams["short_desc"] = title or ""
modified.append(("title", title))
if url is not None:
qparams["bug_file_loc"] = url
modified.append(("url", url))
if severity is not None:
qparams["bug_severity"] = severity
modified.append(("severity", severity))
if priority is not None:
qparams["priority"] = priority
modified.append(("priority", priority))
# cc manipulation
if add_cc is not None:
qparams["newcc"] = ", ".join(add_cc)
modified.append(("newcc", qparams["newcc"]))
if remove_cc is not None:
qparams["cc"] = remove_cc
qparams["removecc"] = "on"
modified.append(("cc", remove_cc))
# bug depend/blocked manipulation
changed_dependson = False
changed_blocked = False
if remove_dependson:
for bug_id in remove_dependson:
qparams["dependson"].remove(str(bug_id))
changed_dependson = True
if remove_blocked:
for bug_id in remove_blocked:
qparams["blocked"].remove(str(bug_id))
changed_blocked = True
if add_dependson:
for bug_id in add_dependson:
qparams["dependson"].append(str(bug_id))
changed_dependson = True
if add_blocked:
for bug_id in add_blocked:
qparams["blocked"].append(str(bug_id))
changed_blocked = True
qparams["dependson"] = ",".join(qparams["dependson"])
qparams["blocked"] = ",".join(qparams["blocked"])
if changed_dependson:
modified.append(("dependson", qparams["dependson"]))
if changed_blocked:
modified.append(("blocked", qparams["blocked"]))
if whiteboard is not None:
qparams["status_whiteboard"] = whiteboard
modified.append(("status_whiteboard", whiteboard))
if keywords is not None:
qparams["keywords"] = keywords
modified.append(("keywords", keywords))
if component is not None:
qparams["component"] = component
modified.append(("component", component))
req_params = urlencode(qparams, True)
req_url = urljoin(self.base, config.urls["modify"])
req = Request(req_url, req_params, config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring("%s:%s" % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
try:
resp = self.opener.open(req)
re_error = re.compile(r'id="error_msg".*>([^<]+)<')
error = re_error.search(resp.read())
if error:
print error.group(1)
return []
return modified
except:
return []
def attachment(self, attachid):
"""Get an attachment by attachment_id
@param attachid: attachment id
@type attachid: int
@return: dict with three keys, 'filename', 'size', 'fd'
@rtype: dict
"""
if not self.authenticated and not self.skip_auth:
self.auth()
qparams = config.params["attach"].copy()
qparams["id"] = attachid
req_params = urlencode(qparams, True)
req_url = urljoin(self.base, config.urls["attach"])
req_url += "?" + req_params
req = Request(req_url, None, config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring("%s:%s" % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
try:
content_type = resp.info()["Content-type"]
namefield = content_type.split(";")[1]
filename = re.search(r"name=\"(.*)\"", namefield).group(1)
content_length = int(resp.info()["Content-length"], 0)
return {"filename": filename, "size": content_length, "fd": resp}
except:
return {}
def post(
self,
product,
component,
title,
description,
url="",
assigned_to="",
cc="",
keywords="",
version="",
dependson="",
blocked="",
priority="",
severity="",
):
"""Post a bug
@param product: product where the bug should be placed
@type product: string
@param component: component where the bug should be placed
@type component: string
@param title: title of the bug.
@type title: string
@param description: description of the bug
@type description: string
@keyword url: optional url to submit with bug
@type url: string
@keyword assigned_to: optional email to assign bug to
@type assigned_to: string.
@keyword cc: option list of CC'd emails
@type: string
@keyword keywords: option list of bugzilla keywords
@type: string
@keyword version: version of the component
@type: string
@keyword dependson: bugs this one depends on
@type: string
@keyword blocked: bugs this one blocks
@type: string
@keyword priority: priority of this bug
@type: string
@keyword severity: severity of this bug
@type: string
@rtype: int
@return: the bug number, or 0 if submission failed.
"""
if not self.authenticated and not self.skip_auth:
self.auth()
qparams = config.params["post"].copy()
qparams["product"] = product
qparams["component"] = component
qparams["short_desc"] = title
qparams["comment"] = description
qparams["assigned_to"] = assigned_to
qparams["cc"] = cc
qparams["bug_file_loc"] = url
qparams["dependson"] = dependson
qparams["blocked"] = blocked
qparams["keywords"] = keywords
# XXX: default version is 'unspecified'
if version != "":
qparams["version"] = version
# XXX: default priority is 'Normal'
if priority != "":
qparams["priority"] = priority
# XXX: default severity is 'normal'
if severity != "":
qparams["bug_severity"] = severity
req_params = urlencode(qparams, True)
req_url = urljoin(self.base, config.urls["post"])
req = Request(req_url, req_params, config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring("%s:%s" % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
try:
re_bug = re.compile(r"(?:\s+)?<title>.*Bug ([0-9]+) Submitted.*</title>")
bug_match = re_bug.search(resp.read())
if bug_match:
return int(bug_match.group(1))
except:
pass
return 0
def attach(self, bugid, title, description, filename, content_type="text/plain", ispatch=False):
"""Attach a file to a bug.
@param bugid: bug id
@type bugid: int
@param title: short description of attachment
@type title: string
@param description: long description of the attachment
@type description: string
@param filename: filename of the attachment
@type filename: string
@keywords content_type: mime-type of the attachment
@type content_type: string
@rtype: bool
@return: True if successful, False if not successful.
"""
if not self.authenticated and not self.skip_auth:
self.auth()
qparams = config.params["attach_post"].copy()
qparams["bugid"] = bugid
qparams["description"] = title
qparams["comment"] = description
if ispatch:
qparams["ispatch"] = "1"
qparams["contenttypeentry"] = "text/plain"
else:
qparams["contenttypeentry"] = content_type
filedata = [("data", filename, open(filename).read())]
content_type, body = encode_multipart_formdata(qparams.items(), filedata)
req_headers = config.headers.copy()
req_headers["Content-type"] = content_type
req_headers["Content-length"] = len(body)
req_url = urljoin(self.base, config.urls["attach_post"])
req = Request(req_url, body, req_headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring("%s:%s" % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
# TODO: return attachment id and success?
try:
re_attach = re.compile(r"<title>(.+)</title>")
# Bugzilla 3/4
re_attach34 = re.compile(r"Attachment \d+ added to Bug \d+")
response = resp.read()
attach_match = re_attach.search(response)
if attach_match:
if attach_match.group(1) == "Changes Submitted" or re_attach34.match(attach_match.group(1)):
return True
else:
return attach_match.group(1)
else:
return False
except:
pass
return False
class CookieTransport(xmlrpclib.Transport):
'''A subclass of xmlrpclib.Transport that supports cookies.'''
cookiejar = None
scheme = 'http'
# Cribbed from xmlrpclib.Transport.send_user_agent
def send_cookies(self, connection, cookie_request):
if self.cookiejar is None:
self.cookiejar = CookieJar()
elif self.cookiejar:
# Let the cookiejar figure out what cookies are appropriate
self.cookiejar.add_cookie_header(cookie_request)
# Pull the cookie headers out of the request object...
cookielist=list()
for h,v in cookie_request.header_items():
if h.startswith('Cookie'):
cookielist.append([h,v])
# ...and put them over the connection
for h,v in cookielist:
connection.putheader(h,v)
# This is the same request() method from xmlrpclib.Transport,
# with a couple additions noted below
def request_with_cookies(self, host, handler, request_body, verbose=0):
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
# ADDED: construct the URL and Request object for proper cookie handling
request_url = "%s://%s%s" % (self.scheme,host,handler)
#log.debug("request_url is %s" % request_url)
cookie_request = urllib2.Request(request_url)
self.send_request(h,handler,request_body)
self.send_host(h,host)
self.send_cookies(h,cookie_request) # ADDED. creates cookiejar if None.
self.send_user_agent(h)
self.send_content(h,request_body)
errcode, errmsg, headers = h.getreply()
# ADDED: parse headers and get cookies here
cookie_response = CookieResponse(headers)
# Okay, extract the cookies from the headers
self.cookiejar.extract_cookies(cookie_response,cookie_request)
#log.debug("cookiejar now contains: %s" % self.cookiejar._cookies)
# And write back any changes
if hasattr(self.cookiejar,'save'):
try:
self.cookiejar.save(self.cookiejar.filename)
except Exception, e:
raise
#log.error("Couldn't write cookiefile %s: %s" % \
# (self.cookiejar.filename,str(e)))
if errcode != 200:
# When runs here, the HTTPS connection isn't useful any more
# before raising an exception to caller
h.close()
raise xmlrpclib.ProtocolError(
host + handler,
errcode, errmsg,
headers
)
self.verbose = verbose
try:
sock = h._conn.sock
except AttributeError:
sock = None
try:
return self._parse_response(h.getfile(), sock)
finally:
h.close()
class Browser(object):
_HEADERS = {
'User-Agent' : ['Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17', ]
}
_POST_HEADERS = {
'Content-Type' : ['application/x-www-form-urlencoded', ]
}
def __init__(self, page_archiver, cookie_file=None):
self._logger = logging.getLogger(__name__)
self._page_archiver = page_archiver
self._logger.debug('Using page archiver: %s. Cookie file: %s',
page_archiver is not None,
cookie_file)
if cookie_file:
umask = os.umask(077)
self._cj = LWPCookieJar(cookie_file)
try:
self._cj.load()
except LoadError:
self._logger.warning('Cannot load cookies from %s' % (cookie_file, ))
os.umask(umask)
else:
self._cj = CookieJar()
pool = HTTPConnectionPool(reactor, persistent=True)
pool.maxPersistentPerHost = 10
self._agent = CookieAgent(ContentDecoderAgent(Agent(reactor, pool=pool),
[('gzip', GzipDecoder)]), self._cj)
self._lock = Lock()
def save_cookies(self):
try:
self._cj.save()
except LoadError:
pass
else:
self._logger.debug('Cookies saved')
@defer.deferredGenerator
def _request(self, request_type, url, referer=None, body=None):
self._logger.debug('Fetching %s', url)
headers = dict(self._HEADERS)
if referer:
headers['Referer'] = [referer, ]
body_prod = None
if body:
headers.update(self._POST_HEADERS)
body_prod = StringProducer(body)
d = defer.waitForDeferred(self._agent.request(request_type, url, Headers(headers),
body_prod))
yield d
response = d.getResult()
receiver = MemoryReceiver()
response.deliverBody(receiver)
if request_type == 'POST' and (response.code >= 300 and response.code < 400):
new_location = '%s/%s' % (
os.path.split(url)[0],
response.headers.getRawHeaders('location')[0])
d = defer.waitForDeferred(self.get(new_location, referer))
yield d
yield d.getResult()
return
else:
d = defer.waitForDeferred(receiver.finished)
yield d
page = d.getResult()
if self._page_archiver:
reactor.callInThread(self._archive_page,
page, url, body, referer)
yield page
def _archive_page(self, page, url, body, referer):
with self._lock:
self._page_archiver.archive(page, url, body, referer)
def get(self, url, referer=None):
return self._request('GET', url, referer)
def post(self, url, data, referer=None):
self._logger.debug('Posting to %s: %s', url, data)
encoded_data = urlencode(data)
return self._request('POST', url, referer, encoded_data)
class CookieTransport(TimeoutTransport):
'''A subclass of xmlrpclib.Transport that supports cookies.'''
cookiejar = None
scheme = 'http'
# Cribbed from xmlrpclib.Transport.send_user_agent
def send_cookies(self, connection, cookie_request):
if self.cookiejar is None:
self.cookiejar = CookieJar()
elif self.cookiejar:
# Let the cookiejar figure out what cookies are appropriate
self.cookiejar.add_cookie_header(cookie_request)
# Pull the cookie headers out of the request object...
cookielist = list()
for h, v in cookie_request.header_items():
if h.startswith('Cookie'):
cookielist.append([h, v])
# ...and put them over the connection
for h, v in cookielist:
connection.putheader(h, v)
def single_request(self, host, handler, request_body, verbose=1):
# issue XML-RPC request
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
request_url = "%s://%s/" % (self.scheme, host)
cookie_request = urllib2.Request(request_url)
try:
self.send_request(h, handler, request_body)
self.send_host(h, host)
self.send_cookies(
h, cookie_request) # ADDED. creates cookiejar if None.
self.send_user_agent(h)
self.send_content(h, request_body)
response = h.getresponse(buffering=True)
# ADDED: parse headers and get cookies here
# fake a response object that we can fill with the headers above
class CookieResponse:
def __init__(self, headers):
self.headers = headers
def info(self):
return self.headers
cookie_response = CookieResponse(response.msg)
# Okay, extract the cookies from the headers
self.cookiejar.extract_cookies(cookie_response, cookie_request)
# And write back any changes
if hasattr(self.cookiejar, 'save'):
self.cookiejar.save(self.cookiejar.filename)
if response.status == 200:
self.verbose = verbose
return self.parse_response(response)
except xmlrpclib.Fault:
raise
except Exception:
# All unexpected errors leave connection in
# a strange state, so we clear it.
self.close()
raise
# discard any response data and raise exception
if (response.getheader("content-length", 0)):
response.read()
raise xmlrpclib.ProtocolError(
host + handler,
response.status,
response.reason,
response.msg,
)
class Bugz:
""" Converts sane method calls to Bugzilla HTTP requests.
@ivar base: base url of bugzilla.
@ivar user: username for authenticated operations.
@ivar password: password for authenticated operations
@ivar cookiejar: for authenticated sessions so we only auth once.
@ivar forget: forget user/password after session.
@ivar authenticated: is this session authenticated already
"""
def __init__(self,
base,
user=None,
password=None,
forget=False,
skip_auth=False,
httpuser=None,
httppassword=None):
"""
{user} and {password} will be prompted if an action needs them
and they are not supplied.
if {forget} is set, the login cookie will be destroyed on quit.
@param base: base url of the bugzilla
@type base: string
@keyword user: username for authenticated actions.
@type user: string
@keyword password: password for authenticated actions.
@type password: string
@keyword forget: forget login session after termination.
@type forget: bool
@keyword skip_auth: do not authenticate
@type skip_auth: bool
"""
self.base = base
scheme, self.host, self.path, query, frag = urlsplit(self.base)
self.authenticated = False
self.forget = forget
if not self.forget:
try:
cookie_file = os.path.join(os.environ['HOME'], COOKIE_FILE)
self.cookiejar = LWPCookieJar(cookie_file)
if forget:
try:
self.cookiejar.load()
self.cookiejar.clear()
self.cookiejar.save()
os.chmod(self.cookiejar.filename, 0600)
except IOError:
pass
except KeyError:
self.warn('Unable to save session cookies in %s' % cookie_file)
self.cookiejar = CookieJar(cookie_file)
else:
self.cookiejar = CookieJar()
self.opener = build_opener(HTTPCookieProcessor(self.cookiejar))
self.user = user
self.password = password
self.httpuser = httpuser
self.httppassword = httppassword
self.skip_auth = skip_auth
def log(self, status_msg):
"""Default logging handler. Expected to be overridden by
the UI implementing subclass.
@param status_msg: status message to print
@type status_msg: string
"""
return
def warn(self, warn_msg):
"""Default logging handler. Expected to be overridden by
the UI implementing subclass.
@param status_msg: status message to print
@type status_msg: string
"""
return
def get_input(self, prompt):
"""Default input handler. Expected to be override by the
UI implementing subclass.
@param prompt: Prompt message
@type prompt: string
"""
return ''
def auth(self):
"""Authenticate a session.
"""
# check if we need to authenticate
if self.authenticated:
return
# try seeing if we really need to request login
if not self.forget:
try:
self.cookiejar.load()
except IOError:
pass
req_url = urljoin(self.base, config.urls['auth'])
req_url += '?GoAheadAndLogIn=1'
req = Request(req_url, None, config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring(
'%s:%s' % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
re_request_login = re.compile(r'<title>.*Log in to .*</title>')
if not re_request_login.search(resp.read()):
self.log('Already logged in.')
self.authenticated = True
return
# prompt for username if we were not supplied with it
if not self.user:
self.log('No username given.')
self.user = self.get_input('Username: '******'No password given.')
self.password = getpass.getpass()
# perform login
qparams = config.params['auth'].copy()
qparams['Bugzilla_login'] = self.user
qparams['Bugzilla_password'] = self.password
if not self.forget:
qparams['Bugzilla_remember'] = 'on'
req_url = urljoin(self.base, config.urls['auth'])
req = Request(req_url, urlencode(qparams), config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring(
'%s:%s' % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
if resp.info().has_key('Set-Cookie'):
self.authenticated = True
if not self.forget:
self.cookiejar.save()
os.chmod(self.cookiejar.filename, 0600)
return True
else:
raise RuntimeError("Failed to login")
def extractResults(self, resp):
# parse the results into dicts.
results = []
columns = []
rows = []
for r in csv.reader(resp):
rows.append(r)
for field in rows[0]:
if config.choices['column_alias'].has_key(field):
columns.append(config.choices['column_alias'][field])
else:
self.log('Unknown field: ' + field)
columns.append(field)
for row in rows[1:]:
if "Missing Search" in row[0]:
self.log('Bugzilla error (Missing search found)')
return None
fields = {}
for i in range(min(len(row), len(columns))):
fields[columns[i]] = row[i]
results.append(fields)
return results
def search(self,
query,
comments=False,
order='number',
assigned_to=None,
reporter=None,
cc=None,
commenter=None,
whiteboard=None,
keywords=None,
status=[],
severity=[],
priority=[],
product=[],
component=[]):
"""Search bugzilla for a bug.
@param query: query string to search in title or {comments}.
@type query: string
@param order: what order to returns bugs in.
@type order: string
@keyword assigned_to: email address which the bug is assigned to.
@type assigned_to: string
@keyword reporter: email address matching the bug reporter.
@type reporter: string
@keyword cc: email that is contained in the CC list
@type cc: string
@keyword commenter: email of a commenter.
@type commenter: string
@keyword whiteboard: string to search in status whiteboard (gentoo?)
@type whiteboard: string
@keyword keywords: keyword to search for
@type keywords: string
@keyword status: bug status to match. default is ['NEW', 'ASSIGNED',
'REOPENED'].
@type status: list
@keyword severity: severity to match, empty means all.
@type severity: list
@keyword priority: priority levels to patch, empty means all.
@type priority: list
@keyword comments: search comments instead of just bug title.
@type comments: bool
@keyword product: search within products. empty means all.
@type product: list
@keyword component: search within components. empty means all.
@type component: list
@return: list of bugs, each bug represented as a dict
@rtype: list of dicts
"""
if not self.authenticated and not self.skip_auth:
self.auth()
qparams = config.params['list'].copy()
qparams['value0-0-0'] = query
if comments:
qparams['type0-0-1'] = qparams['type0-0-0']
qparams['value0-0-1'] = query
qparams['order'] = config.choices['order'].get(order, 'Bug Number')
qparams['bug_severity'] = severity or []
qparams['priority'] = priority or []
if status is None:
# NEW, ASSIGNED and REOPENED is obsolete as of bugzilla 3.x and has
# been removed from bugs.gentoo.org on 2011/05/01
qparams['bug_status'] = [
'NEW', 'ASSIGNED', 'REOPENED', 'UNCONFIRMED', 'CONFIRMED',
'IN_PROGRESS'
]
elif [s.upper() for s in status] == ['ALL']:
qparams['bug_status'] = config.choices['status']
else:
qparams['bug_status'] = [s.upper() for s in status]
qparams['product'] = product or ''
qparams['component'] = component or ''
qparams['status_whiteboard'] = whiteboard or ''
qparams['keywords'] = keywords or ''
# hoops to jump through for emails, since there are
# only two fields, we have to figure out what combinations
# to use if all three are set.
unique = list(set([assigned_to, cc, reporter, commenter]))
unique = [u for u in unique if u]
if len(unique) < 3:
for i in range(len(unique)):
e = unique[i]
n = i + 1
qparams['email%d' % n] = e
qparams['emailassigned_to%d' % n] = int(e == assigned_to)
qparams['emailreporter%d' % n] = int(e == reporter)
qparams['emailcc%d' % n] = int(e == cc)
qparams['emaillongdesc%d' % n] = int(e == commenter)
else:
raise AssertionError('Cannot set assigned_to, cc, and '
'reporter in the same query')
req_params = urlencode(qparams, True)
req_url = urljoin(self.base, config.urls['list'])
req_url += '?' + req_params
req = Request(req_url, None, config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring(
'%s:%s' % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
return self.extractResults(resp)
def namedcmd(self, cmd):
"""Run command stored in Bugzilla by name.
@return: Result from the stored command.
@rtype: list of dicts
"""
if not self.authenticated and not self.skip_auth:
self.auth()
qparams = config.params['namedcmd'].copy()
# Is there a better way of getting a command with a space in its name
# to be encoded as foo%20bar instead of foo+bar or foo%2520bar?
qparams['namedcmd'] = quote(cmd)
req_params = urlencode(qparams, True)
req_params = req_params.replace('%25', '%')
req_url = urljoin(self.base, config.urls['list'])
req_url += '?' + req_params
req = Request(req_url, None, config.headers)
if self.user and self.password:
base64string = base64.encodestring('%s:%s' %
(self.user, self.password))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
return self.extractResults(resp)
def get(self, bugid):
"""Get an ElementTree representation of a bug.
@param bugid: bug id
@type bugid: int
@rtype: ElementTree
"""
if not self.authenticated and not self.skip_auth:
self.auth()
qparams = config.params['show'].copy()
qparams['id'] = bugid
req_params = urlencode(qparams, True)
req_url = urljoin(self.base, config.urls['show'])
req_url += '?' + req_params
req = Request(req_url, None, config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring(
'%s:%s' % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
data = resp.read()
# Get rid of control characters.
data = re.sub('[\x00-\x08\x0e-\x1f\x0b\x0c]', '', data)
fd = StringIO(data)
# workaround for ill-defined XML templates in bugzilla 2.20.2
(major_version, minor_version) = \
(sys.version_info[0], sys.version_info[1])
if major_version > 2 or \
(major_version == 2 and minor_version >= 7):
# If this is 2.7 or greater, then XMLTreeBuilder
# does what we want.
parser = ElementTree.XMLParser()
else:
# Running under Python 2.6, so we need to use our
# subclass of XMLTreeBuilder instead.
parser = ForcedEncodingXMLTreeBuilder(encoding='utf-8')
etree = ElementTree.parse(fd, parser)
bug = etree.find('.//bug')
if bug is not None and bug.attrib.has_key('error'):
return None
else:
return etree
def modify(self,
bugid,
title=None,
comment=None,
url=None,
status=None,
resolution=None,
assigned_to=None,
duplicate=0,
priority=None,
severity=None,
add_cc=[],
remove_cc=[],
add_dependson=[],
remove_dependson=[],
add_blocked=[],
remove_blocked=[],
whiteboard=None,
keywords=None,
component=None):
"""Modify an existing bug
@param bugid: bug id
@type bugid: int
@keyword title: new title for bug
@type title: string
@keyword comment: comment to add
@type comment: string
@keyword url: new url
@type url: string
@keyword status: new status (note, if you are changing it to RESOLVED, you need to set {resolution} as well.
@type status: string
@keyword resolution: new resolution (if status=RESOLVED)
@type resolution: string
@keyword assigned_to: email (needs to exist in bugzilla)
@type assigned_to: string
@keyword duplicate: bug id to duplicate against (if resolution = DUPLICATE)
@type duplicate: int
@keyword priority: new priority for bug
@type priority: string
@keyword severity: new severity for bug
@type severity: string
@keyword add_cc: list of emails to add to the cc list
@type add_cc: list of strings
@keyword remove_cc: list of emails to remove from cc list
@type remove_cc: list of string.
@keyword add_dependson: list of bug ids to add to the depend list
@type add_dependson: list of strings
@keyword remove_dependson: list of bug ids to remove from depend list
@type remove_dependson: list of strings
@keyword add_blocked: list of bug ids to add to the blocked list
@type add_blocked: list of strings
@keyword remove_blocked: list of bug ids to remove from blocked list
@type remove_blocked: list of strings
@keyword whiteboard: set status whiteboard
@type whiteboard: string
@keyword keywords: set keywords
@type keywords: string
@keyword component: set component
@type component: string
@return: list of fields modified.
@rtype: list of strings
"""
if not self.authenticated and not self.skip_auth:
self.auth()
buginfo = Bugz.get(self, bugid)
if not buginfo:
return False
modified = []
qparams = config.params['modify'].copy()
qparams['id'] = bugid
# NOTE: knob has been removed in bugzilla 4 and 3?
qparams['knob'] = 'none'
# copy existing fields
FIELDS = ('bug_file_loc', 'bug_severity', 'short_desc', 'bug_status',
'status_whiteboard', 'keywords', 'resolution', 'op_sys',
'priority', 'version', 'target_milestone', 'assigned_to',
'rep_platform', 'product', 'component', 'token')
FIELDS_MULTI = ('blocked', 'dependson')
for field in FIELDS:
try:
qparams[field] = buginfo.find('.//%s' % field).text
if qparams[field] is None:
del qparams[field]
except:
pass
for field in FIELDS_MULTI:
qparams[field] = [
d.text for d in buginfo.findall('.//%s' % field)
if d is not None and d.text is not None
]
# set 'knob' if we are change the status/resolution
# or trying to reassign bug.
if status:
status = status.upper()
if resolution:
resolution = resolution.upper()
if status and status != qparams['bug_status']:
# Bugzilla >= 3.x
qparams['bug_status'] = status
if status == 'RESOLVED':
qparams['knob'] = 'resolve'
if resolution:
qparams['resolution'] = resolution
else:
qparams['resolution'] = 'FIXED'
modified.append(('status', status))
modified.append(('resolution', qparams['resolution']))
elif status == 'ASSIGNED' or status == 'IN_PROGRESS':
qparams['knob'] = 'accept'
modified.append(('status', status))
elif status == 'REOPENED':
qparams['knob'] = 'reopen'
modified.append(('status', status))
elif status == 'VERIFIED':
qparams['knob'] = 'verified'
modified.append(('status', status))
elif status == 'CLOSED':
qparams['knob'] = 'closed'
modified.append(('status', status))
elif duplicate:
# Bugzilla >= 3.x
qparams['bug_status'] = "RESOLVED"
qparams['resolution'] = "DUPLICATE"
qparams['knob'] = 'duplicate'
qparams['dup_id'] = duplicate
modified.append(('status', 'RESOLVED'))
modified.append(('resolution', 'DUPLICATE'))
elif assigned_to:
qparams['knob'] = 'reassign'
qparams['assigned_to'] = assigned_to
modified.append(('assigned_to', assigned_to))
# setup modification of other bits
if comment:
qparams['comment'] = comment
modified.append(('comment', ellipsis(comment, 60)))
if title:
qparams['short_desc'] = title or ''
modified.append(('title', title))
if url is not None:
qparams['bug_file_loc'] = url
modified.append(('url', url))
if severity is not None:
qparams['bug_severity'] = severity
modified.append(('severity', severity))
if priority is not None:
qparams['priority'] = priority
modified.append(('priority', priority))
# cc manipulation
if add_cc is not None:
qparams['newcc'] = ', '.join(add_cc)
modified.append(('newcc', qparams['newcc']))
if remove_cc is not None:
qparams['cc'] = remove_cc
qparams['removecc'] = 'on'
modified.append(('cc', remove_cc))
# bug depend/blocked manipulation
changed_dependson = False
changed_blocked = False
if remove_dependson:
for bug_id in remove_dependson:
qparams['dependson'].remove(str(bug_id))
changed_dependson = True
if remove_blocked:
for bug_id in remove_blocked:
qparams['blocked'].remove(str(bug_id))
changed_blocked = True
if add_dependson:
for bug_id in add_dependson:
qparams['dependson'].append(str(bug_id))
changed_dependson = True
if add_blocked:
for bug_id in add_blocked:
qparams['blocked'].append(str(bug_id))
changed_blocked = True
qparams['dependson'] = ','.join(qparams['dependson'])
qparams['blocked'] = ','.join(qparams['blocked'])
if changed_dependson:
modified.append(('dependson', qparams['dependson']))
if changed_blocked:
modified.append(('blocked', qparams['blocked']))
if whiteboard is not None:
qparams['status_whiteboard'] = whiteboard
modified.append(('status_whiteboard', whiteboard))
if keywords is not None:
qparams['keywords'] = keywords
modified.append(('keywords', keywords))
if component is not None:
qparams['component'] = component
modified.append(('component', component))
req_params = urlencode(qparams, True)
req_url = urljoin(self.base, config.urls['modify'])
req = Request(req_url, req_params, config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring(
'%s:%s' % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
try:
resp = self.opener.open(req)
re_error = re.compile(r'id="error_msg".*>([^<]+)<')
error = re_error.search(resp.read())
if error:
print error.group(1)
return []
return modified
except:
return []
def attachment(self, attachid):
"""Get an attachment by attachment_id
@param attachid: attachment id
@type attachid: int
@return: dict with three keys, 'filename', 'size', 'fd'
@rtype: dict
"""
if not self.authenticated and not self.skip_auth:
self.auth()
qparams = config.params['attach'].copy()
qparams['id'] = attachid
req_params = urlencode(qparams, True)
req_url = urljoin(self.base, config.urls['attach'])
req_url += '?' + req_params
req = Request(req_url, None, config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring(
'%s:%s' % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
try:
content_type = resp.info()['Content-type']
namefield = content_type.split(';')[1]
filename = re.search(r'name=\"(.*)\"', namefield).group(1)
content_length = int(resp.info()['Content-length'], 0)
return {'filename': filename, 'size': content_length, 'fd': resp}
except:
return {}
def post(self,
product,
component,
title,
description,
url='',
assigned_to='',
cc='',
keywords='',
version='',
dependson='',
blocked='',
priority='',
severity=''):
"""Post a bug
@param product: product where the bug should be placed
@type product: string
@param component: component where the bug should be placed
@type component: string
@param title: title of the bug.
@type title: string
@param description: description of the bug
@type description: string
@keyword url: optional url to submit with bug
@type url: string
@keyword assigned_to: optional email to assign bug to
@type assigned_to: string.
@keyword cc: option list of CC'd emails
@type: string
@keyword keywords: option list of bugzilla keywords
@type: string
@keyword version: version of the component
@type: string
@keyword dependson: bugs this one depends on
@type: string
@keyword blocked: bugs this one blocks
@type: string
@keyword priority: priority of this bug
@type: string
@keyword severity: severity of this bug
@type: string
@rtype: int
@return: the bug number, or 0 if submission failed.
"""
if not self.authenticated and not self.skip_auth:
self.auth()
qparams = config.params['post'].copy()
qparams['product'] = product
qparams['component'] = component
qparams['short_desc'] = title
qparams['comment'] = description
qparams['assigned_to'] = assigned_to
qparams['cc'] = cc
qparams['bug_file_loc'] = url
qparams['dependson'] = dependson
qparams['blocked'] = blocked
qparams['keywords'] = keywords
#XXX: default version is 'unspecified'
if version != '':
qparams['version'] = version
#XXX: default priority is 'Normal'
if priority != '':
qparams['priority'] = priority
#XXX: default severity is 'normal'
if severity != '':
qparams['bug_severity'] = severity
req_params = urlencode(qparams, True)
req_url = urljoin(self.base, config.urls['post'])
req = Request(req_url, req_params, config.headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring(
'%s:%s' % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
try:
re_bug = re.compile(
r'(?:\s+)?<title>.*Bug ([0-9]+) Submitted.*</title>')
bug_match = re_bug.search(resp.read())
if bug_match:
return int(bug_match.group(1))
except:
pass
return 0
def attach(self,
bugid,
title,
description,
filename,
content_type='text/plain',
ispatch=False):
"""Attach a file to a bug.
@param bugid: bug id
@type bugid: int
@param title: short description of attachment
@type title: string
@param description: long description of the attachment
@type description: string
@param filename: filename of the attachment
@type filename: string
@keywords content_type: mime-type of the attachment
@type content_type: string
@rtype: bool
@return: True if successful, False if not successful.
"""
if not self.authenticated and not self.skip_auth:
self.auth()
qparams = config.params['attach_post'].copy()
qparams['bugid'] = bugid
qparams['description'] = title
qparams['comment'] = description
if ispatch:
qparams['ispatch'] = '1'
qparams['contenttypeentry'] = 'text/plain'
else:
qparams['contenttypeentry'] = content_type
filedata = [('data', filename, open(filename).read())]
content_type, body = encode_multipart_formdata(qparams.items(),
filedata)
req_headers = config.headers.copy()
req_headers['Content-type'] = content_type
req_headers['Content-length'] = len(body)
req_url = urljoin(self.base, config.urls['attach_post'])
req = Request(req_url, body, req_headers)
if self.httpuser and self.httppassword:
base64string = base64.encodestring(
'%s:%s' % (self.httpuser, self.httppassword))[:-1]
req.add_header("Authorization", "Basic %s" % base64string)
resp = self.opener.open(req)
# TODO: return attachment id and success?
try:
re_attach = re.compile(r'<title>(.+)</title>')
# Bugzilla 3/4
re_attach34 = re.compile(r'Attachment \d+ added to Bug \d+')
response = resp.read()
attach_match = re_attach.search(response)
if attach_match:
if attach_match.group(
1) == "Changes Submitted" or re_attach34.match(
attach_match.group(1)):
return True
else:
return attach_match.group(1)
else:
return False
except:
pass
return False
class CookieTransport(TimeoutTransport):
'''A subclass of xmlrpclib.Transport that supports cookies.'''
cookiejar = None
scheme = 'http'
# Cribbed from xmlrpclib.Transport.send_user_agent
def send_cookies(self, connection, cookie_request):
if self.cookiejar is None:
self.cookiejar = CookieJar()
elif self.cookiejar:
# Let the cookiejar figure out what cookies are appropriate
self.cookiejar.add_cookie_header(cookie_request)
# Pull the cookie headers out of the request object...
cookielist = list()
for h, v in cookie_request.header_items():
if h.startswith('Cookie'):
cookielist.append([h, v])
# ...and put them over the connection
for h, v in cookielist:
connection.putheader(h, v)
# This is the same request() method from xmlrpclib.Transport,
# with a couple additions noted below
def request(self, host, handler, request_body, verbose=0):
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
request_url = "%s://%s/" % (self.scheme, host)
cookie_request = urllib2.Request(request_url)
self.send_request(h, handler, request_body)
self.send_host(h, host)
self.send_cookies(h,
cookie_request) # ADDED. creates cookiejar if None.
self.send_user_agent(h)
self.send_content(h, request_body)
errcode, errmsg, headers = h.getreply()
# ADDED: parse headers and get cookies here
# fake a response object that we can fill with the headers above
class CookieResponse:
def __init__(self, headers):
self.headers = headers
def info(self):
return self.headers
cookie_response = CookieResponse(headers)
# Okay, extract the cookies from the headers
self.cookiejar.extract_cookies(cookie_response, cookie_request)
# And write back any changes
if hasattr(self.cookiejar, 'save'):
self.cookiejar.save(self.cookiejar.filename)
if errcode != 200:
raise xmlrpclib.ProtocolError(host + handler, errcode, errmsg,
headers)
self.verbose = verbose
try:
sock = h._conn.sock
except AttributeError:
sock = None
return self._parse_response(h.getfile(), sock)
class CookieTransport(TimeoutTransport):
"""A subclass of xmlrpclib.Transport that supports cookies."""
cookiejar = None
scheme = "http"
# Cribbed from xmlrpclib.Transport.send_user_agent
def send_cookies(self, connection, cookie_request):
if self.cookiejar is None:
self.cookiejar = CookieJar()
elif self.cookiejar:
# Let the cookiejar figure out what cookies are appropriate
self.cookiejar.add_cookie_header(cookie_request)
# Pull the cookie headers out of the request object...
cookielist = list()
for h, v in cookie_request.header_items():
if h.startswith("Cookie"):
cookielist.append([h, v])
# ...and put them over the connection
for h, v in cookielist:
connection.putheader(h, v)
def single_request(self, host, handler, request_body, verbose=1):
# issue XML-RPC request
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
request_url = "%s://%s/" % (self.scheme, host)
cookie_request = urllib2.Request(request_url)
try:
self.send_request(h, handler, request_body)
self.send_host(h, host)
self.send_cookies(h, cookie_request) # ADDED. creates cookiejar if None.
self.send_user_agent(h)
self.send_content(h, request_body)
response = h.getresponse(buffering=True)
# ADDED: parse headers and get cookies here
# fake a response object that we can fill with the headers above
class CookieResponse:
def __init__(self, headers):
self.headers = headers
def info(self):
return self.headers
cookie_response = CookieResponse(response.msg)
# Okay, extract the cookies from the headers
self.cookiejar.extract_cookies(cookie_response, cookie_request)
# And write back any changes
if hasattr(self.cookiejar, "save"):
self.cookiejar.save(self.cookiejar.filename)
if response.status == 200:
self.verbose = verbose
return self.parse_response(response)
except xmlrpclib.Fault:
raise
except Exception:
# All unexpected errors leave connection in
# a strange state, so we clear it.
self.close()
raise
# discard any response data and raise exception
if response.getheader("content-length", 0):
response.read()
raise xmlrpclib.ProtocolError(host + handler, response.status, response.reason, response.msg)
