class Bugz: """ Converts sane method calls to Bugzilla HTTP requests. @ivar base: base url of bugzilla. @ivar user: username for authenticated operations. @ivar password: password for authenticated operations @ivar cookiejar: for authenticated sessions so we only auth once. @ivar forget: forget user/password after session. @ivar authenticated: is this session authenticated already """ def __init__(self, base, user = None, password = None, forget = False, skip_auth = False, httpuser = None, httppassword = None ): """ {user} and {password} will be prompted if an action needs them and they are not supplied. if {forget} is set, the login cookie will be destroyed on quit. @param base: base url of the bugzilla @type base: string @keyword user: username for authenticated actions. @type user: string @keyword password: password for authenticated actions. @type password: string @keyword forget: forget login session after termination. @type forget: bool @keyword skip_auth: do not authenticate @type skip_auth: bool """ self.base = base scheme, self.host, self.path, query, frag = urlsplit(self.base) self.authenticated = False self.forget = forget if not self.forget: try: cookie_file = os.path.join(os.environ['HOME'], COOKIE_FILE) self.cookiejar = LWPCookieJar(cookie_file) if forget: try: self.cookiejar.load() self.cookiejar.clear() self.cookiejar.save() os.chmod(self.cookiejar.filename, 0700) except IOError: pass except KeyError: self.warn('Unable to save session cookies in %s' % cookie_file) self.cookiejar = CookieJar(cookie_file) else: self.cookiejar = CookieJar() self.opener = build_opener(HTTPCookieProcessor(self.cookiejar)) self.user = user self.password = password self.httpuser = httpuser self.httppassword = httppassword self.skip_auth = skip_auth def log(self, status_msg): """Default logging handler. Expected to be overridden by the UI implementing subclass. @param status_msg: status message to print @type status_msg: string """ return def warn(self, warn_msg): """Default logging handler. Expected to be overridden by the UI implementing subclass. @param status_msg: status message to print @type status_msg: string """ return def get_input(self, prompt): """Default input handler. Expected to be override by the UI implementing subclass. @param prompt: Prompt message @type prompt: string """ return '' def auth(self): """Authenticate a session. """ # check if we need to authenticate if self.authenticated: return # try seeing if we really need to request login if not self.forget: try: self.cookiejar.load() except IOError: pass req_url = urljoin(self.base, config.urls['auth']) req_url += '?GoAheadAndLogIn=1' req = Request(req_url, None, config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) re_request_login = re.compile(r'<title>.*Log in to Bugzilla</title>') if not re_request_login.search(resp.read()): self.log('Already logged in.') self.authenticated = True return # prompt for username if we were not supplied with it if not self.user: self.log('No username given.') self.user = self.get_input('Username: '******'No password given.') self.password = getpass.getpass() # perform login qparams = config.params['auth'].copy() qparams['Bugzilla_login'] = self.user qparams['Bugzilla_password'] = self.password if not self.forget: qparams['Bugzilla_remember'] = 'on' req_url = urljoin(self.base, config.urls['auth']) req = Request(req_url, urlencode(qparams), config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) if resp.info().has_key('Set-Cookie'): self.authenticated = True if not self.forget: self.cookiejar.save() os.chmod(self.cookiejar.filename, 0700) return True else: raise RuntimeError("Failed to login") def extractResults(self, resp): # parse the results into dicts. results = [] columns = [] rows = [] for r in csv.reader(resp): rows.append(r) for field in rows[0]: if config.choices['column_alias'].has_key(field): columns.append(config.choices['column_alias'][field]) else: self.log('Unknown field: ' + field) columns.append(field) for row in rows[1:]: if row[0].find("Missing Search") != -1: self.log('Bugzilla error (Missing search found)') return None fields = {} for i in range(min(len(row), len(columns))): fields[columns[i]] = row[i] results.append(fields) return results def search(self, query, comments = False, order = 'number', assigned_to = None, reporter = None, cc = None, commenter = None, whiteboard = None, keywords = None, status = [], severity = [], priority = [], product = [], component = []): """Search bugzilla for a bug. @param query: query string to search in title or {comments}. @type query: string @param order: what order to returns bugs in. @type order: string @keyword assigned_to: email address which the bug is assigned to. @type assigned_to: string @keyword reporter: email address matching the bug reporter. @type reporter: string @keyword cc: email that is contained in the CC list @type cc: string @keyword commenter: email of a commenter. @type commenter: string @keyword whiteboard: string to search in status whiteboard (gentoo?) @type whiteboard: string @keyword keywords: keyword to search for @type keywords: string @keyword status: bug status to match. default is ['NEW', 'ASSIGNED', 'REOPENED']. @type status: list @keyword severity: severity to match, empty means all. @type severity: list @keyword priority: priority levels to patch, empty means all. @type priority: list @keyword comments: search comments instead of just bug title. @type comments: bool @keyword product: search within products. empty means all. @type product: list @keyword component: search within components. empty means all. @type component: list @return: list of bugs, each bug represented as a dict @rtype: list of dicts """ if not self.authenticated and not self.skip_auth: self.auth() qparams = config.params['list'].copy() if comments: qparams['long_desc'] = query else: qparams['short_desc'] = query qparams['order'] = config.choices['order'].get(order, 'Bug Number') qparams['bug_severity'] = severity or [] qparams['priority'] = priority or [] if status == None: qparams['bug_status'] = ['NEW', 'ASSIGNED', 'REOPENED'] elif [s.upper() for s in status] == ['ALL']: qparams['bug_status'] = config.choices['status'] else: qparams['bug_status'] = [s.upper() for s in status] qparams['product'] = product or '' qparams['component'] = component or '' qparams['status_whiteboard'] = whiteboard or '' qparams['keywords'] = keywords or '' # hoops to jump through for emails, since there are # only two fields, we have to figure out what combinations # to use if all three are set. unique = list(set([assigned_to, cc, reporter, commenter])) unique = [u for u in unique if u] if len(unique) < 3: for i in range(len(unique)): e = unique[i] n = i + 1 qparams['email%d' % n] = e qparams['emailassigned_to%d' % n] = int(e == assigned_to) qparams['emailreporter%d' % n] = int(e == reporter) qparams['emailcc%d' % n] = int(e == cc) qparams['emaillongdesc%d' % n] = int(e == commenter) else: raise AssertionError('Cannot set assigned_to, cc, and ' 'reporter in the same query') req_params = urlencode(qparams, True) req_url = urljoin(self.base, config.urls['list']) req_url += '?' + req_params req = Request(req_url, None, config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) return self.extractResults(resp) def namedcmd(self, cmd): """Run command stored in Bugzilla by name. @return: Result from the stored command. @rtype: list of dicts """ if not self.authenticated and not self.skip_auth: self.auth() qparams = config.params['namedcmd'].copy() # Is there a better way of getting a command with a space in its name # to be encoded as foo%20bar instead of foo+bar or foo%2520bar? qparams['namedcmd'] = quote(cmd) req_params = urlencode(qparams, True) req_params = req_params.replace('%25','%') req_url = urljoin(self.base, config.urls['list']) req_url += '?' + req_params req = Request(req_url, None, config.headers) if self.user and self.hpassword: base64string = base64.encodestring('%s:%s' % (self.user, self.hpassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) return self.extractResults(resp) def get(self, bugid): """Get an ElementTree representation of a bug. @param bugid: bug id @type bugid: int @rtype: ElementTree """ if not self.authenticated and not self.skip_auth: self.auth() qparams = config.params['show'].copy() qparams['id'] = bugid req_params = urlencode(qparams, True) req_url = urljoin(self.base, config.urls['show']) req_url += '?' + req_params req = Request(req_url, None, config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) fd = StringIO(resp.read()) # workaround for ill-defined XML templates in bugzilla 2.20.2 parser = ForcedEncodingXMLTreeBuilder(encoding = 'utf-8') etree = ElementTree.parse(fd, parser) bug = etree.find('.//bug') if bug and bug.attrib.has_key('error'): return None else: return etree def modify(self, bugid, title = None, comment = None, url = None, status = None, resolution = None, assigned_to = None, duplicate = 0, priority = None, severity = None, add_cc = [], remove_cc = [], add_dependson = [], remove_dependson = [], add_blocked = [], remove_blocked = [], whiteboard = None, keywords = None): """Modify an existing bug @param bugid: bug id @type bugid: int @keyword title: new title for bug @type title: string @keyword comment: comment to add @type comment: string @keyword url: new url @type url: string @keyword status: new status (note, if you are changing it to RESOLVED, you need to set {resolution} as well. @type status: string @keyword resolution: new resolution (if status=RESOLVED) @type resolution: string @keyword assigned_to: email (needs to exist in bugzilla) @type assigned_to: string @keyword duplicate: bug id to duplicate against (if resolution = DUPLICATE) @type duplicate: int @keyword priority: new priority for bug @type priority: string @keyword severity: new severity for bug @type severity: string @keyword add_cc: list of emails to add to the cc list @type add_cc: list of strings @keyword remove_cc: list of emails to remove from cc list @type remove_cc: list of string. @keyword add_dependson: list of bug ids to add to the depend list @type add_dependson: list of strings @keyword remove_dependson: list of bug ids to remove from depend list @type remove_dependson: list of strings @keyword add_blocked: list of bug ids to add to the blocked list @type add_blocked: list of strings @keyword remove_blocked: list of bug ids to remove from blocked list @type remove_blocked: list of strings @keyword whiteboard: set status whiteboard @type whiteboard: string @keyword keywords: set keywords @type keywords: string @return: list of fields modified. @rtype: list of strings """ if not self.authenticated and not self.skip_auth: self.auth() buginfo = Bugz.get(self, bugid) if not buginfo: return False modified = [] qparams = config.params['modify'].copy() qparams['id'] = bugid qparams['knob'] = 'none' # copy existing fields FIELDS = ('bug_file_loc', 'bug_severity', 'short_desc', 'bug_status', 'status_whiteboard', 'keywords', 'op_sys', 'priority', 'version', 'target_milestone', 'assigned_to', 'rep_platform', 'product', 'component') FIELDS_MULTI = ('blocked', 'dependson') for field in FIELDS: try: qparams[field] = buginfo.find('.//%s' % field).text except: pass for field in FIELDS_MULTI: qparams[field] = [d.text for d in buginfo.findall('.//%s' % field)] # set 'knob' if we are change the status/resolution # or trying to reassign bug. if status: status = status.upper() if resolution: resolution = resolution.upper() if status == 'RESOLVED' and status != qparams['bug_status']: qparams['knob'] = 'resolve' if resolution: qparams['resolution'] = resolution else: qparams['resolution'] = 'FIXED' modified.append(('status', status)) modified.append(('resolution', qparams['resolution'])) elif status == 'ASSIGNED' and status != qparams['bug_status']: qparams['knob'] = 'accept' modified.append(('status', status)) elif status == 'REOPENED' and status != qparams['bug_status']: qparams['knob'] = 'reopen' modified.append(('status', status)) elif status == 'VERIFIED' and status != qparams['bug_status']: qparams['knob'] = 'verified' modified.append(('status', status)) elif status == 'CLOSED' and status != qparams['bug_status']: qparams['knob'] = 'closed' modified.append(('status', status)) elif duplicate: qparams['knob'] = 'duplicate' qparams['dup_id'] = duplicate modified.append(('status', 'RESOLVED')) modified.append(('resolution', 'DUPLICATE')) elif assigned_to: qparams['knob'] = 'reassign' qparams['assigned_to'] = assigned_to modified.append(('assigned_to', assigned_to)) # setup modification of other bits if comment: qparams['comment'] = comment modified.append(('comment', ellipsis(comment, 60))) if title: qparams['short_desc'] = title or '' modified.append(('title', title)) if url != None: qparams['bug_file_loc'] = url modified.append(('url', url)) if severity != None: qparams['bug_severity'] = severity modified.append(('severity', severity)) if priority != None: qparams['priority'] = priority modified.append(('priority', priority)) # cc manipulation if add_cc != None: qparams['newcc'] = ', '.join(add_cc) modified.append(('newcc', qparams['newcc'])) if remove_cc != None: qparams['cc'] = remove_cc qparams['removecc'] = 'on' modified.append(('cc', remove_cc)) # bug depend/blocked manipulation changed_dependson = False changed_blocked = False if remove_dependson: for bug_id in remove_dependson: qparams['dependson'].remove(str(bug_id)) changed_dependson = True if remove_blocked: for bug_id in remove_blocked: qparams['blocked'].remove(str(bug_id)) changed_blocked = True if add_dependson: for bug_id in add_dependson: qparams['dependson'].append(str(bug_id)) changed_dependson = True if add_blocked: for bug_id in add_blocked: qparams['blocked'].append(str(bug_id)) changed_blocked = True qparams['dependson'] = ','.join(qparams['dependson']) qparams['blocked'] = ','.join(qparams['blocked']) if changed_dependson: modified.append(('dependson', qparams['dependson'])) if changed_blocked: modified.append(('blocked', qparams['blocked'])) if whiteboard != None: qparams['status_whiteboard'] = whiteboard modified.append(('status_whiteboard', whiteboard)) if keywords != None: qparams['keywords'] = keywords modified.append(('keywords', keywords)) req_params = urlencode(qparams, True) req_url = urljoin(self.base, config.urls['modify']) req = Request(req_url, req_params, config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) try: resp = self.opener.open(req) return modified except: return [] def attachment(self, attachid): """Get an attachment by attachment_id @param attachid: attachment id @type attachid: int @return: dict with three keys, 'filename', 'size', 'fd' @rtype: dict """ if not self.authenticated and not self.skip_auth: self.auth() qparams = config.params['attach'].copy() qparams['id'] = attachid req_params = urlencode(qparams, True) req_url = urljoin(self.base, config.urls['attach']) req_url += '?' + req_params req = Request(req_url, None, config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) try: content_type = resp.info()['Content-type'] namefield = content_type.split(';')[1] filename = re.search(r'name=\"(.*)\"', namefield).group(1) content_length = int(resp.info()['Content-length'], 0) return {'filename': filename, 'size': content_length, 'fd': resp} except: return {} def post(self, product, component, title, description, url = '', assigned_to = '', cc = '', keywords = '', version = '', dependson = '', blocked = '', priority = '', severity = ''): """Post a bug @param product: product where the bug should be placed @type product: string @param component: component where the bug should be placed @type component: string @param title: title of the bug. @type title: string @param description: description of the bug @type description: string @keyword url: optional url to submit with bug @type url: string @keyword assigned_to: optional email to assign bug to @type assigned_to: string. @keyword cc: option list of CC'd emails @type: string @keyword keywords: option list of bugzilla keywords @type: string @keyword version: version of the component @type: string @keyword dependson: bugs this one depends on @type: string @keyword blocked: bugs this one blocks @type: string @keyword priority: priority of this bug @type: string @keyword severity: severity of this bug @type: string @rtype: int @return: the bug number, or 0 if submission failed. """ if not self.authenticated and not self.skip_auth: self.auth() qparams = config.params['post'].copy() qparams['product'] = product qparams['component'] = component qparams['short_desc'] = title qparams['comment'] = description qparams['assigned_to'] = assigned_to qparams['cc'] = cc qparams['bug_file_loc'] = url qparams['dependson'] = dependson qparams['blocked'] = blocked qparams['keywords'] = keywords #XXX: default version is 'unspecified' if version != '': qparams['version'] = version #XXX: default priority is 'P2' if priority != '': qparams['priority'] = priority #XXX: default severity is 'normal' if severity != '': qparams['bug_severity'] = severity req_params = urlencode(qparams, True) req_url = urljoin(self.base, config.urls['post']) req = Request(req_url, req_params, config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) try: re_bug = re.compile(r'<title>.*Bug ([0-9]+) Submitted</title>') bug_match = re_bug.search(resp.read()) if bug_match: return int(bug_match.group(1)) except: pass return 0 def attach(self, bugid, title, description, filename, content_type = 'text/plain'): """Attach a file to a bug. @param bugid: bug id @type bugid: int @param title: short description of attachment @type title: string @param description: long description of the attachment @type description: string @param filename: filename of the attachment @type filename: string @keywords content_type: mime-type of the attachment @type content_type: string @rtype: bool @return: True if successful, False if not successful. """ if not self.authenticated and not self.skip_auth: self.auth() qparams = config.params['attach_post'].copy() qparams['bugid'] = bugid qparams['description'] = title qparams['comment'] = description qparams['contenttypeentry'] = content_type filedata = [('data', filename, open(filename).read())] content_type, body = encode_multipart_formdata(qparams.items(), filedata) req_headers = config.headers.copy() req_headers['Content-type'] = content_type req_headers['Content-length'] = len(body) req_url = urljoin(self.base, config.urls['attach_post']) req = Request(req_url, body, req_headers) if self.httpuser and self.httppassword: base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) # TODO: return attachment id and success? try: re_success = re.compile(r'<title>Changes Submitted</title>') if re_success.search(resp.read()): return True except: pass return False
class _RequestsHandler(object): def __init__(self, cache_dir=None, web_time_out=30, cookie_jar=None, ignore_ssl_errors=False): """ Initialises the UriHandler class Keyword Arguments: :param str cache_dir: A path for http caching. If specified, caching will be used. :param int web_time_out: Timeout for requests in seconds :param str cookie_jar: The path to the cookie jar (in case of file storage) :param ignore_ssl_errors: Ignore any SSL certificate errors. """ self.id = int(time.time()) if cookie_jar: self.cookieJar = MozillaCookieJar(cookie_jar) if not os.path.isfile(cookie_jar): self.cookieJar.save() self.cookieJar.load() self.cookieJarFile = True else: self.cookieJar = CookieJar() self.cookieJarFile = False self.cacheDir = cache_dir self.cacheStore = None if cache_dir: self.cacheStore = StreamCache(cache_dir) Logger.debug("Opened %s", self.cacheStore) else: Logger.debug("No cache-store provided. Cached disabled.") self.userAgent = "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13 (.NET CLR 3.5.30729)" self.webTimeOut = web_time_out # max duration of request self.ignoreSslErrors = ignore_ssl_errors # ignore SSL errors if self.ignoreSslErrors: Logger.warning("Ignoring all SSL errors in Python") # status of the most recent call self.status = UriStatus(code=0, url=None, error=False, reason=None) # for download animation self.__animationIndex = -1 def download(self, uri, filename, folder, progress_callback=None, proxy=None, params="", data="", json="", referer=None, additional_headers=None): """ Downloads a remote file :param str filename: The filename that should be used to store the file. :param str folder: The folder to save the file in. :param str params: Data to send with the request (open(uri, params)). :param str uri: The URI to download. :param dict[str, any]|str data: Data to send with the request (open(uri, data)). :param dict[str, any] json: Json to send with the request (open(uri, params)). :param ProxyInfo proxy: The address and port (proxy.address.ext:port) of a proxy server that should be used. :param str referer: The http referer to use. :param dict additional_headers: The optional headers. :param function progress_callback: The callback for progress update. The format is function(retrievedSize, totalSize, perc, completed, status) :return: The full path of the location to which it was downloaded. :rtype: str """ if not folder or not filename: raise ValueError( "Destination folder and filename should be specified") if not os.path.isdir(folder): raise ValueError("Destination folder is not a valid location") if not progress_callback: raise ValueError("A callback must be specified") download_path = os.path.join(folder, filename) if os.path.isfile(download_path): Logger.info("Url already downloaded to: %s", download_path) return download_path Logger.info("Creating Downloader for url '%s' to filename '%s'", uri, download_path) r = self.__requests(uri, proxy=proxy, params=params, data=data, json=json, referer=referer, additional_headers=additional_headers, no_cache=True, stream=True) if r is None: return "" retrieved_bytes = 0 total_size = int(r.headers.get('Content-Length', '0').strip()) # There is an issue with the way Requests checks for input and it does not like the newInt. if PY2: chunk_size = 10 * 1024 else: chunk_size = 1024 if total_size == 0 else total_size // 100 cancel = False with open(download_path, 'wb') as fd: for chunk in r.iter_content(chunk_size=chunk_size): fd.write(chunk) retrieved_bytes += len(chunk) if progress_callback: cancel = self.__do_progress_callback( progress_callback, retrieved_bytes, total_size, False) if cancel: Logger.warning("Download of %s aborted", uri) break if cancel: if os.path.isfile(download_path): Logger.info("Removing partial download: %s", download_path) os.remove(download_path) return "" if progress_callback: self.__do_progress_callback(progress_callback, retrieved_bytes, total_size, True) return download_path def open(self, uri, proxy=None, params=None, data=None, json=None, referer=None, additional_headers=None, no_cache=False, force_text=False): """ Open an URL Async using a thread :param str uri: The URI to download. :param str params: Data to send with the request (open(uri, params)). :param dict[str, any]|str|bytes data: Data to send with the request (open(uri, data)). :param dict[str, any] json: Json to send with the request (open(uri, params)). :param ProxyInfo proxy: The address and port (proxy.address.ext:port) of a proxy server that should be used. :param str referer: The http referer to use. :param dict|None additional_headers: The optional headers. :param bool no_cache: Should cache be disabled. :param bool force_text: In case no content type is specified, force text. :return: The data that was retrieved from the URI. :rtype: str|unicode """ r = self.__requests(uri, proxy=proxy, params=params, data=data, json=json, referer=referer, additional_headers=additional_headers, no_cache=no_cache, stream=False) if r is None: return "" content_type = r.headers.get("content-type", "") if r.encoding == 'ISO-8859-1' and "text" in content_type: # Requests defaults to ISO-8859-1 for all text content that does not specify an encoding Logger.debug( "Found 'ISO-8859-1' for 'text' content-type. Using UTF-8 instead." ) r.encoding = 'utf-8' elif r.encoding is None and force_text: Logger.debug( "Found missing encoding and 'force_text' was specified. Using UTF-8." ) r.encoding = 'utf-8' elif r.encoding is None and self.__is_text_content_type(content_type): Logger.debug( "Found missing encoding for content type '%s' is considered text. Using UTF-8 instead.", content_type) r.encoding = 'utf-8' # We might need a better mechanism here. if not r.encoding and content_type.lower() in [ "application/json", "application/javascript" ]: return r.text return r.text if r.encoding else r.content def header(self, uri, proxy=None, referer=None, additional_headers=None): """ Retrieves header information only. :param str uri: The URI to fetch the header from. :param ProxyInfo|none proxy: The address and port (proxy.address.ext:port) of a proxy server that should be used. :param str|none referer: The http referer to use. :param dict|none additional_headers: The optional headers. :return: Content-type and the URL to which a redirect could have occurred. :rtype: tuple[str,str] """ with requests.session() as s: s.cookies = self.cookieJar s.verify = not self.ignoreSslErrors proxies = self.__get_proxies(proxy, uri) headers = self.__get_headers(referer, additional_headers) Logger.info("Performing a HEAD for %s", uri) r = s.head(uri, proxies=proxies, headers=headers, allow_redirects=True, timeout=self.webTimeOut) content_type = r.headers.get("Content-Type", "") real_url = r.url self.status = UriStatus(code=r.status_code, url=uri, error=not r.ok, reason=r.reason) if self.cookieJarFile: # noinspection PyUnresolvedReferences self.cookieJar.save() if r.ok: Logger.info("%s resulted in '%s %s' (%s) for %s", r.request.method, r.status_code, r.reason, r.elapsed, r.url) return content_type, real_url else: Logger.error("%s failed with in '%s %s' (%s) for %s", r.request.method, r.status_code, r.reason, r.elapsed, r.url) return "", "" # noinspection PyUnusedLocal def __requests(self, uri, proxy, params, data, json, referer, additional_headers, no_cache, stream): with requests.session() as s: s.cookies = self.cookieJar s.verify = not self.ignoreSslErrors if self.cacheStore and not no_cache: Logger.trace("Adding the %s to the request", self.cacheStore) s.mount("https://", CacheHTTPAdapter(self.cacheStore)) s.mount("http://", CacheHTTPAdapter(self.cacheStore)) proxies = self.__get_proxies(proxy, uri) headers = self.__get_headers(referer, additional_headers) if params is not None: # Old UriHandler behaviour. Set form header to keep compatible if "content-type" not in headers: headers[ "content-type"] = "application/x-www-form-urlencoded" Logger.info("Performing a POST with '%s' for %s", headers["content-type"], uri) r = s.post(uri, data=params, proxies=proxies, headers=headers, stream=stream, timeout=self.webTimeOut) elif data is not None: # Normal Requests compatible data object Logger.info("Performing a POST with '%s' for %s", headers.get("content-type", "<No Content-Type>"), uri) r = s.post(uri, data=data, proxies=proxies, headers=headers, stream=stream, timeout=self.webTimeOut) elif json is not None: Logger.info("Performing a json POST with '%s' for %s", headers.get("content-type", "<No Content-Type>"), uri) r = s.post(uri, json=json, proxies=proxies, headers=headers, stream=stream, timeout=self.webTimeOut) else: Logger.info("Performing a GET for %s", uri) r = s.get(uri, proxies=proxies, headers=headers, stream=stream, timeout=self.webTimeOut) if r.ok: Logger.info("%s resulted in '%s %s' (%s) for %s", r.request.method, r.status_code, r.reason, r.elapsed, r.url) else: Logger.error("%s failed with '%s %s' (%s) for %s", r.request.method, r.status_code, r.reason, r.elapsed, r.url) self.status = UriStatus(code=r.status_code, url=r.url, error=not r.ok, reason=r.reason) if self.cookieJarFile: # noinspection PyUnresolvedReferences self.cookieJar.save() return r def __get_headers(self, referer, additional_headers): headers = {} if additional_headers: for k, v in additional_headers.items(): headers[k.lower()] = v if "user-agent" not in headers: headers["user-agent"] = self.userAgent if referer and "referer" not in headers: headers["referer"] = referer return headers def __get_proxies(self, proxy, url): """ :param ProxyInfo proxy: :param url: :return: :rtype: dict[str, str] """ if proxy is None: return None elif not proxy.use_proxy_for_url(url): Logger.debug("Not using proxy due to filter mismatch") elif proxy.Scheme == "http": Logger.debug("Using a http(s) %s", proxy) proxy_address = proxy.get_proxy_address() return {"http": proxy_address, "https": proxy_address} elif proxy.Scheme == "dns": Logger.debug("Using a DNS %s", proxy) return {"dns": proxy.Proxy} Logger.warning("Unsupported Proxy Scheme: %s", proxy.Scheme) return None def __do_progress_callback(self, progress_callback, retrieved_size, total_size, completed): """ Performs a callback, if the progressCallback was specified. :param progress_callback: The callback method :param retrieved_size: Number of bytes retrieved :param total_size: Total number of bytes :param completed: Are we done? @rtype : Boolean Should we cancel the download? """ if progress_callback is None: # no callback so it was not cancelled return False # calculated some stuff self.__animationIndex = (self.__animationIndex + 1) % 4 bytes_to_mb = 1048576 animation_frames = ["-", "\\", "|", "/"] animation = animation_frames[self.__animationIndex] retrievedsize_mb = 1.0 * retrieved_size / bytes_to_mb totalsize_mb = 1.0 * total_size / bytes_to_mb if total_size > 0: percentage = 100.0 * retrieved_size / total_size else: percentage = 0 status = '%s - %i%% (%.1f of %.1f MB)' % \ (animation, percentage, retrievedsize_mb, totalsize_mb) try: return progress_callback(retrieved_size, total_size, percentage, completed, status) except: Logger.error("Error in Progress Callback", exc_info=True) # cancel the download return True def __is_text_content_type(self, content_type): return content_type.lower() in [ "application/vnd.apple.mpegurl", "application/x-mpegurl" ] def __str__(self): return "UriHandler [id={0}, useCaching={1}, ignoreSslErrors={2}]"\ .format(self.id, self.cacheStore, self.ignoreSslErrors)
class Browser(object): _HEADERS = { 'User-Agent' : ['Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17', ] } _POST_HEADERS = { 'Content-Type' : ['application/x-www-form-urlencoded', ] } def __init__(self, page_archiver, cookie_file=None): self._logger = logging.getLogger(__name__) self._page_archiver = page_archiver self._logger.debug('Using page archiver: %s. Cookie file: %s', page_archiver is not None, cookie_file) if cookie_file: umask = os.umask(077) self._cj = LWPCookieJar(cookie_file) try: self._cj.load() except LoadError: self._logger.warning('Cannot load cookies from %s' % (cookie_file, )) os.umask(umask) else: self._cj = CookieJar() pool = HTTPConnectionPool(reactor, persistent=True) pool.maxPersistentPerHost = 10 self._agent = CookieAgent(ContentDecoderAgent(Agent(reactor, pool=pool), [('gzip', GzipDecoder)]), self._cj) self._lock = Lock() def save_cookies(self): try: self._cj.save() except LoadError: pass else: self._logger.debug('Cookies saved') @defer.deferredGenerator def _request(self, request_type, url, referer=None, body=None): self._logger.debug('Fetching %s', url) headers = dict(self._HEADERS) if referer: headers['Referer'] = [referer, ] body_prod = None if body: headers.update(self._POST_HEADERS) body_prod = StringProducer(body) d = defer.waitForDeferred(self._agent.request(request_type, url, Headers(headers), body_prod)) yield d response = d.getResult() receiver = MemoryReceiver() response.deliverBody(receiver) if request_type == 'POST' and (response.code >= 300 and response.code < 400): new_location = '%s/%s' % ( os.path.split(url)[0], response.headers.getRawHeaders('location')[0]) d = defer.waitForDeferred(self.get(new_location, referer)) yield d yield d.getResult() return else: d = defer.waitForDeferred(receiver.finished) yield d page = d.getResult() if self._page_archiver: reactor.callInThread(self._archive_page, page, url, body, referer) yield page def _archive_page(self, page, url, body, referer): with self._lock: self._page_archiver.archive(page, url, body, referer) def get(self, url, referer=None): return self._request('GET', url, referer) def post(self, url, data, referer=None): self._logger.debug('Posting to %s: %s', url, data) encoded_data = urlencode(data) return self._request('POST', url, referer, encoded_data)
class Bugz: """ Converts sane method calls to Bugzilla HTTP requests. @ivar base: base url of bugzilla. @ivar user: username for authenticated operations. @ivar password: password for authenticated operations @ivar cookiejar: for authenticated sessions so we only auth once. @ivar forget: forget user/password after session. @ivar authenticated: is this session authenticated already """ def __init__(self, base, user=None, password=None, forget=False, skip_auth=False, httpuser=None, httppassword=None): """ {user} and {password} will be prompted if an action needs them and they are not supplied. if {forget} is set, the login cookie will be destroyed on quit. @param base: base url of the bugzilla @type base: string @keyword user: username for authenticated actions. @type user: string @keyword password: password for authenticated actions. @type password: string @keyword forget: forget login session after termination. @type forget: bool @keyword skip_auth: do not authenticate @type skip_auth: bool """ self.base = base scheme, self.host, self.path, query, frag = urlsplit(self.base) self.authenticated = False self.forget = forget if not self.forget: try: cookie_file = os.path.join(os.environ["HOME"], COOKIE_FILE) self.cookiejar = LWPCookieJar(cookie_file) if forget: try: self.cookiejar.load() self.cookiejar.clear() self.cookiejar.save() os.chmod(self.cookiejar.filename, 0600) except IOError: pass except KeyError: self.warn("Unable to save session cookies in %s" % cookie_file) self.cookiejar = CookieJar(cookie_file) else: self.cookiejar = CookieJar() self.opener = build_opener(HTTPCookieProcessor(self.cookiejar)) self.user = user self.password = password self.httpuser = httpuser self.httppassword = httppassword self.skip_auth = skip_auth def log(self, status_msg): """Default logging handler. Expected to be overridden by the UI implementing subclass. @param status_msg: status message to print @type status_msg: string """ return def warn(self, warn_msg): """Default logging handler. Expected to be overridden by the UI implementing subclass. @param status_msg: status message to print @type status_msg: string """ return def get_input(self, prompt): """Default input handler. Expected to be override by the UI implementing subclass. @param prompt: Prompt message @type prompt: string """ return "" def auth(self): """Authenticate a session. """ # check if we need to authenticate if self.authenticated: return # try seeing if we really need to request login if not self.forget: try: self.cookiejar.load() except IOError: pass req_url = urljoin(self.base, config.urls["auth"]) req_url += "?GoAheadAndLogIn=1" req = Request(req_url, None, config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring("%s:%s" % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) re_request_login = re.compile(r"<title>.*Log in to .*</title>") if not re_request_login.search(resp.read()): self.log("Already logged in.") self.authenticated = True return # prompt for username if we were not supplied with it if not self.user: self.log("No username given.") self.user = self.get_input("Username: "******"No password given.") self.password = getpass.getpass() # perform login qparams = config.params["auth"].copy() qparams["Bugzilla_login"] = self.user qparams["Bugzilla_password"] = self.password if not self.forget: qparams["Bugzilla_remember"] = "on" req_url = urljoin(self.base, config.urls["auth"]) req = Request(req_url, urlencode(qparams), config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring("%s:%s" % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) if resp.info().has_key("Set-Cookie"): self.authenticated = True if not self.forget: self.cookiejar.save() os.chmod(self.cookiejar.filename, 0600) return True else: raise RuntimeError("Failed to login") def extractResults(self, resp): # parse the results into dicts. results = [] columns = [] rows = [] for r in csv.reader(resp): rows.append(r) for field in rows[0]: if config.choices["column_alias"].has_key(field): columns.append(config.choices["column_alias"][field]) else: self.log("Unknown field: " + field) columns.append(field) for row in rows[1:]: if "Missing Search" in row[0]: self.log("Bugzilla error (Missing search found)") return None fields = {} for i in range(min(len(row), len(columns))): fields[columns[i]] = row[i] results.append(fields) return results def search( self, query, comments=False, order="number", assigned_to=None, reporter=None, cc=None, commenter=None, whiteboard=None, keywords=None, status=[], severity=[], priority=[], product=[], component=[], ): """Search bugzilla for a bug. @param query: query string to search in title or {comments}. @type query: string @param order: what order to returns bugs in. @type order: string @keyword assigned_to: email address which the bug is assigned to. @type assigned_to: string @keyword reporter: email address matching the bug reporter. @type reporter: string @keyword cc: email that is contained in the CC list @type cc: string @keyword commenter: email of a commenter. @type commenter: string @keyword whiteboard: string to search in status whiteboard (gentoo?) @type whiteboard: string @keyword keywords: keyword to search for @type keywords: string @keyword status: bug status to match. default is ['NEW', 'ASSIGNED', 'REOPENED']. @type status: list @keyword severity: severity to match, empty means all. @type severity: list @keyword priority: priority levels to patch, empty means all. @type priority: list @keyword comments: search comments instead of just bug title. @type comments: bool @keyword product: search within products. empty means all. @type product: list @keyword component: search within components. empty means all. @type component: list @return: list of bugs, each bug represented as a dict @rtype: list of dicts """ if not self.authenticated and not self.skip_auth: self.auth() qparams = config.params["list"].copy() qparams["value0-0-0"] = query if comments: qparams["type0-0-1"] = qparams["type0-0-0"] qparams["value0-0-1"] = query qparams["order"] = config.choices["order"].get(order, "Bug Number") qparams["bug_severity"] = severity or [] qparams["priority"] = priority or [] if status is None: # NEW, ASSIGNED and REOPENED is obsolete as of bugzilla 3.x and has # been removed from bugs.gentoo.org on 2011/05/01 qparams["bug_status"] = ["NEW", "ASSIGNED", "REOPENED", "UNCONFIRMED", "CONFIRMED", "IN_PROGRESS"] elif [s.upper() for s in status] == ["ALL"]: qparams["bug_status"] = config.choices["status"] else: qparams["bug_status"] = [s.upper() for s in status] qparams["product"] = product or "" qparams["component"] = component or "" qparams["status_whiteboard"] = whiteboard or "" qparams["keywords"] = keywords or "" # hoops to jump through for emails, since there are # only two fields, we have to figure out what combinations # to use if all three are set. unique = list(set([assigned_to, cc, reporter, commenter])) unique = [u for u in unique if u] if len(unique) < 3: for i in range(len(unique)): e = unique[i] n = i + 1 qparams["email%d" % n] = e qparams["emailassigned_to%d" % n] = int(e == assigned_to) qparams["emailreporter%d" % n] = int(e == reporter) qparams["emailcc%d" % n] = int(e == cc) qparams["emaillongdesc%d" % n] = int(e == commenter) else: raise AssertionError("Cannot set assigned_to, cc, and " "reporter in the same query") req_params = urlencode(qparams, True) req_url = urljoin(self.base, config.urls["list"]) req_url += "?" + req_params req = Request(req_url, None, config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring("%s:%s" % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) return self.extractResults(resp) def namedcmd(self, cmd): """Run command stored in Bugzilla by name. @return: Result from the stored command. @rtype: list of dicts """ if not self.authenticated and not self.skip_auth: self.auth() qparams = config.params["namedcmd"].copy() # Is there a better way of getting a command with a space in its name # to be encoded as foo%20bar instead of foo+bar or foo%2520bar? qparams["namedcmd"] = quote(cmd) req_params = urlencode(qparams, True) req_params = req_params.replace("%25", "%") req_url = urljoin(self.base, config.urls["list"]) req_url += "?" + req_params req = Request(req_url, None, config.headers) if self.user and self.password: base64string = base64.encodestring("%s:%s" % (self.user, self.password))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) return self.extractResults(resp) def get(self, bugid): """Get an ElementTree representation of a bug. @param bugid: bug id @type bugid: int @rtype: ElementTree """ if not self.authenticated and not self.skip_auth: self.auth() qparams = config.params["show"].copy() qparams["id"] = bugid req_params = urlencode(qparams, True) req_url = urljoin(self.base, config.urls["show"]) req_url += "?" + req_params req = Request(req_url, None, config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring("%s:%s" % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) data = resp.read() # Get rid of control characters. data = re.sub("[\x00-\x08\x0e-\x1f\x0b\x0c]", "", data) fd = StringIO(data) # workaround for ill-defined XML templates in bugzilla 2.20.2 (major_version, minor_version) = (sys.version_info[0], sys.version_info[1]) if major_version > 2 or (major_version == 2 and minor_version >= 7): # If this is 2.7 or greater, then XMLTreeBuilder # does what we want. parser = ElementTree.XMLParser() else: # Running under Python 2.6, so we need to use our # subclass of XMLTreeBuilder instead. parser = ForcedEncodingXMLTreeBuilder(encoding="utf-8") etree = ElementTree.parse(fd, parser) bug = etree.find(".//bug") if bug is not None and bug.attrib.has_key("error"): return None else: return etree def modify( self, bugid, title=None, comment=None, url=None, status=None, resolution=None, assigned_to=None, duplicate=0, priority=None, severity=None, add_cc=[], remove_cc=[], add_dependson=[], remove_dependson=[], add_blocked=[], remove_blocked=[], whiteboard=None, keywords=None, component=None, ): """Modify an existing bug @param bugid: bug id @type bugid: int @keyword title: new title for bug @type title: string @keyword comment: comment to add @type comment: string @keyword url: new url @type url: string @keyword status: new status (note, if you are changing it to RESOLVED, you need to set {resolution} as well. @type status: string @keyword resolution: new resolution (if status=RESOLVED) @type resolution: string @keyword assigned_to: email (needs to exist in bugzilla) @type assigned_to: string @keyword duplicate: bug id to duplicate against (if resolution = DUPLICATE) @type duplicate: int @keyword priority: new priority for bug @type priority: string @keyword severity: new severity for bug @type severity: string @keyword add_cc: list of emails to add to the cc list @type add_cc: list of strings @keyword remove_cc: list of emails to remove from cc list @type remove_cc: list of string. @keyword add_dependson: list of bug ids to add to the depend list @type add_dependson: list of strings @keyword remove_dependson: list of bug ids to remove from depend list @type remove_dependson: list of strings @keyword add_blocked: list of bug ids to add to the blocked list @type add_blocked: list of strings @keyword remove_blocked: list of bug ids to remove from blocked list @type remove_blocked: list of strings @keyword whiteboard: set status whiteboard @type whiteboard: string @keyword keywords: set keywords @type keywords: string @keyword component: set component @type component: string @return: list of fields modified. @rtype: list of strings """ if not self.authenticated and not self.skip_auth: self.auth() buginfo = Bugz.get(self, bugid) if not buginfo: return False modified = [] qparams = config.params["modify"].copy() qparams["id"] = bugid # NOTE: knob has been removed in bugzilla 4 and 3? qparams["knob"] = "none" # copy existing fields FIELDS = ( "bug_file_loc", "bug_severity", "short_desc", "bug_status", "status_whiteboard", "keywords", "resolution", "op_sys", "priority", "version", "target_milestone", "assigned_to", "rep_platform", "product", "component", "token", ) FIELDS_MULTI = ("blocked", "dependson") for field in FIELDS: try: qparams[field] = buginfo.find(".//%s" % field).text if qparams[field] is None: del qparams[field] except: pass for field in FIELDS_MULTI: qparams[field] = [d.text for d in buginfo.findall(".//%s" % field) if d is not None and d.text is not None] # set 'knob' if we are change the status/resolution # or trying to reassign bug. if status: status = status.upper() if resolution: resolution = resolution.upper() if status and status != qparams["bug_status"]: # Bugzilla >= 3.x qparams["bug_status"] = status if status == "RESOLVED": qparams["knob"] = "resolve" if resolution: qparams["resolution"] = resolution else: qparams["resolution"] = "FIXED" modified.append(("status", status)) modified.append(("resolution", qparams["resolution"])) elif status == "ASSIGNED" or status == "IN_PROGRESS": qparams["knob"] = "accept" modified.append(("status", status)) elif status == "REOPENED": qparams["knob"] = "reopen" modified.append(("status", status)) elif status == "VERIFIED": qparams["knob"] = "verified" modified.append(("status", status)) elif status == "CLOSED": qparams["knob"] = "closed" modified.append(("status", status)) elif duplicate: # Bugzilla >= 3.x qparams["bug_status"] = "RESOLVED" qparams["resolution"] = "DUPLICATE" qparams["knob"] = "duplicate" qparams["dup_id"] = duplicate modified.append(("status", "RESOLVED")) modified.append(("resolution", "DUPLICATE")) elif assigned_to: qparams["knob"] = "reassign" qparams["assigned_to"] = assigned_to modified.append(("assigned_to", assigned_to)) # setup modification of other bits if comment: qparams["comment"] = comment modified.append(("comment", ellipsis(comment, 60))) if title: qparams["short_desc"] = title or "" modified.append(("title", title)) if url is not None: qparams["bug_file_loc"] = url modified.append(("url", url)) if severity is not None: qparams["bug_severity"] = severity modified.append(("severity", severity)) if priority is not None: qparams["priority"] = priority modified.append(("priority", priority)) # cc manipulation if add_cc is not None: qparams["newcc"] = ", ".join(add_cc) modified.append(("newcc", qparams["newcc"])) if remove_cc is not None: qparams["cc"] = remove_cc qparams["removecc"] = "on" modified.append(("cc", remove_cc)) # bug depend/blocked manipulation changed_dependson = False changed_blocked = False if remove_dependson: for bug_id in remove_dependson: qparams["dependson"].remove(str(bug_id)) changed_dependson = True if remove_blocked: for bug_id in remove_blocked: qparams["blocked"].remove(str(bug_id)) changed_blocked = True if add_dependson: for bug_id in add_dependson: qparams["dependson"].append(str(bug_id)) changed_dependson = True if add_blocked: for bug_id in add_blocked: qparams["blocked"].append(str(bug_id)) changed_blocked = True qparams["dependson"] = ",".join(qparams["dependson"]) qparams["blocked"] = ",".join(qparams["blocked"]) if changed_dependson: modified.append(("dependson", qparams["dependson"])) if changed_blocked: modified.append(("blocked", qparams["blocked"])) if whiteboard is not None: qparams["status_whiteboard"] = whiteboard modified.append(("status_whiteboard", whiteboard)) if keywords is not None: qparams["keywords"] = keywords modified.append(("keywords", keywords)) if component is not None: qparams["component"] = component modified.append(("component", component)) req_params = urlencode(qparams, True) req_url = urljoin(self.base, config.urls["modify"]) req = Request(req_url, req_params, config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring("%s:%s" % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) try: resp = self.opener.open(req) re_error = re.compile(r'id="error_msg".*>([^<]+)<') error = re_error.search(resp.read()) if error: print error.group(1) return [] return modified except: return [] def attachment(self, attachid): """Get an attachment by attachment_id @param attachid: attachment id @type attachid: int @return: dict with three keys, 'filename', 'size', 'fd' @rtype: dict """ if not self.authenticated and not self.skip_auth: self.auth() qparams = config.params["attach"].copy() qparams["id"] = attachid req_params = urlencode(qparams, True) req_url = urljoin(self.base, config.urls["attach"]) req_url += "?" + req_params req = Request(req_url, None, config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring("%s:%s" % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) try: content_type = resp.info()["Content-type"] namefield = content_type.split(";")[1] filename = re.search(r"name=\"(.*)\"", namefield).group(1) content_length = int(resp.info()["Content-length"], 0) return {"filename": filename, "size": content_length, "fd": resp} except: return {} def post( self, product, component, title, description, url="", assigned_to="", cc="", keywords="", version="", dependson="", blocked="", priority="", severity="", ): """Post a bug @param product: product where the bug should be placed @type product: string @param component: component where the bug should be placed @type component: string @param title: title of the bug. @type title: string @param description: description of the bug @type description: string @keyword url: optional url to submit with bug @type url: string @keyword assigned_to: optional email to assign bug to @type assigned_to: string. @keyword cc: option list of CC'd emails @type: string @keyword keywords: option list of bugzilla keywords @type: string @keyword version: version of the component @type: string @keyword dependson: bugs this one depends on @type: string @keyword blocked: bugs this one blocks @type: string @keyword priority: priority of this bug @type: string @keyword severity: severity of this bug @type: string @rtype: int @return: the bug number, or 0 if submission failed. """ if not self.authenticated and not self.skip_auth: self.auth() qparams = config.params["post"].copy() qparams["product"] = product qparams["component"] = component qparams["short_desc"] = title qparams["comment"] = description qparams["assigned_to"] = assigned_to qparams["cc"] = cc qparams["bug_file_loc"] = url qparams["dependson"] = dependson qparams["blocked"] = blocked qparams["keywords"] = keywords # XXX: default version is 'unspecified' if version != "": qparams["version"] = version # XXX: default priority is 'Normal' if priority != "": qparams["priority"] = priority # XXX: default severity is 'normal' if severity != "": qparams["bug_severity"] = severity req_params = urlencode(qparams, True) req_url = urljoin(self.base, config.urls["post"]) req = Request(req_url, req_params, config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring("%s:%s" % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) try: re_bug = re.compile(r"(?:\s+)?<title>.*Bug ([0-9]+) Submitted.*</title>") bug_match = re_bug.search(resp.read()) if bug_match: return int(bug_match.group(1)) except: pass return 0 def attach(self, bugid, title, description, filename, content_type="text/plain", ispatch=False): """Attach a file to a bug. @param bugid: bug id @type bugid: int @param title: short description of attachment @type title: string @param description: long description of the attachment @type description: string @param filename: filename of the attachment @type filename: string @keywords content_type: mime-type of the attachment @type content_type: string @rtype: bool @return: True if successful, False if not successful. """ if not self.authenticated and not self.skip_auth: self.auth() qparams = config.params["attach_post"].copy() qparams["bugid"] = bugid qparams["description"] = title qparams["comment"] = description if ispatch: qparams["ispatch"] = "1" qparams["contenttypeentry"] = "text/plain" else: qparams["contenttypeentry"] = content_type filedata = [("data", filename, open(filename).read())] content_type, body = encode_multipart_formdata(qparams.items(), filedata) req_headers = config.headers.copy() req_headers["Content-type"] = content_type req_headers["Content-length"] = len(body) req_url = urljoin(self.base, config.urls["attach_post"]) req = Request(req_url, body, req_headers) if self.httpuser and self.httppassword: base64string = base64.encodestring("%s:%s" % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) # TODO: return attachment id and success? try: re_attach = re.compile(r"<title>(.+)</title>") # Bugzilla 3/4 re_attach34 = re.compile(r"Attachment \d+ added to Bug \d+") response = resp.read() attach_match = re_attach.search(response) if attach_match: if attach_match.group(1) == "Changes Submitted" or re_attach34.match(attach_match.group(1)): return True else: return attach_match.group(1) else: return False except: pass return False
class Bugz: """ Converts sane method calls to Bugzilla HTTP requests. @ivar base: base url of bugzilla. @ivar user: username for authenticated operations. @ivar password: password for authenticated operations @ivar cookiejar: for authenticated sessions so we only auth once. @ivar forget: forget user/password after session. @ivar authenticated: is this session authenticated already """ def __init__(self, base, user=None, password=None, forget=False, skip_auth=False, httpuser=None, httppassword=None): """ {user} and {password} will be prompted if an action needs them and they are not supplied. if {forget} is set, the login cookie will be destroyed on quit. @param base: base url of the bugzilla @type base: string @keyword user: username for authenticated actions. @type user: string @keyword password: password for authenticated actions. @type password: string @keyword forget: forget login session after termination. @type forget: bool @keyword skip_auth: do not authenticate @type skip_auth: bool """ self.base = base scheme, self.host, self.path, query, frag = urlsplit(self.base) self.authenticated = False self.forget = forget if not self.forget: try: cookie_file = os.path.join(os.environ['HOME'], COOKIE_FILE) self.cookiejar = LWPCookieJar(cookie_file) if forget: try: self.cookiejar.load() self.cookiejar.clear() self.cookiejar.save() os.chmod(self.cookiejar.filename, 0600) except IOError: pass except KeyError: self.warn('Unable to save session cookies in %s' % cookie_file) self.cookiejar = CookieJar(cookie_file) else: self.cookiejar = CookieJar() self.opener = build_opener(HTTPCookieProcessor(self.cookiejar)) self.user = user self.password = password self.httpuser = httpuser self.httppassword = httppassword self.skip_auth = skip_auth def log(self, status_msg): """Default logging handler. Expected to be overridden by the UI implementing subclass. @param status_msg: status message to print @type status_msg: string """ return def warn(self, warn_msg): """Default logging handler. Expected to be overridden by the UI implementing subclass. @param status_msg: status message to print @type status_msg: string """ return def get_input(self, prompt): """Default input handler. Expected to be override by the UI implementing subclass. @param prompt: Prompt message @type prompt: string """ return '' def auth(self): """Authenticate a session. """ # check if we need to authenticate if self.authenticated: return # try seeing if we really need to request login if not self.forget: try: self.cookiejar.load() except IOError: pass req_url = urljoin(self.base, config.urls['auth']) req_url += '?GoAheadAndLogIn=1' req = Request(req_url, None, config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring( '%s:%s' % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) re_request_login = re.compile(r'<title>.*Log in to .*</title>') if not re_request_login.search(resp.read()): self.log('Already logged in.') self.authenticated = True return # prompt for username if we were not supplied with it if not self.user: self.log('No username given.') self.user = self.get_input('Username: '******'No password given.') self.password = getpass.getpass() # perform login qparams = config.params['auth'].copy() qparams['Bugzilla_login'] = self.user qparams['Bugzilla_password'] = self.password if not self.forget: qparams['Bugzilla_remember'] = 'on' req_url = urljoin(self.base, config.urls['auth']) req = Request(req_url, urlencode(qparams), config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring( '%s:%s' % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) if resp.info().has_key('Set-Cookie'): self.authenticated = True if not self.forget: self.cookiejar.save() os.chmod(self.cookiejar.filename, 0600) return True else: raise RuntimeError("Failed to login") def extractResults(self, resp): # parse the results into dicts. results = [] columns = [] rows = [] for r in csv.reader(resp): rows.append(r) for field in rows[0]: if config.choices['column_alias'].has_key(field): columns.append(config.choices['column_alias'][field]) else: self.log('Unknown field: ' + field) columns.append(field) for row in rows[1:]: if "Missing Search" in row[0]: self.log('Bugzilla error (Missing search found)') return None fields = {} for i in range(min(len(row), len(columns))): fields[columns[i]] = row[i] results.append(fields) return results def search(self, query, comments=False, order='number', assigned_to=None, reporter=None, cc=None, commenter=None, whiteboard=None, keywords=None, status=[], severity=[], priority=[], product=[], component=[]): """Search bugzilla for a bug. @param query: query string to search in title or {comments}. @type query: string @param order: what order to returns bugs in. @type order: string @keyword assigned_to: email address which the bug is assigned to. @type assigned_to: string @keyword reporter: email address matching the bug reporter. @type reporter: string @keyword cc: email that is contained in the CC list @type cc: string @keyword commenter: email of a commenter. @type commenter: string @keyword whiteboard: string to search in status whiteboard (gentoo?) @type whiteboard: string @keyword keywords: keyword to search for @type keywords: string @keyword status: bug status to match. default is ['NEW', 'ASSIGNED', 'REOPENED']. @type status: list @keyword severity: severity to match, empty means all. @type severity: list @keyword priority: priority levels to patch, empty means all. @type priority: list @keyword comments: search comments instead of just bug title. @type comments: bool @keyword product: search within products. empty means all. @type product: list @keyword component: search within components. empty means all. @type component: list @return: list of bugs, each bug represented as a dict @rtype: list of dicts """ if not self.authenticated and not self.skip_auth: self.auth() qparams = config.params['list'].copy() qparams['value0-0-0'] = query if comments: qparams['type0-0-1'] = qparams['type0-0-0'] qparams['value0-0-1'] = query qparams['order'] = config.choices['order'].get(order, 'Bug Number') qparams['bug_severity'] = severity or [] qparams['priority'] = priority or [] if status is None: # NEW, ASSIGNED and REOPENED is obsolete as of bugzilla 3.x and has # been removed from bugs.gentoo.org on 2011/05/01 qparams['bug_status'] = [ 'NEW', 'ASSIGNED', 'REOPENED', 'UNCONFIRMED', 'CONFIRMED', 'IN_PROGRESS' ] elif [s.upper() for s in status] == ['ALL']: qparams['bug_status'] = config.choices['status'] else: qparams['bug_status'] = [s.upper() for s in status] qparams['product'] = product or '' qparams['component'] = component or '' qparams['status_whiteboard'] = whiteboard or '' qparams['keywords'] = keywords or '' # hoops to jump through for emails, since there are # only two fields, we have to figure out what combinations # to use if all three are set. unique = list(set([assigned_to, cc, reporter, commenter])) unique = [u for u in unique if u] if len(unique) < 3: for i in range(len(unique)): e = unique[i] n = i + 1 qparams['email%d' % n] = e qparams['emailassigned_to%d' % n] = int(e == assigned_to) qparams['emailreporter%d' % n] = int(e == reporter) qparams['emailcc%d' % n] = int(e == cc) qparams['emaillongdesc%d' % n] = int(e == commenter) else: raise AssertionError('Cannot set assigned_to, cc, and ' 'reporter in the same query') req_params = urlencode(qparams, True) req_url = urljoin(self.base, config.urls['list']) req_url += '?' + req_params req = Request(req_url, None, config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring( '%s:%s' % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) return self.extractResults(resp) def namedcmd(self, cmd): """Run command stored in Bugzilla by name. @return: Result from the stored command. @rtype: list of dicts """ if not self.authenticated and not self.skip_auth: self.auth() qparams = config.params['namedcmd'].copy() # Is there a better way of getting a command with a space in its name # to be encoded as foo%20bar instead of foo+bar or foo%2520bar? qparams['namedcmd'] = quote(cmd) req_params = urlencode(qparams, True) req_params = req_params.replace('%25', '%') req_url = urljoin(self.base, config.urls['list']) req_url += '?' + req_params req = Request(req_url, None, config.headers) if self.user and self.password: base64string = base64.encodestring('%s:%s' % (self.user, self.password))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) return self.extractResults(resp) def get(self, bugid): """Get an ElementTree representation of a bug. @param bugid: bug id @type bugid: int @rtype: ElementTree """ if not self.authenticated and not self.skip_auth: self.auth() qparams = config.params['show'].copy() qparams['id'] = bugid req_params = urlencode(qparams, True) req_url = urljoin(self.base, config.urls['show']) req_url += '?' + req_params req = Request(req_url, None, config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring( '%s:%s' % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) data = resp.read() # Get rid of control characters. data = re.sub('[\x00-\x08\x0e-\x1f\x0b\x0c]', '', data) fd = StringIO(data) # workaround for ill-defined XML templates in bugzilla 2.20.2 (major_version, minor_version) = \ (sys.version_info[0], sys.version_info[1]) if major_version > 2 or \ (major_version == 2 and minor_version >= 7): # If this is 2.7 or greater, then XMLTreeBuilder # does what we want. parser = ElementTree.XMLParser() else: # Running under Python 2.6, so we need to use our # subclass of XMLTreeBuilder instead. parser = ForcedEncodingXMLTreeBuilder(encoding='utf-8') etree = ElementTree.parse(fd, parser) bug = etree.find('.//bug') if bug is not None and bug.attrib.has_key('error'): return None else: return etree def modify(self, bugid, title=None, comment=None, url=None, status=None, resolution=None, assigned_to=None, duplicate=0, priority=None, severity=None, add_cc=[], remove_cc=[], add_dependson=[], remove_dependson=[], add_blocked=[], remove_blocked=[], whiteboard=None, keywords=None, component=None): """Modify an existing bug @param bugid: bug id @type bugid: int @keyword title: new title for bug @type title: string @keyword comment: comment to add @type comment: string @keyword url: new url @type url: string @keyword status: new status (note, if you are changing it to RESOLVED, you need to set {resolution} as well. @type status: string @keyword resolution: new resolution (if status=RESOLVED) @type resolution: string @keyword assigned_to: email (needs to exist in bugzilla) @type assigned_to: string @keyword duplicate: bug id to duplicate against (if resolution = DUPLICATE) @type duplicate: int @keyword priority: new priority for bug @type priority: string @keyword severity: new severity for bug @type severity: string @keyword add_cc: list of emails to add to the cc list @type add_cc: list of strings @keyword remove_cc: list of emails to remove from cc list @type remove_cc: list of string. @keyword add_dependson: list of bug ids to add to the depend list @type add_dependson: list of strings @keyword remove_dependson: list of bug ids to remove from depend list @type remove_dependson: list of strings @keyword add_blocked: list of bug ids to add to the blocked list @type add_blocked: list of strings @keyword remove_blocked: list of bug ids to remove from blocked list @type remove_blocked: list of strings @keyword whiteboard: set status whiteboard @type whiteboard: string @keyword keywords: set keywords @type keywords: string @keyword component: set component @type component: string @return: list of fields modified. @rtype: list of strings """ if not self.authenticated and not self.skip_auth: self.auth() buginfo = Bugz.get(self, bugid) if not buginfo: return False modified = [] qparams = config.params['modify'].copy() qparams['id'] = bugid # NOTE: knob has been removed in bugzilla 4 and 3? qparams['knob'] = 'none' # copy existing fields FIELDS = ('bug_file_loc', 'bug_severity', 'short_desc', 'bug_status', 'status_whiteboard', 'keywords', 'resolution', 'op_sys', 'priority', 'version', 'target_milestone', 'assigned_to', 'rep_platform', 'product', 'component', 'token') FIELDS_MULTI = ('blocked', 'dependson') for field in FIELDS: try: qparams[field] = buginfo.find('.//%s' % field).text if qparams[field] is None: del qparams[field] except: pass for field in FIELDS_MULTI: qparams[field] = [ d.text for d in buginfo.findall('.//%s' % field) if d is not None and d.text is not None ] # set 'knob' if we are change the status/resolution # or trying to reassign bug. if status: status = status.upper() if resolution: resolution = resolution.upper() if status and status != qparams['bug_status']: # Bugzilla >= 3.x qparams['bug_status'] = status if status == 'RESOLVED': qparams['knob'] = 'resolve' if resolution: qparams['resolution'] = resolution else: qparams['resolution'] = 'FIXED' modified.append(('status', status)) modified.append(('resolution', qparams['resolution'])) elif status == 'ASSIGNED' or status == 'IN_PROGRESS': qparams['knob'] = 'accept' modified.append(('status', status)) elif status == 'REOPENED': qparams['knob'] = 'reopen' modified.append(('status', status)) elif status == 'VERIFIED': qparams['knob'] = 'verified' modified.append(('status', status)) elif status == 'CLOSED': qparams['knob'] = 'closed' modified.append(('status', status)) elif duplicate: # Bugzilla >= 3.x qparams['bug_status'] = "RESOLVED" qparams['resolution'] = "DUPLICATE" qparams['knob'] = 'duplicate' qparams['dup_id'] = duplicate modified.append(('status', 'RESOLVED')) modified.append(('resolution', 'DUPLICATE')) elif assigned_to: qparams['knob'] = 'reassign' qparams['assigned_to'] = assigned_to modified.append(('assigned_to', assigned_to)) # setup modification of other bits if comment: qparams['comment'] = comment modified.append(('comment', ellipsis(comment, 60))) if title: qparams['short_desc'] = title or '' modified.append(('title', title)) if url is not None: qparams['bug_file_loc'] = url modified.append(('url', url)) if severity is not None: qparams['bug_severity'] = severity modified.append(('severity', severity)) if priority is not None: qparams['priority'] = priority modified.append(('priority', priority)) # cc manipulation if add_cc is not None: qparams['newcc'] = ', '.join(add_cc) modified.append(('newcc', qparams['newcc'])) if remove_cc is not None: qparams['cc'] = remove_cc qparams['removecc'] = 'on' modified.append(('cc', remove_cc)) # bug depend/blocked manipulation changed_dependson = False changed_blocked = False if remove_dependson: for bug_id in remove_dependson: qparams['dependson'].remove(str(bug_id)) changed_dependson = True if remove_blocked: for bug_id in remove_blocked: qparams['blocked'].remove(str(bug_id)) changed_blocked = True if add_dependson: for bug_id in add_dependson: qparams['dependson'].append(str(bug_id)) changed_dependson = True if add_blocked: for bug_id in add_blocked: qparams['blocked'].append(str(bug_id)) changed_blocked = True qparams['dependson'] = ','.join(qparams['dependson']) qparams['blocked'] = ','.join(qparams['blocked']) if changed_dependson: modified.append(('dependson', qparams['dependson'])) if changed_blocked: modified.append(('blocked', qparams['blocked'])) if whiteboard is not None: qparams['status_whiteboard'] = whiteboard modified.append(('status_whiteboard', whiteboard)) if keywords is not None: qparams['keywords'] = keywords modified.append(('keywords', keywords)) if component is not None: qparams['component'] = component modified.append(('component', component)) req_params = urlencode(qparams, True) req_url = urljoin(self.base, config.urls['modify']) req = Request(req_url, req_params, config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring( '%s:%s' % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) try: resp = self.opener.open(req) re_error = re.compile(r'id="error_msg".*>([^<]+)<') error = re_error.search(resp.read()) if error: print error.group(1) return [] return modified except: return [] def attachment(self, attachid): """Get an attachment by attachment_id @param attachid: attachment id @type attachid: int @return: dict with three keys, 'filename', 'size', 'fd' @rtype: dict """ if not self.authenticated and not self.skip_auth: self.auth() qparams = config.params['attach'].copy() qparams['id'] = attachid req_params = urlencode(qparams, True) req_url = urljoin(self.base, config.urls['attach']) req_url += '?' + req_params req = Request(req_url, None, config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring( '%s:%s' % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) try: content_type = resp.info()['Content-type'] namefield = content_type.split(';')[1] filename = re.search(r'name=\"(.*)\"', namefield).group(1) content_length = int(resp.info()['Content-length'], 0) return {'filename': filename, 'size': content_length, 'fd': resp} except: return {} def post(self, product, component, title, description, url='', assigned_to='', cc='', keywords='', version='', dependson='', blocked='', priority='', severity=''): """Post a bug @param product: product where the bug should be placed @type product: string @param component: component where the bug should be placed @type component: string @param title: title of the bug. @type title: string @param description: description of the bug @type description: string @keyword url: optional url to submit with bug @type url: string @keyword assigned_to: optional email to assign bug to @type assigned_to: string. @keyword cc: option list of CC'd emails @type: string @keyword keywords: option list of bugzilla keywords @type: string @keyword version: version of the component @type: string @keyword dependson: bugs this one depends on @type: string @keyword blocked: bugs this one blocks @type: string @keyword priority: priority of this bug @type: string @keyword severity: severity of this bug @type: string @rtype: int @return: the bug number, or 0 if submission failed. """ if not self.authenticated and not self.skip_auth: self.auth() qparams = config.params['post'].copy() qparams['product'] = product qparams['component'] = component qparams['short_desc'] = title qparams['comment'] = description qparams['assigned_to'] = assigned_to qparams['cc'] = cc qparams['bug_file_loc'] = url qparams['dependson'] = dependson qparams['blocked'] = blocked qparams['keywords'] = keywords #XXX: default version is 'unspecified' if version != '': qparams['version'] = version #XXX: default priority is 'Normal' if priority != '': qparams['priority'] = priority #XXX: default severity is 'normal' if severity != '': qparams['bug_severity'] = severity req_params = urlencode(qparams, True) req_url = urljoin(self.base, config.urls['post']) req = Request(req_url, req_params, config.headers) if self.httpuser and self.httppassword: base64string = base64.encodestring( '%s:%s' % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) try: re_bug = re.compile( r'(?:\s+)?<title>.*Bug ([0-9]+) Submitted.*</title>') bug_match = re_bug.search(resp.read()) if bug_match: return int(bug_match.group(1)) except: pass return 0 def attach(self, bugid, title, description, filename, content_type='text/plain', ispatch=False): """Attach a file to a bug. @param bugid: bug id @type bugid: int @param title: short description of attachment @type title: string @param description: long description of the attachment @type description: string @param filename: filename of the attachment @type filename: string @keywords content_type: mime-type of the attachment @type content_type: string @rtype: bool @return: True if successful, False if not successful. """ if not self.authenticated and not self.skip_auth: self.auth() qparams = config.params['attach_post'].copy() qparams['bugid'] = bugid qparams['description'] = title qparams['comment'] = description if ispatch: qparams['ispatch'] = '1' qparams['contenttypeentry'] = 'text/plain' else: qparams['contenttypeentry'] = content_type filedata = [('data', filename, open(filename).read())] content_type, body = encode_multipart_formdata(qparams.items(), filedata) req_headers = config.headers.copy() req_headers['Content-type'] = content_type req_headers['Content-length'] = len(body) req_url = urljoin(self.base, config.urls['attach_post']) req = Request(req_url, body, req_headers) if self.httpuser and self.httppassword: base64string = base64.encodestring( '%s:%s' % (self.httpuser, self.httppassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) resp = self.opener.open(req) # TODO: return attachment id and success? try: re_attach = re.compile(r'<title>(.+)</title>') # Bugzilla 3/4 re_attach34 = re.compile(r'Attachment \d+ added to Bug \d+') response = resp.read() attach_match = re_attach.search(response) if attach_match: if attach_match.group( 1) == "Changes Submitted" or re_attach34.match( attach_match.group(1)): return True else: return attach_match.group(1) else: return False except: pass return False