コード例 #1
0
ファイル: bg.py プロジェクト: libresoft/Bicho
    def analyze_bug(self, bug_id, url):
        #Retrieving main bug information
        bug_url = url + "show_bug.cgi?id=" + bug_id + "&ctype=xml"
        printdbg(bug_url)

        handler = BugsHandler()
        parser = xml.sax.make_parser()
        parser.setContentHandler(handler)
        f = urllib.urlopen(bug_url)

        try:
            parser.feed(f.read())
        except Exception:
            printerr("Error parsing URL: %s" % (bug_url))
            raise

        f.close()
        parser.close()
        #handler.print_debug_data()
        issue = handler.get_issue()

        #Retrieving changes
        bug_activity_url = url + "show_activity.cgi?id=" + bug_id
        printdbg( bug_activity_url )
        data_activity = urllib.urlopen(bug_activity_url).read()
        parser = SoupHtmlParser(data_activity, bug_id)
        changes = parser.parse_changes()
        for c in changes:
            issue.add_change(c)
        return issue
コード例 #2
0
    def getReviews (self, limit, start):
                
        args_gerrit ="gerrit query "
        args_gerrit += "project:" + Config.gerrit_project
        args_gerrit += " limit:" + str(limit) 
        if (start != ""): 
            args_gerrit += " resume_sortkey:"+start
        args_gerrit += " --all-approvals --format=JSON"
        
        if vars(Config).has_key('backend_user'):
            cmd = ["ssh", "-p 29418", Config.backend_user+"@"+Config.url, args_gerrit]
            printdbg("Gerrit cmd: " + "ssh -p 29418 "+ Config.backend_user+"@"+Config.url+" "+ args_gerrit)
        else:
            cmd = ["ssh", "-p 29418", Config.url, args_gerrit]
            printdbg("Gerrit cmd: " + "ssh "+ "-p 29418 "+ Config.url+" "+ args_gerrit)
        tickets_raw = subprocess.check_output(cmd)
        tickets_raw = "["+tickets_raw.replace("\n",",")+"]"
        tickets_raw = tickets_raw.replace(",]","]")
        tickets = json.loads(tickets_raw)
        
#        tickets_test = '[{"project":"openstack/nova","branch":"master","topic":"bug/857209","id":"I660532ee5758c7595138d4dcf5a2825ddf898c65","number":"637","subject":"contrib/nova.sh: Updated to latest \\u0027upstream\\u0027 commit:6a8433a resolves bug 857209","owner":{"name":"Dave Walker","email":"*****@*****.**","username":"******"},"url":"https://review.openstack.org/637","createdOn":1316815511,"lastUpdated":1316815646,"sortKey":"0017e78f0000027d","open":false,"status":"ABANDONED","patchSets":[{"number":"1","revision":"95d8d0f75c188f7eabf00ecf6bd5b397852e67b9","ref":"refs/changes/37/637/1","uploader":{"name":"Dave Walker","email":"*****@*****.**","username":"******"},"createdOn":1316815511}]},'
#        tickets_test += '{"project":"openstack/nova","branch":"master","id":"I812e95fb0744ad84abd7ea2ad7d11123667abbc8","number":"635","subject":"Made jenkins email pruning more resilient.","owner":{"name":"Monty Taylor","email":"*****@*****.**","username":"******"},"url":"https://review.openstack.org/635","createdOn":1316813897,"lastUpdated":1316814951,"sortKey":"0017e7830000027b","open":false,"status":"MERGED","patchSets":[{"number":"1","revision":"c586e4ed23846420177802c164f594e021cceea8","ref":"refs/changes/35/635/1","uploader":{"name":"Monty Taylor","email":"*****@*****.**","username":"******"},"createdOn":1316813897,"approvals":[{"type":"SUBM","value":"1","grantedOn":1316814951,"by":{"name":"Jenkins","username":"******"}},{"type":"VRIF","description":"Verified","value":"1","grantedOn":1316814948,"by":{"name":"Jenkins","username":"******"}},{"type":"CRVW","description":"Code Review","value":"2","grantedOn":1316814192,"by":{"name":"Brian Waldon","email":"*****@*****.**","username":"******"}}]}]},'
#        tickets_test += '{"project":"openstack/nova","branch":"master","id":"I495363b44d9da96d66f85c2a621393329830aeb3","number":"630","subject":"Fixing bug 857712","owner":{"name":"Brian Waldon","email":"*****@*****.**","username":"******"},"url":"https://review.openstack.org/630","createdOn":1316810421,"lastUpdated":1316813692,"sortKey":"0017e76e00000276","open":false,"status":"MERGED","patchSets":[{"number":"1","revision":"ddb6945e8fbb8a00d5b67a6a6b8a069b7642022d","ref":"refs/changes/30/630/1","uploader":{"name":"Brian Waldon","email":"*****@*****.**","username":"******"},"createdOn":1316810421,"approvals":[{"type":"SUBM","value":"1","grantedOn":1316813692,"by":{"name":"Jenkins","username":"******"}},{"type":"VRIF","description":"Verified","value":"1","grantedOn":1316813689,"by":{"name":"Jenkins","username":"******"}},{"type":"CRVW","description":"Code Review","value":"1","grantedOn":1316811221,"by":{"name":"Josh Kearney","email":"*****@*****.**","username":"******"}},{"type":"CRVW","description":"Code Review","value":"2","grantedOn":1316812789,"by":{"name":"Brian Lamar","email":"*****@*****.**","username":"******"}},{"type":"CRVW","description":"Code Review","value":"1","grantedOn":1316810744,"by":{"name":"Mark McLoughlin","email":"*****@*****.**","username":"******"}}]}]},'
#        tickets_test += '{"type":"stats","rowCount":67,"runTimeMilliseconds":365}]'        
#        tickets = json.loads(tickets_test)

        
        return tickets
コード例 #3
0
ファイル: sf.py プロジェクト: iKuba/Bicho
    def __parse_issue_attachments(self, soup):
        """
        """
        attachments = []
        try:
            files = soup.find('h4', {
                'id': 'filebar'
            }).findNext('tbody').findAll('tr')
            for f in files:
                attch = {}
                # Each entry contains three fields (td tags) that
                # follow the next order: filename, description and URL.
                aux = f.findAll('td')
                attch['filename'] = self.__clean_str(u''.join(aux[0].contents))
                attch['desc'] = self.__clean_str(u''.join(aux[1].contents))
                attch['url'] = SOURCEFORGE_DOMAIN + aux[2].a.get('href')
                attachments.append(attch)
        except AttributeError:
            # there is no attachment
            pass
        except:
            raise SourceForgeParserError("Error parsing issue's attachments")

        printdbg("%s attachments" % (str(len(attachments))))
        return attachments
コード例 #4
0
ファイル: redmine.py プロジェクト: AlertProject/Bicho
 def parse_changes (self, activity):
     changesList = []
     for entry in activity['entries']:
         # print "changed_by:" + entry['author']
         by = People(entry['author'])
         # print "changed_on:" + entry['updated']
         description = entry['description'].split('updated:')
         changes = description.pop(0)
         field = changes.rpartition('\n')[2].strip()
         while description:                
             changes = description.pop(0).split('\n')
             values = changes[0].split('=>')                
             if (len(values) != 2):
                 printdbg(field + " not supported in changes analysis")
                 old_value = new_value = ""                    
             else:
                 # u'in-progress' => u'closed'
                 values = changes[0].split('=>')
                 old_value = self.remove_unicode(values[0].strip())
                 if old_value == "''": old_value =""
                 new_value = self.remove_unicode(values[1].strip())
                 if new_value == "''": new_value =""
             update = parse(entry['updated'])
             change = Change(unicode(field), unicode(old_value), unicode(new_value), by, update)
             changesList.append(change)
             if (len(changes)>1):
                 field = changes[1].strip()
     return changesList
コード例 #5
0
ファイル: sf.py プロジェクト: iKuba/Bicho
    def __parse_issue_comments(self, soup):
        """
        """
        comments = []
        try:
            artifacts = soup.findAll('tr',
                                     {'class': ISSUE_COMMENT_CLASS_PATTERN})
            for art in artifacts:
                comment = {}

                rawsub, rawdesc = art.findAll('p')
                # Date and sender are content on the first 'p'
                a = rawsub.find('a')
                if a:
                    comment['by'] = {'name': a.get('title'), 'id': a.string}
                else:
                    comment['by'] = {'name': 'nobody', 'id': 'nobody'}

                # Time stamp is the first value of the 'p' contents
                d = self.__clean_str(rawsub.contents[0])
                comment['date'] = self.__str_to_date(
                    ISSUE_COMMENT_DATE_PATTERN.match(d).group(1))

                # Description is content on the second 'p'.
                comment['desc'] = self.__clean_str(u''.join(rawdesc.contents))

                comments.append(comment)
        except:
            SourceForgeParserError('Errror parsing issue comments')
        printdbg("%s comments" % (str(len(comments))))
        return comments
コード例 #6
0
ファイル: sf.py プロジェクト: libresoft/Bicho
    def __parse_issue_comments(self, soup):
        """
        """
        comments = []
        try:
            artifacts = soup.findAll("tr", {"class": ISSUE_COMMENT_CLASS_PATTERN})
            for art in artifacts:
                comment = {}

                rawsub, rawdesc = art.findAll("p")
                # Date and sender are content on the first 'p'
                a = rawsub.find("a")
                if a:
                    comment["by"] = {"name": a.get("title"), "id": a.string}
                else:
                    comment["by"] = {"name": "nobody", "id": "nobody"}

                # Time stamp is the first value of the 'p' contents
                d = self.__clean_str(rawsub.contents[0])
                comment["date"] = self.__str_to_date(ISSUE_COMMENT_DATE_PATTERN.match(d).group(1))

                # Description is content on the second 'p'.
                comment["desc"] = self.__clean_str(u"".join(rawdesc.contents))

                comments.append(comment)
        except:
            SourceForgeParserError("Errror parsing issue comments")
        printdbg("%s comments" % (str(len(comments))))
        return comments
コード例 #7
0
ファイル: allura.py プロジェクト: brainwane/Bicho
    def analyze_bug(self, bug_url):
        #Retrieving main bug information
        printdbg(bug_url)
        bug_number = bug_url.split('/')[-1]

        try:
            f = urllib.urlopen(bug_url)

            # f = urllib.urlopen(bug_url)
            json_ticket = f.read()
            # print json_ticket
            try:
                issue_allura = json.loads(json_ticket)["ticket"]
                issue = self.parse_bug(issue_allura)
                changes = self.analyze_bug_changes(bug_url)
                for c in changes:
                    issue.add_change(c)
                return issue

            except Exception, e:
                print "Problems with Ticket format: " + bug_number
                print e
                return None

        except Exception, e:
            printerr("Error in bug analysis: " + bug_url)
            print(e)
            raise
コード例 #8
0
ファイル: sf.py プロジェクト: iKuba/Bicho
    def __parse_issue_changes(self, soup):
        """
        """
        changes = []
        try:
            entries = soup.find('h4', {
                'id': 'changebar'
            }).findNext('tbody').findAll('tr')
            for e in entries:
                change = {}
                # Each change contains four fields (td tags) that
                # follow the next order: field, old value, date, by.
                aux = e.findAll('td')
                change['field'] = self.__clean_str(aux[0].string)
                change['old_value'] = self.__clean_str(aux[1].string)
                change['date'] = self.__str_to_date(
                    self.__clean_str(aux[2].string))
                if aux[3].a:
                    change['by'] = {
                        'name': self.__clean_str(aux[3].a.get('title')),
                        'id': self.__clean_str(aux[3].a.string)
                    }
                else:
                    change['by'] = {'name': 'nobody', 'id': 'nobody'}

                changes.append(change)
        except AttributeError:
            # there are no changes
            pass
        except:
            raise SourceForgeParserError('Errror parsing issue changes')
        printdbg("%s changes" % (str(len(changes))))
        return changes
コード例 #9
0
    def _retrieve_issues_ids(self,
                             base_url,
                             version,
                             from_date,
                             not_retrieved=True):
        url = self._get_issues_list_url(base_url, version, from_date)
        printdbg("Getting bugzilla issues from %s" % url)

        f = self._urlopen_auth(url)

        # Problems using csv library, not all the fields are delimited by
        # '"' character. Easier using split.
        # Moreover, we drop the first line of the CSV because it contains
        # the headers
        ids = []
        csv = f.read().split('\n')[1:]
        for line in csv:
            # 0: bug_id, 7: changeddate
            values = line.split(',')
            id = values[0]
            change_ts = values[7].strip('"')

            # Filter issues already retrieved
            if not_retrieved:
                if (not self.retrieved.has_key(id)) or (self.retrieved[id] !=
                                                        change_ts):
                    ids.append(id)
            else:
                ids.append(id)
        return ids
コード例 #10
0
    def _login(self):
        """
        Authenticates a user in a bugzilla tracker
        """
        if not (self.backend_user and self.backend_password):
            printdbg("No account data provided. Not logged in bugzilla")
            return

        import cookielib

        cookie_j = cookielib.CookieJar()
        cookie_h = urllib2.HTTPCookieProcessor(cookie_j)

        url = self._get_login_url(self.url)
        values = {
            'Bugzilla_login': self.backend_user,
            'Bugzilla_password': self.backend_password
        }

        opener = urllib2.build_opener(cookie_h)
        urllib2.install_opener(opener)
        data = urllib.urlencode(values)
        request = urllib2.Request(url, data)
        urllib2.urlopen(request)
        for i, c in enumerate(cookie_j):
            self.cookies[c.name] = c.value

        printout("Logged in bugzilla as %s" % self.backend_user)
        printdbg("Bugzilla session cookies: %s" % self.cookies)
コード例 #11
0
ファイル: sf.py プロジェクト: brainwane/Bicho
    def __parse_issue_changes(self, soup):
        """
        """
        changes = []
        try:
            entries = soup.find('h4', {'id': 'changebar'}).findNext('tbody').findAll('tr')
            for e in entries:
                change = {}
                # Each change contains four fields (td tags) that
                # follow the next order: field, old value, date, by.
                aux = e.findAll('td')
                change['field'] = self.__clean_str(aux[0].string)
                change['old_value'] = self.__clean_str(aux[1].string)
                change['date'] = self.__str_to_date(self.__clean_str(aux[2].string))
                if aux[3].a:
                    change['by'] = {'name': self.__clean_str(aux[3].a.get('title')),
                                    'id': self.__clean_str(aux[3].a.string)}
                else:
                    change['by'] = {'name': 'nobody', 'id': 'nobody'}

                changes.append(change)
        except AttributeError:
            # there are no changes
            pass
        except:
            raise SourceForgeParserError('Errror parsing issue changes')
        printdbg("%s changes" % (str(len(changes))))
        return changes
コード例 #12
0
ファイル: jira.py プロジェクト: iKuba/Bicho
 def bugsNumber(self,url):
     oneBug = self.basic_jira_url()
     oneBug += "&tempMax=1"
     printdbg("Getting number of issues: " + oneBug)
     data_url = urllib.urlopen(oneBug).read()
     bugs = data_url.split("<issue")[1].split('\"/>')[0].split("total=\"")[1]
     return int(bugs)
コード例 #13
0
ファイル: jira.py プロジェクト: iKuba/Bicho
    def run(self):
        printout("Running Bicho with delay of %s seconds" % (str(self.delay)))

        issues_per_xml_query = 500
        bugsdb = get_database(DBJiraBackend())

        bugsdb.insert_supported_traker("jira","4.1.2")
        trk = Tracker(self.url.split("-")[0], "jira", "4.1.2")
        dbtrk = bugsdb.insert_tracker(trk)

        serverUrl = self.url.split("/browse/")[0]
        query =  "/si/jira.issueviews:issue-xml/"
        project = self.url.split("/browse/")[1]

        if (project.split("-").__len__() > 1):
            bug_key = project
            project = project.split("-")[0]
            bugs_number = 1

            printdbg(serverUrl + query + bug_key + "/" + bug_key + ".xml")

            parser = xml.sax.make_parser(  )
            handler = BugsHandler(  )
            parser.setContentHandler(handler)
            try:
                parser.parse(serverUrl + query + bug_key + "/" + bug_key + ".xml")
                issue = handler.getIssues()[0]
                bugsdb.insert_issue(issue, dbtrk.id)
            except Exception, e:
                #printerr(e)
                print(e)
コード例 #14
0
    def analyze_bug(self, bug_url):
        #Retrieving main bug information
        printdbg(bug_url)
        bug_number = bug_url.split('/')[-1]

        try:
            f = urllib.urlopen(bug_url)

            # f = urllib.urlopen(bug_url) 
            json_ticket = f.read()
            # print json_ticket
            try:                
                issue_allura = json.loads(json_ticket)["ticket"]
                issue =  self.parse_bug(issue_allura)
                changes = self.analyze_bug_changes(bug_url)
                for c in changes:
                    issue.add_change(c)                 
                return issue

            except Exception, e:
                print "Problems with Ticket format: " + bug_number
                print e
                return None
    
        except Exception, e:
            printerr("Error in bug analysis: " + bug_url);
            print(e)
            raise
コード例 #15
0
    def _retrieve_issues(self, ids, base_url, trk_id):
        # We want to use pop() to get the oldest first so we must reverse the
        # order
        ids.reverse()
        while (ids):
            query_issues = []
            while (len(query_issues) < MAX_ISSUES_PER_XML_QUERY and ids):
                query_issues.append(ids.pop())

            # Retrieving main bug information
            url = self._get_issues_info_url(base_url, query_issues)
            printdbg("Issues to retrieve from: %s" % url)

            handler = BugsHandler()
            self._safe_xml_parse(url, handler)
            issues = handler.get_issues()

            # Retrieving changes
            for issue in issues:
                changes = self._retrieve_issue_activity(base_url, issue.issue)
                for c in changes:
                    issue.add_change(c)

                # We store here the issue once the complete retrieval
                # for each bug is done
                self._store_issue(issue, trk_id)
                self.retrieved[issue.issue] = self._timestamp_to_str(
                    issue.delta_ts)

                time.sleep(self.delay)
コード例 #16
0
ファイル: bg.py プロジェクト: f-breidenstein/Bicho
    def _login(self):
        """
        Authenticates a user in a bugzilla tracker
        """
        if not (self.backend_user and self.backend_password):
            printdbg("No account data provided. Not logged in bugzilla")
            return

        import cookielib

        cookie_j = cookielib.CookieJar()
        cookie_h = urllib2.HTTPCookieProcessor(cookie_j)

        url = self._get_login_url(self.url)
        values = {'Bugzilla_login': self.backend_user,
                  'Bugzilla_password': self.backend_password}

        opener = urllib2.build_opener(cookie_h)
        urllib2.install_opener(opener)
        data = urllib.urlencode(values)
        request = urllib2.Request(url, data)
        urllib2.urlopen(request)
        for i, c in enumerate(cookie_j):
            self.cookies[c.name] = c.value

        printout("Logged in bugzilla as %s" % self.backend_user)
        printdbg("Bugzilla session cookies: %s" % self.cookies)
コード例 #17
0
ファイル: bg.py プロジェクト: f-breidenstein/Bicho
    def _safe_xml_parse(self, bugs_url, handler):
        f = self._urlopen_auth(bugs_url)
        parser = xml.sax.make_parser()
        parser.setContentHandler(handler)

        try:
            contents = f.read()
        except Exception:
            printerr("Error retrieving URL: %s" % (bugs_url))
            raise

        try:
            parser.feed(contents)
            parser.close()
        except Exception:
            # Clean only the invalid XML
            try:
                parser2 = xml.sax.make_parser()
                parser2.setContentHandler(handler)
                parser2.setContentHandler(handler)
                printdbg("Cleaning dirty XML")
                cleaned_contents = ''. \
                    join(c for c in contents if valid_XML_char_ordinal(ord(c)))
                parser2.feed(cleaned_contents)
                parser2.close()
            except Exception:
                printerr("Error parsing URL: %s" % (bugs_url))
                raise
        f.close()
コード例 #18
0
    def insert_new_bugs_created(self, date_from, date_to):
        """
        This method inserts an entry with the data of the creation time
        """
        if (not date_from) and (not date_to):
            issues = self.store.find(DBIssue)
        elif not date_from:
            issues = self.store.find(DBIssue, DBIssue.submitted_on < date_to)
        elif not date_to:
            issues = self.store.find(DBIssue, DBIssue.submitted_on > date_from)
        else:
            issues = self.store.find(
                DBIssue,
                And(DBIssue.submitted_on <= date_to,
                    DBIssue.submitted_on > date_from))

        issues = issues.order_by(Asc(DBIssue.submitted_on))
        ## we store the initial data for each bug found
        for i in issues:
            db_ilog = self.get_last_values(i)  # from issues and change tables
            db_ilog = self.build_initial_state(db_ilog)
            self.store.add(db_ilog)
            printdbg(
                "Issue #%s created at %s - date_from = %s - date_to = %s" %
                (db_ilog.issue, db_ilog.date, date_from, date_to))
コード例 #19
0
ファイル: sf.py プロジェクト: brainwane/Bicho
    def __parse_issue_comments(self, soup):
        """
        """
        comments = []
        try:
            artifacts = soup.findAll('tr', {'class': ISSUE_COMMENT_CLASS_PATTERN})
            for art in artifacts:
                comment = {}

                rawsub, rawdesc = art.findAll('p')
                # Date and sender are content on the first 'p'
                a = rawsub.find('a')
                if a:
                    comment['by'] = {'name': a.get('title'), 'id': a.string}
                else:
                    comment['by'] = {'name': 'nobody', 'id': 'nobody'}

                # Time stamp is the first value of the 'p' contents
                d = self.__clean_str(rawsub.contents[0])
                comment['date'] = self.__str_to_date(ISSUE_COMMENT_DATE_PATTERN.match(d).group(1))

                # Description is content on the second 'p'.
                comment['desc'] = self.__clean_str(u''.join(rawdesc.contents))

                comments.append(comment)
        except:
            SourceForgeParserError('Errror parsing issue comments')
        printdbg("%s comments" % (str(len(comments))))
        return comments
コード例 #20
0
    def run(self):
        printout("Running Bicho with delay of %s seconds" % (str(self.delay)))

        issues_per_xml_query = 500
        bugsdb = get_database(DBJiraBackend())

        bugsdb.insert_supported_traker("jira", "4.1.2")
        trk = Tracker(self.url.split("-")[0], "jira", "4.1.2")
        dbtrk = bugsdb.insert_tracker(trk)

        serverUrl = self.url.split("/browse/")[0]
        query = "/si/jira.issueviews:issue-xml/"
        project = self.url.split("/browse/")[1]

        if (project.split("-").__len__() > 1):
            bug_key = project
            project = project.split("-")[0]
            bugs_number = 1

            printdbg(serverUrl + query + bug_key + "/" + bug_key + ".xml")

            parser = xml.sax.make_parser()
            handler = BugsHandler()
            parser.setContentHandler(handler)
            try:
                parser.parse(serverUrl + query + bug_key + "/" + bug_key +
                             ".xml")
                issue = handler.getIssues()[0]
                bugsdb.insert_issue(issue, dbtrk.id)
            except Exception, e:
                #printerr(e)
                print(e)
コード例 #21
0
ファイル: bg.py プロジェクト: f-breidenstein/Bicho
    def _retrieve_issues(self, ids, base_url, trk_id):
        # We want to use pop() to get the oldest first so we must reverse the
        # order
        ids.reverse()
        while(ids):
            query_issues = []
            while (len(query_issues) < MAX_ISSUES_PER_XML_QUERY and ids):
                query_issues.append(ids.pop())

            # Retrieving main bug information
            url = self._get_issues_info_url(base_url, query_issues)
            printdbg("Issues to retrieve from: %s" % url)

            handler = BugsHandler()
            self._safe_xml_parse(url, handler);
            issues = handler.get_issues()

            # Retrieving changes
            for issue in issues:
                changes = self._retrieve_issue_activity(base_url, issue.issue)
                for c in changes:
                    issue.add_change(c)

                # We store here the issue once the complete retrieval
                # for each bug is done
                self._store_issue(issue, trk_id)
                self.retrieved[issue.issue] = self._timestamp_to_str(issue.delta_ts)

                time.sleep(self.delay)
コード例 #22
0
 def parse_changes (self, activity):
     changesList = []
     for entry in activity['entries']:
         # print "changed_by:" + entry['author']
         by = People(entry['author'])
         # print "changed_on:" + entry['updated']
         description = entry['description'].split('updated:')
         changes = description.pop(0)
         field = changes.rpartition('\n')[2].strip()
         while description:                
             changes = description.pop(0).split('\n')
             values = changes[0].split('=>')                
             if (len(values) != 2):
                 printdbg(field + " not supported in changes analysis")
                 old_value = new_value = ""                    
             else:
                 # u'in-progress' => u'closed'
                 values = changes[0].split('=>')
                 old_value = self.remove_unicode(values[0].strip())
                 if old_value == "''": old_value =""
                 new_value = self.remove_unicode(values[1].strip())
                 if new_value == "''": new_value =""
             update = parse(entry['updated'])
             change = Change(unicode(field), unicode(old_value), unicode(new_value), by, update)
             changesList.append(change)
             if (len(changes)>1):
                 field = changes[1].strip()
     return changesList
コード例 #23
0
ファイル: bg.py プロジェクト: f-breidenstein/Bicho
    def _retrieve_issues_ids(self, base_url, version, from_date, not_retrieved=True):
        url = self._get_issues_list_url(base_url, version, from_date)
        printdbg("Getting bugzilla issues from %s" % url)

        f = self._urlopen_auth(url)

        # Problems using csv library, not all the fields are delimited by
        # '"' character. Easier using split.
        # Moreover, we drop the first line of the CSV because it contains
        # the headers
        ids = []
        csv = f.read().split('\n')[1:]
        for line in csv:
            # 0: bug_id, 7: changeddate
            values = line.split(',')
            id = values[0]
            change_ts = values[7].strip('"')

            # Filter issues already retrieved
            if not_retrieved:
                if (not self.retrieved.has_key(id)) or (self.retrieved[id] != change_ts):
                    ids.append(id)
            else:
                ids.append(id)
        return ids
コード例 #24
0
ファイル: bg.py プロジェクト: f-breidenstein/Bicho
 def _store_issue(self, issue, trk_id):
     try:
         self.bugsdb.insert_issue(issue, trk_id)
         printdbg("Issue #%s stored " % issue.issue)
     except UnicodeEncodeError:
         printerr("UnicodeEncodeError: the issue %s couldn't be stored"
                  % issue.issue)
コード例 #25
0
ファイル: bg.py プロジェクト: f-breidenstein/Bicho
    def _process_issues(self):
        if self._is_issue_url(self.url):
            # FIXME: this only works for one issue, if more id parameters
            # are set, those issues will not be processed
            ids = [self.url.split("show_bug.cgi?id=")[1]]
            printdbg("Issue #%s URL found" % ids[0])
            url = self._get_domain(self.url)
            self._retrieve_issues(ids, url, self.tracker.id)
        else:
            i = 0
            max_rounds = 50 # 50*10000
            url = self._get_domain(self.url)
            last_date, next_date = self._get_last_and_next_dates()

            # Some bugzillas limit the number of results that a query can return.
            # Due to this, bicho will search for new issues/changes until find
            # no one new.
            ids = self._retrieve_issues_ids(self.url, self.version, next_date)

            while(ids):
                if (i>=max_rounds): break
                printout("Round #%d - Total issues to retrieve: %d" % (i, len(ids)))
                self._retrieve_issues(ids, url, self.tracker.id)
                i += 1
                # Search new ids, but first, we have to check whether they are
                # already stored or not
                last_date, next_date = self._get_last_and_next_dates()
                ids = self._retrieve_issues_ids(self.url, self.version, last_date)
                # If there aren't new issues from the same date, ask for a new one
                if not ids:
                    printdbg("No issues found for date %s. Trying with %s" % (last_date, next_date))
                    ids = self._retrieve_issues_ids(self.url, self.version, next_date)

            if i > 0:
                printout("No more issues to retrieve")
コード例 #26
0
ファイル: sf.py プロジェクト: libresoft/Bicho
    def __parse_issue_changes(self, soup):
        """
        """
        changes = []
        try:
            entries = soup.find("h4", {"id": "changebar"}).findNext("tbody").findAll("tr")
            for e in entries:
                change = {}
                # Each change contains four fields (td tags) that
                # follow the next order: field, old value, date, by.
                aux = e.findAll("td")
                change["field"] = self.__clean_str(aux[0].string)
                change["old_value"] = self.__clean_str(aux[1].string)
                change["date"] = self.__str_to_date(self.__clean_str(aux[2].string))
                if aux[3].a:
                    change["by"] = {
                        "name": self.__clean_str(aux[3].a.get("title")),
                        "id": self.__clean_str(aux[3].a.string),
                    }
                else:
                    change["by"] = {"name": "nobody", "id": "nobody"}

                changes.append(change)
        except AttributeError:
            # there are no changes
            pass
        except:
            raise SourceForgeParserError("Errror parsing issue changes")
        printdbg("%s changes" % (str(len(changes))))
        return changes
コード例 #27
0
    def _safe_xml_parse(self, bugs_url, handler):
        f = self._urlopen_auth(bugs_url)
        parser = xml.sax.make_parser()
        parser.setContentHandler(handler)

        try:
            contents = f.read()
        except Exception:
            printerr("Error retrieving URL: %s" % (bugs_url))
            raise

        try:
            parser.feed(contents)
            parser.close()
        except Exception:
            # Clean only the invalid XML
            try:
                parser2 = xml.sax.make_parser()
                parser2.setContentHandler(handler)
                parser2.setContentHandler(handler)
                printdbg("Cleaning dirty XML")
                cleaned_contents = ''. \
                    join(c for c in contents if valid_XML_char_ordinal(ord(c)))
                parser2.feed(cleaned_contents)
                parser2.close()
            except Exception:
                printerr("Error parsing URL: %s" % (bugs_url))
                raise
        f.close()
コード例 #28
0
 def _store_issue(self, issue, trk_id):
     try:
         self.bugsdb.insert_issue(issue, trk_id)
         printdbg("Issue #%s stored " % issue.issue)
     except UnicodeEncodeError:
         printerr("UnicodeEncodeError: the issue %s couldn't be stored" %
                  issue.issue)
コード例 #29
0
ファイル: googlecode.py プロジェクト: AlertProject/Bicho
    def run(self):
        """
        """
        printout("Running Bicho with delay of %s seconds" % (str(self.delay)))
        
        issues_per_query = 250
        start_issue=1

        bugs = [];
        bugsdb = get_database (DBGoogleCodeBackend())
                
        # still useless
        bugsdb.insert_supported_traker("googlecode", "beta")
        trk = Tracker (Config.url, "googlecode", "beta")

        dbtrk = bugsdb.insert_tracker(trk)
        
        self.url = Config.url
        
        
       #  https://code.google.com/feeds/issues/p/mobile-time-care
        self.url_issues = Config.url + "/issues/full?max-results=1" 
        printdbg("URL for getting metadata " + self.url_issues)        
                            
        d = feedparser.parse(self.url_issues)
                
        total_issues = int(d['feed']['opensearch_totalresults'])
        print "Total bugs: ", total_issues
        if  total_issues == 0:
            printout("No bugs found. Did you provide the correct url?")
            sys.exit(0)
        remaining = total_issues                

        print "ETA ", (total_issues*Config.delay)/(60), "m (", (total_issues*Config.delay)/(60*60), "h)"

        while start_issue < total_issues:            
            self.url_issues = Config.url + "/issues/full?max-results=" + str(issues_per_query) 
            self.url_issues += "&start-index=" + str(start_issue)
            
            printdbg("URL for next issues " + self.url_issues) 
                                                            
            d = feedparser.parse(self.url_issues)
                                    
            for entry in d['entries']:
                try:
                    issue = self.analyze_bug(entry)
                    if issue is None:
                        continue
                    bugsdb.insert_issue(issue, dbtrk.id)
                    remaining -= 1
                    print "Remaining time: ", (remaining)*Config.delay/60, "m", " issues ", str(remaining) 
                    time.sleep(Config.delay)
                except Exception, e:
                    printerr("Error in function analyze_bug ")
                    pprint.pprint(entry)
                    traceback.print_exc(file=sys.stdout)
                except UnicodeEncodeError:
                    printerr("UnicodeEncodeError: the issue %s couldn't be stored"
                          % (issue.issue))
コード例 #30
0
 def bugsNumber(self, url):
     oneBug = self.basic_jira_url()
     oneBug += "&tempMax=1"
     printdbg("Getting number of issues: " + oneBug)
     data_url = urllib.urlopen(oneBug).read()
     bugs = data_url.split("<issue")[1].split('\"/>')[0].split(
         "total=\"")[1]
     return int(bugs)
コード例 #31
0
 def _retrieve_issue_activity(self, base_url, id):
     activity_url = self._get_issue_activity_url(base_url, id)
     printdbg("Retrieving activity of issue #%s from %s" %
              (id, activity_url))
     data = self._urlopen_auth(activity_url).read()
     parser = SoupHtmlParser(data, id)
     changes = parser.parse_changes()
     return changes
コード例 #32
0
ファイル: bg.py プロジェクト: f-breidenstein/Bicho
 def _retrieve_issue_activity(self, base_url, id):
     activity_url = self._get_issue_activity_url(base_url, id)
     printdbg("Retrieving activity of issue #%s from %s"
              % (id, activity_url))
     data = self._urlopen_auth(activity_url).read()
     parser = SoupHtmlParser(data, id)
     changes = parser.parse_changes()
     return changes
コード例 #33
0
    def analyze_bug_changes (self, bug_url):
        bug_number = bug_url.split('/')[-1]
        changes_url = bug_url.replace("rest/","")+"/feed.atom"

        printdbg("Analyzing issue changes" + changes_url)

        d = feedparser.parse(changes_url)
        changes = self.parse_changes(d)
        
        return changes
コード例 #34
0
ファイル: redmine.py プロジェクト: AlertProject/Bicho
    def analyze_bug_changes (self, bug_url):
        bug_number = bug_url.split('/')[-1]
        changes_url = bug_url.replace("rest/","")+"/feed.atom"

        printdbg("Analyzing issue changes" + changes_url)

        d = feedparser.parse(changes_url)
        changes = self.parse_changes(d)
        
        return changes
コード例 #35
0
ファイル: redmine.py プロジェクト: iKuba/Bicho
 def _get_issue_changeset(self, bug_id):
     aux = Config.url.rfind('/')
     bug_url = Config.url[:aux + 1]
     bug_url = bug_url + "issues/" + unicode(bug_id) + ".atom"
     
     # changes_url = bug_url.replace("rest/","")+"/feed.atom"
     printdbg("Analyzing issue changes " + bug_url)
     d = feedparser.parse(bug_url)
     changes = self.parse_changes(d)
     
     return changes
コード例 #36
0
ファイル: gerrit.py プロジェクト: brainwane/Bicho
    def run(self):
        """
        """
        printout("Running Bicho with delay of %s seconds" % (str(self.delay)))

        bugs = []
        bugsdb = get_database(DBGerritBackend())

        # still useless in gerrit
        bugsdb.insert_supported_traker("gerrit", "beta")
        trk = Tracker(Config.url + "_" + Config.gerrit_project, "gerrit", "beta")
        dbtrk = bugsdb.insert_tracker(trk)

        last_mod_time = 0
        last_mod_date = bugsdb.get_last_modification_date(dbtrk.id)
        if last_mod_date:
            printdbg("Last reviews analyzed were modified on date: %s"
                     % last_mod_date)
            last_mod_time = time.mktime(time.strptime(last_mod_date, '%Y-%m-%d %H:%M:%S'))

        limit = 500  # gerrit default 500
        last_item = ""
        # last_item = "001f672c00002f80";
        number_results = limit
        total_reviews = 0

        while (number_results == limit or
               number_results == limit + 1):  # wikimedia gerrit returns limit+1
            # ordered by lastUpdated
            tickets = self.getReviews(limit, last_item)
            number_results = 0

            reviews = []
            for entry in tickets:
                if 'project' in entry.keys():
                    if (entry['lastUpdated'] < last_mod_time):
                        break
                    reviews.append(entry["number"])
                    review_data = self.analyze_review(entry)
                    last_item = entry['sortKey']
                    # extra changes not included in gerrit changes
                    # self.add_merged_abandoned_changes_from_comments(entry, review_data)
                    self.add_merged_abandoned_changes(entry, review_data)
                    self.add_new_change(review_data)
                    bugsdb.insert_issue(review_data, dbtrk.id)
                    number_results += 1
                elif 'rowCount' in entry.keys():
                    pprint.pprint(entry)
                    printdbg("CONTINUE FROM: " + last_item)
            total_reviews = total_reviews + int(number_results)
        self.check_merged_abandoned_changes(bugsdb.store, dbtrk.id)

        print("Done. Number of reviews: " + str(total_reviews))
コード例 #37
0
ファイル: database.py プロジェクト: AlertProject/Bicho
    def store_final_relationships(self):
        """
        """
        temp_rels = self.store.find(DBIssueTempRelationship)

        for tr in temp_rels:
            aux_issue_id = self._get_db_issue(tr.issue_id, tr.tracker_id)
            aux_related_to = self._get_db_issue(tr.related_to, tr.tracker_id)
            if (aux_related_to != -1 and aux_issue_id != -1):
                self._insert_relationship(aux_issue_id.id, tr.type, aux_related_to.id)
            else:
                printdbg("Issue %s belongs to a different tracker and won't be stored" % tr.related_to)
コード例 #38
0
ファイル: redmine.py プロジェクト: iKuba/Bicho
 def _get_author_email(self, author_id):
     root = self._get_redmine_root(Config.url)
     author_url = root + "users/" + str(author_id) + ".json"
     #print author_url
     res = None
     try:
         f = urllib2.urlopen(author_url)         
         person = json.loads(f.read())
         res = person['user']['mail']
     except (urllib2.HTTPError, KeyError):
         printdbg("User with id %s has no account information" % author_id)
         res = author_id
     return res
コード例 #39
0
ファイル: bg.py プロジェクト: f-breidenstein/Bicho
    def _get_last_and_next_dates(self):
        last_ts = self.bugsdb.get_last_modification_date(self.tracker.id)

        if not last_ts:
            return None, None
        printdbg("Last issues cached were modified on: %s" % last_ts)

        last_ts_str = self._timestamp_to_str(last_ts)

        # We add one second to the last date to avoid retrieve the same
        # changes modified at that date.
        next_ts = last_ts + timedelta(seconds=1)
        next_ts_str = self._timestamp_to_str(next_ts)
        return last_ts_str, next_ts_str
コード例 #40
0
ファイル: jira.py プロジェクト: AlertProject/Bicho
    def parse_changes(self):
        soup = BeautifulSoup(self.html)
        self.remove_comments(soup)
        remove_tags = ['a', 'span','i']
        try:
            [i.replaceWith(i.contents[0]) for i in soup.findAll(remove_tags)]
        except Exception:
            None
        
        changes = []      
        #FIXME The id of the changes are not stored
        tables = soup.findAll("div", {"class": "actionContainer"})
        table = None
    
        for table in tables:
            change_author = table.find("div", {"class": "action-details"})
            if change_author == None or len(change_author)<3:
                self.changes_lost += 1
                printerr("Change author format not supported. Change lost!")
                continue
            if isinstance(change_author.contents[2], Tag):
                change_author_str = change_author.contents[2]['rel']
            elif isinstance(change_author.contents[2], NavigableString):
                change_author_str = change_author.contents[2]
            else:
                printerr("Change author format not supported")
                printdbg(change_author)
                continue
            author = People(change_author_str.strip())
            author.set_email(BugsHandler.getUserEmail(change_author_str.strip()))
            if isinstance(change_author.contents[4], Tag):
                date_str = change_author.contents[4].find('time')['datetime']
            elif isinstance(change_author.contents[4], NavigableString):
                date_str = change_author.contents[4]
            else:
                printerr("Change date format not supported")
                continue
            date = parse(date_str).replace(tzinfo=None)
 
            rows = list(table.findAll('tr'))
            for row in rows:
                cols = list(row.findAll('td'))
                if len(cols) == 3:
                    field = unicode(cols[0].contents[0].strip())
                    old = unicode(cols[1].contents[0].strip())
                    new = unicode(cols[2].contents[0].strip())
                    
                    change = Change(field, old, new, author, date)
                    changes.append(change)
        return changes
コード例 #41
0
    def _get_last_and_next_dates(self):
        last_ts = self.bugsdb.get_last_modification_date(self.tracker.id)

        if not last_ts:
            return None, None
        printdbg("Last issues cached were modified on: %s" % last_ts)

        last_ts_str = self._timestamp_to_str(last_ts)

        # We add one second to the last date to avoid retrieve the same
        # changes modified at that date.
        next_ts = last_ts + timedelta(seconds=1)
        next_ts_str = self._timestamp_to_str(next_ts)
        return last_ts_str, next_ts_str
コード例 #42
0
    def run(self):
        """
        """
        printout("Running Bicho with delay of %s seconds" % (str(self.delay)))
        
        bugs = [];
        bugsdb = get_database (DBGerritBackend())
                
        # still useless in gerrit
        bugsdb.insert_supported_traker("gerrit", "beta")
        trk = Tracker (Config.url+"_"+Config.gerrit_project, "gerrit", "beta")
        dbtrk = bugsdb.insert_tracker(trk)
        
        last_mod_time = 0        
        last_mod_date = bugsdb.get_last_modification_date(dbtrk.id)                
        if last_mod_date:
            printdbg("Last reviews analyzed were modified on date: %s" 
                     % last_mod_date)
            last_mod_time = time.mktime(time.strptime
                                        (last_mod_date, '%Y-%m-%d %H:%M:%S'))
            
        limit = 500 # gerrit default 500
        last_item = "";
        # last_item = "001f672c00002f80";
        number_results = limit
        total_reviews = 0
        
        while (number_results == limit or
               number_results == limit+1): # wikimedia gerrit returns limit+1
            # ordered by lastUpdated        
            tickets = self.getReviews(limit, last_item)
            number_results = 0
                
            reviews = []
            for entry in tickets:
                if 'project' in entry.keys():
                    if (entry['lastUpdated']<last_mod_time):
                        break
                    reviews.append(entry["number"])
                    review_data = self.analyze_review(entry)
                    last_item = entry['sortKey']
                    bugsdb.insert_issue(review_data, dbtrk.id)
                    number_results = number_results+1
                elif 'rowCount' in entry.keys():
                    pprint.pprint(entry)
                    printdbg("CONTINUE FROM: " + last_item)
            total_reviews = total_reviews + int(number_results)

        print("Done. Number of reviews: " + str(total_reviews))
コード例 #43
0
    def store_final_relationships(self):
        """
        """
        temp_rels = self.store.find(DBIssueTempRelationship)

        for tr in temp_rels:
            aux_issue_id = self._get_db_issue(tr.issue_id, tr.tracker_id)
            aux_related_to = self._get_db_issue(tr.related_to, tr.tracker_id)
            if (aux_related_to != -1 and aux_issue_id != -1):
                self._insert_relationship(aux_issue_id.id, tr.type,
                                          aux_related_to.id)
            else:
                printdbg(
                    "Issue %s belongs to a different tracker and won't be stored"
                    % tr.related_to)
コード例 #44
0
ファイル: jira.py プロジェクト: iKuba/Bicho
    def analyze_bug_list(self, nissues, offset, bugsdb, dbtrk_id):
        url_issues = self.basic_jira_url()
        url_issues += "&tempMax=" + str(nissues) + "&pager/start=" + str(offset)
        printdbg(url_issues)
        
        handler = BugsHandler()
        self.safe_xml_parse(url_issues, handler)

        try:
            issues = handler.getIssues()            
            for issue in issues:
                bugsdb.insert_issue(issue, dbtrk_id)
        except Exception, e:
            import traceback
            traceback.print_exc()
            sys.exit(0)
コード例 #45
0
    def analyze_bug_list(self, nissues, offset, bugsdb, dbtrk_id):
        url_issues = self.basic_jira_url()
        url_issues += "&tempMax=" + str(nissues) + "&pager/start=" + str(
            offset)
        printdbg(url_issues)

        handler = BugsHandler()
        self.safe_xml_parse(url_issues, handler)

        try:
            issues = handler.getIssues()
            for issue in issues:
                bugsdb.insert_issue(issue, dbtrk_id)
        except Exception, e:
            import traceback
            traceback.print_exc()
            sys.exit(0)
コード例 #46
0
ファイル: sf.py プロジェクト: libresoft/Bicho
    def run(self, url):
        """
        """
        printout("Running Bicho with delay of %s seconds" % (str(self.delay)))

        self.url = url
        ids = []
        self.parser = SourceForgeParser()

        # first we take the bugs ids
        if url.find("aid=") > 0:
            aux = url.split("aid=")[1].split("&")[0]
            ids.append(aux)
        else:
            ids = self.__get_issues_list(self.url)

        self.__check_tracker_url(self.url)

        # order the parameters in the url to add the same tracker url
        # to data base without aid parameter
        self.__order_query(self.url)

        self.db = get_database(DBSourceForgeBackend())
        self.db.insert_supported_traker(SUPPORTED_SF_TRACKERS[0], SUPPORTED_SF_TRACKERS[1])
        self.__insert_tracker(self.url)

        nbugs = len(ids)
        if nbugs == 0:
            printout("No bugs found. Did you provide the correct url?")
            sys.exit(0)

        nbugs = len(ids)
        if nbugs == 0:
            printout("No bugs found. Did you provide the correct url?")
            sys.exit(0)

        for id in ids:
            url = self.url + "&func=detail&aid=%s" % id  # FIXME:urls!!!
            printdbg(url)
            issue = self.__get_issue(url)
            self.__insert_issue(issue)

            time.sleep(self.delay)

        printout("Done. %s bugs analyzed" % (nbugs))
コード例 #47
0
ファイル: redmine.py プロジェクト: brainwane/Bicho
    def _get_author_identity(self, author_id):
        if author_id in self.identities:
            return self.identities[author_id]

        root = self._get_redmine_root(Config.url)
        author_url = root + "users/" + str(author_id) + ".json"
        #print author_url
        identity = None
        try:
            f = urllib2.urlopen(author_url)
            person = json.loads(f.read())
            identity = person['user']['mail']
        except (urllib2.HTTPError, KeyError):
            printdbg("User with id %s has no account information" % author_id)
            identity = author_id

        self.identities[author_id] = identity
        return identity
コード例 #48
0
ファイル: issues_log.py プロジェクト: iKuba/Bicho
 def _get_people_id(self, email):
     """
     Gets the id of an user
     """
     try:
         p = self.store.find(DBPeople, DBPeople.email == email).one()
         return p.id
     except (AttributeError, NotOneError):
         p = self.store.find(DBPeople, DBPeople.user_id == email).one()
         try:
             return p.id
         except AttributeError:
             # no person was found in People with the email above, so
             # we include it
             printdbg("Person not found. Inserted with email %s " % (email))
             dp = DBPeople(email)
             self.store.add(dp)
             self.store.commit()
             return dp.id
コード例 #49
0
 def getUserEmail(username):
     return ""
     # http://issues.liferay.com/activity?maxResults=1&streams=user+IS+kalman.vincze
     if not vars(BugsHandler).has_key("_emails"):
         BugsHandler._emails = {}
     if BugsHandler._emails.has_key(username):
         email = BugsHandler._emails[username]
     else:
         serverUrl = Config.url.split("/browse/")[0]
         user_url = serverUrl + "/activity?maxResults=1&streams=user+IS+" + username
         email = ""
         d = feedparser.parse(user_url)
         if d.has_key('entries'):
             if len(d['entries']) > 0:
                 email = d['entries'][0]['author_detail']['email']
                 email = BugsHandler.remove_unicode(email)
                 printdbg(username + " " + email)
         BugsHandler._emails[username] = email
     return email
コード例 #50
0
ファイル: issues_log.py プロジェクト: brainwane/Bicho
 def _get_people_id(self, email):
     """
     Gets the id of an user
     """
     try:
         p = self.store.find(DBPeople, DBPeople.email == email).one()
         return p.id
     except (AttributeError, NotOneError):
         p = self.store.find(DBPeople, DBPeople.user_id == email).one()
         try:
             return p.id
         except AttributeError:
             # no person was found in People with the email above, so
             # we include it
             printdbg("Person not found. Inserted with email %s " % (email))
             dp = DBPeople(email)
             self.store.add(dp)
             self.store.commit()
             return dp.id
コード例 #51
0
ファイル: jira.py プロジェクト: iKuba/Bicho
 def getUserEmail(username):
     return ""
     # http://issues.liferay.com/activity?maxResults=1&streams=user+IS+kalman.vincze
     if not vars(BugsHandler).has_key("_emails"):
         BugsHandler._emails = {}
     if BugsHandler._emails.has_key(username):
         email = BugsHandler._emails[username]
     else:
         serverUrl = Config.url.split("/browse/")[0]
         user_url = serverUrl + "/activity?maxResults=1&streams=user+IS+"+username
         email = ""
         d = feedparser.parse(user_url)
         if d.has_key('entries'):
             if len(d['entries'])>0:
                 email = d['entries'][0]['author_detail']['email']
                 email = BugsHandler.remove_unicode(email)
                 printdbg(username + " " + email)
         BugsHandler._emails[username] = email
     return email
コード例 #52
0
    def _set_version(self):
        if self.version:
            printdbg("Bugzilla version: %s" % self.version)
            return

        info_url = self._get_info_url(self.url)

        f = self._urlopen_auth(info_url)
        try:
            printdbg("Getting bugzilla version from %s" % info_url)
            contents = f.read()
        except Exception:
            printerr("Error retrieving URL %s" % info_url)
            raise
        f.close()

        handler = BugzillaHandler()
        parser = xml.sax.make_parser()
        parser.setContentHandler(handler)
        try:
            cleaned_contents = ''. \
                join(c for c in contents if valid_XML_char_ordinal(ord(c)))
            parser.feed(cleaned_contents)
        except Exception:
            printerr("Error parsing URL %s" % info_url)
            raise
        parser.close()

        self.version = handler.get_version()
        printdbg("Bugzilla version: %s" % self.version)
コード例 #53
0
 def _urlopen_auth(self, url):
     """
     Opens an URL using an authenticated session
     """
     keep_trying = True
     while keep_trying:
         if self._is_auth_session():
             opener = urllib2.build_opener()
             for c in self.cookies:
                 q = str(c) + '=' + self.cookies[c]
                 opener.addheaders.append(('Cookie', q))
         keep_trying = False
         try:
             aux = urllib2.urlopen(url)
         except urllib2.HTTPError as e:
             printerr("The server couldn\'t fulfill the request.")
             printerr("Error code: %s" % e.code)
         except urllib2.URLError as e:
             printdbg("Bicho failed to reach the Bugzilla server")
             printdbg("Reason: %s" % e.reason)
             printdbg("Bicho goes into hibernation for %s seconds" %
                      HIBERNATION_LENGTH)
             time.sleep(HIBERNATION_LENGTH)
             keep_trying = True
     return aux
コード例 #54
0
ファイル: sf.py プロジェクト: iKuba/Bicho
    def run(self):
        """
        """
        printout("Running Bicho with delay of %s seconds" % (str(self.delay)))

        ids = []
        self.parser = SourceForgeParser()

        #first we take the bugs ids
        if url.find("aid=") > 0:
            aux = url.split("aid=")[1].split("&")[0]
            ids.append(aux)
        else:
            ids = self.__get_issues_list(self.url)

        self.__check_tracker_url(self.url)

        #order the parameters in the url to add the same tracker url
        #to data base without aid parameter
        self.__order_query(self.url)

        self.db = get_database(DBSourceForgeBackend())
        self.db.insert_supported_traker(SUPPORTED_SF_TRACKERS[0],
                                        SUPPORTED_SF_TRACKERS[1])
        self.__insert_tracker(self.url)

        nbugs = len(ids)
        if nbugs == 0:
            printout("No bugs found. Did you provide the correct url?")
            sys.exit(0)

        for id in ids:
            url = self.url + '&func=detail&aid=%s' % id  # FIXME:urls!!!
            printdbg(url)
            issue = self.__get_issue(url)
            self.__insert_issue(issue)

            time.sleep(self.delay)

        printout("Done. %s bugs analyzed" % (nbugs))
コード例 #55
0
 def get_people_id(self, email, tracker_id):
     """
     Gets the id of an user
     """
     p = self.store.find(DBPeople, DBPeople.email == email).one()
     ##
     ## the code below was created ad-hoc for KDE solid
     ##
     try:
         return p.id
     except AttributeError:
         p = self.store.find(DBPeople, DBPeople.user_id == email).one()
         try:
             return p.id
         except AttributeError:
             # no person was found in People with the email above, so
             # we include it
             printdbg("Person not found. Inserted with email %s " % (email))
             dp = DBPeople(email, tracker_id)
             self.store.add(dp)
             self.store.commit()
             return dp.id
コード例 #56
0
    def _process_issues(self):
        if self._is_issue_url(self.url):
            # FIXME: this only works for one issue, if more id parameters
            # are set, those issues will not be processed
            ids = [self.url.split("show_bug.cgi?id=")[1]]
            printdbg("Issue #%s URL found" % ids[0])
            url = self._get_domain(self.url)
            self._retrieve_issues(ids, url, self.tracker.id)
        else:
            i = 0
            max_rounds = 50  # 50*10000
            url = self._get_domain(self.url)
            last_date, next_date = self._get_last_and_next_dates()

            # Some bugzillas limit the number of results that a query can return.
            # Due to this, bicho will search for new issues/changes until find
            # no one new.
            ids = self._retrieve_issues_ids(self.url, self.version, next_date)

            while (ids):
                if (i >= max_rounds): break
                printout("Round #%d - Total issues to retrieve: %d" %
                         (i, len(ids)))
                self._retrieve_issues(ids, url, self.tracker.id)
                i += 1
                # Search new ids, but first, we have to check whether they are
                # already stored or not
                last_date, next_date = self._get_last_and_next_dates()
                ids = self._retrieve_issues_ids(self.url, self.version,
                                                last_date)
                # If there aren't new issues from the same date, ask for a new one
                if not ids:
                    printdbg("No issues found for date %s. Trying with %s" %
                             (last_date, next_date))
                    ids = self._retrieve_issues_ids(self.url, self.version,
                                                    next_date)

            if i > 0:
                printout("No more issues to retrieve")
コード例 #57
0
ファイル: googlecode.py プロジェクト: iKuba/Bicho
    def analyze_bug(self, entry):
        people = People(entry['author_detail']['href'])
        people.set_name(entry['author_detail']['name'])

        issue = GoogleCodeIssue(entry['id'], 'issue', entry['title'],
                                entry['content'], people,
                                self._convert_to_datetime(entry['published']))

        # Strange how the parser rename this fields
        if 'issues_uri' in entry.keys():
            people = People(entry['issues_uri'])
            people.set_name(entry['issues_username'])
            issue.assigned_to = people
        issue.status = entry['issues_status']
        issue.resolution = entry['issues_state']
        issue.priority = entry['issues_label']

        # Extended attributes
        # issue.labels = str(issue_googlecode["labels"])
        issue.star = entry['issues_stars']
        issue.ticket_num = entry['issues_id']
        issue.mod_date = self._convert_to_datetime(entry['updated'])
        issue.closed_date = None
        if 'issues_closeddate' in entry.keys():
            issue.closed_date = self._convert_to_datetime(
                entry['issues_closeddate'])

        changes_url = Config.url + "/issues/" + issue.ticket_num + "/comments/full"
        printdbg("Analyzing issue " + changes_url)

        d = feedparser.parse(changes_url)
        changes = self.parse_changes(d, issue.ticket_num)

        for c in changes:
            issue.add_change(c)

        return issue