def analyze_bug(self, bug_url):
#Retrieving main bug information
printdbg(bug_url)
bug_number = bug_url.split('/')[-1]
try:
f = urllib.urlopen(bug_url)
# f = urllib.urlopen(bug_url)
json_ticket = f.read()
# print json_ticket
try:
issue_allura = json.loads(json_ticket)["ticket"]
issue = self.parse_bug(issue_allura)
changes = self.analyze_bug_changes(bug_url)
for c in changes:
issue.add_change(c)
return issue
except Exception, e:
print "Problems with Ticket format: " + bug_number
print e
return None
except Exception, e:
printerr("Error in bug analysis: " + bug_url)
print(e)
raise
def _urlopen_auth(self, url):
"""
Opens an URL using an authenticated session
"""
keep_trying = True
while keep_trying:
if self._is_auth_session():
opener = urllib2.build_opener()
for c in self.cookies:
q = str(c) + '=' + self.cookies[c]
opener.addheaders.append(('Cookie', q))
keep_trying = False
try:
aux = urllib2.urlopen(url)
except urllib2.HTTPError as e:
printerr("The server couldn\'t fulfill the request.")
printerr("Error code: %s" % e.code)
except urllib2.URLError as e:
printdbg("Bicho failed to reach the Bugzilla server")
printdbg("Reason: %s" % e.reason)
printdbg("Bicho goes into hibernation for %s seconds" %
HIBERNATION_LENGTH)
time.sleep(HIBERNATION_LENGTH)
keep_trying = True
return aux
def _store_issue(self, issue, trk_id):
try:
self.bugsdb.insert_issue(issue, trk_id)
printdbg("Issue #%s stored " % issue.issue)
except UnicodeEncodeError:
printerr("UnicodeEncodeError: the issue %s couldn't be stored" %
issue.issue)
def __init__(self):
self.url = Config.url
self.delay = Config.delay
self.max_issues = Config.nissues
self.no_resume = Config.no_resume
if Config.start_from:
# Date format was already checked by config class
from dateutil import parser
self.start_from = parser.parse(Config.start_from)
self.from_id = None
elif Config.from_id:
self.from_id = Config.from_id
self.start_from = None
else:
self.start_from = None
self.from_id = None
self.db = get_database(DBManiphestBackend())
self.identities = {}
self.projects = {}
try:
self.backend_token = Config.backend_token
self.conduit = Conduit(self.url, self.backend_token)
except AttributeError:
printerr("Error: --backend-token is mandatory to download issues from Maniphest\n")
sys.exit(1)
def _store_issue(self, issue, trk_id):
try:
self.bugsdb.insert_issue(issue, trk_id)
printdbg("Issue #%s stored " % issue.issue)
except UnicodeEncodeError:
printerr("UnicodeEncodeError: the issue %s couldn't be stored"
% issue.issue)
def _set_version(self):
if self.version:
printdbg("Bugzilla version: %s" % self.version)
return
info_url = self._get_info_url(self.url)
f = self._urlopen_auth(info_url)
try:
printdbg("Getting bugzilla version from %s" % info_url)
contents = f.read()
except Exception:
printerr("Error retrieving URL %s" % info_url)
raise
f.close()
handler = BugzillaHandler()
parser = xml.sax.make_parser()
parser.setContentHandler(handler)
try:
cleaned_contents = ''. \
join(c for c in contents if valid_XML_char_ordinal(ord(c)))
parser.feed(cleaned_contents)
except Exception:
printerr("Error parsing URL %s" % info_url)
raise
parser.close()
self.version = handler.get_version()
printdbg("Bugzilla version: %s" % self.version)
def __init__(self):
self.url = Config.url
self.delay = Config.delay
self.max_issues = Config.nissues
self.no_resume = Config.no_resume
if Config.start_from:
# Date format was already checked by config class
from dateutil import parser
self.start_from = parser.parse(Config.start_from)
self.from_id = None
elif Config.from_id:
self.from_id = Config.from_id
self.start_from = None
else:
self.start_from = None
self.from_id = None
self.db = get_database(DBManiphestBackend())
self.identities = {}
self.projects = {}
try:
self.backend_token = Config.backend_token
self.conduit = Conduit(self.url, self.backend_token)
except AttributeError:
printerr(
"Error: --backend-token is mandatory to download issues from Maniphest\n"
)
sys.exit(1)
def analyze_bug(self, bug_url):
#Retrieving main bug information
printdbg(bug_url)
bug_number = bug_url.split('/')[-1]
try:
f = urllib.urlopen(bug_url)
# f = urllib.urlopen(bug_url)
json_ticket = f.read()
# print json_ticket
try:
issue_allura = json.loads(json_ticket)["ticket"]
issue = self.parse_bug(issue_allura)
changes = self.analyze_bug_changes(bug_url)
for c in changes:
issue.add_change(c)
return issue
except Exception, e:
print "Problems with Ticket format: " + bug_number
print e
return None
except Exception, e:
printerr("Error in bug analysis: " + bug_url)
print(e)
raise
def _safe_xml_parse(self, bugs_url, handler):
f = self._urlopen_auth(bugs_url)
parser = xml.sax.make_parser()
parser.setContentHandler(handler)
try:
contents = f.read()
except Exception:
printerr("Error retrieving URL: %s" % (bugs_url))
raise
try:
parser.feed(contents)
parser.close()
except Exception:
# Clean only the invalid XML
try:
parser2 = xml.sax.make_parser()
parser2.setContentHandler(handler)
parser2.setContentHandler(handler)
printdbg("Cleaning dirty XML")
cleaned_contents = ''. \
join(c for c in contents if valid_XML_char_ordinal(ord(c)))
parser2.feed(cleaned_contents)
parser2.close()
except Exception:
printerr("Error parsing URL: %s" % (bugs_url))
raise
f.close()
def _safe_xml_parse(self, bugs_url, handler):
f = self._urlopen_auth(bugs_url)
parser = xml.sax.make_parser()
parser.setContentHandler(handler)
try:
contents = f.read()
except Exception:
printerr("Error retrieving URL: %s" % (bugs_url))
raise
try:
parser.feed(contents)
parser.close()
except Exception:
# Clean only the invalid XML
try:
parser2 = xml.sax.make_parser()
parser2.setContentHandler(handler)
parser2.setContentHandler(handler)
printdbg("Cleaning dirty XML")
cleaned_contents = ''. \
join(c for c in contents if valid_XML_char_ordinal(ord(c)))
parser2.feed(cleaned_contents)
parser2.close()
except Exception:
printerr("Error parsing URL: %s" % (bugs_url))
raise
f.close()
def _urlopen_auth(self, url):
"""
Opens an URL using an authenticated session
"""
keep_trying = True
while keep_trying:
if self._is_auth_session():
opener = urllib2.build_opener()
for c in self.cookies:
q = str(c) + '=' + self.cookies[c]
opener.addheaders.append(('Cookie', q))
keep_trying = False
try:
aux = urllib2.urlopen(url)
except urllib2.HTTPError as e:
printerr("The server couldn\'t fulfill the request.")
printerr("Error code: %s" % e.code)
except urllib2.URLError as e:
printdbg("Bicho failed to reach the Bugzilla server")
printdbg("Reason: %s" % e.reason)
printdbg("Bicho goes into hibernation for %s seconds"
% HIBERNATION_LENGTH)
time.sleep(HIBERNATION_LENGTH)
keep_trying = True
return aux
def _set_version(self):
if self.version:
printdbg("Bugzilla version: %s" % self.version)
return
info_url = self._get_info_url(self.url)
f = self._urlopen_auth(info_url)
try:
printdbg("Getting bugzilla version from %s" % info_url)
contents = f.read()
except Exception:
printerr("Error retrieving URL %s" % info_url)
raise
f.close()
handler = BugzillaHandler()
parser = xml.sax.make_parser()
parser.setContentHandler(handler)
try:
cleaned_contents = ''. \
join(c for c in contents if valid_XML_char_ordinal(ord(c)))
parser.feed(cleaned_contents)
except Exception:
printerr("Error parsing URL %s" % info_url)
raise
parser.close()
self.version = handler.get_version()
printdbg("Bugzilla version: %s" % self.version)
def parse_changes(self):
soup = BeautifulSoup(self.html)
self.remove_comments(soup)
remove_tags = ['i']
try:
[i.replaceWith(i.contents[0]) for i in soup.findAll(remove_tags)]
except Exception:
None
changes = []
#FIXME The id of the changes are not stored
tables = soup.findAll("div", {"class": "actionContainer"})
for table in tables:
author_date_text = table.find("div", {"class": "action-details"})
if author_date_text is None:
# no changes have been performed on the issue
continue
elif len(author_date_text) < 3:
self.changes_lost += 1
printerr("Change author format not supported. Change lost!")
continue
a_link = table.find("a", {"class": "user-hover user-avatar"})
if a_link:
# at this point a_link will be similar to the lines below:
#<a class="user-hover user-avatar" rel="kiyoshi.lee"
author_url = a_link['rel']
author = People(author_url)
else:
# instead of <a .. we got a <span ..
span_link = table.find("span", {"class": "user-hover user-avatar"})
author_url = span_link['rel']
author = People(author_url)
# we look for a string similar to:
#<time datetime="2011-11-19T00:27-0800">19/Nov/11 12:27 AM</time>
raw_date = author_date_text.find('time')['datetime']
date = parse(raw_date).replace(tzinfo=None)
rows = list(table.findAll('tr'))
for row in rows:
cols = list(row.findAll('td'))
if len(cols) == 3:
field = unicode(cols[0].contents[0].strip())
if field == "Assignee":
old = unicode(self._get_identifier(cols[1]))
new = unicode(self._get_identifier(cols[2]))
else:
old = unicode(cols[1].contents[0].strip())
new = unicode(cols[2].contents[0].strip())
change = Change(field, old, new, author, date)
changes.append(change)
return changes
def parse_changes(self):
soup = BeautifulSoup(self.html)
self.remove_comments(soup)
remove_tags = ['i']
try:
[i.replaceWith(i.contents[0]) for i in soup.findAll(remove_tags)]
except Exception:
None
changes = []
#FIXME The id of the changes are not stored
tables = soup.findAll("div", {"class": "actionContainer"})
for table in tables:
author_date_text = table.find("div", {"class": "action-details"})
if author_date_text is None:
# no changes have been performed on the issue
continue
elif len(author_date_text) < 3:
self.changes_lost += 1
printerr("Change author format not supported. Change lost!")
continue
auth_link = table.find("a", {"class": "user-hover user-avatar"})
if not auth_link:
# Automated changes usually have this class
auth_link = table.find("a", {"class": "user-hover"})
# instead of <a .. we got a <span>
if not auth_link:
auth_link = table.find("span",
{"class": "user-hover user-avatar"})
author_url = auth_link.get('rel', 'anonymous')
author = People(author_url)
# we look for a string similar to:
#<time datetime="2011-11-19T00:27-0800">19/Nov/11 12:27 AM</time>
raw_date = author_date_text.find('time')['datetime']
date = parse(raw_date).replace(tzinfo=None)
rows = list(table.findAll('tr'))
for row in rows:
cols = list(row.findAll('td'))
if len(cols) == 3:
field = unicode(cols[0].contents[0].strip())
if field == "Assignee":
old = unicode(self._get_identifier(cols[1]))
new = unicode(self._get_identifier(cols[2]))
else:
old = unicode(cols[1].contents[0].strip())
new = unicode(cols[2].contents[0].strip())
change = Change(field, old, new, author, date)
changes.append(change)
return changes
def run(self):
printout("Running Bicho - url: %s" % self.url)
try:
self.fetch_and_store()
except (requests.exceptions.HTTPError, ReviewBoardAPIError), e:
printerr("Error: %s" % e)
sys.exit(1)
def run(self):
printout("Running Bicho with delay of %s seconds - %s" % (self.delay, self.url))
try:
self.fetch_and_store_tickets()
except (requests.exceptions.HTTPError, TracRPCError), e:
printerr("Error: %s" % e)
sys.exit(1)
def run(self):
printout("Running Bicho - %s" % self.url)
try:
self.fetch_and_store_tickets()
except (requests.exceptions.HTTPError, TracRPCError), e:
printerr("Error: %s" % e)
sys.exit(1)
def run(self):
printout("Running Bicho - url: %s" % self.url)
try:
self.fetch_and_store()
except (requests.exceptions.HTTPError, ReviewBoardAPIError), e:
printerr("Error: %s" % e)
sys.exit(1)
def run(self):
"""
"""
printout("Running Bicho with delay of %s seconds" % (str(self.delay)))
issues_per_query = 250
start_issue = 1
bugs = []
bugsdb = get_database(DBGoogleCodeBackend())
# still useless
bugsdb.insert_supported_traker("googlecode", "beta")
trk = Tracker(Config.url, "googlecode", "beta")
dbtrk = bugsdb.insert_tracker(trk)
self.url = Config.url
# https://code.google.com/feeds/issues/p/mobile-time-care
self.url_issues = Config.url + "/issues/full?max-results=1"
printdbg("URL for getting metadata " + self.url_issues)
d = feedparser.parse(self.url_issues)
total_issues = int(d['feed']['opensearch_totalresults'])
print "Total bugs: ", total_issues
if total_issues == 0:
printout("No bugs found. Did you provide the correct url?")
sys.exit(0)
remaining = total_issues
print "ETA ", (total_issues * Config.delay) / (60), "m (", (total_issues * Config.delay) / (60 * 60), "h)"
while start_issue < total_issues:
self.url_issues = Config.url + "/issues/full?max-results=" + str(issues_per_query)
self.url_issues += "&start-index=" + str(start_issue)
printdbg("URL for next issues " + self.url_issues)
d = feedparser.parse(self.url_issues)
for entry in d['entries']:
try:
issue = self.analyze_bug(entry)
if issue is None:
continue
bugsdb.insert_issue(issue, dbtrk.id)
remaining -= 1
print "Remaining time: ", (remaining) * Config.delay / 60, "m", " issues ", str(remaining)
time.sleep(Config.delay)
except Exception, e:
printerr("Error in function analyze_bug ")
pprint.pprint(entry)
traceback.print_exc(file=sys.stdout)
except UnicodeEncodeError:
printerr("UnicodeEncodeError: the issue %s couldn't be stored"
% (issue.issue))
def parse_changes(self):
soup = BeautifulSoup(self.html)
self.remove_comments(soup)
remove_tags = ['a', 'span', 'i']
changes = []
tables = soup.findAll('table')
# We look for the first table with 5 cols
table = None
for table in tables:
if len(table.tr.findAll('th', recursive=False)) == 5:
try:
for i in table.findAll(remove_tags):
i.replaceWith(i.text)
except:
printerr("error removing HTML tags")
break
if table is None:
return changes
rows = list(table.findAll('tr'))
for row in rows[1:]:
cols = list(row.findAll('td'))
if len(cols) == 5:
person_email = cols[0].contents[0].strip()
person_email = unicode(person_email.replace('@', '@'))
date = self._to_datetime_with_secs(cols[1].contents[0].strip())
# when the field contains an Attachment, the list has more
#than a field. For example:
#
# [u'\n', u'Attachment #12723', u'\n Flag\n ']
#
if len(cols[2].contents) > 1:
aux_c = unicode(" ".join(cols[2].contents))
field = unicode(aux_c.replace("\n", "").strip())
else:
field = unicode(cols[2].contents[0].replace("\n",
"").strip())
removed = unicode(cols[3].contents[0].strip())
added = unicode(cols[4].contents[0].strip())
else:
# same as above with the Attachment example
if len(cols[0].contents) > 1:
aux_c = unicode(" ".join(cols[0].contents))
field = aux_c.replace("\n", "").strip()
else:
field = cols[0].contents[0].strip()
removed = cols[1].contents[0].strip()
added = cols[2].contents[0].strip()
field, removed, added = self.sanityze_change(field, removed, added)
by = People(person_email)
by.set_email(person_email)
change = Change(field, removed, added, by, date)
changes.append(change)
return changes
def run(self):
printout("Running Bicho with delay of %s seconds - %s" %
(self.delay, self.url))
try:
self.fetch_and_store_tickets()
except (requests.exceptions.HTTPError, TracRPCError), e:
printerr("Error: %s" % e)
sys.exit(1)
def __parse_issue_summary(self, soup):
"""
"""
try:
m = ISSUE_SUMMARY_PATTERN.match(unicode(soup.title.string))
return m.group(1)
except:
printerr('Error parsing issue summary')
raise SourceForgeParserError('Error parsing issue summary')
def parse_changes(self):
soup = BeautifulSoup(self.html)
self.remove_comments(soup)
remove_tags = ['a', 'span', 'i']
changes = []
tables = soup.findAll('table')
# We look for the first table with 5 cols
table = None
for table in tables:
if len(table.tr.findAll('th', recursive=False)) == 5:
try:
for i in table.findAll(remove_tags):
i.replaceWith(i.text)
except:
printerr("error removing HTML tags")
break
if table is None:
return changes
rows = list(table.findAll('tr'))
for row in rows[1:]:
cols = list(row.findAll('td'))
if len(cols) == 5:
person_email = cols[0].contents[0].strip()
person_email = unicode(person_email.replace('@', '@'))
date = self._to_datetime_with_secs(cols[1].contents[0].strip())
# when the field contains an Attachment, the list has more
#than a field. For example:
#
# [u'\n', u'Attachment #12723', u'\n Flag\n ']
#
if len(cols[2].contents) > 1:
aux_c = unicode(" ".join(cols[2].contents))
field = unicode(aux_c.replace("\n", "").strip())
else:
field = unicode(cols[2].contents[0].replace("\n", "").strip())
removed = unicode(cols[3].contents[0].strip())
added = unicode(cols[4].contents[0].strip())
else:
# same as above with the Attachment example
if len(cols[0].contents) > 1:
aux_c = unicode(" ".join(cols[0].contents))
field = aux_c.replace("\n", "").strip()
else:
field = cols[0].contents[0].strip()
removed = cols[1].contents[0].strip()
added = cols[2].contents[0].strip()
field, removed, added = self.sanityze_change(field, removed, added)
by = People(person_email)
by.set_email(person_email)
change = Change(field, removed, added, by, date)
changes.append(change)
return changes
def __parse_issue_summary(self, soup):
"""
"""
try:
m = ISSUE_SUMMARY_PATTERN.match(unicode(soup.title.string))
return m.group(1)
except:
printerr('Error parsing issue summary')
raise SourceForgeParserError('Error parsing issue summary')
def __parse_issue_assigned_to(self, soup):
"""
"""
try:
assigned = soup.find({'label': True},
text=ISSUE_ASSIGNED_TO_PATTERN).findNext('p')
return assigned.contents[0]
except:
printerr('Error parsing issue assigned to')
raise SourceForgeParserError('Error parsing issue assigned to')
def __parse_issue_resolution(self, soup):
"""
"""
try:
resolution = soup.find({'label': True},
text=ISSUE_RESOLUTION_PATTERN).findNext('p')
return resolution.contents[0]
except:
printerr('Error parsing issue resolution')
raise SourceForgeParserError('Error parsing issue resolution')
def __parse_issue_group(self, soup):
"""
"""
try:
group = soup.find({'label': True},
text=ISSUE_GROUP_PATTERN).findNext('p')
return group.contents[0]
except:
printerr('Error parsing issue group')
raise SourceForgeParserError('Error parsing issue group')
def __parse_issue_status(self, soup):
"""
"""
try:
status = soup.find({'label': True},
text=ISSUE_STATUS_PATTERN).findNext('p')
return status.contents[0]
except:
printerr('Error parsing issue status')
raise SourceForgeParserError('Error parsing issue status')
def __parse_issue_visibility(self, soup):
"""
"""
try:
visibility = soup.find({'label': True},
text=ISSUE_VISIBILITY_PATTERN).findNext('p')
return visibility.contents[0]
except:
printerr('Error parsing issue visibility')
raise SourceForgeParserError('Error parsing issue visibility')
def __parse_issue_priority(self, soup):
"""
"""
try:
priority = soup.find({'label': True},
text=ISSUE_PRIORITY_PATTERN).findNext('p')
return priority.contents[0]
except:
printerr('Error parsing issue priority')
raise SourceForgeParserError('Error parsing issue priority')
def __parse_issue_category(self, soup):
"""
"""
try:
category = soup.find({'label': True},
text=ISSUE_CATEGORY_PATTERN).findNext('p')
return category.contents[0]
except:
printerr('Error parsing issue category')
raise SourceForgeParserError('Error parsing issue category')
def __parse_issue_status(self, soup):
"""
"""
try:
status = soup.find({
'label': True
}, text=ISSUE_STATUS_PATTERN).findNext('p')
return status.contents[0]
except:
printerr('Error parsing issue status')
raise SourceForgeParserError('Error parsing issue status')
def run(self):
printout("Running Bicho - %s" % self.url)
if not self.check_auth():
sys.exit(1)
try:
self.fetch_and_store_tasks()
except (requests.exceptions.HTTPError, ConduitError), e:
printerr("Error: %s" % e)
sys.exit(1)
def check_auth(self):
# Check conduit credentials
try:
printdbg("Checking conduit credentials")
self.conduit.whoami()
printdbg("Credentials checked")
return True
except (requests.exceptions.HTTPError, ConduitError), e:
printerr("Error: %s" % e)
return False
def __parse_issue_group(self, soup):
"""
"""
try:
group = soup.find({
'label': True
}, text=ISSUE_GROUP_PATTERN).findNext('p')
return group.contents[0]
except:
printerr('Error parsing issue group')
raise SourceForgeParserError('Error parsing issue group')
def __parse_issue_category(self, soup):
"""
"""
try:
category = soup.find({
'label': True
}, text=ISSUE_CATEGORY_PATTERN).findNext('p')
return category.contents[0]
except:
printerr('Error parsing issue category')
raise SourceForgeParserError('Error parsing issue category')
def run(self):
printout("Running Bicho - %s" % self.url)
if not self.check_auth():
sys.exit(1)
try:
self.fetch_and_store_tasks()
except (requests.exceptions.HTTPError, ConduitError), e:
printerr("Error: %s" % e)
sys.exit(1)
def __parse_issue_priority(self, soup):
"""
"""
try:
priority = soup.find({
'label': True
}, text=ISSUE_PRIORITY_PATTERN).findNext('p')
return priority.contents[0]
except:
printerr('Error parsing issue priority')
raise SourceForgeParserError('Error parsing issue priority')
def __init__(self):
self.url = Config.url
self.delay = Config.delay
try:
self.backend_password = Config.backend_password
self.backend_user = Config.backend_user
except AttributeError:
printerr("\n--backend-user and --backend-password are mandatory \
to download bugs from Github\n")
sys.exit(1)
self.remaining_ratelimit = 0
def __init__(self):
self.url = Config.url
self.delay = Config.delay
try:
self.backend_password = Config.backend_password
self.backend_user = Config.backend_user
except AttributeError:
printerr("\n--backend-user and --backend-password are mandatory \
to download bugs from Github\n")
sys.exit(1)
self.remaining_ratelimit = 0
def check_auth(self):
# Check conduit credentials
try:
printdbg("Checking conduit credentials")
self.conduit.whoami()
printdbg("Credentials checked")
return True
except (requests.exceptions.HTTPError, ConduitError), e:
printerr("Error: %s" % e)
return False
def run(self):
cfg = Config()
cfg.load_from_file("/home/user/Grimoire/Bicho/bicho/bicho.conf")
# url = 'http://10.137.2.15:8000/test/'
# url = 'http://dev.aubio.org/'
url = 'http://trac.nginx.org/nginx/'
# url = 'http://software.rtcm-ntrip.org/'
# project = "http://trac.nginx.org/nginx/"
# issues = TracIssue
tibi = TracBackend()
issues = tibi.getIDs(url)
bugsdb = get_database(DBTracBackend())
bugsdb.insert_supported_traker("trac", "1.0.6post2")
trk = Tracker(url, "trac", "1.0.6post2")
dbtrk = bugsdb.insert_tracker(trk)
self.start_from = 0 if self.start_from is None else self.start_from
self.end_with = len(issues) if self.end_with is None else self.end_with
for i in range(len(issues)):
if i < self.start_from:
continue
elif i > self.end_with:
break
printdbg("We are trying issue: {}".format(issues[i]))
try:
printdbg("Getting the entry")
raw_data = tibi.getIssue(url, issues[i])
printdbg("Parsing the entry")
issue = tibi.analyzeBug(raw_data)
printdbg("Inserting the issue into the DB")
# Put an issue into the database.
# pprint(vars(issue))
bugsdb.insert_issue(issue, dbtrk.id)
except UnicodeEncodeError, e:
printerr(
"UnicodeEncodeError: the issue %s couldn't be stored"
% (issues[i]))
print e
except Exception, e:
printerr("Error :")
# print e
import traceback
traceback.print_exc()
sys.exit(0)
def __parse_issue_resolution(self, soup):
"""
"""
try:
resolution = soup.find({
'label': True
},
text=ISSUE_RESOLUTION_PATTERN).findNext('p')
return resolution.contents[0]
except:
printerr('Error parsing issue resolution')
raise SourceForgeParserError('Error parsing issue resolution')
def __parse_issue_assigned_to(self, soup):
"""
"""
try:
assigned = soup.find({
'label': True
},
text=ISSUE_ASSIGNED_TO_PATTERN).findNext('p')
return assigned.contents[0]
except:
printerr('Error parsing issue assigned to')
raise SourceForgeParserError('Error parsing issue assigned to')
def __parse_issue_visibility(self, soup):
"""
"""
try:
visibility = soup.find({
'label': True
},
text=ISSUE_VISIBILITY_PATTERN).findNext('p')
return visibility.contents[0]
except:
printerr('Error parsing issue visibility')
raise SourceForgeParserError('Error parsing issue visibility')
def _get_person(self, lpperson):
"""
Returns Bicho People object from Launchpad person object
"""
try:
p = People(lpperson.name)
p.set_name(lpperson.display_name)
if lpperson.confirmed_email_addresses:
for m in lpperson.confirmed_email_addresses:
p.set_email(m.email)
break
except Exception, e:
printerr(str(e))
p = People("unknown")
def analyze_project_bugs(self, bugs, dbtrk, bugsdb):
analyzed = []
nbugs = 0
for bug in bugs:
if bug.web_link in analyzed:
continue # for the bizarre error #338
try:
issue_data = self.analyze_bug(bug)
except Exception, e:
printerr(
"Error in function analyzeBug with URL: ' \
'%s and Bug: %s"
% (str(dbtrk.url), bug)
)
raise e
try:
bugsdb.insert_issue(issue_data, dbtrk.id)
nbugs += 1
except UnicodeEncodeError:
printerr("UnicodeEncodeError: the issue %s couldn't be stored" % (issue_data.issue))
except NotFoundError:
printerr("NotFoundError: the issue %s couldn't be stored" % (issue_data.issue))
except Exception, e:
printerr("Unexpected Error: the issue %s couldn't be stored" % (issue_data.issue))
print e
def __parse_issue_description(self, soup):
"""
"""
try:
# Details is a list of unicode strings, so the
# strings are joined into a string to build the
# description field.
details = soup.find({'label': True},
text=ISSUE_DETAILS_PATTERN).findNext('p')
desc = u''.join(details.contents)
return desc
except:
printerr('Error parsing issue description')
raise SourceForgeParserError('Error parsing issue description')
def _get_person(self, lpperson):
"""
Returns Bicho People object from Launchpad person object
"""
try:
p = People(lpperson.name)
p.set_name(lpperson.display_name)
if lpperson.confirmed_email_addresses:
for m in lpperson.confirmed_email_addresses:
p.set_email(m.email)
break
except Exception, e:
printerr(str(e))
p = People("unknown")
def analyze_project_bugs(self, bugs, dbtrk, bugsdb):
analyzed = []
nbugs = 0
for bug in bugs:
if bug.web_link in analyzed:
continue # for the bizarre error #338
try:
issue_data = self.analyze_bug(bug)
except Exception, e:
printerr("Error in function analyzeBug with URL: ' \
'%s and Bug: %s" % (str(dbtrk.url), bug))
raise e
try:
bugsdb.insert_issue(issue_data, dbtrk.id)
nbugs += 1
except UnicodeEncodeError:
printerr("UnicodeEncodeError: the issue %s couldn't be stored"
% (issue_data.issue))
except NotFoundError:
printerr("NotFoundError: the issue %s couldn't be stored"
% (issue_data.issue))
except Exception, e:
printerr("Unexpected Error: the issue %s couldn't be stored"
% (issue_data.issue))
print e
def __parse_issue_description(self, soup):
"""
"""
try:
# Details is a list of unicode strings, so the
# strings are joined into a string to build the
# description field.
details = soup.find({
'label': True
}, text=ISSUE_DETAILS_PATTERN).findNext('p')
desc = u''.join(details.contents)
return desc
except:
printerr('Error parsing issue description')
raise SourceForgeParserError('Error parsing issue description')
def urlopen_auth(self, url):
"""
Opens an URL using an authenticated session
"""
request = urllib2.Request(url)
opener = urllib2.build_opener()
if self.is_auth_session():
q = [str(c) + '=' + v for c, v in self.cookies.items()]
opener.addheaders.append(('Cookie', '; '.join(q)))
try:
return opener.open(request)
except (urllib2.HTTPError, urllib2.URLError) as e:
printerr("Error code: %s, reason: %s" % (e.code, e.reason))
raise e
def urlopen_auth(self, url):
"""
Opens an URL using an authenticated session
"""
request = urllib2.Request(url)
opener = urllib2.build_opener()
if self.is_auth_session():
q = [str(c) + '=' + v for c, v in self.cookies.items()]
opener.addheaders.append(('Cookie', '; '.join(q)))
try:
return opener.open(request)
except (urllib2.HTTPError, urllib2.URLError) as e:
printerr("Error code: %s, reason: %s" % (e.code, e.reason))
raise e
def __init__(self):
self.url = Config.url
self.delay = Config.delay
self.backend_token = None
self.backend_user = None
self.backend_password = None
if hasattr(Config, 'backend_token'):
self.backend_token = Config.backend_token
elif hasattr(Config, 'backend_user') and hasattr(Config, 'backend_password'):
self.backend_user = Config.backend_user
self.backend_password = Config.backend_password
else:
msg = "\n--backend-user and --backend-password or --backend-token" + \
" are mandatory to download bugs from Github\n"
printerr(msg)
sys.exit(1)
self.newest_first = Config.newest_first
self.remaining_ratelimit = 0
def analyze_project_bugs(self, bugs, dbtrk, bugsdb):
analyzed = []
nbugs = 0
for bug in bugs:
if bug.web_link in analyzed:
continue # for the bizarre error #338
issue_data = self.analyze_bug(bug)
try:
bugsdb.insert_issue(issue_data, dbtrk.id)
nbugs += 1
except UnicodeEncodeError:
printerr("UnicodeEncodeError: the issue %s couldn't be stored"
% (issue_data.issue))
except NotFoundError:
printerr("NotFoundError: the issue %s couldn't be stored"
% (issue_data.issue))
except Exception, e:
printerr("Unexpected Error: the issue %s couldn't be stored"
% (issue_data.issue))
print e
analyzed.append(bug.web_link) # for the bizarre error #338
def __init__(self):
self.url = Config.url
self.delay = Config.delay
self.backend_token = None
self.backend_user = None
self.backend_password = None
self.users = {}
if hasattr(Config, 'backend_token'):
self.backend_token = Config.backend_token
elif hasattr(Config, 'backend_user') and hasattr(
Config, 'backend_password'):
self.backend_user = Config.backend_user
self.backend_password = Config.backend_password
else:
msg = "\n--backend-user and --backend-password or --backend-token" + \
" are mandatory to download bugs from Github\n"
printerr(msg)
sys.exit(1)
self.newest_first = Config.newest_first
self.remaining_ratelimit = 0
def run(self):
"""
"""
printout("Running Bicho with delay of %s seconds" % (str(self.delay)))
issues_per_query = 250
start_issue = 1
bugs = []
bugsdb = get_database(DBGoogleCodeBackend())
# still useless
bugsdb.insert_supported_traker("googlecode", "beta")
trk = Tracker(Config.url, "googlecode", "beta")
dbtrk = bugsdb.insert_tracker(trk)
self.url = Config.url
# https://code.google.com/feeds/issues/p/mobile-time-care
self.url_issues = Config.url + "/issues/full?max-results=1"
printdbg("URL for getting metadata " + self.url_issues)
d = feedparser.parse(self.url_issues)
total_issues = int(d['feed']['opensearch_totalresults'])
print "Total bugs: ", total_issues
if total_issues == 0:
printout("No bugs found. Did you provide the correct url?")
sys.exit(0)
remaining = total_issues
print "ETA ", (total_issues * Config.delay) / (60), "m (", (
total_issues * Config.delay) / (60 * 60), "h)"
while start_issue < total_issues:
self.url_issues = Config.url + "/issues/full?max-results=" + str(
issues_per_query)
self.url_issues += "&start-index=" + str(start_issue)
printdbg("URL for next issues " + self.url_issues)
d = feedparser.parse(self.url_issues)
for entry in d['entries']:
try:
issue = self.analyze_bug(entry)
if issue is None:
continue
bugsdb.insert_issue(issue, dbtrk.id)
remaining -= 1
print "Remaining time: ", (
remaining) * Config.delay / 60, "m", " issues ", str(
remaining)
time.sleep(Config.delay)
except Exception, e:
printerr("Error in function analyze_bug ")
pprint.pprint(entry)
traceback.print_exc(file=sys.stdout)
except UnicodeEncodeError:
printerr(
"UnicodeEncodeError: the issue %s couldn't be stored" %
(issue.issue))
def run(self):
"""
"""
printout("Running Bicho with delay of %s seconds" % (str(self.delay)))
# limit=-1 is NOT recognized as 'all'. 500 is a reasonable limit. - allura code
issues_per_query = 500
start_page = 0
bugs = []
bugsdb = get_database(DBAlluraBackend())
# still useless in allura
bugsdb.insert_supported_traker("allura", "beta")
trk = Tracker(Config.url, "allura", "beta")
dbtrk = bugsdb.insert_tracker(trk)
last_mod_date = bugsdb.get_last_modification_date()
# Date before the first ticket
time_window_start = "1900-01-01T00:00:00Z"
time_window_end = datetime.now().isoformat() + "Z"
if last_mod_date:
time_window_start = last_mod_date
printdbg("Last bugs analyzed were modified on: %s" % last_mod_date)
time_window = time_window_start + " TO " + time_window_end
self.url_issues = Config.url + "/search/?limit=1"
self.url_issues += "&q="
# A time range with all the tickets
self.url_issues += urllib.quote("mod_date_dt:[" + time_window + "]")
printdbg("URL for getting metadata " + self.url_issues)
f = urllib.urlopen(self.url_issues)
ticketTotal = json.loads(f.read())
total_issues = int(ticketTotal['count'])
total_pages = total_issues / issues_per_query
print("Number of tickets: " + str(total_issues))
if total_issues == 0:
printout("No bugs found. Did you provide the correct url?")
sys.exit(0)
remaining = total_issues
print "ETA ", (total_issues * Config.delay) / (60), "m (", (
total_issues * Config.delay) / (60 * 60), "h)"
while start_page <= total_pages:
self.url_issues = Config.url + "/search/?limit=" + str(
issues_per_query)
self.url_issues += "&page=" + str(start_page) + "&q="
# A time range with all the tickets
self.url_issues += urllib.quote("mod_date_dt:[" + time_window +
"]")
# Order by mod_date_dt desc
self.url_issues += "&sort=mod_date_dt+asc"
printdbg("URL for next issues " + self.url_issues)
f = urllib.urlopen(self.url_issues)
ticketList = json.loads(f.read())
bugs = []
for ticket in ticketList["tickets"]:
bugs.append(ticket["ticket_num"])
for bug in bugs:
try:
issue_url = Config.url + "/" + str(bug)
issue_data = self.analyze_bug(issue_url)
if issue_data is None:
continue
bugsdb.insert_issue(issue_data, dbtrk.id)
remaining -= 1
print "Remaining time: ", (
remaining) * Config.delay / 60, "m"
time.sleep(self.delay)
except Exception, e:
printerr("Error in function analyze_bug " + issue_url)
traceback.print_exc(file=sys.stdout)
except UnicodeEncodeError:
printerr(
"UnicodeEncodeError: the issue %s couldn't be stored" %
(issue_data.issue))
def run(self):
print("Running Bicho with delay of %s seconds" % (str(self.delay)))
bugsdb = get_database(DBGithubBackend())
url = self.url
pname = None
pname = self.__get_project_from_url()
printdbg(url)
bugsdb.insert_supported_traker("github", "v3")
trk = Tracker(url, "github", "v3")
dbtrk = bugsdb.insert_tracker(trk)
self.bugs_state = ALL_STATES
self.pagecont = 1
self.mod_date = None
aux_date = bugsdb.get_last_modification_date(tracker_id=dbtrk.id)
if aux_date:
self.mod_date = aux_date.isoformat()
printdbg("Last issue already cached: %s" % self.mod_date)
try:
bugs = self.__get_batch_bugs()
except GitHubRateLimitReached:
printout(
"GitHub rate limit reached. To resume, wait some minutes.")
sys.exit(0)
nbugs = len(bugs)
if len(bugs) == 0:
if aux_date:
printout("Bicho database up to date")
else:
printout("No bugs found. Did you provide the correct url?")
sys.exit(0)
auxcont = 0
while len(bugs) > 0:
for bug in bugs:
try:
issue_data = self.analyze_bug(bug)
except GitHubRateLimitReached:
printout(
"GitHub rate limit reached. To resume, wait some minutes."
)
sys.exit(0)
except Exception:
#FIXME it does not handle the e
msg = "Error in function analyzeBug with URL: %s and bug: %s" % (
url, bug)
printerr(msg)
raise
try:
# we can have meta-trackers but we want to have the
# original tracker name
tr_url = self.__get_tracker_url_from_bug(bug)
if (tr_url != url):
aux_trk = Tracker(tr_url, "github", "v3")
dbtrk = bugsdb.insert_tracker(aux_trk)
bugsdb.insert_issue(issue_data, dbtrk.id)
except UnicodeEncodeError:
printerr(
"UnicodeEncodeError: the issue %s couldn't be stored" %
(issue_data.issue))
except Exception, e:
printerr("ERROR: ")
print e
printdbg("Getting ticket number " + str(bug["number"]))
time.sleep(self.delay)
self.pagecont += 1
try:
bugs = self.__get_batch_bugs()
except GitHubRateLimitReached:
printout(
"GitHub rate limit reached. To resume, wait some minutes.")
sys.exit(0)
nbugs = nbugs + len(bugs)
def run(self):
print("Running Bicho with delay of %s seconds" % (str(self.delay)))
url = self.url
pname = None
pname = self.__get_project_from_url()
bugsdb = get_database(DBLaunchpadBackend())
printdbg(url)
# launchpad needs a temp directory to store cached data
homedir = pwd.getpwuid(os.getuid()).pw_dir
cachedir = os.path.join(homedir, ".cache/bicho/")
if not os.path.exists(cachedir):
os.makedirs(cachedir)
cre_file = os.path.join(cachedir + 'launchpad-credential')
self.lp = Launchpad.login_with('Bicho',
'production',
credentials_file=cre_file)
aux_status = [
"New", "Incomplete", "Opinion", "Invalid", "Won't Fix", "Expired",
"Confirmed", "Triaged", "In Progress", "Fix Committed",
"Fix Released", "Incomplete (with response)",
"Incomplete (without response)"
]
# still useless
bugsdb.insert_supported_traker("launchpad", "x.x")
trk = Tracker(url, "launchpad", "x.x")
dbtrk = bugsdb.insert_tracker(trk)
last_mod_date = bugsdb.get_last_modification_date(tracker_id=dbtrk.id)
if last_mod_date:
bugs = self.lp.projects[pname].searchTasks(
status=aux_status,
omit_duplicates=False,
order_by='date_last_updated',
modified_since=last_mod_date)
else:
bugs = self.lp.projects[pname].searchTasks(
status=aux_status,
omit_duplicates=False,
order_by='date_last_updated')
printdbg("Last bug already cached: %s" % last_mod_date)
nbugs = len(bugs)
if nbugs == 0:
printout("No bugs found. Did you provide the correct url?")
sys.exit(0)
analyzed = []
for bug in bugs:
if bug.web_link in analyzed:
continue # for the bizarre error #338
try:
issue_data = self.analyze_bug(bug)
except Exception:
#FIXME it does not handle the e
printerr("Error in function analyzeBug with URL: ' \
'%s and Bug: %s" % (url, bug))
raise
try:
# we can have meta-trackers but we want to have the original
#tracker name
tr_url = self.__get_tracker_url_from_bug(bug)
if (tr_url != url):
aux_trk = Tracker(tr_url, "launchpad", "x.x")
dbtrk = bugsdb.insert_tracker(aux_trk)
bugsdb.insert_issue(issue_data, dbtrk.id)
except UnicodeEncodeError:
printerr(
"UnicodeEncodeError: the issue %s couldn't be stored" %
(issue_data.issue))
except NotFoundError:
printerr("NotFoundError: the issue %s couldn't be stored" %
(issue_data.issue))
except Exception, e:
printerr("Unexpected Error: the issue %s couldn't be stored" %
(issue_data.issue))
print e
analyzed.append(bug.web_link) # for the bizarre error #338
time.sleep(self.delay)
