def obtainDataPerson(self, obj):
person = People(obj.name)
if hasattr(obj, 'displayName'):
person.set_name(obj.displayName)
if hasattr(obj, 'emailAddress'):
person.set_email(obj.emailAddress)
return person
def check_tasks_events(self):
""" Add event/invalid changes if don't exists and task in this status """
by = People('bicho_tool')
by.set_name('Bicho tool')
issues = self.bugsdb.store.find(DBIssue)
people_none = self.bugsdb.store.find(
DBPeople, DBPeople.user_id == unicode("None")).one()
for task in issues:
if task.status in ['merged', 'invalid']:
# Closed condition that needs and event for it
changes = self.bugsdb.store.find(DBChange,
DBChange.issue_id == task.id)
found_change = False
for change in changes:
if change.new_value == task.status:
found_change = True
break
if not found_change:
task_ext = self.bugsdb.store.find(
DBStoryBoardIssueExt,
DBStoryBoardIssueExt.issue_id == task.id).one()
field = "task_status_changed"
old_value = None
# logging.info("Adding to " + task.summary + " " + task.status + " event")
change = Change(field, old_value, task.status, by,
task_ext.mod_date)
self.bugsdb._insert_change(change, task.id, self.dbtrk.id)
def get_identity(self, username):
if not username:
return None
if username in self.identities:
return self.identities[username]
result = self.api_client.user(username)
raw_user = result['user']
identity = People(username)
if 'user' in result:
if 'fullname' in raw_user:
identity.name = raw_user['fullname']
elif 'first_name' in raw_user:
identity.name = '%s %s' % (raw_user['first_name'],
raw_user['last_name'])
if 'email' in raw_user:
identity.email = raw_user['email']
self.identities[username] = identity
return identity
def parse_changes(self):
soup = BeautifulSoup(self.html)
self.remove_comments(soup)
remove_tags = ['a', 'span', 'i']
changes = []
tables = soup.findAll('table')
# We look for the first table with 5 cols
table = None
for table in tables:
if len(table.tr.findAll('th', recursive=False)) == 5:
try:
for i in table.findAll(remove_tags):
i.replaceWith(i.text)
except:
printerr("error removing HTML tags")
break
if table is None:
return changes
rows = list(table.findAll('tr'))
for row in rows[1:]:
cols = list(row.findAll('td'))
if len(cols) == 5:
person_email = cols[0].contents[0].strip()
person_email = unicode(person_email.replace('@', '@'))
date = self._to_datetime_with_secs(cols[1].contents[0].strip())
# when the field contains an Attachment, the list has more
#than a field. For example:
#
# [u'\n', u'Attachment #12723', u'\n Flag\n ']
#
if len(cols[2].contents) > 1:
aux_c = unicode(" ".join(cols[2].contents))
field = unicode(aux_c.replace("\n", "").strip())
else:
field = unicode(cols[2].contents[0].replace("\n",
"").strip())
removed = unicode(cols[3].contents[0].strip())
added = unicode(cols[4].contents[0].strip())
else:
# same as above with the Attachment example
if len(cols[0].contents) > 1:
aux_c = unicode(" ".join(cols[0].contents))
field = aux_c.replace("\n", "").strip()
else:
field = cols[0].contents[0].strip()
removed = cols[1].contents[0].strip()
added = cols[2].contents[0].strip()
field, removed, added = self.sanityze_change(field, removed, added)
by = People(person_email)
by.set_email(person_email)
change = Change(field, removed, added, by, date)
changes.append(change)
return changes
def parse_changes(self):
soup = BeautifulSoup(self.html)
self.remove_comments(soup)
remove_tags = ['a', 'span', 'i']
changes = []
tables = soup.findAll('table')
# We look for the first table with 5 cols
table = None
for table in tables:
if len(table.tr.findAll('th', recursive=False)) == 5:
try:
for i in table.findAll(remove_tags):
i.replaceWith(i.text)
except:
printerr("error removing HTML tags")
break
if table is None:
return changes
rows = list(table.findAll('tr'))
for row in rows[1:]:
cols = list(row.findAll('td'))
if len(cols) == 5:
person_email = cols[0].contents[0].strip()
person_email = unicode(person_email.replace('@', '@'))
date = self._to_datetime_with_secs(cols[1].contents[0].strip())
# when the field contains an Attachment, the list has more
#than a field. For example:
#
# [u'\n', u'Attachment #12723', u'\n Flag\n ']
#
if len(cols[2].contents) > 1:
aux_c = unicode(" ".join(cols[2].contents))
field = unicode(aux_c.replace("\n", "").strip())
else:
field = unicode(cols[2].contents[0].replace("\n", "").strip())
removed = unicode(cols[3].contents[0].strip())
added = unicode(cols[4].contents[0].strip())
else:
# same as above with the Attachment example
if len(cols[0].contents) > 1:
aux_c = unicode(" ".join(cols[0].contents))
field = aux_c.replace("\n", "").strip()
else:
field = cols[0].contents[0].strip()
removed = cols[1].contents[0].strip()
added = cols[2].contents[0].strip()
field, removed, added = self.sanityze_change(field, removed, added)
by = People(person_email)
by.set_email(person_email)
change = Change(field, removed, added, by, date)
changes.append(change)
return changes
def __get_people_from_uri(self, uri):
# returns People object from uri (person_link)
try:
people_lp = self.lp.people[self._get_nickname_from_uri(uri)]
people_issue = People(people_lp.name)
people_issue.set_name(people_lp.display_name)
except Exception:
# user deleted from Launchpad!
people_issue = People(self._get_nickname_from_uri(uri))
return people_issue
def _get_person(self, lpperson):
"""
Returns Bicho People object from Launchpad person object
"""
p = People(lpperson.name)
p.set_name(lpperson.display_name)
if lpperson.confirmed_email_addresses:
for m in lpperson.confirmed_email_addresses:
p.set_email(m.email)
break
return p
def parse_changes(self, review):
changesList = []
patchSets = review['patchSets']
for activity in patchSets:
if "approvals" not in activity.keys():
continue
patchSetNumber = activity['number']
for entry in activity['approvals']:
# print "changed_by:" + entry['author']
if "username" in entry["by"].keys():
by = People(entry['by']['username'])
elif "email" in entry["by"].keys():
by = People(entry['by']['email'])
elif "name" in entry["by"].keys():
by = People(entry['by']['name'])
else:
by = People(unicode(''))
if "name" in entry["by"].keys():
by.set_name(entry["by"]["name"])
if "email" in entry["by"].keys():
by.set_email(entry["by"]["email"])
# print "changed_on:" + entry['updated']
field = entry['type']
new_value = entry['value']
old_value = patchSetNumber
update = self._convert_to_datetime(entry["grantedOn"])
change = Change(field, old_value, new_value, by, update)
changesList.append(change)
return changesList
def parse_bug(self, issue_allura):
people = People(issue_allura["reported_by_id"])
people.set_name(issue_allura["reported_by"])
issue = AlluraIssue(
issue_allura["_id"], "ticket", issue_allura["summary"],
issue_allura["description"], people,
self._convert_to_datetime(issue_allura["created_date"]))
people = People(issue_allura["assigned_to_id"])
people.set_name(issue_allura["assigned_to"])
issue.assigned_to = people
issue.status = issue_allura["status"]
# No information from Allura for this fields
issue.resolution = None
issue.priority = None
# Extended attributes
issue.labels = str(issue_allura["labels"])
issue.private = issue_allura["private"]
issue.ticket_num = issue_allura["ticket_num"]
issue.discussion_thread_url = issue_allura["discussion_thread_url"]
issue.related_artifacts = str(issue_allura["related_artifacts"])
issue.custom_fields = str(issue_allura["custom_fields"])
issue.mod_date = self._convert_to_datetime(issue_allura["mod_date"])
return issue
def parse_changes(self, activity):
changesList = []
for entry in activity['entries']:
# print "changed_by:" + entry['author']
by = People(entry['author'])
# print "changed_on:" + entry['updated']
description = entry['description'].split('updated:')
changes = description.pop(0)
field = changes.rpartition('\n')[2].strip()
while description:
changes = description.pop(0).split('\n')
values = changes[0].split('=>')
if (len(values) != 2):
printdbg(field + " not supported in changes analysis")
old_value = new_value = ""
else:
# u'in-progress' => u'closed'
values = changes[0].split('=>')
old_value = self.remove_unicode(values[0].strip())
if old_value == "''":
old_value = ""
new_value = self.remove_unicode(values[1].strip())
if new_value == "''":
new_value = ""
update = parse(entry['updated'])
change = Change(unicode(field), unicode(old_value),
unicode(new_value), by, update)
changesList.append(change)
if (len(changes) > 1):
field = changes[1].strip()
return changesList
def add_uploaded_patchset_from_comments(self, review, issue):
# comm.text like '%Patch Set%Verified%' "
UPLOAD_REGEXP_1 = re.compile(r'^Uploaded patch set (.*?).')
UPLOAD_REGEXP_2 = re.compile(r'^Patch Set (.*?):(.*)')
# Review all comments and create MERGE and ABANDONED changes
comments = review['comments']
ncomment = 0
patchset_added = []
for comment in comments:
ncomment += 1
by = People(comment['reviewer']["username"])
if (UPLOAD_REGEXP_2.match(comment["message"])):
patchset = comment["message"].split("Patch Set ")[1]
patchset = patchset.split(":")[0]
elif (UPLOAD_REGEXP_1.match(comment["message"])):
patchset = comment["message"].split(" ")[3]
patchset = patchset.split(".")[0]
else:
continue
# Sometimes we get more than one Verfieid for the same patch
if patchset in patchset_added: continue
patchset_added.append(patchset)
change = Change(unicode("status"), unicode(patchset),
unicode("UPLOADED"), by,
self._convert_to_datetime(comment["timestamp"]))
issue.add_change(change)
def add_merged_abandoned_changes_from_comments(self, review, issue):
# text like 'Patch Set %: Abandoned%' or text like 'Abandoned%'
ABANDONED_REGEXP_1 = re.compile(r'^Patch Set (.*?): Abandoned(.*)')
ABANDONED_REGEXP_2 = re.compile(r'^Abandoned(.*)')
RESTORED_REGEXP_1 = re.compile(r'^Patch Set (.*?): Restored(.*)')
RESTORED_REGEXP_2 = re.compile(r'^Restored(.*)')
abandoned = False
# Review all comments and create MERGE and ABANDONED changes
comments = review['comments']
ncomment = 0
for comment in comments:
ncomment += 1
by = People(comment['reviewer']["username"])
if (comment["message"] ==
"Change has been successfully merged into the git repository."
):
change = Change(
unicode("status"), unicode(""), unicode("MERGED"), by,
self._convert_to_datetime(comment["timestamp"]))
issue.add_change(change)
elif (ABANDONED_REGEXP_1.match(comment["message"])
or ABANDONED_REGEXP_2.match(comment["message"])):
abandoned = True
by_abandoned = by
date_abandoned = self._convert_to_datetime(
comment["timestamp"])
elif (RESTORED_REGEXP_1.match(comment["message"])
or RESTORED_REGEXP_2.match(comment["message"])):
abandoned = False
if (abandoned):
change = Change(unicode("status"), unicode(""),
unicode("ABANDONED"), by_abandoned, date_abandoned)
issue.add_change(change)
def parse_changes(self):
soup = BeautifulSoup(self.html)
self.remove_comments(soup)
remove_tags = ['i']
try:
[i.replaceWith(i.contents[0]) for i in soup.findAll(remove_tags)]
except Exception:
None
changes = []
#FIXME The id of the changes are not stored
tables = soup.findAll("div", {"class": "actionContainer"})
for table in tables:
author_date_text = table.find("div", {"class": "action-details"})
if author_date_text is None:
# no changes have been performed on the issue
continue
elif len(author_date_text) < 3:
self.changes_lost += 1
printerr("Change author format not supported. Change lost!")
continue
auth_link = table.find("a", {"class": "user-hover user-avatar"})
if not auth_link:
# Automated changes usually have this class
auth_link = table.find("a", {"class": "user-hover"})
# instead of <a .. we got a <span>
if not auth_link:
auth_link = table.find("span",
{"class": "user-hover user-avatar"})
author_url = auth_link.get('rel', 'anonymous')
author = People(author_url)
# we look for a string similar to:
#<time datetime="2011-11-19T00:27-0800">19/Nov/11 12:27 AM</time>
raw_date = author_date_text.find('time')['datetime']
date = parse(raw_date).replace(tzinfo=None)
rows = list(table.findAll('tr'))
for row in rows:
cols = list(row.findAll('td'))
if len(cols) == 3:
field = unicode(cols[0].contents[0].strip())
if field == "Assignee":
old = unicode(self._get_identifier(cols[1]))
new = unicode(self._get_identifier(cols[2]))
else:
old = unicode(cols[1].contents[0].strip())
new = unicode(cols[2].contents[0].strip())
change = Change(field, old, new, author, date)
changes.append(change)
return changes
def __get_user(self, username):
if username in self.users:
return self.users[username]
url = "https://api.github.com/users/" + username
raw_user = self.__fetch_data(url)
user = People(username)
if 'name' in raw_user:
user.name = raw_user['name']
if 'email' in raw_user:
user.email = raw_user['email']
self.users[username] = user
return user
def __get_user(self, username):
if username in self.users:
return self.users[username]
url = "https://api.github.com/users/" + username
raw_user = self.__fetch_data(url)
user = People(username)
if 'name' in raw_user:
user.name = raw_user['name']
if 'email' in raw_user:
user.email = raw_user['email']
self.users[username] = user
return user
def parse_comments(self, review):
if "comments" not in review.keys(): return []
commentsList = []
comments = review['comments']
for comment in comments:
if ("username" not in comment['reviewer'].keys()):
comment['reviewer']["username"] = comment['reviewer']["name"]
by = People(comment['reviewer']["username"])
if ("name" in comment['reviewer'].keys()):
by.set_name(comment['reviewer']["name"])
if ("email" in comment['reviewer'].keys()):
by.set_email(comment['reviewer']["email"])
com = Comment(comment["message"], by, self._convert_to_datetime(comment["timestamp"]))
commentsList.append(com)
return commentsList
def get_identities(self, phids):
ids = []
request = []
for phid in phids:
if phid in self.identities:
ids.append(self.identities[phid])
else:
request.append(phid)
# Request non-cached identities
result = self.conduit.users(request)
for r in result:
identity = People(r['userName'])
identity.name = r['realName']
ids.append(identity)
self.identities[r['phid']] = identity
return ids
def get_identity(self, ph_id):
if not ph_id:
return None
if ph_id in self.identities:
return self.identities[ph_id]
result = self.conduit.users([ph_id])
if result:
raw_data = result[0]
identity = People(raw_data['userName'])
identity.name = raw_data['realName']
else:
identity = People(ph_id)
self.identities[ph_id] = identity
return identity
def get_identity(self, username):
identity = self.identities.get(username, None)
if identity:
return identity
identity = People(username)
self.identities[username] = identity
return identity
def get_identities(self, phids):
ids = []
request = []
for phid in phids:
if phid in self.identities:
ids.append(self.identities[phid])
else:
request.append(phid)
# Request non-cached identities
result = self.conduit.users(request)
for r in result:
identity = People(r['userName'])
identity.name = r['realName']
ids.append(identity)
self.identities[r['phid']] = identity
return ids
def get_identity(self, ph_id):
if not ph_id:
return None
if ph_id in self.identities:
return self.identities[ph_id]
result = self.conduit.users([ph_id])
if result:
raw_data = result[0]
identity = People(raw_data['userName'])
identity.name = raw_data['realName']
else:
identity = People(ph_id)
self.identities[ph_id] = identity
return identity
def get_people(self, id):
people = People(id)
people.set_name(id)
for user in self.users:
if user['id'] == id:
people.set_name(user["full_name"])
people.set_email(user["email"])
break
return people
def parse_bug(self, issue_allura):
people = People(issue_allura["reported_by_id"])
people.set_name(issue_allura["reported_by"])
issue = AlluraIssue(issue_allura["_id"],
"ticket",
issue_allura["summary"],
issue_allura["description"],
people,
self._convert_to_datetime(issue_allura["created_date"]))
people = People(issue_allura["assigned_to_id"])
people.set_name(issue_allura["assigned_to"])
issue.assigned_to = people
issue.status = issue_allura["status"]
# No information from Allura for this fields
issue.resolution = None
issue.priority = None
# Extended attributes
issue.labels = str(issue_allura["labels"])
issue.private = issue_allura["private"]
issue.ticket_num = issue_allura["ticket_num"]
issue.discussion_thread_url = issue_allura["discussion_thread_url"]
issue.related_artifacts = str(issue_allura["related_artifacts"])
issue.custom_fields = str(issue_allura["custom_fields"])
issue.mod_date = self._convert_to_datetime(issue_allura["mod_date"])
return issue
def parse_change(self, change):
# print "changed_by:" + entry['author']
field = change['event_type']
by = People(change['author_id'])
by.set_email(self.get_email(change['author_id']))
by.set_name(self.get_fullname(change['author_id']))
old_value = new_value = None
if field == "task_created":
new_value = change['event_info']['task_title']
elif field == "task_priority_changed":
old_value = change['event_info']['old_priority']
new_value = change['event_info']['new_priority']
elif field == "task_status_changed":
old_value = change['event_info']['old_status']
new_value = change['event_info']['new_status']
elif field == "task_assignee_changed":
old_value = change['event_info']['old_assignee_id']
new_value = change['event_info']['new_assignee_id']
elif field == "task_details_changed":
# No data about the old and new details provided in API REST JSON
pass
elif field == "task_deleted":
pass
else:
logging.error(field + " not supported yet in changes.")
logging.info(change)
update = parse(change['created_at'])
change = Change(field, unicode(old_value), unicode(new_value), by,
update)
return change
def _get_person(self, lpperson):
"""
Returns Bicho People object from Launchpad person object
"""
try:
p = People(lpperson.name)
p.set_name(lpperson.display_name)
if lpperson.confirmed_email_addresses:
for m in lpperson.confirmed_email_addresses:
p.set_email(m.email)
break
except Exception, e:
printerr(str(e))
p = People("unknown")
def _parse_journals(self, issue, issue_id):
issue_url = self._get_issue_url(issue_id)
printdbg("Analyzing issue journals " + issue_url)
f = urllib2.urlopen(issue_url)
data = json.loads(f.read())
journals = data["issue"]["journals"]
for journal in journals:
try:
people = People(self._get_author_identity(journal["user"]["id"]))
people.set_name(journal["user"]["name"])
except KeyError:
people = People("None")
dt = self._convert_to_datetime(journal["created_on"])
# Comment
notes = journal.get("notes", None)
if notes:
msg = journal["notes"]
comment = Comment(msg, people, dt)
issue.add_comment(comment)
# Changes
for detail in journal["details"]:
field = detail["name"]
old_value = unicode(detail.get("old_value", unicode(None)))
new_value = unicode(detail.get("new_value", unicode(None)))
# Change status value
if field == u"status_id":
field = unicode("status")
old_value = self.statuses.get(old_value, unicode(None))
new_value = self.statuses.get(new_value, unicode(None))
change = Change(field, old_value, new_value, people, dt)
issue.add_change(change)
def parse_changes(self, activity, bug_id):
changesList = []
for entry in activity['entries']:
if not 'issues_status' in entry.keys():
continue
by = People(entry['author_detail']['href'])
update = parse(entry['updated'])
field = 'Status'
old_value = ''
new_value = entry['issues_status']
change = Change(unicode(field), unicode(old_value),
unicode(new_value), by, update)
changesList.append(change)
return changesList
def add_merged_abandoned_changes(self, review, issue):
if (issue.status <> 'MERGED' and issue.status <> 'ABANDONED'): return
ABANDONED_REGEXP_1 = re.compile(r'^Patch Set (.*?): Abandoned(.*)')
ABANDONED_REGEXP_2 = re.compile(r'^Abandoned(.*)')
patchSets = review['patchSets']
comments = review['comments']
by = None
date = None
patchNumber = unicode('')
# MERGED event searched from approvals
if (issue.status == 'MERGED'):
for activity in patchSets:
if "approvals" not in activity.keys():
continue
patchSetNumber = activity['number']
for entry in activity['approvals']:
if (entry['type'] == 'Code-Review'
and entry['value'] == '2'):
by = People(entry['by']['username'])
date = self._convert_to_datetime(entry["grantedOn"])
patchNumber = patchSetNumber
# TODO: if not by and date, take if from modified_on in issues_ext_gerrit
# ABANDONED event searched from comments
if (issue.status == 'ABANDONED'):
for comment in comments:
if (ABANDONED_REGEXP_1.match(comment["message"])
or ABANDONED_REGEXP_2.match(comment["message"])):
by = People(comment['reviewer']["username"])
date = self._convert_to_datetime(comment["timestamp"])
if (by and date):
change = Change(unicode("status"), patchNumber, issue.status, by,
date)
issue.add_change(change)
def get_identity(self, username):
if not username:
return None
if username in self.identities:
return self.identities[username]
result = self.api_client.user(username)
raw_user = result['user']
identity = People(username)
if 'user' in result:
if 'fullname' in raw_user:
identity.name = raw_user['fullname']
elif 'first_name' in raw_user:
identity.name = '%s %s' % (raw_user['first_name'], raw_user['last_name'])
if 'email' in raw_user:
identity.email = raw_user['email']
self.identities[username] = identity
return identity
def __get_people_from_uri(self, uri):
# returns People object from uri (person_link)
try:
people_lp = self.lp.people[self._get_nickname_from_uri(uri)]
people_issue = People(people_lp.name)
people_issue.set_name(people_lp.display_name)
except KeyError:
# user deleted from Launchpad!
people_issue = People(self._get_nickname_from_uri(uri))
return people_issue
def parse_review(self, review):
if "username" in review["owner"].keys():
people = People(review['owner']['username'])
elif "email" in review["owner"].keys():
people = People(review['owner']['email'])
elif "name" in review["owner"].keys():
people = People(review['owner']['name'])
else:
people = People(unicode(''))
if "name" in review["owner"].keys():
people.set_name(review["owner"]["name"])
if "email" in review["owner"].keys():
people.set_email(review["owner"]["email"])
description = ""
issue = GerritIssue(review["number"],
"review",
review["subject"],
description,
people,
self._convert_to_datetime(review["createdOn"])
)
# people = People(review["assigned_to_id"])
# people.set_name(review["assigned_to"])
# issue.assigned_to = people
issue.status = review["status"]
# No information from Gerrit for this fields
issue.assigned_to = None
issue.resolution = None
issue.priority = None
issue.branch = review["branch"]
issue.url = review["url"]
issue.change_id = review["id"]
if "topic" in review.keys():
issue.related_artifacts = review["topic"]
else:
issue.related_artifacts = None
issue.project = review["project"]
issue.mod_date = self._convert_to_datetime(review["lastUpdated"])
issue.open = review["open"]
return issue
def _get_person(self, lpperson):
"""
Returns Bicho People object from Launchpad person object
"""
try:
p = People(lpperson.name)
p.set_name(lpperson.display_name)
if lpperson.confirmed_email_addresses:
for m in lpperson.confirmed_email_addresses:
p.set_email(m.email)
break
except Exception, e:
printerr(str(e))
p = People("unknown")
def analyze_bug(self, entry):
people = People(entry['author_detail']['href'])
people.set_name(entry['author_detail']['name'])
issue = GoogleCodeIssue(entry['id'], 'issue', entry['title'],
entry['content'], people,
self._convert_to_datetime(entry['published']))
# Strange how the parser rename this fields
if 'issues_uri' in entry.keys():
people = People(entry['issues_uri'])
people.set_name(entry['issues_username'])
issue.assigned_to = people
issue.status = entry['issues_status']
issue.resolution = entry['issues_state']
issue.priority = entry['issues_label']
# Extended attributes
# issue.labels = str(issue_googlecode["labels"])
issue.star = entry['issues_stars']
issue.ticket_num = entry['issues_id']
issue.mod_date = self._convert_to_datetime(entry['updated'])
issue.closed_date = None
if 'issues_closeddate' in entry.keys():
issue.closed_date = self._convert_to_datetime(
entry['issues_closeddate'])
changes_url = Config.url + "/issues/" + issue.ticket_num + "/comments/full"
printdbg("Analyzing issue " + changes_url)
d = feedparser.parse(changes_url)
changes = self.parse_changes(d, issue.ticket_num)
for c in changes:
issue.add_change(c)
return issue
def parse_comments(self, review):
if "comments" not in review.keys(): return []
commentsList = []
comments = review['comments']
for comment in comments:
if ("username" not in comment['reviewer'].keys()):
if ("name" in comment['reviewer'].keys()):
comment['reviewer']["username"] = comment['reviewer']["name"]
else:
comment['reviewer']["username"] = None
by = People(comment['reviewer']["username"])
if ("name" in comment['reviewer'].keys()):
by.set_name(comment['reviewer']["name"])
if ("email" in comment['reviewer'].keys()):
by.set_email(comment['reviewer']["email"])
com = Comment(comment["message"], by, self._convert_to_datetime(comment["timestamp"]))
commentsList.append(com)
return commentsList
def _parse_journals(self, issue, issue_id):
issue_url = self._get_issue_url(issue_id)
printdbg("Analyzing issue journals " + issue_url)
f = urllib2.urlopen(issue_url)
data = json.loads(f.read())
journals = data["issue"]["journals"]
for journal in journals:
try:
people = People(
self._get_author_identity(journal["user"]["id"]))
people.set_name(journal["user"]["name"])
except KeyError:
people = People("None")
dt = self._convert_to_datetime(journal["created_on"])
# Comment
notes = journal.get("notes", None)
if notes:
msg = journal["notes"]
comment = Comment(msg, people, dt)
issue.add_comment(comment)
# Changes
for detail in journal["details"]:
field = detail["name"]
old_value = unicode(detail.get("old_value", unicode(None)))
new_value = unicode(detail.get("new_value", unicode(None)))
# Change status value
if field == u"status_id":
field = unicode("status")
old_value = self.statuses.get(old_value, unicode(None))
new_value = self.statuses.get(new_value, unicode(None))
change = Change(field, old_value, new_value, people, dt)
issue.add_change(change)
def analyze_bug(self, entry):
people = People(entry['author_detail']['href'])
people.set_name(entry['author_detail']['name'])
issue = GoogleCodeIssue(entry['id'],
'issue',
entry['title'],
entry['content'],
people,
self._convert_to_datetime(entry['published']))
# Strange how the parser rename this fields
if 'issues_uri' in entry.keys():
people = People(entry['issues_uri'])
people.set_name(entry['issues_username'])
issue.assigned_to = people
issue.status = entry['issues_status']
issue.resolution = entry['issues_state']
issue.priority = entry['issues_label']
# Extended attributes
# issue.labels = str(issue_googlecode["labels"])
issue.star = entry['issues_stars']
issue.ticket_num = entry['issues_id']
issue.mod_date = self._convert_to_datetime(entry['updated'])
issue.closed_date = None
if 'issues_closeddate' in entry.keys():
issue.closed_date = self._convert_to_datetime(entry['issues_closeddate'])
changes_url = Config.url + "/issues/" + issue.ticket_num + "/comments/full"
printdbg("Analyzing issue " + changes_url)
d = feedparser.parse(changes_url)
changes = self.parse_changes(d, issue.ticket_num)
for c in changes:
issue.add_change(c)
return issue
def get_issue(self):
issue_id = self.atags["bug_id"]
type = self.atags["bug_severity"]
summary = self.atags["short_desc"]
if len(self.ctags["long_desc"]) > 0:
desc = self.ctags["long_desc"][0]["thetext"]
else:
desc = ""
submitted_by = People(self.atags["reporter"])
submitted_by.set_name(self.atags["reporter_name"])
submitted_by.set_email(self.atags["reporter"])
submitted_on = self._convert_to_datetime(self.atags["creation_ts"])
# FIXME: I miss resolution and priority
issue = BugzillaIssue(issue_id, type, summary, desc, submitted_by,
submitted_on)
issue.set_priority(self.atags["priority"])
issue.set_status(self.atags["bug_status"])
assigned_to = People(self.atags["assigned_to"])
assigned_to.set_name(self.atags["assigned_to_name"])
assigned_to.set_email(self.atags["assigned_to"])
issue.set_assigned(assigned_to)
# FIXME = I miss the number of comment and the work_time (useful in
# bugzillas)
# date must be also a datetime
for rc in self._get_raw_comments():
if rc["bug_when"]:
by = People(rc["who"])
by.set_name(rc["who_name"])
by.set_email(rc["who"])
com = Comment(rc["thetext"], by, self._to_datetime_with_secs(rc["bug_when"]))
issue.add_comment(com)
else:
#FIXME bug_when empty
printdbg("ERROR - Comment")
# FIXME TBD: Attachment is not supported so far
## at = Attachment
#issue.add_attachment()
# FIXME TBD: Relations
# fields in btags: dependson, blocked
# issue.add_relationship() # issue_id, type
issue.set_resolution(self.atags["resolution"])
issue.set_alias(self.atags["alias"])
issue.set_delta_ts(self._to_datetime_with_secs(self.atags["delta_ts"]))
issue.set_reporter_accessible(self.atags["reporter_accessible"])
issue.set_cclist_accessible(self.atags["cclist_accessible"])
issue.set_classification_id(self.atags["classification_id"])
issue.set_classification(self.atags["classification"])
issue.set_product(self.atags["product"])
issue.set_component(self.atags["component"])
issue.set_version(self.atags["version"])
issue.set_rep_platform(self.atags["rep_platform"])
issue.set_op_sys(self.atags["op_sys"])
if self.atags["dup_id"]:
issue.set_dup_id(int(self.atags["dup_id"]))
issue.set_bug_file_loc(self.atags["bug_file_loc"])
issue.set_status_whiteboard(self.atags["status_whiteboard"])
issue.set_target_milestone(self.atags["target_milestone"])
issue.set_votes(self.atags["votes"])
issue.set_everconfirmed(self.atags["everconfirmed"])
issue.set_qa_contact(self.atags["qa_contact"])
issue.set_estimated_time(self.atags["estimated_time"])
issue.set_remaining_time(self.atags["remaining_time"])
issue.set_actual_time(self.atags["actual_time"])
if self.atags["deadline"]:
issue.set_deadline(self._convert_to_datetime(self.atags["deadline"]))
issue.set_keywords(self.btags["keywords"])
# we also store the list of watchers/CC
for w in self.btags["cc"]:
auxp = People(w)
issue.add_watcher(auxp)
issue.set_group(self.btags["group"])
issue.set_flag(self.btags["flag"])
return issue
def get_issue(self):
issue_id = self.atags["bug_id"]
type = self.atags["bug_severity"]
summary = self.atags["short_desc"]
if len(self.ctags["long_desc"]) > 0:
desc = self.ctags["long_desc"][0]["thetext"]
else:
desc = ""
submitted_by = People(self.atags["reporter"])
submitted_by.set_name(self.atags["reporter_name"])
submitted_by.set_email(self.atags["reporter"])
submitted_on = self._convert_to_datetime(self.atags["creation_ts"])
# FIXME: I miss resolution and priority
issue = BugzillaIssue(issue_id, type, summary, desc, submitted_by,
submitted_on)
issue.set_priority(self.atags["priority"])
issue.set_status(self.atags["bug_status"])
assigned_to = People(self.atags["assigned_to"])
assigned_to.set_name(self.atags["assigned_to_name"])
assigned_to.set_email(self.atags["assigned_to"])
issue.set_assigned(assigned_to)
# FIXME = I miss the number of comment and the work_time (useful in
# bugzillas)
# date must be also a datetime
for rc in self._get_raw_comments():
if rc["bug_when"]:
by = People(rc["who"])
by.set_name(rc["who_name"])
by.set_email(rc["who"])
com = Comment(rc["thetext"], by,
self._to_datetime_with_secs(rc["bug_when"]))
issue.add_comment(com)
else:
#FIXME bug_when empty
printdbg("ERROR - Comment")
# FIXME TBD: Attachment is not supported so far
## at = Attachment
#issue.add_attachment()
# FIXME TBD: Relations
# fields in btags: dependson, blocked
# issue.add_relationship() # issue_id, type
issue.set_resolution(self.atags["resolution"])
issue.set_alias(self.atags["alias"])
issue.set_delta_ts(self._to_datetime_with_secs(self.atags["delta_ts"]))
issue.set_reporter_accessible(self.atags["reporter_accessible"])
issue.set_cclist_accessible(self.atags["cclist_accessible"])
issue.set_classification_id(self.atags["classification_id"])
issue.set_classification(self.atags["classification"])
issue.set_product(self.atags["product"])
issue.set_component(self.atags["component"])
issue.set_version(self.atags["version"])
issue.set_rep_platform(self.atags["rep_platform"])
issue.set_op_sys(self.atags["op_sys"])
if self.atags["dup_id"]:
issue.set_dup_id(int(self.atags["dup_id"]))
issue.set_bug_file_loc(self.atags["bug_file_loc"])
issue.set_status_whiteboard(self.atags["status_whiteboard"])
issue.set_target_milestone(self.atags["target_milestone"])
issue.set_votes(self.atags["votes"])
issue.set_everconfirmed(self.atags["everconfirmed"])
issue.set_qa_contact(self.atags["qa_contact"])
issue.set_estimated_time(self.atags["estimated_time"])
issue.set_remaining_time(self.atags["remaining_time"])
issue.set_actual_time(self.atags["actual_time"])
if self.atags["deadline"]:
issue.set_deadline(
self._convert_to_datetime(self.atags["deadline"]))
issue.set_keywords(self.btags["keywords"])
# we also store the list of watchers/CC
for w in self.btags["cc"]:
auxp = People(w)
issue.add_watcher(auxp)
issue.set_group(self.btags["group"])
issue.set_flag(self.btags["flag"])
return issue
def parse_issue(self, html):
"""
"""
soup = BeautifulSoup.BeautifulSoup(html,
convertEntities=BeautifulSoup.BeautifulSoup.XHTML_ENTITIES)
self.__prepare_soup(soup)
try:
id = self.__parse_issue_id(soup)
summary = self.__parse_issue_summary(soup)
desc = self.__parse_issue_description(soup)
submission = self.__parse_issue_submission(soup)
priority = self.__parse_issue_priority(soup)
status = self.__parse_issue_status(soup)
resolution = self.__parse_issue_resolution(soup)
asignation = self.__parse_issue_assigned_to(soup)
category = self.__parse_issue_category(soup)
group = self.__parse_issue_group(soup)
# FIXME the visibility var below is never used!!
#visibility = self.__parse_issue_visibility(soup)
try:
comments = self.__parse_issue_comments(soup)
except SourceForgeParserError:
printerr("Error parsing issue's comments")
comments = None
pass
try:
attachments = self.__parse_issue_attachments(soup)
except SourceForgeParserError:
printerr("Error parsing issue's attachments")
attachments = None
pass
try:
changes = self.__parse_issue_changes(soup)
except SourceForgeParserError:
printerr("Error parsing issue's changes")
changes = None
pass
except:
raise
submitted_by = People(submission['id'])
submitted_by.set_name(submission['name'])
submitted_on = submission['date']
#assigned_to = People(asignation)
assigned_to = People('')
assigned_to.set_name(asignation)
issue = SourceForgeIssue(id, 'bug', summary, desc,
submitted_by, submitted_on)
issue.set_priority(priority)
issue.set_status(status, resolution)
issue.set_assigned(assigned_to)
issue.set_category(category)
issue.set_group(group)
if comments:
for comment in comments:
submitted_by = People(comment['by']['id'])
submitted_by.set_name(comment['by']['name'])
issue.add_comment(Comment(comment['desc'], submitted_by,
comment['date']))
if attachments:
for attachment in attachments:
a = Attachment(attachment['url'])
a.set_name(attachment['filename'])
a.set_description(attachment['desc'])
issue.add_attachment(a)
if changes:
for change in changes:
changed_by = People(change['by']['id'])
changed_by.set_name(change['by']['name'])
issue.add_change(Change(change['field'], change['old_value'],
'unknown', changed_by, change['date']))
return issue
def parse_issue(self, html):
"""
"""
soup = BeautifulSoup.BeautifulSoup(
html, convertEntities=BeautifulSoup.BeautifulSoup.XHTML_ENTITIES)
self.__prepare_soup(soup)
try:
id = self.__parse_issue_id(soup)
summary = self.__parse_issue_summary(soup)
desc = self.__parse_issue_description(soup)
submission = self.__parse_issue_submission(soup)
priority = self.__parse_issue_priority(soup)
status = self.__parse_issue_status(soup)
resolution = self.__parse_issue_resolution(soup)
asignation = self.__parse_issue_assigned_to(soup)
category = self.__parse_issue_category(soup)
group = self.__parse_issue_group(soup)
# FIXME the visibility var below is never used!!
#visibility = self.__parse_issue_visibility(soup)
try:
comments = self.__parse_issue_comments(soup)
except SourceForgeParserError:
printerr("Error parsing issue's comments")
comments = None
pass
try:
attachments = self.__parse_issue_attachments(soup)
except SourceForgeParserError:
printerr("Error parsing issue's attachments")
attachments = None
pass
try:
changes = self.__parse_issue_changes(soup)
except SourceForgeParserError:
printerr("Error parsing issue's changes")
changes = None
pass
except:
raise
submitted_by = People(submission['id'])
submitted_by.set_name(submission['name'])
submitted_on = submission['date']
#assigned_to = People(asignation)
assigned_to = People('')
assigned_to.set_name(asignation)
issue = SourceForgeIssue(id, 'bug', summary, desc, submitted_by,
submitted_on)
issue.set_priority(priority)
issue.set_status(status, resolution)
issue.set_assigned(assigned_to)
issue.set_category(category)
issue.set_group(group)
if comments:
for comment in comments:
submitted_by = People(comment['by']['id'])
submitted_by.set_name(comment['by']['name'])
issue.add_comment(
Comment(comment['desc'], submitted_by, comment['date']))
if attachments:
for attachment in attachments:
a = Attachment(attachment['url'])
a.set_name(attachment['filename'])
a.set_description(attachment['desc'])
issue.add_attachment(a)
if changes:
for change in changes:
changed_by = People(change['by']['id'])
changed_by.set_name(change['by']['name'])
issue.add_change(
Change(change['field'], change['old_value'], 'unknown',
changed_by, change['date']))
return issue
def getIssue(self, bug):
#Return the parse data bug into issue object
issue_id = bug.key_id
issue_type = bug.bug_type
summary = bug.summary
description = bug.description
status = bug.status
resolution = bug.resolution
assigned_by = People(bug.assignee_username)
assigned_by.set_name(bug.assignee)
assigned_by.set_email(BugsHandler.getUserEmail(bug.assignee_username))
submitted_by = People(bug.reporter_username)
submitted_by.set_name(bug.reporter)
submitted_by.set_email(BugsHandler.getUserEmail(bug.reporter_username))
submitted_on = parse(bug.created).replace(tzinfo=None)
issue = JiraIssue(issue_id, issue_type, summary, description, submitted_by, submitted_on)
issue.set_assigned(assigned_by)
issue.setIssue_key(bug.issue_key)
issue.setTitle(bug.title)
issue.setLink(bug.link)
issue.setEnvironment(bug.environment)
issue.setSecurity(bug.security)
issue.setUpdated(parse(bug.updated).replace(tzinfo=None))
issue.setVersion(bug.version)
issue.setComponent(bug.component)
issue.setVotes(bug.votes)
issue.setProject(bug.project)
issue.setProject_id(bug.project_id)
issue.setProject_key(bug.project_key)
issue.setStatus(status)
issue.setResolution(resolution)
bug_activity_url = bug.link + '?page=com.atlassian.jira.plugin.system.issuetabpanels%3Achangehistory-tabpanel'
printdbg("Bug activity: " + bug_activity_url)
data_activity = urllib.urlopen(bug_activity_url).read()
parser = SoupHtmlParser(data_activity, bug.key_id)
changes = parser.parse_changes()
for c in changes:
issue.add_change(c)
for comment in bug.comments:
comment_by = People(comment.comment_author)
comment_by.set_email(BugsHandler.getUserEmail(comment.comment_author))
comment_on = parse(comment.comment_created).replace(tzinfo=None)
com = Comment(comment.comment, comment_by, comment_on)
issue.add_comment(com)
for attachment in bug.attachments:
url = "/secure/attachment/" + attachment.attachment_id + "/" + attachment.attachment_name
attachment_by = People(attachment.attachment_author)
attachment_by.set_email(BugsHandler.getUserEmail(attachment.attachment_author))
attachment_on = parse(attachment.attachment_created).replace(tzinfo=None)
attach = Attachment(url, attachment_by, attachment_on)
issue.add_attachment(attach)
#FIXME customfield are not stored in db because is the fields has the same in all the bugs
return issue
def analyze_bug(self, issue_redmine):
#print(issue_redmine)
#print("*** %s " % issue_redmine["author"]["id"])
try:
people = People(
self._get_author_identity(issue_redmine["author"]["id"]))
people.set_name(issue_redmine["author"]["name"])
except KeyError:
people = People("None")
try:
desc = issue_redmine["description"]
except KeyError:
desc = ""
issue = RedmineIssue(
issue_redmine["id"], "ticket", issue_redmine["subject"], desc,
people, self._convert_to_datetime(issue_redmine["created_on"]))
try:
#print("<<< %s " % issue_redmine["assigned_to"]["id"])
people = People(
self._get_author_identity(issue_redmine["assigned_to"]["id"]))
people.set_name(issue_redmine["assigned_to"]["name"])
issue.assigned_to = people
except KeyError:
people = People("None")
issue.assigned_to = people
issue.status = issue_redmine["status"]["name"]
issue.priority = issue_redmine["priority"]["id"]
# No information from Redmine for this field. Included in Status
issue.resolution = None
# Extended attributes
try:
issue.category_id = issue_redmine["category"]["id"]
except KeyError:
issue.category_id = None
issue.done_ratio = issue_redmine["done_ratio"]
# if issue_redmine["due_date"] is None:
# issue.due_date = None
# else:
# issue.due_date = self._convert_to_datetime(issue_redmine["due_date"])
#issue.estimated_hours = issue_redmine["estimated_hours"]
try:
issue.fixed_version_id = issue_redmine["fixed_version"]["id"]
except KeyError:
issue.fixed_version_id = None
#issue.lft = issue_redmine["lft"]
#issue.rgt = issue_redmine["rgt"]
#issue.lock_version = issue_redmine["lock_version"]
#issue.parent_id = issue_redmine["parent_id"]
issue.project_id = issue_redmine["project"]["id"]
#issue.root_id = issue_redmine["root_id"]
# if issue_redmine["start_date"] is None:
# issue.start_date = None
# else:
# issue.start_date = self._convert_to_datetime(issue_redmine["start_date"])
try:
issue.start_date = self._convert_to_datetime(
issue_redmine["start_date"])
except:
issue.start_date = None
issue.tracker_id = issue_redmine["tracker"]["id"]
# if issue_redmine["updated_on"] is None:
# issue.updated_on = None
# else:
# issue.updated_on = self._convert_to_datetime(issue_redmine["updated_on"])
try:
issue.updated_on = self._convert_to_datetime(
issue_redmine["updated_on"])
except KeyError:
issue.updated_on = None
# Parse journals (comments and changes)
self._parse_journals(issue, issue_redmine["id"])
print("Issue #%s updated on %s" %
(issue_redmine["id"], issue.updated_on))
return issue
def parse_task(self, task):
# [u'status', u'assignee_id', u'title', u'story_id', u'created_at', u'updated_at', u'priority', u'creator_id', u'project_id', u'id']
people = People(task["creator_id"])
people.set_email(self.get_email(task["creator_id"]))
people.set_name(self.get_fullname(task["creator_id"]))
description = None
created_at = None
if task["created_at"] is not None:
created_at = self._convert_to_datetime(task["created_at"])
issue = StoryBoardIssue(task["id"], "task", task["title"], description,
people, created_at)
people = People(task["assignee_id"])
people.set_email(self.get_email(task["assignee_id"]))
people.set_name(self.get_fullname(task["assignee_id"]))
issue.assigned_to = people
issue.status = task["status"]
# No information from StoryBoard for this fields
issue.resolution = None
issue.priority = task["priority"]
# Extended attributes
if task["project_id"] is not None:
issue.project_id = task["project_id"]
issue.story_id = task["story_id"]
if task["updated_at"] is not None:
issue.mod_date = self._convert_to_datetime(task["updated_at"])
return issue
def parse_changes(self, review):
changesList = []
patchSets = review['patchSets']
for activity in patchSets:
# Now, we want all patches
# if "approvals" not in activity.keys():
# continue
patchSetNumber = activity['number']
# Add uploaded event
upload = self._convert_to_datetime(activity['createdOn'])
# share this people build logic
if "username" in activity["uploader"].keys():
by = People(activity["uploader"]['username'])
elif "email" in activity["uploader"].keys():
by = People(activity["uploader"]['email'])
elif "name" in activity["uploader"].keys():
by = People(activity["uploader"]['name'])
else:
by = People(unicode(''))
if "name" in activity["uploader"].keys():
by.set_name(activity["uploader"]["name"])
if "email" in activity["uploader"].keys():
by.set_email(activity["uploader"]["email"])
# print "changed_on:" + entry['updated']
field = unicode('Upload')
new_value = unicode('')
old_value = patchSetNumber
change = Change(field, old_value, new_value, by, upload)
changesList.append(change)
if 'approvals' in activity:
for entry in activity['approvals']:
# print "changed_by:" + entry['author']
if "username" in entry["by"].keys():
by = People(entry['by']['username'])
elif "email" in entry["by"].keys():
by = People(entry['by']['email'])
elif "name" in entry["by"].keys():
by = People(entry['by']['name'])
else:
by = People(unicode(''))
if "name" in entry["by"].keys():
by.set_name(entry["by"]["name"])
if "email" in entry["by"].keys():
by.set_email(entry["by"]["email"])
# print "changed_on:" + entry['updated']
field = entry['type']
new_value = entry['value']
old_value = patchSetNumber
update = self._convert_to_datetime(entry["grantedOn"])
change = Change(field, old_value, new_value, by, update)
changesList.append(change)
return changesList
def parse_review(self, review):
if "username" in review["owner"].keys():
people = People(review['owner']['username'])
elif "email" in review["owner"].keys():
people = People(review['owner']['email'])
elif "name" in review["owner"].keys():
people = People(review['owner']['name'])
else:
people = People(unicode(''))
if "name" in review["owner"].keys():
people.set_name(review["owner"]["name"])
if "email" in review["owner"].keys():
people.set_email(review["owner"]["email"])
description = ""
issue = GerritIssue(review["number"], "review", review["subject"],
description, people,
self._convert_to_datetime(review["createdOn"]))
# people = People(review["assigned_to_id"])
# people.set_name(review["assigned_to"])
# issue.assigned_to = people
issue.status = review["status"]
# No information from Gerrit for this fields
issue.assigned_to = None
issue.resolution = None
issue.priority = None
issue.branch = review["branch"]
issue.url = review["url"]
issue.change_id = review["id"]
if "topic" in review.keys():
issue.related_artifacts = review["topic"]
else:
issue.related_artifacts = None
issue.project = review["project"]
issue.mod_date = self._convert_to_datetime(review["lastUpdated"])
issue.open = review["open"]
return issue
def parse_changes(self, review):
changesList = []
patchSets = review['patchSets']
for activity in patchSets:
# Now, we want all patches
# if "approvals" not in activity.keys():
# continue
patchSetNumber = activity['number']
# Add uploaded event
upload = self._convert_to_datetime(activity['createdOn'])
# share this people build logic
if "username" in activity["uploader"].keys():
by = People(activity["uploader"]['username'])
elif "email" in activity["uploader"].keys():
by = People(activity["uploader"]['email'])
elif "name" in activity["uploader"].keys():
by = People(activity["uploader"]['name'])
else:
by = People(unicode(''))
if "name" in activity["uploader"].keys():
by.set_name(activity["uploader"]["name"])
if "email" in activity["uploader"].keys():
by.set_email(activity["uploader"]["email"])
# print "changed_on:" + entry['updated']
field = unicode('Upload')
new_value = unicode('')
old_value = patchSetNumber
change = Change(field, old_value, new_value, by, upload)
changesList.append(change)
if 'approvals' in activity:
for entry in activity['approvals']:
# print "changed_by:" + entry['author']
if "username" in entry["by"].keys():
by = People(entry['by']['username'])
elif "email" in entry["by"].keys():
by = People(entry['by']['email'])
elif "name" in entry["by"].keys():
by = People(entry['by']['name'])
else:
by = People(unicode(''))
if "name" in entry["by"].keys():
by.set_name(entry["by"]["name"])
if "email" in entry["by"].keys():
by.set_email(entry["by"]["email"])
# print "changed_on:" + entry['updated']
field = entry['type']
new_value = entry['value']
old_value = patchSetNumber
update = self._convert_to_datetime(entry["grantedOn"])
change = Change(field, old_value, new_value, by, update)
changesList.append(change)
return changesList
def analyze_bug(self, issue_redmine):
#print(issue_redmine)
#print("*** %s " % issue_redmine["author"]["id"])
try:
people = People(self._get_author_identity(issue_redmine["author"]["id"]))
people.set_name(issue_redmine["author"]["name"])
except KeyError:
people = People("None")
try:
desc = issue_redmine["description"]
except KeyError:
desc = ""
issue = RedmineIssue(issue_redmine["id"],
"ticket",
issue_redmine["subject"],
desc,
people,
self._convert_to_datetime(issue_redmine["created_on"]))
try:
#print("<<< %s " % issue_redmine["assigned_to"]["id"])
people = People(self._get_author_identity(issue_redmine["assigned_to"]["id"]))
people.set_name(issue_redmine["assigned_to"]["name"])
issue.assigned_to = people
except KeyError:
people = People("None")
issue.assigned_to = people
issue.status = issue_redmine["status"]["name"]
issue.priority = issue_redmine["priority"]["id"]
# No information from Redmine for this field. Included in Status
issue.resolution = None
# Extended attributes
try:
issue.category_id = issue_redmine["category"]["id"]
except KeyError:
issue.category_id = None
issue.done_ratio = issue_redmine["done_ratio"]
# if issue_redmine["due_date"] is None:
# issue.due_date = None
# else:
# issue.due_date = self._convert_to_datetime(issue_redmine["due_date"])
#issue.estimated_hours = issue_redmine["estimated_hours"]
try:
issue.fixed_version_id = issue_redmine["fixed_version"]["id"]
except KeyError:
issue.fixed_version_id = None
#issue.lft = issue_redmine["lft"]
#issue.rgt = issue_redmine["rgt"]
#issue.lock_version = issue_redmine["lock_version"]
#issue.parent_id = issue_redmine["parent_id"]
issue.project_id = issue_redmine["project"]["id"]
#issue.root_id = issue_redmine["root_id"]
# if issue_redmine["start_date"] is None:
# issue.start_date = None
# else:
# issue.start_date = self._convert_to_datetime(issue_redmine["start_date"])
try:
issue.start_date = self._convert_to_datetime(issue_redmine["start_date"])
except:
issue.start_date = None
issue.tracker_id = issue_redmine["tracker"]["id"]
# if issue_redmine["updated_on"] is None:
# issue.updated_on = None
# else:
# issue.updated_on = self._convert_to_datetime(issue_redmine["updated_on"])
try:
issue.updated_on = self._convert_to_datetime(issue_redmine["updated_on"])
except KeyError:
issue.updated_on = None
# Parse journals (comments and changes)
self._parse_journals(issue, issue_redmine["id"])
print("Issue #%s updated on %s" % (issue_redmine["id"], issue.updated_on))
return issue
def analyze_bug(self, bug):
#Retrieving main bug information
##
## all the retrieval can be improved. The method bug.lp_attributes
##offers a list of the available attributes for the object
##
printdbg(bug.web_link + " updated at " +
bug.bug.date_last_updated.isoformat())
issue = bug.web_link[bug.web_link.rfind('/') + 1:]
bug_type = bug.importance
summary = bug.bug.title
desc = bug.bug.description
submitted_by = self._get_person(bug.owner)
submitted_on = self.__drop_timezone(bug.date_created)
if bug.assignee:
assignee = self._get_person(bug.assignee)
else:
assignee = People("nobody")
issue = LaunchpadIssue(issue, bug_type, summary, desc, submitted_by,
submitted_on)
issue.set_assigned(assignee)
issue.set_status(bug.status)
issue.set_description(bug.bug.description)
issue.set_web_link(bug.web_link)
issue.set_target_display_name(bug.bug_target_display_name)
issue.set_target_name(bug.bug_target_name)
try:
if bug.date_assigned:
issue.set_date_assigned(self.__drop_timezone(
bug.date_assigned))
except AttributeError:
pass
try:
if bug.date_closed:
issue.set_date_closed(self.__drop_timezone(bug.date_closed))
except AttributeError:
pass
try:
if bug.date_confirmed:
issue.set_date_confirmed(
self.__drop_timezone(bug.date_confirmed))
except AttributeError:
pass
try:
if bug.date_created:
issue.set_date_created(self.__drop_timezone(bug.date_created))
except AttributeError:
pass
try:
if bug.date_fix_committed:
issue.set_date_fix_committed(
self.__drop_timezone(bug.date_fix_committed))
except AttributeError:
pass
try:
if bug.date_fix_released:
issue.set_date_fix_released(
self.__drop_timezone(bug.date_fix_released))
except AttributeError:
pass
try:
if bug.date_in_progress:
issue.set_date_in_progress(
self.__drop_timezone(bug.date_in_progress))
except AttributeError:
pass
try:
if bug.date_incomplete:
issue.set_date_incomplete(
self.__drop_timezone(bug.date_incomplete))
except AttributeError:
pass
try:
if bug.date_left_closed:
issue.set_date_left_closed(
self.__drop_timezone(bug.date_left_closed))
except AttributeError:
pass
try:
if bug.date_left_new:
issue.set_date_left_new(self.__drop_timezone(
bug.date_left_new))
except AttributeError:
pass
try:
if bug.date_triaged:
issue.set_date_triaged(self.__drop_timezone(bug.date_triaged))
except AttributeError:
pass
try:
if bug.date_last_message:
issue.set_date_last_message(
self.__drop_timezone(bug.date_last_message))
except AttributeError:
pass
try:
if bug.bug.date_last_updated:
issue.set_date_last_updated(
self.__drop_timezone(bug.bug.date_last_updated))
except AttributeError:
pass
if bug.milestone:
issue.set_milestone_code_name(bug.milestone.code_name)
issue.set_milestone_data_targeted(bug.milestone.date_targeted)
issue.set_milestone_name(bug.milestone.name)
issue.set_milestone_summary(bug.milestone.summary)
issue.set_milestone_title(bug.milestone.title)
issue.set_milestone_web_link(bug.milestone.web_link)
try:
if bug.bug.duplicate_of:
temp_rel = TempRelationship(bug.bug.id,
unicode('duplicate_of'),
unicode(bug.bug.duplicate_of.id))
issue.add_temp_relationship(temp_rel)
except NotFound:
printdbg(
"Issue %s is a duplicate of a private issue. Ignoring the private issue."
% issue.issue)
issue.set_heat(bug.bug.heat)
issue.set_linked_branches(bug.bug.linked_branches)
# storing the comments:
# first message of the bugs contains the description
if (bug.bug.messages and len(bug.bug.messages) > 1):
skip = 1
for c in bug.bug.messages:
if (skip == 1):
# we skip the first comment which is the description
skip = 0
continue
by = self._get_person(c.owner)
com = Comment(c.content, by, c.date_created)
issue.add_comment(com)
issue.set_tags(bug.bug.tags)
issue.set_title(bug.bug.title)
issue.set_users_affected_count(bug.bug.users_affected_count)
issue.set_web_link_standalone(bug.bug.web_link)
# activity
for entry in bug.bug.activity.entries:
field = entry['whatchanged']
removed = entry['oldvalue']
added = entry['newvalue']
by = self.__get_people_from_uri(entry['person_link'])
date = self.__to_datetime(entry['datechanged'])
change = Change(field, removed, added, by, date)
issue.add_change(change)
for a in bug.bug.attachments.entries:
a_url = a['data_link']
a_name = a['title']
# author and date are stored in the comment object
aux = a['message_link']
comment_id = int(aux[aux.rfind('/') + 1:])
comment = bug.bug.messages[comment_id]
a_by = self._get_person(comment.owner)
a_on = self.__drop_timezone(comment.date_created)
#a_desc = a['']
att = Attachment(a_url, a_by, a_on)
att.set_name(a_name)
#att.set_description()
issue.add_attachment(att)
return issue
def analyze_bug(self, bug):
#Retrieving main bug information
printdbg(bug['url'] + " " + bug['state'] + " updated_at " +
bug['updated_at'] + ' (ratelimit = ' +
str(self.remaining_ratelimit) + ")")
issue = bug['id']
if bug['labels']:
bug_type = bug['labels'][0]['name'] # FIXME
else:
bug_type = unicode('')
summary = bug['title']
desc = bug['body']
submitted_by = self.__get_user(bug['user']['login'])
submitted_on = self.__to_datetime(bug['created_at'])
if bug['assignee']:
assignee = self.__get_user(bug['assignee']['login'])
else:
assignee = People(unicode("nobody"))
issue = GithubIssue(issue, bug_type, summary, desc, submitted_by,
submitted_on)
issue.set_assigned(assignee)
issue.set_status(bug['state'])
issue.set_description(bug['body'])
issue.set_web_link(bug['html_url'])
try:
if bug['closed_at']:
issue.set_closed_at(self.__to_datetime(bug['closed_at']))
except AttributeError:
pass
# updated_at offers ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ
# MySQL doesn't support timezone, we remove it
issue.set_updated_at(self.__to_datetime(bug['updated_at']))
if bug['milestone']:
issue.set_milestone_name(bug['milestone']['id'])
issue.set_milestone_summary(bug['milestone']['description'])
issue.set_milestone_title(bug['milestone']['title'])
issue.set_milestone_web_link(bug['milestone']['url'])
comments = self.__get_batch_comments(bug['number'])
for c in comments:
by = self.__get_user(c['user']['login'])
date = self.__to_datetime(c['created_at'])
com = Comment(c['body'], by, date)
issue.add_comment(com)
# activity
entries = self.__get_batch_activities(bug['number'])
for e in entries:
field = e['event']
added = e['commit_id']
removed = unicode('')
if e['actor']:
by = self.__get_user(e['actor']['login'])
else:
by = People(u"nobody")
## by.setname() FIXME - to be done
date = self.__to_datetime(e['created_at'])
change = Change(field, removed, added, by, date)
issue.add_change(change)
return issue
def getIssue(self, bug, conn):
#Return the parse data bug into issue object
issue_id = bug.key_id
issue_type = bug.bug_type
summary = bug.summary
description = bug.description
status = bug.status
resolution = bug.resolution
assigned_by = People(bug.assignee_username)
assigned_by.set_name(bug.assignee)
assigned_by.set_email(BugsHandler.getUserEmail(bug.assignee_username))
submitted_by = People(bug.reporter_username)
submitted_by.set_name(bug.reporter)
submitted_by.set_email(BugsHandler.getUserEmail(bug.reporter_username))
submitted_on = parse(bug.created).replace(tzinfo=None)
issue = JiraIssue(issue_id, issue_type, summary, description,
submitted_by, submitted_on)
issue.set_assigned(assigned_by)
issue.setIssue_key(bug.issue_key)
issue.setTitle(bug.title)
issue.setLink(bug.link)
issue.setEnvironment(bug.environment)
issue.setSecurity(bug.security)
issue.setUpdated(parse(bug.updated).replace(tzinfo=None))
issue.setVersion(bug.version)
issue.setFixVersion(bug.fix_version)
issue.setComponent(bug.component)
issue.setVotes(bug.votes)
issue.setProject(bug.project)
issue.setProject_id(bug.project_id)
issue.setProject_key(bug.project_key)
issue.setStatus(status)
issue.setResolution(resolution)
bug_activity_url = bug.link + '?page=com.atlassian.jira.plugin.system.issuetabpanels%3Achangehistory-tabpanel'
printdbg("Bug activity: " + bug_activity_url)
f = conn.urlopen_auth(bug_activity_url)
data_activity = f.read()
parser = SoupHtmlParser(data_activity, bug.key_id)
changes = parser.parse_changes()
for c in changes:
issue.add_change(c)
for comment in bug.comments:
comment_by = People(comment.comment_author)
comment_by.set_email(
BugsHandler.getUserEmail(comment.comment_author))
comment_on = parse(comment.comment_created).replace(tzinfo=None)
com = Comment(comment.comment, comment_by, comment_on)
issue.add_comment(com)
for attachment in bug.attachments:
url = "/secure/attachment/" + attachment.attachment_id + "/" + attachment.attachment_name
attachment_by = People(attachment.attachment_author)
attachment_by.set_email(
BugsHandler.getUserEmail(attachment.attachment_author))
attachment_on = parse(
attachment.attachment_created).replace(tzinfo=None)
attach = Attachment(url, attachment_by, attachment_on)
issue.add_attachment(attach)
#FIXME customfield are not stored in db because is the fields has the same in all the bugs
return issue
