def insert_user_data(issues, conf):
"""
Insert user data into database ad update issue data.
:param issues: the issues to retrieve user data from
:param conf: the project configuration
:return: the updated issue data
"""
log.info("Syncing users with ID service...")
# create buffer for users
user_buffer = dict()
# open database connection
dbm = DBManager(conf)
# open ID-service connection
idservice = idManager(dbm, conf)
def get_user_string(name, email):
if not email or email is None:
return "{name}".format(name=name)
# return "{name} <{name}@default.com>".format(name=name) # for debugging only
else:
return "{name} <{email}>".format(name=name, email=email)
def get_or_update_user(user, buffer_db=user_buffer):
# fix encoding for name and e-mail address
if user["name"] is not None:
name = unicode(user["name"]).encode("utf-8")
else:
name = unicode(user["username"]).encode("utf-8")
mail = unicode(user["email"]).encode("utf-8") # empty
# construct string for ID service and send query
user_string = get_user_string(name, mail)
# check buffer to reduce amount of DB queries
if user_string in buffer_db:
log.devinfo("Returning user '{}' from buffer.".format(user_string))
return buffer_db[user_string]
# get person information from ID service
log.devinfo("Passing user '{}' to ID service.".format(user_string))
idx = idservice.getPersonID(user_string)
# update user data with person information from DB
person = idservice.getPersonFromDB(idx)
user["email"] = person["email1"] # column "email1"
user["name"] = person["name"] # column "name"
user["id"] = person["id"] # column "id"
# add user information to buffer
# user_string = get_user_string(user["name"], user["email"]) # update for
buffer_db[user_string] = user
return user
for issue in issues:
# check database for issue author
issue["author"] = get_or_update_user(issue["author"])
# check database for event authors
for comment in issue["comments"]:
# get the event user from the DB
comment["author"] = get_or_update_user(comment["author"])
## get the reference-target user from the DB if needed
# if event["ref_target"] != "":
# event["ref_target"] = get_or_update_user(event["ref_target"])
log.debug("number of issues after insert_user_data: '{}'".format(len(issues)))
return issues
def insert_user_data(issues, conf):
"""
Insert user data into database and update issue data.
:param issues: the issues to retrieve user data from
:param conf: the project configuration
:return: the updated issue data
"""
log.info("Syncing users with ID service...")
# create buffer for users (key: user id)
user_buffer = dict()
# create buffer for user ids (key: user string)
user_id_buffer = dict()
# open database connection
dbm = DBManager(conf)
# open ID-service connection
idservice = idManager(dbm, conf)
def get_user_string(name, email):
if not email or email is None:
return "{name}".format(name=name)
# return "{name} <{name}@default.com>".format(name=name) # for debugging only
else:
return "{name} <{email}>".format(name=name, email=email)
def get_id_and_update_user(user, buffer_db_ids=user_id_buffer):
# fix encoding for name and e-mail address
if user["name"] is not None and user["name"] != "":
name = unicode(user["name"]).encode("utf-8")
else:
name = unicode(user["username"]).encode("utf-8")
mail = unicode(user["email"]).encode("utf-8") # empty
# construct string for ID service and send query
user_string = get_user_string(name, mail)
# check buffer to reduce amount of DB queries
if user_string in buffer_db_ids:
log.devinfo(
"Returning person id for user '{}' from buffer.".format(
user_string))
return buffer_db_ids[user_string]
# get person information from ID service
log.devinfo("Passing user '{}' to ID service.".format(user_string))
idx = idservice.getPersonID(user_string)
# add user information to buffer
# user_string = get_user_string(user["name"], user["email"]) # update for
buffer_db_ids[user_string] = idx
return idx
def get_user_from_id(idx, buffer_db=user_buffer):
# check whether user information is in buffer to reduce amount of DB queries
if idx in buffer_db:
log.devinfo("Returning user '{}' from buffer.".format(idx))
return buffer_db[idx]
# get person information from ID service
log.devinfo("Passing user id '{}' to ID service.".format(idx))
person = idservice.getPersonFromDB(idx)
user = dict()
user["email"] = person["email1"] # column "email1"
user["name"] = person["name"] # column "name"
user["id"] = person["id"] # column "id"
# add user information to buffer
buffer_db[idx] = user
return user
# check and update database for all occurring users
for issue in issues:
# check database for issue author
issue["author"] = get_id_and_update_user(issue["author"])
# check database for comment authors
for comment in issue["comments"]:
comment["author"] = get_id_and_update_user(comment["author"])
# check database for event authors in the history
for event in issue["history"]:
event["author"] = get_id_and_update_user(event["author"])
# check database for target user if needed
if event["event"] == "assigned":
assigned_user = get_id_and_update_user(
create_user(event["event_info_1"], "",
event["event_info_2"]))
event["event_info_1"] = assigned_user
# get all users after database updates having been performed
for issue in issues:
# get issue author
issue["author"] = get_user_from_id(issue["author"])
# get comment authors
for comment in issue["comments"]:
comment["author"] = get_user_from_id(comment["author"])
# get event authors for non-comment events
for event in issue["history"]:
event["author"] = get_user_from_id(event["author"])
# get target user if needed
if event["event"] == "assigned":
assigned_user = get_user_from_id(event["event_info_1"])
event["event_info_1"] = assigned_user["name"]
event["event_info_2"] = assigned_user["email"]
log.debug("number of issues after insert_user_data: '{}'".format(
len(issues)))
return issues
def parse_xml(issue_data, persons, skip_history):
"""
Parse issues from the xml-data
:param issue_data: list of xml-files
:param persons: list of persons from JIRA (incl. e-mail addresses)
:param skip_history: flag if the history will be loaded in a different method
:return: list of parsed issues
"""
log.info("Parse jira issues...")
issues = list()
issuelist = issue_data.getElementsByTagName("item")
# re-process all issues
log.debug("Number of issues:" + str(len(issuelist)))
for issue_x in issuelist:
# temporary container for references
comments = list()
issue = dict()
components = []
# parse values form xml
# add issue values to the issue
key = issue_x.getElementsByTagName("key")[0]
issue["id"] = key.attributes["id"].value
issue["externalId"] = key.firstChild.data
created = issue_x.getElementsByTagName("created")[0]
createDate = created.firstChild.data
issue["creationDate"] = format_time(createDate)
resolved = issue_x.getElementsByTagName("resolved")
issue["resolveDate"] = ""
if (len(resolved) > 0) and (not resolved[0] is None):
resolveDate = resolved[0].firstChild.data
issue["resolveDate"] = format_time(resolveDate)
title = issue_x.getElementsByTagName("title")[0]
issue["title"] = title.firstChild.data
link = issue_x.getElementsByTagName("link")[0]
issue["url"] = link.firstChild.data
type = issue_x.getElementsByTagName("type")[0]
issue["type"] = type.firstChild.data
# TODO new consistent format with GitHub issues. Not supported by the network library yet
issue["type_new"] = ["issue", str(type.firstChild.data.lower())]
status = issue_x.getElementsByTagName("status")[0]
issue["state"] = status.firstChild.data
# TODO new consistent format with GitHub issues. Not supported by the network library yet
issue["state_new"] = status.firstChild.data.lower()
project = issue_x.getElementsByTagName("project")[0]
issue["projectId"] = project.attributes["id"].value
resolution = issue_x.getElementsByTagName("resolution")[0]
issue["resolution"] = resolution.firstChild.data
# new consistent format with GitHub issues. Not supported by the network library yet
issue["resolution_new"] = [str(resolution.firstChild.data.lower())]
# consistency to default GitHub labels
if issue["resolution"] == "Won't Fix":
issue["resolution_new"] = ["wontfix"]
# consistency to default GitHub labels
if issue["resolution"] == "Won't Do":
issue["resolution_new"] = ["wontdo"]
for component in issue_x.getElementsByTagName("component"):
components.append(str(component.firstChild.data))
issue["components"] = components
# if links are not loaded via api, they are added as a history event with less information
if skip_history:
issue["history"] = []
for ref in issue_x.getElementsByTagName("issuelinktype"):
history = dict()
history["event"] = "add_link"
history["author"] = create_user("", "", "")
history["date"] = ""
history["event_info_1"] = ref.getElementsByTagName("issuekey")[0].firstChild.data
history["event_info_2"] = "issue"
issue["history"].append(history)
reporter = issue_x.getElementsByTagName("reporter")[0]
user = create_user(reporter.firstChild.data, reporter.attributes["username"].value, "")
issue["author"] = merge_user_with_user_from_csv(user, persons)
issue["title"] = issue_x.getElementsByTagName("title")[0].firstChild.data
# add comments / issue_changes to the issue
for comment_x in issue_x.getElementsByTagName("comment"):
comment = dict()
comment["id"] = comment_x.attributes["id"].value
user = create_user("", comment_x.attributes["author"].value, "")
comment["author"] = merge_user_with_user_from_csv(user, persons)
comment["state_on_creation"] = issue["state"] # can get updated if history is retrieved
comment["resolution_on_creation"] = issue["resolution"] # can get updated if history is retrieved
created = comment_x.attributes["created"].value
comment["changeDate"] = format_time(created)
comment["text"] = comment_x.firstChild.data
comment["issueId"] = issue["id"]
comments.append(comment)
issue["comments"] = comments
# add relations to the issue
relations = list()
for rel in issue_x.getElementsByTagName("issuelinktype"):
relation = dict()
relation["relation"] = rel.getElementsByTagName("name")[0].firstChild.data
if rel.hasAttribute("inwardlinks"):
left = rel.getElementsByTagName("inwardlinks")
issuekeys = left.getElementsByTagName("issuekey")
for key in issuekeys:
relation["type"] = "inward"
relation["id"] = key.firstChild.data
relations.append(relation)
if rel.hasAttribute("outwardlinks"):
right = rel.getElementsByTagName("outwardlinks")
issuekeys = right.getElementsByTagName("issuekey")
for key in issuekeys:
relation["type"] = "outward"
relation["id"] = key.firstChild.data
relations.append(relation)
issue["relations"] = relations
issues.append(issue)
log.debug("number of issues after parse_xml: '{}'".format(len(issues)))
return issues
def parse_xml(issue_data, persons):
"""Parse issues from the xml-data
:param issue_data: list of xml-files
:param persons: list of persons from JIRA (incl. e-mail addresses)
:return: list of parsed issues
"""
log.info("Parse jira issues...")
issues = list()
log.debug("Number of files:" + str(len(issue_data)))
for issue_file in issue_data:
issuelist = issue_file.getElementsByTagName('item')
# re-process all issues
log.debug("Number of issues:" + str(len(issuelist)))
for issue_x in issuelist:
# temporary container for references
comments = list()
issue = dict()
components = []
refs = []
# parse values form xml
# add issue values to the issue
key = issue_x.getElementsByTagName('key')[0]
issue['id'] = key.attributes['id'].value
issue['externalId'] = key.firstChild.data
created = issue_x.getElementsByTagName('created')[0]
createDate = created.firstChild.data
d = datetime.strptime(createDate, '%a, %d %b %Y %H:%M:%S +0000')
issue['creationDate'] = d.strftime('%Y-%m-%d %H:%M:%S')
resolved = issue_x.getElementsByTagName('resolved')
issue['resolveDate'] = ""
if (len(resolved) > 0) and (not resolved[0] is None):
resolveDate = resolved[0].firstChild.data
d = datetime.strptime(resolveDate,
'%a, %d %b %Y %H:%M:%S +0000')
issue['resolveDate'] = d.strftime('%Y-%m-%d %H:%M:%S')
link = issue_x.getElementsByTagName('link')[0]
issue['url'] = link.firstChild.data
type = issue_x.getElementsByTagName('type')[0]
issue['type'] = type.firstChild.data
status = issue_x.getElementsByTagName('status')[0]
issue['state'] = status.firstChild.data
project = issue_x.getElementsByTagName('project')[0]
issue['projectId'] = project.attributes['id'].value
resolution = issue_x.getElementsByTagName('resolution')[0]
issue['resolution'] = resolution.firstChild.data
for component in issue_x.getElementsByTagName('component'):
components.append(component.firstChild.data)
issue['components'] = components
for ref in issue_x.getElementsByTagName('issuelinktype'):
refId = ref.getElementsByTagName('issuekey')[0].firstChild.data
refType = ref.getElementsByTagName('name')[0].firstChild.data
refs.append((refId, refType))
issue['references'] = refs
reporter = issue_x.getElementsByTagName('reporter')[0]
user = dict()
user["username"] = reporter.attributes['username'].value
user["name"] = reporter.firstChild.data
user["email"] = ""
issue["author"] = merge_user_with_user_from_csv(user, persons)
issue['title'] = issue_x.getElementsByTagName(
'title')[0].firstChild.data
# add comments / issue_changes to the issue
for comment_x in issue_x.getElementsByTagName('comment'):
comment = dict()
comment['id'] = comment_x.attributes['id'].value
user = dict()
user["username"] = comment_x.attributes['author'].value
user["name"] = ""
user["email"] = ""
comment["author"] = merge_user_with_user_from_csv(
user, persons)
comment['state_on_creation'] = issue[
'state'] # can get updated if history is retrieved
comment['resolution_on_creation'] = issue[
'resolution'] # can get updated if history is retrieved
created = comment_x.attributes['created'].value
d = datetime.strptime(created, '%a, %d %b %Y %H:%M:%S +0000')
comment['changeDate'] = d.strftime('%Y-%m-%d %H:%M:%S')
comment['text'] = comment_x.firstChild.data
comment['issueId'] = issue['id']
comments.append(comment)
issue['comments'] = comments
# add relations to the issue
relations = list()
for rel in issue_x.getElementsByTagName('issuelinktype'):
relation = dict()
relation['relation'] = rel.getElementsByTagName(
'name')[0].firstChild.data
if (rel.hasAttribute('inwardlinks')):
left = rel.getElementsByTagName('inwardlinks')
issuekeys = left.getElementsByTagName("issuekey")
for key in issuekeys:
relation['type'] = "inward"
relation['id'] = key.firstChild.data
relations.append(relation)
if (rel.hasAttribute('outwardlinks')):
right = rel.getElementsByTagName('outwardlinks')
issuekeys = right.getElementsByTagName('issuekey')
for key in issuekeys:
relation['type'] = "outward"
relation['id'] = key.firstChild.data
relations.append(relation)
issue["relations"] = relations
issues.append(issue)
log.debug("number of issues after parse_xml: '{}'".format(len(issues)))
return issues