def insert_user_data(issues, conf): """ Insert user data into database ad update issue data. :param issues: the issues to retrieve user data from :param conf: the project configuration :return: the updated issue data """ log.info("Syncing users with ID service...") # create buffer for users user_buffer = dict() # open database connection dbm = DBManager(conf) # open ID-service connection idservice = idManager(dbm, conf) def get_user_string(name, email): if not email or email is None: return "{name}".format(name=name) # return "{name} <{name}@default.com>".format(name=name) # for debugging only else: return "{name} <{email}>".format(name=name, email=email) def get_or_update_user(user, buffer_db=user_buffer): # fix encoding for name and e-mail address if user["name"] is not None: name = unicode(user["name"]).encode("utf-8") else: name = unicode(user["username"]).encode("utf-8") mail = unicode(user["email"]).encode("utf-8") # empty # construct string for ID service and send query user_string = get_user_string(name, mail) # check buffer to reduce amount of DB queries if user_string in buffer_db: log.devinfo("Returning user '{}' from buffer.".format(user_string)) return buffer_db[user_string] # get person information from ID service log.devinfo("Passing user '{}' to ID service.".format(user_string)) idx = idservice.getPersonID(user_string) # update user data with person information from DB person = idservice.getPersonFromDB(idx) user["email"] = person["email1"] # column "email1" user["name"] = person["name"] # column "name" user["id"] = person["id"] # column "id" # add user information to buffer # user_string = get_user_string(user["name"], user["email"]) # update for buffer_db[user_string] = user return user for issue in issues: # check database for issue author issue["author"] = get_or_update_user(issue["author"]) # check database for event authors for comment in issue["comments"]: # get the event user from the DB comment["author"] = get_or_update_user(comment["author"]) ## get the reference-target user from the DB if needed # if event["ref_target"] != "": # event["ref_target"] = get_or_update_user(event["ref_target"]) log.debug("number of issues after insert_user_data: '{}'".format(len(issues))) return issues
def insert_user_data(issues, conf): """ Insert user data into database and update issue data. :param issues: the issues to retrieve user data from :param conf: the project configuration :return: the updated issue data """ log.info("Syncing users with ID service...") # create buffer for users (key: user id) user_buffer = dict() # create buffer for user ids (key: user string) user_id_buffer = dict() # open database connection dbm = DBManager(conf) # open ID-service connection idservice = idManager(dbm, conf) def get_user_string(name, email): if not email or email is None: return "{name}".format(name=name) # return "{name} <{name}@default.com>".format(name=name) # for debugging only else: return "{name} <{email}>".format(name=name, email=email) def get_id_and_update_user(user, buffer_db_ids=user_id_buffer): # fix encoding for name and e-mail address if user["name"] is not None and user["name"] != "": name = unicode(user["name"]).encode("utf-8") else: name = unicode(user["username"]).encode("utf-8") mail = unicode(user["email"]).encode("utf-8") # empty # construct string for ID service and send query user_string = get_user_string(name, mail) # check buffer to reduce amount of DB queries if user_string in buffer_db_ids: log.devinfo( "Returning person id for user '{}' from buffer.".format( user_string)) return buffer_db_ids[user_string] # get person information from ID service log.devinfo("Passing user '{}' to ID service.".format(user_string)) idx = idservice.getPersonID(user_string) # add user information to buffer # user_string = get_user_string(user["name"], user["email"]) # update for buffer_db_ids[user_string] = idx return idx def get_user_from_id(idx, buffer_db=user_buffer): # check whether user information is in buffer to reduce amount of DB queries if idx in buffer_db: log.devinfo("Returning user '{}' from buffer.".format(idx)) return buffer_db[idx] # get person information from ID service log.devinfo("Passing user id '{}' to ID service.".format(idx)) person = idservice.getPersonFromDB(idx) user = dict() user["email"] = person["email1"] # column "email1" user["name"] = person["name"] # column "name" user["id"] = person["id"] # column "id" # add user information to buffer buffer_db[idx] = user return user # check and update database for all occurring users for issue in issues: # check database for issue author issue["author"] = get_id_and_update_user(issue["author"]) # check database for comment authors for comment in issue["comments"]: comment["author"] = get_id_and_update_user(comment["author"]) # check database for event authors in the history for event in issue["history"]: event["author"] = get_id_and_update_user(event["author"]) # check database for target user if needed if event["event"] == "assigned": assigned_user = get_id_and_update_user( create_user(event["event_info_1"], "", event["event_info_2"])) event["event_info_1"] = assigned_user # get all users after database updates having been performed for issue in issues: # get issue author issue["author"] = get_user_from_id(issue["author"]) # get comment authors for comment in issue["comments"]: comment["author"] = get_user_from_id(comment["author"]) # get event authors for non-comment events for event in issue["history"]: event["author"] = get_user_from_id(event["author"]) # get target user if needed if event["event"] == "assigned": assigned_user = get_user_from_id(event["event_info_1"]) event["event_info_1"] = assigned_user["name"] event["event_info_2"] = assigned_user["email"] log.debug("number of issues after insert_user_data: '{}'".format( len(issues))) return issues
def parse_xml(issue_data, persons, skip_history): """ Parse issues from the xml-data :param issue_data: list of xml-files :param persons: list of persons from JIRA (incl. e-mail addresses) :param skip_history: flag if the history will be loaded in a different method :return: list of parsed issues """ log.info("Parse jira issues...") issues = list() issuelist = issue_data.getElementsByTagName("item") # re-process all issues log.debug("Number of issues:" + str(len(issuelist))) for issue_x in issuelist: # temporary container for references comments = list() issue = dict() components = [] # parse values form xml # add issue values to the issue key = issue_x.getElementsByTagName("key")[0] issue["id"] = key.attributes["id"].value issue["externalId"] = key.firstChild.data created = issue_x.getElementsByTagName("created")[0] createDate = created.firstChild.data issue["creationDate"] = format_time(createDate) resolved = issue_x.getElementsByTagName("resolved") issue["resolveDate"] = "" if (len(resolved) > 0) and (not resolved[0] is None): resolveDate = resolved[0].firstChild.data issue["resolveDate"] = format_time(resolveDate) title = issue_x.getElementsByTagName("title")[0] issue["title"] = title.firstChild.data link = issue_x.getElementsByTagName("link")[0] issue["url"] = link.firstChild.data type = issue_x.getElementsByTagName("type")[0] issue["type"] = type.firstChild.data # TODO new consistent format with GitHub issues. Not supported by the network library yet issue["type_new"] = ["issue", str(type.firstChild.data.lower())] status = issue_x.getElementsByTagName("status")[0] issue["state"] = status.firstChild.data # TODO new consistent format with GitHub issues. Not supported by the network library yet issue["state_new"] = status.firstChild.data.lower() project = issue_x.getElementsByTagName("project")[0] issue["projectId"] = project.attributes["id"].value resolution = issue_x.getElementsByTagName("resolution")[0] issue["resolution"] = resolution.firstChild.data # new consistent format with GitHub issues. Not supported by the network library yet issue["resolution_new"] = [str(resolution.firstChild.data.lower())] # consistency to default GitHub labels if issue["resolution"] == "Won't Fix": issue["resolution_new"] = ["wontfix"] # consistency to default GitHub labels if issue["resolution"] == "Won't Do": issue["resolution_new"] = ["wontdo"] for component in issue_x.getElementsByTagName("component"): components.append(str(component.firstChild.data)) issue["components"] = components # if links are not loaded via api, they are added as a history event with less information if skip_history: issue["history"] = [] for ref in issue_x.getElementsByTagName("issuelinktype"): history = dict() history["event"] = "add_link" history["author"] = create_user("", "", "") history["date"] = "" history["event_info_1"] = ref.getElementsByTagName("issuekey")[0].firstChild.data history["event_info_2"] = "issue" issue["history"].append(history) reporter = issue_x.getElementsByTagName("reporter")[0] user = create_user(reporter.firstChild.data, reporter.attributes["username"].value, "") issue["author"] = merge_user_with_user_from_csv(user, persons) issue["title"] = issue_x.getElementsByTagName("title")[0].firstChild.data # add comments / issue_changes to the issue for comment_x in issue_x.getElementsByTagName("comment"): comment = dict() comment["id"] = comment_x.attributes["id"].value user = create_user("", comment_x.attributes["author"].value, "") comment["author"] = merge_user_with_user_from_csv(user, persons) comment["state_on_creation"] = issue["state"] # can get updated if history is retrieved comment["resolution_on_creation"] = issue["resolution"] # can get updated if history is retrieved created = comment_x.attributes["created"].value comment["changeDate"] = format_time(created) comment["text"] = comment_x.firstChild.data comment["issueId"] = issue["id"] comments.append(comment) issue["comments"] = comments # add relations to the issue relations = list() for rel in issue_x.getElementsByTagName("issuelinktype"): relation = dict() relation["relation"] = rel.getElementsByTagName("name")[0].firstChild.data if rel.hasAttribute("inwardlinks"): left = rel.getElementsByTagName("inwardlinks") issuekeys = left.getElementsByTagName("issuekey") for key in issuekeys: relation["type"] = "inward" relation["id"] = key.firstChild.data relations.append(relation) if rel.hasAttribute("outwardlinks"): right = rel.getElementsByTagName("outwardlinks") issuekeys = right.getElementsByTagName("issuekey") for key in issuekeys: relation["type"] = "outward" relation["id"] = key.firstChild.data relations.append(relation) issue["relations"] = relations issues.append(issue) log.debug("number of issues after parse_xml: '{}'".format(len(issues))) return issues
def parse_xml(issue_data, persons): """Parse issues from the xml-data :param issue_data: list of xml-files :param persons: list of persons from JIRA (incl. e-mail addresses) :return: list of parsed issues """ log.info("Parse jira issues...") issues = list() log.debug("Number of files:" + str(len(issue_data))) for issue_file in issue_data: issuelist = issue_file.getElementsByTagName('item') # re-process all issues log.debug("Number of issues:" + str(len(issuelist))) for issue_x in issuelist: # temporary container for references comments = list() issue = dict() components = [] refs = [] # parse values form xml # add issue values to the issue key = issue_x.getElementsByTagName('key')[0] issue['id'] = key.attributes['id'].value issue['externalId'] = key.firstChild.data created = issue_x.getElementsByTagName('created')[0] createDate = created.firstChild.data d = datetime.strptime(createDate, '%a, %d %b %Y %H:%M:%S +0000') issue['creationDate'] = d.strftime('%Y-%m-%d %H:%M:%S') resolved = issue_x.getElementsByTagName('resolved') issue['resolveDate'] = "" if (len(resolved) > 0) and (not resolved[0] is None): resolveDate = resolved[0].firstChild.data d = datetime.strptime(resolveDate, '%a, %d %b %Y %H:%M:%S +0000') issue['resolveDate'] = d.strftime('%Y-%m-%d %H:%M:%S') link = issue_x.getElementsByTagName('link')[0] issue['url'] = link.firstChild.data type = issue_x.getElementsByTagName('type')[0] issue['type'] = type.firstChild.data status = issue_x.getElementsByTagName('status')[0] issue['state'] = status.firstChild.data project = issue_x.getElementsByTagName('project')[0] issue['projectId'] = project.attributes['id'].value resolution = issue_x.getElementsByTagName('resolution')[0] issue['resolution'] = resolution.firstChild.data for component in issue_x.getElementsByTagName('component'): components.append(component.firstChild.data) issue['components'] = components for ref in issue_x.getElementsByTagName('issuelinktype'): refId = ref.getElementsByTagName('issuekey')[0].firstChild.data refType = ref.getElementsByTagName('name')[0].firstChild.data refs.append((refId, refType)) issue['references'] = refs reporter = issue_x.getElementsByTagName('reporter')[0] user = dict() user["username"] = reporter.attributes['username'].value user["name"] = reporter.firstChild.data user["email"] = "" issue["author"] = merge_user_with_user_from_csv(user, persons) issue['title'] = issue_x.getElementsByTagName( 'title')[0].firstChild.data # add comments / issue_changes to the issue for comment_x in issue_x.getElementsByTagName('comment'): comment = dict() comment['id'] = comment_x.attributes['id'].value user = dict() user["username"] = comment_x.attributes['author'].value user["name"] = "" user["email"] = "" comment["author"] = merge_user_with_user_from_csv( user, persons) comment['state_on_creation'] = issue[ 'state'] # can get updated if history is retrieved comment['resolution_on_creation'] = issue[ 'resolution'] # can get updated if history is retrieved created = comment_x.attributes['created'].value d = datetime.strptime(created, '%a, %d %b %Y %H:%M:%S +0000') comment['changeDate'] = d.strftime('%Y-%m-%d %H:%M:%S') comment['text'] = comment_x.firstChild.data comment['issueId'] = issue['id'] comments.append(comment) issue['comments'] = comments # add relations to the issue relations = list() for rel in issue_x.getElementsByTagName('issuelinktype'): relation = dict() relation['relation'] = rel.getElementsByTagName( 'name')[0].firstChild.data if (rel.hasAttribute('inwardlinks')): left = rel.getElementsByTagName('inwardlinks') issuekeys = left.getElementsByTagName("issuekey") for key in issuekeys: relation['type'] = "inward" relation['id'] = key.firstChild.data relations.append(relation) if (rel.hasAttribute('outwardlinks')): right = rel.getElementsByTagName('outwardlinks') issuekeys = right.getElementsByTagName('issuekey') for key in issuekeys: relation['type'] = "outward" relation['id'] = key.firstChild.data relations.append(relation) issue["relations"] = relations issues.append(issue) log.debug("number of issues after parse_xml: '{}'".format(len(issues))) return issues