class ProcessSonar(object): """ handles all incoming requests """ def __init__(self, group, project): """ init process sonar class and set buffers accordingly :param group: group the project belongs to :param project: project name """ self.helper = Helper(group, project) # check log folders existed, if not, create self.helper.checkAllLogs() self.helper.checkQProfileLogReq() def getGitIssuesByState(self, state): projectid = self.helper.getGitlabProjectIDByName( self.helper.GITLAB_GROUP, self.helper.PLAIN_PROJECT, self.helper.TOKEN) if projectid == -1: return [] issues = self.helper.getGitlabIssuesByState(projectid, self.helper.TOKEN, state) return issues def getcategoryoverview(self): """ get an overview of all categories :return: """ issues = self.helper.getIssuesAll() if 'err' in issues: return issues # get all rules associate with quanlity profile rules = [] rules.extend(self.helper.ruleswithdetail) # store details scores = self.helper.calTotalScoreAllCategory() scores_rem = copy.deepcopy(scores) scores_checked_Id = set() for issue in issues: ruleID = issue['rule'] ruleResult = filter(lambda r: r['key'] == ruleID, rules) if len(ruleResult) > 0: # deduct score self.helper.deductscore(ruleID, scores_checked_Id, issue, scores) # cal percentage percentage = self.helper.calPercentByScore(scores, scores_rem) res = self.helper.jsonify(percentage) return res def getcategoryissues(self, main, sub): """ get issues for a specific category :param main: main category :param sub: subcategory :return: issues in that category """ allissues = self.process(False, False) if sub == "": return allissues[main] return allissues[main][sub] def process(self, onlyDup, byAuthor): """ get issues from sonarqube, filter out irrelevant issues, reconstruct the remaining issues according to the predefined 5 main categories and subcategories. :param onlyDup: decide if only duplication issues are returned :return: issues in the selected project """ whichCache = self.helper.LOG_ISSUES_GENERAL_DIR if byAuthor: whichCache = self.helper.LOG_ISSUES_AUTHOR_DIR if onlyDup: whichCache = self.helper.LOG_ISSUES_DUPLICATIONS_DIR cachedissues = self.checkCached(whichCache) if "NOT EXIST" in cachedissues: return self.helper.jsonify({"err": cachedissues}) if not cachedissues == "NO CACHE": return self.helper.jsonify(cachedissues) # get all issues that are open issues = self.helper.getIssuesAll() if 'err' in issues: return issues # get all rules associate with quanlity profile rules = [] if not onlyDup: rules.extend(self.helper.getRulesWithDetail()) else: rules.extend(self.helper.getDuplicationRulesShort()) # store details dup_errmessages = [] scores = self.helper.calTotalScoreAllCategory() scores_rem = copy.deepcopy(scores) scores_checked_Id = set() for issue in issues: ruleID = issue['rule'] ruleResult = filter(lambda r: r['key'] == ruleID, rules) if len(ruleResult) > 0: errmessage = self.helper.makeErrMessage(issue, ruleResult) # deduct score self.helper.deductscore(ruleID, scores_checked_Id, issue, scores) # add code if ruleID == "common-java:DuplicatedBlocks": dup_errmessages.append(errmessage) else: errmessage['code'] = [] #self.helper.storeCodes(issue, errmessage) self.helper.storeCodesBasic(issue, errmessage) self.helper.storeIssue(ruleID, errmessage) # if there is duplication issues, store the issues in separate buffer if len(dup_errmessages) > 0: self.helper.duplicatedBlockHandlerStore(dup_errmessages) # cal percentage percentage = self.helper.calPercentByScore(scores, scores_rem) data = self.helper.dataHandler(percentage, onlyDup) if byAuthor: data = self.getissuebyauthor(data) # store severity list data['severitylist'] = self.helper.getSeverityList() #store list of rules in DRY if duplications if onlyDup: data["rules"] = self.helper.getDuplicationRulesShort() # if not only duplication, store the log self.helper.checkAnalysisLog(whichCache, data) if byAuthor: self.helper.checkAnalysisLog( self.helper.LOG_STATISTICS_AUTHOR_DIR, self.helper.getNumIssuesAllAuthor(data)) res = self.helper.jsonify(data) return res def getissuebyauthor(self, data): """ re-arrange result by author :return: re-arranged issues by author """ errors = data['error'] res = {} for maincategory, possibleissues in errors.iteritems(): # if code smell or java note if type(possibleissues) is list: self.helper.handleAuthorStore(possibleissues, maincategory, "", res) # if other main categories else: for subcategory, issues in possibleissues.iteritems(): self.helper.handleAuthorStore(issues['detail'], maincategory, subcategory, res) return res def getcontributionsbyfile(self, file, start, end): """ get contribution by file :param file: path to the file :param start: start line :param end: end line :return: contributions """ data_list = self.helper.executeShellContributionByFile( file, start, end).split("\n") begin = True res = {} res['all'] = [] for data in data_list: if begin: begin = False continue nameBeginIndex = data.find(" (") + 2 nameEndIndex = data.find(") ") content = data[nameBeginIndex:nameEndIndex].split(" ") content = filter(None, content) if len(content) == 0: continue res['all'].append(content) fullName = content[0] + " " + content[1] if fullName not in res: res[fullName] = [] entry = {} entry['date'] = content[2] entry['time'] = content[3] entry['line number'] = content[5] res[fullName].append(entry) return self.helper.jsonify(res) def getcontributionsbyauthor(self): """ get contributions to classes by author :return: contributions by author """ projectid = self.helper.getGitlabProjectIDByName( self.helper.GITLAB_GROUP, self.helper.PLAIN_PROJECT, self.helper.TOKEN) if projectid == -1: return [] # using project id to get commits commits = self.helper.getCommits(projectid, self.helper.TOKEN) res = {} res["byfile"] = {} res["byauthor"] = {} for commit in commits: author = commit["author_email"] if author not in res["byauthor"]: res["byauthor"][author] = {} diffs = self.helper.getSingleCommitDiff(projectid, self.helper.TOKEN, commit["id"]) for diff in diffs: path = diff["new_path"] if path not in res["byfile"]: res["byfile"][path] = {} if author not in res["byfile"][path]: res["byfile"][path][author] = 0 res["byfile"][path][author] += 1 if path not in res["byauthor"][author]: res["byauthor"][author][path] = 0 res["byauthor"][author][path] += 1 self.helper.displayData(res) return self.helper.jsonify(res) def statistics(self): """ get the statistics of the project :return: statistics in json """ measures = self.helper.getMeasuresReq() res = {} res['measures'] = {} for measure in measures: res['measures'][measure['metric']] = measure['value'] additionalStats = self.helper.executeShellStatsAdditional().split("\n") begin = False startLine = "meaningless catch:" for line in additionalStats: if line[:len(startLine)] == startLine: begin = True if begin: extracts = line.split(":") statname = extracts[0] if len(extracts) > 1: extracts_detail = extracts[1].split() if not statname == startLine[:-1]: res["measures"][statname] = extracts_detail[0] else: res["measures"][statname] = extracts_detail #write logs to local self.helper.checkAnalysisLog(self.helper.LOG_STATISTICS_GENERAL_DIR, res) return self.helper.jsonify(res) def longestmethods(self): """ get the longest methods (at least more than 15 lines of codes) :return: top 10 longest methods names that exceed 15 lines of codes """ total_pages = self.helper.\ getNumOfPagesIssuesReq() # get total number of pages in response issues = self.helper.getIssuesReq( total_pages, "squid:S138") # get issues with method too long entries = [] # store result count = 0 # number of entries # start to iterating issues and store relevant fields for issue in issues: entries.append({}) entries[count]['methodlen'] = int(issue['message'].split()[3]) entries[count]['startline'] = issue['line'] entries[count]['path'] = issue['component'] # get codes from sonar sl = issue['textRange']['startLine'] el = issue['textRange']['endLine'] items = self.helper.getSourceReq(sl, el, issue) entries[count]['code'] = [] title = 0 # strip methond name and store codes for item in items: mname = "" if title == 0: mname = self.helper.stripmethodname(item[1]) entries[count]['methodname'] = mname entries[count]['code'].append(item[1]) count += 1 # sort the result by method length entries.sort(key=lambda x: x['methodlen'], reverse=True) #make res res = {} res['method'] = [] res['method'].extend(entries[:10]) return self.helper.jsonify(res) def getcommit(self, onlyStat): """ get commit information/statistics from gitlab :param onlyStat: whether only statistics is returned :return: information about commits or statistics about commits """ projectid = self.helper.getGitlabProjectIDByName( self.helper.GITLAB_GROUP, self.helper.PLAIN_PROJECT, self.helper.TOKEN) if projectid == -1: return [] # using project id to get commits commits = self.helper.getCommits(projectid, self.helper.TOKEN) res = {} res['authors'] = {} dates = {} # read in student ids and names from csv studentidmaps = self.helper.readStudentInfo() # if only statistics is requested, get the statistics using the student id map if onlyStat: return self.getcommitstatfast(studentidmaps) # iterating through all commits and stores relevant information for commit in commits: # retrieve gitlab id authoremail = commit['author_email'] authorname = self.helper.convertEmailtoGitlabId( authoremail, studentidmaps) # get other info commitdate = commit['committed_date'] commitid = commit['id'] if authorname not in res['authors']: res['authors'][authorname] = {} res['authors'][authorname]['commitlist'] = [] res['authors'][authorname]['commitdates'] = [] dates[authorname] = {} entry = {} entry['commitId'] = commitid entry['date'] = commitdate entry['files'] = [] res['authors'][authorname]['commitlist'].append(entry) # handle date shortdate = commitdate[:10] curnumdates = len(res['authors'][authorname]['commitdates']) if shortdate not in dates[authorname]: dates[authorname][shortdate] = curnumdates datesindex = dates[authorname][shortdate] if datesindex == curnumdates: entry = {} entry[shortdate] = 1 res['authors'][authorname]['commitdates'].append(entry) else: res['authors'][authorname]['commitdates'][datesindex][ shortdate] += 1 totalnumofcommits = len(commits) # sort commitsdates, lists and store other information for author in res['authors']: res['authors'][author]['commitdates'].sort(key=lambda x: x.keys(), reverse=False) res['authors'][author]['commitlist'].sort(key=lambda x: x['date'], reverse=False) numofcommits = len(res['authors'][author]['commitlist']) res['authors'][author]['numofcommits'] = numofcommits res['authors'][author][ 'percentageofcommits'] = 100.00 * numofcommits / totalnumofcommits #self.helper.displayData(res) return self.helper.jsonify(res) def getcommitstatfast(self, studentidmaps): """ get statistics info of commits :param studentidmaps: store student ids and names :return: statistics information of commits """ stats = self.helper.executeShellStats( ) # call shell script to get statistics information parsed = re.split(r'\n--\n', stats) # delete empty lines res = {} # result res_dates = {} # store dates correspond to each author current_author = "" # current author converted_date = "" # dates after conversion left_bound = self.helper.getDateFromTuple( "2099 Dec 31") # initial left bound right_bound = self.helper.getDateFromTuple( "1999 Jan 1") #initial right bound # start to iterating through parsed statistics for p in parsed: lines = p.split("\n") # iterating each line for line in lines: # if keyword author is found in the line, store the author information if "Author:" in line: authorline = line.split() authoremail = authorline[-1] authoremail = authoremail[1:-1] current_author = self.helper.convertEmailtoGitlabId( authoremail, studentidmaps) if current_author not in res: res[current_author] = {} res[current_author]["dates"] = {} res[current_author]["total"] = {} res[current_author]["total"]["files changed"] = 0 res[current_author]["total"]["insertions"] = 0 res[current_author]["total"]["deletions"] = 0 res_dates[current_author] = [] elif "Date:" in line: #check bounds dateline = line.split() current_date = dateline[5] + " " + dateline[ 2] + " " + dateline[3] numerical_date = self.helper.getDateFromTuple(current_date) if numerical_date > right_bound: right_bound = numerical_date elif numerical_date < left_bound: left_bound = numerical_date converted_date = numerical_date.strftime('%Y/%m/%d') #modify res if converted_date not in res_dates[current_author]: res_dates[current_author].append(converted_date) inentry = {} inentry["files changed"] = 0 inentry["insertions"] = 0 inentry["deletions"] = 0 res[current_author]["dates"][converted_date] = inentry elif "file changed," in line or "files changed," in line: statsline = line.split(", ") for stat in statsline: key = "" if "files changed" in stat or 'file changed' in stat: key = "files changed" elif "insertions" in stat or "insertion" in stat: key = "insertions" elif "deletions" in stat or "deletion" in stat: key = "deletions" statdata = stat.split() res[current_author]["dates"][converted_date][ key] += int(statdata[0]) res[current_author]["total"][key] += int(statdata[0]) for authorname, v in res_dates.items(): res[authorname]["sorteddates"] = [] res[authorname]["sorteddates"].extend(res_dates[authorname]) projectdates = self.helper.readProjectDates(self.helper.PLAIN_PROJECT) startdate = projectdates['STARTDATE'] enddate = projectdates['ENDDATE'] res["bounds"] = {"left": startdate, "right": enddate} return self.helper.jsonify(res) def getproject(self): """ check if project exist in sonar and gitlab :return: string contains whether project exists """ res = {} found_project = self.helper.getComponentsReq() if 'errors' in found_project: res['sonar'] = "not found" else: res['sonar'] = "found" GITLAB_URL = "https://coursework.cs.duke.edu/api/v4" URL = GITLAB_URL \ + "/groups/" \ + self.helper.GITLAB_GROUP \ + "/projects?search="\ + self.helper.PLAIN_PROJECT r = requests.get(URL, headers={'PRIVATE-TOKEN': self.helper.TOKEN}) if len(r.json()) == 0: res['gitlab'] = "not found" else: res['gitlab'] = "found" return self.helper.jsonify(res) def getbydirectory(self): """ return all issues corresponding to authors :return: json contains the directories and files and issues in them """ res = json.loads(self.getproject()) if res['sonar'] == 'not found': return json.dumps({}) res = {} path = self.helper.CODES_PATH \ + "/" \ + self.helper.GITLAB_GROUP \ + "/" \ + self.helper.PLAIN_PROJECT for root, subdirs, files in os.walk(path): if "/.git/" in root or root[-4:] == ".git": continue rootshort = re.sub(path, "", root) if rootshort == "": rootshort = "." if rootshort[0] == '/': rootshort = rootshort[1:] if self.helper.shouldSkipDir(rootshort, ["src"]): continue res[rootshort] = {} res[rootshort]['directories'] = self.helper.getFullPath( rootshort, subdirs) res[rootshort]['files'] = self.helper.getFullPath(rootshort, files) issues = json.loads(self.process(False, False)) for category, mainissuelist in issues['error'].items(): if category == "Duplications": continue if isinstance(mainissuelist, dict): for subcategory, subissuelist in mainissuelist.items(): self.helper.makeIssueEntryForDIR(subissuelist['detail'], res) else: self.helper.makeIssueEntryForDIR(mainissuelist, res) return self.helper.jsonify(res) def gethistory(self): """ get history of analysis :return: history of statistics for each author and the whole project """ res = {} res['general'] = self.helper.readLogJSONAll( self.helper.LOG_STATISTICS_GENERAL_DIR) res['author'] = self.helper.readLogJSONAll( self.helper.LOG_STATISTICS_AUTHOR_DIR) return self.helper.jsonify(res) def getcodemaat(self): """ testing method :return: """ return self.helper.executeShellRunCodeMaat() def getcode(self, start, end, path): """ get code for given path :param start: :param end: :param path: :return: code for the given path """ # TODO add cache ultimatePath = self.helper.SONAR_GROUP + self.helper.PLAIN_PROJECT + ":" + path codes = self.helper.storeSingleCodeReq(start, end, ultimatePath) res = {} res['codes'] = codes res['range'] = {} res['range']['start'] = start res['range']['end'] = end res['path'] = path return self.helper.jsonify(res) def checkCached(self, whichCache): """ check whether the cache exists :param whichCache: which cache to check :return: cache or no cache """ cachedissues = {} mostrecenttime = self.helper.getMostRecentAnalysisDateReq() if "errors" in mostrecenttime: return mostrecenttime["errors"] mostrecenttime = self.helper.adjustSonarTime(mostrecenttime) self.helper.readLogJSON(whichCache, mostrecenttime + ".json", cachedissues) if len(cachedissues) > 0: return cachedissues.values()[0] return "NO CACHE"