class ProcessSonar(object):
"""
handles all incoming requests
"""
def __init__(self, group, project):
"""
init process sonar class and set buffers accordingly
:param group: group the project belongs to
:param project: project name
"""
self.helper = Helper(group, project)
# check log folders existed, if not, create
self.helper.checkAllLogs()
self.helper.checkQProfileLogReq()
def getGitIssuesByState(self, state):
projectid = self.helper.getGitlabProjectIDByName(
self.helper.GITLAB_GROUP, self.helper.PLAIN_PROJECT,
self.helper.TOKEN)
if projectid == -1:
return []
issues = self.helper.getGitlabIssuesByState(projectid,
self.helper.TOKEN, state)
return issues
def getcategoryoverview(self):
"""
get an overview of all categories
:return:
"""
issues = self.helper.getIssuesAll()
if 'err' in issues:
return issues
# get all rules associate with quanlity profile
rules = []
rules.extend(self.helper.ruleswithdetail)
# store details
scores = self.helper.calTotalScoreAllCategory()
scores_rem = copy.deepcopy(scores)
scores_checked_Id = set()
for issue in issues:
ruleID = issue['rule']
ruleResult = filter(lambda r: r['key'] == ruleID, rules)
if len(ruleResult) > 0:
# deduct score
self.helper.deductscore(ruleID, scores_checked_Id, issue,
scores)
# cal percentage
percentage = self.helper.calPercentByScore(scores, scores_rem)
res = self.helper.jsonify(percentage)
return res
def getcategoryissues(self, main, sub):
"""
get issues for a specific category
:param main: main category
:param sub: subcategory
:return: issues in that category
"""
allissues = self.process(False, False)
if sub == "":
return allissues[main]
return allissues[main][sub]
def process(self, onlyDup, byAuthor):
"""
get issues from sonarqube, filter out irrelevant issues, reconstruct the remaining issues
according to the predefined 5 main categories and subcategories.
:param onlyDup: decide if only duplication issues are returned
:return: issues in the selected project
"""
whichCache = self.helper.LOG_ISSUES_GENERAL_DIR
if byAuthor:
whichCache = self.helper.LOG_ISSUES_AUTHOR_DIR
if onlyDup:
whichCache = self.helper.LOG_ISSUES_DUPLICATIONS_DIR
cachedissues = self.checkCached(whichCache)
if "NOT EXIST" in cachedissues:
return self.helper.jsonify({"err": cachedissues})
if not cachedissues == "NO CACHE":
return self.helper.jsonify(cachedissues)
# get all issues that are open
issues = self.helper.getIssuesAll()
if 'err' in issues:
return issues
# get all rules associate with quanlity profile
rules = []
if not onlyDup:
rules.extend(self.helper.getRulesWithDetail())
else:
rules.extend(self.helper.getDuplicationRulesShort())
# store details
dup_errmessages = []
scores = self.helper.calTotalScoreAllCategory()
scores_rem = copy.deepcopy(scores)
scores_checked_Id = set()
for issue in issues:
ruleID = issue['rule']
ruleResult = filter(lambda r: r['key'] == ruleID, rules)
if len(ruleResult) > 0:
errmessage = self.helper.makeErrMessage(issue, ruleResult)
# deduct score
self.helper.deductscore(ruleID, scores_checked_Id, issue,
scores)
# add code
if ruleID == "common-java:DuplicatedBlocks":
dup_errmessages.append(errmessage)
else:
errmessage['code'] = []
#self.helper.storeCodes(issue, errmessage)
self.helper.storeCodesBasic(issue, errmessage)
self.helper.storeIssue(ruleID, errmessage)
# if there is duplication issues, store the issues in separate buffer
if len(dup_errmessages) > 0:
self.helper.duplicatedBlockHandlerStore(dup_errmessages)
# cal percentage
percentage = self.helper.calPercentByScore(scores, scores_rem)
data = self.helper.dataHandler(percentage, onlyDup)
if byAuthor:
data = self.getissuebyauthor(data)
# store severity list
data['severitylist'] = self.helper.getSeverityList()
#store list of rules in DRY if duplications
if onlyDup:
data["rules"] = self.helper.getDuplicationRulesShort()
# if not only duplication, store the log
self.helper.checkAnalysisLog(whichCache, data)
if byAuthor:
self.helper.checkAnalysisLog(
self.helper.LOG_STATISTICS_AUTHOR_DIR,
self.helper.getNumIssuesAllAuthor(data))
res = self.helper.jsonify(data)
return res
def getissuebyauthor(self, data):
"""
re-arrange result by author
:return: re-arranged issues by author
"""
errors = data['error']
res = {}
for maincategory, possibleissues in errors.iteritems():
# if code smell or java note
if type(possibleissues) is list:
self.helper.handleAuthorStore(possibleissues, maincategory, "",
res)
# if other main categories
else:
for subcategory, issues in possibleissues.iteritems():
self.helper.handleAuthorStore(issues['detail'],
maincategory, subcategory,
res)
return res
def getcontributionsbyfile(self, file, start, end):
"""
get contribution by file
:param file: path to the file
:param start: start line
:param end: end line
:return: contributions
"""
data_list = self.helper.executeShellContributionByFile(
file, start, end).split("\n")
begin = True
res = {}
res['all'] = []
for data in data_list:
if begin:
begin = False
continue
nameBeginIndex = data.find(" (") + 2
nameEndIndex = data.find(") ")
content = data[nameBeginIndex:nameEndIndex].split(" ")
content = filter(None, content)
if len(content) == 0:
continue
res['all'].append(content)
fullName = content[0] + " " + content[1]
if fullName not in res:
res[fullName] = []
entry = {}
entry['date'] = content[2]
entry['time'] = content[3]
entry['line number'] = content[5]
res[fullName].append(entry)
return self.helper.jsonify(res)
def getcontributionsbyauthor(self):
"""
get contributions to classes by author
:return: contributions by author
"""
projectid = self.helper.getGitlabProjectIDByName(
self.helper.GITLAB_GROUP, self.helper.PLAIN_PROJECT,
self.helper.TOKEN)
if projectid == -1:
return []
# using project id to get commits
commits = self.helper.getCommits(projectid, self.helper.TOKEN)
res = {}
res["byfile"] = {}
res["byauthor"] = {}
for commit in commits:
author = commit["author_email"]
if author not in res["byauthor"]:
res["byauthor"][author] = {}
diffs = self.helper.getSingleCommitDiff(projectid,
self.helper.TOKEN,
commit["id"])
for diff in diffs:
path = diff["new_path"]
if path not in res["byfile"]:
res["byfile"][path] = {}
if author not in res["byfile"][path]:
res["byfile"][path][author] = 0
res["byfile"][path][author] += 1
if path not in res["byauthor"][author]:
res["byauthor"][author][path] = 0
res["byauthor"][author][path] += 1
self.helper.displayData(res)
return self.helper.jsonify(res)
def statistics(self):
"""
get the statistics of the project
:return: statistics in json
"""
measures = self.helper.getMeasuresReq()
res = {}
res['measures'] = {}
for measure in measures:
res['measures'][measure['metric']] = measure['value']
additionalStats = self.helper.executeShellStatsAdditional().split("\n")
begin = False
startLine = "meaningless catch:"
for line in additionalStats:
if line[:len(startLine)] == startLine:
begin = True
if begin:
extracts = line.split(":")
statname = extracts[0]
if len(extracts) > 1:
extracts_detail = extracts[1].split()
if not statname == startLine[:-1]:
res["measures"][statname] = extracts_detail[0]
else:
res["measures"][statname] = extracts_detail
#write logs to local
self.helper.checkAnalysisLog(self.helper.LOG_STATISTICS_GENERAL_DIR,
res)
return self.helper.jsonify(res)
def longestmethods(self):
"""
get the longest methods (at least more than 15 lines of codes)
:return: top 10 longest methods names that exceed 15 lines of codes
"""
total_pages = self.helper.\
getNumOfPagesIssuesReq() # get total number of pages in response
issues = self.helper.getIssuesReq(
total_pages, "squid:S138") # get issues with method too long
entries = [] # store result
count = 0 # number of entries
# start to iterating issues and store relevant fields
for issue in issues:
entries.append({})
entries[count]['methodlen'] = int(issue['message'].split()[3])
entries[count]['startline'] = issue['line']
entries[count]['path'] = issue['component']
# get codes from sonar
sl = issue['textRange']['startLine']
el = issue['textRange']['endLine']
items = self.helper.getSourceReq(sl, el, issue)
entries[count]['code'] = []
title = 0
# strip methond name and store codes
for item in items:
mname = ""
if title == 0:
mname = self.helper.stripmethodname(item[1])
entries[count]['methodname'] = mname
entries[count]['code'].append(item[1])
count += 1
# sort the result by method length
entries.sort(key=lambda x: x['methodlen'], reverse=True)
#make res
res = {}
res['method'] = []
res['method'].extend(entries[:10])
return self.helper.jsonify(res)
def getcommit(self, onlyStat):
"""
get commit information/statistics from gitlab
:param onlyStat: whether only statistics is returned
:return: information about commits or statistics about commits
"""
projectid = self.helper.getGitlabProjectIDByName(
self.helper.GITLAB_GROUP, self.helper.PLAIN_PROJECT,
self.helper.TOKEN)
if projectid == -1:
return []
# using project id to get commits
commits = self.helper.getCommits(projectid, self.helper.TOKEN)
res = {}
res['authors'] = {}
dates = {}
# read in student ids and names from csv
studentidmaps = self.helper.readStudentInfo()
# if only statistics is requested, get the statistics using the student id map
if onlyStat:
return self.getcommitstatfast(studentidmaps)
# iterating through all commits and stores relevant information
for commit in commits:
# retrieve gitlab id
authoremail = commit['author_email']
authorname = self.helper.convertEmailtoGitlabId(
authoremail, studentidmaps)
# get other info
commitdate = commit['committed_date']
commitid = commit['id']
if authorname not in res['authors']:
res['authors'][authorname] = {}
res['authors'][authorname]['commitlist'] = []
res['authors'][authorname]['commitdates'] = []
dates[authorname] = {}
entry = {}
entry['commitId'] = commitid
entry['date'] = commitdate
entry['files'] = []
res['authors'][authorname]['commitlist'].append(entry)
# handle date
shortdate = commitdate[:10]
curnumdates = len(res['authors'][authorname]['commitdates'])
if shortdate not in dates[authorname]:
dates[authorname][shortdate] = curnumdates
datesindex = dates[authorname][shortdate]
if datesindex == curnumdates:
entry = {}
entry[shortdate] = 1
res['authors'][authorname]['commitdates'].append(entry)
else:
res['authors'][authorname]['commitdates'][datesindex][
shortdate] += 1
totalnumofcommits = len(commits)
# sort commitsdates, lists and store other information
for author in res['authors']:
res['authors'][author]['commitdates'].sort(key=lambda x: x.keys(),
reverse=False)
res['authors'][author]['commitlist'].sort(key=lambda x: x['date'],
reverse=False)
numofcommits = len(res['authors'][author]['commitlist'])
res['authors'][author]['numofcommits'] = numofcommits
res['authors'][author][
'percentageofcommits'] = 100.00 * numofcommits / totalnumofcommits
#self.helper.displayData(res)
return self.helper.jsonify(res)
def getcommitstatfast(self, studentidmaps):
"""
get statistics info of commits
:param studentidmaps: store student ids and names
:return: statistics information of commits
"""
stats = self.helper.executeShellStats(
) # call shell script to get statistics information
parsed = re.split(r'\n--\n', stats) # delete empty lines
res = {} # result
res_dates = {} # store dates correspond to each author
current_author = "" # current author
converted_date = "" # dates after conversion
left_bound = self.helper.getDateFromTuple(
"2099 Dec 31") # initial left bound
right_bound = self.helper.getDateFromTuple(
"1999 Jan 1") #initial right bound
# start to iterating through parsed statistics
for p in parsed:
lines = p.split("\n")
# iterating each line
for line in lines:
# if keyword author is found in the line, store the author information
if "Author:" in line:
authorline = line.split()
authoremail = authorline[-1]
authoremail = authoremail[1:-1]
current_author = self.helper.convertEmailtoGitlabId(
authoremail, studentidmaps)
if current_author not in res:
res[current_author] = {}
res[current_author]["dates"] = {}
res[current_author]["total"] = {}
res[current_author]["total"]["files changed"] = 0
res[current_author]["total"]["insertions"] = 0
res[current_author]["total"]["deletions"] = 0
res_dates[current_author] = []
elif "Date:" in line:
#check bounds
dateline = line.split()
current_date = dateline[5] + " " + dateline[
2] + " " + dateline[3]
numerical_date = self.helper.getDateFromTuple(current_date)
if numerical_date > right_bound:
right_bound = numerical_date
elif numerical_date < left_bound:
left_bound = numerical_date
converted_date = numerical_date.strftime('%Y/%m/%d')
#modify res
if converted_date not in res_dates[current_author]:
res_dates[current_author].append(converted_date)
inentry = {}
inentry["files changed"] = 0
inentry["insertions"] = 0
inentry["deletions"] = 0
res[current_author]["dates"][converted_date] = inentry
elif "file changed," in line or "files changed," in line:
statsline = line.split(", ")
for stat in statsline:
key = ""
if "files changed" in stat or 'file changed' in stat:
key = "files changed"
elif "insertions" in stat or "insertion" in stat:
key = "insertions"
elif "deletions" in stat or "deletion" in stat:
key = "deletions"
statdata = stat.split()
res[current_author]["dates"][converted_date][
key] += int(statdata[0])
res[current_author]["total"][key] += int(statdata[0])
for authorname, v in res_dates.items():
res[authorname]["sorteddates"] = []
res[authorname]["sorteddates"].extend(res_dates[authorname])
projectdates = self.helper.readProjectDates(self.helper.PLAIN_PROJECT)
startdate = projectdates['STARTDATE']
enddate = projectdates['ENDDATE']
res["bounds"] = {"left": startdate, "right": enddate}
return self.helper.jsonify(res)
def getproject(self):
"""
check if project exist in sonar and gitlab
:return: string contains whether project exists
"""
res = {}
found_project = self.helper.getComponentsReq()
if 'errors' in found_project:
res['sonar'] = "not found"
else:
res['sonar'] = "found"
GITLAB_URL = "https://coursework.cs.duke.edu/api/v4"
URL = GITLAB_URL \
+ "/groups/" \
+ self.helper.GITLAB_GROUP \
+ "/projects?search="\
+ self.helper.PLAIN_PROJECT
r = requests.get(URL, headers={'PRIVATE-TOKEN': self.helper.TOKEN})
if len(r.json()) == 0:
res['gitlab'] = "not found"
else:
res['gitlab'] = "found"
return self.helper.jsonify(res)
def getbydirectory(self):
"""
return all issues corresponding to authors
:return: json contains the directories and files and issues in them
"""
res = json.loads(self.getproject())
if res['sonar'] == 'not found':
return json.dumps({})
res = {}
path = self.helper.CODES_PATH \
+ "/" \
+ self.helper.GITLAB_GROUP \
+ "/" \
+ self.helper.PLAIN_PROJECT
for root, subdirs, files in os.walk(path):
if "/.git/" in root or root[-4:] == ".git":
continue
rootshort = re.sub(path, "", root)
if rootshort == "":
rootshort = "."
if rootshort[0] == '/':
rootshort = rootshort[1:]
if self.helper.shouldSkipDir(rootshort, ["src"]):
continue
res[rootshort] = {}
res[rootshort]['directories'] = self.helper.getFullPath(
rootshort, subdirs)
res[rootshort]['files'] = self.helper.getFullPath(rootshort, files)
issues = json.loads(self.process(False, False))
for category, mainissuelist in issues['error'].items():
if category == "Duplications":
continue
if isinstance(mainissuelist, dict):
for subcategory, subissuelist in mainissuelist.items():
self.helper.makeIssueEntryForDIR(subissuelist['detail'],
res)
else:
self.helper.makeIssueEntryForDIR(mainissuelist, res)
return self.helper.jsonify(res)
def gethistory(self):
"""
get history of analysis
:return: history of statistics for each author and the whole project
"""
res = {}
res['general'] = self.helper.readLogJSONAll(
self.helper.LOG_STATISTICS_GENERAL_DIR)
res['author'] = self.helper.readLogJSONAll(
self.helper.LOG_STATISTICS_AUTHOR_DIR)
return self.helper.jsonify(res)
def getcodemaat(self):
"""
testing method
:return:
"""
return self.helper.executeShellRunCodeMaat()
def getcode(self, start, end, path):
"""
get code for given path
:param start:
:param end:
:param path:
:return: code for the given path
"""
# TODO add cache
ultimatePath = self.helper.SONAR_GROUP + self.helper.PLAIN_PROJECT + ":" + path
codes = self.helper.storeSingleCodeReq(start, end, ultimatePath)
res = {}
res['codes'] = codes
res['range'] = {}
res['range']['start'] = start
res['range']['end'] = end
res['path'] = path
return self.helper.jsonify(res)
def checkCached(self, whichCache):
"""
check whether the cache exists
:param whichCache: which cache to check
:return: cache or no cache
"""
cachedissues = {}
mostrecenttime = self.helper.getMostRecentAnalysisDateReq()
if "errors" in mostrecenttime:
return mostrecenttime["errors"]
mostrecenttime = self.helper.adjustSonarTime(mostrecenttime)
self.helper.readLogJSON(whichCache, mostrecenttime + ".json",
cachedissues)
if len(cachedissues) > 0:
return cachedissues.values()[0]
return "NO CACHE"
