def computeWhoopsies(self):
judgmentsByQid = judgments_by_qid(self.split.evals)
report = []
for qid, judgList in judgmentsByQid.items():
if len(judgList) > 1:
minGradeDocId = judgList[0].docId
maxGradeDocId = judgList[0].docId
minGrade = maxGrade = judgList[0].grade
for judg in judgList:
if judg.grade < minGrade:
minGrade = judg.grade
minGradeDocId = judg.docId
if judg.grade > maxGrade:
maxGrade = judg.grade
maxGradeDocId = judg.docId
if minGrade != maxGrade:
report.append(
Whoopsie(qid=qid,
judgList=judgList,
minGrade=minGrade,
maxGrade=maxGrade,
minGradeDocId=minGradeDocId,
maxGradeDocId=maxGradeDocId,
output=self.split.output))
report.sort(key=lambda x: x.maxGrade - x.minGrade, reverse=True)
self.whoopsies = report
def loadJudgments(judgFile):
currJudgments = []
existingKws = set()
lastQid = 0
try:
currJudgments = [judg for judg in judgments_from_file(judgFile)]
existingKws = set([judg.keywords for judg in currJudgments])
judgDict = judgments_by_qid(currJudgments)
judgProfile = []
for qid, judglist in judgDict.items():
judgProfile.append((judglist[0], len(judglist)))
judgProfile.sort(key=lambda j: j[1], reverse=True)
for prof in judgProfile:
print("%s has %s judgments" % (prof[0].keywords, prof[1]))
lastQid = currJudgments[-1].qid
except FileNotFoundError:
pass
return (currJudgments, existingKws, lastQid)