def countAgeMatches(self, aAgeTemplates, errorOut): """ count the number of age value matches in a set of annotated age templates """ annotatedAgeValues = { 'min': set([]), 'max': set([]), 'mean': set([]), 'median': set([]) } for template in aAgeTemplates: for type, avList in template.trueValues.items(): for av in avList: # print '@@@ ADDING AGE VALUE:', type, av.value # if len(annotatedAgeValues[type]) > 0: # print '-- Redundant value' annotatedAgeValues[type].add(av) self.nTrueAgeValues = 0 annotatedValueFound = {} for avSet in annotatedAgeValues.values(): for av in avSet: annotatedValueFound[av] = False self.nTrueAgeValues += 1 # count the number of detected values that match annotated ones stats = IRstats() for type, av in self.ageValues.items(): # print '@@@ Checking:', type, av.value if av.source != 'trial_registry': foundAgeValue = False for annotatedValue in annotatedAgeValues[type]: if av.value == annotatedValue.value: stats.incTP() errorOut.write(' +TP: %s = %d\n' % (type, av.value)) # print ' +TP: %s = %d' % (type,av.value) annotatedValueFound[annotatedValue] = True av.evaluation.markCorrect() foundAgeValue = True if foundAgeValue == False: stats.incFP() errorOut.write(' -FP: %s = %d\n' % (type, av.value)) # print ' -FP: %s = %d' % (type, av.value) av.evaluation.markIncorrect() # else: # print '@@@@ AGE VALUE SOURCE IS TRIAL REGISTRY' # count the ones that we missed for av, found in annotatedValueFound.items(): if found == False: stats.incFN() errorOut.write(' -FN: %s = %d\n' % (av.type, av.value)) # print ' -FN: %s = %d' % (av.type, av.value) return stats
def countAgeMatches(self, aAgeTemplates, errorOut): """ count the number of age value matches in a set of annotated age templates """ annotatedAgeValues = {'min':set([]), 'max':set([]), 'mean':set([]), 'median':set([])} for template in aAgeTemplates: for type, avList in template.trueValues.items(): for av in avList: # print '@@@ ADDING AGE VALUE:', type, av.value # if len(annotatedAgeValues[type]) > 0: # print '-- Redundant value' annotatedAgeValues[type].add(av) self.nTrueAgeValues = 0 annotatedValueFound = {} for avSet in annotatedAgeValues.values(): for av in avSet: annotatedValueFound[av] = False self.nTrueAgeValues += 1 # count the number of detected values that match annotated ones stats = IRstats() for type, av in self.ageValues.items(): # print '@@@ Checking:', type, av.value if av.source != 'trial_registry': foundAgeValue = False for annotatedValue in annotatedAgeValues[type]: if av.value == annotatedValue.value: stats.incTP() errorOut.write(' +TP: %s = %d\n' % (type,av.value)) # print ' +TP: %s = %d' % (type,av.value) annotatedValueFound[annotatedValue] = True av.evaluation.markCorrect() foundAgeValue = True if foundAgeValue == False: stats.incFP() errorOut.write(' -FP: %s = %d\n' % (type, av.value)) # print ' -FP: %s = %d' % (type, av.value) av.evaluation.markIncorrect() # else: # print '@@@@ AGE VALUE SOURCE IS TRIAL REGISTRY' # count the ones that we missed for av, found in annotatedValueFound.items(): if found == False: stats.incFN() errorOut.write(' -FN: %s = %d\n' % (av.type, av.value)) # print ' -FN: %s = %d' % (av.type, av.value) return stats
def computeStatistics(self, errorOut): """ Count RPF statistics for each unique AGE, CONDITION, POPULATION entity statOut = file stream for RPF stats for all parts of summarization system errorOut = file stream for TPs, FPs, FNs return hash of IRstats, one for each mention type, keyed by mention type """ stats = {} self.nTrueGroupSizes = 0 aAgeTemplates = createAnnotatedMergedList(self.abstract, 'age') errorOut.write('age:\n') stats['age'] = self.ageInfo.countAgeMatches(aAgeTemplates, errorOut) errorOut.write('condition:\n') aConditionTemplates = self.abstract.annotatedEntities.getList('condition') stats['condition'] = countMatches(aConditionTemplates, \ self.conditionTemplates, errorOut) errorOut.write('group:\n') aGroupTemplates = self.abstract.annotatedEntities.getList('group') stats['group'] = countMatches(aGroupTemplates, self.groupTemplates, errorOut) self.nTrueConditions = len(aConditionTemplates) self.nTrueGroups = len(aGroupTemplates) errorOut.write('group size:\n') gsStats = IRstats() gsFound = set([]) for gTemplate in self.groupTemplates: gSize = gTemplate.getSize(maxSize=True) if gSize != 0: # look for group size match in sizes for annotated group found = False if gTemplate.matchedTemplate != None: for trueGSize in gTemplate.matchedTemplate.sizes: if gSize == trueGSize.value: found = True break if found: # group size is correct gsStats.incTP() errorOut.write(' +TP: %s size = %d\n' % (gTemplate.name, gSize)) gTemplate.groupSizeEvaluation.markCorrect() gsFound.add(gTemplate.matchedTemplate) else: # group size is incorrect gsStats.incFP() errorOut.write(' -FP: %s size = %d\n' % (gTemplate.name, gSize)) gTemplate.groupSizeEvaluation.markIncorrect() # look for false negatives for trueTemplate in aGroupTemplates: if trueTemplate not in gsFound and trueTemplate.matchedTemplate != None and trueTemplate.getSize() > 0: # there should be a group size for this group gsStats.incFN() errorOut.write(' -FN: %s size = %d\n' % \ (trueTemplate.name, trueTemplate.getSize())) stats['group size'] = gsStats self.nTrueGroupSizes = gsStats.tp + gsStats.fn # errorOut.write('population:\n') # templates = createAnnotatedMergedList(self.abstract, 'population') # aPopulationTemplates = [] # for pTemplate in templates: # if pTemplate.isInteresting() > 0: # # term is informative, keep it # aPopulationTemplates.append(pTemplate) # stats['population'] = self.countMatches(aPopulationTemplates, \ # self.populationTemplates, errorOut) return stats