def computeRates(self,alpha=0.05): ''' Populates the distance matrix with more information such as recognition rates for each row. Call this only after all of the data has been added. ''' self.row_headers.sort() self.col_headers.sort() for row in self.classes: successes = 0 total = 0 for col in self.classes: total += self.element(row,col) if row == col: successes += self.element(row,col) rate = float(successes)/total self.setData(row,'Rate',rate) self.setData(row,'Bar',"#"*int(10*rate+0.5)) self.setData(row,'Lower',cibinom(total,successes,alpha)[0]) self.setData(row,'Upper',cibinom(total,successes,alpha)[1]) for col in self.classes: successes = 0 total = 0 for row in self.classes: total += self.element(row,col) if row == col: successes += self.element(row,col) rate = float(successes)/total self.setData('Total',col,"%0.4f"%rate) self.setData('Total','Rate',self.update_rate()) self.setData('Total','Bar',"#"*int(10*self.update_rate()+0.5)) self.setData('Total','Lower',self.confidenceInterval(alpha)[0]) self.setData('Total','Upper',self.confidenceInterval(alpha)[1])
def computeRates(self, alpha=0.05): ''' Populates the distance matrix with more information such as recognition rates for each row. Call this only after all of the data has been added. ''' self.row_headers.sort() self.col_headers.sort() for row in self.classes: successes = 0 total = 0 for col in self.classes: total += self.element(row, col) if row == col: successes += self.element(row, col) rate = float(successes) / total self.setData(row, 'Rate', rate) self.setData(row, 'Bar', "#" * int(10 * rate + 0.5)) self.setData(row, 'Lower', cibinom(total, successes, alpha)[0]) self.setData(row, 'Upper', cibinom(total, successes, alpha)[1]) for col in self.classes: successes = 0 total = 0 for row in self.classes: total += self.element(row, col) if row == col: successes += self.element(row, col) rate = float(successes) / total self.setData('Total', col, "%0.4f" % rate) self.setData('Total', 'Rate', self.update_rate()) self.setData('Total', 'Bar', "#" * int(10 * self.update_rate() + 0.5)) self.setData('Total', 'Lower', self.confidenceInterval(alpha)[0]) self.setData('Total', 'Upper', self.confidenceInterval(alpha)[1])
def addSample(self,probe_id,scores): ''' Adds a sample to the test. The similarity scores is a list of tuples that contain tuples of (gallery_id, score) ''' comp_func = lambda a,b: (a > b) - (a < b) if self.score_type == SCORE_TYPE_LOW: scores.sort( lambda x,y: comp_func(x[1],y[1]) ) elif self.score_type == SCORE_TYPE_HIGH: scores.sort( lambda x,y: -comp_func(x[1],y[1]) ) else: raise ValueError("Unknown score type: %s"%self.score_type) best_match_id = None best_match_score = None best_match_rank = None best_nonmatch_id = None best_nonmatch_score = None best_nonmatch_rank = None pid = self.id_func(probe_id) rank = 0 for gallery_id,score in scores: if probe_id == gallery_id: # Probe and gallery should not be the same image continue gid = self.id_func(gallery_id) match = False if pid == gid: match = True if match and best_match_id == None: best_match_id = gallery_id best_match_score = score best_match_rank = rank if not match and best_nonmatch_id == None: best_nonmatch_id = gallery_id best_nonmatch_score = score best_nonmatch_rank = rank if match: self.positives.append(score) else: self.negatives.append(score) rank += 1 self.total_probes += 1 self.total_gallery = max(self.total_gallery,rank) if best_match_rank==0: self.rank1_success += 1 self.probes_table.setElement(probe_id,'MatchId',best_match_id) self.probes_table.setElement(probe_id,'MatchScore',best_match_score) self.probes_table.setElement(probe_id,'MatchRank',best_match_rank) self.probes_table.setElement(probe_id,'Success',best_match_rank==0) self.probes_table.setElement(probe_id,'NonMatchId',best_nonmatch_id) self.probes_table.setElement(probe_id,'NonMatchScore',best_nonmatch_score) self.probes_table.setElement(probe_id,'NonMatchRank',best_nonmatch_rank) self.rank1_rate = float(self.rank1_success)/float(self.total_probes) self.rank1_bounds = cibinom(self.total_probes,self.rank1_success)
def addSample(self, truth_rects, detected_rects, im=None, annotate=False): ''' Adds a sample to face detection test. Input: truth_rects - truth for an image. detected_rects - output of the detector im - the image or filename to assciate with the sample. annotate - add diagnostic annotations to the images. ''' self.images += 1 name = None detected_rects = copy.copy(detected_rects) if isinstance(im,Image): name = im.filename if self.pixels != None: self.pixels += im.asPIL().size[0] * im.asPIL().size[1] elif isinstance(im,str): name = im self.pixels = None else: name = "%d"%self.sample_id self.pixels = None table = self.table for i in range(len(truth_rects)): truth = truth_rects[i] self.positives += 1 success = False best_overlap = 0.0 best_detection = None for j in range(len(detected_rects)): detected = detected_rects[j] overlap = overlap_score(truth,detected) if overlap >= self.threshold and overlap > best_overlap: success = True best_overlap = overlap best_detection = j table.setData(self.sample_id,'id',self.sample_id) table.setData(self.sample_id,'name',name) table.setData(self.sample_id,'truth_rect',str(truth)) if best_detection != None: table.setData(self.sample_id,'detection_rect',str(detected_rects[best_detection])) else: table.setData(self.sample_id,'detection_rect',None) table.setData(self.sample_id,'success',success) table.setData(self.sample_id,'overlap',best_overlap) self.sample_id+=1 if success: self.successes += 1 if annotate and isinstance(im,Image): if success: im.annotateEllipse(truth,color='green') else: im.annotateEllipse(truth,color='red') if best_detecion != None: im.annototeRect(detected_rects[best_detection],color='green') # Remove the best detection if success if best_detection != None: del detected_rects[best_detection] if annotate: for each in detected_rects: im.annotateRect(each,color='red') self.negatives += len(detected_rects) self.end_time = time.time() self.total_time = self.end_time - self.start_time self.image_time = self.total_time/float(self.images) # Update summary statistics if self.positives > 0: self.pos_rate = float(self.successes)/self.positives self.pos_bounds = cibinom(self.positives,self.successes,alpha=0.05) if self.pixels != None: self.neg_rate = float(self.negatives)/float(1.0e-6*self.pixels) self.createSummary()
def confidenceInterval(self,alpha=0.05): ''' Returns the estimated a confidence interval for the success update_rate by modeling the success update_rate as a binomial distribution. ''' return cibinom(self.total,self.successes,alpha=alpha)
def addSample(self, probe_id, scores): ''' Adds a sample to the test. The similarity scores is a list of tuples that contain tuples of (gallery_id, score) ''' comp_func = lambda a, b: (a > b) - (a < b) if self.score_type == SCORE_TYPE_LOW: scores.sort(lambda x, y: comp_func(x[1], y[1])) elif self.score_type == SCORE_TYPE_HIGH: scores.sort(lambda x, y: -comp_func(x[1], y[1])) else: raise ValueError("Unknown score type: %s" % self.score_type) best_match_id = None best_match_score = None best_match_rank = None best_nonmatch_id = None best_nonmatch_score = None best_nonmatch_rank = None pid = self.id_func(probe_id) rank = 0 for gallery_id, score in scores: if probe_id == gallery_id: # Probe and gallery should not be the same image continue gid = self.id_func(gallery_id) match = False if pid == gid: match = True if match and best_match_id == None: best_match_id = gallery_id best_match_score = score best_match_rank = rank if not match and best_nonmatch_id == None: best_nonmatch_id = gallery_id best_nonmatch_score = score best_nonmatch_rank = rank if match: self.positives.append(score) else: self.negatives.append(score) rank += 1 self.total_probes += 1 self.total_gallery = max(self.total_gallery, rank) if best_match_rank == 0: self.rank1_success += 1 self.probes_table.setElement(probe_id, 'MatchId', best_match_id) self.probes_table.setElement(probe_id, 'MatchScore', best_match_score) self.probes_table.setElement(probe_id, 'MatchRank', best_match_rank) self.probes_table.setElement(probe_id, 'Success', best_match_rank == 0) self.probes_table.setElement(probe_id, 'NonMatchId', best_nonmatch_id) self.probes_table.setElement(probe_id, 'NonMatchScore', best_nonmatch_score) self.probes_table.setElement(probe_id, 'NonMatchRank', best_nonmatch_rank) self.rank1_rate = float(self.rank1_success) / float(self.total_probes) self.rank1_bounds = cibinom(self.total_probes, self.rank1_success)
def addSample(self, truth_rects, detected_rects, im=None, annotate=False): ''' Adds a sample to face detection test. @param truth_rects: truth for an image. @param detected_rects: output of the detector @param im: the image or filename to assciate with the sample. @param annotate: add diagnostic annotations to the images. ''' self.images += 1 name = None detected_rects = copy.copy(detected_rects) if isinstance(im,Image): name = im.filename if self.pixels != None: self.pixels += im.asPIL().size[0] * im.asPIL().size[1] elif isinstance(im,str): name = im self.pixels = None else: name = "%d"%self.sample_id self.pixels = None table = self.table for i in range(len(truth_rects)): truth = truth_rects[i] self.positives += 1 success = False best_overlap = 0.0 best_detection = None for j in range(len(detected_rects)): detected = detected_rects[j] overlap = overlap_score(truth,detected) if overlap >= self.threshold and overlap > best_overlap: success = True best_overlap = overlap best_detection = j table.setData(self.sample_id,'id',self.sample_id) table.setData(self.sample_id,'name',name) table.setData(self.sample_id,'truth_rect',str(truth)) if best_detection != None: table.setData(self.sample_id,'detection_rect',str(detected_rects[best_detection])) else: table.setData(self.sample_id,'detection_rect',None) table.setData(self.sample_id,'success',success) table.setData(self.sample_id,'overlap',best_overlap) self.sample_id+=1 if success: self.successes += 1 if annotate and isinstance(im,Image): if success: im.annotateEllipse(truth,color='green') else: im.annotateEllipse(truth,color='red') if best_detecion != None: im.annototeRect(detected_rects[best_detection],color='green') # Remove the best detection if success if best_detection != None: del detected_rects[best_detection] if annotate: for each in detected_rects: im.annotateRect(each,color='red') self.negatives += len(detected_rects) self.end_time = time.time() self.total_time = self.end_time - self.start_time self.image_time = self.total_time/float(self.images) # Update summary statistics if self.positives > 0: self.pos_rate = float(self.successes)/self.positives self.pos_bounds = cibinom(self.positives,self.successes,alpha=0.05) if self.pixels != None: self.neg_rate = float(self.negatives)/float(1.0e-6*self.pixels) self.createSummary()
def confidenceInterval(self, alpha=0.05): ''' Returns the estimated a confidence interval for the success update_rate by modeling the success update_rate as a binomial distribution. ''' return cibinom(self.total, self.successes, alpha=alpha)