def computeRates(self,alpha=0.05):
'''
Populates the distance matrix with more information such as
recognition rates for each row. Call this only after all of the
data has been added.
'''
self.row_headers.sort()
self.col_headers.sort()
for row in self.classes:
successes = 0
total = 0
for col in self.classes:
total += self.element(row,col)
if row == col:
successes += self.element(row,col)
rate = float(successes)/total
self.setData(row,'Rate',rate)
self.setData(row,'Bar',"#"*int(10*rate+0.5))
self.setData(row,'Lower',cibinom(total,successes,alpha)[0])
self.setData(row,'Upper',cibinom(total,successes,alpha)[1])
for col in self.classes:
successes = 0
total = 0
for row in self.classes:
total += self.element(row,col)
if row == col:
successes += self.element(row,col)
rate = float(successes)/total
self.setData('Total',col,"%0.4f"%rate)
self.setData('Total','Rate',self.update_rate())
self.setData('Total','Bar',"#"*int(10*self.update_rate()+0.5))
self.setData('Total','Lower',self.confidenceInterval(alpha)[0])
self.setData('Total','Upper',self.confidenceInterval(alpha)[1])
def computeRates(self, alpha=0.05):
'''
Populates the distance matrix with more information such as
recognition rates for each row. Call this only after all of the
data has been added.
'''
self.row_headers.sort()
self.col_headers.sort()
for row in self.classes:
successes = 0
total = 0
for col in self.classes:
total += self.element(row, col)
if row == col:
successes += self.element(row, col)
rate = float(successes) / total
self.setData(row, 'Rate', rate)
self.setData(row, 'Bar', "#" * int(10 * rate + 0.5))
self.setData(row, 'Lower', cibinom(total, successes, alpha)[0])
self.setData(row, 'Upper', cibinom(total, successes, alpha)[1])
for col in self.classes:
successes = 0
total = 0
for row in self.classes:
total += self.element(row, col)
if row == col:
successes += self.element(row, col)
rate = float(successes) / total
self.setData('Total', col, "%0.4f" % rate)
self.setData('Total', 'Rate', self.update_rate())
self.setData('Total', 'Bar', "#" * int(10 * self.update_rate() + 0.5))
self.setData('Total', 'Lower', self.confidenceInterval(alpha)[0])
self.setData('Total', 'Upper', self.confidenceInterval(alpha)[1])
def addSample(self,probe_id,scores):
'''
Adds a sample to the test. The similarity scores is a list of
tuples that contain tuples of (gallery_id, score)
'''
comp_func = lambda a,b: (a > b) - (a < b)
if self.score_type == SCORE_TYPE_LOW:
scores.sort( lambda x,y: comp_func(x[1],y[1]) )
elif self.score_type == SCORE_TYPE_HIGH:
scores.sort( lambda x,y: -comp_func(x[1],y[1]) )
else:
raise ValueError("Unknown score type: %s"%self.score_type)
best_match_id = None
best_match_score = None
best_match_rank = None
best_nonmatch_id = None
best_nonmatch_score = None
best_nonmatch_rank = None
pid = self.id_func(probe_id)
rank = 0
for gallery_id,score in scores:
if probe_id == gallery_id:
# Probe and gallery should not be the same image
continue
gid = self.id_func(gallery_id)
match = False
if pid == gid:
match = True
if match and best_match_id == None:
best_match_id = gallery_id
best_match_score = score
best_match_rank = rank
if not match and best_nonmatch_id == None:
best_nonmatch_id = gallery_id
best_nonmatch_score = score
best_nonmatch_rank = rank
if match:
self.positives.append(score)
else:
self.negatives.append(score)
rank += 1
self.total_probes += 1
self.total_gallery = max(self.total_gallery,rank)
if best_match_rank==0:
self.rank1_success += 1
self.probes_table.setElement(probe_id,'MatchId',best_match_id)
self.probes_table.setElement(probe_id,'MatchScore',best_match_score)
self.probes_table.setElement(probe_id,'MatchRank',best_match_rank)
self.probes_table.setElement(probe_id,'Success',best_match_rank==0)
self.probes_table.setElement(probe_id,'NonMatchId',best_nonmatch_id)
self.probes_table.setElement(probe_id,'NonMatchScore',best_nonmatch_score)
self.probes_table.setElement(probe_id,'NonMatchRank',best_nonmatch_rank)
self.rank1_rate = float(self.rank1_success)/float(self.total_probes)
self.rank1_bounds = cibinom(self.total_probes,self.rank1_success)
def addSample(self, truth_rects, detected_rects, im=None, annotate=False):
'''
Adds a sample to face detection test.
Input:
truth_rects - truth for an image.
detected_rects - output of the detector
im - the image or filename to assciate with the sample.
annotate - add diagnostic annotations to the images.
'''
self.images += 1
name = None
detected_rects = copy.copy(detected_rects)
if isinstance(im,Image):
name = im.filename
if self.pixels != None:
self.pixels += im.asPIL().size[0] * im.asPIL().size[1]
elif isinstance(im,str):
name = im
self.pixels = None
else:
name = "%d"%self.sample_id
self.pixels = None
table = self.table
for i in range(len(truth_rects)):
truth = truth_rects[i]
self.positives += 1
success = False
best_overlap = 0.0
best_detection = None
for j in range(len(detected_rects)):
detected = detected_rects[j]
overlap = overlap_score(truth,detected)
if overlap >= self.threshold and overlap > best_overlap:
success = True
best_overlap = overlap
best_detection = j
table.setData(self.sample_id,'id',self.sample_id)
table.setData(self.sample_id,'name',name)
table.setData(self.sample_id,'truth_rect',str(truth))
if best_detection != None:
table.setData(self.sample_id,'detection_rect',str(detected_rects[best_detection]))
else:
table.setData(self.sample_id,'detection_rect',None)
table.setData(self.sample_id,'success',success)
table.setData(self.sample_id,'overlap',best_overlap)
self.sample_id+=1
if success:
self.successes += 1
if annotate and isinstance(im,Image):
if success:
im.annotateEllipse(truth,color='green')
else:
im.annotateEllipse(truth,color='red')
if best_detecion != None:
im.annototeRect(detected_rects[best_detection],color='green')
# Remove the best detection if success
if best_detection != None:
del detected_rects[best_detection]
if annotate:
for each in detected_rects:
im.annotateRect(each,color='red')
self.negatives += len(detected_rects)
self.end_time = time.time()
self.total_time = self.end_time - self.start_time
self.image_time = self.total_time/float(self.images)
# Update summary statistics
if self.positives > 0:
self.pos_rate = float(self.successes)/self.positives
self.pos_bounds = cibinom(self.positives,self.successes,alpha=0.05)
if self.pixels != None:
self.neg_rate = float(self.negatives)/float(1.0e-6*self.pixels)
self.createSummary()
def confidenceInterval(self,alpha=0.05):
'''
Returns the estimated a confidence interval for the success update_rate by
modeling the success update_rate as a binomial distribution.
'''
return cibinom(self.total,self.successes,alpha=alpha)
def addSample(self, probe_id, scores):
'''
Adds a sample to the test. The similarity scores is a list of
tuples that contain tuples of (gallery_id, score)
'''
comp_func = lambda a, b: (a > b) - (a < b)
if self.score_type == SCORE_TYPE_LOW:
scores.sort(lambda x, y: comp_func(x[1], y[1]))
elif self.score_type == SCORE_TYPE_HIGH:
scores.sort(lambda x, y: -comp_func(x[1], y[1]))
else:
raise ValueError("Unknown score type: %s" % self.score_type)
best_match_id = None
best_match_score = None
best_match_rank = None
best_nonmatch_id = None
best_nonmatch_score = None
best_nonmatch_rank = None
pid = self.id_func(probe_id)
rank = 0
for gallery_id, score in scores:
if probe_id == gallery_id:
# Probe and gallery should not be the same image
continue
gid = self.id_func(gallery_id)
match = False
if pid == gid:
match = True
if match and best_match_id == None:
best_match_id = gallery_id
best_match_score = score
best_match_rank = rank
if not match and best_nonmatch_id == None:
best_nonmatch_id = gallery_id
best_nonmatch_score = score
best_nonmatch_rank = rank
if match:
self.positives.append(score)
else:
self.negatives.append(score)
rank += 1
self.total_probes += 1
self.total_gallery = max(self.total_gallery, rank)
if best_match_rank == 0:
self.rank1_success += 1
self.probes_table.setElement(probe_id, 'MatchId', best_match_id)
self.probes_table.setElement(probe_id, 'MatchScore', best_match_score)
self.probes_table.setElement(probe_id, 'MatchRank', best_match_rank)
self.probes_table.setElement(probe_id, 'Success', best_match_rank == 0)
self.probes_table.setElement(probe_id, 'NonMatchId', best_nonmatch_id)
self.probes_table.setElement(probe_id, 'NonMatchScore',
best_nonmatch_score)
self.probes_table.setElement(probe_id, 'NonMatchRank',
best_nonmatch_rank)
self.rank1_rate = float(self.rank1_success) / float(self.total_probes)
self.rank1_bounds = cibinom(self.total_probes, self.rank1_success)
def addSample(self, truth_rects, detected_rects, im=None, annotate=False):
'''
Adds a sample to face detection test.
@param truth_rects: truth for an image.
@param detected_rects: output of the detector
@param im: the image or filename to assciate with the sample.
@param annotate: add diagnostic annotations to the images.
'''
self.images += 1
name = None
detected_rects = copy.copy(detected_rects)
if isinstance(im,Image):
name = im.filename
if self.pixels != None:
self.pixels += im.asPIL().size[0] * im.asPIL().size[1]
elif isinstance(im,str):
name = im
self.pixels = None
else:
name = "%d"%self.sample_id
self.pixels = None
table = self.table
for i in range(len(truth_rects)):
truth = truth_rects[i]
self.positives += 1
success = False
best_overlap = 0.0
best_detection = None
for j in range(len(detected_rects)):
detected = detected_rects[j]
overlap = overlap_score(truth,detected)
if overlap >= self.threshold and overlap > best_overlap:
success = True
best_overlap = overlap
best_detection = j
table.setData(self.sample_id,'id',self.sample_id)
table.setData(self.sample_id,'name',name)
table.setData(self.sample_id,'truth_rect',str(truth))
if best_detection != None:
table.setData(self.sample_id,'detection_rect',str(detected_rects[best_detection]))
else:
table.setData(self.sample_id,'detection_rect',None)
table.setData(self.sample_id,'success',success)
table.setData(self.sample_id,'overlap',best_overlap)
self.sample_id+=1
if success:
self.successes += 1
if annotate and isinstance(im,Image):
if success:
im.annotateEllipse(truth,color='green')
else:
im.annotateEllipse(truth,color='red')
if best_detecion != None:
im.annototeRect(detected_rects[best_detection],color='green')
# Remove the best detection if success
if best_detection != None:
del detected_rects[best_detection]
if annotate:
for each in detected_rects:
im.annotateRect(each,color='red')
self.negatives += len(detected_rects)
self.end_time = time.time()
self.total_time = self.end_time - self.start_time
self.image_time = self.total_time/float(self.images)
# Update summary statistics
if self.positives > 0:
self.pos_rate = float(self.successes)/self.positives
self.pos_bounds = cibinom(self.positives,self.successes,alpha=0.05)
if self.pixels != None:
self.neg_rate = float(self.negatives)/float(1.0e-6*self.pixels)
self.createSummary()
def confidenceInterval(self, alpha=0.05):
'''
Returns the estimated a confidence interval for the success update_rate by
modeling the success update_rate as a binomial distribution.
'''
return cibinom(self.total, self.successes, alpha=alpha)
