def evaluate_detections(self, all_boxes, output_dir, det_salt = '', eval_salt = '', overlap_thresh = 0.5):
num_classes = self.num_classes
num_images = self.num_images
gt_roidb = self.gt_roidb()
ap = [[]]; prec = [[]]; rec = [[]]
ap_file = os.path.join(output_dir, 'eval' + det_salt + eval_salt + '.txt')
with open(ap_file, 'wt') as f:
for i in xrange(1, self.num_classes):
dt = []; gt = [];
# Prepare the output
for j in xrange(0,num_images):
bs = all_boxes[i][j]
if len(bs) == 0:
bb = np.zeros((0,4)).astype(np.float32)
sc = np.zeros((0,1)).astype(np.float32)
else:
bb = bs[:,:4].reshape(bs.shape[0],4)
sc = bs[:,4].reshape(bs.shape[0],1)
dtI = dict({'sc': sc, 'boxInfo': bb})
dt.append(dtI)
# Prepare the annotations
for j in xrange(0,num_images):
cls_ind = np.where(gt_roidb[j]['gt_classes'] == i)[0]
bb = gt_roidb[j]['boxes'][cls_ind,:]
diff = np.zeros((len(cls_ind),1)).astype(np.bool)
gt.append(dict({'diff': diff, 'boxInfo': bb}))
bOpts = dict({'minoverlap': overlap_thresh})
ap_i, rec_i, prec_i = eval.inst_bench(dt, gt, bOpts)[:3]
ap.append(ap_i[0]); prec.append(prec_i); rec.append(rec_i)
ap_str = '{:20s}:{:10f}'.format(self.classes[i], ap_i[0]*100)
f.write(ap_str + '\n')
print ap_str
ap_str = '{:20s}:{:10f}'.format('mean', np.mean(ap[1:])*100)
f.write(ap_str + '\n')
print ap_str
eval_file = os.path.join(output_dir, 'eval' + det_salt + eval_salt + '.pkl')
g_utils.save_variables(eval_file, [ap, prec, rec, self._classes, self._class_to_ind], \
['ap', 'prec', 'rec', 'classes', 'class_to_ind'], overwrite = True)
eval_file = os.path.join(output_dir, 'eval' + det_salt + eval_salt + '.mat')
g_utils.scio.savemat(eval_file, {'ap': ap, 'prec': prec, 'rec': rec, 'classes': self._classes}, do_compression = True);
return ap, prec, rec, self._classes, self._class_to_ind
def evaluate_detections(self, all_boxes, output_dir, det_salt = '', eval_salt = '', overlap_thresh = 0.5):
num_classes = self.num_classes
num_images = self.num_images
gt_roidb = self.gt_roidb()
ap = [[]]; prec = [[]]; rec = [[]]
ap_file = os.path.join(output_dir, 'eval' + det_salt + eval_salt + '.txt')
with open(ap_file, 'wt') as f:
for i in xrange(1, self.num_classes):
dt = []; gt = [];
# Prepare the output
for j in xrange(0,num_images):
bs = all_boxes[i][j]
if len(bs) == 0:
bb = np.zeros((0,4)).astype(np.float32)
sc = np.zeros((0,1)).astype(np.float32)
else:
bb = bs[:,:4].reshape(bs.shape[0],4)
sc = bs[:,4].reshape(bs.shape[0],1)
dtI = dict({'sc': sc, 'boxInfo': bb})
dt.append(dtI)
# Prepare the annotations
for j in xrange(0,num_images):
cls_ind = np.where(gt_roidb[j]['gt_classes'] == i)[0]
bb = gt_roidb[j]['boxes'][cls_ind,:]
diff = np.zeros((len(cls_ind),1)).astype(np.bool)
gt.append(dict({'diff': diff, 'boxInfo': bb}))
bOpts = dict({'minoverlap': overlap_thresh})
ap_i, rec_i, prec_i = eval.inst_bench(dt, gt, bOpts)[:3]
ap.append(ap_i[0]); prec.append(prec_i); rec.append(rec_i)
ap_str = '{:20s}:{:10f}'.format(self.classes[i], ap_i[0]*100)
f.write(ap_str + '\n')
print ap_str
ap_str = '{:20s}:{:10f}'.format('mean', np.mean(ap[1:])*100)
f.write(ap_str + '\n')
print ap_str
eval_file = os.path.join(output_dir, 'eval' + det_salt + eval_salt + '.pkl')
g_utils.save_variables(eval_file, [ap, prec, rec, self._classes, self._class_to_ind], \
['ap', 'prec', 'rec', 'classes', 'class_to_ind'], overwrite = True)
eval_file = os.path.join(output_dir, 'eval' + det_salt + eval_salt + '.mat')
g_utils.scio.savemat(eval_file, {'ap': ap, 'prec': prec, 'rec': rec, 'classes': self._classes}, do_compression = True);
return ap, prec, rec, self._classes, self._class_to_ind
def test_net(net, imdb):
"""Test a Fast R-CNN network on an image database."""
num_images = len(imdb.image_index)
# heuristic: keep an average of 40 detections per class per images prior
# to NMS
max_per_set = cfg.TEST.MAX_PER_SET_F * num_images
# heuristic: keep at most 100 detection per class per image prior to NMS
max_per_image = cfg.TEST.MAX_PER_IMAGE
# detection thresold for each class (this is adaptively set based on the
# max_per_set constraint)
thresh = -np.inf * np.ones(imdb.num_classes)
# top_scores will hold one minheap of scores per class (used to enforce
# the max_per_set constraint)
top_scores = [[] for _ in xrange(imdb.num_classes)]
# all detections are collected into:
# all_boxes[cls][image] = N x 5 array of detections in
# (x1, y1, x2, y2, score)
all_boxes = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
output_dir = get_output_dir(imdb, net)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# timers
_t = {'im_detect' : Timer(), 'misc' : Timer()}
roidb = imdb.roidb
for i in xrange(num_images):
image_paths = imdb.image_path_at(i); im = [];
for image_path in image_paths:
im1 = cv2.imread(image_path)
im1 = im1[:, :, ::-1]
im.append(im1)
_t['im_detect'].tic()
scores, boxes = im_detect(net, im, roidb[i]['boxes'])
_t['im_detect'].toc()
_t['misc'].tic()
for j in xrange(1, imdb.num_classes):
inds = np.where((scores[:, j] > thresh[j]) &
(roidb[i]['gt_classes'] == 0))[0]
cls_scores = scores[inds, j]
cls_boxes = boxes[inds, j*4:(j+1)*4]
top_inds = np.argsort(-cls_scores)[:max_per_image]
cls_scores = cls_scores[top_inds]
cls_boxes = cls_boxes[top_inds, :]
# push new scores onto the minheap
for val in cls_scores:
heapq.heappush(top_scores[j], val)
# if we've collected more than the max number of detection,
# then pop items off the minheap and update the class threshold
if len(top_scores[j]) > max_per_set:
while len(top_scores[j]) > max_per_set:
heapq.heappop(top_scores[j])
thresh[j] = top_scores[j][0]
all_boxes[j][i] = \
np.hstack((cls_boxes, cls_scores[:, np.newaxis])) \
.astype(np.float32, copy=False)
if 0:
keep = nms(all_boxes[j][i], 0.3)
vis_detections(im, imdb.classes[j], all_boxes[j][i][keep, :])
_t['misc'].toc()
print 'im_detect: {:d}/{:d} {:.3f}s {:.3f}s' \
.format(i + 1, num_images, _t['im_detect'].average_time,
_t['misc'].average_time)
for j in xrange(1, imdb.num_classes):
for i in xrange(num_images):
inds = np.where(all_boxes[j][i][:, -1] > thresh[j])[0]
all_boxes[j][i] = all_boxes[j][i][inds, :]
det_file = os.path.join(output_dir, 'detections' + cfg.TEST.DET_SALT + '.pkl')
g_utils.save_variables(det_file, [all_boxes], ['all_boxes'], overwrite = True)
det_file = os.path.join(output_dir, 'detections' + cfg.TEST.DET_SALT + '.pkl')
g_utils.scio.savemat(det_file, {'all_boxes': all_boxes}, do_compression = True)
print 'Applying NMS to all detections'
nms_dets = apply_nms(all_boxes, cfg.TEST.NMS)
print 'Evaluating detections'
ap, prec, rec, classes, class_to_ind = imdb.evaluate_detections(nms_dets, output_dir, cfg.TEST.DET_SALT, cfg.TEST.EVAL_SALT)
