def evaluate_detections(self, all_boxes, output_dir, det_salt = '', eval_salt = '', overlap_thresh = 0.5): num_classes = self.num_classes num_images = self.num_images gt_roidb = self.gt_roidb() ap = [[]]; prec = [[]]; rec = [[]] ap_file = os.path.join(output_dir, 'eval' + det_salt + eval_salt + '.txt') with open(ap_file, 'wt') as f: for i in xrange(1, self.num_classes): dt = []; gt = []; # Prepare the output for j in xrange(0,num_images): bs = all_boxes[i][j] if len(bs) == 0: bb = np.zeros((0,4)).astype(np.float32) sc = np.zeros((0,1)).astype(np.float32) else: bb = bs[:,:4].reshape(bs.shape[0],4) sc = bs[:,4].reshape(bs.shape[0],1) dtI = dict({'sc': sc, 'boxInfo': bb}) dt.append(dtI) # Prepare the annotations for j in xrange(0,num_images): cls_ind = np.where(gt_roidb[j]['gt_classes'] == i)[0] bb = gt_roidb[j]['boxes'][cls_ind,:] diff = np.zeros((len(cls_ind),1)).astype(np.bool) gt.append(dict({'diff': diff, 'boxInfo': bb})) bOpts = dict({'minoverlap': overlap_thresh}) ap_i, rec_i, prec_i = eval.inst_bench(dt, gt, bOpts)[:3] ap.append(ap_i[0]); prec.append(prec_i); rec.append(rec_i) ap_str = '{:20s}:{:10f}'.format(self.classes[i], ap_i[0]*100) f.write(ap_str + '\n') print ap_str ap_str = '{:20s}:{:10f}'.format('mean', np.mean(ap[1:])*100) f.write(ap_str + '\n') print ap_str eval_file = os.path.join(output_dir, 'eval' + det_salt + eval_salt + '.pkl') g_utils.save_variables(eval_file, [ap, prec, rec, self._classes, self._class_to_ind], \ ['ap', 'prec', 'rec', 'classes', 'class_to_ind'], overwrite = True) eval_file = os.path.join(output_dir, 'eval' + det_salt + eval_salt + '.mat') g_utils.scio.savemat(eval_file, {'ap': ap, 'prec': prec, 'rec': rec, 'classes': self._classes}, do_compression = True); return ap, prec, rec, self._classes, self._class_to_ind
def test_net(net, imdb): """Test a Fast R-CNN network on an image database.""" num_images = len(imdb.image_index) # heuristic: keep an average of 40 detections per class per images prior # to NMS max_per_set = cfg.TEST.MAX_PER_SET_F * num_images # heuristic: keep at most 100 detection per class per image prior to NMS max_per_image = cfg.TEST.MAX_PER_IMAGE # detection thresold for each class (this is adaptively set based on the # max_per_set constraint) thresh = -np.inf * np.ones(imdb.num_classes) # top_scores will hold one minheap of scores per class (used to enforce # the max_per_set constraint) top_scores = [[] for _ in xrange(imdb.num_classes)] # all detections are collected into: # all_boxes[cls][image] = N x 5 array of detections in # (x1, y1, x2, y2, score) all_boxes = [[[] for _ in xrange(num_images)] for _ in xrange(imdb.num_classes)] output_dir = get_output_dir(imdb, net) if not os.path.exists(output_dir): os.makedirs(output_dir) # timers _t = {'im_detect' : Timer(), 'misc' : Timer()} roidb = imdb.roidb for i in xrange(num_images): image_paths = imdb.image_path_at(i); im = []; for image_path in image_paths: im1 = cv2.imread(image_path) im1 = im1[:, :, ::-1] im.append(im1) _t['im_detect'].tic() scores, boxes = im_detect(net, im, roidb[i]['boxes']) _t['im_detect'].toc() _t['misc'].tic() for j in xrange(1, imdb.num_classes): inds = np.where((scores[:, j] > thresh[j]) & (roidb[i]['gt_classes'] == 0))[0] cls_scores = scores[inds, j] cls_boxes = boxes[inds, j*4:(j+1)*4] top_inds = np.argsort(-cls_scores)[:max_per_image] cls_scores = cls_scores[top_inds] cls_boxes = cls_boxes[top_inds, :] # push new scores onto the minheap for val in cls_scores: heapq.heappush(top_scores[j], val) # if we've collected more than the max number of detection, # then pop items off the minheap and update the class threshold if len(top_scores[j]) > max_per_set: while len(top_scores[j]) > max_per_set: heapq.heappop(top_scores[j]) thresh[j] = top_scores[j][0] all_boxes[j][i] = \ np.hstack((cls_boxes, cls_scores[:, np.newaxis])) \ .astype(np.float32, copy=False) if 0: keep = nms(all_boxes[j][i], 0.3) vis_detections(im, imdb.classes[j], all_boxes[j][i][keep, :]) _t['misc'].toc() print 'im_detect: {:d}/{:d} {:.3f}s {:.3f}s' \ .format(i + 1, num_images, _t['im_detect'].average_time, _t['misc'].average_time) for j in xrange(1, imdb.num_classes): for i in xrange(num_images): inds = np.where(all_boxes[j][i][:, -1] > thresh[j])[0] all_boxes[j][i] = all_boxes[j][i][inds, :] det_file = os.path.join(output_dir, 'detections' + cfg.TEST.DET_SALT + '.pkl') g_utils.save_variables(det_file, [all_boxes], ['all_boxes'], overwrite = True) det_file = os.path.join(output_dir, 'detections' + cfg.TEST.DET_SALT + '.pkl') g_utils.scio.savemat(det_file, {'all_boxes': all_boxes}, do_compression = True) print 'Applying NMS to all detections' nms_dets = apply_nms(all_boxes, cfg.TEST.NMS) print 'Evaluating detections' ap, prec, rec, classes, class_to_ind = imdb.evaluate_detections(nms_dets, output_dir, cfg.TEST.DET_SALT, cfg.TEST.EVAL_SALT)