def labelme_eval_sds(det_file, seg_file, annot_path, image_list, cls_name, cache_dir, class_names, mask_size, binary_thresh, ov_thresh=0.5): # 1. Check whether ground truth cache file exists assert(type(image_list) is list) image_names = [i[:i.rfind(".")] for i in image_list] # image_list cache_gt_file_format = "%s_mask_gt.pkl" check_labelme_sds_cache(cache_dir, annot_path, image_names, class_names, cache_gt_file_format) print("***Performing evaluation on %s***"%(cls_name)) gt_cache = os.path.join(cache_dir, cache_gt_file_format%(cls_name)) with open(gt_cache, 'rb') as f: print("Loading ground-truth %s..."%(gt_cache)) gt_pkl = cPickle.load(f) # 2. Get predict pickle file for this class with open(det_file, 'rb') as f: print("Loading det predictions %s..."%(det_file)) boxes_pkl = cPickle.load(f) with open(seg_file, 'rb') as f: print("Loading seg predictions %s..."%(seg_file)) masks_pkl = cPickle.load(f) # 3. Pre-compute number of total instances to allocate memory num_image = len(image_names) box_num = 0 for im_i in xrange(num_image): box_num += len(boxes_pkl[im_i]) # 4. Re-organize all the predicted boxes new_boxes = np.zeros((box_num, 5)) new_masks = np.zeros((box_num, mask_size, mask_size)) new_image = [] cnt = 0 for image_ind in xrange(len(image_names)): boxes = boxes_pkl[image_ind] masks = masks_pkl[image_ind] num_instance = len(boxes) for box_ind in xrange(num_instance): new_boxes[cnt] = boxes[box_ind] new_masks[cnt] = masks[box_ind] new_image.append(image_names[image_ind]) cnt += 1 # 5. Rearrange boxes according to their scores seg_scores = new_boxes[:, -1] keep_inds = np.argsort(-seg_scores) new_boxes = new_boxes[keep_inds, :] new_masks = new_masks[keep_inds, :, :] num_pred = new_boxes.shape[0] # 6. Calculate t/f positive fp = np.zeros((num_pred, 1)) tp = np.zeros((num_pred, 1)) for i in xrange(num_pred): pred_box = np.round(new_boxes[i, :4]).astype(int) pred_mask = new_masks[i] pred_mask = cv2.resize(pred_mask.astype(np.float32), (pred_box[2] - pred_box[0] + 1, pred_box[3] - pred_box[1] + 1)) pred_mask = pred_mask >= binary_thresh image_index = new_image[keep_inds[i]] # load from cache if image_index not in gt_pkl: fp[i] = 1 continue gt_dict_list = gt_pkl[image_index] # annot_file = os.path.join(annot_path, image_index+".xml") # gt_dict_list = parse_rec(annot_file) # calculate max region overlap cur_overlap = -1000 cur_overlap_ind = -1 for ind2, gt_dict in enumerate(gt_dict_list): if gt_dict['mask_cls'] != cls_name: continue gt_mask_bound = np.round(gt_dict['mask_bound']).astype(int) gt_mask = gt_dict['mask'] pred_mask_bound = pred_box ov = mask_overlap(gt_mask_bound, pred_mask_bound, gt_mask, pred_mask) # try: # ov = mask_overlap(gt_mask_bound, pred_mask_bound, gt_mask, pred_mask) # except IndexError, e: # print('gt_mask_bound', gt_mask_bound) # print('gt_mask', gt_mask) # print('pred_mask',pred_mask) # print('pred_mask_bound', pred_mask_bound) # print(ind2) # print(annot_file) # gt_dict_list = parse_rec(annot_file, verbose=True) # raise IndexError, e if ov > cur_overlap: cur_overlap = ov cur_overlap_ind = ind2 if cur_overlap >= ov_thresh: # if 'already_detect' in gt_dict_list[cur_overlap_ind]: if gt_dict_list[cur_overlap_ind]['already_detect']: fp[i] = 1 else: tp[i] = 1 gt_dict_list[cur_overlap_ind]['already_detect'] = 1 else: fp[i] = 1 # 7. Calculate precision num_pos = 0 for key, val in gt_pkl.iteritems(): num_pos += len(val) fp = np.cumsum(fp) tp = np.cumsum(tp) rec = tp / float(num_pos) # avoid divide by zero in case the first matches a difficult gt prec = tp / np.maximum(fp+tp, np.finfo(np.float64).eps) ap = voc_ap(rec, prec, True) return ap
def voc_eval_sds(det_file, seg_file, devkit_path, image_list, cls_name, cache_dir, class_names, mask_size, binary_thresh, ov_thresh=0.5): # 1. Check whether ground truth cache file exists with open(image_list, 'r') as f: lines = f.readlines() image_names = [x.strip() for x in lines] check_voc_sds_cache(cache_dir, devkit_path, image_names, class_names) gt_cache = cache_dir + '/' + cls_name + '_mask_gt.pkl' with open(gt_cache, 'rb') as f: gt_pkl = pickle.load(f) # 2. Get predict pickle file for this class with open(det_file, 'rb') as f: boxes_pkl = pickle.load(f) with open(seg_file, 'rb') as f: masks_pkl = pickle.load(f) # 3. Pre-compute number of total instances to allocate memory num_image = len(image_names) box_num = 0 for im_i in range(num_image): box_num += len(boxes_pkl[im_i]) # 4. Re-organize all the predicted boxes new_boxes = np.zeros((box_num, 5)) new_masks = np.zeros((box_num, mask_size, mask_size)) new_image = [] cnt = 0 for image_ind in range(len(image_names)): boxes = boxes_pkl[image_ind] masks = masks_pkl[image_ind] num_instance = len(boxes) for box_ind in range(num_instance): new_boxes[cnt] = boxes[box_ind] new_masks[cnt] = masks[box_ind] new_image.append(image_names[image_ind]) cnt += 1 # 5. Rearrange boxes according to their scores seg_scores = new_boxes[:, -1] keep_inds = np.argsort(-seg_scores) new_boxes = new_boxes[keep_inds, :] new_masks = new_masks[keep_inds, :, :] num_pred = new_boxes.shape[0] import cv2 # 6. Calculate t/f positive fp = np.zeros((num_pred, 1)) tp = np.zeros((num_pred, 1)) for i in range(num_pred): pred_box = np.round(new_boxes[i, :4]).astype(int) pred_mask = new_masks[i] pred_mask = cv2.resize( pred_mask.astype(np.float32), (pred_box[2] - pred_box[0] + 1, pred_box[3] - pred_box[1] + 1)) pred_mask = pred_mask >= binary_thresh image_index = new_image[keep_inds[i]] if image_index not in gt_pkl: fp[i] = 1 continue gt_dict_list = gt_pkl[image_index] # calculate max region overlap cur_overlap = -1000 cur_overlap_ind = -1 for ind2, gt_dict in enumerate(gt_dict_list): gt_mask_bound = np.round(gt_dict['mask_bound']).astype(int) pred_mask_bound = pred_box ov = mask_overlap(gt_mask_bound, pred_mask_bound, gt_dict['mask'], pred_mask) if ov > cur_overlap: cur_overlap = ov cur_overlap_ind = ind2 if cur_overlap >= ov_thresh: if gt_dict_list[cur_overlap_ind]['already_detect']: fp[i] = 1 else: tp[i] = 1 gt_dict_list[cur_overlap_ind]['already_detect'] = 1 else: fp[i] = 1 # 7. Calculate precision num_pos = 0 for key, val in gt_pkl.items(): num_pos += len(val) fp = np.cumsum(fp) tp = np.cumsum(tp) rec = tp / float(num_pos) # avoid divide by zero in case the first matches a difficult gt prec = tp / np.maximum(fp + tp, np.finfo(np.float64).eps) ap = voc_ap(rec, prec, True) return ap
def voc_eval_sds(det_file, seg_file, devkit_path, image_list, cls_name, cache_dir, class_names, mask_size, binary_thresh, ov_thresh=0.5): # 1. Check whether ground truth cache file exists with open(image_list, 'r') as f: lines = f.readlines() image_names = [x.strip() for x in lines] check_voc_sds_cache(cache_dir, devkit_path, image_names, class_names) gt_cache = cache_dir + '/' + cls_name + '_mask_gt.pkl' with open(gt_cache, 'rb') as f: gt_pkl = cPickle.load(f) # 2. Get predict pickle file for this class with open(det_file, 'rb') as f: boxes_pkl = cPickle.load(f) with open(seg_file, 'rb') as f: masks_pkl = cPickle.load(f) # 3. Pre-compute number of total instances to allocate memory num_image = len(image_names) box_num = 0 for im_i in xrange(num_image): box_num += len(boxes_pkl[im_i]) # 4. Re-organize all the predicted boxes new_boxes = np.zeros((box_num, 5)) new_masks = np.zeros((box_num, mask_size, mask_size)) new_image = [] cnt = 0 for image_ind in xrange(len(image_names)): boxes = boxes_pkl[image_ind] masks = masks_pkl[image_ind] num_instance = len(boxes) for box_ind in xrange(num_instance): new_boxes[cnt] = boxes[box_ind] new_masks[cnt] = masks[box_ind] new_image.append(image_names[image_ind]) cnt += 1 # 5. Rearrange boxes according to their scores seg_scores = new_boxes[:, -1] keep_inds = np.argsort(-seg_scores) new_boxes = new_boxes[keep_inds, :] new_masks = new_masks[keep_inds, :, :] num_pred = new_boxes.shape[0] import cv2 # 6. Calculate t/f positive fp = np.zeros((num_pred, 1)) tp = np.zeros((num_pred, 1)) for i in xrange(num_pred): pred_box = np.round(new_boxes[i, :4]).astype(int) pred_mask = new_masks[i] pred_mask = cv2.resize(pred_mask.astype(np.float32), (pred_box[2] - pred_box[0] + 1, pred_box[3] - pred_box[1] + 1)) pred_mask = pred_mask >= binary_thresh image_index = new_image[keep_inds[i]] if image_index not in gt_pkl: fp[i] = 1 continue gt_dict_list = gt_pkl[image_index] # calculate max region overlap cur_overlap = -1000 cur_overlap_ind = -1 for ind2, gt_dict in enumerate(gt_dict_list): gt_mask_bound = np.round(gt_dict['mask_bound']).astype(int) pred_mask_bound = pred_box ov = mask_overlap(gt_mask_bound, pred_mask_bound, gt_dict['mask'], pred_mask) if ov > cur_overlap: cur_overlap = ov cur_overlap_ind = ind2 if cur_overlap >= ov_thresh: if gt_dict_list[cur_overlap_ind]['already_detect']: fp[i] = 1 else: tp[i] = 1 gt_dict_list[cur_overlap_ind]['already_detect'] = 1 else: fp[i] = 1 # 7. Calculate precision num_pos = 0 for key, val in gt_pkl.iteritems(): num_pos += len(val) fp = np.cumsum(fp) tp = np.cumsum(tp) rec = tp / float(num_pos) # avoid divide by zero in case the first matches a difficult gt prec = tp / np.maximum(fp+tp, np.finfo(np.float64).eps) ap = voc_ap(rec, prec, True) return ap