def labelme_eval_sds(det_file, seg_file, annot_path, image_list, cls_name, cache_dir,
class_names, mask_size, binary_thresh, ov_thresh=0.5):
# 1. Check whether ground truth cache file exists
assert(type(image_list) is list)
image_names = [i[:i.rfind(".")] for i in image_list] # image_list
cache_gt_file_format = "%s_mask_gt.pkl"
check_labelme_sds_cache(cache_dir, annot_path, image_names, class_names, cache_gt_file_format)
print("***Performing evaluation on %s***"%(cls_name))
gt_cache = os.path.join(cache_dir, cache_gt_file_format%(cls_name))
with open(gt_cache, 'rb') as f:
print("Loading ground-truth %s..."%(gt_cache))
gt_pkl = cPickle.load(f)
# 2. Get predict pickle file for this class
with open(det_file, 'rb') as f:
print("Loading det predictions %s..."%(det_file))
boxes_pkl = cPickle.load(f)
with open(seg_file, 'rb') as f:
print("Loading seg predictions %s..."%(seg_file))
masks_pkl = cPickle.load(f)
# 3. Pre-compute number of total instances to allocate memory
num_image = len(image_names)
box_num = 0
for im_i in xrange(num_image):
box_num += len(boxes_pkl[im_i])
# 4. Re-organize all the predicted boxes
new_boxes = np.zeros((box_num, 5))
new_masks = np.zeros((box_num, mask_size, mask_size))
new_image = []
cnt = 0
for image_ind in xrange(len(image_names)):
boxes = boxes_pkl[image_ind]
masks = masks_pkl[image_ind]
num_instance = len(boxes)
for box_ind in xrange(num_instance):
new_boxes[cnt] = boxes[box_ind]
new_masks[cnt] = masks[box_ind]
new_image.append(image_names[image_ind])
cnt += 1
# 5. Rearrange boxes according to their scores
seg_scores = new_boxes[:, -1]
keep_inds = np.argsort(-seg_scores)
new_boxes = new_boxes[keep_inds, :]
new_masks = new_masks[keep_inds, :, :]
num_pred = new_boxes.shape[0]
# 6. Calculate t/f positive
fp = np.zeros((num_pred, 1))
tp = np.zeros((num_pred, 1))
for i in xrange(num_pred):
pred_box = np.round(new_boxes[i, :4]).astype(int)
pred_mask = new_masks[i]
pred_mask = cv2.resize(pred_mask.astype(np.float32), (pred_box[2] - pred_box[0] + 1, pred_box[3] - pred_box[1] + 1))
pred_mask = pred_mask >= binary_thresh
image_index = new_image[keep_inds[i]]
# load from cache
if image_index not in gt_pkl:
fp[i] = 1
continue
gt_dict_list = gt_pkl[image_index]
# annot_file = os.path.join(annot_path, image_index+".xml")
# gt_dict_list = parse_rec(annot_file)
# calculate max region overlap
cur_overlap = -1000
cur_overlap_ind = -1
for ind2, gt_dict in enumerate(gt_dict_list):
if gt_dict['mask_cls'] != cls_name:
continue
gt_mask_bound = np.round(gt_dict['mask_bound']).astype(int)
gt_mask = gt_dict['mask']
pred_mask_bound = pred_box
ov = mask_overlap(gt_mask_bound, pred_mask_bound, gt_mask, pred_mask)
# try:
# ov = mask_overlap(gt_mask_bound, pred_mask_bound, gt_mask, pred_mask)
# except IndexError, e:
# print('gt_mask_bound', gt_mask_bound)
# print('gt_mask', gt_mask)
# print('pred_mask',pred_mask)
# print('pred_mask_bound', pred_mask_bound)
# print(ind2)
# print(annot_file)
# gt_dict_list = parse_rec(annot_file, verbose=True)
# raise IndexError, e
if ov > cur_overlap:
cur_overlap = ov
cur_overlap_ind = ind2
if cur_overlap >= ov_thresh:
# if 'already_detect' in gt_dict_list[cur_overlap_ind]:
if gt_dict_list[cur_overlap_ind]['already_detect']:
fp[i] = 1
else:
tp[i] = 1
gt_dict_list[cur_overlap_ind]['already_detect'] = 1
else:
fp[i] = 1
# 7. Calculate precision
num_pos = 0
for key, val in gt_pkl.iteritems():
num_pos += len(val)
fp = np.cumsum(fp)
tp = np.cumsum(tp)
rec = tp / float(num_pos)
# avoid divide by zero in case the first matches a difficult gt
prec = tp / np.maximum(fp+tp, np.finfo(np.float64).eps)
ap = voc_ap(rec, prec, True)
return ap
def voc_eval_sds(det_file,
seg_file,
devkit_path,
image_list,
cls_name,
cache_dir,
class_names,
mask_size,
binary_thresh,
ov_thresh=0.5):
# 1. Check whether ground truth cache file exists
with open(image_list, 'r') as f:
lines = f.readlines()
image_names = [x.strip() for x in lines]
check_voc_sds_cache(cache_dir, devkit_path, image_names, class_names)
gt_cache = cache_dir + '/' + cls_name + '_mask_gt.pkl'
with open(gt_cache, 'rb') as f:
gt_pkl = pickle.load(f)
# 2. Get predict pickle file for this class
with open(det_file, 'rb') as f:
boxes_pkl = pickle.load(f)
with open(seg_file, 'rb') as f:
masks_pkl = pickle.load(f)
# 3. Pre-compute number of total instances to allocate memory
num_image = len(image_names)
box_num = 0
for im_i in range(num_image):
box_num += len(boxes_pkl[im_i])
# 4. Re-organize all the predicted boxes
new_boxes = np.zeros((box_num, 5))
new_masks = np.zeros((box_num, mask_size, mask_size))
new_image = []
cnt = 0
for image_ind in range(len(image_names)):
boxes = boxes_pkl[image_ind]
masks = masks_pkl[image_ind]
num_instance = len(boxes)
for box_ind in range(num_instance):
new_boxes[cnt] = boxes[box_ind]
new_masks[cnt] = masks[box_ind]
new_image.append(image_names[image_ind])
cnt += 1
# 5. Rearrange boxes according to their scores
seg_scores = new_boxes[:, -1]
keep_inds = np.argsort(-seg_scores)
new_boxes = new_boxes[keep_inds, :]
new_masks = new_masks[keep_inds, :, :]
num_pred = new_boxes.shape[0]
import cv2
# 6. Calculate t/f positive
fp = np.zeros((num_pred, 1))
tp = np.zeros((num_pred, 1))
for i in range(num_pred):
pred_box = np.round(new_boxes[i, :4]).astype(int)
pred_mask = new_masks[i]
pred_mask = cv2.resize(
pred_mask.astype(np.float32),
(pred_box[2] - pred_box[0] + 1, pred_box[3] - pred_box[1] + 1))
pred_mask = pred_mask >= binary_thresh
image_index = new_image[keep_inds[i]]
if image_index not in gt_pkl:
fp[i] = 1
continue
gt_dict_list = gt_pkl[image_index]
# calculate max region overlap
cur_overlap = -1000
cur_overlap_ind = -1
for ind2, gt_dict in enumerate(gt_dict_list):
gt_mask_bound = np.round(gt_dict['mask_bound']).astype(int)
pred_mask_bound = pred_box
ov = mask_overlap(gt_mask_bound, pred_mask_bound, gt_dict['mask'],
pred_mask)
if ov > cur_overlap:
cur_overlap = ov
cur_overlap_ind = ind2
if cur_overlap >= ov_thresh:
if gt_dict_list[cur_overlap_ind]['already_detect']:
fp[i] = 1
else:
tp[i] = 1
gt_dict_list[cur_overlap_ind]['already_detect'] = 1
else:
fp[i] = 1
# 7. Calculate precision
num_pos = 0
for key, val in gt_pkl.items():
num_pos += len(val)
fp = np.cumsum(fp)
tp = np.cumsum(tp)
rec = tp / float(num_pos)
# avoid divide by zero in case the first matches a difficult gt
prec = tp / np.maximum(fp + tp, np.finfo(np.float64).eps)
ap = voc_ap(rec, prec, True)
return ap
def voc_eval_sds(det_file, seg_file, devkit_path, image_list, cls_name, cache_dir,
class_names, mask_size, binary_thresh, ov_thresh=0.5):
# 1. Check whether ground truth cache file exists
with open(image_list, 'r') as f:
lines = f.readlines()
image_names = [x.strip() for x in lines]
check_voc_sds_cache(cache_dir, devkit_path, image_names, class_names)
gt_cache = cache_dir + '/' + cls_name + '_mask_gt.pkl'
with open(gt_cache, 'rb') as f:
gt_pkl = cPickle.load(f)
# 2. Get predict pickle file for this class
with open(det_file, 'rb') as f:
boxes_pkl = cPickle.load(f)
with open(seg_file, 'rb') as f:
masks_pkl = cPickle.load(f)
# 3. Pre-compute number of total instances to allocate memory
num_image = len(image_names)
box_num = 0
for im_i in xrange(num_image):
box_num += len(boxes_pkl[im_i])
# 4. Re-organize all the predicted boxes
new_boxes = np.zeros((box_num, 5))
new_masks = np.zeros((box_num, mask_size, mask_size))
new_image = []
cnt = 0
for image_ind in xrange(len(image_names)):
boxes = boxes_pkl[image_ind]
masks = masks_pkl[image_ind]
num_instance = len(boxes)
for box_ind in xrange(num_instance):
new_boxes[cnt] = boxes[box_ind]
new_masks[cnt] = masks[box_ind]
new_image.append(image_names[image_ind])
cnt += 1
# 5. Rearrange boxes according to their scores
seg_scores = new_boxes[:, -1]
keep_inds = np.argsort(-seg_scores)
new_boxes = new_boxes[keep_inds, :]
new_masks = new_masks[keep_inds, :, :]
num_pred = new_boxes.shape[0]
import cv2
# 6. Calculate t/f positive
fp = np.zeros((num_pred, 1))
tp = np.zeros((num_pred, 1))
for i in xrange(num_pred):
pred_box = np.round(new_boxes[i, :4]).astype(int)
pred_mask = new_masks[i]
pred_mask = cv2.resize(pred_mask.astype(np.float32), (pred_box[2] - pred_box[0] + 1, pred_box[3] - pred_box[1] + 1))
pred_mask = pred_mask >= binary_thresh
image_index = new_image[keep_inds[i]]
if image_index not in gt_pkl:
fp[i] = 1
continue
gt_dict_list = gt_pkl[image_index]
# calculate max region overlap
cur_overlap = -1000
cur_overlap_ind = -1
for ind2, gt_dict in enumerate(gt_dict_list):
gt_mask_bound = np.round(gt_dict['mask_bound']).astype(int)
pred_mask_bound = pred_box
ov = mask_overlap(gt_mask_bound, pred_mask_bound, gt_dict['mask'], pred_mask)
if ov > cur_overlap:
cur_overlap = ov
cur_overlap_ind = ind2
if cur_overlap >= ov_thresh:
if gt_dict_list[cur_overlap_ind]['already_detect']:
fp[i] = 1
else:
tp[i] = 1
gt_dict_list[cur_overlap_ind]['already_detect'] = 1
else:
fp[i] = 1
# 7. Calculate precision
num_pos = 0
for key, val in gt_pkl.iteritems():
num_pos += len(val)
fp = np.cumsum(fp)
tp = np.cumsum(tp)
rec = tp / float(num_pos)
# avoid divide by zero in case the first matches a difficult gt
prec = tp / np.maximum(fp+tp, np.finfo(np.float64).eps)
ap = voc_ap(rec, prec, True)
return ap
