def positive_and_negative_selecting_strategy(proposals, logits, gt_boxes, gt_labels, neg_pos_ratio=3, min_size=5): # overlaps between the anchors and the gt boxes # overlaps (ex, gt) overlaps = bbox_overlaps(np.ascontiguousarray(proposals, dtype=np.float), np.ascontiguousarray(gt_boxes, dtype=np.float)) # max overlap index of each proposal argmax_overlaps = overlaps.argmax(axis=1) max_overlaps = overlaps[np.arange(proposals.shape[0]), argmax_overlaps] # max overlap index of each gt box gt_argmax_overlaps = overlaps.argmax(axis=0) gt_max_overlaps = overlaps[gt_argmax_overlaps, np.arange(overlaps.shape[1])] proposal_labels = np.empty((proposals.shape[0], ), dtype=np.int32) proposal_labels.fill(-1) proposal_bbox_index = np.zeros((proposals.shape[0], ), dtype=np.int32) # negative proposal_labels[max_overlaps < 0.3] = 0 # strategy 2 (positive) proposal_labels[max_overlaps > 0.5] = gt_labels[argmax_overlaps[ max_overlaps > 0.5]] proposal_bbox_index[max_overlaps > 0.5] = argmax_overlaps[ max_overlaps > 0.5] # strategy 1 (positive) proposal_labels[gt_argmax_overlaps] = gt_labels proposal_bbox_index[gt_argmax_overlaps] = np.arange(0, gt_boxes.shape[0]) # remove invalid proposals disable_keep_index = inv_filter_boxes(proposals, min_size) proposal_labels[disable_keep_index] = -1 # subsample negative labels if we have too many fg_inds = np.where(proposal_labels >= 1)[0] num_fg = len(fg_inds) bg_inds = np.where(proposal_labels == 0)[0] if len(bg_inds) > num_fg * neg_pos_ratio: disable_inds = npr.choice(bg_inds, size=(len(bg_inds) - num_fg * neg_pos_ratio), replace=False) proposal_labels[disable_inds] = -1 proposal_bbox_targets = bbox_transform(proposals, gt_boxes[proposal_bbox_index, :]) proposal_bbox_targets[proposal_labels < 1] = 0 return proposal_labels, proposal_bbox_targets
def eva(self, data, label): if label is not None: data = zip(data, label) bingo_num = 0 pool_size = 0 for det_predict, det_gt in data: # detected bbox det_bbox = det_predict['det-bbox'] # l2 distance between det bbox and gt bbox det_score = np.array(det_predict['det-score']) det_sorted_inds = np.argsort(det_score, axis=1).tolist() gt_bbox = det_gt['bbox'] gt_category = det_gt['category_id'] overlaps = bbox_overlaps( np.ascontiguousarray(det_bbox, dtype=np.float), np.ascontiguousarray(gt_bbox, dtype=np.float)) pool_size += len(gt_bbox) gtm = np.ones((len(gt_bbox))) * (-1) for dind, d in enumerate(det_bbox): best_match = -1 iou_thres = 0.5 for gind, g in enumerate(gt_bbox): if gtm[gind] >= 0: continue if overlaps[dind, gind] < iou_thres: continue iou_thres = overlaps[dind, gind] best_match = gind if best_match > -1: gtm[best_match] = dind if gt_category[best_match] in det_sorted_inds[dind][0:self. top_k]: bingo_num += 1 cmc = float(bingo_num) / float(pool_size) return { 'statistic': { 'name': self.name, 'value': [ { 'name': 'CMC', 'value': cmc, 'type': 'SCALAR', 'x': '', 'y': '' }, ] } }
def eva(self, data, label): # video list if label is not None: data = zip(data, label) all_trajectory_list = [] for video_predict, video_gt in data: trajectory_list = [] for det_predict, det_gt in zip(video_predict, video_gt): # predict bbox in frame det_bbox = np.array(det_predict['det-bbox']) det_socre = np.array(det_predict['det-score']) det_label = np.array( det_predict['det-label']).astype(dtype=np.int32) # gt bbox in frame gt_bbox = det_gt['bbox'] gt_category = det_gt['category_id'] gt_bbox_num = len(gt_bbox) if len(trajectory_list) == 0: trajectory_list = [[] for _ in range(gt_bbox_num)] overlaps = bbox_overlaps( np.ascontiguousarray(det_bbox, dtype=np.float), np.ascontiguousarray(gt_bbox, dtype=np.float)) gtm = np.ones((gt_bbox_num)) * (-1) for dind, d in enumerate(det_bbox.tolist()): best_match = -1 iou_thres = self.iou_thres for gind, g in enumerate(gt_bbox): if int(gt_category[gind]) != int(det_label[dind]): continue if gtm[gind] >= 0: continue if overlaps[dind, gind] < iou_thres: continue iou_thres = overlaps[dind, gind] best_match = gind if best_match > -1: gtm[best_match] = dind trajectory_list[int(gt_category[best_match])].append( (det_bbox[dind], gt_bbox[best_match], det_socre[dind], True)) for gind, g in enumerate(gt_bbox): if gtm[gind] == -1: trajectory_list[int(gt_category[gind])].append( ([], gt_bbox[gind], 0.0, False)) all_trajectory_list.extend(trajectory_list) trajectory_error_list = [] trajectory_f_error_list = [] trajectory_cp_error_list = [] trajectory_sr_error_list = [] for trajectory in all_trajectory_list: ordered_trajectory = sorted(trajectory, key=lambda x: x[2], reverse=True) error_f = self._fragment_error_fast(ordered_trajectory) error_cpe = self._center_position_error_fast(ordered_trajectory) error_sre = self._scale_ratio_error_fast(ordered_trajectory) trajectory_f_error_list.append(error_f.tolist()) trajectory_cp_error_list.append(error_cpe.tolist()) trajectory_sr_error_list.append(error_sre.tolist()) trajectory_error_list.append( (error_f + error_cpe + error_sre).tolist()) trajectory_error_mean = np.mean(trajectory_error_list, 0) trajectory_f_error_mean = np.mean(trajectory_f_error_list, 0) trajectory_cp_error_mean = np.mean(trajectory_cp_error_list, 0) trajectory_sr_error_mean = np.mean(trajectory_sr_error_list, 0) stability_e = np.sum(trajectory_error_mean) return { 'statistic': { 'name': self.name, 'value': [{ 'name': 'TRAJECTORY_STABILITY', 'value': stability_e, 'type': 'SCALAR', 'x': 'class', 'y': 'STABILITY' }, { 'name': 'TRAJECTORY_FRAGMENT_ERROR', 'value': trajectory_f_error_mean.tolist(), 'type': 'SCALAR', 'x': 'Detect Threshold', 'y': 'Fragment Error' }, { 'name': 'TRAJECTORY_CENTER_POSITION_ERROR', 'value': trajectory_cp_error_mean.tolist(), 'type': 'SCALAR', 'x': 'Detect Threshold', 'y': 'Center Position Error' }, { 'name': 'TRAJECTORY_SCALE_RATIO_ERROR', 'value': trajectory_sr_error_mean.tolist(), 'type': 'SCALAR', 'x': 'Detect Threshold', 'y': 'Scale Ratio Error' }, { 'name': 'TRAJECTORY_ERROR', 'value': trajectory_error_mean.tolist(), 'type': 'SCALAR', 'x': 'Detect Threshold', 'y': 'Total Error' }] } }
def eva(self, data, label): category_num = len(self.task.class_label) detection_score = [[] for _ in range(category_num)] detection_label = [[] for _ in range(category_num)] if label is not None: data = zip(data, label) # assert(len(data) == len(label)) # overlap_thre = float(getattr(self.task, 'APRF_overlap', 0.5)) # 1.step positive sample is overlap > overlap_thre predict_box_total = 0 for predict, gt in data: if predict is None: for missed_gt_bi in range(len(gt['bbox'])): detection_label[gt['category_id'][missed_gt_bi]].append(1) detection_score[gt['category_id'][missed_gt_bi]].append( -float("inf")) continue det_bbox = predict['det-bbox'] det_score = predict['det-score'].flatten() det_label = predict['det-label'].flatten() # ground truth bbox and categories gt_bbox = np.array(gt['bbox']) gt_category = np.array(gt['category_id']).astype(dtype=np.int32) gt_bbox_num = gt_bbox.shape[0] # predict position, category, and score predict_box = np.array(det_bbox) predict_category = np.array(det_label).astype(dtype=np.int32) predict_score = np.array(det_score) predict_box_total += predict_score.shape[0] overlaps = bbox_overlaps( np.ascontiguousarray(predict_box, dtype=np.float), np.ascontiguousarray(gt_bbox, dtype=np.float)) # distinguish positive and negative samples and scores (overlap > 0.5) gtm = np.ones((gt_bbox_num)) * (-1) for dind, d in enumerate(predict_box.tolist()): # information about best match so far (m=-1 -> unmatched) m = -1 iou = 0.5 for gind, g in enumerate(gt_bbox.tolist()): if gt_category[gind] != predict_category[dind]: continue # if this gt already matched continue if gtm[gind] >= 0: continue # continue to next gt unless better match made if overlaps[dind, gind] < iou: continue # if match successful and best so far, store appropriately iou = overlaps[dind, gind] m = gind # if match made store id of match for both dt and gt if m > -1: gtm[m] = dind # success to match detection_score[gt_category[m]].append(predict_score[dind]) detection_label[gt_category[m]].append(1) # process none matched det for dind, d in enumerate(predict_box.tolist()): if dind not in gtm: detection_score[predict_category[dind]].append( predict_score[dind]) detection_label[predict_category[dind]].append(0) # process missed gt bbox missed_gt_bbox = [_ for _ in range(gt_bbox_num) if gtm[_] == -1] for missed_gt_bi in missed_gt_bbox: detection_label[gt_category[missed_gt_bi]].append(1) detection_score[gt_category[missed_gt_bi]].append( -float("inf")) ######################################################################### # 2.step compute true positive(TP), false negative(FN), true negative(TN), false positive(FP) category_TP = [] category_FN = [] category_TN = [] category_FP = [] for category_detection_label, category_detection_score in zip( detection_label, detection_score): # skip 0 predicated_label = [ 1 if s > -float("inf") else 0 for s in category_detection_score ] res = binary_c_stats(actual=category_detection_label, predicated=predicated_label) TP, FN, TN, FP = res if res is not None else 0, 0, 0, 0 category_TP.append(TP) category_FN.append(FN) category_TN.append(TN) category_FP.append(FP) return { 'statistic': { 'name': self.name, 'value': [{ 'name': 'true-positive', 'value': category_TP, 'type': 'SCALAR', 'x': 'class', 'y': 'TP' }, { 'name': 'false-negative', 'value': category_FN, 'type': 'SCALAR', 'x': 'class', 'y': 'FN' }, { 'name': 'true-negative', 'value': category_TN, 'type': 'SCALAR', 'x': 'class', 'y': 'TN' }, { 'name': 'false-positive', 'value': category_FP, 'type': 'SCALAR', 'x': 'class', 'y': 'FP' }] }, }
def eva(self, data, label): category_num = len(self.task.class_label) detection_score = [[] for _ in range(category_num)] detection_label = [[] for _ in range(category_num)] if label is not None: data = zip(data, label) # assert(len(data) == len(label)) predict_box_total = 0 samples_scores = [] # 1.step positive sample is overlap > 0.5 for predict, gt in data: # sample id sample_id = gt['id'] if predict is None: for missed_gt_bi in range(len(gt['bbox'])): detection_label[gt['category_id'][missed_gt_bi]].append(1) detection_score[gt['category_id'][missed_gt_bi]].append( -float("inf")) continue det_bbox = predict['det-bbox'] det_score = predict['det-score'].flatten() det_label = predict['det-label'].flatten() # ground truth bbox and categories gt_bbox = np.array(gt['bbox']) gt_category = np.array(gt['category_id']).astype(dtype=np.int32) gt_bbox_num = gt_bbox.shape[0] # predict position, category, and score predict_box = np.array(det_bbox) predict_category = np.array(det_label).astype(dtype=np.int32) predict_score = np.array(det_score) predict_box_total += predict_score.shape[0] overlaps = bbox_overlaps( np.ascontiguousarray(predict_box, dtype=np.float), np.ascontiguousarray(gt_bbox, dtype=np.float)) #overlaps = _mask.iou(predict_box.tolist(), gt_bbox.tolist(), [0 for _ in range(gt_bbox_num)]) # distinguish positive and negative samples and scores (overlap > 0.5) gtm = np.ones((gt_bbox_num)) * (-1) for dind, d in enumerate(predict_box.tolist()): # information about best match so far (m=-1 -> unmatched) m = -1 iou = 0.5 for gind, g in enumerate(gt_bbox.tolist()): if gt_category[gind] != predict_category[dind]: continue # if this gt already matched continue if gtm[gind] >= 0: continue # continue to next gt unless better match made if overlaps[dind, gind] < iou: continue # if match successful and best so far, store appropriately iou = overlaps[dind, gind] m = gind # if match made store id of match for both dt and gt if m > -1: gtm[m] = dind # success to match detection_score[gt_category[m]].append(predict_score[dind]) detection_label[gt_category[m]].append(1) # record sample samples_scores.append({ 'id': sample_id, 'score': 1, 'category': gt_category[m], 'box': gt_bbox[m].tolist(), 'index': sample_id * 100 + m }) # process none matched det for dind, d in enumerate(predict_box.tolist()): if dind not in gtm: detection_score[predict_category[dind]].append( predict_score[dind]) detection_label[predict_category[dind]].append(0) # process missed gt bbox missed_gt_bbox = [_ for _ in range(gt_bbox_num) if gtm[_] == -1] for missed_gt_bi in missed_gt_bbox: detection_label[gt_category[missed_gt_bi]].append(1) detection_score[gt_category[missed_gt_bi]].append( -float("inf")) # record sample samples_scores.append({ 'id': sample_id, 'score': 0, 'category': gt_category[missed_gt_bi], 'box': gt_bbox[missed_gt_bi].tolist(), 'index': sample_id * 100 + missed_gt_bi }) ######################################################################### # 2.step compute mean average precision voc_mean_map = [] for predict, gt in zip(detection_label, detection_score): # skip 0 result = vmap(predict, gt) if result is None: result = 0.0 voc_mean_map.append(result) # 3.step make json voc_map = float(np.mean(voc_mean_map)) return { 'statistic': { 'name': self.name, 'value': [{ 'name': 'Mean-MAP', 'value': voc_map, 'type': 'SCALAR' }, { 'name': 'MAP', 'value': voc_mean_map, 'type': 'SCALAR', 'x': 'class', 'y': 'Mean Average Precision' }] }, 'info': samples_scores }
def eva(self, data, label): category_num = len(self.task.class_label) detection_score = [[] for _ in range(category_num)] detection_label = [[] for _ in range(category_num)] if label is not None: data = zip(data, label) # assert(len(data) == len(label)) # overlap_thre = float(getattr(self.task, 'pr_overlap', 0.5)) # 1.step positive sample is overlap > overlap_thre predict_box_total = 0 for predict, gt in data: if predict is None: for missed_gt_bi in range(len(gt['bbox'])): detection_label[gt['category_id'][missed_gt_bi]].append(1) detection_score[gt['category_id'][missed_gt_bi]].append( -float("inf")) continue det_bbox = predict['det-bbox'] det_score = predict['det-score'].flatten() det_label = predict['det-label'].flatten() # ground truth bbox and categories gt_bbox = np.array(gt['bbox']) gt_category = np.array(gt['category_id']).astype(dtype=np.int32) gt_bbox_num = gt_bbox.shape[0] # predict position, category, and score predict_box = np.array(det_bbox) predict_category = np.array(det_label).astype(dtype=np.int32) predict_score = np.array(det_score) predict_box_total += predict_score.shape[0] overlaps = bbox_overlaps( np.ascontiguousarray(predict_box, dtype=np.float), np.ascontiguousarray(gt_bbox, dtype=np.float)) # distinguish positive and negative samples and scores (overlap > 0.5) gtm = np.ones((gt_bbox_num)) * (-1) for dind, d in enumerate(predict_box.tolist()): # information about best match so far (m=-1 -> unmatched) m = -1 iou = 0.5 for gind, g in enumerate(gt_bbox.tolist()): if gt_category[gind] != predict_category[dind]: continue # if this gt already matched continue if gtm[gind] >= 0: continue # continue to next gt unless better match made if overlaps[dind, gind] < iou: continue # if match successful and best so far, store appropriately iou = overlaps[dind, gind] m = gind # if match made store id of match for both dt and gt if m > -1: gtm[m] = dind # success to match detection_score[gt_category[m]].append(predict_score[dind]) detection_label[gt_category[m]].append(1) # process none matched det for dind, d in enumerate(predict_box.tolist()): if dind not in gtm: detection_score[predict_category[dind]].append( predict_score[dind]) detection_label[predict_category[dind]].append(0) # process missed gt bbox missed_gt_bbox = [_ for _ in range(gt_bbox_num) if gtm[_] == -1] for missed_gt_bi in missed_gt_bbox: detection_label[gt_category[missed_gt_bi]].append(1) detection_score[gt_category[missed_gt_bi]].append( -float("inf")) ######################################################################### # 2.step compute Precicion Recall curve category_pr_curves = [] for predict, gt in zip(detection_label, detection_score): # skip 0 pr_curve = pr(predict, gt) if pr_curve is None: pr_curve = np.array(()) category_pr_curves.append(pr_curve.tolist()) return { 'statistic': { 'name': self.name, 'value': [{ 'name': 'Precision-Recall', 'value': category_pr_curves, 'type': 'CURVE', 'x': 'recall', 'y': 'precision' }] }, }