def compute_average_precision_per_class(args, num_true_cases, gt_boxes, difficult_cases, prediction_file, iou_threshold, use_2007_metric): with open(prediction_file) as f: image_ids = [] boxes = [] scores = [] for line in f: t = line.rstrip().split(" ") image_ids.append(t[0]) scores.append(float(t[1])) box = torch.tensor([float(v) for v in t[2:]]).unsqueeze(0) box -= 1.0 # convert to python format where indexes start from 0 boxes.append(box) scores = np.array(scores) sorted_indexes = np.argsort(-scores) boxes = [boxes[i] for i in sorted_indexes] image_ids = [image_ids[i] for i in sorted_indexes] true_positive = np.zeros(len(image_ids)) false_positive = np.zeros(len(image_ids)) matched = set() for i, image_id in enumerate(image_ids): box = boxes[i] if args['flow_control']['dataset_type'] == "voc": pass else: image_id = int(image_id) if image_id not in gt_boxes.keys(): false_positive[i] = 1 continue gt_box = gt_boxes[image_id] ious = box_utils.iou_of(box, gt_box) max_iou = torch.max(ious).item() max_arg = torch.argmax(ious).item() if max_iou > iou_threshold: if difficult_cases[image_id][max_arg] == 0: if (image_id, max_arg) not in matched: true_positive[i] = 1 matched.add((image_id, max_arg)) else: false_positive[i] = 1 else: false_positive[i] = 1 true_positive = true_positive.cumsum() false_positive = false_positive.cumsum() precision = true_positive / (true_positive + false_positive + 1e-8) recall = true_positive / num_true_cases # import pdb;pdb.set_trace() _vtp = true_positive[-1] if len(true_positive) != 0 else 0 _vfp = false_positive[-1] if len(false_positive) != 0 else 0 if use_2007_metric: return measurements.compute_voc2007_average_precision( precision, recall), _vtp / (_vtp + _vfp + 1e-8), _vtp / num_true_cases else: return measurements.compute_average_precision(precision, recall)
def compute_average_precision_per_class(num_true_cases, gt_boxes, difficult_cases, prediction_file, iou_threshold, use_2007_metric): """ num_true_cases : 実際にそのクラスに属するtarget総数 """ with open(prediction_file) as f: image_ids = [] boxes = [] scores = [] for line in f: # そのクラス分のprediction box 全部読み込み t = line.rstrip().split(" ") # t: [image_id, scores, boxes] image_ids.append(t[0]) scores.append(float(t[1])) box = torch.tensor([float(v) for v in t[2:]]).unsqueeze(0) box -= 1.0 # convert to python format where indexes start from 0 boxes.append(box) scores = np.array(scores) sorted_indexes = np.argsort( -scores ) # -付きなのでlarger is firstでindex // 全ボックス内でconfidenceの高い順にsort boxes = [boxes[i] for i in sorted_indexes] image_ids = [image_ids[i] for i in sorted_indexes] true_positive = np.zeros(len(image_ids)) false_positive = np.zeros(len(image_ids)) matched = set() for i, image_id in enumerate( image_ids ): # image_ids : このクラスと判断されたimageのid / gt_boxes : 実際にこのクラスだったボックス box = boxes[i] if image_id not in gt_boxes: false_positive[i] = 1 # このクラスの物体box,と予測して実際は違った continue gt_box = gt_boxes[image_id] # 多分複数ボックス(そのimage_id中のこのクラスの正解ボックス) ious = box_utils.iou_of(box, gt_box) max_iou = torch.max(ious).item() max_arg = torch.argmax(ious).item() if max_iou > iou_threshold: # 少なくともどれかのgtboxに対してIoU>しきい値であったpredictionが存在 if difficult_cases[image_id][max_arg] == 0: if ( image_id, max_arg ) not in matched: # これまで,同じimage_id中で今見てるボックス(max_arg)が検出されていない場合(not in match) true_positive[i] = 1 # max_argのボックス = true positive matched.add((image_id, max_arg)) else: # 同じgt_boxが既に検出され,true positiveとして数えられている: 同じ物体を重複してpredicしている場合など. false_positive[i] = 1 else: false_positive[i] = 1 true_positive = true_positive.cumsum() false_positive = false_positive.cumsum() precision = true_positive / (true_positive + false_positive) recall = true_positive / num_true_cases if use_2007_metric: return measurements.compute_voc2007_average_precision( precision, recall) else: return measurements.compute_average_precision(precision, recall)
def compute_average_precision_per_class( num_true_cases, gt_boxes, difficult_cases, prediction_file, iou_threshold, use_2007_metric, ): with open(str(prediction_file)) as f: image_ids = [] boxes = [] scores = [] for line in f: t = line.rstrip().split(" ") image_ids.append(t[0]) scores.append(float(t[1])) box = torch.tensor([float(v) for v in t[2:]]).unsqueeze(0) box -= 1.0 # convert to python format where indexes start from 0 boxes.append(box) scores = np.array(scores) sorted_indexes = np.argsort(-scores) boxes = [boxes[i] for i in sorted_indexes] image_ids = [image_ids[i] for i in sorted_indexes] true_positive = np.zeros(len(image_ids)) false_positive = np.zeros(len(image_ids)) matched = set() for i, image_id in enumerate(image_ids): box = boxes[i] if image_id not in gt_boxes: false_positive[i] = 1 continue gt_box = gt_boxes[image_id] ious = box_utils.iou_of(box, gt_box) max_iou = torch.max(ious).item() max_arg = torch.argmax(ious).item() if max_iou > iou_threshold: if difficult_cases[image_id][max_arg] == 0: if (image_id, max_arg) not in matched: true_positive[i] = 1 matched.add((image_id, max_arg)) else: false_positive[i] = 1 else: false_positive[i] = 1 true_positive = true_positive.cumsum() false_positive = false_positive.cumsum() precision = true_positive / (true_positive + false_positive) recall = true_positive / num_true_cases if use_2007_metric: return measurements.compute_voc2007_average_precision( precision, recall) else: return measurements.compute_average_precision(precision, recall)
def compute_average_precision_per_class(num_true_cases, gt_boxes, difficult_cases, prediction_file, iou_threshold, use_2007_metric): with open(prediction_file) as f: image_ids = [] boxes = [] scores = [] for line in f: #ImageID,Source,LabelName,Confidence,XMin,XMax,YMin,YMax,IsOccluded,IsTruncated,IsGroupOf,IsDepiction,IsInside,ClassId,ClassName t = line.rstrip().split(",") image_ids.append(t[0]) scores.append(float(t[3])) box = torch.tensor( [float(t[4]), float(t[6]), float(t[5]), float(t[7])]).unsqueeze(0) box -= 1.0 # convert to python format where indexes start from 0 boxes.append(box) scores = np.array(scores) sorted_indexes = np.argsort(-scores) boxes = [boxes[i] for i in sorted_indexes] image_ids = [image_ids[i] for i in sorted_indexes] true_positive = np.zeros(len(image_ids)) false_positive = np.zeros(len(image_ids)) matched = set() for i, image_id in enumerate(image_ids): box = boxes[i] if image_id not in gt_boxes: false_positive[i] = 1 continue gt_box = gt_boxes[image_id] ious = box_utils.iou_of(box, gt_box) max_iou = torch.max(ious).item() max_arg = torch.argmax(ious).item() if max_iou > iou_threshold: if difficult_cases[image_id][max_arg] == 0: if (image_id, max_arg) not in matched: true_positive[i] = 1 matched.add((image_id, max_arg)) else: false_positive[i] = 1 else: false_positive[i] = 1 true_positive = true_positive.cumsum() false_positive = false_positive.cumsum() precision = true_positive / (true_positive + false_positive) recall = true_positive / num_true_cases if use_2007_metric: return measurements.compute_voc2007_average_precision( precision, recall) else: return measurements.compute_average_precision(precision, recall)
def compute_average_precision_per_class(num_true_cases, gt_boxes, difficult_cases, id_list, score_list, box_list, iou_threshold, use_2007_metric): image_ids = id_list boxes = [] scores = score_list for bl in box_list: box = torch.tensor(bl).unsqueeze(0) box -= 1.0 # convert to python format where indexes start from 0 boxes.append(box) scores = np.array(scores) sorted_indexes = np.argsort(-scores) boxes = [boxes[i] for i in sorted_indexes] image_ids = [image_ids[i] for i in sorted_indexes] true_positive = np.zeros(len(image_ids)) false_positive = np.zeros(len(image_ids)) matched = set() for i, image_id in enumerate(image_ids): box = boxes[i] if image_id not in gt_boxes: false_positive[i] = 1 continue gt_box = gt_boxes[image_id] ious = box_utils.iou_of(box, gt_box) max_iou = torch.max(ious).item() max_arg = torch.argmax(ious).item() if max_iou > iou_threshold: if difficult_cases[image_id][max_arg] == 0: if (image_id, max_arg) not in matched: true_positive[i] = 1 matched.add((image_id, max_arg)) else: false_positive[i] = 1 else: false_positive[i] = 1 true_positive = true_positive.cumsum() false_positive = false_positive.cumsum() precision = true_positive / (true_positive + false_positive) recall = true_positive / num_true_cases if use_2007_metric: return compute_voc2007_average_precision(precision, recall) else: return compute_average_precision(precision, recall)
def coco_ap_per_class(num_true_cases, gt_boxes, difficult_cases, prediction_file, use_2007_metric): aps, precs, recalls = [], [], [] _measurement_func = measurements.compute_voc2007_average_precision if use_2007_metric else measurements.compute_average_precision with open(prediction_file) as f: image_ids = [] boxes = [] scores = [] for line in f: t = line.rstrip().split(" ") image_ids.append(t[0]) scores.append(float(t[1])) box = torch.tensor([float(v) for v in t[2:]]).unsqueeze(0) box -= 1.0 # convert to python format where indexes start from 0 boxes.append(box) scores = np.array(scores) sorted_indexes = np.argsort(-scores) boxes = [boxes[i] for i in sorted_indexes] image_ids = [image_ids[i] for i in sorted_indexes] max_ious = [] max_args = [] for i, image_id in enumerate(image_ids): box = boxes[i] image_id = int(image_id) # if image_id not in gt_boxes.keys(): # false_positive[i] = 1 # continue if image_id in gt_boxes.keys(): gt_box = gt_boxes[image_id] ious = box_utils.iou_of(box, gt_box) max_iou = torch.max(ious).item() max_arg = torch.argmax(ious).item() else: max_iou = 0.0 max_arg = 0 max_ious.append(max_iou) max_args.append(max_arg) for _iou_thresh in [0.5 + 0.05 * i for i in range(0, 10)]: true_positive = np.zeros(len(image_ids)) false_positive = np.zeros(len(image_ids)) matched = set() for i, (max_iou, max_arg, image_id) in enumerate(zip(max_ious, max_args, image_ids)): image_id = int(image_id) if max_iou > _iou_thresh: if difficult_cases[image_id][max_arg] == 0: if (image_id, max_arg) not in matched: true_positive[i] = 1 matched.add((image_id, max_arg)) else: false_positive[i] = 1 else: false_positive[i] = 1 true_positive = true_positive.cumsum() false_positive = false_positive.cumsum() precision = true_positive / (true_positive + false_positive + 1e-8) recall = true_positive / num_true_cases _vtp = true_positive[-1] if len(true_positive) != 0 else 0 _vfp = false_positive[-1] if len(false_positive) != 0 else 0 ap = _measurement_func(precision, recall) _prec = _vtp / (_vtp + _vfp + 1e-8) _recall = _vtp / num_true_cases aps.append(ap) precs.append(_prec) recalls.append(_recall) return sum(aps) / len(aps), sum(precs) / len(precs), sum(recalls) / len( recalls)
def calc_mAP(weights, batch_size=16, img_size=640, iou_thres=0.5, conf_thres=0.001, nms_thres=0.5, save_json=False, model=None): label_path = "models/train-coco_person_face-0.0.3-RFB-160/coco-person-face-labels.txt" class_names = [name.strip() for name in open(label_path).readlines()] num_classes = len(class_names) if opt.net_type == 'RFB': net = create_Mb_Tiny_RFB_fd(len(class_names), is_test=True, device="cuda:0") predictor = create_Mb_Tiny_RFB_fd_predictor(net, candidate_size=100, device="cuda:0") elif opt.net_type == 'slim': net = create_mb_tiny_fd(len(class_names), is_test=True, device="cuda:0") predictor = create_mb_tiny_fd_predictor(net, candidate_size=100, device="cuda:0") net.load(weights) net.eval() device = "cuda:0" # target_transform = MatchPrior(config.priors, config.center_variance, # config.size_variance, 0.34999999404) # # test_transform = TestTransform(config.image_size, config.image_mean_test, config.image_std) # val_dataset = COCODataset("/media/test/data/coco", transform=test_transform, # target_transform=target_transform, is_test=True) # logging.info("validation dataset size: {}".format(len(val_dataset))) # # val_loader = DataLoader(val_dataset, batch_size, # num_workers=4, # shuffle=False) data_list = json.load(open("/media/test/data/coco/test_datalist.json")) all_correct = None all_p_prob = None all_p_label = None all_g_label = None seen = 0 for data in tqdm(data_list): image_id = data['image_file'] gt_boxes = np.array(data['boxes'], dtype=np.float32) gt_labels = np.array(data['labels'], dtype=np.int64) image = cv2.imread(image_id) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) p_boxes, p_labels, p_probs = predictor.predict(image) nl = gt_labels.shape[0] correct = np.array([0] * p_boxes.shape[0]) for i, gt_box in enumerate(gt_boxes): seen += 1 p_index = np.array(range(p_boxes.shape[0])) gt_label = gt_labels[i] valid_p_boxes = p_boxes[correct == 0] # remove matched predic box valid_p_index = p_index[correct == 0] valid_p_probs = p_probs[correct == 0] valid_p_labels = p_labels[correct == 0] valid_p_boxes = valid_p_boxes[ valid_p_labels == gt_label] # select predict label == gt label valid_p_index = valid_p_index[valid_p_labels == gt_label] valid_p_probs = valid_p_probs[valid_p_labels == gt_label] if valid_p_boxes.shape[0] == 0: continue iou = iou_of(torch.tensor(valid_p_boxes), torch.tensor(np.expand_dims(gt_box, axis=0))) max_val = torch.max(iou) if max_val.item() > iou_thres: correct[valid_p_index[torch.argmax(iou).item()]] = 1 all_correct = np.concatenate( [all_correct, correct], axis=0) if all_correct is not None else correct all_p_prob = np.concatenate( [all_p_prob, p_probs], axis=0) if all_p_prob is not None else p_probs all_p_label = np.concatenate( [all_p_label, p_labels], axis=0) if all_p_label is not None else p_labels all_g_label = np.concatenate( [all_g_label, gt_labels], axis=0) if all_g_label is not None else gt_labels p, r, ap, f1, ap_class = ap_per_class(all_correct, all_p_prob, all_p_label, all_g_label) mp, mr, map, mf1 = p.mean(), r.mean(), ap.mean(), f1.mean() nt = np.bincount(all_g_label.astype(np.int64), minlength=2) # number of targets per class # Print results phead = '%30s' + '%10s' * 6 print(phead % ('type', 'total', 'total', 'mp', 'mr', 'map', 'mf1')) pf = '%30s' + '%10.3g' * 6 # print format print(pf % ('all', seen, nt.sum(), mp, mr, map, mf1)) # Print results per class names = ['BACKGROUND', 'person', 'face'] for i, c in enumerate(ap_class): print(pf % (names[c], seen, nt[c], p[i], r[i], ap[i], f1[i]))