def compute_gt_annotations(anchors,
annotations,
negative_overlap=0.4,
positive_overlap=0.5):
""" Obtain indices of gt annotations with the greatest overlap.
Args
anchors: np.array of annotations of shape (N, 4) for (x1, y1, x2, y2).
annotations: np.array of shape (N, 5) for (x1, y1, x2, y2, label).
negative_overlap: IoU overlap for negative anchors (all anchors with overlap < negative_overlap are negative).
positive_overlap: IoU overlap or positive anchors (all anchors with overlap > positive_overlap are positive).
Returns
positive_indices: indices of positive anchors
ignore_indices: indices of ignored anchors
argmax_overlaps_inds: ordered overlaps indices
"""
overlaps = compute_overlap(anchors.astype(np.float64),
annotations.astype(np.float64))
argmax_overlaps_inds = np.argmax(overlaps, axis=1)
max_overlaps = overlaps[np.arange(overlaps.shape[0]), argmax_overlaps_inds]
# assign "dont care" labels
positive_indices = max_overlaps >= positive_overlap
ignore_indices = (max_overlaps > negative_overlap) & ~positive_indices
return positive_indices, ignore_indices, argmax_overlaps_inds
def evaluate_batch(pred, target, score, iou_threshold):
pred_result = {}
pred_annotation = {}
pred_size = pred.shape[0]
detected_annotations = []
for i in range(pred_size):
pred_label = int(pred[i, 0])
if pred_label not in pred_result:
pred_result[pred_label] = []
# record [score, true or false]
s_ = float(score[i, 0])
box = np.expand_dims(pred[i, 1:5].astype(np.float64), axis=0)
overlap = compute_overlap(box, target[:, 1:5].astype(np.float64))
assigned_target = np.argmax(overlap, axis=1)
max_overlap = overlap[0, assigned_target]
if max_overlap >= iou_threshold and assigned_target not in detected_annotations:
record = (s_, 1)
detected_annotations.append(assigned_target)
else:
record = (s_, 0)
pred_result[pred_label].append(record)
for j in range(target.shape[0]):
target_label = int(target[j, 0])
if target_label not in pred_annotation:
pred_annotation[target_label] = 0
pred_annotation[target_label] += 1
return pred_result, pred_annotation
def compute_gt_annotations(anchors,
annotations,
negative_overlap=0.4,
positive_overlap=0.5):
# (N, K)
overlaps = compute_overlap(anchors.astype(np.float64),
annotations.astype(np.float64))
# (N, )
argmax_overlaps_indices = np.argmax(overlaps, axis=1)
# (N, )
max_overlaps = overlaps[np.arange(overlaps.shape[0]),
argmax_overlaps_indices]
# assign "dont care" labels
# (N, )
positive_indices = max_overlaps >= positive_overlap
# in case of there are gt boxes has no matched positive anchors
nonzero_indices = np.nonzero(overlaps == np.max(overlaps, axis=0))
positive_indices[nonzero_indices[0]] = 1
# (N, )
ignore_indices = (max_overlaps > negative_overlap) & ~positive_indices
return positive_indices, ignore_indices, argmax_overlaps_indices
def get_gt_indices(anchors,
annotations,
positive_overlap=0.5,
negative_overlap=0.4):
overlaps = compute_overlap(anchors.astype(np.float64),
annotations.astype(np.float64))
# 获得每个anchor对应的最大gt的idx
best_overlap_each_anchor_indices = np.argmax(overlaps, axis=1)
best_overlap_each_anchor = overlaps[np.arange(overlaps.shape[0]),
best_overlap_each_anchor_indices]
# print(np.max(best_overlap_each_anchor))
positive_indices = best_overlap_each_anchor > positive_overlap
ignore_indices = (best_overlap_each_anchor >
negative_overlap) & ~positive_indices
return positive_indices, ignore_indices, best_overlap_each_anchor_indices
def handle_multi_labels(self, boxes, labels, scores):
overlap = compute_overlap(boxes, boxes)
boxes_number = boxes.shape[0]
fathers = []
for i in range(0, boxes_number):
for j in range(i, boxes_number):
father, _ = self.data_loader.judge_similar(labels[i, 0], labels[j, 0])
if father != -1 and overlap[i, j] > 0.5:
fathers.append(father)
fathers = set(fathers)
sons = list(set(range(boxes_number))-fathers)
sons.sort()
boxes = boxes[sons]
scores = scores[sons]
labels = labels[sons]
return boxes, labels, scores
def compute_gt_annotations(
anchors,
annotations,
negative_overlap=0.4,
positive_overlap=0.5
):
"""
Obtain indices of gt annotations with the greatest overlap.
Args
anchors: np.array of annotations of shape (N, 4) for (x1, y1, x2, y2).
annotations: np.array of shape (K, 5) for (x1, y1, x2, y2, label).
negative_overlap: IoU overlap for negative anchors (all anchors with overlap < negative_overlap are negative).
positive_overlap: IoU overlap or positive anchors (all anchors with overlap > positive_overlap are positive).
Returns
positive_indices: indices of positive anchors, (N, )
ignore_indices: indices of ignored anchors, (N, )
argmax_overlaps_inds: ordered overlaps indices, (N, )
"""
# (N, K)
overlaps = compute_overlap(anchors.astype(np.float64), annotations.astype(np.float64))
# (N, )
argmax_overlaps_inds = np.argmax(overlaps, axis=1)
# (N, )
max_overlaps = overlaps[np.arange(overlaps.shape[0]), argmax_overlaps_inds]
# assign "dont care" labels
# (N, )
positive_indices = max_overlaps >= positive_overlap
#get the max overlapping anchor indices for each gt box and set them to true so that each gt box has at least one positive anchor box
max_overlapping_anchor_box_indices = np.argmax(overlaps, axis = 0)
positive_indices[max_overlapping_anchor_box_indices] = True
# (N, )
ignore_indices = (max_overlaps > negative_overlap) & ~positive_indices
return positive_indices, ignore_indices, argmax_overlaps_inds
def compute_gt_annotations(anchors,
annotations,
negative_overlap=0.4,
positive_overlap=0.5):
"""
Obtain indices of gt annotations with the greatest overlap.
Args
anchors: np.array of annotations of shape (N, 4) for (x1, y1, x2, y2).
annotations: np.array of shape (K, 5) for (x1, y1, x2, y2, label).
negative_overlap: IoU overlap for negative anchors (all anchors with overlap < negative_overlap are negative).
positive_overlap: IoU overlap or positive anchors (all anchors with overlap > positive_overlap are positive).
Returns
positive_indices: indices of positive anchors, (N, )
ignore_indices: indices of ignored anchors, (N, )
argmax_overlaps_inds: ordered overlaps indices, (N, )
"""
# (N, K)
overlaps = compute_overlap(anchors.astype(np.float64),
annotations.astype(np.float64))
# (N, )
argmax_overlaps_inds = np.argmax(overlaps, axis=1)
# (N, )
max_overlaps = overlaps[np.arange(overlaps.shape[0]), argmax_overlaps_inds]
# assign "dont care" labels
# (N, )
positive_indices = max_overlaps >= positive_overlap
# adam: in case of there are gt boxes has no matched positive anchors
# nonzero_inds = np.nonzero(overlaps == np.max(overlaps, axis=0))
# positive_indices[nonzero_inds[0]] = 1
# (N, )
ignore_indices = (max_overlaps > negative_overlap) & ~positive_indices
return positive_indices, ignore_indices, argmax_overlaps_inds
def evaluate(generator,
model,
iou_threshold=0.5,
score_threshold=0.01,
max_detections=100,
visualize=False,
epoch=0):
"""
Evaluate a given dataset using a given model.
Args:
generator: The generator that represents the dataset to evaluate.
model: The model to evaluate.
iou_threshold: The threshold used to consider when a detection is positive or negative.
score_threshold: The score confidence threshold to use for detections.
max_detections: The maximum number of detections to use per image.
visualize: Show the visualized detections or not.
Returns:
A dict mapping class names to mAP scores.
"""
# gather all detections and annotations
all_detections = _get_detections(generator,
model,
score_threshold=score_threshold,
max_detections=max_detections,
visualize=visualize)
all_annotations = _get_annotations(generator)
average_precisions = {}
num_tp = 0
num_fp = 0
# process detections and annotations
for label in range(generator.num_classes()):
if not generator.has_label(label):
continue
false_positives = np.zeros((0, ))
true_positives = np.zeros((0, ))
scores = np.zeros((0, ))
num_annotations = 0.0
for i in range(generator.size()):
detections = all_detections[i][label]
annotations = all_annotations[i][label]
num_annotations += annotations.shape[0]
detected_annotations = []
for d in detections:
scores = np.append(scores, d[4])
if annotations.shape[0] == 0:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
continue
overlaps = compute_overlap(np.expand_dims(d, axis=0),
annotations)
assigned_annotation = np.argmax(overlaps, axis=1)
max_overlap = overlaps[0, assigned_annotation]
if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:
false_positives = np.append(false_positives, 0)
true_positives = np.append(true_positives, 1)
detected_annotations.append(assigned_annotation)
else:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
# no annotations -> AP for this class is 0 (is this correct?)
if num_annotations == 0:
average_precisions[label] = 0, 0
continue
# sort by score
indices = np.argsort(-scores)
false_positives = false_positives[indices]
true_positives = true_positives[indices]
# compute false positives and true positives
false_positives = np.cumsum(false_positives)
true_positives = np.cumsum(true_positives)
if false_positives.shape[0] == 0:
num_fp += 0
else:
num_fp += false_positives[-1]
if true_positives.shape[0] == 0:
num_tp += 0
else:
num_tp += true_positives[-1]
# compute recall and precision
recall = true_positives / num_annotations
precision = true_positives / np.maximum(
true_positives + false_positives,
np.finfo(np.float64).eps)
print("Precision: " + str(precision))
print("Recall: " + str(recall))
# compute average precision
average_precision = _compute_ap(recall, precision)
average_precisions[label] = average_precision, num_annotations
print('num_fp={}, num_tp={}, num_annotations={}'.format(
num_fp, num_tp, num_annotations))
return average_precisions
def evaluate(generator,
model,
iou_threshold=0.25,
score_threshold=0.05,
max_detections=500,
save_path=None):
""" Evaluate a given dataset using a given model.
# Arguments
generator : The generator that represents the dataset to evaluate.
model : The model to evaluate.
iou_threshold : The threshold used to consider when a detection is positive or negative.
score_threshold : The score confidence threshold to use for detections.
max_detections : The maximum number of detections to use per image.
save_path : The path to save images with visualized detections to.
# Returns
A dict mapping class names to mAP scores.
"""
# gather all detections and annotations
## all detections include all those that don't reach the treshold
all_detections, image_names, detection_list, scores_list, labels_list = _get_detections(
generator,
model,
score_threshold=score_threshold,
max_detections=max_detections,
save_path=save_path)
all_annotations = _get_annotations(generator)
#print(image_names)
#print(detection_list)
## average_precisions is initialized as dictionary
average_precisions = {}
true_positives_dict = {}
false_positives_dict = {}
iou_dict = {}
# all_detections = pickle.load(open('all_detections.pkl', 'rb'))
# all_annotations = pickle.load(open('all_annotations.pkl', 'rb'))
# pickle.dump(all_detections, open('all_detections.pkl', 'wb'))
# pickle.dump(all_annotations, open('all_annotations.pkl', 'wb'))
## create different lists that I need
iou = []
#all_completed_detections = []
#image_index = []
#object_type_df = []
# process detections and annotations
## all this part is done for each class
## but only for classes that are actually present
## so it doesn't cover classes that detections were made on but which were not in the picture
## however, generator.num_classes()=7
for label in range(generator.num_classes()):
if not generator.has_label(label):
continue
false_positives = np.zeros((0, ))
true_positives = np.zeros((0, ))
scores = np.zeros((0, ))
num_annotations = 0.0
num_detections = 0.0
iou_per_class = []
for i in range(generator.size()):
detections = all_detections[i][label]
annotations = all_annotations[i][label]
num_annotations += annotations.shape[0]
detected_annotations = []
for d in detections:
scores = np.append(scores, d[4])
if annotations.shape[0] == 0:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
continue
overlaps = compute_overlap(np.expand_dims(d, axis=0),
annotations)
assigned_annotation = np.argmax(overlaps, axis=1)
max_overlap = overlaps[0, assigned_annotation]
## here we check if box if it is true or false positive
if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:
## I assume that IoU is only calculated for TP boxes
false_positives = np.append(false_positives, 0)
true_positives = np.append(true_positives, 1)
#print("bbox overlap: ",max_overlap)
iou.append(np.asscalar(max_overlap))
iou_per_class.append(np.asscalar(max_overlap))
#print("iou list: ",iou)
detected_annotations.append(assigned_annotation)
#all_completed_detections.append(d)
#image_index.append(i)
#print(image_index)
num_detections += 1
#print(detections)
else:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
## the way (I think) they do it in EAD: if the overlap doesn't reach treshold, it's 0
iou.append(0)
iou_per_class.append(0)
#print("Scores: ",scores)
# no annotations -> AP for this class is 0 (is this correct?)
## hid continue to see # of FP
if num_annotations == 0:
average_precisions[label] = 0, 0
#continue
# sort by score
indices = np.argsort(-scores)
false_positives = false_positives[indices]
true_positives = true_positives[indices]
# compute false positives and true positives
## cumsum returns the cumulative sum of the elements along a given axis
false_positives = np.cumsum(false_positives)
true_positives = np.cumsum(true_positives)
# compute recall and precision
## we divide an array by a scalar
recall = true_positives / num_annotations
precision = true_positives / np.maximum(
true_positives + false_positives,
np.finfo(np.float64).eps)
# compute average precision
average_precision = _compute_ap(recall, precision)
## average_precision will be in the following dictionary format: {0: (0.31672005587085506, 1112.0), 1: (0.08074755526818446, 107.0), 2: (0.19361940213603135, 291.0), 3: (0.12725467367537643, 57.0), 4: (0.20030495872509274, 121.0), 5: (0.06083609353943108, 481.0), 6: (0.41498412085028863, 89.0)}
average_precisions[label] = average_precision, num_annotations
## added dictionaries for TP and FP
## I use max, because true_positives is an array with accumulating TP's
if true_positives.size != 0:
true_positives_dict[label] = max(true_positives), num_annotations
else:
true_positives_dict[label] = 0, num_annotations
if false_positives.size != 0:
false_positives_dict[label] = max(false_positives), num_annotations
else:
false_positives_dict[label] = 0, num_annotations
iou_dict[label] = np.mean(iou), num_annotations
#print("Label: ",generator.label_to_name(label))
#print("FP: ",false_positives_dict)
#print("TP: ",true_positives_dict)
#print("AP: ", average_precision)
#print("precision: ",precision)
#print("recall: ",recall)
return false_positives_dict, true_positives_dict, iou_dict, average_precisions, iou, image_names, detection_list, scores_list, labels_list
def evaluate(generator,
model,
iou_threshold=0.5,
score_threshold=0.01,
max_detections=100,
visualize=False,
flip_test=False,
keep_resolution=False):
all_detections = _get_detections(generator,
model,
score_threshold=score_threshold,
max_detections=max_detections,
visualize=visualize,
flip_test=flip_test,
keep_resolution=keep_resolution)
all_annotations = _get_annotations(generator)
average_precisions = {}
# process detections and annotations
for label in range(generator.num_classes()):
if not generator.has_label(label):
continue
false_positives = np.zeros((0, ))
true_positives = np.zeros((0, ))
scores = np.zeros((0, ))
num_annotations = 0.0
for i in range(generator.size()):
detections = all_detections[i][label]
annotations = all_annotations[i][label]
num_annotations += annotations.shape[0]
detected_annotations = []
for d in detections:
scores = np.append(scores, d[4])
if annotations.shape[0] == 0:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
continue
overlaps = compute_overlap(np.expand_dims(d, axis=0),
annotations)
assigned_annotation = np.argmax(overlaps, axis=1)
max_overlap = overlaps[0, assigned_annotation]
if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:
false_positives = np.append(false_positives, 0)
true_positives = np.append(true_positives, 1)
detected_annotations.append(assigned_annotation)
else:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
if num_annotations == 0:
average_precisions[label] = 0, 0
continue
# sort by score
indices = np.argsort(-scores)
false_positives = false_positives[indices]
true_positives = true_positives[indices]
# compute false positives and true positives
false_positives = np.cumsum(false_positives)
true_positives = np.cumsum(true_positives)
# compute recall and precision
recall = true_positives / num_annotations
precision = true_positives / np.maximum(
true_positives + false_positives,
np.finfo(np.float64).eps)
# compute average precision
average_precision = _compute_ap(recall, precision)
average_precisions[label] = average_precision, num_annotations
return average_precisions
def evalution(generator,
model,
iou_threshold=0.5,
score_threshold=0.05,
max_detections=100,
save_path=None):
""" Evaluate a given dataset using a given model.
# Arguments
generator : The generator that represents the dataset to evaluate.
model : The model to evaluate.
iou_threshold : The threshold used to consider when a detection is positive or negative.
score_threshold : The score confidence threshold to use for detections.
max_detections : The maximum number of detections to use per image.
save_path : The path to save images with visualized detections to.
# Returns
A dict mapping class names to mAP scores.
"""
all_detections, all_inferences_time = _get_detections(
generator,
model,
score_threshold=score_threshold,
max_detections=max_detections,
save_path=save_path)
all_annotations = _get_annotations(generator)
average_precisions = {}
for label in range(generator.num_classes()):
if not generator.has(label):
continue
false_positives = np.zeros((0, ))
true_positives = np.zeros((0, ))
scores = np.zeros((0, ))
num_annotations = 0.0
num_detections = 0.0
for i in range(generator.size()):
detections = all_detections[i][label]
annotations = all_annotations[i][label]
# number of boxes * 4
num_annotations += annotations.shape[0]
num_detections += detections.shape[0]
detected_annotations = []
for d in detections:
scores = np.append(scores, d[4])
if annotations.shape[0] == 0:
np.append(false_positives, 1)
np.append(true_positives, 0)
continue
overlaps = compute_overlap(np.expand_dims(d[:-1], axis=0),
annotations)
assigned_labels = np.argmax(overlaps, axis=1)[0]
max_overlap = overlaps[0, assigned_labels]
# 如果多个检测框对应某一个GT的IoU都大于阈值？ detections本来就是排过序的
if max_overlap > iou_threshold and assigned_labels not in detected_annotations:
false_positives = np.append(false_positives, 0)
true_positives = np.append(true_positives, 1)
detected_annotations.append(assigned_labels)
else:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
if num_annotations == 0:
if num_detections == 0:
average_precisions[label] = 1, 0
continue
else:
average_precisions[label] = 0, 0
continue
indices = np.argsort(-scores)
false_positives = false_positives[indices]
true_positives = true_positives[indices]
# np.cumsum([0,1,0,1,1])
# array([0, 1, 1, 2, 3])
false_positives = np.cumsum(false_positives)
true_positives = np.cumsum(true_positives)
# recall = TP / number of gt
# prescision = TP /
recall = true_positives / num_annotations
prescision = true_positives / np.maximum(
true_positives + false_positives,
np.finfo(np.float64).eps)
average_precision = _compute_ap(recall, prescision)
average_precisions[label] = average_precision, num_annotations
inference_time_mean = np.sum(all_inferences_time) / generator.size()
return average_precisions, inference_time_mean
def evaluate(generator,
model,
iou_threshold=0.5,
score_threshold=0.05,
max_detections=100,
save_path=None,
epoch=0):
"""
Evaluate a given dataset using a given model.
Args:
generator: The generator that represents the dataset to evaluate.
model: The model to evaluate.
iou_threshold: The threshold used to consider when a detection is positive or negative.
score_threshold: The score confidence threshold to use for detections.
max_detections: The maximum number of detections to use per image.
save_path: The path to save images with visualized detections to.
epoch: epoch index
Returns:
A dict mapping class names to mAP scores.
"""
# gather all detections and annotations
all_detections = _get_detections(generator,
model,
score_threshold=score_threshold,
max_detections=max_detections,
save_path=save_path)
all_annotations = _get_annotations(generator)
average_precisions = {}
# all_detections = pickle.load(open('fcos/all_detections_11.pkl', 'rb'))
# all_annotations = pickle.load(open('fcos/all_annotations.pkl', 'rb'))
# pickle.dump(all_detections, open('fcos/all_detections_{}.pkl'.format(epoch + 1), 'wb'))
# pickle.dump(all_annotations, open('fcos/all_annotations_{}.pkl'.format(epoch + 1), 'wb'))
# process detections and annotations
for label in range(generator.num_classes()):
if not generator.has_label(label):
continue
false_positives = np.zeros((0, ))
true_positives = np.zeros((0, ))
scores = np.zeros((0, ))
num_annotations = 0.0
for i in range(generator.size()):
detections = all_detections[i][label]
annotations = all_annotations[i][label]
num_annotations += annotations.shape[0]
detected_annotations = []
for d in detections:
scores = np.append(scores, d[4])
if annotations.shape[0] == 0:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
continue
overlaps = compute_overlap(np.expand_dims(d, axis=0),
annotations)
assigned_annotation = np.argmax(overlaps, axis=1)
max_overlap = overlaps[0, assigned_annotation]
if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:
false_positives = np.append(false_positives, 0)
true_positives = np.append(true_positives, 1)
detected_annotations.append(assigned_annotation)
else:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
# no annotations -> AP for this class is 0 (is this correct?)
if num_annotations == 0:
average_precisions[label] = 0, 0
continue
# sort by score
indices = np.argsort(-scores)
false_positives = false_positives[indices]
true_positives = true_positives[indices]
# compute false positives and true positives
false_positives = np.cumsum(false_positives)
true_positives = np.cumsum(true_positives)
# compute recall and precision
recall = true_positives / num_annotations
precision = true_positives / np.maximum(
true_positives + false_positives,
np.finfo(np.float64).eps)
# compute average precision
average_precision = _compute_ap(recall, precision)
average_precisions[label] = average_precision, num_annotations
return average_precisions
with open('car.txt', 'r') as f:
img_lines = f.readlines()
save_path = '/Users/yanyan/data/plot_img_0'
if not os.path.exists(save_path):
os.makedirs(save_path)
bbox = []
for img_line in img_lines:
img_info = img_line.strip('\n').split('\t')
img_path = img_info[0]
img_boxes = img_info[1:]
img = cv2.imread(root + img_path)
img, scale = preprocess_image(img, image_size)
for img_box in img_boxes:
box = list(map(int, img_box.split(',')[:-2]))
bbox.append(box)
bbox_array = np.array(bbox) * scale
# (644, 49104)
overlaps = compute_overlap(bbox_array.astype(np.float64),
anchors.astype(np.float64))
# overlaps = compute_overlap(bbox_array.astype(anchors.astype(np.float64), np.float64))
print(overlaps.shape)
argmax_overlaps_inds = np.argmax(overlaps, axis=1)
max_overlaps = overlaps[np.arange(overlaps.shape[0]), argmax_overlaps_inds]
ingore = max_overlaps < 0.5
print(sum(ingore))
def evaluate(
generator,
model,
iou_threshold=0.5,
score_threshold=0.05,
max_detections=100,
save_path=None,
diameter_threshold=0.1,
):
""" Evaluate a given dataset using a given model.
# Arguments
generator: The generator that represents the dataset to evaluate.
model: The model to evaluate.
iou_threshold: The threshold used to consider when a detection is positive or negative.
score_threshold: The score confidence threshold to use for detections.
max_detections: The maximum number of detections to use per image.
save_path: The path to save images with visualized detections to.
diameter_threshold: Threshold relative to the object's diameter when a prdicted 6D pose in considered to be correct
# Returns
Several dictionaries mapping class names to the computed metrics.
"""
# gather all detections and annotations
all_detections = _get_detections(generator,
model,
score_threshold=score_threshold,
max_detections=max_detections,
save_path=save_path)
all_annotations = _get_annotations(generator)
all_3d_models = generator.get_models_3d_points_dict()
all_3d_model_diameters = generator.get_objects_diameter_dict()
average_precisions = {}
add_metric = {}
add_s_metric = {}
metric_5cm_5degree = {}
translation_diff_metric = {}
rotation_diff_metric = {}
metric_2d_projection = {}
mixed_add_and_add_s_metric = {}
average_point_distance_error_metric = {}
average_sym_point_distance_error_metric = {}
mixed_average_point_distance_error_metric = {}
# process detections and annotations
for label in range(generator.num_classes()):
if not generator.has_label(label):
continue
false_positives = np.zeros((0, ))
true_positives = np.zeros((0, ))
scores = np.zeros((0, ))
num_annotations = 0.0
true_positives_add = np.zeros((0, ))
true_positives_add_s = np.zeros((0, ))
model_3d_points = all_3d_models[label]
model_3d_diameter = all_3d_model_diameters[label]
true_positives_5cm_5degree = np.zeros((0, ))
translation_diffs = np.zeros((0, ))
rotation_diffs = np.zeros((0, ))
true_positives_2d_projection = np.zeros((0, ))
point_distance_errors = np.zeros((0, ))
point_sym_distance_errors = np.zeros((0, ))
for i in tqdm(range(generator.size())):
detections = all_detections[i][label][0]
detections_rotations = all_detections[i][label][1]
detections_translations = all_detections[i][label][2]
annotations = all_annotations[i][label][0]
annotations_rotations = all_annotations[i][label][1]
annotations_translations = all_annotations[i][label][2]
num_annotations += annotations.shape[0]
detected_annotations = []
for d, d_rotation, d_translation in zip(detections,
detections_rotations,
detections_translations):
scores = np.append(scores, d[4])
if annotations.shape[0] == 0:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
continue
overlaps = compute_overlap(np.expand_dims(d, axis=0),
annotations)
assigned_annotation = np.argmax(overlaps, axis=1)
max_overlap = overlaps[0, assigned_annotation]
assigned_rotation = annotations_rotations[
assigned_annotation, :3]
assigned_translation = annotations_translations[
assigned_annotation, :]
if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:
false_positives = np.append(false_positives, 0)
true_positives = np.append(true_positives, 1)
detected_annotations.append(assigned_annotation)
#correct 2d object detection => check if the 6d pose is also correct
is_correct_6d_pose_add, mean_distances_add, transformed_points_gt, transformed_points_pred = check_6d_pose_add(
model_3d_points,
model_3d_diameter,
rotation_gt=generator.axis_angle_to_rotation_mat(
assigned_rotation),
translation_gt=np.squeeze(assigned_translation),
rotation_pred=generator.axis_angle_to_rotation_mat(
d_rotation),
translation_pred=d_translation,
diameter_threshold=diameter_threshold)
is_correct_6d_pose_add_s, mean_distances_add_s = check_6d_pose_add_s(
model_3d_points,
model_3d_diameter,
rotation_gt=generator.axis_angle_to_rotation_mat(
assigned_rotation),
translation_gt=np.squeeze(assigned_translation),
rotation_pred=generator.axis_angle_to_rotation_mat(
d_rotation),
translation_pred=d_translation,
diameter_threshold=diameter_threshold)
is_correct_6d_pose_5cm_5degree, translation_distance, rotation_distance = check_6d_pose_5cm_5degree(
rotation_gt=generator.axis_angle_to_rotation_mat(
assigned_rotation),
translation_gt=np.squeeze(assigned_translation),
rotation_pred=generator.axis_angle_to_rotation_mat(
d_rotation),
translation_pred=d_translation)
is_correct_2d_projection = check_6d_pose_2d_reprojection(
model_3d_points,
rotation_gt=generator.axis_angle_to_rotation_mat(
assigned_rotation),
translation_gt=np.squeeze(assigned_translation),
rotation_pred=generator.axis_angle_to_rotation_mat(
d_rotation),
translation_pred=d_translation,
camera_matrix=generator.load_camera_matrix(i),
pixel_threshold=5.0)
# #draw transformed gt points in image to test the transformation
test_draw(generator.load_image(i),
generator.load_camera_matrix(i),
transformed_points_pred, i)
if is_correct_6d_pose_add:
true_positives_add = np.append(true_positives_add, 1)
if is_correct_6d_pose_add_s:
true_positives_add_s = np.append(
true_positives_add_s, 1)
if is_correct_6d_pose_5cm_5degree:
true_positives_5cm_5degree = np.append(
true_positives_5cm_5degree, 1)
if is_correct_2d_projection:
true_positives_2d_projection = np.append(
true_positives_2d_projection, 1)
translation_diffs = np.append(translation_diffs,
translation_distance)
rotation_diffs = np.append(rotation_diffs,
rotation_distance)
point_distance_errors = np.append(point_distance_errors,
mean_distances_add)
point_sym_distance_errors = np.append(
point_sym_distance_errors, mean_distances_add_s)
else:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
# no annotations -> AP for this class is 0 (is this correct?)
if num_annotations == 0:
average_precisions[label] = 0, 0
continue
# sort by score
indices = np.argsort(-scores)
false_positives = false_positives[indices]
true_positives = true_positives[indices]
# compute false positives and true positives
false_positives = np.cumsum(false_positives)
true_positives = np.cumsum(true_positives)
# compute recall and precision
recall = true_positives / num_annotations
precision = true_positives / np.maximum(
true_positives + false_positives,
np.finfo(np.float64).eps)
# compute average precision
average_precision = _compute_ap(recall, precision)
average_precisions[label] = average_precision, num_annotations
#compute add accuracy
add_accuracy = np.sum(true_positives_add) / num_annotations
add_metric[label] = add_accuracy, num_annotations
#compute add-s accuracy
add_s_accuracy = np.sum(true_positives_add_s) / num_annotations
add_s_metric[label] = add_s_accuracy, num_annotations
#compute 5cm 5degree accuracy
accuracy_5cm_5degree = np.sum(
true_positives_5cm_5degree) / num_annotations
metric_5cm_5degree[label] = accuracy_5cm_5degree, num_annotations
#compute the mean and std of the translation- and rotation differences
mean_translations = np.mean(translation_diffs)
std_translations = np.std(translation_diffs)
translation_diff_metric[label] = mean_translations, std_translations
mean_rotations = np.mean(rotation_diffs)
std_rotations = np.std(rotation_diffs)
rotation_diff_metric[label] = mean_rotations, std_rotations
#compute 2d projection accuracy
accuracy_2d_projection = np.sum(
true_positives_2d_projection) / num_annotations
metric_2d_projection[label] = accuracy_2d_projection, num_annotations
#compute the mean and std of the transformed point errors
mean_point_distance_errors = np.mean(point_distance_errors)
std_point_distance_errors = np.std(point_distance_errors)
average_point_distance_error_metric[
label] = mean_point_distance_errors, std_point_distance_errors
#compute the mean and std of the symmetric transformed point errors
mean_point_sym_distance_errors = np.mean(point_sym_distance_errors)
std_point_sym_distance_errors = np.std(point_sym_distance_errors)
average_sym_point_distance_error_metric[
label] = mean_point_sym_distance_errors, std_point_sym_distance_errors
#fill in the add values for asymmetric objects and add-s for symmetric objects
for label, add_tuple in add_metric.items():
add_s_tuple = add_s_metric[label]
if generator.class_labels_to_object_ids[
label] in generator.symmetric_objects:
mixed_add_and_add_s_metric[label] = add_s_tuple
else:
mixed_add_and_add_s_metric[label] = add_tuple
#fill in the average point distance values for asymmetric objects and the corresponding average sym point distances for symmetric objects
for label, asym_tuple in average_point_distance_error_metric.items():
sym_tuple = average_sym_point_distance_error_metric[label]
if generator.class_labels_to_object_ids[
label] in generator.symmetric_objects:
mixed_average_point_distance_error_metric[label] = sym_tuple
else:
mixed_average_point_distance_error_metric[label] = asym_tuple
return average_precisions, add_metric, add_s_metric, metric_5cm_5degree, translation_diff_metric, rotation_diff_metric, metric_2d_projection, mixed_add_and_add_s_metric, average_point_distance_error_metric, average_sym_point_distance_error_metric, mixed_average_point_distance_error_metric
def evaluate_mAP(generator,
model,
method='iou',
start_threshold=0.5,
score_threshold=0.01,
max_detections=100,
visualize=False,
epoch=0):
"""
Evaluate a given dataset using a given model for mAPstart_threshold:0.95.
Args:
generator: The generator that represents the dataset to evaluate.
model: The model to evaluate.
iou_threshold: The threshold used to consider when a detection is positive or negative.
score_threshold: The score confidence threshold to use for detections.
max_detections: The maximum number of detections to use per image.
visualize: Show the visualized detections or not.
Returns:
A dict mapping class names to mAP scores.
"""
if method == 'iou':
from utils.compute_overlap import compute_overlap
elif method == 'piou':
from utils.compute_overlap_piou import compute_overlap
# gather all detections and annotations
all_detections = _get_detections(generator,
model,
score_threshold=score_threshold,
max_detections=max_detections)
all_annotations = _get_annotations(generator)
average_precisions = {}
num_tp = 0
num_fp = 0
# process detections and annotations
for iou_threshold in np.arange(start_threshold, 1.0, step=0.05):
for label in progressbar.progressbar(
range(generator.num_classes()),
prefix=f'Evaluating threshold {iou_threshold:.2f}: '):
if not generator.has_label(label):
continue
false_positives = np.zeros((0, ))
true_positives = np.zeros((0, ))
scores = np.zeros((0, ))
num_annotations = 0.0
for i in range(generator.size()):
detections = all_detections[i][label]
annotations = all_annotations[i][label]
num_annotations += annotations.shape[0]
detected_annotations = []
for d in detections:
scores = np.append(scores, d[4])
if annotations.shape[0] == 0:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
continue
overlaps = compute_overlap(np.expand_dims(d, axis=0),
annotations)
assigned_annotation = np.argmax(overlaps, axis=1)
max_overlap = overlaps[0, assigned_annotation]
if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:
false_positives = np.append(false_positives, 0)
true_positives = np.append(true_positives, 1)
detected_annotations.append(assigned_annotation)
else:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
# no annotations -> AP for this class is 0 (is this correct?)
if num_annotations == 0:
average_precisions[label] = 0, 0
continue
# sort by score
indices = np.argsort(-scores)
false_positives = false_positives[indices]
true_positives = true_positives[indices]
# compute false positives and true positives
false_positives = np.cumsum(false_positives)
true_positives = np.cumsum(true_positives)
if false_positives.shape[0] == 0:
num_fp += 0
else:
num_fp += false_positives[-1]
if true_positives.shape[0] == 0:
num_tp += 0
else:
num_tp += true_positives[-1]
# compute recall and precision
recall = true_positives / num_annotations
precision = true_positives / np.maximum(
true_positives + false_positives,
np.finfo(np.float64).eps)
# store precision for give threshold
average_precision = _compute_ap(recall, precision)
if not (label in average_precisions):
average_precisions[label] = {'map': [], 'ann': []}
average_precisions[label]['map'].append([average_precision])
average_precisions[label]['ann'].append([num_annotations])
# average precision for the given iou threshold
ap_iou = [
average_precisions[label]['map'][-1]
for label in average_precisions
]
print(f'AP{iou_threshold:.2f} = {np.mean(ap_iou):.4f}')
# compute average precision
for label in average_precisions:
average_precision = average_precisions[label]['map']
num_annotations = average_precisions[label]['ann']
average_precisions[label] = (np.mean(average_precision),
np.mean(num_annotations))
print('num_fp={}, num_tp={}'.format(num_fp, num_tp))
return average_precisions
