def test_exact_iou():
#One box entire inside another
prediction = box(minx=0, miny=0, maxx=10, maxy=10)
truth = box(minx=0, miny=0, maxx=5, maxy=5)
source_iou = average_precision.IoU(truth, prediction)
assert source_iou == 25 / 100
true_array = np.expand_dims(np.array(truth.bounds), axis=0)
prediction_array = np.expand_dims(np.array([*prediction.bounds]), axis=0)
retinanet_iou = compute_overlap(prediction_array, true_array)
assert retinanet_iou[0][0] == source_iou
#Poorly overlapping
prediction = box(minx=0, miny=0, maxx=5, maxy=5)
truth = box(minx=4, miny=4, maxx=9, maxy=9)
source_iou = average_precision.IoU(truth, prediction)
assert source_iou == (1 / 49)
true_array = np.expand_dims(np.array(truth.bounds), axis=0)
prediction_array = np.expand_dims(np.array([*prediction.bounds]), axis=0)
retinanet_iou = compute_overlap(prediction_array, true_array)
assert retinanet_iou == source_iou
def compute_measures(generator, label, iou_threshold, all_detections, all_annotations):
false_positives = np.zeros((0,))
true_positives = np.zeros((0,))
scores = np.zeros((0,))
num_annotations = 0.0
for i in range(generator.size()):
detections = all_detections[i][label]
annotations = all_annotations[i][label]
num_annotations += annotations.shape[0]
detected_annotations = []
sort_by_score_index = detections[:,4].argsort()[::-1]
for d in detections[sort_by_score_index]:
scores = np.append(scores, d[4])
if num_annotations == 0:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
continue
overlaps = compute_overlap(np.expand_dims(d, axis=0), annotations)
assigned_annotation = np.argmax(overlaps, axis=1) if overlaps.size else 0
max_overlap = overlaps[0, assigned_annotation] if overlaps.size else 0
if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:
false_positives = np.append(false_positives, 0)
true_positives = np.append(true_positives, 1)
detected_annotations.append(assigned_annotation)
else:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
return true_positives, false_positives, scores, num_annotations
def retinant_true_positive(from_retinanet):
all_detections, all_annotations, validation_generator = from_retinanet
iou_threshold = 0.5
boxes = []
##Retinanet
#walk through entire eval to find why the next test fails
# process detections and annotations
for label in range(validation_generator.num_classes()):
if not validation_generator.has_label(label):
continue
false_positives = np.zeros((0, ))
true_positives = np.zeros((0, ))
scores = np.zeros((0, ))
num_annotations = 0.0
iou_score = []
for i in range(validation_generator.size()):
detections = all_detections[i][label]
annotations = all_annotations[i][label]
num_annotations += annotations.shape[0]
detected_annotations = []
for d in detections:
scores = np.append(scores, d[4])
boxes.append(box(*d[:4]))
if annotations.shape[0] == 0:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
continue
overlaps = compute_overlap(np.expand_dims(d, axis=0),
annotations)
assigned_annotation = np.argmax(overlaps, axis=1)
max_overlap = overlaps[0, assigned_annotation]
iou_score.append(max_overlap)
if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:
false_positives = np.append(false_positives, 0)
true_positives = np.append(true_positives, 1)
detected_annotations.append(assigned_annotation)
else:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
result = pd.DataFrame({
"box": boxes,
"match": true_positives,
"iou": iou_score
})
return result
def average_overlap(values,
entries,
state,
image_shape,
mode="focal",
ratio_count=3,
include_stride=False):
anchor_params = calculate_config(values, ratio_count)
if include_stride:
anchors = anchors_for_shape(image_shape, anchor_params=anchor_params)
else:
anchors = base_anchors_for_shape(anchor_params=anchor_params)
overlap = compute_overlap(entries, anchors)
max_overlap = np.amax(overlap, axis=1)
not_matched = len(np.where(max_overlap < 0.5)[0])
if mode == "avg":
result = 1 - np.average(max_overlap)
elif mode == "ce":
result = np.average(-np.log(max_overlap))
elif mode == "focal":
result = np.average(-(1 - max_overlap)**2 * np.log(max_overlap))
else:
raise Exception("Invalid mode.")
if "iteration" not in state:
state["iteration"] = 0
else:
state["iteration"] += 1
if result < state["best_result"]:
state["best_result"] = result
print(f"Current best anchor configuration {result}",
state["iteration"])
print(f"Ratios: {sorted(np.round(anchor_params.ratios, 3))}")
print(f"Scales: {sorted(np.round(anchor_params.scales, 3))}")
if include_stride:
print(f"Average overlap: {np.round(np.average(max_overlap), 3)}")
print(
f"Number of labels that don't have any matching anchor: {not_matched}"
)
print()
return result, not_matched
for i in range(generator.size()-9400):
>>>>>>> 19c53023491d92692b7d44f1343b74e2d6623f42
detections = all_detections[i][label]
annotations = all_annotations[i][label]
num_annotations += annotations.shape[0]
detected_annotations = []
for d in detections:
scores = np.append(scores, d[4])
if annotations.shape[0] == 0:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
continue
overlaps = compute_overlap(np.expand_dims(d, axis=0), annotations)
assigned_annotation = np.argmax(overlaps, axis=1)
max_overlap = overlaps[0, assigned_annotation]
if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:
false_positives = np.append(false_positives, 0)
true_positives = np.append(true_positives, 1)
detected_annotations.append(assigned_annotation)
else:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
# no annotations -> AP for this class is 0 (is this correct?)
if num_annotations == 0:
average_precisions[label] = 0, 0
continue
def evaluate(generator,
model,
iou_threshold=0.5,
score_threshold=0.05,
max_detections=200,
save_path=None,
experiment=None):
""" Evaluate the mAP for given dataset using a given model.
# Arguments
generator : The generator that represents the dataset to evaluate.
model : The model to evaluate.
iou_threshold : The threshold used to consider when a detection is positive or negative.
score_threshold : The score confidence threshold to use for detections.
max_detections : The maximum number of detections to use per image.
save_path : The path to save images with visualized detections to.
experiment : Comet ml experiment to evaluate
# Returns
A dict mapping class names to mAP scores.
"""
# gather all detections and annotations
all_detections = _get_detections(generator,
model,
score_threshold=score_threshold,
max_detections=max_detections,
save_path=save_path,
experiment=experiment)
all_annotations = _get_annotations(generator)
average_precisions = {}
# all_detections = pickle.load(open('all_detections.pkl', 'rb'))
# all_annotations = pickle.load(open('all_annotations.pkl', 'rb'))
# pickle.dump(all_detections, open('all_detections.pkl', 'wb'))
# pickle.dump(all_annotations, open('all_annotations.pkl', 'wb'))
# process detections and annotations
for label in range(generator.num_classes()):
false_positives = np.zeros((0, ))
true_positives = np.zeros((0, ))
scores = np.zeros((0, ))
num_annotations = 0.0
for i in range(generator.size()):
detections = all_detections[i][label]
annotations = all_annotations[i][label]
num_annotations += annotations.shape[0]
detected_annotations = []
try:
_ = len(detections)
except:
print("No detections")
continue
for d in detections:
scores = np.append(scores, d[4])
if annotations.shape[0] == 0:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
continue
overlaps = compute_overlap(np.expand_dims(d, axis=0),
annotations)
assigned_annotation = np.argmax(overlaps, axis=1)
max_overlap = overlaps[0, assigned_annotation]
if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:
false_positives = np.append(false_positives, 0)
true_positives = np.append(true_positives, 1)
detected_annotations.append(assigned_annotation)
else:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
# no annotations -> AP for this class is 0 (is this correct?)
if num_annotations == 0:
average_precisions[label] = 0, 0
continue
# sort by score
indices = np.argsort(-scores)
false_positives = false_positives[indices]
true_positives = true_positives[indices]
# compute false positives and true positives
false_positives = np.cumsum(false_positives)
true_positives = np.cumsum(true_positives)
# compute recall and precision
recall = true_positives / num_annotations
precision = true_positives / np.maximum(
true_positives + false_positives,
np.finfo(np.float64).eps)
if len(recall) > 0:
print(
f"At score threshold {score_threshold}, the IoU recall is {recall[-1]} and precision is {precision[-1]}"
)
else:
print("None of the annotations exceeded score threshold")
# compute average precision
average_precision = _compute_ap(recall, precision)
average_precisions[label] = average_precision, num_annotations
return average_precisions
def test_iou():
true_array = np.expand_dims(np.array([0., 0., 5., 5.]), axis=0)
prediction_array = np.expand_dims(np.array([0., 0., 10., 10.]), axis=0)
retinanet_iou = compute_overlap(prediction_array, true_array)
assert retinanet_iou[0][0] == (5**2 / 10**2)
def find_validation_score():
from keras_retinanet.utils.anchors import compute_overlap
iou_threshold = 0.5
tc1, tc2 = get_class_name_mappings()
d1, d2 = get_description_for_labels()
all_files = glob.glob(DATASET_PATH + 'validation_big/*.jpg')
all_ids = []
for a in all_files:
all_ids.append(os.path.basename(a)[:-4])
print('Total image files: {} {}'.format(len(all_files), len(all_ids)))
valid = pd.read_csv(DATASET_PATH +
'annotations/validation-annotations-bbox.csv')
print('Number of files in annotations: {}'.format(
len(valid['ImageID'].unique())))
boxes_files = glob.glob(OUTPUT_PATH + 'cache_tensorflow_validation/*.pklz')
print('Predictions found: {}'.format(len(boxes_files)))
unique_classes = valid['LabelName'].unique()
print('Unique classes: {}'.format(len(unique_classes)))
print('Read detections...')
all_detections = get_detections(boxes_files)
print('Read annotations...')
all_annotations = get_real_annotations(valid, unique_classes)
average_precisions = {}
for zz, label in enumerate(unique_classes):
print('Go for: {} ({})'.format(d1[label], label))
if label not in tc2:
average_precisions[label] = 0, 1
continue
false_positives = np.zeros((0, ))
true_positives = np.zeros((0, ))
scores = np.zeros((0, ))
num_annotations = 0.0
for i in range(len(all_ids)):
detections = []
annotations = []
id = all_ids[i]
if id in all_detections:
if label in all_detections[id]:
detections = all_detections[id][label]
if id in all_annotations:
if label in all_annotations[id]:
annotations = all_annotations[id][label]
if len(detections) == 0 and len(annotations) == 0:
continue
if 0:
if len(detections) > 0 and len(annotations) > 0:
print(detections)
print(annotations)
print('----')
num_annotations += len(annotations)
detected_annotations = []
for d in detections:
scores = np.append(scores, d[4])
if len(annotations) == 0:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
continue
overlaps = compute_overlap(
np.expand_dims(np.array(d, dtype=np.float64), axis=0),
np.array(annotations, dtype=np.float64))
assigned_annotation = np.argmax(overlaps, axis=1)
max_overlap = overlaps[0, assigned_annotation]
if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:
false_positives = np.append(false_positives, 0)
true_positives = np.append(true_positives, 1)
detected_annotations.append(assigned_annotation)
else:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
if num_annotations == 0:
average_precisions[label] = 0, 0
continue
# sort by score
indices = np.argsort(-scores)
false_positives = false_positives[indices]
true_positives = true_positives[indices]
# compute false positives and true positives
false_positives = np.cumsum(false_positives)
true_positives = np.cumsum(true_positives)
# compute recall and precision
recall = true_positives / num_annotations
precision = true_positives / np.maximum(
true_positives + false_positives,
np.finfo(np.float64).eps)
# compute average precision
average_precision = _compute_ap(recall, precision)
average_precisions[label] = average_precision, num_annotations
print(d1[label], average_precision, num_annotations)
present_classes = 0
precision = 0
for label, (average_precision,
num_annotations) in average_precisions.items():
print('{:.0f} instances of class'.format(num_annotations), d1[label],
'with average precision: {:.4f}'.format(average_precision))
if num_annotations > 0:
present_classes += 1
precision += average_precision
mean_ap = precision / present_classes
print('mAP: {}'.format(mean_ap))
def evaluate(self):
# Initialise Ground Truth data
self.load_ground_truths()
#Initialise Detections
self.populate_detections()
for l in range(self.generator.num_classes()):
tp = np.zeros((0, ))
fp = np.zeros((0, ))
scores = np.zeros((0, ))
npos = 0
for i in range(self.generator.size()):
dets = self.detections[i][l]
gt = self.ground_truth[i][l]
npos += gt.shape[0]
detected_record = []
for detection in dets:
# Append the score first as we might skip!
scores = np.append(scores, detection[4])
# Ensure shape.
if gt.shape[0] == 0:
fp = np.append(fp, 1)
tp = np.append(tp, 0)
continue
#Calculate overlap
overlaps = compute_overlap(
np.expand_dims(detection, axis=0), gt)
best_overlap = np.argmax(overlaps, axis=1)
if best_overlap not in detected_record and overlaps[
0, best_overlap] > self.iou_threshold:
# Only capture once
detected_record.append(best_overlap)
# Positive count.
fp = np.append(fp, 0)
tp = np.append(tp, 1)
else:
# Negative count.
fp = np.append(fp, 1)
tp = np.append(tp, 0)
indices = np.argsort(-scores)
tp = tp[indices]
fp = fp[indices]
tp_score = np.cumsum(tp)
fp_score = np.cumsum(fp)
recall = tp_score / npos
precision = tp_score / np.maximum(tp_score + fp_score,
np.finfo(np.float64).eps)
ap = self._compute_ap(recall, precision)
self.average_precisions = np.append(self.average_precisions, ap)
print(self.generator.label_to_name(l), ap)
print("mAP: {}".format(self.average_precisions.mean()))
