def test_mean_average_precision_6():
"""
Multiple wrong prediction because of wrong location (box coordinates), but all with higher scores.
In this case AP should be the precision value @recall=1.0
"""
gt = GroundTruthBoundingBox(clazz="a", box=np.array([10, 10, 20, 20]), image_id="1")
pred1 = PredictedBoundingBox(clazz="a", box=np.array([10, 10, 20, 20]), image_id="1", score=0.8)
pred2 = PredictedBoundingBox(
clazz="a", box=np.array([50, 50, 60, 60]), image_id="1", score=0.85
)
pred3 = PredictedBoundingBox(
clazz="a", box=np.array([50, 50, 60, 60]), image_id="1", score=0.86
)
pred4 = PredictedBoundingBox(
clazz="a", box=np.array([50, 50, 60, 60]), image_id="1", score=0.87
)
pred5 = PredictedBoundingBox(
clazz="a", box=np.array([50, 50, 60, 60]), image_id="1", score=0.88
)
pred6 = PredictedBoundingBox(
clazz="a", box=np.array([50, 50, 60, 60]), image_id="1", score=0.89
)
pred7 = PredictedBoundingBox(clazz="a", box=np.array([50, 50, 60, 60]), image_id="1", score=0.9)
mAP = BoxMeanAveragePrecision({gt}, {pred1, pred2, pred3, pred4, pred5, pred6, pred7}).mAP(
iou_threshold=0.5
)
assert math.isclose(mAP, 1 / 7)
def test_mean_average_precision_8():
gt = GroundTruthBoundingBox(clazz="a", box=np.array([10, 10, 20, 20]), image_id="1")
pred1 = PredictedBoundingBox(clazz="a", box=np.array([11, 11, 20, 20]), image_id="1", score=0.9)
pred2 = PredictedBoundingBox(clazz="b", box=np.array([10, 10, 20, 20]), image_id="1", score=0.9)
mAP = BoxMeanAveragePrecision({gt}, {pred1, pred2}).mAP(iou_threshold=0.5)
assert mAP == 0.5
def test_mean_average_precision_5():
"""
One wrong prediction because of wrong location (image_id), but has higher score.
In this case AP should be the precision value @recall=1.0
"""
gt = GroundTruthBoundingBox(clazz="a", box=np.array([10, 10, 20, 20]), image_id="1")
pred1 = PredictedBoundingBox(clazz="a", box=np.array([10, 10, 20, 20]), image_id="1", score=0.8)
pred2 = PredictedBoundingBox(clazz="a", box=np.array([10, 10, 20, 20]), image_id="2", score=0.9)
mAP = BoxMeanAveragePrecision({gt}, {pred1, pred2}).mAP(iou_threshold=0.5)
assert mAP == 0.5
def test_mean_average_precision_4():
"""
One wrong prediction because of wrong location (image_id), but has lower score.
In this case AP is still 1.0 because the wrong prediction has lower score
"""
gt = GroundTruthBoundingBox(clazz="a", box=np.array([10, 10, 20, 20]), image_id="1")
pred1 = PredictedBoundingBox(clazz="a", box=np.array([10, 10, 20, 20]), image_id="1", score=0.9)
pred2 = PredictedBoundingBox(clazz="a", box=np.array([10, 10, 20, 20]), image_id="2", score=0.8)
mAP = BoxMeanAveragePrecision({gt}, {pred1, pred2}).mAP(iou_threshold=0.5)
assert mAP == 1.0
def test_mean_average_precision_3():
"""
No correct predictions because the class were wrong.
AP should be 0.
"""
gt = GroundTruthBoundingBox(clazz="a", box=np.array([10, 10, 20, 20]), image_id="1")
pred1 = PredictedBoundingBox(clazz="c", box=np.array([10, 10, 20, 20]), image_id="1", score=0.9)
pred2 = PredictedBoundingBox(clazz="b", box=np.array([10, 10, 20, 20]), image_id="1", score=0.9)
mAP = BoxMeanAveragePrecision({gt}, {pred1, pred2}).mAP(iou_threshold=0.5)
assert mAP == 0.0
def test_mean_average_precision_2():
"""
One correct box and one wrong box which got the class wrong
In this case AP should be 0.5 because:
for class "a" AP is 1.0
for class "b" AP is 0.0
"""
gt = GroundTruthBoundingBox(clazz="a", box=np.array([10, 10, 20, 20]), image_id="1")
pred1 = PredictedBoundingBox(clazz="a", box=np.array([10, 10, 20, 20]), image_id="1", score=0.9)
pred2 = PredictedBoundingBox(clazz="b", box=np.array([10, 10, 20, 20]), image_id="1", score=0.9)
mAP = BoxMeanAveragePrecision({gt}, {pred1, pred2}).mAP(iou_threshold=0.5)
assert mAP == 0.5
def test_mean_average_precision():
"""
Two correct predictions for one ground truths.
In this case even though there should only be one prediction for one ground truth,
the AP should still be 1.0 because:
precision is 1.0 @ recall=1.0
"""
gt = GroundTruthBoundingBox(clazz="a", box=np.array([10, 10, 20, 20]), image_id="1")
pred1 = PredictedBoundingBox(clazz="a", box=np.array([10, 10, 20, 20]), image_id="1", score=0.9)
pred2 = PredictedBoundingBox(clazz="a", box=np.array([10, 10, 20, 20]), image_id="1", score=0.9)
mAP = BoxMeanAveragePrecision({gt}, {pred1, pred2}).mAP(iou_threshold=0.5)
assert mAP == 1.0
def string_to_boundingbox(
string: str,
box_type: str,
str_format: str = "xyxy"
) -> Union[GroundTruthBoundingBox, PredictedBoundingBox]:
"""
Args:
string: a string representation of a bounding box with format "class image_id xmin ymin xmax ymax score"
box_type: whether the bounding box is a predicted one or a ground truth.
str_format: the bounding box format, one of ["xyxy", "xxyy", "xywh"].
Return:
Either a GTBoundingBox or a PredictedBoundingBox object
"""
# TODO: support normalized boxes, i.e. value between 0, 1.
if str_format == "xyxy":
clazz, image_id, score, xmin, ymin, xmax, ymax = string.split(" ")
elif str_format == "xxyy":
clazz, image_id, score, xmin, xmax, ymin, ymax = string.split(" ")
elif str_format == "xywh":
clazz, image_id, score, xmin, ymin, w, h = string.split(" ")
xmax = int(xmin) + int(w)
ymax = int(ymin) + int(h)
else:
raise ValueError(f"Unknown string format: {str_format}")
xmin = int(xmin)
ymin = int(ymin)
xmax = int(xmax)
ymax = int(ymax)
score = float(score)
if box_type == "pred":
return PredictedBoundingBox(
clazz=str(clazz),
image_id=str(image_id),
score=score,
box=np.array([xmin, ymin, xmax, ymax]),
)
elif box_type == "gt":
return GroundTruthBoundingBox(
clazz=str(clazz),
image_id=str(image_id),
box=np.array([xmin, ymin, xmax, ymax]),
)
else:
raise ValueError(f"Unknown bounding box type: {box_type}.")