def _evaluate(self, image_id, groundtruth, prediction):
"""Evaluates an single image example.
Args:
image_id: A unique identifier for the image.
groundtruth: A dictionary containing the following fields.
- `subject`, `subject/box`, `predicate`, `object`, `object/box`.
prediction: A dictionary containing the following fields.
- `subject`, `subject/box`, `predicate`, `object`, `object/box`.
Returns:
n_triples: Number of annotated triples in the groundtruth.
recall50: Number of recalled groundtruth in the top-50 prediction.
recall100: Number of recalled groundtruth in the top-100 prediction.
"""
gt_subject = groundtruth['subject']
gt_object = groundtruth['object']
gt_predicate = groundtruth['predicate']
n_gt = len(gt_subject)
pred_subject = prediction['subject']
pred_object = prediction['object']
pred_predicate = prediction['predicate']
n_pred = len(pred_subject)
# Compute the iou between prediction and groundtruth.
# - `subject_iou` shape = [n_pred, n_gt].
# - `object_iou` shape = [n_pred, n_gt].
subject_iou = box_ops.py_iou(np.expand_dims(prediction['subject/box'], 1),
np.expand_dims(groundtruth['subject/box'], 0))
object_iou = box_ops.py_iou(np.expand_dims(prediction['object/box'], 1),
np.expand_dims(groundtruth['object/box'], 0))
recall50 = 0
recall100 = 0
recalled = set()
for i in range(min(n_pred, 100)):
for j in range(n_gt):
if (not j in recalled and pred_subject[i] == gt_subject[j] and
pred_object[i] == gt_object[j] and
pred_predicate[i] == gt_predicate[j] and
subject_iou[i, j] >= self._iou_threshold and
object_iou[i, j] >= self._iou_threshold):
recalled.add(j)
if i < 50:
recall50 += 1
if i < 100:
recall100 += 1
return n_gt, recall50, recall100
def _insert_entity(entity_names, entity_boxes, box, name):
for i in range(len(entity_names)):
if name == entity_names[i] and py_iou(box, entity_boxes[i]) > 0.5:
return i
entity_names.append(name)
entity_boxes.append(box)
return len(entity_names) - 1
def test_py_iou(self):
self.assertAllClose(
box_ops.py_iou(box1=np.array([[0.0, 0.0, 2.0, 2.0],
[0.0, 0.0, 2.0, 3.0],
[1.0, 1.0, 2.0, 2.0],
[0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 1.0, 1.0]]),
box2=np.array([[1.0, 1.0, 2.0, 2.0],
[1.0, 1.0, 2.0, 2.0],
[0.0, 0.0, 2.0, 3.0],
[1.0, 1.0, 1.0, 1.0],
[2.0, 2.0, 1.0, 1.0]])),
[0.25, 1.0 / 6, 1.0 / 6, 0.0, 0.0])
def _py_per_image_nms(n_example, scores, sub, sub_score, sub_box, pred,
pred_score, obj, obj_score, obj_box):
dedup_index = []
dedup_score = []
dedup_subject = []
dedup_subject_score = []
dedup_subject_box = []
dedup_predicate = []
dedup_predicate_score = []
dedup_object = []
dedup_object_score = []
dedup_object_box = []
subject_iou = box_ops.py_iou(np.expand_dims(sub_box, 1),
np.expand_dims(sub_box, 0))
object_iou = box_ops.py_iou(np.expand_dims(obj_box, 1),
np.expand_dims(obj_box, 0))
for i in range(n_example):
j = 0
while j < len(dedup_score):
if (sub[i] == dedup_subject[j]
and pred[i] == dedup_predicate[j]
and obj[i] == dedup_object[j]
and subject_iou[i, dedup_index[j]] > iou_thresh
and object_iou[i, dedup_index[j]] > iou_thresh):
break
j += 1
if j == len(dedup_score):
dedup_index.append(i)
dedup_score.append(scores[i])
dedup_subject.append(sub[i])
dedup_subject_score.append(sub_score[i])
dedup_subject_box.append(sub_box[i])
dedup_predicate.append(pred[i])
dedup_predicate_score.append(pred_score[i])
dedup_object.append(obj[i])
dedup_object_score.append(obj_score[i])
dedup_object_box.append(obj_box[i])
if len(dedup_score) >= max_total_size:
break
def _pad_fn(x, dtype=np.float32):
if isinstance(x, list):
x = np.array(x, dtype=dtype)
if len(x) < max_total_size:
pad = max_total_size - len(x)
if len(x.shape) == 1:
x = np.concatenate([x, np.zeros((pad), dtype=dtype)], 0)
elif len(x.shape) == 2 and x.shape[-1] == 4:
x = np.concatenate([x, np.zeros((pad, 4), dtype=dtype)], 0)
else:
raise ValueError('Not supported')
return x[:max_total_size]
if len(dedup_score):
return [
np.array(len(dedup_score), np.int32),
_pad_fn(dedup_score),
_pad_fn(dedup_subject, np.int32),
_pad_fn(dedup_subject_score),
_pad_fn(dedup_subject_box),
_pad_fn(dedup_predicate, np.int32),
_pad_fn(dedup_predicate_score),
_pad_fn(dedup_object, np.int32),
_pad_fn(dedup_object_score),
_pad_fn(dedup_object_box),
]
else:
return [
np.array(0, np.int32),
np.zeros((max_total_size), np.float32),
np.zeros((max_total_size), np.int32),
np.zeros((max_total_size), np.float32),
np.zeros((max_total_size, 4), np.float32),
np.zeros((max_total_size), np.int32),
np.zeros((max_total_size), np.float32),
np.zeros((max_total_size), np.int32),
np.zeros((max_total_size), np.float32),
np.zeros((max_total_size, 4), np.float32),
]