def evaluate_from_dict(gt_entry,
pred_entry,
mode,
result_dict,
multiple_preds=False,
viz_dict=None,
**kwargs):
"""
Shortcut to doing evaluate_recall from dict
:param gt_entry: Dictionary containing gt_relations, gt_boxes, gt_classes
:param pred_entry: Dictionary containing pred_rels, pred_boxes (if detection), pred_classes
:param mode: 'det' or 'cls'
:param result_dict:
:param viz_dict:
:param kwargs:
:return:
"""
gt_rels = gt_entry['gt_relations']
gt_boxes = gt_entry['gt_boxes'].astype(float)
gt_classes = gt_entry['gt_classes']
pred_rel_inds = pred_entry['pred_rel_inds']
rel_scores = pred_entry['rel_scores']
if mode == 'predcls':
pred_boxes = gt_boxes
pred_classes = gt_classes
obj_scores = np.ones(gt_classes.shape[0])
elif mode == 'sgcls':
pred_boxes = gt_boxes
pred_classes = pred_entry['pred_classes']
obj_scores = pred_entry['obj_scores']
elif mode == 'sgdet' or mode == 'phrdet':
pred_boxes = pred_entry['pred_boxes'].astype(float)
pred_classes = pred_entry['pred_classes']
obj_scores = pred_entry['obj_scores']
elif mode == 'preddet':
# Only extract the indices that appear in GT
prc = intersect_2d(pred_rel_inds, gt_rels[:, :2])
if prc.size == 0:
for k in result_dict[mode + '_recall']:
result_dict[mode + '_recall'][k].append(0.0)
return None, None, None
pred_inds_per_gt = prc.argmax(0)
pred_rel_inds = pred_rel_inds[pred_inds_per_gt]
rel_scores = rel_scores[pred_inds_per_gt]
# Now sort the matching ones
rel_scores_sorted = argsort_desc(rel_scores[:, 1:])
rel_scores_sorted[:, 1] += 1
rel_scores_sorted = np.column_stack(
(pred_rel_inds[rel_scores_sorted[:, 0]], rel_scores_sorted[:, 1]))
matches = intersect_2d(rel_scores_sorted, gt_rels)
for k in result_dict[mode + '_recall']:
rec_i = float(matches[:k].any(0).sum()) / float(gt_rels.shape[0])
result_dict[mode + '_recall'][k].append(rec_i)
return None, None, None
else:
raise ValueError('invalid mode')
if multiple_preds:
obj_scores_per_rel = obj_scores[pred_rel_inds].prod(1)
overall_scores = obj_scores_per_rel[:, None] * rel_scores[:, 1:]
score_inds = argsort_desc(overall_scores)[:100]
pred_rels = np.column_stack(
(pred_rel_inds[score_inds[:, 0]], score_inds[:, 1] + 1))
predicate_scores = rel_scores[score_inds[:, 0], score_inds[:, 1] + 1]
else:
pred_rels = np.column_stack(
(pred_rel_inds, 1 + rel_scores[:, 1:].argmax(1)))
predicate_scores = rel_scores[:, 1:].max(1)
pred_to_gt, pred_5ples, rel_scores = evaluate_recall(
gt_rels,
gt_boxes,
gt_classes,
pred_rels,
pred_boxes,
pred_classes,
predicate_scores,
obj_scores,
phrdet=mode == 'phrdet',
**kwargs)
for k in result_dict[mode + '_recall']:
match = reduce(np.union1d, pred_to_gt[:k])
rec_i = float(len(match)) / float(gt_rels.shape[0])
result_dict[mode + '_recall'][k].append(rec_i)
return pred_to_gt, pred_5ples, rel_scores
def val_batch(sgg_model,
batch_num,
b,
evaluator,
eval_m,
val_dataset,
evaluator_list,
evaluator_multiple_preds_list,
vis=False,
max_obj=10,
max_rels=20,
train=None,
test_zs=None,
predicate_weights=None):
if val_dataset.torch_detector:
scale = 1.
box_threshs = [0.2, 0.05, 0.01]
else:
scale = BOX_SCALE / IM_SCALE
box_threshs = [None]
pred_entries = []
for box_score_thresh in box_threshs:
sgg_model.set_box_score_thresh(box_score_thresh)
try:
det_res = sgg_model(
b.scatter()) # keep as it was in the original code
det_res = [det_res]
for i, (boxes_i, objs_i, obj_scores_i, rels_i,
pred_scores_i) in enumerate(det_res):
if vis and len(
val_dataset.gt_classes[batch_num + i]) > max_obj:
print('skipping a scene graph with too many objects')
continue
if VG.split == 'stanford':
w, h = b[i][1][0, :2]
scale_gt = 1. / (BOX_SCALE / max(w, h))
else:
scale_gt = 1.
gt_entry = {
'gt_classes':
val_dataset.gt_classes[batch_num + i].copy(),
'gt_relations':
val_dataset.relationships[batch_num + i].copy(),
'gt_boxes':
val_dataset.gt_boxes[batch_num + i].copy() * scale_gt,
}
pred_entry = {
'pred_boxes': boxes_i * scale,
'pred_classes': objs_i,
'pred_rel_inds': rels_i,
'obj_scores': obj_scores_i,
'rel_scores': pred_scores_i, # hack for now.
}
if predicate_weights is not None:
p = 1. / predicate_weights[1:]
pred_entry[
'rel_scores'][:,
1:] = pred_entry['rel_scores'][:, 1:] * p
pred_entry[
'rel_scores'] = pred_entry['rel_scores'] / np.sum(
pred_entry['rel_scores'], axis=1, keepdims=True)
assert (abs(pred_entry['rel_scores'].sum(1) - 1) <
1e-5).all(), pred_entry['rel_scores'].sum(1)
pred_entries.append(pred_entry)
for sfx in ['', '_nogc']:
evaluator[eval_m + sfx].evaluate_scene_graph_entry(
gt_entry, pred_entry)
if evaluator_list is not None and len(evaluator_list) > 0:
eval_entry(eval_m, gt_entry, pred_entry, evaluator_list,
evaluator_multiple_preds_list)
if vis:
print(val_dataset.filenames[batch_num + i],
'showing ground truth')
im_gt = draw_boxes(
b[0][0][0].permute(1, 2,
0).data.cpu().numpy().copy(), [
train.ind_to_classes[c]
for c in gt_entry['gt_classes']
],
gt_entry['gt_boxes'],
torch_detector=val_dataset.torch_detector)
plt.figure(figsize=(10, 10))
plt.imshow(im_gt)
plt.axis(False)
plt.show()
show_nx(gt_entry['gt_classes'],
gt_entry['gt_boxes'],
gt_entry['gt_relations'],
train_set=train,
test_set=test_zs,
torch_detector=val_dataset.torch_detector)
print(val_dataset.filenames[batch_num + i],
'showing top %d relationships' % max_rels)
im_pred = draw_boxes(
b[0][0][0].permute(1, 2, 0).data.cpu().numpy().copy(),
[
train.ind_to_classes[c]
for c in pred_entry['pred_classes']
],
pred_entry['pred_boxes'],
torch_detector=val_dataset.torch_detector)
plt.figure(figsize=(10, 10))
plt.imshow(im_pred)
plt.axis(False)
plt.show()
obj_scores = pred_entry['obj_scores']
pred_rel_inds = rels_i
obj_scores_per_rel = obj_scores[pred_rel_inds].prod(1)
overall_scores = obj_scores_per_rel[:, None] * pred_entry[
'rel_scores'][:, 1:]
score_inds = argsort_desc(overall_scores)[:max_rels]
pred_rels = np.column_stack(
(pred_rel_inds[score_inds[:,
0]], score_inds[:, 1] + 1))
show_nx(pred_entry['pred_classes'],
pred_entry['pred_boxes'],
pred_rels,
train_set=train,
test_set=test_zs,
torch_detector=val_dataset.torch_detector)
return pred_entries
except (ValueError, IndexError) as e:
print('no objects or relations found'.upper(), e, b[0][-1],
'trying a smaller threshold')
def evaluate_from_dict(gt_entry, pred_entry, mode, result_dict, multiple_preds=False,
viz_dict=None, **kwargs):
"""
Shortcut to doing evaluate_recall from dict
:param gt_entry: Dictionary containing gt_relations, gt_boxes, gt_classes
:param pred_entry: Dictionary containing pred_rels, pred_boxes (if detection), pred_classes
:param mode: 'det' or 'cls'
:param result_dict:
:param viz_dict:
:param kwargs:
:return:
"""
gt_rels = gt_entry['gt_relations']
gt_boxes = gt_entry['gt_boxes'].astype(float)
gt_classes = gt_entry['gt_classes']
pred_rel_inds = pred_entry['pred_rel_inds']
rel_scores = pred_entry['rel_scores']
if mode == 'predcls':
pred_boxes = gt_boxes
pred_classes = gt_classes
obj_scores = np.ones(gt_classes.shape[0])
elif mode == 'sgcls':
pred_boxes = gt_boxes
pred_classes = pred_entry['pred_classes']
obj_scores = pred_entry['obj_scores']
elif mode == 'sgdet' or mode == 'phrdet':
pred_boxes = pred_entry['pred_boxes'].astype(float)
pred_classes = pred_entry['pred_classes']
obj_scores = pred_entry['obj_scores']
elif mode == 'preddet':
# Only extract the indices that appear in GT
prc = intersect_2d(pred_rel_inds, gt_rels[:, :2])
if prc.size == 0:
for k in result_dict[mode + '_recall']:
result_dict[mode + '_recall'][k].append(0.0)
return None, None, None
pred_inds_per_gt = prc.argmax(0)
pred_rel_inds = pred_rel_inds[pred_inds_per_gt]
rel_scores = rel_scores[pred_inds_per_gt]
# Now sort the matching ones
rel_scores_sorted = argsort_desc(rel_scores[:,1:])
rel_scores_sorted[:,1] += 1
rel_scores_sorted = np.column_stack((pred_rel_inds[rel_scores_sorted[:,0]], rel_scores_sorted[:,1]))
matches = intersect_2d(rel_scores_sorted, gt_rels)
for k in result_dict[mode + '_recall']:
rec_i = float(matches[:k].any(0).sum()) / float(gt_rels.shape[0])
result_dict[mode + '_recall'][k].append(rec_i)
return None, None, None
else:
raise ValueError('invalid mode')
# multi_pred controls whether Allow multiple predicates per pair of box0, box1
if multiple_preds:
# compute overall_score which is used to sort @100
obj_scores_per_rel = obj_scores[pred_rel_inds].prod(1)
overall_scores = obj_scores_per_rel[:,None] * rel_scores[:,1:]
# sort and get the 100 highest overall_scores
score_inds = argsort_desc(overall_scores)[:100]
pred_rels = np.column_stack((pred_rel_inds[score_inds[:,0]], score_inds[:,1]+1))
predicate_scores = rel_scores[score_inds[:,0], score_inds[:,1]+1]
else:
# only one relationship: get the predicate index and score whose score is the highest
# pred_rels is [num_rel, 2+1]; predicate_scores is [num_rel, 1]
pred_rels = np.column_stack((pred_rel_inds, 1+rel_scores[:,1:].argmax(1))) # merge matrix; .argmax return the index of maximum
predicate_scores = rel_scores[:,1:].max(1) # maximum of each row (most likely relation between certain 2 objects)
# print("pred_rels is", pred_rels)
# compare gt and prediction: boxes, classes, predicate
pred_to_gt, pred_5ples, rel_scores = evaluate_recall(
gt_rels, gt_boxes, gt_classes,
pred_rels, pred_boxes, pred_classes,
predicate_scores, obj_scores, phrdet= mode=='phrdet',
**kwargs)
#######################################################
# get k = @20 @50 @100 results from length of num_rel which is predicted
for k in result_dict[mode + '_recall']:
# how many are correct in the first k prdicitons; get a union set, element can be reduplicate;
# shape: (#correct,)
match = reduce(np.union1d, pred_to_gt[:k])
# attention: match / gt
rec_i = float(len(match)) / float(gt_rels.shape[0])
# recall value in @20, @50, @100; stored in self.result_dict
result_dict[mode + '_recall'][k].append(rec_i)
# pred_to_gt: list len is num_rel, length of list inside the whole lis may not be 1
# pred_5ples: (num_rel, 5)， (id0, id1, cls0, cls1, rel)
# rel_scores: (num_rel, 3), (box1, box2, predicate score), sorted by overall score
return pred_to_gt, pred_5ples, rel_scores
def evaluate_from_dict(gt_entry, pred_entry, mode, result_dict, multiple_preds=False,
viz_dict=None, **kwargs):
"""
Shortcut to doing evaluate_recall from dict
:param gt_entry: Dictionary containing gt_relations, gt_boxes, gt_classes
:param pred_entry: Dictionary containing pred_rels, pred_boxes (if detection), pred_classes
:param mode: 'det' or 'cls'
:param result_dict:
:param viz_dict:
:param kwargs:
:return:
"""
gt_rels = gt_entry['gt_relations']
gt_boxes = gt_entry['gt_boxes'].astype(float)
gt_classes = gt_entry['gt_classes']
pred_rel_inds = pred_entry['pred_rel_inds']
rel_scores = pred_entry['rel_scores']
pred_predicates = np.zeros((rel_scores.shape[-1]-1,))
gt_predicates = np.zeros((rel_scores.shape[-1]-1,))
vaild_mask = np.bincount(gt_rels[:, :-1].reshape(-1), minlength=len(gt_classes)) > 0
for i in gt_rels[:, 2]:
gt_predicates[i - 1] += 1.0
if mode == 'predcls':
pred_boxes = gt_boxes
pred_classes = gt_classes
obj_scores = np.ones(gt_classes.shape[0])
pred_rel_inds, rel_scores = filter_out_ind(pred_rel_inds, rel_scores, gt_classes.shape[0])
elif mode == 'sgcls':
pred_boxes = gt_boxes
pred_classes = pred_entry['pred_classes']
obj_scores = pred_entry['obj_scores']
elif mode == 'sgdet' or mode == 'phrdet':
pred_boxes = pred_entry['pred_boxes'].astype(float)
pred_classes = pred_entry['pred_classes']
obj_scores = pred_entry['obj_scores']
elif mode == 'preddet':
# Only extract the indices that appear in GT
prc = intersect_2d(pred_rel_inds, gt_rels[:, :2])
if prc.size == 0:
for k in result_dict[mode + '_recall']:
result_dict[mode + '_recall'][k].append(0.0)
return None, None, None
pred_inds_per_gt = np.where(prc)[0]
pred_rel_inds = pred_rel_inds[pred_inds_per_gt]
rel_scores = rel_scores[pred_inds_per_gt]
# Now sort the matching ones
rel_scores_sorted = argsort_desc(rel_scores[:,1:])
rel_scores = np.sort(np.ravel(rel_scores[:,1:]))[::-1]
rel_scores_sorted[:,1] += 1
rel_scores_sorted = np.column_stack((pred_rel_inds[rel_scores_sorted[:,0]], rel_scores_sorted[:,1]))
matches = intersect_2d(rel_scores_sorted, gt_rels)
for k in result_dict[mode + '_recall']:
rec_i = float(matches[:k].any(0).sum()) / float(gt_rels.shape[0])
result_dict[mode + '_recall'][k].append(rec_i)
return rel_scores_sorted, None, rel_scores
else:
raise ValueError('invalid mode')
if multiple_preds:
obj_scores_per_rel = obj_scores[pred_rel_inds].prod(1)
overall_scores = obj_scores_per_rel[:,None] * rel_scores[:,1:]
score_inds = argsort_desc(overall_scores)[:100]
pred_rels = np.column_stack((pred_rel_inds[score_inds[:,0]], score_inds[:,1]+1))
predicate_scores = rel_scores[score_inds[:,0], score_inds[:,1]+1]
else:
pred_rels = np.column_stack((pred_rel_inds, 1+rel_scores[:,1:].argmax(1)))
predicate_scores = rel_scores[:,1:].max(1)
pred_to_gt, pred_5ples, rel_scores = evaluate_recall(
gt_rels, gt_boxes, gt_classes,
pred_rels, pred_boxes, pred_classes,
predicate_scores, obj_scores, phrdet= mode=='phrdet',
**kwargs)
match_indices = reduce(np.union1d, pred_to_gt[:100]).astype(np.int64)
for i in gt_rels[match_indices, 2]:
pred_predicates[i - 1] += 1.0
objs_mAP = (gt_classes[vaild_mask]==pred_classes[vaild_mask]).mean()
result_dict[mode + '_objs_mAP'].append(objs_mAP)
result_dict[mode + '_pred_predicates'].append(pred_predicates)
result_dict[mode + '_gt_predicates'].append(gt_predicates)
result_dict[mode + '_objs_mAP'].append(objs_mAP)
for k in result_dict[mode + '_recall']:
match = reduce(np.union1d, pred_to_gt[:k])
rec_i = float(len(match)) / float(gt_rels.shape[0])
result_dict[mode + '_recall'][k].append(rec_i)
prc = intersect_2d(gt_rels[:, :-1], pred_rel_inds[:k]).any(1).sum()
p_i = float(prc) / float(gt_rels.shape[0])
result_dict[mode + '_pairrecall'][k].append(p_i)
return pred_to_gt, pred_5ples, predicate_scores, pred_rels
def evaluate_from_dict(gt_entry, pred_entry, mode, eval_result_dict, eval_result_dict2,
gtrel_dist_dict, gtkeyrel_dist_dict,
predrel_dist_dict, predmatchedrel_dist_dict, predmatchedkeyrel_dist_dict,
multiple_preds=False, predrel_treedeep_scores_dict=None,
viz_dict=None, num_predicates=50, **kwargs):
"""
Shortcut to doing evaluate_recall from dict
:param gt_entry: Dictionary containing gt_relations, gt_boxes, gt_classes
:param pred_entry: Dictionary containing pred_rels, pred_boxes (if detection), pred_classes
:param mode: 'det' or 'cls'
:param result_dict:
:param viz_dict:
:param kwargs:
:return:
"""
gt_rels = gt_entry['gt_relations']
gt_boxes = gt_entry['gt_boxes'].astype(float)
gt_classes = gt_entry['gt_classes']
gt_key_rels = gt_entry['gt_key_rels'] if 'gt_key_rels' in gt_entry else None
pred_rel_inds = pred_entry['pred_rel_inds']
rel_scores = pred_entry['rel_scores']
rel_rank_scores = pred_entry['rel_rank_scores']
tree = pred_entry['forest']
# get each node's depth
tree_depth_dict = None
tree_depth = None
tree_width = None
if tree is not None:
root = tree[0]
tree_depth_dict = {}
get_treeNodes_depth(tree_depth_dict, root)
tree_depth = root.max_depth()
tree_width = root.max_width()
# if rel_rank_scores is not None:
# rel_scores *= rel_rank_scores[:, None]
if mode == 'predcls':
pred_boxes = gt_boxes
pred_classes = gt_classes
obj_scores = np.ones(gt_classes.shape[0])
elif mode == 'sgcls':
pred_boxes = gt_boxes
pred_classes = pred_entry['pred_classes']
obj_scores = pred_entry['obj_scores']
elif mode == 'sgdet' or mode == 'phrdet':
pred_boxes = pred_entry['pred_boxes'].astype(float)
pred_classes = pred_entry['pred_classes']
obj_scores = pred_entry['obj_scores']
elif mode == 'preddet':
# Only extract the indices that appear in GT
prc = intersect_2d(pred_rel_inds, gt_rels[:, :2])
if prc.size == 0:
for k in eval_result_dict[mode + '_recall']:
eval_result_dict[mode + '_recall'][k].append(0.0)
return None, None, None
pred_inds_per_gt = prc.argmax(0)
pred_rel_inds = pred_rel_inds[pred_inds_per_gt]
rel_scores = rel_scores[pred_inds_per_gt]
# Now sort the matching ones
rel_scores_sorted = argsort_desc(rel_scores[:, 1:])
rel_scores_sorted[:, 1] += 1
rel_scores_sorted = np.column_stack((pred_rel_inds[rel_scores_sorted[:, 0]], rel_scores_sorted[:, 1]))
matches = intersect_2d(rel_scores_sorted, gt_rels)
for k in eval_result_dict[mode + '_recall']:
rec_i = float(matches[:k].any(0).sum()) / float(gt_rels.shape[0])
eval_result_dict[mode + '_recall'][k].append(rec_i)
return None, None, None
else:
raise ValueError('invalid mode')
if multiple_preds:
obj_scores_per_rel = obj_scores[pred_rel_inds].prod(1)
overall_scores = obj_scores_per_rel[:, None] * rel_scores[:, 1:]
score_inds = argsort_desc(overall_scores)[:100]
pred_rels = np.column_stack((pred_rel_inds[score_inds[:, 0]], score_inds[:, 1] + 1))
predicate_scores = rel_scores[score_inds[:, 0], score_inds[:, 1] + 1]
else:
pred_rels = np.column_stack((pred_rel_inds, 1 + rel_scores[:, 1:].argmax(1)))
predicate_scores = rel_scores[:, 1:].max(1)
# note: for relrank
if rel_rank_scores is not None:
predicate_scores *= rel_rank_scores
RES_pred_to_gt, RES_subobj_to_gt, pred_5ples, rel_scores = evaluate_recall(
gt_rels, gt_boxes, gt_classes,
pred_rels, pred_boxes, pred_classes,
predicate_scores, obj_scores, phrdet=mode == 'phrdet',
**kwargs)
# evaluate the tree: get the distribution over tree hierarchy
if tree is not None:
# get gt distribution, only do it in predcls or sgcls mode, use the gt box
if mode == 'sgcls' or mode == 'predcls':
for pair in gt_rels[:, :2]:
gtrel_dist_dict[_gen_key(tree_depth_dict[pair[0]], tree_depth_dict[pair[1]])] += 1
if gt_key_rels is not None:
for ind in gt_key_rels:
pair = gt_rels[ind, :2]
gtkeyrel_dist_dict[_gen_key(tree_depth_dict[pair[0]], tree_depth_dict[pair[1]])] += 1
# get the predicted rels distribution
box_pair_inds = pred_rel_inds[:5]
for ind in box_pair_inds:
predrel_dist_dict[mode][_gen_key(tree_depth_dict[ind[0]], tree_depth_dict[ind[1]])] += 1
norm_triplet_rel_scores = rel_scores.prod(1)
norm_triplet_rel_scores = norm_triplet_rel_scores / np.max(norm_triplet_rel_scores)
for ind_idx, ind in enumerate(pred_rel_inds):
predrel_treedeep_scores_dict[mode][_gen_key(tree_depth_dict[ind[0]], tree_depth_dict[ind[1]])].append(norm_triplet_rel_scores[ind_idx])
# get the match ones, only do it in predcls or sgcls mode, use the gt box
match = reduce(np.union1d, RES_subobj_to_gt[:5])
for m in match:
predmatchedrel_dist_dict[mode][
_gen_key(tree_depth_dict[gt_rels[int(m), 0]], tree_depth_dict[gt_rels[int(m), 1]])] += 1
if gt_key_rels is not None:
key_match = np.intersect1d(match, gt_key_rels)
for m in key_match:
predmatchedkeyrel_dist_dict[mode][
_gen_key(tree_depth_dict[gt_rels[int(m), 0]], tree_depth_dict[gt_rels[int(m), 1]])] += 1
for pred_to_gt, result_dict in [(RES_pred_to_gt, eval_result_dict), (RES_subobj_to_gt, eval_result_dict2)]:
for k in result_dict[mode + '_recall']:
match = reduce(np.union1d, pred_to_gt[:k])
key_match = np.intersect1d(match, gt_key_rels) if gt_key_rels is not None else None
if gt_key_rels is not None:
for idx in range(len(key_match)):
local_label = gt_rels[int(key_match[idx]), 2]
if (mode + '_key_recall_hit') not in result_dict:
result_dict[mode + '_key_recall_hit'] = {}
if k not in result_dict[mode + '_key_recall_hit']:
result_dict[mode + '_key_recall_hit'][k] = [0] * (num_predicates + 1)
result_dict[mode + '_key_recall_hit'][k][int(local_label)] += 1
result_dict[mode + '_key_recall_hit'][k][0] += 1
for idx in range(gt_key_rels.shape[0]):
local_label = gt_rels[int(gt_key_rels[idx]), 2]
if (mode + '_key_recall_count') not in result_dict:
result_dict[mode + '_key_recall_count'] = {}
if k not in result_dict[mode + '_key_recall_count']:
result_dict[mode + '_key_recall_count'][k] = [0] * (num_predicates + 1)
result_dict[mode + '_key_recall_count'][k][int(local_label)] += 1
result_dict[mode + '_key_recall_count'][k][0] += 1
for idx in range(len(match)):
local_label = gt_rels[int(match[idx]), 2]
if (mode + '_recall_hit') not in result_dict:
result_dict[mode + '_recall_hit'] = {}
if k not in result_dict[mode + '_recall_hit']:
result_dict[mode + '_recall_hit'][k] = [0] * (num_predicates + 1)
result_dict[mode + '_recall_hit'][k][int(local_label)] += 1
result_dict[mode + '_recall_hit'][k][0] += 1
for idx in range(gt_rels.shape[0]):
local_label = gt_rels[idx, 2]
if (mode + '_recall_count') not in result_dict:
result_dict[mode + '_recall_count'] = {}
if k not in result_dict[mode + '_recall_count']:
result_dict[mode + '_recall_count'][k] = [0] * (num_predicates + 1)
result_dict[mode + '_recall_count'][k][int(local_label)] += 1
result_dict[mode + '_recall_count'][k][0] += 1
rec_i = float(len(match)) / float(gt_rels.shape[0])
result_dict[mode + '_recall'][k].append(rec_i)
if gt_key_rels is not None:
key_rec_i = float(len(key_match)) / float(gt_key_rels.shape[0])
result_dict[mode + '_key_recall'][k].append(key_rec_i)
return RES_pred_to_gt, RES_subobj_to_gt, pred_5ples, rel_scores, tree_depth, tree_width
def evaluate_from_dict(gt_entry,
pred_entry,
mode,
eval_result_dict,
multiple_preds=1,
num_predicates=50,
**kwargs):
"""
Shortcut to doing evaluate_recall from dict
:param gt_entry: Dictionary containing gt_relations, gt_boxes, gt_classes
:param pred_entry: Dictionary containing pred_rels, pred_boxes (if detection), pred_classes
:param mode: 'det' or 'cls'
:param result_dict:
:param viz_dict:
:param kwargs:
:return:
"""
gt_rels = gt_entry['gt_relations']
gt_boxes = gt_entry['gt_boxes'].astype(float)
gt_classes = gt_entry['gt_classes']
pred_rel_inds = pred_entry['pred_rel_inds']
rel_scores = pred_entry['rel_scores']
if mode == 'predcls':
pred_boxes = gt_boxes
pred_classes = gt_classes
obj_scores = np.ones(gt_classes.shape[0])
elif mode == 'sgcls':
pred_boxes = gt_boxes
pred_classes = pred_entry['pred_classes']
obj_scores = pred_entry['obj_scores']
elif mode == 'sgdet' or mode == 'phrdet':
pred_boxes = pred_entry['pred_boxes'].astype(float)
pred_classes = pred_entry['pred_classes']
obj_scores = pred_entry['obj_scores']
elif mode == 'preddet':
# Only extract the indices that appear in GT
prc = intersect_2d(pred_rel_inds, gt_rels[:, :2])
if prc.size == 0:
for k in eval_result_dict[mode + '_recall']:
eval_result_dict[mode + '_recall'][k].append(0.0)
return None, None, None
pred_inds_per_gt = prc.argmax(0)
pred_rel_inds = pred_rel_inds[pred_inds_per_gt]
rel_scores = rel_scores[pred_inds_per_gt]
# Now sort the matching ones
rel_scores_sorted = argsort_desc(rel_scores[:, 1:])
rel_scores_sorted[:, 1] += 1
rel_scores_sorted = np.column_stack(
(pred_rel_inds[rel_scores_sorted[:, 0]], rel_scores_sorted[:, 1]))
matches = intersect_2d(rel_scores_sorted, gt_rels)
for k in eval_result_dict[mode + '_recall']:
rec_i = float(matches[:k].any(0).sum()) / float(gt_rels.shape[0])
eval_result_dict[mode + '_recall'][k].append(rec_i)
return None, None, None
else:
raise ValueError('invalid mode')
if multiple_preds > 1:
if multiple_preds == rel_scores.shape[1] - 1: # all predicates
obj_scores_per_rel = obj_scores[pred_rel_inds].prod(1)
overall_scores = obj_scores_per_rel[:, None] * rel_scores[:, 1:]
score_inds = argsort_desc(overall_scores)[:100]
pred_rels = np.column_stack(
(pred_rel_inds[score_inds[:, 0]], score_inds[:, 1] + 1))
predicate_scores = rel_scores[score_inds[:, 0],
score_inds[:, 1] + 1]
else:
# between 1 and all predictes
obj_scores_per_rel = obj_scores[pred_rel_inds].prod(1) # Nr
overall_scores = obj_scores_per_rel[:,
None] * rel_scores[:,
1:] # Nr, 70
# sort predicate scores for each pair
sorted_predicates_idx = np.argsort(
-overall_scores,
axis=1)[:, :multiple_preds] # Nr, multiple_preds
sorted_predicates_scores = np.sort(
overall_scores, axis=1)[:, ::-1][:, :multiple_preds]
score_inds = argsort_desc(sorted_predicates_scores)[:100]
pred_rels = np.column_stack(
(pred_rel_inds[score_inds[:, 0]],
sorted_predicates_idx[score_inds[:, 0], score_inds[:, 1]] +
1))
predicate_scores = rel_scores[
score_inds[:, 0],
sorted_predicates_idx[score_inds[:, 0], score_inds[:, 1]] + 1]
else:
pred_rels = np.column_stack(
(pred_rel_inds, 1 + rel_scores[:, 1:].argmax(1)))
predicate_scores = rel_scores[:, 1:].max(1)
RES_pred_to_gt, pred_5ples, rel_scores = evaluate_recall(
gt_rels,
gt_boxes,
gt_classes,
pred_rels,
pred_boxes,
pred_classes,
predicate_scores,
obj_scores,
phrdet=mode == 'phrdet',
**kwargs)
pred_to_gt = RES_pred_to_gt
result_dict = eval_result_dict
for k in result_dict[mode + '_recall']:
match = reduce(np.union1d, pred_to_gt[:k])
for idx in range(len(match)):
local_label = gt_rels[int(match[idx]), 2]
if (mode + '_recall_hit') not in result_dict:
result_dict[mode + '_recall_hit'] = {}
if k not in result_dict[mode + '_recall_hit']:
result_dict[mode +
'_recall_hit'][k] = [0] * (num_predicates + 1)
result_dict[mode + '_recall_hit'][k][int(local_label)] += 1
result_dict[mode + '_recall_hit'][k][0] += 1
for idx in range(gt_rels.shape[0]):
local_label = gt_rels[idx, 2]
if (mode + '_recall_count') not in result_dict:
result_dict[mode + '_recall_count'] = {}
if k not in result_dict[mode + '_recall_count']:
result_dict[mode +
'_recall_count'][k] = [0] * (num_predicates + 1)
result_dict[mode + '_recall_count'][k][int(local_label)] += 1
result_dict[mode + '_recall_count'][k][0] += 1
rec_i = float(len(match)) / float(gt_rels.shape[0])
result_dict[mode + '_recall'][k].append(rec_i)
return RES_pred_to_gt, pred_5ples, rel_scores
def evaluate_from_dict(self,
gt_entry,
pred_entry,
mode,
result_dict,
multiple_preds=False,
viz_dict=None,
**kwargs):
"""
Shortcut to doing evaluate_recall from dict
:param gt_entry: Dictionary containing gt_relations, gt_boxes, gt_classes
:param pred_entry: Dictionary containing pred_rels, pred_boxes (if detection), pred_classes
:param mode: 'det' or 'cls'
:param result_dict:
:param viz_dict:
:param kwargs:
:return:
"""
gt_rels = gt_entry['gt_relations']
gt_boxes = gt_entry['gt_boxes'].astype(float)
gt_classes = gt_entry['gt_classes']
pred_rel_inds = pred_entry['pred_rel_inds']
rel_scores = pred_entry['rel_scores']
if mode == 'predcls':
pred_boxes = gt_boxes
pred_classes = gt_classes
obj_scores = np.ones(gt_classes.shape[0])
elif mode == 'sgcls':
pred_boxes = gt_boxes
pred_classes = pred_entry['pred_classes']
obj_scores = pred_entry['obj_scores']
elif mode == 'objcls':
pred_boxes = gt_boxes
pred_classes = pred_entry['pred_classes']
obj_scores = pred_entry['obj_scores']
# same as sgcls but assume perfect predicate recognition
pred_rel_inds = gt_rels[:, :2]
rel_scores = np.zeros((len(gt_rels), rel_scores.shape[1]))
rel_scores[np.arange(len(gt_rels)), gt_rels[:, 2]] = 1
elif mode == 'sgdet' or mode == 'phrdet':
pred_boxes = pred_entry['pred_boxes'].astype(float)
pred_classes = pred_entry['pred_classes']
obj_scores = pred_entry['obj_scores']
elif mode == 'preddet':
# Only extract the indices that appear in GT
prc = intersect_2d(pred_rel_inds, gt_rels[:, :2])
if prc.size == 0:
for k in result_dict[mode + '_recall']:
result_dict[mode + '_recall'][k].append(0.0)
if self.per_triplet:
for k in result_dict[mode + '_recall_norm']:
result_dict[mode + '_recall_norm'][k].append(0.0)
return None, None, None
pred_inds_per_gt = prc.argmax(0)
pred_rel_inds = pred_rel_inds[pred_inds_per_gt]
rel_scores = rel_scores[pred_inds_per_gt]
# Now sort the matching ones
rel_scores_sorted = argsort_desc(rel_scores[:, 1:])
rel_scores_sorted[:, 1] += 1
rel_scores_sorted = np.column_stack(
(pred_rel_inds[rel_scores_sorted[:, 0]], rel_scores_sorted[:,
1]))
matches = intersect_2d(rel_scores_sorted, gt_rels)
for k in result_dict[mode + '_recall']:
rec_i = float(matches[:k].any(0).sum()) / float(
gt_rels.shape[0])
result_dict[mode + '_recall'][k].append(rec_i)
if self.per_triplet:
for k in result_dict[mode + '_recall_norm']:
rec_i = float(matches[:k].any(0).sum()) / float(
gt_rels.shape[0])
result_dict[mode + '_recall_norm'][k].append(rec_i)
return None, None, None
else:
raise ValueError('invalid mode')
if multiple_preds:
obj_scores_per_rel = obj_scores[pred_rel_inds].prod(1)
overall_scores = obj_scores_per_rel[:, None] * rel_scores[:, 1:]
score_inds = argsort_desc(overall_scores)[:MAX_RECALL_K]
pred_rels = np.column_stack(
(pred_rel_inds[score_inds[:, 0]], score_inds[:, 1] + 1))
predicate_scores = rel_scores[score_inds[:, 0],
score_inds[:, 1] + 1]
else:
pred_rels = np.column_stack(
(pred_rel_inds, 1 + rel_scores[:, 1:].argmax(1)))
predicate_scores = rel_scores[:, 1:].max(1)
# print('eval', gt_rels.shape, pred_rels.shape, predicate_scores.shape, gt_boxes.shape)
pred_to_gt, pred_5ples, rel_scores = evaluate_recall(
gt_rels,
gt_boxes,
gt_classes,
pred_rels,
pred_boxes,
pred_classes,
predicate_scores,
obj_scores,
phrdet=mode == 'phrdet',
**kwargs)
if self.per_triplet:
counts = np.zeros(len(gt_rels))
for rel_i, gt_rel in enumerate(gt_rels):
o, s, R = gt_rel
tri_str = '{}_{}_{}'.format(gt_classes[o], R, gt_classes[s])
if tri_str in self.triplet_counts:
counts[rel_i] = self.triplet_counts[tri_str]
weights = self.normalize_counts(counts)
for k in result_dict[mode + '_recall']:
match = reduce(np.union1d, pred_to_gt[:k])
# print('match', match, type(match))
match = np.array(match).astype(np.int)
rec_i = float(len(match)) / float(gt_rels.shape[0])
result_dict[mode + '_recall'][k].append(rec_i)
if self.per_triplet:
result_dict[mode + '_recall_norm'][k].append(
np.sum(weights[match]))
if self.per_triplet:
# TODO: this looks similar to preddet, reuse that code
score_inds = argsort_desc(overall_scores)
pred_rels = np.column_stack(
(pred_rel_inds[score_inds[:, 0]], score_inds[:, 1] + 1))
# Naive and slow code to get per triplet ranks
ranks, counts = np.zeros(len(gt_rels)) - 1, np.zeros(len(gt_rels))
for rel_i, gt_rel in enumerate(gt_rels):
o, s, R = gt_rel
tri_str = '{}_{}_{}'.format(gt_classes[o], R, gt_classes[s])
if tri_str in self.triplet_counts:
counts[rel_i] = self.triplet_counts[tri_str]
# select only pairs with this bounding boxes
ind = np.where((pred_rels[:, 0] == o) & (pred_rels[:, 1] == s)
| (pred_rels[:, 0] == s)
& (pred_rels[:, 1] == o))[0]
pred_to_gt_triplet, _, _ = evaluate_recall(
gt_rel.reshape(1, -1), gt_boxes, gt_classes,
pred_rels[ind], pred_boxes, pred_classes)
for r, p in enumerate(pred_to_gt_triplet):
if len(p) > 0:
assert p == [0], (p, gt_rel, pred_to_gt_triplet)
ranks[rel_i] = r
break
if ranks[rel_i] < 0:
ranks[rel_i] = MAX_RECALL_K + 1
# For sgcls not all combinations are present, so take some max rank as the default value
if tri_str not in self.triplet_ranks:
self.triplet_ranks[tri_str] = []
self.triplet_ranks[tri_str].append(ranks[rel_i])
result_dict[mode + '_rank'].extend(ranks)
result_dict[mode + '_counts'].extend(
counts) # save count to normalize later
return pred_to_gt, pred_5ples, rel_scores
def evaluate_from_dict(gt_entry, pred_entry, mode, result_dict, multiple_preds=False,
viz_dict=None, **kwargs):
"""
Shortcut to doing evaluate_recall from dict
:param gt_entry: Dictionary containing gt_relations, gt_boxes, gt_classes
:param pred_entry: Dictionary containing pred_rels, pred_boxes (if detection), pred_classes
:param mode: 'det' or 'cls'
:param result_dict:
:param viz_dict:
:param kwargs:
:return:
"""
gt_rels = gt_entry['gt_relations']
gt_boxes = gt_entry['gt_boxes'].astype(float)
gt_classes = gt_entry['gt_classes']
# gt_filenames = gt_entry['filenames']
pred_rel_inds = pred_entry['pred_rel_inds']
rel_scores = pred_entry['rel_scores']
if mode == 'predcls':
pred_boxes = gt_boxes
pred_classes = gt_classes
obj_scores = np.ones(gt_classes.shape[0])
elif mode == 'sgcls':
pred_boxes = gt_boxes
pred_classes = pred_entry['pred_classes']
obj_scores = pred_entry['obj_scores']
elif mode == 'sgdet' or mode == 'phrdet':
pred_boxes = pred_entry['pred_boxes'].astype(float)
pred_classes = pred_entry['pred_classes']
obj_scores = pred_entry['obj_scores']
elif mode == 'preddet':
# Only extract the indices that appear in GT
prc = intersect_2d(pred_rel_inds, gt_rels[:, :2])
if prc.size == 0:
for k in result_dict[mode + '_recall']:
result_dict[mode + '_recall'][k].append(0.0)
return None, None, None
pred_inds_per_gt = prc.argmax(0)
pred_rel_inds = pred_rel_inds[pred_inds_per_gt]
rel_scores = rel_scores[pred_inds_per_gt]
# Now sort the matching ones
rel_scores_sorted = argsort_desc(rel_scores[:,1:])
rel_scores_sorted[:,1] += 1
rel_scores_sorted = np.column_stack((pred_rel_inds[rel_scores_sorted[:,0]], rel_scores_sorted[:,1]))
matches = intersect_2d(rel_scores_sorted, gt_rels)
for k in result_dict[mode + '_recall']:
rec_i = float(matches[:k].any(0).sum()) / float(gt_rels.shape[0])
result_dict[mode + '_recall'][k].append(rec_i)
return None, None, None
else:
raise ValueError('invalid mode')
if multiple_preds:
obj_scores_per_rel = obj_scores[pred_rel_inds].prod(1)
overall_scores = obj_scores_per_rel[:,None] * rel_scores[:,1:]
score_inds = argsort_desc(overall_scores)[:100]
pred_rels = np.column_stack((pred_rel_inds[score_inds[:,0]], score_inds[:,1]+1))
predicate_scores = rel_scores[score_inds[:,0], score_inds[:,1]+1]
else:
pred_rels = np.column_stack((pred_rel_inds, 1+rel_scores[:,1:].argmax(1)))
predicate_scores = rel_scores[:,1:].max(1)
pred_to_gt, pred_5ples, rel_scores = evaluate_recall(
gt_rels, gt_boxes, gt_classes,
pred_rels, pred_boxes, pred_classes,
predicate_scores, obj_scores, phrdet= mode=='phrdet',
**kwargs)
for k in result_dict[mode + '_recall']:
match = reduce(np.union1d, pred_to_gt[:k])
# FIXME: I think this part of original code is wrong. We shouldn't do union.
#: stores tuples (hits, count)
hits_per_rel = dict()
# gt_rels: shape: (m, 3), (s, p, r)
for i in range(gt_rels.shape[0]):
gt_s, gt_o, gt_r = gt_rels[i]
hits_per_rel.setdefault(gt_r, [0, 0])
hits_per_rel[gt_r][1] += 1
hits_per_rel[gt_r][0] += i in match
rec_per_rel = {r: (hits, cnt) for r, (hits, cnt) in hits_per_rel.items()}
rec_i = float(len(match)) / float(gt_rels.shape[0])
result_dict[mode + '_recall'][k].append(rec_i)
result_dict[mode + '_recall_per_rel'][k].append(rec_per_rel)
return pred_to_gt, pred_5ples, rel_scores
