def evaluate_from_dict(gt_entry, pred_entry, mode, result_dict, multiple_preds=False, viz_dict=None, **kwargs): """ Shortcut to doing evaluate_recall from dict :param gt_entry: Dictionary containing gt_relations, gt_boxes, gt_classes :param pred_entry: Dictionary containing pred_rels, pred_boxes (if detection), pred_classes :param mode: 'det' or 'cls' :param result_dict: :param viz_dict: :param kwargs: :return: """ gt_rels = gt_entry['gt_relations'] gt_boxes = gt_entry['gt_boxes'].astype(float) gt_classes = gt_entry['gt_classes'] pred_rel_inds = pred_entry['pred_rel_inds'] rel_scores = pred_entry['rel_scores'] if mode == 'predcls': pred_boxes = gt_boxes pred_classes = gt_classes obj_scores = np.ones(gt_classes.shape[0]) elif mode == 'sgcls': pred_boxes = gt_boxes pred_classes = pred_entry['pred_classes'] obj_scores = pred_entry['obj_scores'] elif mode == 'sgdet' or mode == 'phrdet': pred_boxes = pred_entry['pred_boxes'].astype(float) pred_classes = pred_entry['pred_classes'] obj_scores = pred_entry['obj_scores'] elif mode == 'preddet': # Only extract the indices that appear in GT prc = intersect_2d(pred_rel_inds, gt_rels[:, :2]) if prc.size == 0: for k in result_dict[mode + '_recall']: result_dict[mode + '_recall'][k].append(0.0) return None, None, None pred_inds_per_gt = prc.argmax(0) pred_rel_inds = pred_rel_inds[pred_inds_per_gt] rel_scores = rel_scores[pred_inds_per_gt] # Now sort the matching ones rel_scores_sorted = argsort_desc(rel_scores[:, 1:]) rel_scores_sorted[:, 1] += 1 rel_scores_sorted = np.column_stack( (pred_rel_inds[rel_scores_sorted[:, 0]], rel_scores_sorted[:, 1])) matches = intersect_2d(rel_scores_sorted, gt_rels) for k in result_dict[mode + '_recall']: rec_i = float(matches[:k].any(0).sum()) / float(gt_rels.shape[0]) result_dict[mode + '_recall'][k].append(rec_i) return None, None, None else: raise ValueError('invalid mode') if multiple_preds: obj_scores_per_rel = obj_scores[pred_rel_inds].prod(1) overall_scores = obj_scores_per_rel[:, None] * rel_scores[:, 1:] score_inds = argsort_desc(overall_scores)[:100] pred_rels = np.column_stack( (pred_rel_inds[score_inds[:, 0]], score_inds[:, 1] + 1)) predicate_scores = rel_scores[score_inds[:, 0], score_inds[:, 1] + 1] else: pred_rels = np.column_stack( (pred_rel_inds, 1 + rel_scores[:, 1:].argmax(1))) predicate_scores = rel_scores[:, 1:].max(1) pred_to_gt, pred_5ples, rel_scores = evaluate_recall( gt_rels, gt_boxes, gt_classes, pred_rels, pred_boxes, pred_classes, predicate_scores, obj_scores, phrdet=mode == 'phrdet', **kwargs) for k in result_dict[mode + '_recall']: match = reduce(np.union1d, pred_to_gt[:k]) rec_i = float(len(match)) / float(gt_rels.shape[0]) result_dict[mode + '_recall'][k].append(rec_i) return pred_to_gt, pred_5ples, rel_scores
def val_batch(sgg_model, batch_num, b, evaluator, eval_m, val_dataset, evaluator_list, evaluator_multiple_preds_list, vis=False, max_obj=10, max_rels=20, train=None, test_zs=None, predicate_weights=None): if val_dataset.torch_detector: scale = 1. box_threshs = [0.2, 0.05, 0.01] else: scale = BOX_SCALE / IM_SCALE box_threshs = [None] pred_entries = [] for box_score_thresh in box_threshs: sgg_model.set_box_score_thresh(box_score_thresh) try: det_res = sgg_model( b.scatter()) # keep as it was in the original code det_res = [det_res] for i, (boxes_i, objs_i, obj_scores_i, rels_i, pred_scores_i) in enumerate(det_res): if vis and len( val_dataset.gt_classes[batch_num + i]) > max_obj: print('skipping a scene graph with too many objects') continue if VG.split == 'stanford': w, h = b[i][1][0, :2] scale_gt = 1. / (BOX_SCALE / max(w, h)) else: scale_gt = 1. gt_entry = { 'gt_classes': val_dataset.gt_classes[batch_num + i].copy(), 'gt_relations': val_dataset.relationships[batch_num + i].copy(), 'gt_boxes': val_dataset.gt_boxes[batch_num + i].copy() * scale_gt, } pred_entry = { 'pred_boxes': boxes_i * scale, 'pred_classes': objs_i, 'pred_rel_inds': rels_i, 'obj_scores': obj_scores_i, 'rel_scores': pred_scores_i, # hack for now. } if predicate_weights is not None: p = 1. / predicate_weights[1:] pred_entry[ 'rel_scores'][:, 1:] = pred_entry['rel_scores'][:, 1:] * p pred_entry[ 'rel_scores'] = pred_entry['rel_scores'] / np.sum( pred_entry['rel_scores'], axis=1, keepdims=True) assert (abs(pred_entry['rel_scores'].sum(1) - 1) < 1e-5).all(), pred_entry['rel_scores'].sum(1) pred_entries.append(pred_entry) for sfx in ['', '_nogc']: evaluator[eval_m + sfx].evaluate_scene_graph_entry( gt_entry, pred_entry) if evaluator_list is not None and len(evaluator_list) > 0: eval_entry(eval_m, gt_entry, pred_entry, evaluator_list, evaluator_multiple_preds_list) if vis: print(val_dataset.filenames[batch_num + i], 'showing ground truth') im_gt = draw_boxes( b[0][0][0].permute(1, 2, 0).data.cpu().numpy().copy(), [ train.ind_to_classes[c] for c in gt_entry['gt_classes'] ], gt_entry['gt_boxes'], torch_detector=val_dataset.torch_detector) plt.figure(figsize=(10, 10)) plt.imshow(im_gt) plt.axis(False) plt.show() show_nx(gt_entry['gt_classes'], gt_entry['gt_boxes'], gt_entry['gt_relations'], train_set=train, test_set=test_zs, torch_detector=val_dataset.torch_detector) print(val_dataset.filenames[batch_num + i], 'showing top %d relationships' % max_rels) im_pred = draw_boxes( b[0][0][0].permute(1, 2, 0).data.cpu().numpy().copy(), [ train.ind_to_classes[c] for c in pred_entry['pred_classes'] ], pred_entry['pred_boxes'], torch_detector=val_dataset.torch_detector) plt.figure(figsize=(10, 10)) plt.imshow(im_pred) plt.axis(False) plt.show() obj_scores = pred_entry['obj_scores'] pred_rel_inds = rels_i obj_scores_per_rel = obj_scores[pred_rel_inds].prod(1) overall_scores = obj_scores_per_rel[:, None] * pred_entry[ 'rel_scores'][:, 1:] score_inds = argsort_desc(overall_scores)[:max_rels] pred_rels = np.column_stack( (pred_rel_inds[score_inds[:, 0]], score_inds[:, 1] + 1)) show_nx(pred_entry['pred_classes'], pred_entry['pred_boxes'], pred_rels, train_set=train, test_set=test_zs, torch_detector=val_dataset.torch_detector) return pred_entries except (ValueError, IndexError) as e: print('no objects or relations found'.upper(), e, b[0][-1], 'trying a smaller threshold')
def evaluate_from_dict(gt_entry, pred_entry, mode, result_dict, multiple_preds=False, viz_dict=None, **kwargs): """ Shortcut to doing evaluate_recall from dict :param gt_entry: Dictionary containing gt_relations, gt_boxes, gt_classes :param pred_entry: Dictionary containing pred_rels, pred_boxes (if detection), pred_classes :param mode: 'det' or 'cls' :param result_dict: :param viz_dict: :param kwargs: :return: """ gt_rels = gt_entry['gt_relations'] gt_boxes = gt_entry['gt_boxes'].astype(float) gt_classes = gt_entry['gt_classes'] pred_rel_inds = pred_entry['pred_rel_inds'] rel_scores = pred_entry['rel_scores'] if mode == 'predcls': pred_boxes = gt_boxes pred_classes = gt_classes obj_scores = np.ones(gt_classes.shape[0]) elif mode == 'sgcls': pred_boxes = gt_boxes pred_classes = pred_entry['pred_classes'] obj_scores = pred_entry['obj_scores'] elif mode == 'sgdet' or mode == 'phrdet': pred_boxes = pred_entry['pred_boxes'].astype(float) pred_classes = pred_entry['pred_classes'] obj_scores = pred_entry['obj_scores'] elif mode == 'preddet': # Only extract the indices that appear in GT prc = intersect_2d(pred_rel_inds, gt_rels[:, :2]) if prc.size == 0: for k in result_dict[mode + '_recall']: result_dict[mode + '_recall'][k].append(0.0) return None, None, None pred_inds_per_gt = prc.argmax(0) pred_rel_inds = pred_rel_inds[pred_inds_per_gt] rel_scores = rel_scores[pred_inds_per_gt] # Now sort the matching ones rel_scores_sorted = argsort_desc(rel_scores[:,1:]) rel_scores_sorted[:,1] += 1 rel_scores_sorted = np.column_stack((pred_rel_inds[rel_scores_sorted[:,0]], rel_scores_sorted[:,1])) matches = intersect_2d(rel_scores_sorted, gt_rels) for k in result_dict[mode + '_recall']: rec_i = float(matches[:k].any(0).sum()) / float(gt_rels.shape[0]) result_dict[mode + '_recall'][k].append(rec_i) return None, None, None else: raise ValueError('invalid mode') # multi_pred controls whether Allow multiple predicates per pair of box0, box1 if multiple_preds: # compute overall_score which is used to sort @100 obj_scores_per_rel = obj_scores[pred_rel_inds].prod(1) overall_scores = obj_scores_per_rel[:,None] * rel_scores[:,1:] # sort and get the 100 highest overall_scores score_inds = argsort_desc(overall_scores)[:100] pred_rels = np.column_stack((pred_rel_inds[score_inds[:,0]], score_inds[:,1]+1)) predicate_scores = rel_scores[score_inds[:,0], score_inds[:,1]+1] else: # only one relationship: get the predicate index and score whose score is the highest # pred_rels is [num_rel, 2+1]; predicate_scores is [num_rel, 1] pred_rels = np.column_stack((pred_rel_inds, 1+rel_scores[:,1:].argmax(1))) # merge matrix; .argmax return the index of maximum predicate_scores = rel_scores[:,1:].max(1) # maximum of each row (most likely relation between certain 2 objects) # print("pred_rels is", pred_rels) # compare gt and prediction: boxes, classes, predicate pred_to_gt, pred_5ples, rel_scores = evaluate_recall( gt_rels, gt_boxes, gt_classes, pred_rels, pred_boxes, pred_classes, predicate_scores, obj_scores, phrdet= mode=='phrdet', **kwargs) ####################################################### # get k = @20 @50 @100 results from length of num_rel which is predicted for k in result_dict[mode + '_recall']: # how many are correct in the first k prdicitons; get a union set, element can be reduplicate; # shape: (#correct,) match = reduce(np.union1d, pred_to_gt[:k]) # attention: match / gt rec_i = float(len(match)) / float(gt_rels.shape[0]) # recall value in @20, @50, @100; stored in self.result_dict result_dict[mode + '_recall'][k].append(rec_i) # pred_to_gt: list len is num_rel, length of list inside the whole lis may not be 1 # pred_5ples: (num_rel, 5), (id0, id1, cls0, cls1, rel) # rel_scores: (num_rel, 3), (box1, box2, predicate score), sorted by overall score return pred_to_gt, pred_5ples, rel_scores
def evaluate_from_dict(gt_entry, pred_entry, mode, result_dict, multiple_preds=False, viz_dict=None, **kwargs): """ Shortcut to doing evaluate_recall from dict :param gt_entry: Dictionary containing gt_relations, gt_boxes, gt_classes :param pred_entry: Dictionary containing pred_rels, pred_boxes (if detection), pred_classes :param mode: 'det' or 'cls' :param result_dict: :param viz_dict: :param kwargs: :return: """ gt_rels = gt_entry['gt_relations'] gt_boxes = gt_entry['gt_boxes'].astype(float) gt_classes = gt_entry['gt_classes'] pred_rel_inds = pred_entry['pred_rel_inds'] rel_scores = pred_entry['rel_scores'] pred_predicates = np.zeros((rel_scores.shape[-1]-1,)) gt_predicates = np.zeros((rel_scores.shape[-1]-1,)) vaild_mask = np.bincount(gt_rels[:, :-1].reshape(-1), minlength=len(gt_classes)) > 0 for i in gt_rels[:, 2]: gt_predicates[i - 1] += 1.0 if mode == 'predcls': pred_boxes = gt_boxes pred_classes = gt_classes obj_scores = np.ones(gt_classes.shape[0]) pred_rel_inds, rel_scores = filter_out_ind(pred_rel_inds, rel_scores, gt_classes.shape[0]) elif mode == 'sgcls': pred_boxes = gt_boxes pred_classes = pred_entry['pred_classes'] obj_scores = pred_entry['obj_scores'] elif mode == 'sgdet' or mode == 'phrdet': pred_boxes = pred_entry['pred_boxes'].astype(float) pred_classes = pred_entry['pred_classes'] obj_scores = pred_entry['obj_scores'] elif mode == 'preddet': # Only extract the indices that appear in GT prc = intersect_2d(pred_rel_inds, gt_rels[:, :2]) if prc.size == 0: for k in result_dict[mode + '_recall']: result_dict[mode + '_recall'][k].append(0.0) return None, None, None pred_inds_per_gt = np.where(prc)[0] pred_rel_inds = pred_rel_inds[pred_inds_per_gt] rel_scores = rel_scores[pred_inds_per_gt] # Now sort the matching ones rel_scores_sorted = argsort_desc(rel_scores[:,1:]) rel_scores = np.sort(np.ravel(rel_scores[:,1:]))[::-1] rel_scores_sorted[:,1] += 1 rel_scores_sorted = np.column_stack((pred_rel_inds[rel_scores_sorted[:,0]], rel_scores_sorted[:,1])) matches = intersect_2d(rel_scores_sorted, gt_rels) for k in result_dict[mode + '_recall']: rec_i = float(matches[:k].any(0).sum()) / float(gt_rels.shape[0]) result_dict[mode + '_recall'][k].append(rec_i) return rel_scores_sorted, None, rel_scores else: raise ValueError('invalid mode') if multiple_preds: obj_scores_per_rel = obj_scores[pred_rel_inds].prod(1) overall_scores = obj_scores_per_rel[:,None] * rel_scores[:,1:] score_inds = argsort_desc(overall_scores)[:100] pred_rels = np.column_stack((pred_rel_inds[score_inds[:,0]], score_inds[:,1]+1)) predicate_scores = rel_scores[score_inds[:,0], score_inds[:,1]+1] else: pred_rels = np.column_stack((pred_rel_inds, 1+rel_scores[:,1:].argmax(1))) predicate_scores = rel_scores[:,1:].max(1) pred_to_gt, pred_5ples, rel_scores = evaluate_recall( gt_rels, gt_boxes, gt_classes, pred_rels, pred_boxes, pred_classes, predicate_scores, obj_scores, phrdet= mode=='phrdet', **kwargs) match_indices = reduce(np.union1d, pred_to_gt[:100]).astype(np.int64) for i in gt_rels[match_indices, 2]: pred_predicates[i - 1] += 1.0 objs_mAP = (gt_classes[vaild_mask]==pred_classes[vaild_mask]).mean() result_dict[mode + '_objs_mAP'].append(objs_mAP) result_dict[mode + '_pred_predicates'].append(pred_predicates) result_dict[mode + '_gt_predicates'].append(gt_predicates) result_dict[mode + '_objs_mAP'].append(objs_mAP) for k in result_dict[mode + '_recall']: match = reduce(np.union1d, pred_to_gt[:k]) rec_i = float(len(match)) / float(gt_rels.shape[0]) result_dict[mode + '_recall'][k].append(rec_i) prc = intersect_2d(gt_rels[:, :-1], pred_rel_inds[:k]).any(1).sum() p_i = float(prc) / float(gt_rels.shape[0]) result_dict[mode + '_pairrecall'][k].append(p_i) return pred_to_gt, pred_5ples, predicate_scores, pred_rels
def evaluate_from_dict(gt_entry, pred_entry, mode, eval_result_dict, eval_result_dict2, gtrel_dist_dict, gtkeyrel_dist_dict, predrel_dist_dict, predmatchedrel_dist_dict, predmatchedkeyrel_dist_dict, multiple_preds=False, predrel_treedeep_scores_dict=None, viz_dict=None, num_predicates=50, **kwargs): """ Shortcut to doing evaluate_recall from dict :param gt_entry: Dictionary containing gt_relations, gt_boxes, gt_classes :param pred_entry: Dictionary containing pred_rels, pred_boxes (if detection), pred_classes :param mode: 'det' or 'cls' :param result_dict: :param viz_dict: :param kwargs: :return: """ gt_rels = gt_entry['gt_relations'] gt_boxes = gt_entry['gt_boxes'].astype(float) gt_classes = gt_entry['gt_classes'] gt_key_rels = gt_entry['gt_key_rels'] if 'gt_key_rels' in gt_entry else None pred_rel_inds = pred_entry['pred_rel_inds'] rel_scores = pred_entry['rel_scores'] rel_rank_scores = pred_entry['rel_rank_scores'] tree = pred_entry['forest'] # get each node's depth tree_depth_dict = None tree_depth = None tree_width = None if tree is not None: root = tree[0] tree_depth_dict = {} get_treeNodes_depth(tree_depth_dict, root) tree_depth = root.max_depth() tree_width = root.max_width() # if rel_rank_scores is not None: # rel_scores *= rel_rank_scores[:, None] if mode == 'predcls': pred_boxes = gt_boxes pred_classes = gt_classes obj_scores = np.ones(gt_classes.shape[0]) elif mode == 'sgcls': pred_boxes = gt_boxes pred_classes = pred_entry['pred_classes'] obj_scores = pred_entry['obj_scores'] elif mode == 'sgdet' or mode == 'phrdet': pred_boxes = pred_entry['pred_boxes'].astype(float) pred_classes = pred_entry['pred_classes'] obj_scores = pred_entry['obj_scores'] elif mode == 'preddet': # Only extract the indices that appear in GT prc = intersect_2d(pred_rel_inds, gt_rels[:, :2]) if prc.size == 0: for k in eval_result_dict[mode + '_recall']: eval_result_dict[mode + '_recall'][k].append(0.0) return None, None, None pred_inds_per_gt = prc.argmax(0) pred_rel_inds = pred_rel_inds[pred_inds_per_gt] rel_scores = rel_scores[pred_inds_per_gt] # Now sort the matching ones rel_scores_sorted = argsort_desc(rel_scores[:, 1:]) rel_scores_sorted[:, 1] += 1 rel_scores_sorted = np.column_stack((pred_rel_inds[rel_scores_sorted[:, 0]], rel_scores_sorted[:, 1])) matches = intersect_2d(rel_scores_sorted, gt_rels) for k in eval_result_dict[mode + '_recall']: rec_i = float(matches[:k].any(0).sum()) / float(gt_rels.shape[0]) eval_result_dict[mode + '_recall'][k].append(rec_i) return None, None, None else: raise ValueError('invalid mode') if multiple_preds: obj_scores_per_rel = obj_scores[pred_rel_inds].prod(1) overall_scores = obj_scores_per_rel[:, None] * rel_scores[:, 1:] score_inds = argsort_desc(overall_scores)[:100] pred_rels = np.column_stack((pred_rel_inds[score_inds[:, 0]], score_inds[:, 1] + 1)) predicate_scores = rel_scores[score_inds[:, 0], score_inds[:, 1] + 1] else: pred_rels = np.column_stack((pred_rel_inds, 1 + rel_scores[:, 1:].argmax(1))) predicate_scores = rel_scores[:, 1:].max(1) # note: for relrank if rel_rank_scores is not None: predicate_scores *= rel_rank_scores RES_pred_to_gt, RES_subobj_to_gt, pred_5ples, rel_scores = evaluate_recall( gt_rels, gt_boxes, gt_classes, pred_rels, pred_boxes, pred_classes, predicate_scores, obj_scores, phrdet=mode == 'phrdet', **kwargs) # evaluate the tree: get the distribution over tree hierarchy if tree is not None: # get gt distribution, only do it in predcls or sgcls mode, use the gt box if mode == 'sgcls' or mode == 'predcls': for pair in gt_rels[:, :2]: gtrel_dist_dict[_gen_key(tree_depth_dict[pair[0]], tree_depth_dict[pair[1]])] += 1 if gt_key_rels is not None: for ind in gt_key_rels: pair = gt_rels[ind, :2] gtkeyrel_dist_dict[_gen_key(tree_depth_dict[pair[0]], tree_depth_dict[pair[1]])] += 1 # get the predicted rels distribution box_pair_inds = pred_rel_inds[:5] for ind in box_pair_inds: predrel_dist_dict[mode][_gen_key(tree_depth_dict[ind[0]], tree_depth_dict[ind[1]])] += 1 norm_triplet_rel_scores = rel_scores.prod(1) norm_triplet_rel_scores = norm_triplet_rel_scores / np.max(norm_triplet_rel_scores) for ind_idx, ind in enumerate(pred_rel_inds): predrel_treedeep_scores_dict[mode][_gen_key(tree_depth_dict[ind[0]], tree_depth_dict[ind[1]])].append(norm_triplet_rel_scores[ind_idx]) # get the match ones, only do it in predcls or sgcls mode, use the gt box match = reduce(np.union1d, RES_subobj_to_gt[:5]) for m in match: predmatchedrel_dist_dict[mode][ _gen_key(tree_depth_dict[gt_rels[int(m), 0]], tree_depth_dict[gt_rels[int(m), 1]])] += 1 if gt_key_rels is not None: key_match = np.intersect1d(match, gt_key_rels) for m in key_match: predmatchedkeyrel_dist_dict[mode][ _gen_key(tree_depth_dict[gt_rels[int(m), 0]], tree_depth_dict[gt_rels[int(m), 1]])] += 1 for pred_to_gt, result_dict in [(RES_pred_to_gt, eval_result_dict), (RES_subobj_to_gt, eval_result_dict2)]: for k in result_dict[mode + '_recall']: match = reduce(np.union1d, pred_to_gt[:k]) key_match = np.intersect1d(match, gt_key_rels) if gt_key_rels is not None else None if gt_key_rels is not None: for idx in range(len(key_match)): local_label = gt_rels[int(key_match[idx]), 2] if (mode + '_key_recall_hit') not in result_dict: result_dict[mode + '_key_recall_hit'] = {} if k not in result_dict[mode + '_key_recall_hit']: result_dict[mode + '_key_recall_hit'][k] = [0] * (num_predicates + 1) result_dict[mode + '_key_recall_hit'][k][int(local_label)] += 1 result_dict[mode + '_key_recall_hit'][k][0] += 1 for idx in range(gt_key_rels.shape[0]): local_label = gt_rels[int(gt_key_rels[idx]), 2] if (mode + '_key_recall_count') not in result_dict: result_dict[mode + '_key_recall_count'] = {} if k not in result_dict[mode + '_key_recall_count']: result_dict[mode + '_key_recall_count'][k] = [0] * (num_predicates + 1) result_dict[mode + '_key_recall_count'][k][int(local_label)] += 1 result_dict[mode + '_key_recall_count'][k][0] += 1 for idx in range(len(match)): local_label = gt_rels[int(match[idx]), 2] if (mode + '_recall_hit') not in result_dict: result_dict[mode + '_recall_hit'] = {} if k not in result_dict[mode + '_recall_hit']: result_dict[mode + '_recall_hit'][k] = [0] * (num_predicates + 1) result_dict[mode + '_recall_hit'][k][int(local_label)] += 1 result_dict[mode + '_recall_hit'][k][0] += 1 for idx in range(gt_rels.shape[0]): local_label = gt_rels[idx, 2] if (mode + '_recall_count') not in result_dict: result_dict[mode + '_recall_count'] = {} if k not in result_dict[mode + '_recall_count']: result_dict[mode + '_recall_count'][k] = [0] * (num_predicates + 1) result_dict[mode + '_recall_count'][k][int(local_label)] += 1 result_dict[mode + '_recall_count'][k][0] += 1 rec_i = float(len(match)) / float(gt_rels.shape[0]) result_dict[mode + '_recall'][k].append(rec_i) if gt_key_rels is not None: key_rec_i = float(len(key_match)) / float(gt_key_rels.shape[0]) result_dict[mode + '_key_recall'][k].append(key_rec_i) return RES_pred_to_gt, RES_subobj_to_gt, pred_5ples, rel_scores, tree_depth, tree_width
def evaluate_from_dict(gt_entry, pred_entry, mode, eval_result_dict, multiple_preds=1, num_predicates=50, **kwargs): """ Shortcut to doing evaluate_recall from dict :param gt_entry: Dictionary containing gt_relations, gt_boxes, gt_classes :param pred_entry: Dictionary containing pred_rels, pred_boxes (if detection), pred_classes :param mode: 'det' or 'cls' :param result_dict: :param viz_dict: :param kwargs: :return: """ gt_rels = gt_entry['gt_relations'] gt_boxes = gt_entry['gt_boxes'].astype(float) gt_classes = gt_entry['gt_classes'] pred_rel_inds = pred_entry['pred_rel_inds'] rel_scores = pred_entry['rel_scores'] if mode == 'predcls': pred_boxes = gt_boxes pred_classes = gt_classes obj_scores = np.ones(gt_classes.shape[0]) elif mode == 'sgcls': pred_boxes = gt_boxes pred_classes = pred_entry['pred_classes'] obj_scores = pred_entry['obj_scores'] elif mode == 'sgdet' or mode == 'phrdet': pred_boxes = pred_entry['pred_boxes'].astype(float) pred_classes = pred_entry['pred_classes'] obj_scores = pred_entry['obj_scores'] elif mode == 'preddet': # Only extract the indices that appear in GT prc = intersect_2d(pred_rel_inds, gt_rels[:, :2]) if prc.size == 0: for k in eval_result_dict[mode + '_recall']: eval_result_dict[mode + '_recall'][k].append(0.0) return None, None, None pred_inds_per_gt = prc.argmax(0) pred_rel_inds = pred_rel_inds[pred_inds_per_gt] rel_scores = rel_scores[pred_inds_per_gt] # Now sort the matching ones rel_scores_sorted = argsort_desc(rel_scores[:, 1:]) rel_scores_sorted[:, 1] += 1 rel_scores_sorted = np.column_stack( (pred_rel_inds[rel_scores_sorted[:, 0]], rel_scores_sorted[:, 1])) matches = intersect_2d(rel_scores_sorted, gt_rels) for k in eval_result_dict[mode + '_recall']: rec_i = float(matches[:k].any(0).sum()) / float(gt_rels.shape[0]) eval_result_dict[mode + '_recall'][k].append(rec_i) return None, None, None else: raise ValueError('invalid mode') if multiple_preds > 1: if multiple_preds == rel_scores.shape[1] - 1: # all predicates obj_scores_per_rel = obj_scores[pred_rel_inds].prod(1) overall_scores = obj_scores_per_rel[:, None] * rel_scores[:, 1:] score_inds = argsort_desc(overall_scores)[:100] pred_rels = np.column_stack( (pred_rel_inds[score_inds[:, 0]], score_inds[:, 1] + 1)) predicate_scores = rel_scores[score_inds[:, 0], score_inds[:, 1] + 1] else: # between 1 and all predictes obj_scores_per_rel = obj_scores[pred_rel_inds].prod(1) # Nr overall_scores = obj_scores_per_rel[:, None] * rel_scores[:, 1:] # Nr, 70 # sort predicate scores for each pair sorted_predicates_idx = np.argsort( -overall_scores, axis=1)[:, :multiple_preds] # Nr, multiple_preds sorted_predicates_scores = np.sort( overall_scores, axis=1)[:, ::-1][:, :multiple_preds] score_inds = argsort_desc(sorted_predicates_scores)[:100] pred_rels = np.column_stack( (pred_rel_inds[score_inds[:, 0]], sorted_predicates_idx[score_inds[:, 0], score_inds[:, 1]] + 1)) predicate_scores = rel_scores[ score_inds[:, 0], sorted_predicates_idx[score_inds[:, 0], score_inds[:, 1]] + 1] else: pred_rels = np.column_stack( (pred_rel_inds, 1 + rel_scores[:, 1:].argmax(1))) predicate_scores = rel_scores[:, 1:].max(1) RES_pred_to_gt, pred_5ples, rel_scores = evaluate_recall( gt_rels, gt_boxes, gt_classes, pred_rels, pred_boxes, pred_classes, predicate_scores, obj_scores, phrdet=mode == 'phrdet', **kwargs) pred_to_gt = RES_pred_to_gt result_dict = eval_result_dict for k in result_dict[mode + '_recall']: match = reduce(np.union1d, pred_to_gt[:k]) for idx in range(len(match)): local_label = gt_rels[int(match[idx]), 2] if (mode + '_recall_hit') not in result_dict: result_dict[mode + '_recall_hit'] = {} if k not in result_dict[mode + '_recall_hit']: result_dict[mode + '_recall_hit'][k] = [0] * (num_predicates + 1) result_dict[mode + '_recall_hit'][k][int(local_label)] += 1 result_dict[mode + '_recall_hit'][k][0] += 1 for idx in range(gt_rels.shape[0]): local_label = gt_rels[idx, 2] if (mode + '_recall_count') not in result_dict: result_dict[mode + '_recall_count'] = {} if k not in result_dict[mode + '_recall_count']: result_dict[mode + '_recall_count'][k] = [0] * (num_predicates + 1) result_dict[mode + '_recall_count'][k][int(local_label)] += 1 result_dict[mode + '_recall_count'][k][0] += 1 rec_i = float(len(match)) / float(gt_rels.shape[0]) result_dict[mode + '_recall'][k].append(rec_i) return RES_pred_to_gt, pred_5ples, rel_scores
def evaluate_from_dict(self, gt_entry, pred_entry, mode, result_dict, multiple_preds=False, viz_dict=None, **kwargs): """ Shortcut to doing evaluate_recall from dict :param gt_entry: Dictionary containing gt_relations, gt_boxes, gt_classes :param pred_entry: Dictionary containing pred_rels, pred_boxes (if detection), pred_classes :param mode: 'det' or 'cls' :param result_dict: :param viz_dict: :param kwargs: :return: """ gt_rels = gt_entry['gt_relations'] gt_boxes = gt_entry['gt_boxes'].astype(float) gt_classes = gt_entry['gt_classes'] pred_rel_inds = pred_entry['pred_rel_inds'] rel_scores = pred_entry['rel_scores'] if mode == 'predcls': pred_boxes = gt_boxes pred_classes = gt_classes obj_scores = np.ones(gt_classes.shape[0]) elif mode == 'sgcls': pred_boxes = gt_boxes pred_classes = pred_entry['pred_classes'] obj_scores = pred_entry['obj_scores'] elif mode == 'objcls': pred_boxes = gt_boxes pred_classes = pred_entry['pred_classes'] obj_scores = pred_entry['obj_scores'] # same as sgcls but assume perfect predicate recognition pred_rel_inds = gt_rels[:, :2] rel_scores = np.zeros((len(gt_rels), rel_scores.shape[1])) rel_scores[np.arange(len(gt_rels)), gt_rels[:, 2]] = 1 elif mode == 'sgdet' or mode == 'phrdet': pred_boxes = pred_entry['pred_boxes'].astype(float) pred_classes = pred_entry['pred_classes'] obj_scores = pred_entry['obj_scores'] elif mode == 'preddet': # Only extract the indices that appear in GT prc = intersect_2d(pred_rel_inds, gt_rels[:, :2]) if prc.size == 0: for k in result_dict[mode + '_recall']: result_dict[mode + '_recall'][k].append(0.0) if self.per_triplet: for k in result_dict[mode + '_recall_norm']: result_dict[mode + '_recall_norm'][k].append(0.0) return None, None, None pred_inds_per_gt = prc.argmax(0) pred_rel_inds = pred_rel_inds[pred_inds_per_gt] rel_scores = rel_scores[pred_inds_per_gt] # Now sort the matching ones rel_scores_sorted = argsort_desc(rel_scores[:, 1:]) rel_scores_sorted[:, 1] += 1 rel_scores_sorted = np.column_stack( (pred_rel_inds[rel_scores_sorted[:, 0]], rel_scores_sorted[:, 1])) matches = intersect_2d(rel_scores_sorted, gt_rels) for k in result_dict[mode + '_recall']: rec_i = float(matches[:k].any(0).sum()) / float( gt_rels.shape[0]) result_dict[mode + '_recall'][k].append(rec_i) if self.per_triplet: for k in result_dict[mode + '_recall_norm']: rec_i = float(matches[:k].any(0).sum()) / float( gt_rels.shape[0]) result_dict[mode + '_recall_norm'][k].append(rec_i) return None, None, None else: raise ValueError('invalid mode') if multiple_preds: obj_scores_per_rel = obj_scores[pred_rel_inds].prod(1) overall_scores = obj_scores_per_rel[:, None] * rel_scores[:, 1:] score_inds = argsort_desc(overall_scores)[:MAX_RECALL_K] pred_rels = np.column_stack( (pred_rel_inds[score_inds[:, 0]], score_inds[:, 1] + 1)) predicate_scores = rel_scores[score_inds[:, 0], score_inds[:, 1] + 1] else: pred_rels = np.column_stack( (pred_rel_inds, 1 + rel_scores[:, 1:].argmax(1))) predicate_scores = rel_scores[:, 1:].max(1) # print('eval', gt_rels.shape, pred_rels.shape, predicate_scores.shape, gt_boxes.shape) pred_to_gt, pred_5ples, rel_scores = evaluate_recall( gt_rels, gt_boxes, gt_classes, pred_rels, pred_boxes, pred_classes, predicate_scores, obj_scores, phrdet=mode == 'phrdet', **kwargs) if self.per_triplet: counts = np.zeros(len(gt_rels)) for rel_i, gt_rel in enumerate(gt_rels): o, s, R = gt_rel tri_str = '{}_{}_{}'.format(gt_classes[o], R, gt_classes[s]) if tri_str in self.triplet_counts: counts[rel_i] = self.triplet_counts[tri_str] weights = self.normalize_counts(counts) for k in result_dict[mode + '_recall']: match = reduce(np.union1d, pred_to_gt[:k]) # print('match', match, type(match)) match = np.array(match).astype(np.int) rec_i = float(len(match)) / float(gt_rels.shape[0]) result_dict[mode + '_recall'][k].append(rec_i) if self.per_triplet: result_dict[mode + '_recall_norm'][k].append( np.sum(weights[match])) if self.per_triplet: # TODO: this looks similar to preddet, reuse that code score_inds = argsort_desc(overall_scores) pred_rels = np.column_stack( (pred_rel_inds[score_inds[:, 0]], score_inds[:, 1] + 1)) # Naive and slow code to get per triplet ranks ranks, counts = np.zeros(len(gt_rels)) - 1, np.zeros(len(gt_rels)) for rel_i, gt_rel in enumerate(gt_rels): o, s, R = gt_rel tri_str = '{}_{}_{}'.format(gt_classes[o], R, gt_classes[s]) if tri_str in self.triplet_counts: counts[rel_i] = self.triplet_counts[tri_str] # select only pairs with this bounding boxes ind = np.where((pred_rels[:, 0] == o) & (pred_rels[:, 1] == s) | (pred_rels[:, 0] == s) & (pred_rels[:, 1] == o))[0] pred_to_gt_triplet, _, _ = evaluate_recall( gt_rel.reshape(1, -1), gt_boxes, gt_classes, pred_rels[ind], pred_boxes, pred_classes) for r, p in enumerate(pred_to_gt_triplet): if len(p) > 0: assert p == [0], (p, gt_rel, pred_to_gt_triplet) ranks[rel_i] = r break if ranks[rel_i] < 0: ranks[rel_i] = MAX_RECALL_K + 1 # For sgcls not all combinations are present, so take some max rank as the default value if tri_str not in self.triplet_ranks: self.triplet_ranks[tri_str] = [] self.triplet_ranks[tri_str].append(ranks[rel_i]) result_dict[mode + '_rank'].extend(ranks) result_dict[mode + '_counts'].extend( counts) # save count to normalize later return pred_to_gt, pred_5ples, rel_scores
def evaluate_from_dict(gt_entry, pred_entry, mode, result_dict, multiple_preds=False, viz_dict=None, **kwargs): """ Shortcut to doing evaluate_recall from dict :param gt_entry: Dictionary containing gt_relations, gt_boxes, gt_classes :param pred_entry: Dictionary containing pred_rels, pred_boxes (if detection), pred_classes :param mode: 'det' or 'cls' :param result_dict: :param viz_dict: :param kwargs: :return: """ gt_rels = gt_entry['gt_relations'] gt_boxes = gt_entry['gt_boxes'].astype(float) gt_classes = gt_entry['gt_classes'] # gt_filenames = gt_entry['filenames'] pred_rel_inds = pred_entry['pred_rel_inds'] rel_scores = pred_entry['rel_scores'] if mode == 'predcls': pred_boxes = gt_boxes pred_classes = gt_classes obj_scores = np.ones(gt_classes.shape[0]) elif mode == 'sgcls': pred_boxes = gt_boxes pred_classes = pred_entry['pred_classes'] obj_scores = pred_entry['obj_scores'] elif mode == 'sgdet' or mode == 'phrdet': pred_boxes = pred_entry['pred_boxes'].astype(float) pred_classes = pred_entry['pred_classes'] obj_scores = pred_entry['obj_scores'] elif mode == 'preddet': # Only extract the indices that appear in GT prc = intersect_2d(pred_rel_inds, gt_rels[:, :2]) if prc.size == 0: for k in result_dict[mode + '_recall']: result_dict[mode + '_recall'][k].append(0.0) return None, None, None pred_inds_per_gt = prc.argmax(0) pred_rel_inds = pred_rel_inds[pred_inds_per_gt] rel_scores = rel_scores[pred_inds_per_gt] # Now sort the matching ones rel_scores_sorted = argsort_desc(rel_scores[:,1:]) rel_scores_sorted[:,1] += 1 rel_scores_sorted = np.column_stack((pred_rel_inds[rel_scores_sorted[:,0]], rel_scores_sorted[:,1])) matches = intersect_2d(rel_scores_sorted, gt_rels) for k in result_dict[mode + '_recall']: rec_i = float(matches[:k].any(0).sum()) / float(gt_rels.shape[0]) result_dict[mode + '_recall'][k].append(rec_i) return None, None, None else: raise ValueError('invalid mode') if multiple_preds: obj_scores_per_rel = obj_scores[pred_rel_inds].prod(1) overall_scores = obj_scores_per_rel[:,None] * rel_scores[:,1:] score_inds = argsort_desc(overall_scores)[:100] pred_rels = np.column_stack((pred_rel_inds[score_inds[:,0]], score_inds[:,1]+1)) predicate_scores = rel_scores[score_inds[:,0], score_inds[:,1]+1] else: pred_rels = np.column_stack((pred_rel_inds, 1+rel_scores[:,1:].argmax(1))) predicate_scores = rel_scores[:,1:].max(1) pred_to_gt, pred_5ples, rel_scores = evaluate_recall( gt_rels, gt_boxes, gt_classes, pred_rels, pred_boxes, pred_classes, predicate_scores, obj_scores, phrdet= mode=='phrdet', **kwargs) for k in result_dict[mode + '_recall']: match = reduce(np.union1d, pred_to_gt[:k]) # FIXME: I think this part of original code is wrong. We shouldn't do union. #: stores tuples (hits, count) hits_per_rel = dict() # gt_rels: shape: (m, 3), (s, p, r) for i in range(gt_rels.shape[0]): gt_s, gt_o, gt_r = gt_rels[i] hits_per_rel.setdefault(gt_r, [0, 0]) hits_per_rel[gt_r][1] += 1 hits_per_rel[gt_r][0] += i in match rec_per_rel = {r: (hits, cnt) for r, (hits, cnt) in hits_per_rel.items()} rec_i = float(len(match)) / float(gt_rels.shape[0]) result_dict[mode + '_recall'][k].append(rec_i) result_dict[mode + '_recall_per_rel'][k].append(rec_per_rel) return pred_to_gt, pred_5ples, rel_scores