class MultipleChoiceEvaluater(object):
def __init__(self,
subset='val',
num_eval=None,
need_im_feat=True,
need_attr=False,
use_ans_type=False,
feat_type='res152'):
anno_file = 'data/MultipleChoicesQuestionsKarpathy%sV2.0.json' % subset.title(
)
self._subset = subset
d = load_json(anno_file)
self._id2type = d['candidate_types']
self._annotations = d['annotation']
if num_eval == 0:
num_eval = len(self._annotations)
self._num_to_eval = num_eval
self._idx = 0
self._need_attr = need_attr
self._need_im_feat = need_im_feat
self._quest_encoder = SentenceEncoder('question')
self._answer_encoder = SentenceEncoder('answer')
self._im_encoder = MCDataFetcher(subset='kp%s' % subset,
feat_type=feat_type)
self.num_samples = len(self._annotations)
self._mc_ctx = MultiChoiceQuestionManger(subset='val')
self._group_by_answer_type()
self._use_ans_type = use_ans_type
def get_task_data(self):
info = self._annotations[self._idx]
questions = info['questions']
answer = info['answer']
answer_idx = info['answer_id']
image_id = info['image_id']
quest_id = int(info['coco_question_ids'][0])
# prepare for output
outputs = []
if self._need_im_feat:
im_feat = self._im_encoder.get_image_feature(image_id)
outputs.append(im_feat)
if self._need_attr:
attr = self._im_encoder.get_attribute_feature(image_id)
outputs.append(attr)
quest, quest_len = self._quest_encoder.encode_sentences(questions)
if self._use_ans_type:
ans_type = add_answer_type(quest_id, self._mc_ctx)
outputs += [quest, quest_len, None, ans_type, answer_idx, image_id]
else:
ans, ans_len = self._answer_encoder.encode_sentences(answer)
outputs += [
quest, quest_len, None, ans, ans_len, answer_idx, image_id
]
self._idx += 1
return outputs
def get_labels(self, answer_ids):
answer_id2labels = {
info['answer_id']: info['labels']
for info in self._annotations
}
type_mat = []
for ans_id in answer_ids:
labels = np.array(answer_id2labels[ans_id])
type_mat.append(labels[np.newaxis, :])
type_mat = np.concatenate(type_mat, axis=0)
return (type_mat == 0).argmax(axis=1)
def _group_by_answer_type(self):
self.answer_ids_per_type = {}
for info in self._annotations:
for quest_id in info['coco_question_ids']:
answer_id = info['answer_id']
type_str = self._mc_ctx.get_answer_type(quest_id)
self.answer_ids_per_type.setdefault(type_str,
[]).append(answer_id)
@staticmethod
def _get_intersect_table(pool, target):
# create hashing table
hash_tab = {k: 0 for k in target}
return np.array([c in hash_tab for c in pool])
def evaluate_results(self, answer_ids, scores, model_type=None):
types, results = [], []
# ALL
cmc = self._evaluate_worker(answer_ids, scores, 'ALL')
results.append(cmc)
types.append('all')
# per answer type
for type in self.answer_ids_per_type.keys():
target = np.array(self.answer_ids_per_type[type])
sel_tab = self._get_intersect_table(answer_ids, target)
cmc = self._evaluate_worker(answer_ids[sel_tab],
scores[sel_tab, :], type)
results.append(cmc)
types.append(type)
results = np.concatenate(results, axis=0)
if model_type is not None:
from scipy.io import savemat
res_file = 'result/mc_%s_result.mat' % model_type.lower()
savemat(res_file, {'cmc': results, 'types': types})
def _evaluate_worker(self, answer_ids, scores, type):
answer_id2labels = {
info['answer_id']: info['labels']
for info in self._annotations
}
type_mat = []
for ans_id in answer_ids:
labels = np.array(answer_id2labels[ans_id])
type_mat.append(labels[np.newaxis, :])
type_mat = np.concatenate(type_mat, axis=0)
gt_mask = np.equal(type_mat, 0)
gt_scores = []
for i, (gt, score) in enumerate(zip(gt_mask, scores)):
gt_scores.append(score[gt].max())
# find the rank of gt scores
gt_scores = np.array(gt_scores)[:, np.newaxis]
sorted_scores = -np.sort(-scores, axis=1)
gt_rank = np.equal(sorted_scores, gt_scores).argmax(axis=1)
# print('\nMean rank: %0.2f' % gt_rank.mean())
# compute cmc
num, num_cands = gt_mask.shape
cmc = np.zeros(num_cands, dtype=np.float32)
for i in range(num_cands):
cmc[i] = np.less_equal(gt_rank, i).sum()
cmc = cmc / num * 100.
print('\n======= type %s =======' % type.upper())
print('---------- cmc -----------')
print('Top 1: %0.2f' % cmc[0])
print('Top 3: %0.2f' % cmc[2])
print('Top 10: %0.2f' % cmc[9])
# top 1 analysis
self.top1_analysis(scores, type_mat)
return cmc[np.newaxis, :]
def top1_analysis(self, scores, type_mat):
# print('======= Top 1 analysis =======')
print('--------- top 1 -----------')
pred_labels = scores.argmax(axis=1)
types = np.zeros_like(pred_labels)
for i, idx in enumerate(pred_labels):
types[i] = type_mat[i, idx]
bin_count = np.bincount(types)
num = pred_labels.size
for i, c in enumerate(bin_count):
type_str = self._id2type[str(i)]
pnt = float(c) * 100. / num
print('%s: %02.2f' % (type_str, pnt))
print('\n')
def prediction_examples(self):
pass
class MultipleChoiceEvaluater(object):
def __init__(self, subset='val'):
anno_file = 'data/MultipleChoicesQuestionsKarpathy%s.json' % subset.title()
self._subset = subset
d = load_json(anno_file)
self._id2type = d['candidate_types']
self._annotations = d['annotation']
self._idx = 0
self.num_samples = len(self._annotations)
self._mc_ctx = MultiChoiceQuestionManger(subset='val')
self._group_by_answer_type()
def update_annotation(self, do_update=True):
man_file = 'data/distractor_analysis.json'
anno = load_json(man_file)['annotation']
hash_tab = {d['answer_idx']: d['confused'] for d in anno}
if not do_update:
return np.array([d['answer_idx'] for d in anno if d['confused']])
# return np.array(hash_tab.keys())
for datum in self._annotations:
ans_id = datum['answer_id']
datum['labels'] = np.array(datum['labels'])
if ans_id in hash_tab:
conf_ids = hash_tab[ans_id]
if conf_ids:
tmp_ids = np.array(conf_ids)
datum['labels'][tmp_ids] = 0
return np.array(hash_tab.keys())
def get_labels(self, answer_ids):
answer_id2labels = {info['answer_id']: info['labels'] for info in self._annotations}
type_mat = []
for ans_id in answer_ids:
labels = np.array(answer_id2labels[ans_id])
type_mat.append(labels[np.newaxis, :])
type_mat = np.concatenate(type_mat, axis=0)
return (type_mat == 0).argmax(axis=1)
def _group_by_answer_type(self):
self.answer_ids_per_type = {}
for info in self._annotations:
for quest_id in info['coco_question_ids']:
answer_id = info['answer_id']
type_str = self._mc_ctx.get_answer_type(quest_id)
self.answer_ids_per_type.setdefault(type_str, []).append(answer_id)
@staticmethod
def _get_intersect_table(pool, target):
# create hashing table
hash_tab = {k: 0 for k in target}
return np.array([c in hash_tab for c in pool])
def evaluate_results(self, answer_ids, scores, model_type=None):
types, results = [], []
# ALL
cmc = self._evaluate_worker(answer_ids, scores, 'ALL')
results.append(cmc)
types.append('all')
# per answer type
for type in self.answer_ids_per_type.keys():
target = np.array(self.answer_ids_per_type[type])
sel_tab = self._get_intersect_table(answer_ids, target)
cmc = self._evaluate_worker(answer_ids[sel_tab],
scores[sel_tab, :], type)
results.append(cmc)
types.append(type)
results = np.concatenate(results, axis=0)
if model_type is not None:
from scipy.io import savemat
res_file = 'result/mc_%s_result.mat' % model_type.lower()
savemat(res_file, {'cmc': results, 'types': types})
def _evaluate_worker(self, answer_ids, scores, type):
answer_id2labels = {info['answer_id']: info['labels'] for info in self._annotations}
type_mat = []
for ans_id in answer_ids:
labels = np.array(answer_id2labels[ans_id])
type_mat.append(labels[np.newaxis, :])
type_mat = np.concatenate(type_mat, axis=0)
gt_mask = np.equal(type_mat, 0)
gt_scores = []
for i, (gt, score) in enumerate(zip(gt_mask, scores)):
gt_scores.append(score[gt].max())
# find the rank of gt scores
gt_scores = np.array(gt_scores)[:, np.newaxis]
sorted_scores = -np.sort(-scores, axis=1)
gt_rank = np.equal(sorted_scores, gt_scores).argmax(axis=1)
# print('\nMean rank: %0.2f' % gt_rank.mean())
# compute cmc
num, num_cands = gt_mask.shape
cmc = np.zeros(num_cands, dtype=np.float32)
for i in range(num_cands):
cmc[i] = np.less_equal(gt_rank, i).sum()
cmc = cmc / num * 100.
print('\n======= type %s =======' % type.upper())
print('---------- cmc -----------')
print('Top 1: %0.3f' % cmc[0])
print('Top 5: %0.3f' % cmc[4])
print('Top 10: %0.3f' % cmc[9])
# top 1 analysis
self.top1_analysis(scores, type_mat)
return cmc[np.newaxis, :]
def top1_analysis(self, scores, type_mat):
# print('======= Top 1 analysis =======')
print('--------- top 1 -----------')
pred_labels = scores.argmax(axis=1)
types = np.zeros_like(pred_labels)
for i, idx in enumerate(pred_labels):
types[i] = type_mat[i, idx]
bin_count = np.bincount(types)
num = pred_labels.size
for i, c in enumerate(bin_count):
type_str = self._id2type[str(i)]
pnt = float(c) * 100. / num
print('%s: %02.2f' % (type_str, pnt))
print('\n')
