def process(method, inf_type='rand'):
if inf_type == 'rand':
res_file = 'result/tmp_bs_RL2_final_%s.json' % method
else:
res_file = 'result/tmp_bs_RL2_final_%s_BEAM.json' % method
if os.path.exists(res_file):
print('File %s already exist, skipped' % res_file)
return
# cands = load_results()
model = _TYPE2Model[method]()
mc_ctx = MultiChoiceQuestionManger(subset='val')
task_data = load_lm_outputs(method, inf_type)
belief_sets = {}
t = time()
num = len(task_data)
for i, ans_key in enumerate(task_data.keys()):
# time it
avg_time = (time() - t)
print('%d/%d (%0.2f sec/sample)' % (i, num, avg_time))
t = time()
# extract basis info
cands = task_data[ans_key]
quest_id = cands[0]['question_id']
# gt_answer = mc_ctx.get_gt_answer(quest_id)
image_id = mc_ctx.get_image_id(quest_id)
image = mc_ctx.get_image_file(quest_id)
# process
gt_question = mc_ctx.get_question(quest_id)
i_scores, i_questions = [], []
for item in cands:
target = item['question']
pred_ans, vqa_score = model.get_score(image_id, target)
# inset check
is_valid = compare_answer(pred_ans, ans_key)
if not is_valid:
continue
i_questions.append(target)
i_scores.append([float(vqa_score), item['score']])
print('%d/%d' % (len(i_questions), len(cands)))
bs_i = {
'image': image,
'image_id': image_id,
'question': gt_question,
'answer': ans_key,
'belief_sets': i_questions,
'belief_strength': i_scores
}
belief_sets[ans_key] = bs_i
save_json(res_file, belief_sets)
class PredictionVisualiser(object):
def __init__(self, model_name, K=3, do_plot=True):
self._gt_mgr = MultiChoiceQuestionManger(subset='trainval',
load_ans=True)
self._rev_map = SentenceGenerator(trainset='trainval')
self._top_k = K
self._do_plot = do_plot
self._model_name = model_name
self._cache_dir = 'att_maps/%s' % self._model_name
mkdir_if_missing(self._cache_dir)
def plot(self, quest_id, scores, att_map):
if type(quest_id) != int:
quest_id = int(quest_id)
scores = scores.flatten()
if scores.size == 2001:
scores[-1] = 0
# show question and gt answer
question = self._gt_mgr.get_question(quest_id)
gt_ans = self._gt_mgr.get_gt_answer(quest_id)
print('\n====================================')
print('Q: %s' % question)
print('A: %s' % gt_ans)
# show top k prediction
index = (-scores).argsort()[:self._top_k]
for idx in index:
pred_ans = self._rev_map.index_to_top_answer(idx)
print('P: %-20s\t(%0.2f)' % (pred_ans, scores[idx]))
print('\n')
# show image
im_file = self._gt_mgr.get_image_file(quest_id)
im = imread(im_file)
if np.rank(im) == 2:
im = np.tile(im[::, np.newaxis], [1, 1, 3])
if self._do_plot:
imshow(im)
plt.show()
else:
self.save_cache_file(quest_id, im, att_map, question)
return
# show attention map
tokens = _tokenize_sentence(question)
self._show_attention_map(im, att_map, tokens)
def save_cache_file(self, quest_id, im, att_map, question):
from scipy.io import savemat
sv_path = os.path.join(self._cache_dir, '%d.mat' % quest_id)
savemat(sv_path, {'im': im, 'att_map': att_map, 'quest': question})
def _show_attention_map(self, im, att_map, tokens):
att_map = att_map.reshape([-1, 14, 14])
num = att_map.shape[0]
if num == 1:
tokens = [' '.join(tokens)] # merge to a sentence
else:
tokens = [' '.join(tokens)] # merge to a sentence
# mean_map = att_map.mean(axis=0)[np.newaxis, ::]
# att_map = np.concatenate([att_map, mean_map], axis=0)
# tokens.append('average')
# render and plot
for i, am in enumerate(att_map):
am = resize(am, im.shape[:2], preserve_range=True)
am = am / am.max()
v = im * am[:, :, np.newaxis]
v = np.minimum(np.round(v).astype(np.uint8), 255)
if self._do_plot:
imshow(v)
plt.title('%s <%d>' % (tokens[0], i))
plt.show()
