def process(method, inf_type='rand'): if inf_type == 'rand': res_file = 'result/tmp_bs_RL2_final_%s.json' % method else: res_file = 'result/tmp_bs_RL2_final_%s_BEAM.json' % method if os.path.exists(res_file): print('File %s already exist, skipped' % res_file) return # cands = load_results() model = _TYPE2Model[method]() mc_ctx = MultiChoiceQuestionManger(subset='val') task_data = load_lm_outputs(method, inf_type) belief_sets = {} t = time() num = len(task_data) for i, ans_key in enumerate(task_data.keys()): # time it avg_time = (time() - t) print('%d/%d (%0.2f sec/sample)' % (i, num, avg_time)) t = time() # extract basis info cands = task_data[ans_key] quest_id = cands[0]['question_id'] # gt_answer = mc_ctx.get_gt_answer(quest_id) image_id = mc_ctx.get_image_id(quest_id) image = mc_ctx.get_image_file(quest_id) # process gt_question = mc_ctx.get_question(quest_id) i_scores, i_questions = [], [] for item in cands: target = item['question'] pred_ans, vqa_score = model.get_score(image_id, target) # inset check is_valid = compare_answer(pred_ans, ans_key) if not is_valid: continue i_questions.append(target) i_scores.append([float(vqa_score), item['score']]) print('%d/%d' % (len(i_questions), len(cands))) bs_i = { 'image': image, 'image_id': image_id, 'question': gt_question, 'answer': ans_key, 'belief_sets': i_questions, 'belief_strength': i_scores } belief_sets[ans_key] = bs_i save_json(res_file, belief_sets)
class PredictionVisualiser(object): def __init__(self, model_name, K=3, do_plot=True): self._gt_mgr = MultiChoiceQuestionManger(subset='trainval', load_ans=True) self._rev_map = SentenceGenerator(trainset='trainval') self._top_k = K self._do_plot = do_plot self._model_name = model_name self._cache_dir = 'att_maps/%s' % self._model_name mkdir_if_missing(self._cache_dir) def plot(self, quest_id, scores, att_map): if type(quest_id) != int: quest_id = int(quest_id) scores = scores.flatten() if scores.size == 2001: scores[-1] = 0 # show question and gt answer question = self._gt_mgr.get_question(quest_id) gt_ans = self._gt_mgr.get_gt_answer(quest_id) print('\n====================================') print('Q: %s' % question) print('A: %s' % gt_ans) # show top k prediction index = (-scores).argsort()[:self._top_k] for idx in index: pred_ans = self._rev_map.index_to_top_answer(idx) print('P: %-20s\t(%0.2f)' % (pred_ans, scores[idx])) print('\n') # show image im_file = self._gt_mgr.get_image_file(quest_id) im = imread(im_file) if np.rank(im) == 2: im = np.tile(im[::, np.newaxis], [1, 1, 3]) if self._do_plot: imshow(im) plt.show() else: self.save_cache_file(quest_id, im, att_map, question) return # show attention map tokens = _tokenize_sentence(question) self._show_attention_map(im, att_map, tokens) def save_cache_file(self, quest_id, im, att_map, question): from scipy.io import savemat sv_path = os.path.join(self._cache_dir, '%d.mat' % quest_id) savemat(sv_path, {'im': im, 'att_map': att_map, 'quest': question}) def _show_attention_map(self, im, att_map, tokens): att_map = att_map.reshape([-1, 14, 14]) num = att_map.shape[0] if num == 1: tokens = [' '.join(tokens)] # merge to a sentence else: tokens = [' '.join(tokens)] # merge to a sentence # mean_map = att_map.mean(axis=0)[np.newaxis, ::] # att_map = np.concatenate([att_map, mean_map], axis=0) # tokens.append('average') # render and plot for i, am in enumerate(att_map): am = resize(am, im.shape[:2], preserve_range=True) am = am / am.max() v = im * am[:, :, np.newaxis] v = np.minimum(np.round(v).astype(np.uint8), 255) if self._do_plot: imshow(v) plt.title('%s <%d>' % (tokens[0], i)) plt.show()