def visualise(): mc_ctx = MultiChoiceQuestionManger() to_sentence = SentenceGenerator(trainset='trainval') # writer = ExperimentWriter('latex/examples_replay_buffer_rescore') writer = ExperimentWriter('latex/examples_replay_buffer_rescore_prior') # d = load_json('vqa_replay_buffer/vqa_replay_low_rescore.json') d = load_json('vqa_replay_buffer/vqa_replay_low_rescore_prior_05_04.json') memory = d['memory'] # show random 100 keys = deepcopy(memory.keys()) np.random.seed(123) np.random.shuffle(keys) vis_keys = keys[:100] for i, quest_key in enumerate(vis_keys): pathes = memory[quest_key] if len(pathes) == 0: continue # if it has valid questions quest_id = int(quest_key) image_id = mc_ctx.get_image_id(quest_id) gt_question = mc_ctx.get_question(quest_id) answer = mc_ctx.get_gt_answer(quest_id) head = 'Q: %s A: %s' % (gt_question, answer) im_file = '%s2014/COCO_%s2014_%012d.jpg' % ('val', 'val', image_id) im_path = os.path.join(IM_ROOT, im_file) questions = [] for p in pathes.keys(): conf1, conf2 = pathes[p] _tokens = [int(t) for t in p.split(' ')] sentence = to_sentence.index_to_question(_tokens) descr = '%s (%0.2f-%0.2f)' % (sentence, conf1, conf2) questions.append(descr) writer.add_result(image_id, quest_id, im_path, head, questions) writer.render()
def process(method, inf_type='rand'): if inf_type == 'rand': res_file = 'result/tmp_bs_RL2_final_%s.json' % method else: res_file = 'result/tmp_bs_RL2_final_%s_BEAM.json' % method if os.path.exists(res_file): print('File %s already exist, skipped' % res_file) return # cands = load_results() model = _TYPE2Model[method]() mc_ctx = MultiChoiceQuestionManger(subset='val') task_data = load_lm_outputs(method, inf_type) belief_sets = {} t = time() num = len(task_data) for i, ans_key in enumerate(task_data.keys()): # time it avg_time = (time() - t) print('%d/%d (%0.2f sec/sample)' % (i, num, avg_time)) t = time() # extract basis info cands = task_data[ans_key] quest_id = cands[0]['question_id'] # gt_answer = mc_ctx.get_gt_answer(quest_id) image_id = mc_ctx.get_image_id(quest_id) image = mc_ctx.get_image_file(quest_id) # process gt_question = mc_ctx.get_question(quest_id) i_scores, i_questions = [], [] for item in cands: target = item['question'] pred_ans, vqa_score = model.get_score(image_id, target) # inset check is_valid = compare_answer(pred_ans, ans_key) if not is_valid: continue i_questions.append(target) i_scores.append([float(vqa_score), item['score']]) print('%d/%d' % (len(i_questions), len(cands))) bs_i = { 'image': image, 'image_id': image_id, 'question': gt_question, 'answer': ans_key, 'belief_sets': i_questions, 'belief_strength': i_scores } belief_sets[ans_key] = bs_i save_json(res_file, belief_sets)
def process(delta=0.2): # w2v_ncoder = SentenceEncoder() # load gt and answer manager ctx = MultiChoiceQuestionManger(subset='val') # load candidates candidates = load_json('result/var_vaq_beam_VAQ-VARDSDC_full.json') # load candidate scores score_list = load_json( 'result/var_vaq_beam_VAQ-VARDSDC_full_oracle_dump.json') score_d = {item['aug_quest_id']: item['CIDEr'] for item in score_list} # loop over questions dataset = {} unk_image_ids = [] question_id2image_id = {} for item in candidates: aug_id = item['question_id'] question = item['question'] image_id = item['image_id'] unk_image_ids.append(image_id) question_id = int(aug_id / 1000) score = score_d[aug_id] question_id2image_id[question_id] = image_id if question_id in dataset: assert (question not in dataset[question_id]) dataset[question_id][question] = score else: dataset[question_id] = {question: score} # get stat unk_image_ids = set(unk_image_ids) num_images = len(unk_image_ids) print('Find %d unique keys from %d images' % (len(dataset), num_images)) print('%0.3f questions on average' % (len(dataset) / float(num_images))) # visualise vis_keys = dataset.keys() np.random.shuffle(vis_keys) for quest_id in vis_keys[:20]: ans = ctx.get_gt_answer(quest_id) image_id = ctx.get_image_id(quest_id) gt = ctx.get_question(quest_id).lower() print('\ngt: %s' % gt) for quest, sc in dataset[quest_id].items(): print('%s (%0.3f)' % (quest, sc))
def process(delta = 0.2): w2v_ncoder = SentenceEncoder() # load gt and answer manager ctx = MultiChoiceQuestionManger(subset='train') # load candidates candidates = load_json('result/var_vaq_beam_VAQ-VAR_full_kptrain.json') # load candidate scores score_list = load_json('result/var_vaq_beam_VAQ-VAR_full_kptrain_oracle_dump.json') score_d = {item['aug_quest_id']: item['CIDEr'] for item in score_list} # loop over questions dataset = {} unk_image_ids = [] question_id2image_id = {} for item in candidates: aug_id = item['question_id'] question = item['question'] image_id = item['image_id'] unk_image_ids.append(image_id) question_id = int(aug_id / 1000) score = score_d[aug_id] question_id2image_id[question_id] = image_id if question_id in dataset: assert (question not in dataset[question_id]) dataset[question_id][question] = score else: dataset[question_id] = {question: score} # get stat unk_image_ids = set(unk_image_ids) num_images = len(unk_image_ids) print('Find %d unique keys from %d images' % (len(dataset), num_images)) print('%0.3f questions on average' % (len(dataset) / float(num_images))) # build tuple num_pairs = 0 offset = 0 cst_pairs = [] image_ids, quest_ids, question_w2v, answer_w2v = [], [], [], [] num_task = len(dataset) t = time() for _i, (quest_id, item) in enumerate(dataset.items()): if _i % 1000 == 0: print('processed: %d/%d (%0.2f sec./batch)' % (_i, num_task, time()-t)) t = time() ans = ctx.get_gt_answer(quest_id) image_id = ctx.get_image_id(quest_id) assert(image_id == question_id2image_id[quest_id]) gt = ctx.get_question(quest_id).lower() gt = ' '.join(word_tokenize(gt)) include_gt = np.any(np.array(item.values()) == 10.) sc, ps = [], [] if gt not in item and not include_gt: item[gt] = 10. for q, s in item.items(): sc.append(s) ps.append(q) sc = np.array(sc, dtype=np.float32) _this_n = len(ps) path_ind = np.arange(_this_n) + offset # data checking and assertion try: assert (np.sum(sc == 10.) <= 1) # only one gt except Exception as e: ind = np.where(sc == 10.)[0] for _idx in ind: print('%s' % (ps[_idx])) raise e # find contrastive pairs diff = sc[np.newaxis, :] - sc[:, np.newaxis] valid_entries = diff >= delta neg, pos = np.where(valid_entries) assert (np.all(np.greater_equal(sc[pos] - sc[neg], delta))) pos_q_ind = path_ind[pos] neg_q_ind = path_ind[neg] # save _this_pairs = [[p, n] for p, n in zip(pos_q_ind, neg_q_ind)] cst_pairs += _this_pairs # encode answer _ans_w2v = w2v_ncoder.encode(ans) ans_w2v = np.tile(_ans_w2v, [_this_n, 1]) answer_w2v.append(ans_w2v) # encode questions for p in ps: _q_w2v = w2v_ncoder.encode(p) question_w2v.append(_q_w2v) image_ids.append(image_id) quest_ids.append(quest_id) # update pointer offset += _this_n num_pairs += _this_n print('Total pairs: %d' % num_pairs) # merge cst_pairs = np.array(cst_pairs, dtype=np.int32) image_ids = np.array(image_ids, dtype=np.int32) quest_ids = np.array(quest_ids, dtype=np.int32) answer_w2v = np.concatenate(answer_w2v, axis=0).astype(np.float32) question_w2v = np.concatenate(question_w2v, axis=0).astype(np.float32) from util import save_hdf5 sv_file = 'result/cst_ranking_kptrain_delta%g.data' % delta save_hdf5(sv_file, {'cst_pairs': cst_pairs, 'image_ids': image_ids, 'quest_ids': quest_ids, 'answer_w2v': answer_w2v, 'question_w2v': question_w2v})
def process(): w2v_ncoder = SentenceEncoder() # load gt and answer manager ctx = MultiChoiceQuestionManger(subset='val') # load candidates candidates = load_json('result/var_vaq_beam_VAQ-VAR_full_kptest.json') # load candidate scores score_list = load_json( 'result/var_vaq_beam_VAQ-VAR_full_kptest_oracle_dump.json') score_d = {item['aug_quest_id']: item['CIDEr'] for item in score_list} # loop over questions dataset = {} unk_image_ids = [] question_id2image_id = {} for item in candidates: aug_id = item['question_id'] question = item['question'] image_id = item['image_id'] unk_image_ids.append(image_id) question_id = int(aug_id / 1000) score = score_d[aug_id] question_id2image_id[question_id] = image_id if question_id in dataset: assert (question not in dataset[question_id]) dataset[question_id][question] = (score, aug_id) else: dataset[question_id] = {question: (score, aug_id)} # get stat unk_image_ids = set(unk_image_ids) num_images = len(unk_image_ids) print('Find %d unique keys from %d images' % (len(dataset), num_images)) print('%0.3f questions on average' % (len(dataset) / float(num_images))) # build tuple num_pairs = 0 offset = 0 image_ids, quest_ids, aug_quest_ids, question_w2v, answer_w2v, scores = [], [], [], [], [], [] num_task = len(dataset) t = time() for _i, (quest_id, item) in enumerate(dataset.items()): if _i % 1000 == 0: print('processed: %d/%d (%0.2f sec./batch)' % (_i, num_task, time() - t)) t = time() ans = ctx.get_gt_answer(quest_id) image_id = ctx.get_image_id(quest_id) assert (image_id == question_id2image_id[quest_id]) ps = [] for q, (s, aug_id) in item.items(): ps.append(q) aug_quest_ids.append(aug_id) scores.append(s) _this_n = len(ps) # encode answer _ans_w2v = w2v_ncoder.encode(ans) ans_w2v = np.tile(_ans_w2v, [_this_n, 1]) answer_w2v.append(ans_w2v) # encode questions for p in ps: _q_w2v = w2v_ncoder.encode(p) question_w2v.append(_q_w2v) image_ids.append(image_id) quest_ids.append(quest_id) # update pointer offset += _this_n num_pairs += _this_n print('Total pairs: %d' % num_pairs) # merge image_ids = np.array(image_ids, dtype=np.int32) quest_ids = np.array(quest_ids, dtype=np.int32) scores = np.array(scores, dtype=np.float32) aug_quest_ids = np.array(aug_quest_ids, dtype=np.int64) answer_w2v = np.concatenate(answer_w2v, axis=0).astype(np.float32) question_w2v = np.concatenate(question_w2v, axis=0).astype(np.float32) from util import save_hdf5 sv_file = 'result/cst_ranking_kptest.data' save_hdf5( sv_file, { 'image_ids': image_ids, 'quest_ids': quest_ids, 'aug_quest_ids': aug_quest_ids, 'scores': scores, 'answer_w2v': answer_w2v, 'question_w2v': question_w2v })