def _load_data(self): meta_file = 'data/%svqa_std_mscoco_%s.meta' % (self._version_suffix, self._subset) data_file = 'data/%svqa_std_mscoco_%s.data' % (self._version_suffix, self._subset) # load meta file d = load_json(meta_file) self._images = d['images'] self._load_answer_type(d['quest_id']) self._quest_ids = np.array(d['quest_id']) vqa_image_ids = [ find_image_id_from_fname(im_name) for im_name in self._images ] self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32) # load data file d = load_hdf5(data_file) self._quest = d['quest_arr'].astype(np.int32) self._quest_len = d['quest_len'].astype(np.int32) self._answer = d['answer'].astype(np.int32) self._num = self._answer.size self._check_valid_answers() vqa_image_ids = [ find_image_id_from_fname(im_name) for im_name in self._images ] self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32) # load caption data self._load_caption_data() # load attributes self._load_attributes() # load answer sequences self._load_answer_sequence()
def _load_data(self): meta_file = os.path.join( _DATA_ROOT, 'data/%svqa_std_mscoco_%s.meta' % (self._version_suffix, self._subset)) data_file = os.path.join( _DATA_ROOT, 'data/%svqa_std_mscoco_%s.data' % (self._version_suffix, self._subset)) # load meta d = load_json(meta_file) self._images = d['images'] self._quest_ids = np.array(d['quest_id']) vqa_image_ids = [ find_image_id_from_fname(im_name) for im_name in self._images ] self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32) # load QA data d = load_hdf5(data_file) self._quest = d['quest_arr'].astype(np.int32) self._quest_len = d['quest_len'].astype(np.int32) self._answer = d['answer'].astype(np.int32) self._check_valid_answers() # self._load_caption_feature() self._load_global_image_feature()
def _load_data(self): meta_file = os.path.join( _DATA_ROOT, 'data/%svqa_std_mscoco_%s.meta' % (self._version_suffix, self._subset)) data_file = os.path.join( _DATA_ROOT, 'data/%svqa_std_mscoco_%s.data' % (self._version_suffix, self._subset)) # load meta d = load_json(meta_file) self._images = d['images'] self._quest_ids = np.array(d['quest_id']) vqa_image_ids = [ find_image_id_from_fname(im_name) for im_name in self._images ] self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32) # load Question data d = load_hdf5(data_file) self._quest = d['quest_arr'].astype(np.int32) self._quest_len = d['quest_len'].astype(np.int32) # self._quests = d['quest_w2v'].astype(np.float32) self._num = self._quest.shape[0] # load Answer data self._load_answer_proposals() self._load_global_image_feature()
def _load_data(self): meta_file = os.path.join(_DATA_ROOT, 'data/%svqa_std_mscoco_%s.meta' % (self._version_suffix, self._subset)) data_file = os.path.join(_DATA_ROOT, 'data/%svqa_std_mscoco_%s.data' % (self._version_suffix, self._subset)) # load meta d = load_json(meta_file) self._images = d['images'] self._quest_ids = np.array(d['quest_id']) self.quest_id2index = {qid: i for i, qid in enumerate(self._quest_ids)} vqa_image_ids = [find_image_id_from_fname(im_name) for im_name in self._images] self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32) image_id2quest_ids = {} for qid, im_id in zip(self._quest_ids, self._vqa_image_ids): if im_id in image_id2quest_ids: # already create key image_id2quest_ids[im_id].append(qid) else: image_id2quest_ids[im_id] = [qid] self.image_id2quest_ids = image_id2quest_ids # load QA data d = load_hdf5(data_file) self._quest = d['quest_arr'].astype(np.int32) self._quest_len = d['quest_len'].astype(np.int32) self._num = self._quest_len.size self._answer = d['answer'].astype(np.int32) self._check_valid_answers() # self._load_caption_feature() self._load_global_image_feature()
def _load_data(self): meta_file = os.path.join(_DATA_ROOT, 'data/%svqa_std_mscoco_%s.meta' % (self._version_suffix, self._subset)) data_file = os.path.join(_DATA_ROOT, 'data4/%svar_ivqa_%s_question_answers.data' % (self._version_suffix, self._subset)) # load meta d = load_json(meta_file) images = d['images'] quest_ids = np.array(d['quest_id']) vqa_image_ids = [find_image_id_from_fname(im_name) for im_name in images] quest_id2image_id = {qid: im_id for (qid, im_id) in zip(quest_ids, vqa_image_ids)} self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32) # load QA data d = load_hdf5(data_file) self._quest_ids = d['ext_quest_ids'].astype(np.int32) self._quest = d['ext_quest_arr'].astype(np.int32) self._quest_len = d['ext_quest_len'].astype(np.int32) self._answer = d['ext_top_answer'].astype(np.int32) self._check_valid_answers() # sort images abs_quest_ids = self._quest_ids[:, 0] self._vqa_image_ids = np.array([quest_id2image_id[_id] for _id in abs_quest_ids], dtype=np.int32) # self._load_caption_feature() self._load_global_image_feature()
def _load_data(self): meta_file = os.path.join(_DATA_ROOT, 'data/%svqa_std_mscoco_%s.meta' % (self._version_suffix, self._subset)) data_file = os.path.join(_DATA_ROOT, 'data3/%svqa_mc_w2v_coding_%s.data' % (self._version_suffix, self._subset)) # load meta d = load_json(meta_file) self._images = d['images'] self._quest_ids = np.array(d['quest_id']) vqa_image_ids = [find_image_id_from_fname(im_name) for im_name in self._images] self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32) # load QA data d = load_hdf5(data_file) self._quests = d['quest_w2v'].astype(np.float32) self._answer = d['cands_w2v'].astype(np.float32) self._labels = d['labels'] self._num = self._labels.size # double check question ids assert (np.all(self._quest_ids == d['quest_ids'])) self._post_proc_answers() self._load_global_image_feature()
def get_subset_image_ids(subset): from util import find_image_id_from_fname, load_json print('Parsing image index of set %s...' % subset) meta_file = '../VQA-tensorflow/data/vqa_std_mscoco_%s.meta' % subset images = load_json(meta_file)['images'] # get image ids image_ids = [find_image_id_from_fname(fname) for fname in images] unique_ids = np.unique(np.array(image_ids, dtype=np.int32)) # print('Finshed, find %d unique images.\n' % unique_ids.size) return unique_ids.tolist()
def _load_subset(): fpath = os.path.join(fdir, 'unique_vtfq.json') d = load_json(fpath) dataset = [] for item in d.values(): i_label = int(item['label']) if i_label == 2: # irrelevant image = os.path.basename(item['image']) image_id = find_image_id_from_fname(image) dataset.append({ 'image': 'val2014/' + image, 'image_id': image_id, 'question': item['question'] + '?' }) return dataset
def _load_subset(subset): fpath = os.path.join(fdir, 'input/qrpe_%s.json' % subset) d = load_json(fpath) dataset = [] db_type = 'train' if subset == 'train' else 'val' for item in d.values(): i_from = item['imtype'] i_label = int(item['label']) if i_label == 2 and 'coco' in i_from: image_id = find_image_id_from_fname(item['image']) dataset.append({ 'image': '%s2014/%s' % (db_type, item['image']), 'image_id': image_id, 'question': item['question'] }) return dataset
def _load_data(self): meta_file = os.path.join('data/QRD_irrelevant_meta_test.json') data_file = os.path.join('data/QRD_irrelevant_data_test.data') # load meta d = load_json(meta_file) self._images = d['images'] vqa_image_ids = [ find_image_id_from_fname(im_name) for im_name in self._images ] self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32) # load QA data d = load_hdf5(data_file) self._quest = d['quest'].astype(np.int32) self._quest_len = d['quest_len'].astype(np.int32) self._labels = d['labels'].astype(np.float32) # self._check_valid_answers() self._num = self._quest_len.size
def load_qa_data(subset): # load meta file meta_file = os.path.join(_DATA_ROOT, 'data/vqa_std_mscoco_%s.meta' % subset) d = load_json(meta_file) image_ids = [find_image_id_from_fname(im_name) for im_name in d['images']] image_id2qa_index = {} for i, image_id in enumerate(image_ids): if image_id in image_id2qa_index: image_id2qa_index[image_id].append(i) else: image_id2qa_index[image_id] = [i] data_file = os.path.join(_DATA_ROOT, 'data/vqa_std_mscoco_%s.data' % subset) d = load_hdf5(data_file) labels = d['answer'] return image_id2qa_index, labels
def _load_data(self): meta_file = 'data/vqa_std_mscoco_%s.meta' % self._subset data_file = 'data/vqa_std_mscoco_%s.data' % self._subset # load meta d = load_json(meta_file) self._images = d['images'] self._quest_ids = np.array(d['quest_id']) self.quest_id2index = {qid: i for i, qid in enumerate(self._quest_ids)} vqa_image_ids = [ find_image_id_from_fname(im_name) for im_name in self._images ] self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32) # load QA data d = load_hdf5(data_file) self._quest = d['quest_arr'].astype(np.int32) self._quest_len = d['quest_len'].astype(np.int32) self._answer = d['answer'].astype(np.int32)
def process(model_type='MLB2-att'): result_format = 'result/vqa_score_on_mbs_%s_v2.json' questions = load_task_data() model = load_model(model_type) results = [] num = len(questions) t = time() for i, item in enumerate(questions): if i % 100 == 0: print('Testing [%s]: %d/%d (speed: %0.2f sec/100 samples)' % (model_type, i, num, time() - t)) t = time() image_id = find_image_id_from_fname(item['image']) question = item['target'] answer = item['answer'] score = model.query_score(image_id, question, answer) results.append(score) res_file = result_format % model_type save_json(res_file, results)
def test_irrelevant_management(): from util import load_json, find_image_id_from_fname d = load_json('data/vqa_std_mscoco_trainval.meta') image_ids = [find_image_id_from_fname(fname) for fname in d['images']] irrel_man = IrrelevantManager(image_ids) num = len(image_ids) batch_size = 4 num_batches = 4 from time import time t = time() for i in range(num_batches): print('\nBatch %d: ' % i) index = np.random.choice(num, size=(batch_size,), replace=False) neg_index = irrel_man.query_irrelevant(index) print('pos:') print(index) print('neg:') print(neg_index) tot = time() - t print('Time: %0.2f ms/batch' % (tot * 1000. / float(num_batches)))
def split_subset(subset, inputs): print('Processing split %s' % subset) images, quest_id, quest_arr, quest_len, answer, attr_image_ids, attr_arr = inputs vqa_image_ids = [find_image_id_from_fname(fname) for fname in images] # get coco ids coco_ids = get_image_id(subset) # build coco id hashing table coco_ids = {image_id: i for i, image_id in enumerate(coco_ids)} # split vqa data keep_tab = np.array([im_id in coco_ids for im_id in vqa_image_ids]) images = [im for im, keep in zip(images, keep_tab) if keep] quest_id = [q_id for q_id, keep in zip(quest_id, keep_tab) if keep] quest_arr = quest_arr[keep_tab] quest_len = quest_len[keep_tab] answer = answer[keep_tab] # split attribute data keep_tab = np.array([im_id in coco_ids for im_id in attr_image_ids]) attr_image_ids = attr_image_ids[keep_tab] attr_arr = attr_arr[keep_tab] # process answers encode_answers(quest_id, subset) # save to files vqa_meta_file = 'data/vqa_std_mscoco_kp%s.meta' % subset save_json(vqa_meta_file, {'images': images, 'quest_id': quest_id}) vqa_data_file = 'data/vqa_std_mscoco_kp%s.data' % subset save_hdf5(vqa_data_file, { 'quest_arr': quest_arr, 'quest_len': quest_len, 'answer': answer }) attr_data_file = 'data/attribute_std_mscoco_kp%s.data' % subset save_hdf5(attr_data_file, { 'image_ids': attr_image_ids, 'att_arr': attr_arr })
def _load_relevant_data(self): meta_file = os.path.join(_DATA_ROOT, 'data/vqa_std_mscoco_%s.meta' % self._subset) data_file = os.path.join(_DATA_ROOT, 'data/vqa_std_mscoco_%s.data' % self._subset) # load meta d = load_json(meta_file) self._r_images = d['images'] image_ids = [ find_image_id_from_fname(fname) for fname in self._r_images ] self.ezir_ctx = IrrelevantManager(image_ids) # load QA data d = load_hdf5(data_file) self._r_arr = d['quest_arr'].astype(np.int32) self._r_arr_len = d['quest_len'].astype(np.int32) self._num_rel = len(self._r_images) # load blacklist blist = load_json('data/kptest_blacklist.json') is_valid = np.array([image_id not in blist for image_id in image_ids]) print('%d/%d valid' % (is_valid.sum(), is_valid.size)) self.valid_ids = np.where(is_valid)[0]
def process(model_type='MLB2-att'): # result_format = 'result/vqa_pred_bsadv_%s_v2.json' result_format = 'result/vqa_pred_bsadv_%s.json' questions = load_task_data() model = load_model(model_type) results = [] num = len(questions) t = time() acc = [] for i, item in enumerate(questions): if i % 100 == 0: print('Testing [%s]: %d/%d (speed: %0.2f sec/100 samples)' % (model_type, i, num, time() - t)) t = time() image_id = find_image_id_from_fname(item['image']) question = item['target'] answer = item['answer'] ans, _ = model.get_score(image_id, question) acc.append(ans == answer) # score = model.query_score(image_id, question, answer) results.append(ans) res_file = result_format % model_type save_json(res_file, results) print('%s: Acc: %0.2f' % (model_type, 100. * sum(acc) / float(len(acc))))
def split_subset(subset, inputs): print('Processing split %s' % subset) images, quest_id, quest_arr, quest_len, answer, attr_image_ids, attr_arr = inputs vqa_image_ids = [find_image_id_from_fname(fname) for fname in images] # get coco ids coco_ids = get_image_id(subset) # build coco id hashing table coco_ids = {image_id: i for i, image_id in enumerate(coco_ids)} # split vqa data keep_tab = np.array([im_id in coco_ids for im_id in vqa_image_ids]) images = [im for im, keep in zip(images, keep_tab) if keep] quest_id = [q_id for q_id, keep in zip(quest_id, keep_tab) if keep] quest_arr = quest_arr[keep_tab] quest_len = quest_len[keep_tab] answer = answer[keep_tab] # split attribute data keep_tab = np.array([im_id in coco_ids for im_id in attr_image_ids]) attr_image_ids = attr_image_ids[keep_tab] attr_arr = attr_arr[keep_tab] # process answers encode_answers(quest_id, subset)
def make_blacklist(): d = load_json('data/vqa_std_mscoco_kptest.meta') # question_ids = d['quest_id'] images = d['images'] image_ids = [find_image_id_from_fname(im_name) for im_name in images] return {image_id: None for image_id in image_ids}