def _load_data(self): meta_file = os.path.join(_DATA_ROOT, 'data/%svqa_std_mscoco_%s.meta' % (self._version_suffix, self._subset)) # load meta d = load_json(meta_file) self._images = d['images'] self._quest_ids = np.array(d['quest_id']) vqa_image_ids = [find_image_id_from_fname(im_name) for im_name in self._images] self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32) # load questions data_file = os.path.join(_DATA_ROOT, 'data/%svqa_std_mscoco_%s.data' % (self._version_suffix, self._subset)) d = load_hdf5(data_file) self._quest = d['quest_arr'].astype(np.int32) self._quest_len = d['quest_len'].astype(np.int32) # load QA data data_file = os.path.join(_DATA_ROOT, 'data3/%svqa_mc_w2v_coding_%s.data' % (self._version_suffix, self._subset)) d = load_hdf5(data_file) self._answer = d['cands_w2v'].astype(np.float32) self._labels = d['labels'] self._num = self._labels.size # double check question ids assert (np.all(self._quest_ids == d['quest_ids'])) self._post_proc_answers() self._load_global_image_feature()
def __init__(self, subset='kpval', feat_type='res152'): self._subset = subset # load attribute file if feat_type != 'res152': data_file = 'data/attribute_std_mscoco_%s.data' % self._subset d = load_hdf5(data_file) self._attributes = d['att_arr'].astype(np.float32) else: data_file = 'data/res152_std_mscoco_%s.data' % self._subset d = load_hdf5(data_file) self._attributes = d['features'].astype(np.float32) image_ids = d['image_ids'] self._image_id2index = {image_id: i for i, image_id in enumerate(image_ids)}
def load_dataset(subset='dev', split=None): # load VQA results print('Dataset: Loading VQA results...') vqa_file = 'data/vqa-advrl_vqa_score2000_%s.hdf5' % subset print('Loading %s' % vqa_file) d = load_hdf5(vqa_file) vqa_ids = d['quest_ids'] vqa_scores = d['confidence'][:, :2000] gt_labels = d['labels'] # Load SPLIT 0 print('Dataset: Loading VAQ results <SPLIT 0>...') vaq_file = 'data/vaq-van_vaq_cond_score1000_%s.hdf5' % subset d = load_hdf5(vaq_file) vaq_loss = d['nll'] vaq_ids = d['quest_ids'] quest_id2vaq_index = {qid: i for i, qid in enumerate(vaq_ids)} new_index = [quest_id2vaq_index[qid] for qid in vqa_ids] vaq_loss_0 = vaq_loss[new_index] # Load SPLIT 1 print('Dataset: Loading VAQ results <SPLIT 1>...') vaq_file = 'data/vaq-van_vaq_cond_score1000-2000_%s.hdf5' % subset d = load_hdf5(vaq_file) vaq_loss = d['nll'] vaq_ids = d['quest_ids'] quest_id2vaq_index = {qid: i for i, qid in enumerate(vaq_ids)} new_index = [quest_id2vaq_index[qid] for qid in vqa_ids] vaq_loss_1 = vaq_loss[new_index] vaq_loss = np.concatenate([vaq_loss_0, vaq_loss_1], 1) # Split dataset because training set has over-fitted if subset == 'train' or split is None: return vqa_scores, vaq_loss, gt_labels, vqa_ids else: num = gt_labels.size num_train = int(num * 0.7) if split == 0: print('Loading training split') vqa_scores = vqa_scores[:num_train, :] vaq_loss = vaq_loss[:num_train, :] gt_labels = gt_labels[:num_train] else: print('Loading testing split') vqa_scores = vqa_scores[num_train:, :] vaq_loss = vaq_loss[num_train:, :] gt_labels = gt_labels[num_train:] return vqa_scores, vaq_loss, gt_labels, vqa_ids
def _load_image_data(self): feat_file = os.path.join(FEAT_ROOT, 'mscoco_res152_%s.h5' % self._subset) d = load_hdf5(feat_file) self._im_feats = d['features'] self._feats_image_ids = d['image_ids'] self._num = self._im_feats.shape[0]
def load_answer_data(subset): # get question ids d = load_json('data/vqa_std_mscoco_%s.meta' % subset) quest_ids = d['quest_id'] # load top answers d = load_hdf5('data/vqa_std_mscoco_%s.data' % subset) top_answers = d['answer'] # load answer sequence d = load_hdf5('data/answer_std_mscoco_%s.data' % subset) answer_arr = d['ans_arr'] answer_arr_len = d['ans_len'] quest_id2index = {qid: i for i, qid in enumerate(quest_ids)} return quest_id2index, quest_ids, top_answers, answer_arr, answer_arr_len
def _load_data(self): meta_file = os.path.join(_DATA_ROOT, 'data/%svqa_std_mscoco_%s.meta' % (self._version_suffix, self._subset)) data_file = os.path.join(_DATA_ROOT, 'data/%svqa_std_mscoco_%s.data' % (self._version_suffix, self._subset)) # load meta d = load_json(meta_file) self._images = d['images'] self._quest_ids = np.array(d['quest_id']) self.quest_id2index = {qid: i for i, qid in enumerate(self._quest_ids)} vqa_image_ids = [find_image_id_from_fname(im_name) for im_name in self._images] self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32) image_id2quest_ids = {} for qid, im_id in zip(self._quest_ids, self._vqa_image_ids): if im_id in image_id2quest_ids: # already create key image_id2quest_ids[im_id].append(qid) else: image_id2quest_ids[im_id] = [qid] self.image_id2quest_ids = image_id2quest_ids # load QA data d = load_hdf5(data_file) self._quest = d['quest_arr'].astype(np.int32) self._quest_len = d['quest_len'].astype(np.int32) self._num = self._quest_len.size self._answer = d['answer'].astype(np.int32) self._check_valid_answers() # self._load_caption_feature() self._load_global_image_feature()
def _init_exemplars(self, subset): from util import load_hdf5 print('Initialising statastics with ground truth') d = load_hdf5('data/vqa_std_mscoco_%s.data' % subset) gts = self.parse_gt_questions(d['quest_arr'], d['quest_len']) # update stat self._update_samples(gts, generate_key=True)
def __init__(self, top_ans_file, k): print('Answer layer: Loading top answer sequences') d = load_hdf5(top_ans_file) self.k = k self.data_len = d['answer_seq'].shape[-1] self._answer_seq = PinnedDataManager(d['answer_seq']) self._answer_len = PinnedDataManager(d['answer_seq_len'])
def _load_answer_sequence(self): if not self._output_ans_seq: return data_file = self._data_root + 'data/%sanswer_std_mscoco_%s.data' % (self._version_suffix, self._subset) d = load_hdf5(data_file) self._answer_seq = d['ans_arr'] self._ans_seq_len = d['ans_len']
def _load_data(self): meta_file = os.path.join(_DATA_ROOT, 'data/%svqa_std_mscoco_%s.meta' % (self._version_suffix, self._subset)) data_file = os.path.join(_DATA_ROOT, 'data4/%svar_ivqa_%s_question_answers.data' % (self._version_suffix, self._subset)) # load meta d = load_json(meta_file) images = d['images'] quest_ids = np.array(d['quest_id']) vqa_image_ids = [find_image_id_from_fname(im_name) for im_name in images] quest_id2image_id = {qid: im_id for (qid, im_id) in zip(quest_ids, vqa_image_ids)} self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32) # load QA data d = load_hdf5(data_file) self._quest_ids = d['ext_quest_ids'].astype(np.int32) self._quest = d['ext_quest_arr'].astype(np.int32) self._quest_len = d['ext_quest_len'].astype(np.int32) self._answer = d['ext_top_answer'].astype(np.int32) self._check_valid_answers() # sort images abs_quest_ids = self._quest_ids[:, 0] self._vqa_image_ids = np.array([quest_id2image_id[_id] for _id in abs_quest_ids], dtype=np.int32) # self._load_caption_feature() self._load_global_image_feature()
def _load_data(self): meta_file = 'data/%svqa_std_mscoco_%s.meta' % (self._version_suffix, self._subset) data_file = 'data/%svqa_std_mscoco_%s.data' % (self._version_suffix, self._subset) self._images = load_json(meta_file)['images'] d = load_hdf5(data_file) self._quest = d['quest_arr'].astype(np.int32) self._quest_len = d['quest_len'].astype(np.int32) self._answer = d['answer'].astype(np.int32) self._num = self._answer.size self._check_valid_answers() vqa_image_ids = [ find_image_id_from_fname(im_name) for im_name in self._images ] self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32) # load caption data self._load_caption_data() # load attributes if self._attr_type == 'res152': self._load_global_image_feature() else: self._load_attributes() # load answer sequences self._load_answer_sequence()
def load_image_data(subset): # load images data_file = 'data/res152_std_mscoco_%s.data' % subset d = load_hdf5(data_file) image_ids = d['image_ids'] feats = d['features'] return feats, image_ids
def initialise_with_gt(self): # load data from util import load_hdf5 from time import time t = time() print('Initialising statastics with ground truth') d = load_hdf5('data/vqa_std_mscoco_kprestval.data') gts = parse_gt_questions(d['quest_arr'], d['quest_len']) # update stat self._update_samples(gts, generate_key=True, update_reference=True) print('Initialisation finished, time %0.2fs' % (time() - t)) print('Total number of questions: %d' % self.num_total) print('Number of unique questions: %d' % self.num_unique) print('Average question counts: %0.2f' % self.average_count) num_counts = self.history.values() max_count = max(num_counts) min_count = min(num_counts) for t in [0.01, 0.02, 0.03, 0.05]: r_min = np.exp(-max_count / self.average_count * t) r_max = np.exp(-min_count / self.average_count * t) print( '[t=%0.4f] Max question counts: %d, ratio %0.3f, estimated reward % 0.3f' % (t, max_count, max_count / self.average_count, r_min)) print( '[t=%0.4f] Min question counts: %d, ratio %0.3f, estimated reward: %0.3f' % (t, min_count, min_count / self.average_count, r_max)) self.gt_keys = {k: None for k in self.history.keys()}
def _load_data(self): meta_file = os.path.join( _DATA_ROOT, 'data/%svqa_std_mscoco_%s.meta' % (self._version_suffix, self._subset)) data_file = os.path.join( _DATA_ROOT, 'data/%svqa_std_mscoco_%s.data' % (self._version_suffix, self._subset)) # load meta d = load_json(meta_file) self._images = d['images'] self._quest_ids = np.array(d['quest_id']) vqa_image_ids = [ find_image_id_from_fname(im_name) for im_name in self._images ] self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32) # load QA data d = load_hdf5(data_file) self._quest = d['quest_arr'].astype(np.int32) self._quest_len = d['quest_len'].astype(np.int32) self._answer = d['answer'].astype(np.int32) self._check_valid_answers() # self._load_caption_feature() self._load_global_image_feature()
def _load_data(self): meta_file = os.path.join( _DATA_ROOT, 'data/%svqa_std_mscoco_%s.meta' % (self._version_suffix, self._subset)) data_file = os.path.join( _DATA_ROOT, 'data3/%svqa_mc_w2v_coding_%s.data' % (self._version_suffix, self._subset)) # load meta d = load_json(meta_file) self._images = d['images'] self._quest_ids = np.array(d['quest_id']) vqa_image_ids = [ find_image_id_from_fname(im_name) for im_name in self._images ] self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32) # load Question data d = load_hdf5(data_file) self._quests = d['quest_w2v'].astype(np.float32) self._num = self._quests.shape[0] # double check question ids assert (np.all(self._quest_ids == d['quest_ids'])) # load Answer data self._load_answer_proposals() self._load_global_image_feature()
def _load_subset(subset): fname = 'mscoco_res152_%s.h5' % subset print('Loading file %s' % fname) d = load_hdf5(os.path.join(data_root, fname)) ids = d['image_ids'].tolist() f = d['features'] return ids, f
def _load_data(self): data_file = 'data/vqa_std_mscoco_multilabel_%s.data' % self._subset d = load_hdf5(data_file) self._image_ids = d['image_ids'] self._feat = d['features'] self._labels = d['labels'] self._num = self._labels.shape[0]
def _load_global_image_feature(self): data_file = self._data_root + 'data/res152_std_mscoco_%s.data' % self._subset d = load_hdf5(data_file) image_ids = d['image_ids'] image_id2att_index = {image_id: i for i, image_id in enumerate(image_ids)} vqa_index2att_index = [image_id2att_index[image_id] for image_id in self._vqa_image_ids] self._vqa_index2att_index = np.array(vqa_index2att_index, dtype=np.int32) self._attributes = d['features']
def _init_image_cache(self): from util import load_hdf5 d = load_hdf5('data/attribute_std_mscoco_kpval.data') # d = load_hdf5('data/res152_std_mscoco_kpval.data') image_ids = d['image_ids'] self.im_feats = d['att_arr'] image_id2index = {image_id: idx for idx, image_id in enumerate(image_ids)} self.image_id2index = image_id2index
def _load_attributes(self): data_file = self._data_root + 'data/attribute_std_mscoco_%s.data' % self._subset d = load_hdf5(data_file) self._attributes = d['att_arr'].astype(np.float32) image_ids = d['image_ids'] image_id2att_index = {image_id: i for i, image_id in enumerate(image_ids)} vqa_index2att_index = [image_id2att_index[image_id] for image_id in self._vqa_image_ids] self._vqa_index2att_index = np.array(vqa_index2att_index, dtype=np.int32)
def _load_caption_feature(self): data_file = 'data/capt1k_std_mscoco_%s.data' % self._subset d = load_hdf5(data_file) image_ids = d['image_ids'] image_id2att_index = {image_id: i for i, image_id in enumerate(image_ids)} vqa_index2att_index = [image_id2att_index[image_id] for image_id in self._vqa_image_ids] self._vqa_index2att_index = np.array(vqa_index2att_index, dtype=np.int32) self._attributes = d['att_arr']
def _load_irrelevant_data(self): # load meta d = load_json('data/QRD_irrelevant_meta.json') self._ir_images = d['images'] # load QA data d = load_hdf5('data/QRD_irrelevant_data.data') self._ir_arr = d['quest'].astype(np.int32) self._ir_arr_len = d['quest_len'].astype(np.int32) self._num_ir = len(self._ir_images)
def __init__(self, subset): data_file = 'data/res152_std_mscoco_%s.data' % subset d = load_hdf5(data_file) image_ids = d['image_ids'] self.data = d['features'] self.image_id2index = { image_id: i for i, image_id in enumerate(image_ids) }
def __init__(self, top_k): prop_file = 'data/vqa-st_vqa_score2000_kpval.hdf5' d = load_hdf5(prop_file) quest_ids = d['quest_ids'] self.quest_id2score_index = {int(qid): i for i, qid in enumerate(quest_ids)} self.scores = d['confidence'][:, :2000] self.top_k = top_k self.encoder = SentenceEncoder('answer') self.reverse_top_ans_dict = _load_reverse_answer_vocab()
def _load_global_image_feature(self): data_file = 'data/res152_std_mscoco_%s.data' % self._subset d = load_hdf5(data_file) image_ids = d['image_ids'] self._feat = d['features'] self.image_id2att_index = { image_id: i for i, image_id in enumerate(image_ids) }
def _load_question_data(self): data_file = 'data/vqa_retrieval_mscoco_%s.data' % self._subset d = load_hdf5(data_file) self._quest_image_ids = d['image_ids'].astype(np.int32) self._quest = d['quest_arr'].astype(np.int32) self._quest_len = d['quest_len'].astype(np.int32) self._answer = d['ans_arr'].astype(np.int32) self._answer_len = d['ans_len'].astype(np.int32) self._quest_ids = d['quest_ids'].astype(np.int32)
def _load_image_data(self): feat_file = os.path.join(FEAT_ROOT, 'mscoco_res152_dev_full.h5') d = load_hdf5(feat_file) self._im_feats = d['features'] self._feats_image_ids = d['image_ids'] self._num = self._im_feats.shape[0] self._image_id2image_index = { im_id: i for i, im_id in enumerate(self._feats_image_ids) }
def _load_qa_data(self): data_file = 'data/att_rerank/qa_arr.h5' d = load_hdf5(data_file) self._quest_arr = d['quest_arr'] self._quest_len = d['quest_len'] self._top2000_arr = d['top2000_arr'] self._top2000_len = d['top2000_len'] self._model_topk_pred = d['att_cand_arr'] self.quest_ids = d['quest_id'] self._image_ids = d['image_id']
def __init__(self, batch_size, pad_token): self.batch_size = batch_size from util import load_hdf5 data_file = 'data/vqa_std_mscoco_kprestval.data' d = load_hdf5(data_file) gts = _parse_gt_questions(d['quest_arr'], d['quest_len']) gts = [_g + [2] for _g in gts] self._quest, self._quest_len = put_to_array(gts, pad_token, max_length=20) self.num = self._quest_len.size
def _load_contrastive_answers(self): if self._is_test: return meta_file = 'data/vqa_retrieval_cst_ans_mscoco_%s.meta' % self._subset data_file = 'data/vqa_retrieval_cst_ans_mscoco_%s.data' % self._subset d = load_json(meta_file)['quest_id2cand_index'] self.quest_id2cand_index = {int(k): v for k, v in d.iteritems()} d = load_hdf5(data_file) self.cst_answers = d['cand_arr'].astype(np.int32) self.cst_ans_len = d['cand_len'].astype(np.int32)