def _load_data(self):
meta_file = os.path.join(_DATA_ROOT,
'data/%svqa_std_mscoco_%s.meta' %
(self._version_suffix, self._subset))
# load meta
d = load_json(meta_file)
self._images = d['images']
self._quest_ids = np.array(d['quest_id'])
vqa_image_ids = [find_image_id_from_fname(im_name) for im_name in self._images]
self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32)
# load questions
data_file = os.path.join(_DATA_ROOT,
'data/%svqa_std_mscoco_%s.data' %
(self._version_suffix, self._subset))
d = load_hdf5(data_file)
self._quest = d['quest_arr'].astype(np.int32)
self._quest_len = d['quest_len'].astype(np.int32)
# load QA data
data_file = os.path.join(_DATA_ROOT,
'data3/%svqa_mc_w2v_coding_%s.data' %
(self._version_suffix, self._subset))
d = load_hdf5(data_file)
self._answer = d['cands_w2v'].astype(np.float32)
self._labels = d['labels']
self._num = self._labels.size
# double check question ids
assert (np.all(self._quest_ids == d['quest_ids']))
self._post_proc_answers()
self._load_global_image_feature()
def __init__(self, subset='kpval', feat_type='res152'):
self._subset = subset
# load attribute file
if feat_type != 'res152':
data_file = 'data/attribute_std_mscoco_%s.data' % self._subset
d = load_hdf5(data_file)
self._attributes = d['att_arr'].astype(np.float32)
else:
data_file = 'data/res152_std_mscoco_%s.data' % self._subset
d = load_hdf5(data_file)
self._attributes = d['features'].astype(np.float32)
image_ids = d['image_ids']
self._image_id2index = {image_id: i for i, image_id in enumerate(image_ids)}
def load_dataset(subset='dev', split=None):
# load VQA results
print('Dataset: Loading VQA results...')
vqa_file = 'data/vqa-advrl_vqa_score2000_%s.hdf5' % subset
print('Loading %s' % vqa_file)
d = load_hdf5(vqa_file)
vqa_ids = d['quest_ids']
vqa_scores = d['confidence'][:, :2000]
gt_labels = d['labels']
# Load SPLIT 0
print('Dataset: Loading VAQ results <SPLIT 0>...')
vaq_file = 'data/vaq-van_vaq_cond_score1000_%s.hdf5' % subset
d = load_hdf5(vaq_file)
vaq_loss = d['nll']
vaq_ids = d['quest_ids']
quest_id2vaq_index = {qid: i for i, qid in enumerate(vaq_ids)}
new_index = [quest_id2vaq_index[qid] for qid in vqa_ids]
vaq_loss_0 = vaq_loss[new_index]
# Load SPLIT 1
print('Dataset: Loading VAQ results <SPLIT 1>...')
vaq_file = 'data/vaq-van_vaq_cond_score1000-2000_%s.hdf5' % subset
d = load_hdf5(vaq_file)
vaq_loss = d['nll']
vaq_ids = d['quest_ids']
quest_id2vaq_index = {qid: i for i, qid in enumerate(vaq_ids)}
new_index = [quest_id2vaq_index[qid] for qid in vqa_ids]
vaq_loss_1 = vaq_loss[new_index]
vaq_loss = np.concatenate([vaq_loss_0, vaq_loss_1], 1)
# Split dataset because training set has over-fitted
if subset == 'train' or split is None:
return vqa_scores, vaq_loss, gt_labels, vqa_ids
else:
num = gt_labels.size
num_train = int(num * 0.7)
if split == 0:
print('Loading training split')
vqa_scores = vqa_scores[:num_train, :]
vaq_loss = vaq_loss[:num_train, :]
gt_labels = gt_labels[:num_train]
else:
print('Loading testing split')
vqa_scores = vqa_scores[num_train:, :]
vaq_loss = vaq_loss[num_train:, :]
gt_labels = gt_labels[num_train:]
return vqa_scores, vaq_loss, gt_labels, vqa_ids
def _load_image_data(self):
feat_file = os.path.join(FEAT_ROOT,
'mscoco_res152_%s.h5' % self._subset)
d = load_hdf5(feat_file)
self._im_feats = d['features']
self._feats_image_ids = d['image_ids']
self._num = self._im_feats.shape[0]
def load_answer_data(subset):
# get question ids
d = load_json('data/vqa_std_mscoco_%s.meta' % subset)
quest_ids = d['quest_id']
# load top answers
d = load_hdf5('data/vqa_std_mscoco_%s.data' % subset)
top_answers = d['answer']
# load answer sequence
d = load_hdf5('data/answer_std_mscoco_%s.data' % subset)
answer_arr = d['ans_arr']
answer_arr_len = d['ans_len']
quest_id2index = {qid: i for i, qid in enumerate(quest_ids)}
return quest_id2index, quest_ids, top_answers, answer_arr, answer_arr_len
def _load_data(self):
meta_file = os.path.join(_DATA_ROOT,
'data/%svqa_std_mscoco_%s.meta' %
(self._version_suffix, self._subset))
data_file = os.path.join(_DATA_ROOT,
'data/%svqa_std_mscoco_%s.data' %
(self._version_suffix, self._subset))
# load meta
d = load_json(meta_file)
self._images = d['images']
self._quest_ids = np.array(d['quest_id'])
self.quest_id2index = {qid: i for i, qid in enumerate(self._quest_ids)}
vqa_image_ids = [find_image_id_from_fname(im_name) for im_name in self._images]
self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32)
image_id2quest_ids = {}
for qid, im_id in zip(self._quest_ids, self._vqa_image_ids):
if im_id in image_id2quest_ids: # already create key
image_id2quest_ids[im_id].append(qid)
else:
image_id2quest_ids[im_id] = [qid]
self.image_id2quest_ids = image_id2quest_ids
# load QA data
d = load_hdf5(data_file)
self._quest = d['quest_arr'].astype(np.int32)
self._quest_len = d['quest_len'].astype(np.int32)
self._num = self._quest_len.size
self._answer = d['answer'].astype(np.int32)
self._check_valid_answers()
# self._load_caption_feature()
self._load_global_image_feature()
def _init_exemplars(self, subset):
from util import load_hdf5
print('Initialising statastics with ground truth')
d = load_hdf5('data/vqa_std_mscoco_%s.data' % subset)
gts = self.parse_gt_questions(d['quest_arr'], d['quest_len'])
# update stat
self._update_samples(gts, generate_key=True)
def __init__(self, top_ans_file, k):
print('Answer layer: Loading top answer sequences')
d = load_hdf5(top_ans_file)
self.k = k
self.data_len = d['answer_seq'].shape[-1]
self._answer_seq = PinnedDataManager(d['answer_seq'])
self._answer_len = PinnedDataManager(d['answer_seq_len'])
def _load_answer_sequence(self):
if not self._output_ans_seq:
return
data_file = self._data_root + 'data/%sanswer_std_mscoco_%s.data' % (self._version_suffix, self._subset)
d = load_hdf5(data_file)
self._answer_seq = d['ans_arr']
self._ans_seq_len = d['ans_len']
def _load_data(self):
meta_file = os.path.join(_DATA_ROOT,
'data/%svqa_std_mscoco_%s.meta' %
(self._version_suffix, self._subset))
data_file = os.path.join(_DATA_ROOT,
'data4/%svar_ivqa_%s_question_answers.data' %
(self._version_suffix, self._subset))
# load meta
d = load_json(meta_file)
images = d['images']
quest_ids = np.array(d['quest_id'])
vqa_image_ids = [find_image_id_from_fname(im_name) for im_name in images]
quest_id2image_id = {qid: im_id for (qid, im_id) in zip(quest_ids, vqa_image_ids)}
self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32)
# load QA data
d = load_hdf5(data_file)
self._quest_ids = d['ext_quest_ids'].astype(np.int32)
self._quest = d['ext_quest_arr'].astype(np.int32)
self._quest_len = d['ext_quest_len'].astype(np.int32)
self._answer = d['ext_top_answer'].astype(np.int32)
self._check_valid_answers()
# sort images
abs_quest_ids = self._quest_ids[:, 0]
self._vqa_image_ids = np.array([quest_id2image_id[_id] for _id in abs_quest_ids],
dtype=np.int32)
# self._load_caption_feature()
self._load_global_image_feature()
def _load_data(self):
meta_file = 'data/%svqa_std_mscoco_%s.meta' % (self._version_suffix,
self._subset)
data_file = 'data/%svqa_std_mscoco_%s.data' % (self._version_suffix,
self._subset)
self._images = load_json(meta_file)['images']
d = load_hdf5(data_file)
self._quest = d['quest_arr'].astype(np.int32)
self._quest_len = d['quest_len'].astype(np.int32)
self._answer = d['answer'].astype(np.int32)
self._num = self._answer.size
self._check_valid_answers()
vqa_image_ids = [
find_image_id_from_fname(im_name) for im_name in self._images
]
self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32)
# load caption data
self._load_caption_data()
# load attributes
if self._attr_type == 'res152':
self._load_global_image_feature()
else:
self._load_attributes()
# load answer sequences
self._load_answer_sequence()
def load_image_data(subset):
# load images
data_file = 'data/res152_std_mscoco_%s.data' % subset
d = load_hdf5(data_file)
image_ids = d['image_ids']
feats = d['features']
return feats, image_ids
def initialise_with_gt(self):
# load data
from util import load_hdf5
from time import time
t = time()
print('Initialising statastics with ground truth')
d = load_hdf5('data/vqa_std_mscoco_kprestval.data')
gts = parse_gt_questions(d['quest_arr'], d['quest_len'])
# update stat
self._update_samples(gts, generate_key=True, update_reference=True)
print('Initialisation finished, time %0.2fs' % (time() - t))
print('Total number of questions: %d' % self.num_total)
print('Number of unique questions: %d' % self.num_unique)
print('Average question counts: %0.2f' % self.average_count)
num_counts = self.history.values()
max_count = max(num_counts)
min_count = min(num_counts)
for t in [0.01, 0.02, 0.03, 0.05]:
r_min = np.exp(-max_count / self.average_count * t)
r_max = np.exp(-min_count / self.average_count * t)
print(
'[t=%0.4f] Max question counts: %d, ratio %0.3f, estimated reward % 0.3f'
% (t, max_count, max_count / self.average_count, r_min))
print(
'[t=%0.4f] Min question counts: %d, ratio %0.3f, estimated reward: %0.3f'
% (t, min_count, min_count / self.average_count, r_max))
self.gt_keys = {k: None for k in self.history.keys()}
def _load_data(self):
meta_file = os.path.join(
_DATA_ROOT, 'data/%svqa_std_mscoco_%s.meta' %
(self._version_suffix, self._subset))
data_file = os.path.join(
_DATA_ROOT, 'data/%svqa_std_mscoco_%s.data' %
(self._version_suffix, self._subset))
# load meta
d = load_json(meta_file)
self._images = d['images']
self._quest_ids = np.array(d['quest_id'])
vqa_image_ids = [
find_image_id_from_fname(im_name) for im_name in self._images
]
self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32)
# load QA data
d = load_hdf5(data_file)
self._quest = d['quest_arr'].astype(np.int32)
self._quest_len = d['quest_len'].astype(np.int32)
self._answer = d['answer'].astype(np.int32)
self._check_valid_answers()
# self._load_caption_feature()
self._load_global_image_feature()
def _load_data(self):
meta_file = os.path.join(
_DATA_ROOT, 'data/%svqa_std_mscoco_%s.meta' %
(self._version_suffix, self._subset))
data_file = os.path.join(
_DATA_ROOT, 'data3/%svqa_mc_w2v_coding_%s.data' %
(self._version_suffix, self._subset))
# load meta
d = load_json(meta_file)
self._images = d['images']
self._quest_ids = np.array(d['quest_id'])
vqa_image_ids = [
find_image_id_from_fname(im_name) for im_name in self._images
]
self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32)
# load Question data
d = load_hdf5(data_file)
self._quests = d['quest_w2v'].astype(np.float32)
self._num = self._quests.shape[0]
# double check question ids
assert (np.all(self._quest_ids == d['quest_ids']))
# load Answer data
self._load_answer_proposals()
self._load_global_image_feature()
def _load_subset(subset):
fname = 'mscoco_res152_%s.h5' % subset
print('Loading file %s' % fname)
d = load_hdf5(os.path.join(data_root, fname))
ids = d['image_ids'].tolist()
f = d['features']
return ids, f
def _load_data(self):
data_file = 'data/vqa_std_mscoco_multilabel_%s.data' % self._subset
d = load_hdf5(data_file)
self._image_ids = d['image_ids']
self._feat = d['features']
self._labels = d['labels']
self._num = self._labels.shape[0]
def _load_global_image_feature(self):
data_file = self._data_root + 'data/res152_std_mscoco_%s.data' % self._subset
d = load_hdf5(data_file)
image_ids = d['image_ids']
image_id2att_index = {image_id: i for i, image_id in enumerate(image_ids)}
vqa_index2att_index = [image_id2att_index[image_id] for image_id in self._vqa_image_ids]
self._vqa_index2att_index = np.array(vqa_index2att_index, dtype=np.int32)
self._attributes = d['features']
def _init_image_cache(self):
from util import load_hdf5
d = load_hdf5('data/attribute_std_mscoco_kpval.data')
# d = load_hdf5('data/res152_std_mscoco_kpval.data')
image_ids = d['image_ids']
self.im_feats = d['att_arr']
image_id2index = {image_id: idx for idx, image_id in enumerate(image_ids)}
self.image_id2index = image_id2index
def _load_attributes(self):
data_file = self._data_root + 'data/attribute_std_mscoco_%s.data' % self._subset
d = load_hdf5(data_file)
self._attributes = d['att_arr'].astype(np.float32)
image_ids = d['image_ids']
image_id2att_index = {image_id: i for i, image_id in enumerate(image_ids)}
vqa_index2att_index = [image_id2att_index[image_id] for image_id in self._vqa_image_ids]
self._vqa_index2att_index = np.array(vqa_index2att_index, dtype=np.int32)
def _load_caption_feature(self):
data_file = 'data/capt1k_std_mscoco_%s.data' % self._subset
d = load_hdf5(data_file)
image_ids = d['image_ids']
image_id2att_index = {image_id: i for i, image_id in enumerate(image_ids)}
vqa_index2att_index = [image_id2att_index[image_id] for image_id in self._vqa_image_ids]
self._vqa_index2att_index = np.array(vqa_index2att_index, dtype=np.int32)
self._attributes = d['att_arr']
def _load_irrelevant_data(self):
# load meta
d = load_json('data/QRD_irrelevant_meta.json')
self._ir_images = d['images']
# load QA data
d = load_hdf5('data/QRD_irrelevant_data.data')
self._ir_arr = d['quest'].astype(np.int32)
self._ir_arr_len = d['quest_len'].astype(np.int32)
self._num_ir = len(self._ir_images)
def __init__(self, subset):
data_file = 'data/res152_std_mscoco_%s.data' % subset
d = load_hdf5(data_file)
image_ids = d['image_ids']
self.data = d['features']
self.image_id2index = {
image_id: i
for i, image_id in enumerate(image_ids)
}
def __init__(self, top_k):
prop_file = 'data/vqa-st_vqa_score2000_kpval.hdf5'
d = load_hdf5(prop_file)
quest_ids = d['quest_ids']
self.quest_id2score_index = {int(qid): i for i, qid in enumerate(quest_ids)}
self.scores = d['confidence'][:, :2000]
self.top_k = top_k
self.encoder = SentenceEncoder('answer')
self.reverse_top_ans_dict = _load_reverse_answer_vocab()
def _load_global_image_feature(self):
data_file = 'data/res152_std_mscoco_%s.data' % self._subset
d = load_hdf5(data_file)
image_ids = d['image_ids']
self._feat = d['features']
self.image_id2att_index = {
image_id: i
for i, image_id in enumerate(image_ids)
}
def _load_question_data(self):
data_file = 'data/vqa_retrieval_mscoco_%s.data' % self._subset
d = load_hdf5(data_file)
self._quest_image_ids = d['image_ids'].astype(np.int32)
self._quest = d['quest_arr'].astype(np.int32)
self._quest_len = d['quest_len'].astype(np.int32)
self._answer = d['ans_arr'].astype(np.int32)
self._answer_len = d['ans_len'].astype(np.int32)
self._quest_ids = d['quest_ids'].astype(np.int32)
def _load_image_data(self):
feat_file = os.path.join(FEAT_ROOT, 'mscoco_res152_dev_full.h5')
d = load_hdf5(feat_file)
self._im_feats = d['features']
self._feats_image_ids = d['image_ids']
self._num = self._im_feats.shape[0]
self._image_id2image_index = {
im_id: i
for i, im_id in enumerate(self._feats_image_ids)
}
def _load_qa_data(self):
data_file = 'data/att_rerank/qa_arr.h5'
d = load_hdf5(data_file)
self._quest_arr = d['quest_arr']
self._quest_len = d['quest_len']
self._top2000_arr = d['top2000_arr']
self._top2000_len = d['top2000_len']
self._model_topk_pred = d['att_cand_arr']
self.quest_ids = d['quest_id']
self._image_ids = d['image_id']
def __init__(self, batch_size, pad_token):
self.batch_size = batch_size
from util import load_hdf5
data_file = 'data/vqa_std_mscoco_kprestval.data'
d = load_hdf5(data_file)
gts = _parse_gt_questions(d['quest_arr'], d['quest_len'])
gts = [_g + [2] for _g in gts]
self._quest, self._quest_len = put_to_array(gts, pad_token,
max_length=20)
self.num = self._quest_len.size
def _load_contrastive_answers(self):
if self._is_test:
return
meta_file = 'data/vqa_retrieval_cst_ans_mscoco_%s.meta' % self._subset
data_file = 'data/vqa_retrieval_cst_ans_mscoco_%s.data' % self._subset
d = load_json(meta_file)['quest_id2cand_index']
self.quest_id2cand_index = {int(k): v for k, v in d.iteritems()}
d = load_hdf5(data_file)
self.cst_answers = d['cand_arr'].astype(np.int32)
self.cst_ans_len = d['cand_len'].astype(np.int32)
