def _load_data(self):
meta_file = 'data/%svqa_std_mscoco_%s.meta' % (self._version_suffix,
self._subset)
data_file = 'data/%svqa_std_mscoco_%s.data' % (self._version_suffix,
self._subset)
# load meta file
d = load_json(meta_file)
self._images = d['images']
self._load_answer_type(d['quest_id'])
self._quest_ids = np.array(d['quest_id'])
vqa_image_ids = [
find_image_id_from_fname(im_name) for im_name in self._images
]
self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32)
# load data file
d = load_hdf5(data_file)
self._quest = d['quest_arr'].astype(np.int32)
self._quest_len = d['quest_len'].astype(np.int32)
self._answer = d['answer'].astype(np.int32)
self._num = self._answer.size
self._check_valid_answers()
vqa_image_ids = [
find_image_id_from_fname(im_name) for im_name in self._images
]
self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32)
# load caption data
self._load_caption_data()
# load attributes
self._load_attributes()
# load answer sequences
self._load_answer_sequence()
def _load_data(self):
meta_file = os.path.join(
_DATA_ROOT, 'data/%svqa_std_mscoco_%s.meta' %
(self._version_suffix, self._subset))
data_file = os.path.join(
_DATA_ROOT, 'data/%svqa_std_mscoco_%s.data' %
(self._version_suffix, self._subset))
# load meta
d = load_json(meta_file)
self._images = d['images']
self._quest_ids = np.array(d['quest_id'])
vqa_image_ids = [
find_image_id_from_fname(im_name) for im_name in self._images
]
self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32)
# load QA data
d = load_hdf5(data_file)
self._quest = d['quest_arr'].astype(np.int32)
self._quest_len = d['quest_len'].astype(np.int32)
self._answer = d['answer'].astype(np.int32)
self._check_valid_answers()
# self._load_caption_feature()
self._load_global_image_feature()
def _load_data(self):
meta_file = os.path.join(
_DATA_ROOT, 'data/%svqa_std_mscoco_%s.meta' %
(self._version_suffix, self._subset))
data_file = os.path.join(
_DATA_ROOT, 'data/%svqa_std_mscoco_%s.data' %
(self._version_suffix, self._subset))
# load meta
d = load_json(meta_file)
self._images = d['images']
self._quest_ids = np.array(d['quest_id'])
vqa_image_ids = [
find_image_id_from_fname(im_name) for im_name in self._images
]
self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32)
# load Question data
d = load_hdf5(data_file)
self._quest = d['quest_arr'].astype(np.int32)
self._quest_len = d['quest_len'].astype(np.int32)
# self._quests = d['quest_w2v'].astype(np.float32)
self._num = self._quest.shape[0]
# load Answer data
self._load_answer_proposals()
self._load_global_image_feature()
def _load_data(self):
meta_file = os.path.join(_DATA_ROOT,
'data/%svqa_std_mscoco_%s.meta' %
(self._version_suffix, self._subset))
data_file = os.path.join(_DATA_ROOT,
'data/%svqa_std_mscoco_%s.data' %
(self._version_suffix, self._subset))
# load meta
d = load_json(meta_file)
self._images = d['images']
self._quest_ids = np.array(d['quest_id'])
self.quest_id2index = {qid: i for i, qid in enumerate(self._quest_ids)}
vqa_image_ids = [find_image_id_from_fname(im_name) for im_name in self._images]
self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32)
image_id2quest_ids = {}
for qid, im_id in zip(self._quest_ids, self._vqa_image_ids):
if im_id in image_id2quest_ids: # already create key
image_id2quest_ids[im_id].append(qid)
else:
image_id2quest_ids[im_id] = [qid]
self.image_id2quest_ids = image_id2quest_ids
# load QA data
d = load_hdf5(data_file)
self._quest = d['quest_arr'].astype(np.int32)
self._quest_len = d['quest_len'].astype(np.int32)
self._num = self._quest_len.size
self._answer = d['answer'].astype(np.int32)
self._check_valid_answers()
# self._load_caption_feature()
self._load_global_image_feature()
def _load_data(self):
meta_file = os.path.join(_DATA_ROOT,
'data/%svqa_std_mscoco_%s.meta' %
(self._version_suffix, self._subset))
data_file = os.path.join(_DATA_ROOT,
'data4/%svar_ivqa_%s_question_answers.data' %
(self._version_suffix, self._subset))
# load meta
d = load_json(meta_file)
images = d['images']
quest_ids = np.array(d['quest_id'])
vqa_image_ids = [find_image_id_from_fname(im_name) for im_name in images]
quest_id2image_id = {qid: im_id for (qid, im_id) in zip(quest_ids, vqa_image_ids)}
self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32)
# load QA data
d = load_hdf5(data_file)
self._quest_ids = d['ext_quest_ids'].astype(np.int32)
self._quest = d['ext_quest_arr'].astype(np.int32)
self._quest_len = d['ext_quest_len'].astype(np.int32)
self._answer = d['ext_top_answer'].astype(np.int32)
self._check_valid_answers()
# sort images
abs_quest_ids = self._quest_ids[:, 0]
self._vqa_image_ids = np.array([quest_id2image_id[_id] for _id in abs_quest_ids],
dtype=np.int32)
# self._load_caption_feature()
self._load_global_image_feature()
def _load_data(self):
meta_file = os.path.join(_DATA_ROOT,
'data/%svqa_std_mscoco_%s.meta' %
(self._version_suffix, self._subset))
data_file = os.path.join(_DATA_ROOT,
'data3/%svqa_mc_w2v_coding_%s.data' %
(self._version_suffix, self._subset))
# load meta
d = load_json(meta_file)
self._images = d['images']
self._quest_ids = np.array(d['quest_id'])
vqa_image_ids = [find_image_id_from_fname(im_name) for im_name in self._images]
self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32)
# load QA data
d = load_hdf5(data_file)
self._quests = d['quest_w2v'].astype(np.float32)
self._answer = d['cands_w2v'].astype(np.float32)
self._labels = d['labels']
self._num = self._labels.size
# double check question ids
assert (np.all(self._quest_ids == d['quest_ids']))
self._post_proc_answers()
self._load_global_image_feature()
def get_subset_image_ids(subset):
from util import find_image_id_from_fname, load_json
print('Parsing image index of set %s...' % subset)
meta_file = '../VQA-tensorflow/data/vqa_std_mscoco_%s.meta' % subset
images = load_json(meta_file)['images']
# get image ids
image_ids = [find_image_id_from_fname(fname) for fname in images]
unique_ids = np.unique(np.array(image_ids, dtype=np.int32))
# print('Finshed, find %d unique images.\n' % unique_ids.size)
return unique_ids.tolist()
def _load_subset():
fpath = os.path.join(fdir, 'unique_vtfq.json')
d = load_json(fpath)
dataset = []
for item in d.values():
i_label = int(item['label'])
if i_label == 2: # irrelevant
image = os.path.basename(item['image'])
image_id = find_image_id_from_fname(image)
dataset.append({
'image': 'val2014/' + image,
'image_id': image_id,
'question': item['question'] + '?'
})
return dataset
def _load_subset(subset):
fpath = os.path.join(fdir, 'input/qrpe_%s.json' % subset)
d = load_json(fpath)
dataset = []
db_type = 'train' if subset == 'train' else 'val'
for item in d.values():
i_from = item['imtype']
i_label = int(item['label'])
if i_label == 2 and 'coco' in i_from:
image_id = find_image_id_from_fname(item['image'])
dataset.append({
'image': '%s2014/%s' % (db_type, item['image']),
'image_id': image_id,
'question': item['question']
})
return dataset
def _load_data(self):
meta_file = os.path.join('data/QRD_irrelevant_meta_test.json')
data_file = os.path.join('data/QRD_irrelevant_data_test.data')
# load meta
d = load_json(meta_file)
self._images = d['images']
vqa_image_ids = [
find_image_id_from_fname(im_name) for im_name in self._images
]
self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32)
# load QA data
d = load_hdf5(data_file)
self._quest = d['quest'].astype(np.int32)
self._quest_len = d['quest_len'].astype(np.int32)
self._labels = d['labels'].astype(np.float32)
# self._check_valid_answers()
self._num = self._quest_len.size
def load_qa_data(subset):
# load meta file
meta_file = os.path.join(_DATA_ROOT,
'data/vqa_std_mscoco_%s.meta' % subset)
d = load_json(meta_file)
image_ids = [find_image_id_from_fname(im_name) for im_name in d['images']]
image_id2qa_index = {}
for i, image_id in enumerate(image_ids):
if image_id in image_id2qa_index:
image_id2qa_index[image_id].append(i)
else:
image_id2qa_index[image_id] = [i]
data_file = os.path.join(_DATA_ROOT,
'data/vqa_std_mscoco_%s.data' % subset)
d = load_hdf5(data_file)
labels = d['answer']
return image_id2qa_index, labels
def _load_data(self):
meta_file = 'data/vqa_std_mscoco_%s.meta' % self._subset
data_file = 'data/vqa_std_mscoco_%s.data' % self._subset
# load meta
d = load_json(meta_file)
self._images = d['images']
self._quest_ids = np.array(d['quest_id'])
self.quest_id2index = {qid: i for i, qid in enumerate(self._quest_ids)}
vqa_image_ids = [
find_image_id_from_fname(im_name) for im_name in self._images
]
self._vqa_image_ids = np.array(vqa_image_ids, dtype=np.int32)
# load QA data
d = load_hdf5(data_file)
self._quest = d['quest_arr'].astype(np.int32)
self._quest_len = d['quest_len'].astype(np.int32)
self._answer = d['answer'].astype(np.int32)
def process(model_type='MLB2-att'):
result_format = 'result/vqa_score_on_mbs_%s_v2.json'
questions = load_task_data()
model = load_model(model_type)
results = []
num = len(questions)
t = time()
for i, item in enumerate(questions):
if i % 100 == 0:
print('Testing [%s]: %d/%d (speed: %0.2f sec/100 samples)' %
(model_type, i, num, time() - t))
t = time()
image_id = find_image_id_from_fname(item['image'])
question = item['target']
answer = item['answer']
score = model.query_score(image_id, question, answer)
results.append(score)
res_file = result_format % model_type
save_json(res_file, results)
def test_irrelevant_management():
from util import load_json, find_image_id_from_fname
d = load_json('data/vqa_std_mscoco_trainval.meta')
image_ids = [find_image_id_from_fname(fname) for fname in d['images']]
irrel_man = IrrelevantManager(image_ids)
num = len(image_ids)
batch_size = 4
num_batches = 4
from time import time
t = time()
for i in range(num_batches):
print('\nBatch %d: ' % i)
index = np.random.choice(num, size=(batch_size,), replace=False)
neg_index = irrel_man.query_irrelevant(index)
print('pos:')
print(index)
print('neg:')
print(neg_index)
tot = time() - t
print('Time: %0.2f ms/batch' % (tot * 1000. / float(num_batches)))
def split_subset(subset, inputs):
print('Processing split %s' % subset)
images, quest_id, quest_arr, quest_len, answer, attr_image_ids, attr_arr = inputs
vqa_image_ids = [find_image_id_from_fname(fname) for fname in images]
# get coco ids
coco_ids = get_image_id(subset)
# build coco id hashing table
coco_ids = {image_id: i for i, image_id in enumerate(coco_ids)}
# split vqa data
keep_tab = np.array([im_id in coco_ids for im_id in vqa_image_ids])
images = [im for im, keep in zip(images, keep_tab) if keep]
quest_id = [q_id for q_id, keep in zip(quest_id, keep_tab) if keep]
quest_arr = quest_arr[keep_tab]
quest_len = quest_len[keep_tab]
answer = answer[keep_tab]
# split attribute data
keep_tab = np.array([im_id in coco_ids for im_id in attr_image_ids])
attr_image_ids = attr_image_ids[keep_tab]
attr_arr = attr_arr[keep_tab]
# process answers
encode_answers(quest_id, subset)
# save to files
vqa_meta_file = 'data/vqa_std_mscoco_kp%s.meta' % subset
save_json(vqa_meta_file, {'images': images, 'quest_id': quest_id})
vqa_data_file = 'data/vqa_std_mscoco_kp%s.data' % subset
save_hdf5(vqa_data_file, {
'quest_arr': quest_arr,
'quest_len': quest_len,
'answer': answer
})
attr_data_file = 'data/attribute_std_mscoco_kp%s.data' % subset
save_hdf5(attr_data_file, {
'image_ids': attr_image_ids,
'att_arr': attr_arr
})
def _load_relevant_data(self):
meta_file = os.path.join(_DATA_ROOT,
'data/vqa_std_mscoco_%s.meta' % self._subset)
data_file = os.path.join(_DATA_ROOT,
'data/vqa_std_mscoco_%s.data' % self._subset)
# load meta
d = load_json(meta_file)
self._r_images = d['images']
image_ids = [
find_image_id_from_fname(fname) for fname in self._r_images
]
self.ezir_ctx = IrrelevantManager(image_ids)
# load QA data
d = load_hdf5(data_file)
self._r_arr = d['quest_arr'].astype(np.int32)
self._r_arr_len = d['quest_len'].astype(np.int32)
self._num_rel = len(self._r_images)
# load blacklist
blist = load_json('data/kptest_blacklist.json')
is_valid = np.array([image_id not in blist for image_id in image_ids])
print('%d/%d valid' % (is_valid.sum(), is_valid.size))
self.valid_ids = np.where(is_valid)[0]
def process(model_type='MLB2-att'):
# result_format = 'result/vqa_pred_bsadv_%s_v2.json'
result_format = 'result/vqa_pred_bsadv_%s.json'
questions = load_task_data()
model = load_model(model_type)
results = []
num = len(questions)
t = time()
acc = []
for i, item in enumerate(questions):
if i % 100 == 0:
print('Testing [%s]: %d/%d (speed: %0.2f sec/100 samples)' %
(model_type, i, num, time() - t))
t = time()
image_id = find_image_id_from_fname(item['image'])
question = item['target']
answer = item['answer']
ans, _ = model.get_score(image_id, question)
acc.append(ans == answer)
# score = model.query_score(image_id, question, answer)
results.append(ans)
res_file = result_format % model_type
save_json(res_file, results)
print('%s: Acc: %0.2f' % (model_type, 100. * sum(acc) / float(len(acc))))
def split_subset(subset, inputs):
print('Processing split %s' % subset)
images, quest_id, quest_arr, quest_len, answer, attr_image_ids, attr_arr = inputs
vqa_image_ids = [find_image_id_from_fname(fname) for fname in images]
# get coco ids
coco_ids = get_image_id(subset)
# build coco id hashing table
coco_ids = {image_id: i for i, image_id in enumerate(coco_ids)}
# split vqa data
keep_tab = np.array([im_id in coco_ids for im_id in vqa_image_ids])
images = [im for im, keep in zip(images, keep_tab) if keep]
quest_id = [q_id for q_id, keep in zip(quest_id, keep_tab) if keep]
quest_arr = quest_arr[keep_tab]
quest_len = quest_len[keep_tab]
answer = answer[keep_tab]
# split attribute data
keep_tab = np.array([im_id in coco_ids for im_id in attr_image_ids])
attr_image_ids = attr_image_ids[keep_tab]
attr_arr = attr_arr[keep_tab]
# process answers
encode_answers(quest_id, subset)
def make_blacklist():
d = load_json('data/vqa_std_mscoco_kptest.meta')
# question_ids = d['quest_id']
images = d['images']
image_ids = [find_image_id_from_fname(im_name) for im_name in images]
return {image_id: None for image_id in image_ids}
