def main():
assert config.mode in ['train', 'trainval']
train_answers = utils.get_file('train', answer=True)
val_answers = utils.get_file('val', answer=True)
occurence = filter_answers(train_answers + val_answers, 5)
# occurence = get_top_answers(answers,3000)
ans2label = create_ans2label(occurence, 'trainval')
compute_target(train_answers, ans2label, 'train')
compute_target(val_answers, ans2label, 'val')
compute_target(train_answers + val_answers, ans2label, 'trainval')
def _load_entries(self):
"""Load entries
img_id2idx: dict {img_id -> idx} val can be used to retrieve image or features
dataroot: root path of dataset
split: 'train', 'val', 'trainval', 'test'
"""
def _create_entry(img_id, question, answer):
entry = {
'question_id': question['question_id'],
'image_id': img_id,
'img_idx': self.img_id2idx[img_id],
'question': self.encode_question(question['question']),
'answer': self.encode_answer(answer)
}
return entry
questions = utils.get_file(self.split, question=True)
questions = sorted(questions, key=lambda x: x['question_id'])
if self.split != 'test':
answer_path = os.path.join(self.dataroot, '%s_target.json' % self.split)
with open(answer_path, 'r') as fd:
answers = json.load(fd)
utils.assert_eq(len(questions), len(answers))
answers = sorted(answers, key=lambda x: x['question_id'])
entries = []
for question, answer in zip(questions, answers):
img_id = answer.pop('image_id')
ques_id = answer.pop('question_id')
utils.assert_eq(question['question_id'], ques_id)
utils.assert_eq(question['image_id'], img_id)
entries.append(_create_entry(img_id, question, answer))
else:
entries = [_create_entry(question['image_id'], question, 0) for question in questions]
print(len(entries))
return entries
def create_vqa_dictionary():
dictionary = Dictionary()
if config.type == 'cp':
ques_file_types = ['train', 'test']
else:
ques_file_types = ['train', 'val', 'test']
for type in ques_file_types:
questions = utils.get_file(type, question=True)
for q in questions:
dictionary.tokenize(q['question'], True)
return dictionary
def create_vqa_exp_dictionary():
dictionary = Dictionary()
qid2exp = {}
if config.type == 'cp':
ques_file_types = ['train', 'test']
else:
ques_file_types = ['train', 'val']
for type in ques_file_types:
questions = utils.get_file(type, question=True)
questions = sorted(questions, key=lambda x: x['question_id'])
answers = utils.get_file(type, answer=True)
answers = sorted(answers, key=lambda x: x['question_id'])
for q, a in zip(questions, answers):
ques_id = a.pop('question_id')
utils.assert_eq(q['question_id'], ques_id)
dictionary.tokenize(q['question'], True)
dictionary.tokenize(a['multiple_choice_answer'], True)
qid2exp[int(ques_id)] = {
'question': q['question'],
'answer': a['multiple_choice_answer']
}
print(ques_id, a['image_id'])
return dictionary, qid2exp
def __init__(self,
results_path,
finetune_results_path,
img_path,
dest_dir='visuals/%s' % (config.version),
show_words=True,
skip_results=False,
preprocess_results=True):
if skip_results:
answers_json = utils.get_file(split="val", answer=True)
self.results = {
idx: [item['question_id'], item['multiple_choice_answer']]
for idx, item in enumerate(answers_json)
}
else:
self.results = []
if not preprocess_results:
with open(os.path.join(results_path, 'cross_results.json'),
'r') as f:
self.results = json.load(f)
with open(os.path.join(results_path, 'qid_to_iq.json'),
'r') as f:
self.q_iq = json.load(f)
else:
questions_json = utils.get_file(split="val", question=True)
answers_json = utils.get_file(split="val", answer=True)
self.q_iq = {
ques['question_id']: [
ques['image_id'], ques['question'],
ans['multiple_choice_answer'], ans['question_type'],
ans['answer_type']
]
for ques, ans in zip(questions_json, answers_json)
}
with open(os.path.join(results_path, 'qid_to_iq.json'),
'w') as f:
json.dump(self.q_iq, f)
with open(os.path.join(results_path, 'eval_results.json'),
'r') as f:
false_results = json.load(f)
with open(
os.path.join(finetune_results_path,
'eval_results.json'), 'r') as f:
true_results = json.load(f)
for f_idx, f_ques in enumerate(false_results):
for t_idx, t_ques in enumerate(true_results):
if f_ques['question_id'] == t_ques['question_id']:
self.results.append({
'question_id':
t_ques['question_id'],
'true_answer':
t_ques['answer'],
'false_answer':
f_ques['answer'],
'true_idx':
t_idx,
'false_idx':
f_idx
})
print(len(self.results))
with open(os.path.join(results_path, 'cross_results.json'),
'w') as f:
json.dump(self.results, f)
self.train_img_ids = utils.load_imageid(img_path + "train2014")
self.val_img_ids = utils.load_imageid(img_path + "val2014")
with h5py.File(
os.path.join(finetune_results_path, 'att_weights.h5'),
'r') as f:
self.true_weight = f['weights'][:]
self.true_spatials = f['spatials'][:]
self.hints = f['hints'][:]
with h5py.File(os.path.join(results_path, 'att_weights.h5'),
'r') as f:
self.false_weight = f['weights'][:]
self.false_spatials = f['spatials'][:]
self.train_image_fmt = img_path + "train2014/COCO_train2014_%s.jpg"
self.val_image_fmt = img_path + "val2014/COCO_val2014_%s.jpg"
self.skip_results = skip_results
self.dest_dir = dest_dir
if not os.path.isdir(dest_dir + ''):
os.system('mkdir -p ' + dest_dir)
self.show_words = show_words
self.font = cv2.FONT_HERSHEY_SIMPLEX
self.length = self.__len__()
self.color = {
"blue": (255, 0, 0),
"yellow": (0, 255, 255),
"green": (0, 255, 0),
"red": (0, 0, 255)
}
def _load_entries(self):
"""Load entries
img_id2idx: dict {img_id -> idx} val can be used to retrieve image or features
dataroot: root path of dataset
split: 'train', 'val', 'trainval', 'test'
"""
def _create_entry(img_id, question, answer):
entry = {
'question_id':
question['question_id'],
'image_id':
img_id,
'img_idx':
self.img_id2idx[img_id],
'question':
self.encode_question(question['question']),
'question_type':
answer['question_type'],
'answer':
self.encode_answer(answer),
'bias':
self.question_type_to_prob_array[answer['question_type']]
if self.split not in ['val', 'test'] else 0,
'has_hint':
torch.tensor(False),
'hint':
torch.zeros(36).bool(),
# 'hint': torch.zeros(36),
}
return entry
questions = utils.get_file(self.split, question=True)
questions = sorted(questions, key=lambda x: x['question_id'])
if self.split != 'test':
answer_path = os.path.join(self.dataroot,
'%s_target.json' % self.split)
with open(answer_path, 'r') as fd:
answers = json.load(fd)
utils.assert_eq(len(questions), len(answers))
answers = sorted(answers, key=lambda x: x['question_id'])
# compute the bias for train dataset
if self.split not in ['val', 'test']:
self.compute_bias_with_qty(answers)
entries = []
for question, answer in zip(questions, answers):
img_id = answer.pop('image_id')
ques_id = answer.pop('question_id')
utils.assert_eq(question['question_id'], ques_id)
utils.assert_eq(question['image_id'], img_id)
entries.append(_create_entry(img_id, question, answer))
else:
answer = {'question_type': 'Null'}
entries = [
_create_entry(question['image_id'], question, answer)
for question in questions
]
print(len(entries))
return entries