def load_combined_vocab(cls, data_dir='data/CLEVR_v1.0'):
data_dir = to_path(data_dir)
q_vocab_file = data_dir.joinpath(f'{cls.name}/q_vocab.json')
a_vocab_file = data_dir.joinpath(f'{cls.name}/a_vocab.json')
if q_vocab_file.exists() and a_vocab_file.exists():
q_vocab = Vocab.build_from_counter_file('q_vocab', data_dir.joinpath(cls.name), q_vocab_file,
special_symbols='<padding>')
a_vocab = Vocab.build_from_counter_file('a_vocab', data_dir.joinpath(cls.name), a_vocab_file)
return q_vocab, a_vocab
ann_files = [data_dir.joinpath(f'questions/CLEVR_{split}_questions.json') for split in ('train', 'val')]
q_anns = [json.load(a_file.open())['questions'] for a_file in ann_files]
if not q_vocab_file.exists():
questions = [ann_item['question'] for q_ann in q_anns for ann_item in q_ann]
q_tokens = []
for question in tqdm(questions, desc='Tokenizing questions'):
q_tokens.append(cls.q_tokenize(question))
q_vocab = Vocab.build_from_sequences('q_vocab', data_dir.joinpath(f'{cls.name}'), q_tokens,
special_symbols='<padding>')
q_vocab.dump_to_file(q_vocab_file)
else:
q_vocab = Vocab.build_from_counter_file('q_vocab', data_dir.joinpath(cls.name), q_vocab_file,
special_symbols='<padding>')
if not a_vocab_file.exists():
answers = [ann_item['answer'] for ann in q_anns for ann_item in ann]
a_tokens = []
for answer in tqdm(answers, desc='Tokenizing answers'):
a_tokens.append(cls.a_tokenize(answer))
a_vocab = Vocab.build_from_sequences('a_vocab', data_dir.joinpath(f'{cls.name}'), a_tokens)
a_vocab.dump_to_file(a_vocab_file)
else:
a_vocab = Vocab.build_from_counter_file('a_vocab', data_dir.joinpath(cls.name), a_vocab_file)
q_vocab.strict = True
a_vocab.strict = True
return q_vocab, a_vocab
def __init__(self, work_dir: str, step2log: int = 1, writer_kwargs=None):
super().__init__()
self.work_dir = to_path(work_dir)
self.step2log = step2log
if not self.work_dir.exists():
self.work_dir.mkdir(parents=True)
self.writer = SummaryWriter(**writer_kwargs)
def auto_config_file(self, params):
config_dir = to_path(
f'./{params.work_dir}/{params.proj_name}/{params.exp_name}/{params.exp_version}/experiment'
)
if not config_dir.exists():
config_dir.mkdir(parents=True)
return config_dir.joinpath('params.json')
def load_combined_vocab(cls, data_dir='work_dir/data/vqa2'):
data_dir = utils.to_path(data_dir)
q_vocab_file = data_dir.joinpath(f'{cls.name}/q_vocab.json')
a_vocab_file = data_dir.joinpath(f'{cls.name}/a_vocab.json')
if not q_vocab_file.exists():
logger.info(f'Creating question vocab {q_vocab_file.name}')
q_dict = json.load(
data_dir.joinpath(cls.name, 'train_q_dict.json').open())
id2words = ['<padding>'] + list(q_dict['itow'].values())
word2ids = {word: idx for idx, word in enumerate(id2words)}
q_vocab = Vocab.build_from_statistic(
id2words, word2ids, 'q_vocab',
data_dir.joinpath(f'{cls.name}'))
q_vocab.save(q_vocab_file)
if not a_vocab_file.exists():
logger.info(f'Creating answer vocab {a_vocab_file.name}')
a_dict = json.load(
data_dir.joinpath(cls.name, 'train_a_dict.json').open())
id2words = list(a_dict['itow'].values()) + ['<padding>']
word2ids = {word: idx for idx, word in enumerate(id2words)}
a_vocab = Vocab.build_from_statistic(
id2words, word2ids, 'a_vocab',
data_dir.joinpath(f'{cls.name}'))
a_vocab.save(a_vocab_file)
q_vocab = Vocab.from_file('q_vocab', data_dir.joinpath(cls.name),
q_vocab_file)
a_vocab = Vocab.from_file('a_vocab', data_dir.joinpath(cls.name),
a_vocab_file)
return q_vocab, a_vocab
def load_combined_anns(cls, data_dir, data_split):
data_dir = to_path(data_dir)
ann_file = data_dir.joinpath(f'{cls.name}/{data_split}_ann.pt')
if ann_file.exists():
anns_t = torch.load(ann_file)
else:
sys_logger.info(f'Creating combined annotation files {ann_file.name}')
q_vocab, a_vocab = cls.load_combined_vocab(data_dir)
if data_split in ('valB_30k', 'valB_120k'):
q_anns = json.load(data_dir.joinpath('questions', f'CLEVR_valB_questions.json').open())['questions']
if data_split == 'valB_30k':
q_anns = q_anns[:30000]
else:
q_anns = q_anns[30000:]
else:
q_anns = json.load(data_dir.joinpath('questions', f'CLEVR_{data_split}_questions.json').open())['questions']
anns = defaultdict(list)
for q_item in tqdm(q_anns, desc=f'Creating {ann_file.name}'):
anns['q_ids'].append(q_item['question_index'])
anns['img_ids'].append(q_item['image_index'])
anns['img_names'].append(q_item['image_filename'])
anns['q_tokens'].append(cls.q_tokenize(q_item['question']))
anns['q_types'].append(q_item['question_family_index'])
anns['a_tokens'].append(q_item['answer'])
anns['q_lens'].append(len(anns['q_tokens'][-1]))
q_label = [q_vocab[token] for token in anns['q_tokens'][-1]]
q_label_t = torch.empty(47).fill_(q_vocab.padding_idx).long()
q_label_t[:len(q_label)] = torch.tensor(q_label).long()
anns['q_labels'].append(q_label_t)
anns['a_labels'].append(a_vocab[q_item['answer']])
anns_t = {name: to_tensor(value) for name, value in anns.items()}
torch.save(anns_t, ann_file)
return anns_t
def __init__(
self,
save_dir: str,
restore_file: str = None,
epoch2save: int = 1,
only_best: bool = True,
strict_mode: str = False,
monitor: str = 'val_loss',
):
super().__init__()
self.save_dir = to_path(save_dir)
if not self.save_dir.exists():
self.save_dir.mkdir(parents=True)
self.restore_file = to_path(restore_file)
self.epoch2save = epoch2save
self.only_best = only_best
self.strict_mode = strict_mode
self.monitor = monitor
self.best_val_acc = 0
def __init__(self,
img_dir,
req_field_names=None,
file2idx_fn=None,
img_size=None,
transforms=None):
self.img_dir = to_path(img_dir)
assert self.img_dir.exists()
self.file2idx_fn = file2idx_fn or self._file2idx_fn
self.transforms = transforms or self.load_transforms(img_size)
super().__init__(req_field_names)
def __init__(self, name, logger_dir):
self.name = name
self.logger_dir = to_path(logger_dir)
self.json_file = self.logger_dir.joinpath(f'{name}.json')
self.histories = dict() if not self.json_file.exists() else json.load(
self.json_file.open())
self.histories = {
int(key): value
for key, value in self.histories.items()
}
self.is_first = True
def __init__(self,
data_dir: str,
split: str,
req_field_names: List[str],
is_lazy: bool = False):
self.data_dir = to_path(data_dir)
self.split = split
self.loaders = None
self._length = None
py_data.Dataset.__init__(self)
Reader.__init__(self, req_field_names, is_lazy)
PtBase.__init__(self)
def __init__(
self,
data_dir,
split,
req_field_names=None,
is_load=False,
):
self.data_dir = to_path(data_dir)
h5_file = self.data_dir.joinpath(f'{split}_images.h5')
if not h5_file.exists():
self.create_h5_file(self.data_dir, split)
super().__init__(h5_file, req_field_names, is_load=is_load)
def __init__(
self,
data_dir: str,
split: str,
req_field_names: List[str],
):
self.data_dir = to_path(data_dir)
self.split = split
self.loaders = None
py_data.Dataset.__init__(self)
Reader.__init__(self, req_field_names)
PtBase.__init__(self)
def load_combined_vocab(cls, data_dir='work_dir/data/vqa2'):
data_dir = to_path(data_dir)
q_vocab_file = data_dir.joinpath(f'{cls.name}/q_vocab.json')
a_vocab_file = data_dir.joinpath(f'{cls.name}/a_vocab.json')
if not q_vocab_file.exists():
logger.info(f'Creating question vocab {q_vocab_file.name}')
q_files = (f'vqacp_v2_{key}_questions.json'
for key in ('train', 'test'))
q_files = (data_dir.joinpath(f'annotations/{q_file}')
for q_file in q_files)
questions = functools.reduce(lambda x, y: x + y,
(json.load(q_file.open())
for q_file in q_files))
q_tokens = []
for q in tqdm.tqdm(questions):
tokens = cls.q_tokenize(q['question'])
q_tokens.append(tokens)
q_vocab = Vocab.build_from_sequences(
'q_vocab',
data_dir.joinpath(f'{cls.name}'),
q_tokens,
special_symbols='<padding>')
q_vocab.dump_to_file(q_vocab_file)
if not a_vocab_file.exists():
logger.info(f'Creating answer vocab {a_vocab_file.name}')
a_files = (f'vqacp_v2_{split}_annotations.json'
for split in ('train', 'test'))
a_files = (data_dir.joinpath(f'annotations/{a_file}')
for a_file in a_files)
answers = functools.reduce(lambda x, y: x + y,
(json.load(a_file.open())
for a_file in a_files))
a_tokens = []
for answer in tqdm.tqdm(answers):
gtruth = cls.a_tokenize(answer['multiple_choice_answer'])
a_tokens.append(gtruth)
a_vocab = Vocab.build_from_sequences(
'a_vocab',
data_dir.joinpath(f'{cls.name}'),
a_tokens,
max_word_num=3000,
special_symbols='<padding>')
a_vocab.dump_to_file(a_vocab_file)
q_vocab = Vocab.build_from_counter_file('q_vocab',
data_dir.joinpath(cls.name),
q_vocab_file,
special_symbols='<padding>')
a_vocab = Vocab.build_from_counter_file('a_vocab',
data_dir.joinpath(cls.name),
a_vocab_file,
special_symbols='<padding>')
return q_vocab, a_vocab
def __init__(
self,
data_dir,
split,
req_field_names=None,
transforms=None,
is_load=False,
):
self.data_dir = to_path(data_dir)
if split in ('valB_30k', 'valB_120k'):
split = 'valB'
super().__init__(self.data_dir.parent.joinpath('images', f'{split}'),
req_field_names,
transforms=transforms)
def load_combined_anns(cls,
data_dir: str = 'work_dir/data/gqa',
data_split: str = 'train',
q_max=30):
data_dir = to_path(data_dir).joinpath(cls.name)
ann_file = data_dir.joinpath(f'{data_split}_ann.pt')
if ann_file.exists():
return torch.load(ann_file)
logger.info(f'Creating combined annotation files {ann_file.name}')
q_vocab, a_vocab = cls.load_combined_vocab(data_dir.parent)
if data_split in ('train', 'val'):
q_file = data_dir.parent.joinpath(
f'questions/{data_split}_balanced_questions.json')
obj_infos = json.load(
data_dir.parent.joinpath(
'objects/gqa_objects_info.json').open())
# scene_file = data_dir.joinpath(f'sceneGraphs/{data_split}_sceneGraphs.json')
q_anns = json.load(q_file.open())
else:
raise NotImplementedError()
anns = collections.defaultdict(list)
pbar = tqdm.tqdm(
range(len(q_anns)),
desc=f'Creating combined annotation files {ann_file.name}')
for q_id, q_ann in q_anns.items():
anns['splits'].append(data_split)
anns['q_ids'].append(q_id)
anns['img_ids'].append(q_ann['imageId'])
anns['q_tokens'].append(cls.q_tokenize(q_ann['question']))
q_label = [
q_vocab[token] for token in anns['q_tokens'][-1][:q_max]
]
anns['q_lens'].append(len(q_label))
q_label_t = torch.empty(q_max).fill_(q_vocab.padding_idx).long()
q_label_t[:len(q_label)] = torch.tensor(q_label).long()
anns['q_labels'].append(q_label_t)
obj_info = obj_infos[q_ann['imageId']]
anns['img_shapes'].append(
torch.tensor((obj_info['width'], obj_info['height'])).float())
anns['img_obj_nums'].append(torch.tensor(obj_info['objectsNum']))
if 'answer' in q_ann:
anns['a_tokens'].append(q_ann['answer'])
anns['a_labels'].append(torch.tensor(a_vocab[q_ann['answer']]))
pbar.update(1)
anns_t = {key: to_tensor(value) for key, value in anns.items()}
torch.save(anns_t, ann_file, pickle_module=dill)
return anns_t
def __init__(
self,
name: str,
work_dir: str,
step2log: int,
splits: List[str],
):
self.name = name
self.work_dir = to_path(work_dir)
if not self.work_dir.exists():
self.work_dir.mkdir(parents=True)
self.step2log = step2log
self.metric = Metric(name, self.work_dir)
self.splits = splits
super().__init__()
def init_args(cls, params, sub_cls=None):
cls.default_init_args(params)
try_set_attr(params, f'{cls.prefix_name()}_name', 'logger_group')
try_set_attr(
params, f'{cls.prefix_name()}_logger_dir',
to_path(params.root_dir).joinpath(f'loggers/{params.proj_name}'))
logger_cls = cls.load_cls(
try_get_attr(params,
f'{cls.prefix_name()}_logger_cls',
check=False))
if logger_cls is not None:
logger_cls.init_args(params)
setattr(
params, f'{cls.prefix_name()}_logger_kwargs',
load_func_kwargs(params, logger_cls.__init__,
cls.prefix_name()))
def load_combined_vocab(cls, data_dir='work_dir/data/gqa'):
data_dir = to_path(data_dir).joinpath(cls.name)
q_vocab_file = data_dir.joinpath('q_vocab.json')
a_vocab_file = data_dir.joinpath('a_vocab.json')
if not q_vocab_file.exists():
logger.info(f'Creating question vocab {q_vocab_file.name}')
q_vocab = Vocab.build_from_txt(
'q_vocab', data_dir.joinpath('vocabulary_gqa.txt'))
q_vocab.save(q_vocab_file)
else:
q_vocab = Vocab.from_file('q_vocab', q_vocab_file)
if not a_vocab_file.exists():
logger.info(f'Creating answer vocab {a_vocab_file.name}')
a_vocab = Vocab.build_from_txt(
'a_vocab', data_dir.joinpath('answers_gqa.txt'))
a_vocab.save(a_vocab_file)
else:
a_vocab = Vocab.from_file('a_vocab', a_vocab_file)
q_vocab.embed_init = np.load(
data_dir.joinpath('gloves_gqa_no_pad.npy'))
return q_vocab, a_vocab
def load_combined_anns(
cls,
data_dir: str = 'data/vqa2',
split: str = 'train',
):
data_dir = utils.to_path(data_dir)
ann_file = data_dir.joinpath(f'{cls.name}/{split}_ann.pt')
if ann_file.exists():
anns_t = torch.load(ann_file)
else:
logger.info(f'Creating combined annotation files {ann_file.name}')
q_vocab, a_vocab = cls.load_combined_vocab(data_dir)
if split in ('train', 'val'):
json_anns = json.load(
data_dir.joinpath(cls.name,
f'vqa_{split}_final_3000.json').open())
ins_anns = json.load(
data_dir.joinpath(
f'annotations/instances_{split}2014.json').open())
img_anns = {ann['id']: ann for ann in ins_anns['images']}
elif split == 'test':
json_anns = json.load(
data_dir.joinpath(cls.name, f'vqa_test_toked.json').open())
q_anns = json.load(
data_dir.joinpath(
f'annotations/v2_OpenEnded_mscoco_{split}2015_questions.json'
).open())['questions']
ins_anns = json.load(
data_dir.joinpath(
f'annotations/image_info_{split}2015.json').open())
img_anns = {ann['id']: ann for ann in ins_anns['images']}
elif split == 'train_val':
train_anns, val_anns = [
cls.load_combined_anns(data_dir, split)
for split in ('train', 'val')
]
anns_t = {}
for key in train_anns.keys():
if torch.is_tensor(train_anns[key]):
anns_t[key] = torch.cat(
(train_anns[key], val_anns[key]), dim=0)
elif isinstance(train_anns[key], (list, tuple)):
anns_t[key] = list(train_anns[key]) + list(
val_anns[key])
else:
raise NotImplementedError()
return anns_t
else:
raise NotImplementedError()
anns = collections.defaultdict(list)
for idx, json_ann in enumerate(tqdm(json_anns)):
anns['splits'].append(split)
anns['q_ids'].append(json_ann['question_id'])
anns['img_ids'].append(int(json_ann['image_id']))
anns['q_tokens'].append(json_ann['question_toked'])
anns['q_lens'].append(len(anns['q_tokens'][-1]))
q_label = [q_vocab[token] for token in anns['q_tokens'][-1]]
q_label_t = torch.empty(30).fill_(q_vocab.padding_idx).long()
q_label_t[:len(q_label)] = torch.tensor(q_label).long()
anns['q_labels'].append(q_label_t)
img_ann = img_anns[int(json_ann['image_id'])]
anns['img_names'].append(img_ann['file_name'])
anns['img_shapes'].append(
(img_ann['width'], img_ann['height']))
if split == 'test':
continue
anns['a_counts'].append([
a_count for a_count in json_ann['answers']
if a_count[0] in a_vocab.word2idx
])
anns['a_tokens'].append(json_ann['answer'])
anns['a_scores'].append([
a_score for a_score in json_ann['answers_w_scores']
if a_score[0] in a_vocab.word2idx
])
anns_t = {
key: utils.to_tensor(value)
for key, value in anns.items()
}
torch.save(anns_t, ann_file)
return anns_t
def load_config(self, config_file):
config_file = to_path(config_file)
assert config_file.exists()
config = json.load(open(config_file))
return config
def init_args(cls, params, sub_cls=None):
cls.default_init_args(params)
try_set_attr(
params, f'{cls.prefix_name()}_work_dir',
to_path(params.root_dir).joinpath(f'loggers/{params.proj_name}'))
try_set_attr(params, f'{cls.prefix_name()}_env', params.proj_name)
def load_combined_anns(
cls,
data_dir: str = 'data/vqa2',
data_split: str = 'train',
):
data_dir = to_path(data_dir)
ann_file = data_dir.joinpath(f'{cls.name}/{data_split}_ann.pt')
if ann_file.exists():
anns_t = torch.load(ann_file)
if 'splits' not in anns_t:
anns_t['splits'] = [data_split] * len(anns_t['q_ids'])
torch.save(anns_t, ann_file)
else:
logger.info(f'Creating combined annotation files {ann_file.name}')
q_vocab, a_vocab = cls.load_combined_vocab(data_dir)
if data_split in ('train', 'val'):
origin_anns = json.load(
data_dir.joinpath(
f'annotations/v2_mscoco_{data_split}2014_annotations.json'
).open())['annotations']
q_anns = json.load(
data_dir.joinpath(
f'annotations/v2_OpenEnded_mscoco_{data_split}2014_questions.json'
).open())['questions']
ins_anns = json.load(
data_dir.joinpath(
f'annotations/instances_{data_split}2014.json').open())
img_anns = {ann['id']: ann for ann in ins_anns['images']}
elif data_split == 'test':
origin_anns = None
q_anns = json.load(
data_dir.joinpath(
f'annotations/v2_OpenEnded_mscoco_{data_split}2015_questions.json'
).open())['questions']
ins_anns = json.load(
data_dir.joinpath(
f'annotations/image_info_{data_split}2015.json').open(
))
img_anns = {ann['id']: ann for ann in ins_anns['images']}
elif data_split == 'train_val':
train_anns, val_anns = [
cls.load_combined_anns(data_dir, split)
for split in ('train', 'val')
]
anns_t = {}
for key in train_anns.keys():
if torch.is_tensor(train_anns[key]):
anns_t[key] = torch.cat(
(train_anns[key], val_anns[key]), dim=0)
elif isinstance(train_anns[key], (list, tuple)):
anns_t[key] = list(train_anns[key]) + list(
val_anns[key])
else:
raise NotImplementedError()
return anns_t
else:
raise NotImplementedError()
anns = collections.defaultdict(list)
for idx, q in enumerate(tqdm.tqdm(q_anns)):
anns['splits'].append(data_split)
anns['q_ids'].append(q['question_id'])
anns['img_ids'].append(q['image_id'])
anns['q_tokens'].append(cls.q_tokenize(q['question']))
anns['q_lens'].append(len(anns['q_tokens'][-1]))
q_label = [q_vocab[token] for token in anns['q_tokens'][-1]]
q_label_t = torch.empty(24).fill_(q_vocab.padding_idx).long()
q_label_t[:len(q_label)] = torch.tensor(q_label).long()
anns['q_labels'].append(q_label_t)
img_ann = img_anns[q['image_id']]
anns['img_names'].append(img_ann['file_name'])
anns['img_shapes'].append(
(img_ann['width'], img_ann['height']))
if origin_anns is None:
continue
assert q['question_id'] == origin_anns[idx]['question_id']
ori_ann = origin_anns[idx]
anns['a_tokens'].append(
cls.a_tokenize(ori_ann['multiple_choice_answer']))
answers = []
for ans in ori_ann['answers']:
answ = cls.a_tokenize(ans['answer'])
if answ in a_vocab.words:
answers.append(answ)
anns['a_counts'].append(
collections.Counter(answers).most_common())
# a_label_count = torch.empty(len(a_vocab)).fill_(a_vocab.padding_idx).float()
# for a_token, a_count in anns['a_counts'][-1]:
# a_label_count[a_vocab[a_token]] = a_count
# anns['a_label_counts'].append(a_label_count)
accepted_answers = sum(
[ans[1] for ans in anns['a_counts'][-1]])
anns['a_scores'].append([(ans[0], ans[1] / accepted_answers)
for ans in anns['a_counts'][-1]])
# a_label_score = torch.empty(len(a_vocab)).fill_(a_vocab.padding_idx).float()
# for a_token, a_score in anns['a_scores'][-1]:
# a_label_score[a_vocab[a_token]] = a_score
# anns['a_label_scores'].append(a_label_score)
anns_t = {key: to_tensor(value) for key, value in anns.items()}
torch.save(anns_t, ann_file)
return anns_t
def load_combined_anns(
cls,
data_dir: str = 'data/vqa2',
data_split: str = 'train',
):
data_dir = to_path(data_dir)
ann_file = data_dir.joinpath(f'{cls.name}/{data_split}_ann.pt')
if ann_file.exists():
anns_t = torch.load(ann_file)
if 'splits' not in anns_t:
anns_t['splits'] = [data_split] * len(anns_t['q_ids'])
torch.save(anns_t, ann_file)
else:
logger.info(f'Creating combined annotation files {ann_file.name}')
q_vocab, a_vocab = cls.load_combined_vocab(data_dir)
img_ann_dict = {}
for split in ('train', 'val'):
ins_anns = json.load(
data_dir.joinpath(
f'annotations/instances_{split}2014.json').open())
img_anns = {ann['id']: ann for ann in ins_anns['images']}
img_ann_dict[split] = img_anns
if data_split in ('train', 'test'):
origin_anns = json.load(
data_dir.joinpath(
f'annotations/vqacp_v2_{data_split}_annotations.json').
open())
q_anns = json.load(
data_dir.joinpath(
f'annotations/vqacp_v2_{data_split}_questions.json').
open())
else:
raise NotImplementedError()
anns = collections.defaultdict(list)
for idx, q in enumerate(tqdm.tqdm(q_anns)):
anns['splits'].append(q['coco_split'][:-4])
anns['q_ids'].append(q['question_id'])
anns['img_ids'].append(q['image_id'])
anns['q_tokens'].append(cls.q_tokenize(q['question']))
anns['q_lens'].append(len(anns['q_tokens'][-1]))
q_label = [q_vocab[token] for token in anns['q_tokens'][-1]]
q_label_t = torch.empty(24).fill_(q_vocab.padding_idx).long()
q_label_t[:len(q_label)] = torch.tensor(q_label).long()
anns['q_labels'].append(q_label_t)
img_anns = img_ann_dict[anns['splits'][-1]]
img_ann = img_anns[q['image_id']]
anns['img_names'].append(img_ann['file_name'])
anns['img_shapes'].append(
(img_ann['width'], img_ann['height']))
if origin_anns is None:
continue
assert q['question_id'] == origin_anns[idx]['question_id']
ori_ann = origin_anns[idx]
anns['a_tokens'].append(
cls.a_tokenize(ori_ann['multiple_choice_answer']))
answers = []
for ans in ori_ann['answers']:
answ = cls.a_tokenize(ans['answer'])
if answ in a_vocab.words:
answers.append(answ)
anns['a_counts'].append(
collections.Counter(answers).most_common())
accepted_answers = sum(
[ans[1] for ans in anns['a_counts'][-1]])
anns['a_scores'].append([(ans[0], ans[1] / accepted_answers)
for ans in anns['a_counts'][-1]])
anns_t = {key: to_tensor(value) for key, value in anns.items()}
torch.save(anns_t, ann_file)
return anns_t
