def create_dictionary(entries, dataset):
dictionary = Dictionary()
for ent in entries:
if dataset == 'vqd':
qs = ent['question']
else:
qs = ent['sentence']['sent']
dictionary.tokenize(qs, True)
return dictionary
def create_dictionary(ds):
dictionary = Dictionary()
entries = []
for group in ['train','test']:
with open( dataset[ds][group],'rb') as f:
d = pickle.load(f)
entries.extend(d)
for ent in entries:
qs = ent['question']
dictionary.tokenize(qs, True)
return dictionary
