def build_word_dict(args, examples, fields, dict_size=None,
no_special_token=False):
"""Return a dictionary from question and document words in
provided examples.
"""
word_dict = Vocabulary(no_special_token)
for w in load_words(args, examples, fields, dict_size):
word_dict.add(w)
return word_dict
def build_word_dict(args, examples, fields, dict_size=None,
special_token="pad_unk", attrname="tokens"):
"""Return a dictionary from question and document words in
provided examples.
"""
word_dict = Vocabulary(no_special_token)
for w in load_words(args, examples, fields, dict_size, \
num_spec_tokens=len(special_tokens.split("_")),\
attrname=attrname):
word_dict.add(w)
return word_dict
