def build_word_dict(args, examples, fields, dict_size=None, no_special_token=False): """Return a dictionary from question and document words in provided examples. """ word_dict = Vocabulary(no_special_token) for w in load_words(args, examples, fields, dict_size): word_dict.add(w) return word_dict
def build_word_dict(args, examples, fields, dict_size=None, special_token="pad_unk", attrname="tokens"): """Return a dictionary from question and document words in provided examples. """ word_dict = Vocabulary(no_special_token) for w in load_words(args, examples, fields, dict_size, \ num_spec_tokens=len(special_tokens.split("_")),\ attrname=attrname): word_dict.add(w) return word_dict