def initial_dict(filename, with_unk=False): d = Dict(filename) d.addSpecial(NULL_WORD) if with_unk: d.addSpecial(UNK_WORD) # d.addSpecial(BOS_WORD) return d
def initial_dict(filename, with_unk=False): """ initial a dict with filename, add special NULL_WORD (""), and unknown OOV word. """ d = Dict(filename) # NULL_WORD is always 0, UNK_WORD is 1 d.addSpecial(NULL_WORD) if with_unk: d.addSpecial(UNK_WORD) # d.addSpecial(BOS_WORD) return d
rel_dict = rel_dict.pruneByThreshold(threshold) aux_dict = aux_dict.pruneByThreshold(threshold) category_dict = category_dict.pruneByThreshold(threshold) # print (rel_dict) word_dict.save() lemma_dict.save() pos_dict.save() aux_dict.save() ner_dict.save() high_dict.save() category_dict.save() rel_dict.save() sensed_dict.save() else: word_dict = Dict("data/word_dict") lemma_dict = Dict("data/lemma_dict") aux_dict = Dict("data/aux_dict") high_dict = Dict("data/high_dict") pos_dict = Dict("data/pos_dict") ner_dict = Dict("data/ner_dict") rel_dict = Dict("data/rel_dict") category_dict = Dict("data/category_dict") sensed_dict = Dict("data/sensed_dict") word_dict.load() lemma_dict.load() pos_dict.load() ner_dict.load() rel_dict.load() category_dict.load()
d = Dict(filename) # NULL_WORD is always 0, UNK_WORD is 1 d.addSpecial(NULL_WORD) if with_unk: d.addSpecial(UNK_WORD) # d.addSpecial(BOS_WORD) return d if not opt.skip: # skip means if dict existed, not build it. # init all the dict, which can be used as vocabulary of the neual model. # used as input source if opt.merge_common_dicts: try: word_dict = Dict(opt.build_folder + "dicts/word_dict") word_dict.load() except: word_dict = initial_dict(opt.build_folder + "dicts/word_dict", with_unk=True) try: lemma_dict = Dict(opt.build_folder + "dicts/lemma_dict") lemma_dict.load() except: lemma_dict = initial_dict(opt.build_folder + "dicts/lemma_dict", with_unk=True) try: source_pos_dict = Dict(opt.build_folder + "dicts/pos_dict") source_pos_dict.load() except:
""" d = Dict(filename) # NULL_WORD is always 0, UNK_WORD is 1 d.addSpecial(NULL_WORD) if with_unk: d.addSpecial(UNK_WORD) # d.addSpecial(BOS_WORD) return d if not opt.skip: # skip means if dict existed, not build it. # init all the dict, which can be used as vocabulary of the neual model. if opt.merge_common_dicts: try: amr_word_dict = Dict(opt.build_folder + "dicts/word_dict") amr_word_dict.load() except: amr_word_dict = initial_dict(opt.build_folder + "dicts/word_dict", with_unk=True) try: amr_lemma_dict = Dict(opt.build_folder + "dicts/lemma_dict") amr_lemma_dict.load() except: amr_lemma_dict = initial_dict(opt.build_folder + "dicts/lemma_dict", with_unk=True) try: amr_pos_dict = Dict(opt.build_folder + "dicts/pos_dict")