def initial_dict(filename, with_unk=False):
d = Dict(filename)
d.addSpecial(NULL_WORD)
if with_unk:
d.addSpecial(UNK_WORD)
# d.addSpecial(BOS_WORD)
return d
def initial_dict(filename, with_unk=False):
"""
initial a dict with filename, add special NULL_WORD (""), and unknown OOV word.
"""
d = Dict(filename)
# NULL_WORD is always 0, UNK_WORD is 1
d.addSpecial(NULL_WORD)
if with_unk:
d.addSpecial(UNK_WORD)
# d.addSpecial(BOS_WORD)
return d
rel_dict = rel_dict.pruneByThreshold(threshold)
aux_dict = aux_dict.pruneByThreshold(threshold)
category_dict = category_dict.pruneByThreshold(threshold)
# print (rel_dict)
word_dict.save()
lemma_dict.save()
pos_dict.save()
aux_dict.save()
ner_dict.save()
high_dict.save()
category_dict.save()
rel_dict.save()
sensed_dict.save()
else:
word_dict = Dict("data/word_dict")
lemma_dict = Dict("data/lemma_dict")
aux_dict = Dict("data/aux_dict")
high_dict = Dict("data/high_dict")
pos_dict = Dict("data/pos_dict")
ner_dict = Dict("data/ner_dict")
rel_dict = Dict("data/rel_dict")
category_dict = Dict("data/category_dict")
sensed_dict = Dict("data/sensed_dict")
word_dict.load()
lemma_dict.load()
pos_dict.load()
ner_dict.load()
rel_dict.load()
category_dict.load()
d = Dict(filename)
# NULL_WORD is always 0, UNK_WORD is 1
d.addSpecial(NULL_WORD)
if with_unk:
d.addSpecial(UNK_WORD)
# d.addSpecial(BOS_WORD)
return d
if not opt.skip:
# skip means if dict existed, not build it.
# init all the dict, which can be used as vocabulary of the neual model.
# used as input source
if opt.merge_common_dicts:
try:
word_dict = Dict(opt.build_folder + "dicts/word_dict")
word_dict.load()
except:
word_dict = initial_dict(opt.build_folder + "dicts/word_dict",
with_unk=True)
try:
lemma_dict = Dict(opt.build_folder + "dicts/lemma_dict")
lemma_dict.load()
except:
lemma_dict = initial_dict(opt.build_folder + "dicts/lemma_dict",
with_unk=True)
try:
source_pos_dict = Dict(opt.build_folder + "dicts/pos_dict")
source_pos_dict.load()
except:
"""
d = Dict(filename)
# NULL_WORD is always 0, UNK_WORD is 1
d.addSpecial(NULL_WORD)
if with_unk:
d.addSpecial(UNK_WORD)
# d.addSpecial(BOS_WORD)
return d
if not opt.skip:
# skip means if dict existed, not build it.
# init all the dict, which can be used as vocabulary of the neual model.
if opt.merge_common_dicts:
try:
amr_word_dict = Dict(opt.build_folder + "dicts/word_dict")
amr_word_dict.load()
except:
amr_word_dict = initial_dict(opt.build_folder + "dicts/word_dict",
with_unk=True)
try:
amr_lemma_dict = Dict(opt.build_folder + "dicts/lemma_dict")
amr_lemma_dict.load()
except:
amr_lemma_dict = initial_dict(opt.build_folder +
"dicts/lemma_dict",
with_unk=True)
try:
amr_pos_dict = Dict(opt.build_folder + "dicts/pos_dict")
d = Dict(filename)
# NULL_WORD is always 0, UNK_WORD is 1
d.addSpecial(NULL_WORD)
if with_unk:
d.addSpecial(UNK_WORD)
# d.addSpecial(BOS_WORD)
return d
if not opt.skip:
# skip means if dict existed, not build it.
# init all the dict, which can be used as vocabulary of the neual model.
# used as input source
if opt.merge_common_dicts:
try:
word_dict = Dict(opt.build_folder + "dicts/word_dict")
word_dict.load()
except:
word_dict = initial_dict(opt.build_folder + "dicts/word_dict",
with_unk=True)
try:
lemma_dict = Dict(opt.build_folder + "dicts/lemma_dict")
lemma_dict.load()
except:
lemma_dict = initial_dict(opt.build_folder + "dicts/lemma_dict",
with_unk=True)
try:
source_pos_dict = Dict(opt.build_folder + "dicts/pos_dict")
source_pos_dict.load()
except:
d = Dict(filename)
# NULL_WORD is always 0, UNK_WORD is 1
d.addSpecial(NULL_WORD)
if with_unk:
d.addSpecial(UNK_WORD)
# d.addSpecial(BOS_WORD)
return d
if not opt.skip:
# skip means if dict existed, not build it.
# init all the dict, which can be used as vocabulary of the neual model.
# used as input source
if opt.merge_common_dicts:
try:
word_dict = Dict(opt.build_folder + "dicts/word_dict")
word_dict.load()
except:
word_dict = initial_dict(opt.build_folder + "dicts/word_dict",
with_unk=True)
try:
lemma_dict = Dict(opt.build_folder + "dicts/lemma_dict")
lemma_dict.load()
except:
lemma_dict = initial_dict(opt.build_folder + "dicts/lemma_dict",
with_unk=True)
try:
source_pos_dict = Dict(opt.build_folder + "dicts/pos_dict")
source_pos_dict.load()
except: