def establish_qh_dict():
qh_typed_words = get_qh_typed_words()
entity_type_dict = {}
for word_type in ['IT', '动物', '医药', '历史人名', '地名', '成语', '法律', '财经', '食物']:
for word in qh_typed_words[word_type]:
entity_type_dict[word] = "其他专名"
print("清华词典构建完毕")
return entity_type_dict
def using_typed_words():
from harvesttext.resources import get_qh_typed_words, get_baidu_stopwords
ht0 = HarvestText()
typed_words, stopwords = get_qh_typed_words(), get_baidu_stopwords()
ht0.add_typed_words(typed_words)
print("加载清华领域词典，并使用停用词")
print("全部类型", typed_words.keys())
sentence = "THUOCL是自然语言处理的一套中文词库，词表来自主流网站的社会标签、搜索热词、输入法词库等。"
print(sentence)
print(ht0.posseg(sentence, stopwords=stopwords))
print("一些词语被赋予特殊类型IT,而“是”等词语被筛出。")
def test_using_typed_words():
sys.stdout, expected = open(get_current_function_name()+"_current","w"), open(get_current_function_name()+"_expected").read()
from harvesttext.resources import get_qh_typed_words,get_baidu_stopwords
ht0 = HarvestText()
typed_words, stopwords = get_qh_typed_words(), get_baidu_stopwords()
ht0.add_typed_words(typed_words)
print("加载清华领域词典，并使用停用词")
print("全部类型",typed_words.keys())
sentence = "THUOCL是自然语言处理的一套中文词库，词表来自主流网站的社会标签、搜索热词、输入法词库等。"
print(sentence)
print(ht0.posseg(sentence,stopwords=stopwords))
print("一些词语被赋予特殊类型IT,而“是”等词语被筛出。")
sys.stdout.close()
assert open(get_current_function_name() + "_current").read() == expected