# -*- coding:utf-8 -*-
# Author：hankcs
# Date: 2018-07-01 19:15
# 《自然语言处理入门》6.4 HanLP 中的 CRF++ API
# 配套书籍：http://nlp.hankcs.com/book.php
# 讨论答疑：https://bbs.hankcs.com/
from pyhanlp import *
from pyhanlp.static import HANLP_JAR_PATH
from tests.book.ch03.demo_corpus_loader import my_cws_corpus
from tests.test_utility import test_data_path
CRFSegmenter = JClass('com.hankcs.hanlp.model.crf.CRFSegmenter')
TXT_CORPUS_PATH = my_cws_corpus()
TSV_CORPUS_PATH = TXT_CORPUS_PATH + ".tsv"
TEMPLATE_PATH = test_data_path() + "/cws-template.txt"
CRF_MODEL_PATH = test_data_path() + "/crf-cws-model"
CRF_MODEL_TXT_PATH = test_data_path() + "/crf-cws-model.txt"
def train_or_load(corpus_txt_path=TXT_CORPUS_PATH,
model_txt_path=CRF_MODEL_TXT_PATH):
if os.path.isfile(model_txt_path): # 已训练，直接加载
segmenter = CRFSegmenter(model_txt_path)
return segmenter
else:
segmenter = CRFSegmenter() # 创建空白分词器
segmenter.convertCorpus(corpus_txt_path, TSV_CORPUS_PATH) # 执行转换
segmenter.dumpTemplate(TEMPLATE_PATH) # 导出特征模板
# 交给CRF++训练
print("语料已转换为 %s ，特征模板已导出为 %s" % (TSV_CORPUS_PATH, TEMPLATE_PATH))
break
wordnet.add(i, Vertex.newPunctuationInstance(sent[i - 1: j - 1])) # 填充[i, j)之间的空白行
i = j
else:
i += len(vertexes[i][-1].realWord)
return wordnet
def viterbi(wordnet):
nodes = wordnet.getVertexes()
# 前向遍历
for i in range(0, len(nodes) - 1):
for node in nodes[i]:
for to in nodes[i + len(node.realWord)]:
to.updateFrom(node) # 根据距离公式计算节点距离，并维护最短路径上的前驱指针from
# 后向回溯
path = [] # 最短路径
f = nodes[len(nodes) - 1].getFirst() # 从终点回溯
while f:
path.insert(0, f)
f = f.getFrom() # 按前驱指针from回溯
return [v.realWord for v in path]
if __name__ == '__main__':
corpus_path = my_cws_corpus()
model_path = os.path.join(test_data_path(), 'my_cws_model')
train_bigram(corpus_path, model_path)
load_bigram(model_path)