def new_generate_ltp_results():
# 加载模型
ltp_model = '../../ltp_models/base1'
ltp = LTP(path=ltp_model)
# 读取原句子
data = read_file_in_ltp('../data/train_base.json')
sentences = list(map(lambda x: x['content'], data))
segmented, pos, ner, srl, dep, sdp_tree, sdp_graph = [], [], [], [], [], [], []
for sent in tqdm(sentences):
# 分词
segmented0, hidden = ltp.seg([sent])
# 词性标注
cur_pos = ltp.pos(hidden)
# 命名实体识别
cur_ner = ltp.ner(hidden)
# 语义角色标注
cur_srl = ltp.srl(hidden)
# 依存句法分析
cur_dep = ltp.dep(hidden)
# 语义依存分析 (树)
cur_sdp_tree = ltp.sdp(hidden, mode='tree')
# 语义依存分析 (图)
cur_sdp_graph = ltp.sdp(hidden, mode='graph')
segmented.append(segmented0[0])
pos.append(cur_pos[0])
ner.append(cur_ner[0])
srl.append(cur_srl[0])
dep.append(cur_dep[0])
sdp_tree.append(cur_sdp_tree[0])
sdp_graph.append(cur_sdp_graph[0])
# 生成句子与分词的对应
sent_seg_matches = sentence_segment_match(data, segmented)
pickle.dump([segmented, pos, ner, srl, dep, sdp_tree, sdp_graph, sent_seg_matches], open('new_ltp_results.pk', 'wb'))
return segmented, pos, ner, srl, dep, sdp_tree, sdp_graph, sent_seg_matches
def WriteTest(readfile, savefile):
with open(readfile, "r", encoding="utf-8") as rfp:
ltp = LTP()
logger.info("Processing file:%s ." % (readfile))
with open(savefile, 'w', encoding='utf-8') as wfp:
for row in tqdm(rfp, desc="file %s process" % (readfile)):
sent1, sent2 = row.split('\t')
seg, hid = ltp.seg([sent1, sent2])
sdp = ltp.sdp(hid, mode='tree')
pos = ltp.pos(hid)
tmpitem = {
'sentence1': [seg[0], pos[0], sdp[0]],
'sentence2': [seg[1], pos[1], sdp[1]]
}
jsonline = json.dumps(tmpitem)
wfp.write(jsonline + "\n")
class Server(object):
def __init__(self,
path: str = 'small',
batch_size: int = 50,
device: str = None,
onnx: bool = False):
self.ltp = LTP(path=path, device=device)
self.split = lambda a: map(lambda b: a[b:b + batch_size],
range(0, len(a), batch_size))
def _build_words(self, words, pos, dep):
res = [{'id': -1, 'length': 0, 'offset': 0, 'text': 'root'}]
for word, p, (id, parent, relation) in zip(words, pos, dep):
offset = res[-1]['offset'] + res[-1]['length']
res.append({
'id': id - 1,
'length': len(word),
'offset': offset,
'text': word,
'pos': p,
'parent': parent - 1,
'relation': relation,
'roles': [],
'parents': []
})
return res[1:]
def _predict(self, sentences: List[str]):
result = []
for sentences_batch in self.split(sentences):
batch_seg, hidden = self.ltp.seg(sentences_batch)
batch_pos = self.ltp.pos(hidden)
batch_ner = self.ltp.ner(hidden)
batch_srl = self.ltp.srl(hidden)
batch_dep = self.ltp.dep(hidden, fast=False)
batch_sdp = self.ltp.sdp(hidden, mode='mix')
for sent, seg, pos, ner, srl, dep, sdp in \
zip(sentences_batch, batch_seg, batch_pos, batch_ner, batch_srl, batch_dep, batch_sdp):
words = self._build_words(seg, pos, dep)
for word, token_srl in zip(words, srl):
for role, start, end in token_srl:
text = "".join(seg[start:end + 1])
offset = words[start]['offset']
word['roles'].append({
'text': text,
'offset': offset,
'length': len(text),
'type': role
})
for start, end, label in sdp:
words[start - 1]['parents'].append({
'parent': end - 1,
'relate': label
})
nes = []
for role, start, end in ner:
text = "".join(seg[start:end + 1])
nes.append({
'text': text,
'offset': start,
'ne': role.lower(),
'length': len(text)
})
result.append({'text': sent, 'nes': nes, 'words': words})
return result
def serve(self, port: int = 5000, n_process: int = None):
if n_process is None:
n_process = 1 if sys.platform == 'win32' else 8
fmt = LogFormatter(fmt='%(asctime)s - %(levelname)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
color=True)
root_logger = logging.getLogger()
console_handler = logging.StreamHandler()
file_handler = logging.FileHandler('server.log')
console_handler.setFormatter(fmt)
file_handler.setFormatter(fmt)
root_logger.addHandler(console_handler)
root_logger.addHandler(file_handler)
app_log.setLevel(logging.INFO)
gen_log.setLevel(logging.INFO)
access_log.setLevel(logging.INFO)
app_log.info("Model is loading...")
app_log.info("Model Has Been Loaded!")
app = Application([(r"/.*", LTPHandler, dict(ltp=self))])
server = HTTPServer(app)
server.bind(port)
server.start(n_process)
ioloop.IOLoop.instance().start()
# tiaozhuan=searchKG(kglist=['地点','地址','大小','颜色','老婆','丈夫'],text='我家住在和平区哪个地方')
# print(tiaozhuan,"jieguo shi !!!!!!!!!!!!!!!!")
##
# 加入句子成分跳转.
seg, hidden = ltp.seg([text])
# sdp = ltp.sdp(hidden, graph=False)
print(seg, "seg")
pos = ltp.pos(hidden)
ner = ltp.ner(hidden)
print("ner", ner)
srl = ltp.srl(hidden)
dep = ltp.dep(hidden)
sdp = ltp.sdp(hidden)
print(ner, "ner结果")
seg = seg[0]
dep = dep[0]
sdp = sdp[0]
print(sdp, "语义分析!!!!!!!!!!!!!!!!!!!") # 太难用了.
print(dep)
for i in dep: #dep算法目前识别不出来老婆的跳转.
print(i, seg[i[0] - 1], seg[i[1] - 1]) # 注意下标会多一个, 箭1后为真正下标.
'''
下面我们根据跳跃图简历bfs算法
'''
#dep 就是我们需要的图
tmp3 = []
for i in kglist:
t = (cosine_distance(vec2(i), vec2(text)))
tmp3.append(t)
tmp3 = np.array(tmp3)
# 查询到的最近kg 3元组是!!!!!!!!!!!!!!!!
dix = np.argmin(tmp3)
print('最近的3元组是', kglist[dix])
print(33333333333333)
##
seg, hidden = ltp.seg([text])
sdp = ltp.sdp(hidden, graph=False)
print(seg, "seg")
pos = ltp.pos(hidden)
ner = ltp.ner(hidden)
print("ner", ner)
srl = ltp.srl(hidden)
dep = ltp.dep(hidden)
sdp = ltp.sdp(hidden)
seg = seg[0]
for i in sdp[0]:
print(i, seg[i[0] - 1], seg[i[1] - 1]) # 注意下标会多一个, 箭1后为真正下标.
class Run(object):
def __init__(self,
path: str = 'small',
batch_size: int = 50,
device: str = None,
onnx: bool = False):
if onnx:
self.ltp = FastLTP(path=path, device=device, need_config=True)
else:
self.ltp = LTP(path=path, device=device, need_config=True)
self.split = lambda a: map(lambda b: a[b:b + batch_size],
range(0, len(a), batch_size))
def _build_words(self, words, pos, dep):
res = [{'id': -1, 'length': 0, 'offset': 0, 'text': 'root'}]
for word, p, (id, parent, relation) in zip(words, pos, dep):
offset = res[-1]['offset'] + res[-1]['length']
res.append({
'id': id - 1,
'length': len(word),
'offset': offset,
'text': word,
'pos': p,
'parent': parent - 1,
'relation': relation,
'roles': [],
'parents': []
})
return res[1:]
def _predict(self, sentences: List[str]):
result = []
for sentences_batch in self.split(sentences):
batch_seg, hidden = self.ltp.seg(sentences_batch)
batch_pos = self.ltp.pos(hidden)
batch_ner = self.ltp.ner(hidden)
batch_srl = self.ltp.srl(hidden)
batch_dep = self.ltp.dep(hidden)
batch_sdp = self.ltp.sdp(hidden)
for sent, seg, pos, ner, srl, dep, sdp in \
zip(sentences_batch, batch_seg, batch_pos, batch_ner, batch_srl, batch_dep, batch_sdp):
words = self._build_words(seg, pos, dep)
for word, token_srl in zip(words, srl):
for role, start, end in token_srl:
text = "".join(seg[start:end + 1])
offset = words[start]['offset']
word['roles'].append({
'text': text,
'offset': offset,
'length': len(text),
'type': role
})
for start, end, label in sdp:
words[start - 1]['parents'].append({
'parent': end - 1,
'relate': label
})
nes = []
for role, start, end in ner:
text = "".join(seg[start:end + 1])
nes.append({
'text': text,
'offset': start,
'ne': role.lower(),
'length': len(text)
})
result.append({'text': sent, 'nes': nes, 'words': words})
return result
def test(self, sentences: List[str] = None):
self.ltp.add_words("DMI与主机通讯中断")
if sentences is None:
sentences = ["他叫汤姆去拿外衣。"]
res = self._predict([sentence.strip() for sentence in sentences])
print(json.dumps(res, indent=2, sort_keys=True, ensure_ascii=False))
def save(self, out='ltp.npz'):
import numpy as np
nps = {}
for k, v in self.ltp.model.state_dict().items():
k = k.replace("gamma", "weight").replace("beta", "bias")
nps[k] = np.ascontiguousarray(v.cpu().numpy())
np.savez(out, **nps)
config = self.ltp.config
with open('config.json', 'w', encoding='utf-8') as f:
json.dump(config, f, indent=2)
def test_seged(self):
import torch
sentences = [
'My name is tom.', 'He called Tom to get coats.', '他叫Tom去拿外衣。',
'他叫汤姆去拿外衣。', "我去长江大桥玩。"
]
seg, hidden = self.ltp.seg(sentences)
seged, hidden_seged = self.ltp.seg(seg, is_preseged=True)
hidden: dict
hidden_seged: dict
for key, value in hidden.items():
if isinstance(value, torch.Tensor):
test = torch.sum(value.float() -
hidden_seged[key].float()).numpy()
print(key, test)
print(seg == seged)
class Server(object):
def __init__(self,
path: str = 'small',
batch_size: int = 50,
device: str = None,
onnx: bool = False):
if onnx:
self.ltp = FastLTP(path=path, device=device)
else:
self.ltp = LTP(path=path, device=device)
self.split = lambda a: map(lambda b: a[b:b + batch_size],
range(0, len(a), batch_size))
def _build_words(self, words, pos, dep):
res = [{'id': -1, 'length': 0, 'offset': 0, 'text': 'root'}]
for word, p, (id, parent, relation) in zip(words, pos, dep):
offset = res[-1]['offset'] + res[-1]['length']
res.append({
'id': id - 1,
'length': len(word),
'offset': offset,
'text': word,
'pos': p,
'parent': parent - 1,
'relation': relation,
'roles': [],
'parents': []
})
return res[1:]
def _predict(self, sentences: List[str]):
result = []
for sentences_batch in self.split(sentences):
batch_seg, hidden = self.ltp.seg(sentences_batch)
batch_pos = self.ltp.pos(hidden)
batch_ner = self.ltp.ner(hidden)
batch_srl = self.ltp.srl(hidden)
batch_dep = self.ltp.dep(hidden)
batch_sdp = self.ltp.sdp(hidden)
for sent, seg, pos, ner, srl, dep, sdp in \
zip(sentences_batch, batch_seg, batch_pos, batch_ner, batch_srl, batch_dep, batch_sdp):
words = self._build_words(seg, pos, dep)
for word, token_srl in zip(words, srl):
for role, start, end in token_srl:
text = "".join(seg[start:end + 1])
offset = words[start]['offset']
word['roles'].append({
'text': text,
'offset': offset,
'length': len(text),
'type': role
})
for start, end, label in sdp:
words[start - 1]['parents'].append({
'parent': end - 1,
'relate': label
})
nes = []
for role, start, end in ner:
text = "".join(seg[start:end + 1])
nes.append({
'text': text,
'offset': start,
'ne': role.lower(),
'length': len(text)
})
result.append({'text': sent, 'nes': nes, 'words': words})
return result
#初始化分词工具
ltp = LTP(proxies=proxies)
sentences = pdfreader.getTestFromPdf()['Text']
seg = []
sdp = []
dep = []
pos = []
cluster = []
#对句子进行分词以及语义依存分析
for st in sentences:
if (st != ''):
seg_temp, hidden = ltp.seg([st])
# 获得语义依存关系
sdp.append(ltp.sdp(hidden)[0])
#获得词性列表
pos.append(ltp.pos(hidden)[0])
#获得分词列表
seg.append(seg_temp[0])
#获得语法依存关系
dep.append(ltp.dep(hidden)[0])
#初始化储存三元组的list
resultTriad = []
for index in range(len(dep)):
r = getTriad(dep[index], seg[index], pos[index])
resultTriad.append(r)
#初始化数据库连接
graph = getConnection()
class Conllu(object):
"""
:param path: 模型路径，或者自动从网上下载 ['base', 'small', 'tiny']
:param batch_size: 最大 Batch Size 自动切分
:param device: ['cpu', 'cuda']
:param onnx: 是否启用 onnx
"""
def __init__(self,
path: str = 'small',
batch_size: int = 50,
device: str = None,
onnx: bool = False):
if onnx:
self.ltp = FastLTP(path=path, device=device, need_config=True)
else:
self.ltp = LTP(path=path, device=device, need_config=True)
self._split = lambda a: map(lambda b: a[b:b + batch_size],
range(0, len(a), batch_size))
def _predict(self,
sentences: List[str],
pos=True,
ner=True,
srl=True,
dep=True,
sdp=True):
result = []
for sentences_batch in self._split(sentences):
batch_seg, hidden = self.ltp.seg(sentences_batch)
batch_size = len(sentences_batch)
batch_pos = self.ltp.pos(hidden) if pos else ([[]] * batch_size)
batch_ner = self.ltp.ner(hidden) if ner else ([None] * batch_size)
batch_srl = self.ltp.srl(
hidden, keep_empty=False) if srl else ([None] * batch_size)
batch_dep = self.ltp.dep(hidden) if dep else ([None] * batch_size)
batch_sdp = self.ltp.sdp(hidden) if sdp else ([None] * batch_size)
result += list(
zip(batch_seg, batch_pos, batch_ner, batch_dep, batch_sdp,
batch_srl))
return result
def predict(self,
input: str,
output: Optional[str] = None,
pos: bool = True,
ner: bool = False,
srl: bool = False,
dep: bool = True,
sdp: bool = False):
"""
预测文本并输出为 conllu 格式
:param input: 要预测的文件，每行一句话
:param output: 输出的结果文件，默认是输入文件添加 .conll 后缀
:param pos: 是否输出 词性标注 结果 ['True','False']
:param ner: 是否输出 命名实体识别 结果 ['True','False'], 占用 conllu feats 列
:param srl: 是否输出 语义角色标注 结果 ['True','False'], 占用 conllu misc 列
:param dep: 是否输出 依存句法分析 结果 ['True','False']
:param sdp: 是否输出 语义依存分析 结果 ['True','False']
"""
if output is None:
output = f"{input}.conllu"
with open(output, mode='w', encoding='utf-8') as f:
sentences = sum([sent for idx, sent in iter_lines(input)], [])
results = self._predict(sentences, pos, ner, srl, dep, sdp)
for text, (seg_s, pos_s, ner_s, dep_s, sdp_s,
srl_s) in zip(sentences, results):
tokens = conllu.TokenList([
conllu.models.Token(id=idx + 1,
form=token,
lemma=token,
upos=pos if pos else '_',
xpos=pos if pos else '_',
feats='O' if ner else '_',
head=idx,
deprel='_',
deps='' if sdp else '_',
misc='SpaceAfter=No')
for idx, (token,
pos) in enumerate(zip_longest(seg_s, pos_s))
], conllu.models.Metadata(text=text))
if ner:
for tag, start, end in ner_s:
tokens[start]['feats'] = f'B-{tag}'
for i in range(start + 1, end):
tokens[start]['feats'] = f'I-{tag}'
if dep:
for id, head, tag in dep_s:
tokens[id - 1]['head'] = head
tokens[id - 1]['deprel'] = tag
if sdp:
for id, head, tag in sdp_s:
if tokens[id - 1]['deps']:
tokens[id - 1]['deps'] = tokens[
id - 1]['deps'] + f"|{head}:{tag}"
else:
tokens[id - 1]['deps'] = f"{head}:{tag}"
if srl:
srl_predicate, srl_roles = list(zip(*srl_s))
srl_predicate_num = len(srl_predicate)
if srl_predicate_num > 0:
srl_misc = [[
f'Predicate={"Y" if i in srl_predicate else "_"}',
['O'] * srl_predicate_num
] for i in range(len(tokens))]
for idx, srl_role in enumerate(srl_roles):
for tag, start, end in srl_role:
srl_misc[start][-1][idx] = f'B-{tag}'
for i in range(start + 1, end):
srl_misc[start][-1][idx] = f'I-{tag}'
srl_misc = [
"|".join([s[0], "Role=" + ",".join(s[-1])])
for s in srl_misc
]
for token, misc in zip(tokens, srl_misc):
token['misc'] = f"{token['misc']}|{misc}"
f.write(tokens.serialize())
