def task(param):
print('加载模型中...')
LtpModel = ltpModel()
W2vModel = w2vModel(LtpModel)
TripleModel = tripleModel(LtpModel)
QAModel = qaModel('qa_data\\weibokb' + str(param['id']) + '.json',
LtpModel, TripleModel, W2vModel)
scoreHist = [0] * 10
scores = []
lines = param['lines']
sentence_number = 0
i = 0
for line in lines:
try:
qaList = line.strip().split('\t')
if len(qaList) != 2:
scores.append(-1)
continue
question = qaList[0]
answer = qaList[1]
score, reason = QAModel.getMatchScore(question, answer)
index = int(score * 10)
if index >= 10:
index = 9
scoreHist[index] += 1
scores.append(score)
except:
scores.append(-1)
print('process:' + str(param['id']) + ' error ' + line)
traceback.print_exc()
sentence_number += 1
if sentence_number % 100 == 0:
print('process:%d done%d %s' %
(param['id'], sentence_number, str(scoreHist)))
print('process:%d finish%d %s' %
(param['id'], sentence_number, str(scoreHist)))
QAModel.saveKB()
result = {
'sentence_number': sentence_number,
'scoreHist': scoreHist,
'scores': scores,
'id': param['id']
}
return result
# -*- coding: utf-8 -*-
import sys
import traceback
from model.ltpModel import ltpModel
from model.tripleModel import tripleModel
#win10 1709版本控制台存在bug，需要引入这个包防止print意外报错
import win_unicode_console
win_unicode_console.enable()
print('加载模型中...')
LtpModel = ltpModel()
TripleModel = tripleModel(LtpModel)
print('开始读取文件')
#读取文件
in_file_name = "test_file\\rdf_input.txt"
out_file_name = "test_file\\rdf_output.txt"
if len(sys.argv) > 1:
in_file_name = sys.argv[1]
if len(sys.argv) > 2:
out_file_name = sys.argv[2]
#默认一行只允许存在一句话
with open(in_file_name, 'r',
encoding='utf-8') as in_file, open(out_file_name,
'w',
encoding='utf-8') as out_file:
sentence_number = 0