def task(param):
print('加载模型中...')
LtpModel = ltpModel()
W2vModel = w2vModel(LtpModel)
TripleModel = tripleModel(LtpModel)
QAModel = qaModel('qa_data\\weibokb' + str(param['id']) + '.json',
LtpModel, TripleModel, W2vModel)
scoreHist = [0] * 10
scores = []
lines = param['lines']
sentence_number = 0
i = 0
for line in lines:
try:
qaList = line.strip().split('\t')
if len(qaList) != 2:
scores.append(-1)
continue
question = qaList[0]
answer = qaList[1]
score, reason = QAModel.getMatchScore(question, answer)
index = int(score * 10)
if index >= 10:
index = 9
scoreHist[index] += 1
scores.append(score)
except:
scores.append(-1)
print('process:' + str(param['id']) + ' error ' + line)
traceback.print_exc()
sentence_number += 1
if sentence_number % 100 == 0:
print('process:%d done%d %s' %
(param['id'], sentence_number, str(scoreHist)))
print('process:%d finish%d %s' %
(param['id'], sentence_number, str(scoreHist)))
QAModel.saveKB()
result = {
'sentence_number': sentence_number,
'scoreHist': scoreHist,
'scores': scores,
'id': param['id']
}
return result
# -*- coding: utf-8 -*-
import sys
import traceback
from model.ltpModel import ltpModel
from model.qaModel import qaModel
from model.w2vModel import w2vModel
from model.tripleModel import tripleModel
modelname='nmt'
print('加载模型中...')
LtpModel=ltpModel()
W2vModel=w2vModel(LtpModel)
TripleModel=tripleModel(LtpModel)
QAModel=qaModel('qa_data\\weibokb.json',LtpModel,TripleModel,W2vModel)
print('开始读取文件')
scoreFiles=[]
scoreNums=[0]*11
for i in range(0,11):
f=open('weibo.100w.test.'+modelname+'.score'+str(i),'w')
scoreFiles.append(f)
scoreSum=0
with open('weibo.100w.test.key', 'r') as keyFile, open('weibo.100w.test.'+modelname+'.output', 'r') as valueFile,\
open('weibo.100w.test.'+modelname+'.score','w') as out_file:
id = 1