def ner_train_data():
ent2mention = json_load(Config.ent2mention_json)
# recognizer = hanlp.load(hanlp.pretrained.ner.MSRA_NER_BERT_BASE_ZH)
tokenizer = hanlp.load('PKU_NAME_MERGED_SIX_MONTHS_CONVSEG')
recognizer = hanlp.load(hanlp.pretrained.ner.MSRA_NER_BERT_BASE_ZH)
from LAC import LAC
# 装载LAC模型
lac = LAC(mode='lac')
jieba.enable_paddle() # 启动paddle模式。 0.40版之后开始支持，早期版本不支持
_ent_patten = re.compile(r'["<](.*?)[>"]')
for q, sparql, a in load_data():
q_text = question_patten.findall(q)[0]
hanlp_entities = recognizer([list(q_text)])
hanlp_words = tokenizer(q_text)
lac_results = lac.run(q_text)
q_entities = _ent_patten.findall(sparql)
jieba_results = list(jieba.cut_for_search(q_text))
mentions = [ent2mention.get(ent) for ent in q_entities]
print(f"q_text: {q_text}\nq_entities: {q_entities}, "
f"\nlac_results:{lac_results}"
f"\nhanlp_words: {hanlp_words}, "
f"\njieba_results: {jieba_results}, "
f"\nhanlp_entities: {hanlp_entities}, "
f"\nmentions: {mentions}")
import ipdb
ipdb.set_trace()
def train_qa():
"""训练数据进行回答；做指标测试"""
logging_config('train_qa.log', stream_log=True)
from ckbqa.qa.qa import QA
from ckbqa.dataset.data_prepare import load_data, question_patten, entity_pattern, attr_pattern
# from ckbqa.qa.evaluation_matrics import get_metrics
#
logging.info('* start run ...')
qa = QA()
data = []
for question, sparql, answer in load_data(tqdm_prefix='test qa'):
print('\n' * 2)
print('*****' * 20)
print(f" question : {question}")
print(f" sparql : {sparql}")
print(f" standard answer : {answer}")
q_text = question_patten.findall(question)[0]
standard_subject_entities = entity_pattern.findall(
sparql) + attr_pattern.findall(sparql)
standard_answer_entities = entity_pattern.findall(
answer) + attr_pattern.findall(answer)
try:
(result_entities, candidate_entities, candidate_out_paths,
candidate_in_paths) = qa.run(q_text, return_candidates=True)
except KeyboardInterrupt:
exit('Ctrl C , exit')
except:
logging.info(f'ERROR: {traceback.format_exc()}')
result_entities = []
candidate_entities = []
# print(f" result answer : {result_entities}")
# precision, recall, f1 = get_metrics(subject_entities, candidate_entities)
# if recall == 0 or len(set(standard_entities) & set(candidate_entities)) == 0:
# print(f"question: {question}\n"
# f"subject_entities: {subject_entities}, candidate_entities: {candidate_entities}"
# f"precision: {precision:.4f}, recall: {recall:.4f}, f1: {f1:.4f}\n\n")
# import ipdb
# ipdb.set_trace()
data.append([
question, standard_subject_entities,
list(candidate_entities), standard_answer_entities, result_entities
])
data_df = pd.DataFrame(data,
columns=[
'question', 'standard_subject_entities',
'candidate_entities',
'standard_answer_entities', 'result_entities'
])
data_df.to_csv(ResultSaver().train_result_csv,
index=False,
encoding='utf_8_sig')
def ceg():
logging_config('train_evaluate.log', stream_log=True)
from ckbqa.models.evaluation_matrics import get_metrics
from ckbqa.qa.el import CEG
from ckbqa.dataset.data_prepare import load_data, question_patten, entity_pattern, attr_pattern #
ceg = CEG() # Candidate Entity Generation
ceg_precisions, ceg_recalls, ceg_f1_scores = [], [], []
ceg_csv = "./ceg.csv"
data = []
for q, sparql, a in load_data(tqdm_prefix='ceg evaluate '):
q_entities = entity_pattern.findall(sparql) + attr_pattern.findall(
sparql)
q_text = ''.join(question_patten.findall(q))
# 修复之前把实体<>去掉造成的问题；问题解析时去掉，但预测时未去掉；
# 所以需要匹配文字，不匹配 <>, ""
ent2mention = ceg.get_ent2mention(q_text)
# CEG Candidate Entity Generation
precision, recall, f1 = get_metrics(q_entities, ent2mention)
ceg_precisions.append(precision)
ceg_recalls.append(recall)
ceg_f1_scores.append(f1)
#
data.append([q, q_entities, list(ent2mention.keys())])
if recall == 0:
# ceg.memory.entity2id
# ceg.memory.mention2entity
print(
f"question: {q}\n"
f"subject_entities: {q_entities}, candidate_entities: {ent2mention}"
f"precision: {precision:.4f}, recall: {recall:.4f}, f1: {f1:.4f}\n\n"
)
# import ipdb
# ipdb.set_trace()
print('\n\n')
# import time
# time.sleep(2)
pd.DataFrame(data, columns=['question', 'q_entities',
'ceg']).to_csv(ceg_csv,
index=False,
encoding='utf_8_sig')
ave_precision = sum(ceg_precisions) / len(ceg_precisions)
ave_recall = sum(ceg_recalls) / len(ceg_recalls)
ave_f1_score = sum(ceg_f1_scores) / len(ceg_f1_scores)
print(f"ave_precision: {ave_precision:.3f}, "
f"ave_recall: {ave_recall:.3f}, "
f"ave_f1_score:{ave_f1_score:.3f}")
def test():
logging_config('test.log', stream_log=True)
from ckbqa.qa.qa import QA
from ckbqa.dataset.data_prepare import question_patten
# from ckbqa.qa.evaluation_matrics import get_metrics
#
logging.info('* start run ...')
qa = QA()
q188 = 'q188:墨冰仙是哪个门派的？'
q189 = 'q189:在金庸小说《天龙八部》中，斗转星移的修习者是谁？'
q190 = 'q190:《基督山伯爵》的作者是谁？' # 这里跑没问题
for question in [q188, q189, q190]:
q_text = question_patten.findall(question)[0]
(result_entities, candidate_entities, candidate_out_paths,
candidate_in_paths) = qa.run(q_text, return_candidates=True)
print(question)
import ipdb
ipdb.set_trace()
def gen_train_data(self):
X_train = []
Y_label = []
from ckbqa.qa.el import EL # 避免循环导入
el = EL()
for q, sparql, a in load_data(tqdm_prefix='EntityScore train data '):
# a_entities = entity_pattern.findall(a)
q_entities = set(
entity_pattern.findall(sparql) +
attr_pattern.findall(sparql)) # attr
q_text = question_patten.findall(q)[0]
candidate_entities = el.el(q_text)
for ent_name, feature_dict in candidate_entities.items():
feature = feature_dict['feature']
label = 1 if ent_name in q_entities else 0 # 候选实体有的不在答案中
X_train.append(feature)
Y_label.append(label)
pkl_dump({
'x_data': X_train,
'y_label': Y_label
}, Config.entity_score_data_pkl)
def valid_qa():
"""验证数据答案；验证集做回答；得到提交数据 """
logging_config('valid_qa.log', stream_log=True)
from ckbqa.qa.qa import QA
from ckbqa.dataset.data_prepare import question_patten
#
data_df = pd.DataFrame([
question.strip()
for question in open(valid_question_txt, 'r', encoding='utf-8')
],
columns=['question'])
logging.info(f"data_df.shape: {data_df.shape}")
qa = QA()
valid_datas = {'question': [], 'result': []}
with open(ResultSaver().submit_result_txt, 'w', encoding='utf-8') as f:
for index, row in tqdm(data_df.iterrows(),
total=data_df.shape[0],
desc='qa answer'):
question = row['question']
q_text = question_patten.findall(question)[0]
try:
result_entities = qa.run(q_text)
result_entities = [
ent if ent.startswith('<') else f'"{ent}"'
for ent in result_entities
]
f.write('\t'.join(result_entities) + '\n')
except KeyboardInterrupt:
exit('Ctrl C , exit')
except:
result_entities = []
f.write('""\n')
logging.info(traceback.format_exc())
f.flush()
valid_datas['question'].append(question)
valid_datas['result'].append(result_entities)
pd.DataFrame(valid_datas).to_csv(ResultSaver().valid_result_csv,
index=False,
encoding='utf_8_sig')
def data2samples(self, neg_rate=3, test_size=0.1):
if os.path.isfile(
Config.get_relation_score_sample_csv('train', neg_rate)):
return
questions = []
sim_questions = []
labels = []
all_relations = list(json_load(Config.relation2id))
_entity_pattern = re.compile(r'["<](.*?)[>"]')
for q, sparql, a in load_data(tqdm_prefix='data2samples '):
q_text = question_patten.findall(q)[0]
q_entities = _entity_pattern.findall(sparql)
questions.append(q_text)
sim_questions.append('的'.join(q_entities))
labels.append(1)
#
for neg_relation in random.sample(all_relations, neg_rate):
questions.append(q_text)
neg_question = '的'.join(q_entities[:-1] +
[neg_relation]) # 随机替换 <关系>
sim_questions.append(neg_question)
labels.append(0)
data_df = pd.DataFrame({
'question': questions,
'sim_question': sim_questions,
'label': labels
})
data_df.to_csv(Config.relation_score_sample_csv,
encoding='utf_8_sig',
index=False)
train_df, test_df = train_test_split(data_df, test_size=test_size)
test_df.to_csv(Config.get_relation_score_sample_csv('test', neg_rate),
encoding='utf_8_sig',
index=False)
train_df.to_csv(Config.get_relation_score_sample_csv(
'train', neg_rate),
encoding='utf_8_sig',
index=False)