def ner_train_data(): ent2mention = json_load(Config.ent2mention_json) # recognizer = hanlp.load(hanlp.pretrained.ner.MSRA_NER_BERT_BASE_ZH) tokenizer = hanlp.load('PKU_NAME_MERGED_SIX_MONTHS_CONVSEG') recognizer = hanlp.load(hanlp.pretrained.ner.MSRA_NER_BERT_BASE_ZH) from LAC import LAC # 装载LAC模型 lac = LAC(mode='lac') jieba.enable_paddle() # 启动paddle模式。 0.40版之后开始支持,早期版本不支持 _ent_patten = re.compile(r'["<](.*?)[>"]') for q, sparql, a in load_data(): q_text = question_patten.findall(q)[0] hanlp_entities = recognizer([list(q_text)]) hanlp_words = tokenizer(q_text) lac_results = lac.run(q_text) q_entities = _ent_patten.findall(sparql) jieba_results = list(jieba.cut_for_search(q_text)) mentions = [ent2mention.get(ent) for ent in q_entities] print(f"q_text: {q_text}\nq_entities: {q_entities}, " f"\nlac_results:{lac_results}" f"\nhanlp_words: {hanlp_words}, " f"\njieba_results: {jieba_results}, " f"\nhanlp_entities: {hanlp_entities}, " f"\nmentions: {mentions}") import ipdb ipdb.set_trace()
def train_qa(): """训练数据进行回答;做指标测试""" logging_config('train_qa.log', stream_log=True) from ckbqa.qa.qa import QA from ckbqa.dataset.data_prepare import load_data, question_patten, entity_pattern, attr_pattern # from ckbqa.qa.evaluation_matrics import get_metrics # logging.info('* start run ...') qa = QA() data = [] for question, sparql, answer in load_data(tqdm_prefix='test qa'): print('\n' * 2) print('*****' * 20) print(f" question : {question}") print(f" sparql : {sparql}") print(f" standard answer : {answer}") q_text = question_patten.findall(question)[0] standard_subject_entities = entity_pattern.findall( sparql) + attr_pattern.findall(sparql) standard_answer_entities = entity_pattern.findall( answer) + attr_pattern.findall(answer) try: (result_entities, candidate_entities, candidate_out_paths, candidate_in_paths) = qa.run(q_text, return_candidates=True) except KeyboardInterrupt: exit('Ctrl C , exit') except: logging.info(f'ERROR: {traceback.format_exc()}') result_entities = [] candidate_entities = [] # print(f" result answer : {result_entities}") # precision, recall, f1 = get_metrics(subject_entities, candidate_entities) # if recall == 0 or len(set(standard_entities) & set(candidate_entities)) == 0: # print(f"question: {question}\n" # f"subject_entities: {subject_entities}, candidate_entities: {candidate_entities}" # f"precision: {precision:.4f}, recall: {recall:.4f}, f1: {f1:.4f}\n\n") # import ipdb # ipdb.set_trace() data.append([ question, standard_subject_entities, list(candidate_entities), standard_answer_entities, result_entities ]) data_df = pd.DataFrame(data, columns=[ 'question', 'standard_subject_entities', 'candidate_entities', 'standard_answer_entities', 'result_entities' ]) data_df.to_csv(ResultSaver().train_result_csv, index=False, encoding='utf_8_sig')
def ceg(): logging_config('train_evaluate.log', stream_log=True) from ckbqa.models.evaluation_matrics import get_metrics from ckbqa.qa.el import CEG from ckbqa.dataset.data_prepare import load_data, question_patten, entity_pattern, attr_pattern # ceg = CEG() # Candidate Entity Generation ceg_precisions, ceg_recalls, ceg_f1_scores = [], [], [] ceg_csv = "./ceg.csv" data = [] for q, sparql, a in load_data(tqdm_prefix='ceg evaluate '): q_entities = entity_pattern.findall(sparql) + attr_pattern.findall( sparql) q_text = ''.join(question_patten.findall(q)) # 修复之前把实体<>去掉造成的问题;问题解析时去掉,但预测时未去掉; # 所以需要匹配文字,不匹配 <>, "" ent2mention = ceg.get_ent2mention(q_text) # CEG Candidate Entity Generation precision, recall, f1 = get_metrics(q_entities, ent2mention) ceg_precisions.append(precision) ceg_recalls.append(recall) ceg_f1_scores.append(f1) # data.append([q, q_entities, list(ent2mention.keys())]) if recall == 0: # ceg.memory.entity2id # ceg.memory.mention2entity print( f"question: {q}\n" f"subject_entities: {q_entities}, candidate_entities: {ent2mention}" f"precision: {precision:.4f}, recall: {recall:.4f}, f1: {f1:.4f}\n\n" ) # import ipdb # ipdb.set_trace() print('\n\n') # import time # time.sleep(2) pd.DataFrame(data, columns=['question', 'q_entities', 'ceg']).to_csv(ceg_csv, index=False, encoding='utf_8_sig') ave_precision = sum(ceg_precisions) / len(ceg_precisions) ave_recall = sum(ceg_recalls) / len(ceg_recalls) ave_f1_score = sum(ceg_f1_scores) / len(ceg_f1_scores) print(f"ave_precision: {ave_precision:.3f}, " f"ave_recall: {ave_recall:.3f}, " f"ave_f1_score:{ave_f1_score:.3f}")
def test(): logging_config('test.log', stream_log=True) from ckbqa.qa.qa import QA from ckbqa.dataset.data_prepare import question_patten # from ckbqa.qa.evaluation_matrics import get_metrics # logging.info('* start run ...') qa = QA() q188 = 'q188:墨冰仙是哪个门派的?' q189 = 'q189:在金庸小说《天龙八部》中,斗转星移的修习者是谁?' q190 = 'q190:《基督山伯爵》的作者是谁?' # 这里跑没问题 for question in [q188, q189, q190]: q_text = question_patten.findall(question)[0] (result_entities, candidate_entities, candidate_out_paths, candidate_in_paths) = qa.run(q_text, return_candidates=True) print(question) import ipdb ipdb.set_trace()
def gen_train_data(self): X_train = [] Y_label = [] from ckbqa.qa.el import EL # 避免循环导入 el = EL() for q, sparql, a in load_data(tqdm_prefix='EntityScore train data '): # a_entities = entity_pattern.findall(a) q_entities = set( entity_pattern.findall(sparql) + attr_pattern.findall(sparql)) # attr q_text = question_patten.findall(q)[0] candidate_entities = el.el(q_text) for ent_name, feature_dict in candidate_entities.items(): feature = feature_dict['feature'] label = 1 if ent_name in q_entities else 0 # 候选实体有的不在答案中 X_train.append(feature) Y_label.append(label) pkl_dump({ 'x_data': X_train, 'y_label': Y_label }, Config.entity_score_data_pkl)
def valid_qa(): """验证数据答案;验证集做回答;得到提交数据 """ logging_config('valid_qa.log', stream_log=True) from ckbqa.qa.qa import QA from ckbqa.dataset.data_prepare import question_patten # data_df = pd.DataFrame([ question.strip() for question in open(valid_question_txt, 'r', encoding='utf-8') ], columns=['question']) logging.info(f"data_df.shape: {data_df.shape}") qa = QA() valid_datas = {'question': [], 'result': []} with open(ResultSaver().submit_result_txt, 'w', encoding='utf-8') as f: for index, row in tqdm(data_df.iterrows(), total=data_df.shape[0], desc='qa answer'): question = row['question'] q_text = question_patten.findall(question)[0] try: result_entities = qa.run(q_text) result_entities = [ ent if ent.startswith('<') else f'"{ent}"' for ent in result_entities ] f.write('\t'.join(result_entities) + '\n') except KeyboardInterrupt: exit('Ctrl C , exit') except: result_entities = [] f.write('""\n') logging.info(traceback.format_exc()) f.flush() valid_datas['question'].append(question) valid_datas['result'].append(result_entities) pd.DataFrame(valid_datas).to_csv(ResultSaver().valid_result_csv, index=False, encoding='utf_8_sig')
def data2samples(self, neg_rate=3, test_size=0.1): if os.path.isfile( Config.get_relation_score_sample_csv('train', neg_rate)): return questions = [] sim_questions = [] labels = [] all_relations = list(json_load(Config.relation2id)) _entity_pattern = re.compile(r'["<](.*?)[>"]') for q, sparql, a in load_data(tqdm_prefix='data2samples '): q_text = question_patten.findall(q)[0] q_entities = _entity_pattern.findall(sparql) questions.append(q_text) sim_questions.append('的'.join(q_entities)) labels.append(1) # for neg_relation in random.sample(all_relations, neg_rate): questions.append(q_text) neg_question = '的'.join(q_entities[:-1] + [neg_relation]) # 随机替换 <关系> sim_questions.append(neg_question) labels.append(0) data_df = pd.DataFrame({ 'question': questions, 'sim_question': sim_questions, 'label': labels }) data_df.to_csv(Config.relation_score_sample_csv, encoding='utf_8_sig', index=False) train_df, test_df = train_test_split(data_df, test_size=test_size) test_df.to_csv(Config.get_relation_score_sample_csv('test', neg_rate), encoding='utf_8_sig', index=False) train_df.to_csv(Config.get_relation_score_sample_csv( 'train', neg_rate), encoding='utf_8_sig', index=False)