def main(): parser = argparse.ArgumentParser() parser.add_argument("--model", default="BILSTM", help="model in [HMM,CRF,BILSTM,BILSTM-CRF,ENSEMBLE]", type=str) params = vars(parser.parse_args()) """训练模型,评估结果""" # 读取数据 print("读取数据...") train_word_lists, train_tag_lists, word2id, tag2id = build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) if params["model"] == "HMM": # 训练评估HMM模型 print("正在训练评估HMM模型...") hmm_pred = hmm_train_eval((train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists), word2id, tag2id) elif params["model"] == "CRF": # 训练评估CRF模型 print("正在训练评估CRF模型...") crf_pred = crf_train_eval((train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists)) elif params["model"] == "BILSTM": # 训练评估BI-LSTM模型 print("正在训练评估BI-LSTM模型...") # LSTM模型训练的时候需要在word2id和tag2id加入PAD和UNK bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) lstm_pred = bilstm_train_and_eval((train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), bilstm_word2id, bilstm_tag2id, crf=False) elif params["model"] == "BILSTM-CRF": print("正在训练评估Bi-LSTM+CRF模型...") # 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到) crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) # 还需要额外的一些数据处理 train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf( train_word_lists, train_tag_lists) dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf( dev_word_lists, dev_tag_lists) test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred = bilstm_train_and_eval( (train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), crf_word2id, crf_tag2id) elif params["model"] == "ENSEMBLE": # 模型集成 print()
def main(): print("读取数据...") train_word_lists, train_tag_lists, word2id, tag2id = \ build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) dev_word_lists_, dev_word_lists_raw, article_id = loadDevFile("development_2.txt") print("加载并评估hmm模型...") hmm_model = load_model(HMM_MODEL_PATH) #hmm_pred = hmm_model.test(test_word_lists, # word2id, # tag2id) hmm_pred_dev = hmm_model.test(dev_word_lists_, word2id, tag2id) output_pred(hmm_pred_dev, article_id, dev_word_lists_raw) metrics = Metrics(test_tag_lists, hmm_pred, remove_O=REMOVE_O) metrics.report_scores() # 打印每个标记的精确度、召回率、f1分数 metrics.report_confusion_matrix() # 打印混淆矩阵 # 加载并评估CRF模型 print("加载并评估crf模型...") crf_model = load_model(CRF_MODEL_PATH) crf_pred = crf_model.test(test_word_lists) metrics = Metrics(test_tag_lists, crf_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() # bilstm模型 print("加载并评估bilstm模型...") bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) bilstm_model = load_model(BiLSTM_MODEL_PATH) bilstm_model.model.bilstm.flatten_parameters() # remove warning lstm_pred, target_tag_list = bilstm_model.test(test_word_lists, test_tag_lists, bilstm_word2id, bilstm_tag2id) metrics = Metrics(target_tag_list, lstm_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() print("加载并评估bilstm+crf模型...") crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) bilstm_model = load_model(BiLSTMCRF_MODEL_PATH) bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True ) lstmcrf_pred, target_tag_list = bilstm_model.test(test_word_lists, test_tag_lists, crf_word2id, crf_tag2id) metrics = Metrics(target_tag_list, lstmcrf_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() ensemble_evaluate( [hmm_pred, crf_pred, lstm_pred, lstmcrf_pred], test_tag_lists )
def main(): """训练模型,评估结果""" # 读取数据 print("读取数据...") data_folder = "./data123" train_word_lists, train_tag_lists, word2id, tag2id = \ build_corpus("train", data_dir=data_folder) dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False, data_dir=data_folder) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False, data_dir=data_folder) # 训练评估hmm模型 print("正在训练评估HMM模型...") hmm_pred = hmm_train_eval((train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists), word2id, tag2id) # 训练评估CRF模型 print("正在训练评估CRF模型...") crf_pred = crf_train_eval((train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists)) # 训练评估BI-LSTM模型 print("正在训练评估双向LSTM模型...") # LSTM模型训练的时候需要在word2id和tag2id加入PAD和UNK bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) lstm_pred = bilstm_train_and_eval((train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), bilstm_word2id, bilstm_tag2id, crf=False) print("正在训练评估Bi-LSTM+CRF模型...") # 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到) crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) # 还需要额外的一些数据处理 train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf( train_word_lists, train_tag_lists) dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf( dev_word_lists, dev_tag_lists) test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred = bilstm_train_and_eval( (train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), crf_word2id, crf_tag2id) ensemble_evaluate([hmm_pred, crf_pred, lstm_pred, lstmcrf_pred], test_tag_lists)
def decode(input_status, output_file): print("读取数据...") train_word_lists, train_tag_lists, word2id1, tag2id = build_corpus( "train", make_vocab=True) decode_word_lists, decode_tag_lists, word2id2, tag2id1 = build_corpus( input_status, make_vocab=True) lists = train_word_lists + decode_word_lists word2id = build_map(lists) print("加载并评估hmm模型...") hmm_model = load_model(HMM_MODEL_PATH) hmm_pred = hmm_model.test(decode_word_lists, word2id, tag2id) print("加载并评估crf模型...") crf_model = load_model(CRF_MODEL_PATH) crf_pred = crf_model.test(decode_word_lists) print("加载并评估bilstm模型...") bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) bilstm_model = load_model(BiLSTM_MODEL_PATH) bilstm_model.model.bilstm.flatten_parameters() # remove warning lstm_pred, target_tag_list = bilstm_model.test(decode_word_lists, decode_tag_lists, bilstm_word2id, bilstm_tag2id) print("加载并评估bilstm+crf模型...") crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) bilstm_model = load_model(BiLSTMCRF_MODEL_PATH) bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning decode_word_lists, decode_tag_lists = prepocess_data_for_lstmcrf( decode_word_lists, decode_tag_lists, test=True) lstmcrf_pred, target_tag_list = bilstm_model.test(decode_word_lists, decode_tag_lists, crf_word2id, crf_tag2id) print("加载并评估lattice lstm模型...") latticelstm_pred = run(status='decode') print("加载并评估ensemble模型...") predict_results = ensemble_evaluate(hmm_pred, crf_pred, lstm_pred, lstmcrf_pred, latticelstm_pred, decode_tag_lists, status='decode') print("输出解码结果...") write_decoded_results(output_file, predict_results, decode_word_lists)
def main(): print("Read data...") train_word_lists, train_tag_lists, word2id, tag2id = \ build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) print("Load and evaluate the hmm model...") hmm_model = load_model(HMM_MODEL_PATH) hmm_pred = hmm_model.test(test_word_lists, word2id, tag2id) metrics = Metrics(test_tag_lists, hmm_pred, remove_O=REMOVE_O) metrics.report_scores( ) # Print the accuracy of each mark, recall rate, f1 score metrics.report_confusion_matrix() #Print confusion matrix # Load and evaluate the CRF model print("Load and evaluate the crf model...") crf_model = load_model(CRF_MODEL_PATH) crf_pred = crf_model.test(test_word_lists) metrics = Metrics(test_tag_lists, crf_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() # bilstm Model print("Load and evaluate the bilstm model...") bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) bilstm_model = load_model(BiLSTM_MODEL_PATH) bilstm_model.model.bilstm.flatten_parameters() # remove warning lstm_pred, target_tag_list = bilstm_model.test(test_word_lists, test_tag_lists, bilstm_word2id, bilstm_tag2id) metrics = Metrics(target_tag_list, lstm_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() print("Load and evaluate the bilstm+crf model...") crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) bilstm_model = load_model(BiLSTMCRF_MODEL_PATH) bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred, target_tag_list = bilstm_model.test(test_word_lists, test_tag_lists, crf_word2id, crf_tag2id) metrics = Metrics(target_tag_list, lstmcrf_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() ensemble_evaluate([hmm_pred, crf_pred, lstm_pred, lstmcrf_pred], test_tag_lists)
def main(): print("读取数据...") train_word_lists, train_data_lists, train_wordlabel_lists, train_datalabel_lists, train_dataptr_lists, word2id, data2id = build_corpus( "train") dev_word_lists, dev_data_lists, dev_wordlabel_lists, dev_datalabel_lists, dev_dataptr_lists = build_corpus( "dev", make_vocab=False) test_word_lists, test_data_lists, test_wordlabel_lists, test_datalabel_lists, test_dataptr_lists = build_corpus( "test", make_vocab=False) # bilstm模型 print("加载并评估bilstm模型...") bilstm_word2id, bilstm_data2id = extend_maps(word2id, data2id, for_crf=False) bilstm_model = load_model(BiLSTM_MODEL_PATH) bilstm_model.model.bilstm.flatten_parameters() # remove warning lstm_pred, target_tag_list = bilstm_model.test( test_word_lists, test_data_lists, test_wordlabel_lists, test_datalabel_lists, test_dataptr_lists, bilstm_word2id, bilstm_data2id) allnum = 0 correct = 0 f = open('test.tgt.dataptr1', 'w') for pred, gold in zip(lstm_pred, target_tag_list): pred = pred.cpu().numpy().tolist()[:len(gold)] f.write(' '.join([str(x) for x in pred]) + '\n') for x, y in zip(pred, gold): if x == y: correct += 1 allnum += 1 f.close() # TODO print(correct / allnum)
def run_lstm_crf(train=True): train_word_lists, train_tag_lists = build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev") test_word_lists, test_tag_lists = build_corpus("test") word2id, tag2id = build_vocab(train_word_lists, train_tag_lists) # 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到) crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) # 还需要额外的一些数据处理 train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf( train_word_lists, train_tag_lists) dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf( dev_word_lists, dev_tag_lists) test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) if train: print("正在训练Bi-LSTM+CRF模型...") bilstm_crf_train((train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), crf_word2id, crf_tag2id) print("正在评估Bi-LSTM+CRF模型...") pred = bilstm_crf_eval((test_word_lists, test_tag_lists), crf_word2id, crf_tag2id) return pred
def main(): """训练模型,评估结果""" # 读取数据 print("读取数据...") train_word_lists, train_tag_lists, word2id, tag2id = \ build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) # 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到) crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) # 还需要额外的一些数据处理 train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf( train_word_lists, train_tag_lists) dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf( dev_word_lists, dev_tag_lists) test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) mode = 'train' if mode == 'train': # # 训练评估hmm模型 # print("正在训练评估HMM模型...") # hmm_pred = hmm_train_eval( # (train_word_lists, train_tag_lists), # (test_word_lists, test_tag_lists), # word2id, # tag2id # ) # # 训练评估CRF模型 # print("正在训练评估CRF模型...") # crf_pred = crf_train_eval( # (train_word_lists, train_tag_lists), # (test_word_lists, test_tag_lists) # ) # # 训练评估BI-LSTM模型 # print("正在训练评估双向LSTM模型...") # # LSTM模型训练的时候需要在word2id和tag2id加入PAD和UNK # bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) # lstm_pred = bilstm_train_and_eval( # (train_word_lists, train_tag_lists), # (dev_word_lists, dev_tag_lists), # (test_word_lists, test_tag_lists), # bilstm_word2id, bilstm_tag2id, # crf=False # ) print("正在训练评估Bi-LSTM+CRF模型...") lstmcrf_pred = bilstm_train_and_eval( (train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), crf_word2id, crf_tag2id) elif mode == 'generate': lstmcrf_pred = bilstm_eval((test_word_lists, test_tag_lists), crf_word2id, crf_tag2id) print(lstmcrf_pred)
def main(): print("读取数据...") train_word_lists, train_tag_lists, word2id, tag2id = build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) word_string = "傅城州,博士,广东药科大学医药信息工程学院教师,中国计算机学会会员,中国计算机学会青年计算机科技论坛(YOCSEF)广州分论坛AC委员、学术秘书。本科和硕士(推免)毕业于华南师范大学计算机学院软件工程专业,2017年6月获得华南师范大学服务计算理论与技术理学博士学位(导师:汤庸教授)。" word_list = [] test_word_lists = [] for word in word_string: word_list.append(word) test_word_lists.append(word_list) print("加载并评估bilstm+crf模型...") crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) bilstm_model = load_model(BiLSTMCRF_MODEL_PATH) bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning # test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( # test_word_lists, test_tag_lists, test=True # ) print(crf_tag2id) id2tag = {v: k for k, v in crf_tag2id.items()} lstmcrf_pred = bilstm_model.test(test_word_lists, crf_word2id, crf_tag2id) for word_list, tag_list in zip(test_word_lists, lstmcrf_pred): for word, tag in zip(word_list, tag_list): print(word, " ", tag.item(), " ", id2tag[tag.item()]) print(lstmcrf_pred)
def main(): """模型训练与评估""" # 读取数据 print("读取数据中...") train_word_lists, train_tag_lists, word2id, tag2id = build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) #训练并评估hmm模型 print("正在训练评估HMM模型") hmm_pred = hmm_train_eval((train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists), word2id, tag2id) # 训练并评估crf模型 crf_pred = crf_train_eval((train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists)) #训练并评估bilstm模型 bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) lstm_pred = bilstm_train_and_eval((train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), bilstm_word2id, bilstm_tag2id, crf=False) print("正在训练评估Bi-LSTM+CRF模型...") # 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到) crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) print(' '.join([i[0] for i in crf_tag2id.items()])) # 还需要额外的一些数据处理 train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf( train_word_lists, train_tag_lists) dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf( dev_word_lists, dev_tag_lists) test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred = bilstm_train_and_eval( (train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), crf_word2id, crf_tag2id) ensemble_evaluate([hmm_pred, crf_pred, lstm_pred, lstmcrf_pred], test_tag_lists)
def main(): print("读取数据...") train_word_lists, train_tag_lists, word2id, tag2id = \ build_corpus("train") # dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) # test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("train", make_vocab=False) # print("加载并评估hmm模型...") # hmm_model = load_model(HMM_MODEL_PATH) # hmm_pred = hmm_model.test(test_word_lists, # word2id, # tag2id) # metrics = Metrics(test_tag_lists, hmm_pred, remove_O=REMOVE_O) # metrics.report_scores() # 打印每个标记的精确度、召回率、f1分数 # metrics.report_confusion_matrix() # 打印混淆矩阵 # 加载并评估CRF模型 # print("加载并评估crf模型...") # crf_model = load_model(CRF_MODEL_PATH) # crf_pred = crf_model.test(test_word_lists) # metrics = Metrics(test_tag_lists, crf_pred, remove_O=REMOVE_O) # metrics.report_scores() # metrics.report_confusion_matrix() # bilstm模型 # print("加载并评估bilstm模型...") # bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) # bilstm_model = load_model(BiLSTM_MODEL_PATH) # bilstm_model.model.bilstm.flatten_parameters() # remove warning # lstm_pred, target_tag_list = bilstm_model.test(test_word_lists, test_tag_lists, # bilstm_word2id, bilstm_tag2id) # metrics = Metrics(target_tag_list, lstm_pred, remove_O=REMOVE_O) # metrics.report_scores() # metrics.report_confusion_matrix() print("加载并评估bilstm+crf模型...") crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) bilstm_model = load_model(BiLSTMCRF_MODEL_PATH) bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning test_word_lists = test_word_lists[:10] test_tag_lists = test_tag_lists[:10] test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred, target_tag_list = bilstm_model.test(test_word_lists, test_tag_lists, crf_word2id, crf_tag2id) print(target_tag_list) print(lstmcrf_pred) metrics = Metrics(target_tag_list, lstmcrf_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix()
def run_lstm(train=True): train_word_lists, train_tag_lists = build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev") test_word_lists, test_tag_lists = build_corpus("test") word2id, tag2id = build_vocab(train_word_lists, train_tag_lists) # LSTM模型训练的时候需要在word2id和tag2id加入PAD和UNK bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) if train: print("正在训练双向LSTM模型...") bilstm_train((train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), bilstm_word2id, bilstm_tag2id) print("正在评估双向LSTM模型...") pred = bilstm_eval((test_word_lists, test_tag_lists), bilstm_word2id, bilstm_tag2id) return pred
def main(): print("读取数据...") train_word_lists, train_data_lists, train_wordlabel_lists, train_datalabel_lists, train_dataptr_lists, word2id, data2id = build_corpus( "train") dev_word_lists, dev_data_lists, dev_wordlabel_lists, dev_datalabel_lists, dev_dataptr_lists = build_corpus( "dev", make_vocab=False) test_word_lists, test_data_lists, test_wordlabel_lists, test_datalabel_lists, test_dataptr_lists = build_corpus( "test", make_vocab=False) # # 训练评估hmm模型 # print("正在训练评估HMM模型...") # hmm_pred = hmm_train_eval( # (train_word_lists, train_tag_lists), # (test_word_lists, test_tag_lists), # word2id, # tag2id # ) # # 训练评估CRF模型 # print("正在训练评估CRF模型...") # crf_pred = crf_train_eval( # (train_word_lists, train_tag_lists), # (test_word_lists, test_tag_lists) # ) # 训练评估BI-LSTM模型 print("正在训练评估双向LSTM模型...") # LSTM模型训练的时候需要在word2id和tag2id加入PAD和UNK bilstm_word2id, bilstm_data2id = extend_maps(word2id, data2id, for_crf=False) lstm_pred = bilstm_train_and_eval( (train_word_lists, train_data_lists, train_wordlabel_lists, train_datalabel_lists, train_dataptr_lists), (dev_word_lists, dev_data_lists, dev_wordlabel_lists, dev_datalabel_lists, dev_dataptr_lists), (test_word_lists, test_data_lists, test_wordlabel_lists, test_datalabel_lists, test_dataptr_lists), bilstm_word2id, bilstm_data2id, crf=False)
def main(corpus_dir, test_file): # 读取数据 # print("读取数据...") train_word_lists, train_tag_lists, word2id, tag2id = \ build_corpus("train", data_dir=corpus_dir) test_word_lists, test_tag_lists = build_corpus_test(test_file) # print(test_word_lists, test_tag_lists) # print("加载并评估bilstm+crf模型...") crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) bilstm_model = load_model(BiLSTMCRF_MODEL_PATH) bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred, target_tag_list = bilstm_model.test(test_word_lists, test_tag_lists, crf_word2id, crf_tag2id) for i in range(len(lstmcrf_pred)): for j in range(len(lstmcrf_pred[i])): print(test_word_lists[i][j] + "\t" + lstmcrf_pred[i][j]) print()
''' @Autor: xujiahuan @Date: 2020-05-06 11:00:17 @LastEditors: xujiahuan @LastEditTime: 2020-05-17 16:51:08 ''' from sklearn.externals import joblib import json from utils import extend_maps test_word_lists = [['上', '海', '浦', '东']] with open('tag2id.json') as f: tag2id = json.load(f) with open('word2id.json', encoding='utf-8') as f: word2id = json.load(f) bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=True) path = "ckpts/lstm_crf.pkl" model = joblib.load(path) test_tag_lists = [] length = len(test_word_lists[0]) temp = ['O' for i in range(length)] test_tag_lists.append(temp) pred_tag_lists, test_tag_lists = model.test(test_word_lists, test_tag_lists, word2id, tag2id) print(pred_tag_lists)
from utils import load_model, extend_maps, prepocess_data_for_lstmcrf from data import build_corpus import glob HMM_MODEL_PATH = './ckpts/hmm.pkl' CRF_MODEL_PATH = './ckpts/crf.pkl' BiLSTM_MODEL_PATH = './ckpts/bilstm.pkl' BiLSTMCRF_MODEL_PATH = './ckpts/bilstm_crf.pkl' print("读取数据...") train_word_lists, train_tag_lists, word2id, tag2id = \ build_corpus("train") print("加载并评估bilstm+crf模型...") crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) bilstm_model = load_model(BiLSTMCRF_MODEL_PATH) bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning for item in glob.glob("data/chusai_xuanshou/" + "*.txt"): with open(item, encoding="utf-8") as f: origin_text = "".join(f.readlines()) test_word_list = list(origin_text) test_word_lists = [test_word_list] test_tag_list = ["N" for _ in test_word_list] test_tag_lists = [test_tag_list] test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True ) lstmcrf_pred = bilstm_model.testA(test_word_lists, crf_word2id, crf_tag2id) print(test_word_list)
def main(): """训练模型,评估结果""" # 读取数据 print("读取数据...") # train_word_lists, train_tag_lists, word2id, tag2id = \ # build_corpus("train") # dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) # test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) # 训练评估hmm模型 # print("正在训练评估HMM模型...") with open('corpus_word2id', 'rb') as f: word2id = pickle.load(f) with open('tag2id', 'rb') as f: tag2id = pickle.load(f) # print('word2id',word2id) # print('tag2id',tag2id) train_word_lists = [] train_tag_lists = [] f = open('normal_daguan_train.txt') datas = f.readlines() f.close() for data in datas: temps = data.strip().split('|||') words = temps[0] tags = temps[1] train_word_lists.append(words.split(' ')) train_tag_lists.append(tags.split(' ')) test_word_lists = [] test_tag_lists = [] f = open('normal_daguan_test.txt') datas = f.readlines() f.close() for data in datas: words = data.strip() test_word_lists.append(words.split(' ')) test_tag_lists.append([None for i in range(len(words))]) '''前3000数据集划分''' dev_word_lists = train_word_lists[:1500] dev_tag_lists = train_tag_lists[:1500] # test_word_lists = train_word_lists[1500:3000] # test_tag_lists = train_tag_lists[1500:3000] train_word_lists = train_word_lists[1500:] train_tag_lists = train_tag_lists[1500:] '''后3000数据集划分''' # dev_word_lists = train_word_lists[-3000:-1500] # dev_tag_lists = train_tag_lists[-3000:-1500] # test_word_lists = train_word_lists[-1500:] # test_tag_lists = train_tag_lists[-1500:] # train_word_lists = train_word_lists[:-3000] # train_tag_lists = train_tag_lists[:-3000] # print(train_word_lists[0]) # print(train_tag_lists[0]) # print(test_word_lists[0]) # print(len(train_tag_lists) # print("正在训练评估Bi-LSTM+CRF模型...") # 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到) crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) # 还需要额外的一些数据处理 train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf( train_word_lists, train_tag_lists) dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf( dev_word_lists, dev_tag_lists, ) test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, ) model = Word2Vec.load('charEmbedding_300dim') weights = np.zeros(shape=(len(crf_word2id), 300)) for word in word2id.keys(): if word in model.wv.vocab: weights[word2id[word]] = model[word] else: weights[word2id[word]] = np.random.uniform(-0.1, 0.1, size=(300, )) lstmcrf_pred = bilstm_train_and_eval( (train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), # (dev_word_lists, dev_tag_lists), weights, crf_word2id, crf_tag2id)
def main(args): """训练模型,评估结果""" output_directory = os.path.join('ckpts', args.name) if not os.path.isdir(output_directory): os.makedirs(output_directory) os.chmod(output_directory, 0o775) shutil.copy2('models/config.py', output_directory) # 读取数据 print("读取数据...") train_word_lists, train_tag_lists, word2id, tag2id = build_corpus("train", fix_length=-1) dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) # 训练评估HMM模型 print("正在训练评估HMM模型...") hmm_pred = hmm_train_eval( (train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists), word2id, tag2id, output_directory ) # 训练评估CRF模型 print("正在训练评估CRF模型...") crf_pred = crf_train_eval( (train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists), output_directory ) # 训练评估BI-LSTM模型 print("正在训练评估双向LSTM模型...") # LSTM模型训练的时候需要在word2id和tag2id加入PAD和UNK bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) lstm_pred = bilstm_train_and_eval( (train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), bilstm_word2id, bilstm_tag2id, output_directory, crf=False ) print("正在训练评估Bi-LSTM+CRF模型...") # 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到) crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) # 还需要额外的一些数据处理 train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf( train_word_lists, train_tag_lists ) dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf( dev_word_lists, dev_tag_lists ) test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True ) lstmcrf_pred = bilstm_train_and_eval( (train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), crf_word2id, crf_tag2id, output_directory ) ensemble_evaluate( [hmm_pred, crf_pred, lstm_pred, lstmcrf_pred], test_tag_lists )
def main_rep1(x, y): if x == 'train': # select data according to args.process print("Read data...") train_word_lists, train_tag_lists, word2id, tag2id = \ build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) ###### if y == 'crf': crf_pred = crf_train_eval((train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists)) ensemble_evaluate([crf_pred], test_tag_lists) elif y == 'bilstm': bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) lstm_pred = bilstm_train_and_eval( (train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), bilstm_word2id, bilstm_tag2id, crf=False) ensemble_evaluate([lstm_pred], test_tag_lists) elif y == 'bilstm-crf': crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) # more data processing train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf( train_word_lists, train_tag_lists) dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf( dev_word_lists, dev_tag_lists) test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred = bilstm_train_and_eval( (train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), crf_word2id, crf_tag2id) ensemble_evaluate([lstmcrf_pred], test_tag_lists) else: HMM_MODEL_PATH = './ckpts/hmm.pkl' CRF_MODEL_PATH = './ckpts/crf.pkl' BiLSTM_MODEL_PATH = './ckpts/bilstm.pkl' BiLSTMCRF_MODEL_PATH = './ckpts/bilstm_crf.pkl' REMOVE_O = False # Whether to remove the O mark at the time of evaluation # select data according to args.process print("Read data...") train_word_lists, train_tag_lists, word2id, tag2id = \ build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) if y == 'crf': crf_model = load_model_1(CRF_MODEL_PATH) crf_pred = crf_model.test(test_word_lists) metrics = Metrics(test_tag_lists, crf_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() elif y == 'bilstm': bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) bilstm_model = load_model_1(BiLSTM_MODEL_PATH) bilstm_model.model.bilstm.flatten_parameters() # remove warning lstm_pred, target_tag_list = bilstm_model.test( test_word_lists, test_tag_lists, bilstm_word2id, bilstm_tag2id) metrics = Metrics(target_tag_list, lstm_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() elif y == 'bilstm-crf': crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) bilstm_model = load_model_1(BiLSTMCRF_MODEL_PATH) bilstm_model.model.bilstm.bilstm.flatten_parameters( ) # remove warning test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred, target_tag_list = bilstm_model.test( test_word_lists, test_tag_lists, crf_word2id, crf_tag2id) metrics = Metrics(target_tag_list, lstmcrf_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() exit()
def main(): """训练模型,评估结果""" # 读取数据 print("读取数据...") train_word_lists, train_tag_lists, word2id, tag2id = build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) #test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) test_word_lists, test_tag_lists, article_id = loadDevFile( "development_2.txt") # # 训练评估hmm模型 # # print("正在训练评估HMM模型...") # # hmm_pred = hmm_train_eval( # # (train_word_lists, train_tag_lists), # # (dev_word_lists_, test_tag_lists), # # word2id, # # tag2id, # # remove_O=True # # ) # # output_pred(hmm_pred, article_id, dev_word_lists_raw) # # 训练评估CRF模型 # print("正在训练评估CRF模型...") # crf_pred = crf_train_eval( # (train_word_lists, train_tag_lists), # (dev_word_lists_, test_tag_lists), # remove_O=True # ) # output_pred(crf_pred, article_id, dev_word_lists_raw,output_path = 'output_crf.tsv') # 训练评估BI-LSTM模型 print("正在训练评估双向LSTM模型...") # LSTM模型训练的时候需要在word2id和tag2id加入PAD和UNK bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) lstm_pred = bilstm_train_and_eval((train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), bilstm_word2id, bilstm_tag2id, crf=False, remove_O=True) print("正在训练评估Bi-LSTM+CRF模型...") # 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到) crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) # 还需要额外的一些数据处理 train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf( train_word_lists, train_tag_lists) dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf( dev_word_lists, dev_tag_lists) test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred = bilstm_train_and_eval((train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), crf_word2id, crf_tag2id, remove_O=True) ensemble_evaluate([hmm_pred, crf_pred, lstm_pred, lstmcrf_pred], test_tag_lists)
def main(): print("读取数据...") train_word_lists, train_tag_lists, word2id, tag2id = \ build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) # print("加载并评估hmm模型...") # hmm_model = load_model(HMM_MODEL_PATH) # hmm_pred = hmm_model.test(test_word_lists, # word2id, # tag2id) # metrics = Metrics(test_tag_lists, hmm_pred, remove_O=REMOVE_O) # metrics.report_scores() # 打印每个标记的精确度、召回率、f1分数 # metrics.report_confusion_matrix() # 打印混淆矩阵 # # # 加载并评估CRF模型 # print("加载并评估crf模型...") # crf_model = load_model(CRF_MODEL_PATH) # crf_pred = crf_model.test(test_word_lists) # metrics = Metrics(test_tag_lists, crf_pred, remove_O=REMOVE_O) # metrics.report_scores() # metrics.report_confusion_matrix() # # # bilstm模型 # print("加载并评估bilstm模型...") # bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) # bilstm_model = load_model(BiLSTM_MODEL_PATH) # bilstm_model.model.bilstm.flatten_parameters() # remove warning # lstm_pred, target_tag_list = bilstm_model.test(test_word_lists, test_tag_lists, # bilstm_word2id, bilstm_tag2id) # metrics = Metrics(target_tag_list, lstm_pred, remove_O=REMOVE_O) # metrics.report_scores() # metrics.report_confusion_matrix() print("加载并评估bilstm+crf模型...") crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) bilstm_model = load_model(BiLSTMCRF_MODEL_PATH) bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred, target_tag_list = bilstm_model.test(test_word_lists, test_tag_lists, crf_word2id, crf_tag2id) metrics = Metrics(target_tag_list, lstmcrf_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() # ensemble_evaluate( # [hmm_pred, crf_pred, lstm_pred, lstmcrf_pred], # test_tag_lists # ) ls = ['B-SYM', 'M-SYM', 'E-SYM'] selected = [ i for i in range(len(test_tag_lists[0])) if test_tag_lists[0][i] in ls ] selected_word = [test_word_lists[0][i] for i in selected] selected_predict = [ i for i in range(len(lstmcrf_pred[0])) if lstmcrf_pred[0][i] in ls ] selected_predict_word = [test_word_lists[0][i] for i in selected_predict] for tag_list, doc in zip(train_tag_lists, train_word_lists): selected_train = [i for i in range(len(tag_list)) if tag_list[i] in ls] selected_train_word = [doc[i] for i in selected_train] # print(selected_train_word) print('preditct list:', lstmcrf_pred) print('target list:', target_tag_list) print(selected_word) print(selected_predict_word)
def main(): import argparse parser = argparse.ArgumentParser(description='main.py') parser.add_argument('--hmm', action='store_true', default=False, help='Test HMM') parser.add_argument('--crf', action='store_true', default=False, help='Test CRF') parser.add_argument('--bilstm', action='store_true', default=False, help='Test BiLSTM') parser.add_argument('--bilstm-crf', action='store_true', default=False, help='Test BiLSTM-CRF') parser.add_argument('--cbow', action='store_true', default=False, help='Use CBOW embedding for BiLSTM-CRF') args = parser.parse_args() print("读取数据...") train_word_lists, train_tag_lists, word2id, tag2id = \ build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) if args.hmm: print("加载并评估hmm模型...") hmm_model = load_model(HMM_MODEL_PATH) hmm_pred = hmm_model.test(test_word_lists, word2id, tag2id) metrics = Metrics(test_tag_lists, hmm_pred, remove_O=REMOVE_O) metrics.report_scores() # 打印每个标记的精确度、召回率、f1分数 metrics.report_confusion_matrix() # 打印混淆矩阵 # 加载并评估CRF模型 if args.crf: print("加载并评估crf模型...") crf_model = load_model(CRF_MODEL_PATH) crf_pred = crf_model.test(test_word_lists) metrics = Metrics(test_tag_lists, crf_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() # bilstm模型 if args.bilstm: print("加载并评估bilstm模型...") bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) bilstm_model = load_model(BiLSTM_MODEL_PATH) bilstm_model.model.bilstm.flatten_parameters() # remove warning lstm_pred, target_tag_list = bilstm_model.test(test_word_lists, test_tag_lists, bilstm_word2id, bilstm_tag2id) metrics = Metrics(target_tag_list, lstm_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() if args.bilstm_crf: print("加载并评估bilstm+crf模型...") crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) bilstm_model = load_model(BiLSTMCRF_MODEL_PATH) bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred, target_tag_list = bilstm_model.test( test_word_lists, test_tag_lists, crf_word2id, crf_tag2id) metrics = Metrics(target_tag_list, lstmcrf_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix()
def main(): import argparse parser = argparse.ArgumentParser(description='main.py') parser.add_argument('--hmm', action='store_true', default=False, help='Train HMM') parser.add_argument('--crf', action='store_true', default=False, help='Train CRF') parser.add_argument('--bilstm', action='store_true', default=False, help='Train BiLSTM') parser.add_argument('--bilstm-crf', action='store_true', default=False, help='Train BiLSTM-CRF') parser.add_argument('--cbow', action='store_true', default=False, help='Train or use CBOW embedding for BiLSTM-CRF') args = parser.parse_args() """训练模型,评估结果""" # 读取数据 print("读取数据...") train_word_lists, train_tag_lists, word2id, tag2id = \ build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) # 训练评估hmm模型 if args.hmm: print("正在训练评估HMM模型...") hmm_pred = hmm_train_eval((train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists), word2id, tag2id) # 训练评估CRF模型 if args.crf: print("正在训练评估CRF模型...") crf_pred = crf_train_eval((train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists)) if args.bilstm: # 训练评估BI-LSTM模型 print("正在训练评估双向LSTM模型...") # LSTM模型训练的时候需要在word2id和tag2id加入PAD和UNK bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) lstm_pred = bilstm_train_and_eval((train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), bilstm_word2id, bilstm_tag2id, crf=False) # 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到) crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) # 还需要额外的一些数据处理 train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf( train_word_lists, train_tag_lists) dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf( dev_word_lists, dev_tag_lists) test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) if args.bilstm_crf: print("正在训练评估Bi-LSTM+CRF模型...") cbow_emb = None if args.cbow: print('Loading CBOW model') cbow_model = load_model('ckpts/cbow.pkl') cbow_emb = cbow_model.model.lookup_embedding() del cbow_model lstmcrf_pred = bilstm_train_and_eval( (train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), crf_word2id, crf_tag2id, cbow_emb=cbow_emb) elif args.cbow: print("正在训练CBOW模型...") cbow.CBOW_Model(len(crf_word2id)).train(train_word_lists, crf_word2id)