def main():
"""训练模型，评估结果"""
# 读取数据
print("读取数据...")
train_word_lists, train_tag_lists, word2id, tag2id = \
build_corpus("train")
dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False)
test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False)
# 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到)
crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True)
# 还需要额外的一些数据处理
train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf(
train_word_lists, train_tag_lists)
dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf(
dev_word_lists, dev_tag_lists)
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(
test_word_lists, test_tag_lists, test=True)
mode = 'train'
if mode == 'train':
# # 训练评估ｈｍｍ模型
# print("正在训练评估HMM模型...")
# hmm_pred = hmm_train_eval(
# (train_word_lists, train_tag_lists),
# (test_word_lists, test_tag_lists),
# word2id,
# tag2id
# )
# # 训练评估CRF模型
# print("正在训练评估CRF模型...")
# crf_pred = crf_train_eval(
# (train_word_lists, train_tag_lists),
# (test_word_lists, test_tag_lists)
# )
# # 训练评估BI-LSTM模型
# print("正在训练评估双向LSTM模型...")
# # LSTM模型训练的时候需要在word2id和tag2id加入PAD和UNK
# bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False)
# lstm_pred = bilstm_train_and_eval(
# (train_word_lists, train_tag_lists),
# (dev_word_lists, dev_tag_lists),
# (test_word_lists, test_tag_lists),
# bilstm_word2id, bilstm_tag2id,
# crf=False
# )
print("正在训练评估Bi-LSTM+CRF模型...")
lstmcrf_pred = bilstm_train_and_eval(
(train_word_lists, train_tag_lists),
(dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists),
crf_word2id, crf_tag2id)
elif mode == 'generate':
lstmcrf_pred = bilstm_eval((test_word_lists, test_tag_lists),
crf_word2id, crf_tag2id)
print(lstmcrf_pred)
def run_lstm_crf(train=True):
train_word_lists, train_tag_lists = build_corpus("train")
dev_word_lists, dev_tag_lists = build_corpus("dev")
test_word_lists, test_tag_lists = build_corpus("test")
word2id, tag2id = build_vocab(train_word_lists, train_tag_lists)
# 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到)
crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True)
# 还需要额外的一些数据处理
train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf(
train_word_lists, train_tag_lists)
dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf(
dev_word_lists, dev_tag_lists)
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(
test_word_lists, test_tag_lists, test=True)
if train:
print("正在训练Bi-LSTM+CRF模型...")
bilstm_crf_train((train_word_lists, train_tag_lists),
(dev_word_lists, dev_tag_lists), crf_word2id,
crf_tag2id)
print("正在评估Bi-LSTM+CRF模型...")
pred = bilstm_crf_eval((test_word_lists, test_tag_lists), crf_word2id,
crf_tag2id)
return pred
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--model",
default="BILSTM",
help="model in [HMM,CRF,BILSTM,BILSTM-CRF,ENSEMBLE]",
type=str)
params = vars(parser.parse_args())
"""训练模型，评估结果"""
# 读取数据
print("读取数据...")
train_word_lists, train_tag_lists, word2id, tag2id = build_corpus("train")
dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False)
test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False)
if params["model"] == "HMM":
# 训练评估HMM模型
print("正在训练评估HMM模型...")
hmm_pred = hmm_train_eval((train_word_lists, train_tag_lists),
(test_word_lists, test_tag_lists), word2id,
tag2id)
elif params["model"] == "CRF":
# 训练评估CRF模型
print("正在训练评估CRF模型...")
crf_pred = crf_train_eval((train_word_lists, train_tag_lists),
(test_word_lists, test_tag_lists))
elif params["model"] == "BILSTM":
# 训练评估BI-LSTM模型
print("正在训练评估BI-LSTM模型...")
# LSTM模型训练的时候需要在word2id和tag2id加入PAD和UNK
bilstm_word2id, bilstm_tag2id = extend_maps(word2id,
tag2id,
for_crf=False)
lstm_pred = bilstm_train_and_eval((train_word_lists, train_tag_lists),
(dev_word_lists, dev_tag_lists),
(test_word_lists, test_tag_lists),
bilstm_word2id,
bilstm_tag2id,
crf=False)
elif params["model"] == "BILSTM-CRF":
print("正在训练评估Bi-LSTM+CRF模型...")
# 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到)
crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True)
# 还需要额外的一些数据处理
train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf(
train_word_lists, train_tag_lists)
dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf(
dev_word_lists, dev_tag_lists)
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(
test_word_lists, test_tag_lists, test=True)
lstmcrf_pred = bilstm_train_and_eval(
(train_word_lists, train_tag_lists),
(dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists),
crf_word2id, crf_tag2id)
elif params["model"] == "ENSEMBLE":
# 模型集成
print()
def main():
"""训练模型，评估结果"""
# 读取数据
print("读取数据...")
data_folder = "./data123"
train_word_lists, train_tag_lists, word2id, tag2id = \
build_corpus("train", data_dir=data_folder)
dev_word_lists, dev_tag_lists = build_corpus("dev",
make_vocab=False,
data_dir=data_folder)
test_word_lists, test_tag_lists = build_corpus("test",
make_vocab=False,
data_dir=data_folder)
# 训练评估ｈｍｍ模型
print("正在训练评估HMM模型...")
hmm_pred = hmm_train_eval((train_word_lists, train_tag_lists),
(test_word_lists, test_tag_lists), word2id,
tag2id)
# 训练评估CRF模型
print("正在训练评估CRF模型...")
crf_pred = crf_train_eval((train_word_lists, train_tag_lists),
(test_word_lists, test_tag_lists))
# 训练评估BI-LSTM模型
print("正在训练评估双向LSTM模型...")
# LSTM模型训练的时候需要在word2id和tag2id加入PAD和UNK
bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False)
lstm_pred = bilstm_train_and_eval((train_word_lists, train_tag_lists),
(dev_word_lists, dev_tag_lists),
(test_word_lists, test_tag_lists),
bilstm_word2id,
bilstm_tag2id,
crf=False)
print("正在训练评估Bi-LSTM+CRF模型...")
# 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到)
crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True)
# 还需要额外的一些数据处理
train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf(
train_word_lists, train_tag_lists)
dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf(
dev_word_lists, dev_tag_lists)
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(
test_word_lists, test_tag_lists, test=True)
lstmcrf_pred = bilstm_train_and_eval(
(train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists),
(test_word_lists, test_tag_lists), crf_word2id, crf_tag2id)
ensemble_evaluate([hmm_pred, crf_pred, lstm_pred, lstmcrf_pred],
test_tag_lists)
def main():
"""模型训练与评估"""
# 读取数据
print("读取数据中...")
train_word_lists, train_tag_lists, word2id, tag2id = build_corpus("train")
dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False)
test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False)
#训练并评估hmm模型
print("正在训练评估HMM模型")
hmm_pred = hmm_train_eval((train_word_lists, train_tag_lists),
(test_word_lists, test_tag_lists), word2id,
tag2id)
# 训练并评估crf模型
crf_pred = crf_train_eval((train_word_lists, train_tag_lists),
(test_word_lists, test_tag_lists))
#训练并评估bilstm模型
bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False)
lstm_pred = bilstm_train_and_eval((train_word_lists, train_tag_lists),
(dev_word_lists, dev_tag_lists),
(test_word_lists, test_tag_lists),
bilstm_word2id,
bilstm_tag2id,
crf=False)
print("正在训练评估Bi-LSTM+CRF模型...")
# 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到)
crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True)
print(' '.join([i[0] for i in crf_tag2id.items()]))
# 还需要额外的一些数据处理
train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf(
train_word_lists, train_tag_lists)
dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf(
dev_word_lists, dev_tag_lists)
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(
test_word_lists, test_tag_lists, test=True)
lstmcrf_pred = bilstm_train_and_eval(
(train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists),
(test_word_lists, test_tag_lists), crf_word2id, crf_tag2id)
ensemble_evaluate([hmm_pred, crf_pred, lstm_pred, lstmcrf_pred],
test_tag_lists)
def main():
print("读取数据...")
train_word_lists, train_tag_lists, word2id, tag2id = \
build_corpus("train")
dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False)
test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False)
dev_word_lists_, dev_word_lists_raw, article_id = loadDevFile("development_2.txt")
print("加载并评估hmm模型...")
hmm_model = load_model(HMM_MODEL_PATH)
#hmm_pred = hmm_model.test(test_word_lists,
# word2id,
# tag2id)
hmm_pred_dev = hmm_model.test(dev_word_lists_,
word2id,
tag2id)
output_pred(hmm_pred_dev, article_id, dev_word_lists_raw)
metrics = Metrics(test_tag_lists, hmm_pred, remove_O=REMOVE_O)
metrics.report_scores() # 打印每个标记的精确度、召回率、f1分数
metrics.report_confusion_matrix() # 打印混淆矩阵
# 加载并评估CRF模型
print("加载并评估crf模型...")
crf_model = load_model(CRF_MODEL_PATH)
crf_pred = crf_model.test(test_word_lists)
metrics = Metrics(test_tag_lists, crf_pred, remove_O=REMOVE_O)
metrics.report_scores()
metrics.report_confusion_matrix()
# bilstm模型
print("加载并评估bilstm模型...")
bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False)
bilstm_model = load_model(BiLSTM_MODEL_PATH)
bilstm_model.model.bilstm.flatten_parameters() # remove warning
lstm_pred, target_tag_list = bilstm_model.test(test_word_lists, test_tag_lists,
bilstm_word2id, bilstm_tag2id)
metrics = Metrics(target_tag_list, lstm_pred, remove_O=REMOVE_O)
metrics.report_scores()
metrics.report_confusion_matrix()
print("加载并评估bilstm+crf模型...")
crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True)
bilstm_model = load_model(BiLSTMCRF_MODEL_PATH)
bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(
test_word_lists, test_tag_lists, test=True
)
lstmcrf_pred, target_tag_list = bilstm_model.test(test_word_lists, test_tag_lists,
crf_word2id, crf_tag2id)
metrics = Metrics(target_tag_list, lstmcrf_pred, remove_O=REMOVE_O)
metrics.report_scores()
metrics.report_confusion_matrix()
ensemble_evaluate(
[hmm_pred, crf_pred, lstm_pred, lstmcrf_pred],
test_tag_lists
)
def main():
print("读取数据...")
train_word_lists, train_tag_lists, word2id, tag2id = \
build_corpus("train")
# dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False)
# test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False)
test_word_lists, test_tag_lists = build_corpus("train", make_vocab=False)
# print("加载并评估hmm模型...")
# hmm_model = load_model(HMM_MODEL_PATH)
# hmm_pred = hmm_model.test(test_word_lists,
# word2id,
# tag2id)
# metrics = Metrics(test_tag_lists, hmm_pred, remove_O=REMOVE_O)
# metrics.report_scores() # 打印每个标记的精确度、召回率、f1分数
# metrics.report_confusion_matrix() # 打印混淆矩阵
# 加载并评估CRF模型
# print("加载并评估crf模型...")
# crf_model = load_model(CRF_MODEL_PATH)
# crf_pred = crf_model.test(test_word_lists)
# metrics = Metrics(test_tag_lists, crf_pred, remove_O=REMOVE_O)
# metrics.report_scores()
# metrics.report_confusion_matrix()
# bilstm模型
# print("加载并评估bilstm模型...")
# bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False)
# bilstm_model = load_model(BiLSTM_MODEL_PATH)
# bilstm_model.model.bilstm.flatten_parameters() # remove warning
# lstm_pred, target_tag_list = bilstm_model.test(test_word_lists, test_tag_lists,
# bilstm_word2id, bilstm_tag2id)
# metrics = Metrics(target_tag_list, lstm_pred, remove_O=REMOVE_O)
# metrics.report_scores()
# metrics.report_confusion_matrix()
print("加载并评估bilstm+crf模型...")
crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True)
bilstm_model = load_model(BiLSTMCRF_MODEL_PATH)
bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning
test_word_lists = test_word_lists[:10]
test_tag_lists = test_tag_lists[:10]
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(
test_word_lists, test_tag_lists, test=True)
lstmcrf_pred, target_tag_list = bilstm_model.test(test_word_lists,
test_tag_lists,
crf_word2id, crf_tag2id)
print(target_tag_list)
print(lstmcrf_pred)
metrics = Metrics(target_tag_list, lstmcrf_pred, remove_O=REMOVE_O)
metrics.report_scores()
metrics.report_confusion_matrix()
def main():
print("Read data...")
train_word_lists, train_tag_lists, word2id, tag2id = \
build_corpus("train")
dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False)
test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False)
print("Load and evaluate the hmm model...")
hmm_model = load_model(HMM_MODEL_PATH)
hmm_pred = hmm_model.test(test_word_lists, word2id, tag2id)
metrics = Metrics(test_tag_lists, hmm_pred, remove_O=REMOVE_O)
metrics.report_scores(
) # Print the accuracy of each mark, recall rate, f1 score
metrics.report_confusion_matrix() #Print confusion matrix
# Load and evaluate the CRF model
print("Load and evaluate the crf model...")
crf_model = load_model(CRF_MODEL_PATH)
crf_pred = crf_model.test(test_word_lists)
metrics = Metrics(test_tag_lists, crf_pred, remove_O=REMOVE_O)
metrics.report_scores()
metrics.report_confusion_matrix()
# bilstm Model
print("Load and evaluate the bilstm model...")
bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False)
bilstm_model = load_model(BiLSTM_MODEL_PATH)
bilstm_model.model.bilstm.flatten_parameters() # remove warning
lstm_pred, target_tag_list = bilstm_model.test(test_word_lists,
test_tag_lists,
bilstm_word2id,
bilstm_tag2id)
metrics = Metrics(target_tag_list, lstm_pred, remove_O=REMOVE_O)
metrics.report_scores()
metrics.report_confusion_matrix()
print("Load and evaluate the bilstm+crf model...")
crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True)
bilstm_model = load_model(BiLSTMCRF_MODEL_PATH)
bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(
test_word_lists, test_tag_lists, test=True)
lstmcrf_pred, target_tag_list = bilstm_model.test(test_word_lists,
test_tag_lists,
crf_word2id, crf_tag2id)
metrics = Metrics(target_tag_list, lstmcrf_pred, remove_O=REMOVE_O)
metrics.report_scores()
metrics.report_confusion_matrix()
ensemble_evaluate([hmm_pred, crf_pred, lstm_pred, lstmcrf_pred],
test_tag_lists)
def decode(input_status, output_file):
print("读取数据...")
train_word_lists, train_tag_lists, word2id1, tag2id = build_corpus(
"train", make_vocab=True)
decode_word_lists, decode_tag_lists, word2id2, tag2id1 = build_corpus(
input_status, make_vocab=True)
lists = train_word_lists + decode_word_lists
word2id = build_map(lists)
print("加载并评估hmm模型...")
hmm_model = load_model(HMM_MODEL_PATH)
hmm_pred = hmm_model.test(decode_word_lists, word2id, tag2id)
print("加载并评估crf模型...")
crf_model = load_model(CRF_MODEL_PATH)
crf_pred = crf_model.test(decode_word_lists)
print("加载并评估bilstm模型...")
bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False)
bilstm_model = load_model(BiLSTM_MODEL_PATH)
bilstm_model.model.bilstm.flatten_parameters() # remove warning
lstm_pred, target_tag_list = bilstm_model.test(decode_word_lists,
decode_tag_lists,
bilstm_word2id,
bilstm_tag2id)
print("加载并评估bilstm+crf模型...")
crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True)
bilstm_model = load_model(BiLSTMCRF_MODEL_PATH)
bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning
decode_word_lists, decode_tag_lists = prepocess_data_for_lstmcrf(
decode_word_lists, decode_tag_lists, test=True)
lstmcrf_pred, target_tag_list = bilstm_model.test(decode_word_lists,
decode_tag_lists,
crf_word2id, crf_tag2id)
print("加载并评估lattice lstm模型...")
latticelstm_pred = run(status='decode')
print("加载并评估ensemble模型...")
predict_results = ensemble_evaluate(hmm_pred,
crf_pred,
lstm_pred,
lstmcrf_pred,
latticelstm_pred,
decode_tag_lists,
status='decode')
print("输出解码结果...")
write_decoded_results(output_file, predict_results, decode_word_lists)
def main(corpus_dir, test_file):
# 读取数据
# print("读取数据...")
train_word_lists, train_tag_lists, word2id, tag2id = \
build_corpus("train", data_dir=corpus_dir)
test_word_lists, test_tag_lists = build_corpus_test(test_file)
# print(test_word_lists, test_tag_lists)
# print("加载并评估bilstm+crf模型...")
crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True)
bilstm_model = load_model(BiLSTMCRF_MODEL_PATH)
bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(
test_word_lists, test_tag_lists, test=True)
lstmcrf_pred, target_tag_list = bilstm_model.test(test_word_lists,
test_tag_lists,
crf_word2id, crf_tag2id)
for i in range(len(lstmcrf_pred)):
for j in range(len(lstmcrf_pred[i])):
print(test_word_lists[i][j] + "\t" + lstmcrf_pred[i][j])
print()
build_corpus("train")
print("加载并评估bilstm+crf模型...")
crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True)
bilstm_model = load_model(BiLSTMCRF_MODEL_PATH)
bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning
for item in glob.glob("data/chusai_xuanshou/" + "*.txt"):
with open(item, encoding="utf-8") as f:
origin_text = "".join(f.readlines())
test_word_list = list(origin_text)
test_word_lists = [test_word_list]
test_tag_list = ["N" for _ in test_word_list]
test_tag_lists = [test_tag_list]
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(
test_word_lists, test_tag_lists, test=True
)
lstmcrf_pred = bilstm_model.testA(test_word_lists,
crf_word2id, crf_tag2id)
print(test_word_list)
print(lstmcrf_pred)
with open(item.replace("txt", "ann"), 'w', encoding='utf-8') as ann_f:
flag = False
counter = 0
word = ""
start = 0
end = 0
for idx, tag in enumerate(lstmcrf_pred[0]):
if tag != "O":
if not flag:
def main(args):
"""训练模型，评估结果"""
output_directory = os.path.join('ckpts', args.name)
if not os.path.isdir(output_directory):
os.makedirs(output_directory)
os.chmod(output_directory, 0o775)
shutil.copy2('models/config.py', output_directory)
# 读取数据
print("读取数据...")
train_word_lists, train_tag_lists, word2id, tag2id = build_corpus("train", fix_length=-1)
dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False)
test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False)
# 训练评估HMM模型
print("正在训练评估HMM模型...")
hmm_pred = hmm_train_eval(
(train_word_lists, train_tag_lists),
(test_word_lists, test_tag_lists),
word2id,
tag2id,
output_directory
)
# 训练评估CRF模型
print("正在训练评估CRF模型...")
crf_pred = crf_train_eval(
(train_word_lists, train_tag_lists),
(test_word_lists, test_tag_lists),
output_directory
)
# 训练评估BI-LSTM模型
print("正在训练评估双向LSTM模型...")
# LSTM模型训练的时候需要在word2id和tag2id加入PAD和UNK
bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False)
lstm_pred = bilstm_train_and_eval(
(train_word_lists, train_tag_lists),
(dev_word_lists, dev_tag_lists),
(test_word_lists, test_tag_lists),
bilstm_word2id, bilstm_tag2id,
output_directory,
crf=False
)
print("正在训练评估Bi-LSTM+CRF模型...")
# 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到)
crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True)
# 还需要额外的一些数据处理
train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf(
train_word_lists, train_tag_lists
)
dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf(
dev_word_lists, dev_tag_lists
)
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(
test_word_lists, test_tag_lists, test=True
)
lstmcrf_pred = bilstm_train_and_eval(
(train_word_lists, train_tag_lists),
(dev_word_lists, dev_tag_lists),
(test_word_lists, test_tag_lists),
crf_word2id, crf_tag2id,
output_directory
)
ensemble_evaluate(
[hmm_pred, crf_pred, lstm_pred, lstmcrf_pred],
test_tag_lists
)
def main():
"""训练模型，评估结果"""
# 读取数据
print("读取数据...")
# train_word_lists, train_tag_lists, word2id, tag2id = \
# build_corpus("train")
# dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False)
# test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False)
# 训练评估ｈｍｍ模型
# print("正在训练评估HMM模型...")
with open('corpus_word2id', 'rb') as f:
word2id = pickle.load(f)
with open('tag2id', 'rb') as f:
tag2id = pickle.load(f)
# print('word2id',word2id)
# print('tag2id',tag2id)
train_word_lists = []
train_tag_lists = []
f = open('normal_daguan_train.txt')
datas = f.readlines()
f.close()
for data in datas:
temps = data.strip().split('|||')
words = temps[0]
tags = temps[1]
train_word_lists.append(words.split(' '))
train_tag_lists.append(tags.split(' '))
test_word_lists = []
test_tag_lists = []
f = open('normal_daguan_test.txt')
datas = f.readlines()
f.close()
for data in datas:
words = data.strip()
test_word_lists.append(words.split(' '))
test_tag_lists.append([None for i in range(len(words))])
'''前3000数据集划分'''
dev_word_lists = train_word_lists[:1500]
dev_tag_lists = train_tag_lists[:1500]
# test_word_lists = train_word_lists[1500:3000]
# test_tag_lists = train_tag_lists[1500:3000]
train_word_lists = train_word_lists[1500:]
train_tag_lists = train_tag_lists[1500:]
'''后3000数据集划分'''
# dev_word_lists = train_word_lists[-3000:-1500]
# dev_tag_lists = train_tag_lists[-3000:-1500]
# test_word_lists = train_word_lists[-1500:]
# test_tag_lists = train_tag_lists[-1500:]
# train_word_lists = train_word_lists[:-3000]
# train_tag_lists = train_tag_lists[:-3000]
# print(train_word_lists[0])
# print(train_tag_lists[0])
# print(test_word_lists[0])
# print(len(train_tag_lists)
#
print("正在训练评估Bi-LSTM+CRF模型...")
# 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到)
crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True)
# 还需要额外的一些数据处理
train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf(
train_word_lists, train_tag_lists)
dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf(
dev_word_lists,
dev_tag_lists,
)
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(
test_word_lists,
test_tag_lists,
)
model = Word2Vec.load('charEmbedding_300dim')
weights = np.zeros(shape=(len(crf_word2id), 300))
for word in word2id.keys():
if word in model.wv.vocab:
weights[word2id[word]] = model[word]
else:
weights[word2id[word]] = np.random.uniform(-0.1, 0.1, size=(300, ))
lstmcrf_pred = bilstm_train_and_eval(
(train_word_lists, train_tag_lists),
(dev_word_lists, dev_tag_lists),
(test_word_lists, test_tag_lists),
# (dev_word_lists, dev_tag_lists),
weights,
crf_word2id,
crf_tag2id)
def main():
"""训练模型，评估结果"""
# 读取数据
print("读取数据...")
train_word_lists, train_tag_lists, word2id, tag2id = build_corpus("train")
dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False)
#test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False)
test_word_lists, test_tag_lists, article_id = loadDevFile(
"development_2.txt")
# # 训练评估ｈｍｍ模型
# # print("正在训练评估HMM模型...")
# # hmm_pred = hmm_train_eval(
# # (train_word_lists, train_tag_lists),
# # (dev_word_lists_, test_tag_lists),
# # word2id,
# # tag2id,
# # remove_O=True
# # )
# # output_pred(hmm_pred, article_id, dev_word_lists_raw)
# # 训练评估CRF模型
# print("正在训练评估CRF模型...")
# crf_pred = crf_train_eval(
# (train_word_lists, train_tag_lists),
# (dev_word_lists_, test_tag_lists),
# remove_O=True
# )
# output_pred(crf_pred, article_id, dev_word_lists_raw,output_path = 'output_crf.tsv')
# 训练评估BI-LSTM模型
print("正在训练评估双向LSTM模型...")
# LSTM模型训练的时候需要在word2id和tag2id加入PAD和UNK
bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False)
lstm_pred = bilstm_train_and_eval((train_word_lists, train_tag_lists),
(dev_word_lists, dev_tag_lists),
(test_word_lists, test_tag_lists),
bilstm_word2id,
bilstm_tag2id,
crf=False,
remove_O=True)
print("正在训练评估Bi-LSTM+CRF模型...")
# 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到)
crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True)
# 还需要额外的一些数据处理
train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf(
train_word_lists, train_tag_lists)
dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf(
dev_word_lists, dev_tag_lists)
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(
test_word_lists, test_tag_lists, test=True)
lstmcrf_pred = bilstm_train_and_eval((train_word_lists, train_tag_lists),
(dev_word_lists, dev_tag_lists),
(test_word_lists, test_tag_lists),
crf_word2id,
crf_tag2id,
remove_O=True)
ensemble_evaluate([hmm_pred, crf_pred, lstm_pred, lstmcrf_pred],
test_tag_lists)
def main():
print("读取数据...")
train_word_lists, train_tag_lists, word2id, tag2id = \
build_corpus("train")
dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False)
test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False)
# print("加载并评估hmm模型...")
# hmm_model = load_model(HMM_MODEL_PATH)
# hmm_pred = hmm_model.test(test_word_lists,
# word2id,
# tag2id)
# metrics = Metrics(test_tag_lists, hmm_pred, remove_O=REMOVE_O)
# metrics.report_scores() # 打印每个标记的精确度、召回率、f1分数
# metrics.report_confusion_matrix() # 打印混淆矩阵
#
# # 加载并评估CRF模型
# print("加载并评估crf模型...")
# crf_model = load_model(CRF_MODEL_PATH)
# crf_pred = crf_model.test(test_word_lists)
# metrics = Metrics(test_tag_lists, crf_pred, remove_O=REMOVE_O)
# metrics.report_scores()
# metrics.report_confusion_matrix()
#
# # bilstm模型
# print("加载并评估bilstm模型...")
# bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False)
# bilstm_model = load_model(BiLSTM_MODEL_PATH)
# bilstm_model.model.bilstm.flatten_parameters() # remove warning
# lstm_pred, target_tag_list = bilstm_model.test(test_word_lists, test_tag_lists,
# bilstm_word2id, bilstm_tag2id)
# metrics = Metrics(target_tag_list, lstm_pred, remove_O=REMOVE_O)
# metrics.report_scores()
# metrics.report_confusion_matrix()
print("加载并评估bilstm+crf模型...")
crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True)
bilstm_model = load_model(BiLSTMCRF_MODEL_PATH)
bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(
test_word_lists, test_tag_lists, test=True)
lstmcrf_pred, target_tag_list = bilstm_model.test(test_word_lists,
test_tag_lists,
crf_word2id, crf_tag2id)
metrics = Metrics(target_tag_list, lstmcrf_pred, remove_O=REMOVE_O)
metrics.report_scores()
metrics.report_confusion_matrix()
# ensemble_evaluate(
# [hmm_pred, crf_pred, lstm_pred, lstmcrf_pred],
# test_tag_lists
# )
ls = ['B-SYM', 'M-SYM', 'E-SYM']
selected = [
i for i in range(len(test_tag_lists[0])) if test_tag_lists[0][i] in ls
]
selected_word = [test_word_lists[0][i] for i in selected]
selected_predict = [
i for i in range(len(lstmcrf_pred[0])) if lstmcrf_pred[0][i] in ls
]
selected_predict_word = [test_word_lists[0][i] for i in selected_predict]
for tag_list, doc in zip(train_tag_lists, train_word_lists):
selected_train = [i for i in range(len(tag_list)) if tag_list[i] in ls]
selected_train_word = [doc[i] for i in selected_train]
# print(selected_train_word)
print('preditct list:', lstmcrf_pred)
print('target list:', target_tag_list)
print(selected_word)
print(selected_predict_word)
def main_rep1(x, y):
if x == 'train':
# select data according to args.process
print("Read data...")
train_word_lists, train_tag_lists, word2id, tag2id = \
build_corpus("train")
dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False)
test_word_lists, test_tag_lists = build_corpus("test",
make_vocab=False)
######
if y == 'crf':
crf_pred = crf_train_eval((train_word_lists, train_tag_lists),
(test_word_lists, test_tag_lists))
ensemble_evaluate([crf_pred], test_tag_lists)
elif y == 'bilstm':
bilstm_word2id, bilstm_tag2id = extend_maps(word2id,
tag2id,
for_crf=False)
lstm_pred = bilstm_train_and_eval(
(train_word_lists, train_tag_lists),
(dev_word_lists, dev_tag_lists),
(test_word_lists, test_tag_lists),
bilstm_word2id,
bilstm_tag2id,
crf=False)
ensemble_evaluate([lstm_pred], test_tag_lists)
elif y == 'bilstm-crf':
crf_word2id, crf_tag2id = extend_maps(word2id,
tag2id,
for_crf=True)
# more data processing
train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf(
train_word_lists, train_tag_lists)
dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf(
dev_word_lists, dev_tag_lists)
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(
test_word_lists, test_tag_lists, test=True)
lstmcrf_pred = bilstm_train_and_eval(
(train_word_lists, train_tag_lists),
(dev_word_lists, dev_tag_lists),
(test_word_lists, test_tag_lists), crf_word2id, crf_tag2id)
ensemble_evaluate([lstmcrf_pred], test_tag_lists)
else:
HMM_MODEL_PATH = './ckpts/hmm.pkl'
CRF_MODEL_PATH = './ckpts/crf.pkl'
BiLSTM_MODEL_PATH = './ckpts/bilstm.pkl'
BiLSTMCRF_MODEL_PATH = './ckpts/bilstm_crf.pkl'
REMOVE_O = False # Whether to remove the O mark at the time of evaluation
# select data according to args.process
print("Read data...")
train_word_lists, train_tag_lists, word2id, tag2id = \
build_corpus("train")
dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False)
test_word_lists, test_tag_lists = build_corpus("test",
make_vocab=False)
if y == 'crf':
crf_model = load_model_1(CRF_MODEL_PATH)
crf_pred = crf_model.test(test_word_lists)
metrics = Metrics(test_tag_lists, crf_pred, remove_O=REMOVE_O)
metrics.report_scores()
metrics.report_confusion_matrix()
elif y == 'bilstm':
bilstm_word2id, bilstm_tag2id = extend_maps(word2id,
tag2id,
for_crf=False)
bilstm_model = load_model_1(BiLSTM_MODEL_PATH)
bilstm_model.model.bilstm.flatten_parameters() # remove warning
lstm_pred, target_tag_list = bilstm_model.test(
test_word_lists, test_tag_lists, bilstm_word2id, bilstm_tag2id)
metrics = Metrics(target_tag_list, lstm_pred, remove_O=REMOVE_O)
metrics.report_scores()
metrics.report_confusion_matrix()
elif y == 'bilstm-crf':
crf_word2id, crf_tag2id = extend_maps(word2id,
tag2id,
for_crf=True)
bilstm_model = load_model_1(BiLSTMCRF_MODEL_PATH)
bilstm_model.model.bilstm.bilstm.flatten_parameters(
) # remove warning
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(
test_word_lists, test_tag_lists, test=True)
lstmcrf_pred, target_tag_list = bilstm_model.test(
test_word_lists, test_tag_lists, crf_word2id, crf_tag2id)
metrics = Metrics(target_tag_list, lstmcrf_pred, remove_O=REMOVE_O)
metrics.report_scores()
metrics.report_confusion_matrix()
exit()
def main():
import argparse
parser = argparse.ArgumentParser(description='main.py')
parser.add_argument('--hmm',
action='store_true',
default=False,
help='Test HMM')
parser.add_argument('--crf',
action='store_true',
default=False,
help='Test CRF')
parser.add_argument('--bilstm',
action='store_true',
default=False,
help='Test BiLSTM')
parser.add_argument('--bilstm-crf',
action='store_true',
default=False,
help='Test BiLSTM-CRF')
parser.add_argument('--cbow',
action='store_true',
default=False,
help='Use CBOW embedding for BiLSTM-CRF')
args = parser.parse_args()
print("读取数据...")
train_word_lists, train_tag_lists, word2id, tag2id = \
build_corpus("train")
dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False)
test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False)
if args.hmm:
print("加载并评估hmm模型...")
hmm_model = load_model(HMM_MODEL_PATH)
hmm_pred = hmm_model.test(test_word_lists, word2id, tag2id)
metrics = Metrics(test_tag_lists, hmm_pred, remove_O=REMOVE_O)
metrics.report_scores() # 打印每个标记的精确度、召回率、f1分数
metrics.report_confusion_matrix() # 打印混淆矩阵
# 加载并评估CRF模型
if args.crf:
print("加载并评估crf模型...")
crf_model = load_model(CRF_MODEL_PATH)
crf_pred = crf_model.test(test_word_lists)
metrics = Metrics(test_tag_lists, crf_pred, remove_O=REMOVE_O)
metrics.report_scores()
metrics.report_confusion_matrix()
# bilstm模型
if args.bilstm:
print("加载并评估bilstm模型...")
bilstm_word2id, bilstm_tag2id = extend_maps(word2id,
tag2id,
for_crf=False)
bilstm_model = load_model(BiLSTM_MODEL_PATH)
bilstm_model.model.bilstm.flatten_parameters() # remove warning
lstm_pred, target_tag_list = bilstm_model.test(test_word_lists,
test_tag_lists,
bilstm_word2id,
bilstm_tag2id)
metrics = Metrics(target_tag_list, lstm_pred, remove_O=REMOVE_O)
metrics.report_scores()
metrics.report_confusion_matrix()
if args.bilstm_crf:
print("加载并评估bilstm+crf模型...")
crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True)
bilstm_model = load_model(BiLSTMCRF_MODEL_PATH)
bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(
test_word_lists, test_tag_lists, test=True)
lstmcrf_pred, target_tag_list = bilstm_model.test(
test_word_lists, test_tag_lists, crf_word2id, crf_tag2id)
metrics = Metrics(target_tag_list, lstmcrf_pred, remove_O=REMOVE_O)
metrics.report_scores()
metrics.report_confusion_matrix()
def main():
import argparse
parser = argparse.ArgumentParser(description='main.py')
parser.add_argument('--hmm',
action='store_true',
default=False,
help='Train HMM')
parser.add_argument('--crf',
action='store_true',
default=False,
help='Train CRF')
parser.add_argument('--bilstm',
action='store_true',
default=False,
help='Train BiLSTM')
parser.add_argument('--bilstm-crf',
action='store_true',
default=False,
help='Train BiLSTM-CRF')
parser.add_argument('--cbow',
action='store_true',
default=False,
help='Train or use CBOW embedding for BiLSTM-CRF')
args = parser.parse_args()
"""训练模型，评估结果"""
# 读取数据
print("读取数据...")
train_word_lists, train_tag_lists, word2id, tag2id = \
build_corpus("train")
dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False)
test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False)
# 训练评估ｈｍｍ模型
if args.hmm:
print("正在训练评估HMM模型...")
hmm_pred = hmm_train_eval((train_word_lists, train_tag_lists),
(test_word_lists, test_tag_lists), word2id,
tag2id)
# 训练评估CRF模型
if args.crf:
print("正在训练评估CRF模型...")
crf_pred = crf_train_eval((train_word_lists, train_tag_lists),
(test_word_lists, test_tag_lists))
if args.bilstm:
# 训练评估BI-LSTM模型
print("正在训练评估双向LSTM模型...")
# LSTM模型训练的时候需要在word2id和tag2id加入PAD和UNK
bilstm_word2id, bilstm_tag2id = extend_maps(word2id,
tag2id,
for_crf=False)
lstm_pred = bilstm_train_and_eval((train_word_lists, train_tag_lists),
(dev_word_lists, dev_tag_lists),
(test_word_lists, test_tag_lists),
bilstm_word2id,
bilstm_tag2id,
crf=False)
# 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到)
crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True)
# 还需要额外的一些数据处理
train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf(
train_word_lists, train_tag_lists)
dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf(
dev_word_lists, dev_tag_lists)
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(
test_word_lists, test_tag_lists, test=True)
if args.bilstm_crf:
print("正在训练评估Bi-LSTM+CRF模型...")
cbow_emb = None
if args.cbow:
print('Loading CBOW model')
cbow_model = load_model('ckpts/cbow.pkl')
cbow_emb = cbow_model.model.lookup_embedding()
del cbow_model
lstmcrf_pred = bilstm_train_and_eval(
(train_word_lists, train_tag_lists),
(dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists),
crf_word2id,
crf_tag2id,
cbow_emb=cbow_emb)
elif args.cbow:
print("正在训练CBOW模型...")
cbow.CBOW_Model(len(crf_word2id)).train(train_word_lists, crf_word2id)
