def main(): print("读取数据...") train_word_lists, train_tag_lists, word2id, tag2id = \ build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) dev_word_lists_, dev_word_lists_raw, article_id = loadDevFile("development_2.txt") print("加载并评估hmm模型...") hmm_model = load_model(HMM_MODEL_PATH) #hmm_pred = hmm_model.test(test_word_lists, # word2id, # tag2id) hmm_pred_dev = hmm_model.test(dev_word_lists_, word2id, tag2id) output_pred(hmm_pred_dev, article_id, dev_word_lists_raw) metrics = Metrics(test_tag_lists, hmm_pred, remove_O=REMOVE_O) metrics.report_scores() # 打印每个标记的精确度、召回率、f1分数 metrics.report_confusion_matrix() # 打印混淆矩阵 # 加载并评估CRF模型 print("加载并评估crf模型...") crf_model = load_model(CRF_MODEL_PATH) crf_pred = crf_model.test(test_word_lists) metrics = Metrics(test_tag_lists, crf_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() # bilstm模型 print("加载并评估bilstm模型...") bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) bilstm_model = load_model(BiLSTM_MODEL_PATH) bilstm_model.model.bilstm.flatten_parameters() # remove warning lstm_pred, target_tag_list = bilstm_model.test(test_word_lists, test_tag_lists, bilstm_word2id, bilstm_tag2id) metrics = Metrics(target_tag_list, lstm_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() print("加载并评估bilstm+crf模型...") crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) bilstm_model = load_model(BiLSTMCRF_MODEL_PATH) bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True ) lstmcrf_pred, target_tag_list = bilstm_model.test(test_word_lists, test_tag_lists, crf_word2id, crf_tag2id) metrics = Metrics(target_tag_list, lstmcrf_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() ensemble_evaluate( [hmm_pred, crf_pred, lstm_pred, lstmcrf_pred], test_tag_lists )
def main(): """训练模型,评估结果""" # 读取数据 print("读取数据...") train_word_lists, train_tag_lists, word2id, tag2id = \ build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) #### # 训练评估hmm模型 #### print("正在训练评估HMM模型...") #### hmm_pred = hmm_train_eval( #### (train_word_lists, train_tag_lists), #### (test_word_lists, test_tag_lists), #### word2id, #### tag2id #### ) #### #### # 训练评估CRF模型 #### print("正在训练评估CRF模型...") #### crf_pred = crf_train_eval( #### (train_word_lists, train_tag_lists), #### (test_word_lists, test_tag_lists) #### ) #### #### # 训练评估BI-LSTM模型 #### print("正在训练评估双向LSTM模型...") #### # LSTM模型训练的时候需要在word2id和tag2id加入PAD和UNK #### bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) #### lstm_pred = bilstm_train_and_eval( #### (train_word_lists, train_tag_lists), #### (dev_word_lists, dev_tag_lists), #### (test_word_lists, test_tag_lists), #### bilstm_word2id, bilstm_tag2id, #### crf=False #### ) print("正在训练评估Bi-LSTM+CRF模型...") # 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到) crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) # 还需要额外的一些数据处理 train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf( train_word_lists, train_tag_lists) dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf( dev_word_lists, dev_tag_lists) test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred = bilstm_train_and_eval( (train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), crf_word2id, crf_tag2id) ensemble_evaluate( # [hmm_pred, crf_pred, lstm_pred, lstmcrf_pred], [lstmcrf_pred], test_tag_lists)
def main(): """训练模型,评估结果""" # 读取数据 print("读取数据...") data_folder = "./data123" train_word_lists, train_tag_lists, word2id, tag2id = \ build_corpus("train", data_dir=data_folder) dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False, data_dir=data_folder) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False, data_dir=data_folder) # 训练评估hmm模型 print("正在训练评估HMM模型...") hmm_pred = hmm_train_eval((train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists), word2id, tag2id) # 训练评估CRF模型 print("正在训练评估CRF模型...") crf_pred = crf_train_eval((train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists)) # 训练评估BI-LSTM模型 print("正在训练评估双向LSTM模型...") # LSTM模型训练的时候需要在word2id和tag2id加入PAD和UNK bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) lstm_pred = bilstm_train_and_eval((train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), bilstm_word2id, bilstm_tag2id, crf=False) print("正在训练评估Bi-LSTM+CRF模型...") # 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到) crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) # 还需要额外的一些数据处理 train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf( train_word_lists, train_tag_lists) dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf( dev_word_lists, dev_tag_lists) test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred = bilstm_train_and_eval( (train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), crf_word2id, crf_tag2id) ensemble_evaluate([hmm_pred, crf_pred, lstm_pred, lstmcrf_pred], test_tag_lists)
def main(): print("Read data...") train_word_lists, train_tag_lists, word2id, tag2id = \ build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) print("Load and evaluate the hmm model...") hmm_model = load_model(HMM_MODEL_PATH) hmm_pred = hmm_model.test(test_word_lists, word2id, tag2id) metrics = Metrics(test_tag_lists, hmm_pred, remove_O=REMOVE_O) metrics.report_scores( ) # Print the accuracy of each mark, recall rate, f1 score metrics.report_confusion_matrix() #Print confusion matrix # Load and evaluate the CRF model print("Load and evaluate the crf model...") crf_model = load_model(CRF_MODEL_PATH) crf_pred = crf_model.test(test_word_lists) metrics = Metrics(test_tag_lists, crf_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() # bilstm Model print("Load and evaluate the bilstm model...") bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) bilstm_model = load_model(BiLSTM_MODEL_PATH) bilstm_model.model.bilstm.flatten_parameters() # remove warning lstm_pred, target_tag_list = bilstm_model.test(test_word_lists, test_tag_lists, bilstm_word2id, bilstm_tag2id) metrics = Metrics(target_tag_list, lstm_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() print("Load and evaluate the bilstm+crf model...") crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) bilstm_model = load_model(BiLSTMCRF_MODEL_PATH) bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred, target_tag_list = bilstm_model.test(test_word_lists, test_tag_lists, crf_word2id, crf_tag2id) metrics = Metrics(target_tag_list, lstmcrf_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() ensemble_evaluate([hmm_pred, crf_pred, lstm_pred, lstmcrf_pred], test_tag_lists)
def main(): """模型训练与评估""" # 读取数据 print("读取数据中...") train_word_lists, train_tag_lists, word2id, tag2id = build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) #训练并评估hmm模型 print("正在训练评估HMM模型") hmm_pred = hmm_train_eval((train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists), word2id, tag2id) # 训练并评估crf模型 crf_pred = crf_train_eval((train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists)) #训练并评估bilstm模型 bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) lstm_pred = bilstm_train_and_eval((train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), bilstm_word2id, bilstm_tag2id, crf=False) print("正在训练评估Bi-LSTM+CRF模型...") # 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到) crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) print(' '.join([i[0] for i in crf_tag2id.items()])) # 还需要额外的一些数据处理 train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf( train_word_lists, train_tag_lists) dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf( dev_word_lists, dev_tag_lists) test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred = bilstm_train_and_eval( (train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), crf_word2id, crf_tag2id) ensemble_evaluate([hmm_pred, crf_pred, lstm_pred, lstmcrf_pred], test_tag_lists)
def decode(input_status, output_file): print("读取数据...") train_word_lists, train_tag_lists, word2id1, tag2id = build_corpus( "train", make_vocab=True) decode_word_lists, decode_tag_lists, word2id2, tag2id1 = build_corpus( input_status, make_vocab=True) lists = train_word_lists + decode_word_lists word2id = build_map(lists) print("加载并评估hmm模型...") hmm_model = load_model(HMM_MODEL_PATH) hmm_pred = hmm_model.test(decode_word_lists, word2id, tag2id) print("加载并评估crf模型...") crf_model = load_model(CRF_MODEL_PATH) crf_pred = crf_model.test(decode_word_lists) print("加载并评估bilstm模型...") bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) bilstm_model = load_model(BiLSTM_MODEL_PATH) bilstm_model.model.bilstm.flatten_parameters() # remove warning lstm_pred, target_tag_list = bilstm_model.test(decode_word_lists, decode_tag_lists, bilstm_word2id, bilstm_tag2id) print("加载并评估bilstm+crf模型...") crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) bilstm_model = load_model(BiLSTMCRF_MODEL_PATH) bilstm_model.model.bilstm.bilstm.flatten_parameters() # remove warning decode_word_lists, decode_tag_lists = prepocess_data_for_lstmcrf( decode_word_lists, decode_tag_lists, test=True) lstmcrf_pred, target_tag_list = bilstm_model.test(decode_word_lists, decode_tag_lists, crf_word2id, crf_tag2id) print("加载并评估lattice lstm模型...") latticelstm_pred = run(status='decode') print("加载并评估ensemble模型...") predict_results = ensemble_evaluate(hmm_pred, crf_pred, lstm_pred, lstmcrf_pred, latticelstm_pred, decode_tag_lists, status='decode') print("输出解码结果...") write_decoded_results(output_file, predict_results, decode_word_lists)
def main(args): """训练模型,评估结果""" output_directory = os.path.join('ckpts', args.name) if not os.path.isdir(output_directory): os.makedirs(output_directory) os.chmod(output_directory, 0o775) shutil.copy2('models/config.py', output_directory) # 读取数据 print("读取数据...") train_word_lists, train_tag_lists, word2id, tag2id = build_corpus("train", fix_length=-1) dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) # 训练评估HMM模型 print("正在训练评估HMM模型...") hmm_pred = hmm_train_eval( (train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists), word2id, tag2id, output_directory ) # 训练评估CRF模型 print("正在训练评估CRF模型...") crf_pred = crf_train_eval( (train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists), output_directory ) # 训练评估BI-LSTM模型 print("正在训练评估双向LSTM模型...") # LSTM模型训练的时候需要在word2id和tag2id加入PAD和UNK bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) lstm_pred = bilstm_train_and_eval( (train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), bilstm_word2id, bilstm_tag2id, output_directory, crf=False ) print("正在训练评估Bi-LSTM+CRF模型...") # 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到) crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) # 还需要额外的一些数据处理 train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf( train_word_lists, train_tag_lists ) dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf( dev_word_lists, dev_tag_lists ) test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True ) lstmcrf_pred = bilstm_train_and_eval( (train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), crf_word2id, crf_tag2id, output_directory ) ensemble_evaluate( [hmm_pred, crf_pred, lstm_pred, lstmcrf_pred], test_tag_lists )
def main(): """训练模型,评估结果""" # 读取数据 print("读取数据...") train_word_lists, train_tag_lists, word2id, tag2id = build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) #test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) test_word_lists, test_tag_lists, article_id = loadDevFile( "development_2.txt") # # 训练评估hmm模型 # # print("正在训练评估HMM模型...") # # hmm_pred = hmm_train_eval( # # (train_word_lists, train_tag_lists), # # (dev_word_lists_, test_tag_lists), # # word2id, # # tag2id, # # remove_O=True # # ) # # output_pred(hmm_pred, article_id, dev_word_lists_raw) # # 训练评估CRF模型 # print("正在训练评估CRF模型...") # crf_pred = crf_train_eval( # (train_word_lists, train_tag_lists), # (dev_word_lists_, test_tag_lists), # remove_O=True # ) # output_pred(crf_pred, article_id, dev_word_lists_raw,output_path = 'output_crf.tsv') # 训练评估BI-LSTM模型 print("正在训练评估双向LSTM模型...") # LSTM模型训练的时候需要在word2id和tag2id加入PAD和UNK bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) lstm_pred = bilstm_train_and_eval((train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), bilstm_word2id, bilstm_tag2id, crf=False, remove_O=True) print("正在训练评估Bi-LSTM+CRF模型...") # 如果是加了CRF的lstm还要加入<start>和<end> (解码的时候需要用到) crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) # 还需要额外的一些数据处理 train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf( train_word_lists, train_tag_lists) dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf( dev_word_lists, dev_tag_lists) test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred = bilstm_train_and_eval((train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), crf_word2id, crf_tag2id, remove_O=True) ensemble_evaluate([hmm_pred, crf_pred, lstm_pred, lstmcrf_pred], test_tag_lists)
def main(): """Training model and evaluating results!""" # selecting model do_hmm_in_main = False do_crf_in_main = False do_bilstm_in_main = False do_bilstmcrf_in_main = True do_ensemble_in_main = False ensemble_model_list = [] # Data print("Reading data:") ner_data_dir = "./datasets/FA_NER_Data_IOB" train_word_lists, train_tag_lists, word2id, tag2id = build_corpus( "train", data_dir=ner_data_dir) dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False, data_dir=ner_data_dir) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False, data_dir=ner_data_dir) print("len(train_word_lists):", len(train_word_lists)) print("len(word2id=vocab):", len(word2id)) if do_hmm_in_main: # Training and Evaluating HMM model print("Training and Evaluating HMM model:") hmm_pred = hmm_train_eval((train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists), word2id, tag2id) ensemble_model_list.append(hmm_pred) if do_crf_in_main: # Training and evaluating CRF model print("Training and evaluating CRF model:") crf_pred = crf_train_eval((train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists)) ensemble_model_list.append(crf_pred) if do_bilstm_in_main: # Training and evaluating BI-LSTM model print("Training and evaluating Bi-LSTM model:") # We need to put 'PAD' and 'UNK' in word2id and tag2id, when we train LSTM model. bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) lstm_pred = bilstm_train_and_eval((train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), bilstm_word2id, bilstm_tag2id, crf=False) ensemble_model_list.append(lstm_pred) if do_bilstmcrf_in_main: # Training and evaluating Bi-LSTM+CRF model print("Training and evaluating Bi-LSTM-CRF model:") # We need to add <start> and <end>, when we use lstm model with CRF (will be used during decoder processing). crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) # data processing train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf( train_word_lists, train_tag_lists) dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf( dev_word_lists, dev_tag_lists) test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred = bilstm_train_and_eval( (train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), crf_word2id, crf_tag2id, remove_O=False, reload_model=True) ensemble_model_list.append(lstmcrf_pred) if do_ensemble_in_main: ensemble_evaluate(ensemble_model_list, test_tag_lists)
def main_rep1(x, y): if x == 'train': # select data according to args.process print("Read data...") train_word_lists, train_tag_lists, word2id, tag2id = \ build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) ###### if y == 'crf': crf_pred = crf_train_eval((train_word_lists, train_tag_lists), (test_word_lists, test_tag_lists)) ensemble_evaluate([crf_pred], test_tag_lists) elif y == 'bilstm': bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) lstm_pred = bilstm_train_and_eval( (train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), bilstm_word2id, bilstm_tag2id, crf=False) ensemble_evaluate([lstm_pred], test_tag_lists) elif y == 'bilstm-crf': crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) # more data processing train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf( train_word_lists, train_tag_lists) dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf( dev_word_lists, dev_tag_lists) test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred = bilstm_train_and_eval( (train_word_lists, train_tag_lists), (dev_word_lists, dev_tag_lists), (test_word_lists, test_tag_lists), crf_word2id, crf_tag2id) ensemble_evaluate([lstmcrf_pred], test_tag_lists) else: HMM_MODEL_PATH = './ckpts/hmm.pkl' CRF_MODEL_PATH = './ckpts/crf.pkl' BiLSTM_MODEL_PATH = './ckpts/bilstm.pkl' BiLSTMCRF_MODEL_PATH = './ckpts/bilstm_crf.pkl' REMOVE_O = False # Whether to remove the O mark at the time of evaluation # select data according to args.process print("Read data...") train_word_lists, train_tag_lists, word2id, tag2id = \ build_corpus("train") dev_word_lists, dev_tag_lists = build_corpus("dev", make_vocab=False) test_word_lists, test_tag_lists = build_corpus("test", make_vocab=False) if y == 'crf': crf_model = load_model_1(CRF_MODEL_PATH) crf_pred = crf_model.test(test_word_lists) metrics = Metrics(test_tag_lists, crf_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() elif y == 'bilstm': bilstm_word2id, bilstm_tag2id = extend_maps(word2id, tag2id, for_crf=False) bilstm_model = load_model_1(BiLSTM_MODEL_PATH) bilstm_model.model.bilstm.flatten_parameters() # remove warning lstm_pred, target_tag_list = bilstm_model.test( test_word_lists, test_tag_lists, bilstm_word2id, bilstm_tag2id) metrics = Metrics(target_tag_list, lstm_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() elif y == 'bilstm-crf': crf_word2id, crf_tag2id = extend_maps(word2id, tag2id, for_crf=True) bilstm_model = load_model_1(BiLSTMCRF_MODEL_PATH) bilstm_model.model.bilstm.bilstm.flatten_parameters( ) # remove warning test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf( test_word_lists, test_tag_lists, test=True) lstmcrf_pred, target_tag_list = bilstm_model.test( test_word_lists, test_tag_lists, crf_word2id, crf_tag2id) metrics = Metrics(target_tag_list, lstmcrf_pred, remove_O=REMOVE_O) metrics.report_scores() metrics.report_confusion_matrix() exit()