valid_label_documents[i] for i in batch_ids ] valid_word_features = get_word_features(batch_word_documents, word_dic, config_dic.get("gpu")) #valid_char_features = get_char_features(batch_word_documents, char_dic, config_dic.get("gpu")) valid_sw_features_list = [] for sp_key, sp in sps.items(): valid_sw_features_list.append( get_sw_features(batch_word_documents, sw_dicts[sp_key], sp, config_dic.get("gpu"))) valid_label_features = get_label_features(batch_label_documents, label_dic, config_dic.get("gpu")) valid_tag_seq = seq_model.forward(valid_word_features, None, valid_sw_features_list) masks = valid_word_features.get("masks") rt, tt = predict_check(valid_tag_seq, valid_label_features.get("label_ids"), masks) right_token += rt total_token += tt ################ evaluate by precision, recall and fscore ################### true_seqs.extend([ label_dic.id2token.get(int(label_id), label_dic.token2id["O"]) for label_id in valid_label_features.get( "label_ids").masked_select(masks) ]) pred_seqs.extend([ label_dic.id2token.get(int(label_id), label_dic.token2id["O"]) for label_id in valid_tag_seq.masked_select(masks)
test_word_features = get_word_features(batch_word_documents, word_dic, config_dic.get("gpu")) test_char_features = get_char_features(batch_word_documents, char_dic, config_dic.get("gpu")) test_sw_features_list = [] for sp_key, sp in sps.items(): test_sw_features_list.append( get_sw_features(batch_word_documents, sw_dicts[sp_key], sp, config_dic.get("gpu"))) test_label_features = get_label_features(batch_label_documents, label_dic, config_dic.get("gpu")) test_tag_seq = seq_model.forward(test_word_features, test_char_features, test_sw_features_list) rt, tt = predict_check(test_tag_seq, test_label_features.get("label_ids"), test_word_features.get("masks")) right_token += rt total_token += tt ################ evaluate by precision, recall and fscore ################### masks = test_word_features.get("masks") word_seqs.extend([ word for word_document in batch_word_documents for word in word_document ]) true_seqs.extend([ label_dic.id2token[int(label_id)] for label_id in