valid_label_documents[i] for i in batch_ids
]
valid_word_features = get_word_features(batch_word_documents,
word_dic,
config_dic.get("gpu"))
#valid_char_features = get_char_features(batch_word_documents, char_dic, config_dic.get("gpu"))
valid_sw_features_list = []
for sp_key, sp in sps.items():
valid_sw_features_list.append(
get_sw_features(batch_word_documents, sw_dicts[sp_key], sp,
config_dic.get("gpu")))
valid_label_features = get_label_features(batch_label_documents,
label_dic,
config_dic.get("gpu"))
valid_tag_seq = seq_model.forward(valid_word_features, None,
valid_sw_features_list)
masks = valid_word_features.get("masks")
rt, tt = predict_check(valid_tag_seq,
valid_label_features.get("label_ids"),
masks)
right_token += rt
total_token += tt
################ evaluate by precision, recall and fscore ###################
true_seqs.extend([
label_dic.id2token.get(int(label_id), label_dic.token2id["O"])
for label_id in valid_label_features.get(
"label_ids").masked_select(masks)
])
pred_seqs.extend([
label_dic.id2token.get(int(label_id), label_dic.token2id["O"])
for label_id in valid_tag_seq.masked_select(masks)
test_word_features = get_word_features(batch_word_documents, word_dic,
config_dic.get("gpu"))
test_char_features = get_char_features(batch_word_documents, char_dic,
config_dic.get("gpu"))
test_sw_features_list = []
for sp_key, sp in sps.items():
test_sw_features_list.append(
get_sw_features(batch_word_documents, sw_dicts[sp_key], sp,
config_dic.get("gpu")))
test_label_features = get_label_features(batch_label_documents,
label_dic,
config_dic.get("gpu"))
test_tag_seq = seq_model.forward(test_word_features,
test_char_features,
test_sw_features_list)
rt, tt = predict_check(test_tag_seq,
test_label_features.get("label_ids"),
test_word_features.get("masks"))
right_token += rt
total_token += tt
################ evaluate by precision, recall and fscore ###################
masks = test_word_features.get("masks")
word_seqs.extend([
word for word_document in batch_word_documents
for word in word_document
])
true_seqs.extend([
label_dic.id2token[int(label_id)] for label_id in
