def get_prediction(model, sentence, save_dir, mark='Eval', verbose=False):
"""Evaluate the model on `steps` batches."""
# set model to evaluation mode
model.eval()
idx2lbl = load_obj(save_dir + "idx2lbl.json")
idx2cls = load_obj(save_dir + "idx2cls.json")
enc = sentence.to(device)
enc_self_attn_mask = get_attn_pad_mask(enc, enc)
enc_self_attn_mask.to(device)
# get results from model
logits_tgt, logits_clsf = model(enc, enc_self_attn_mask)
# get sentence length
pad_num = enc.data.eq(0).sum(axis=1)
score_cls, cls_idx = torch.max(logits_clsf, dim=-1)
pred_cls = cls_idx[0].data.tolist()
# get valid slot for a specific intent
idx_mask = load_mask(save_dir)
masked_logits_tgt = softmax_mask(logits_tgt, cls_idx, idx_mask)
score_tgt, tgt_idx = torch.max(masked_logits_tgt, dim=-1)
pred_tags = tgt_idx[0, 0:-pad_num].data.tolist()
pred_lbls = []
for idx in pred_tags:
pred_lbls.append(idx2lbl[str(idx)])
pred_cls = idx2cls[str(pred_cls)]
return pred_cls, pred_lbls
def save_dict_onnx(save_dir, save_onnx):
config = load_obj(save_dir + "Config.json")
word2idx = load_obj(save_dir + "dict.json")
idx2lbl = load_obj(save_dir + "idx2lbl.json")
idx2cls = load_obj(save_dir + "idx2cls.json")
config_onnx = {}
config_onnx['max_len'] = config['max_len']
config_onnx['WORD'] = WORD
config_onnx['BOS'] = BOS
config_onnx['UNK'] = UNK
config_onnx['PAD'] = PAD
save_obj(word2idx, save_onnx + 'dict.json')
save_obj(config_onnx, save_onnx + 'Config.json')
save_obj(idx2lbl, save_onnx + "idx2lbl.json")
save_obj(idx2cls, save_onnx + "idx2cls.json")
def load_save_mask(save_dir, save_onnx):
config = load_obj(save_dir + "Config.json")
num_label = config['num_label']
dict_lbl = load_obj(save_dir + "dict_lbl.json")
dict_clsf = load_obj(save_dir + "dict_clsf.json")
lbl_mask = load_obj(save_dir + "lbl_mask.json")
idx_mask = {}
idx_mask_onnx = {}
for intent in lbl_mask:
valid_slot = lbl_mask[intent]
mask = [0] * num_label
for s in valid_slot:
idx = dict_lbl[s]
mask[idx] = 1
idx_mask_onnx[dict_clsf[intent]] = mask
idx_mask[dict_clsf[intent]] = torch.LongTensor(mask).to(device)
save_obj(idx_mask_onnx, save_onnx + 'idx_mask_onnx.json')
torch.save(idx_mask, save_dir + 'idx_mask.json')
def predict():
parser = argparse.ArgumentParser(description='Transformer NER')
# parser.add_argument('--corpus-data', type=str, default='../data/auto_only-nav-distance_BOI.txt',
# help='path to corpus data')
parser.add_argument('--save-dir',
type=str,
default='./data_char/',
help='path to save processed data')
parser.add_argument('--pre-w2v', type=str, default='../data/w2v')
args = parser.parse_args()
pre_w2v = torch.load(args.save_dir + 'pre_w2v')
pre_w2v = torch.Tensor(pre_w2v).to(device)
model_ckpt = torch.load(os.path.join(
args.save_dir, '{}.pyt'.format("Transformer_NER_best")),
map_location=torch.device(device))
config = load_obj(args.save_dir + 'Config.json')
model = Transformer_Mix(config, pre_w2v).to(device)
model.load_state_dict(model_ckpt['model'])
model.eval()
data_loader = DataLoader_test(args.save_dir)
input_sentence = request.args.get('text')
tokens, test_data = data_loader.load_sentences(input_sentence)
pred_cls, pred_lbls = get_prediction(model,
test_data,
args.save_dir,
mark='Test',
verbose=True)
slot = pretty_print(tokens, pred_lbls, pred_cls)
return render_template('result.html',
input_sentence=input_sentence,
pred_cls=''.join(pred_cls),
pred_lbls=' '.join(pred_lbls),
slot=slot)
def load_mask(save_dir):
config = load_obj(save_dir + "Config.json")
num_label = config['num_label']
idx2lbl = load_obj(save_dir + "idx2lbl.json")
idx2cls = load_obj(save_dir + "idx2cls.json")
dict_lbl = load_obj(save_dir + "dict_lbl.json")
dict_clsf = load_obj(save_dir + "dict_clsf.json")
lbl_mask = load_obj(save_dir + "lbl_mask.json")
idx_mask = {}
for intent in lbl_mask:
valid_slot = lbl_mask[intent]
mask = [0] * num_label
for s in valid_slot:
idx = dict_lbl[s]
mask[idx] = 1
idx_mask[dict_clsf[intent]] = torch.LongTensor(mask).to(device)
# print(idx_mask)
return idx_mask
def __init__(self, save_dir):
self.save_dir = save_dir
self.word2idx = load_obj(self.save_dir + "dict.json")
self.config = load_obj(self.save_dir + "Config.json")
self.max_len = self.config["max_len"]
def evaluate_f1_no_mask(model, dl_test, save_dir, criterion_clsf = nn.CrossEntropyLoss().to(device), criterion_tgt = nn.CrossEntropyLoss(ignore_index=PAD).to(device), verbose = False):
loss_test = 0
pred_tags = []
true_tags = []
pred_clss = []
true_clss = []
criterion_clsf = criterion_clsf
criterion_tgt = criterion_tgt
idx2lbl = load_obj(save_dir+'idx2lbl.json')
for enc, tgt, cls in dl_test[:]:
model.eval()
with torch.no_grad():
enc = enc.to(device)
tgt = tgt.to(device)
cls = cls.to(device)
enc_self_attn_mask = get_attn_pad_mask(enc, enc)
enc_self_attn_mask.to(device)
logits_tgt, logits_clsf = model(enc,enc_self_attn_mask)
loss_tgt = criterion_tgt(logits_tgt.transpose(1, 2), tgt) # for masked LM
loss_tgt = (loss_tgt.float()).mean()
loss_clsf = criterion_clsf(logits_clsf, cls)# for sentence classification
loss = loss_clsf + loss_tgt
# loss = loss_clsf
loss_test+=loss
pad_mask = enc.data.eq(0).sum(axis = 1)
score_tgt, tgt_idx = torch.max(logits_tgt,dim = -1)
score_cls, cls_idx = torch.max(logits_clsf, dim = -1)
for pre, true, pad_num in zip(tgt_idx, tgt, pad_mask):
pred_tags += pre[0:-pad_num].data.tolist()
true_tags += true[0:-pad_num].data.tolist()
# print(cls_idx.size())
pred_clss += cls_idx.tolist()
true_clss += cls.tolist()
# print(len(pred_tags), len(true_tags))
# print(pred_tags)
# print(true_tags)
# print(len(pred_clss), len(true_clss))
# print(pred_clss)
# print(true_clss)
assert len(pred_tags) == len(true_tags)
assert len(pred_clss) == len(true_clss)
# print(pred_clss[-20:])
# print(true_clss[-20:])
# print(pred_tags[-20:])
# print(true_tags[-20:])
# print(enc[-20:])
f1_tgt = f1_score(pred_tags, true_tags, average='micro')
f1_cls = f1_score(pred_clss, true_clss, average='micro')
# logging loss, f1 and report
metrics = {}
true_lbls = []
pred_lbls = []
for t,p in zip(true_tags,pred_tags):
true_lbls.append(idx2lbl[str(t)])
pred_lbls.append(idx2lbl[str(p)])
f1_tgt_merged = f1_score_merged(true_lbls, pred_lbls)
if verbose:
report = classification_report(true_lbls, pred_lbls)
print("============no_mask_slot================")
print(report, flush=True)
return loss_test/len(dl_test), f1_cls*100, f1_tgt*100, f1_tgt_merged
help='path to save processed data')
parser.add_argument('--pre-w2v', type=str, default='../data/w2v')
args = parser.parse_args()
args.corpus_data = args.save_dir + args.corpus_name
# corpus = Corpus(args.corpus_data, args.pre_w2v, args.save_dir)
dl = DataLoader(args.save_dir, batch_size = 128)()
dl_train, dl_test = train_test_split(dl, test_size=0.33)
pre_w2v = torch.load(args.save_dir + 'pre_w2v')
pre_w2v = torch.Tensor(pre_w2v).to(device)
model_ckpt = torch.load(os.path.join(args.save_dir, '{}.pyt'.format("Transformer_NER_best")),map_location=torch.device(device))
config = load_obj(args.save_dir+'Config.json')
# cls_size = config['num_class']
# tgt_size = config['num_label']
model =Transformer_Mix(config, pre_w2v).to(device)
model.load_state_dict (model_ckpt['model'])
optimizer = optim.Adam(model.parameters(), lr=0.01)
optimizer.load_state_dict(model_ckpt['model_opt'])
loss_epoch_test = 0
pred_tags = []
true_tags = []
criterion_clsf = nn.CrossEntropyLoss().to(device)
criterion_tgt = nn.CrossEntropyLoss(ignore_index=PAD).to(device)
loss_epoch_test, f1_cls, f1_tgt, f1_tgt_merged = evaluate_f1(model, dl_test, args.save_dir, verbose=1)
def generate_report_txt(
model,
dl_test,
save_dir,
criterion_clsf=nn.CrossEntropyLoss().to(device),
criterion_tgt=nn.CrossEntropyLoss(ignore_index=PAD).to(device),
verbose=False):
loss_test = 0
pred_tags = []
true_tags = []
pred_clss = []
true_clss = []
criterion_clsf = criterion_clsf
criterion_tgt = criterion_tgt
idx2lbl = load_obj(save_dir + 'idx2lbl.json')
idx2cls = load_obj(save_dir + "idx2cls.json")
sents = load_obj(save_dir + "TestDataSentence.txt")
for enc, tgt, cls in dl_test[:]:
model.eval()
with torch.no_grad():
enc = enc.to(device)
tgt = tgt.to(device)
cls = cls.to(device)
enc_self_attn_mask = get_attn_pad_mask(enc, enc)
enc_self_attn_mask.to(device)
logits_tgt, logits_clsf = model(enc, enc_self_attn_mask)
loss_tgt = criterion_tgt(logits_tgt.transpose(1, 2),
tgt) # for masked LM
loss_tgt = (loss_tgt.float()).mean()
loss_clsf = criterion_clsf(logits_clsf,
cls) # for sentence classification
loss = loss_clsf + loss_tgt
# loss = loss_clsf
loss_test += loss
pad_mask = enc.data.eq(0).sum(axis=1)
score_cls, cls_idx = torch.max(logits_clsf, dim=-1)
# get valid slot for a specific intent
idx_mask = load_mask(save_dir)
masked_logits_tgt = softmax_mask(logits_tgt, cls_idx, idx_mask)
score_tgt, tgt_idx = torch.max(masked_logits_tgt, dim=-1)
for pre, true, pad_num in zip(tgt_idx, tgt, pad_mask):
pred_tags.append(pre[0:-pad_num].data.tolist())
true_tags.append(true[0:-pad_num].data.tolist())
pred_clss += cls_idx.tolist()
true_clss += cls.tolist()
print("Prediction completed", flush=True)
lines_correct = []
lines_intent_error = []
lines_slot_error = []
for idx in range(len(true_clss)):
tokens = sents[idx].split(' ')
true_lbls = []
pred_lbls = []
true_tags_idx = true_tags[idx]
pred_tags_idx = pred_tags[idx]
for t, p in zip(true_tags_idx, pred_tags_idx):
true_lbls.append(idx2lbl[str(t)])
pred_lbls.append(idx2lbl[str(p)])
true_entities = get_entities(true_lbls)
pred_entities = get_entities(pred_lbls)
slots_true = []
slots_pred = []
for chunk_true, chunk_pred, cls_true, cls_pred in zip(
true_entities, pred_entities, true_clss, pred_clss):
tag, start, end = chunk_true[0], chunk_true[1], chunk_true[2]
tok = ''.join(tokens[start:end + 1])
slot_true = '<{0}>: {1}'.format(tag, tok)
slots_true.append(slot_true)
tag, start, end = chunk_pred[0], chunk_pred[1], chunk_pred[2]
tok = ''.join(tokens[start:end + 1])
slot_pred = '<{0}>: {1}'.format(tag, tok)
slots_pred.append(slot_pred)
intent_true = idx2cls[str(true_clss[idx])]
intent_pred = idx2cls[str(pred_clss[idx])]
line = "Sentence:{0:}\nExpect: \t{1}\t{2}\nPredict:\t{3}\t{4}\n".format(
sents[idx], intent_true, slots_true, intent_pred, slots_pred)
if intent_true != intent_pred:
lines_intent_error.append(line)
elif slots_true != slots_pred:
lines_slot_error.append(line)
else:
lines_correct.append(line)
correct_num = len(lines_correct)
intent_w_num = len(lines_intent_error)
slot_w_num = len(lines_slot_error)
total_line = len(lines_intent_error) + len(lines_correct) + len(
lines_slot_error)
score1 = 'total line = {0}; Exact match = {1}, with intent fail = {2}, with slot fail = {3};'.format(
total_line, correct_num, intent_w_num, slot_w_num)
score2 = 'Accuracy = {0:.4f}'.format(correct_num / total_line)
scores = [score1, score2]
# saveing report
print("Saving reports...", flush=True)
report_dir = os.path.join(save_dir, 'reports', '')
create_dir(report_dir)
remove_old_file(report_dir + 'reports_correct.txt')
remove_old_file(report_dir + 'reports_intent_error.txt')
remove_old_file(report_dir + 'reports_slot_error.txt')
remove_old_file(report_dir + 'scores.txt')
with open(report_dir + 'reports_correct.txt', 'w', encoding='utf-8') as f:
for line in lines_correct:
f.write("{0}".format(line + '\n'))
with open(report_dir + 'reports_intent_error.txt', 'w',
encoding='utf-8') as f:
for line in lines_intent_error:
f.write("{0}".format(line + '\n'))
with open(report_dir + 'reports_slot_error.txt', 'w',
encoding='utf-8') as f:
for line in lines_slot_error:
f.write("{0}".format(line + '\n'))
with open(report_dir + 'scores.txt', 'w', encoding='utf-8') as f:
for line in scores:
f.write("{0}".format(line + '\n'))
