def calc_f1_score(ner_model, dataset_loader, action2idx, if_cuda):
idx2action = {v: k for k, v in action2idx.items()}
ner_model.eval()
correct = 0
total_correct_entity = 0
total_act = 0
total_entity_in_gold = 0
total_entity_in_pre = 0
for feature, label, action in itertools.chain.from_iterable(
dataset_loader): # feature : torch.Size([4, 17])
fea_v, tg_v, ac_v = utils.repack_vb(if_cuda, feature, label, action)
# loss, pre_action, right_num = ner_model.forward(fea_v, ac_v) # loss torch.Size([1, seq_len, action_size+1, action_size+1])
_, pre_actions, right_num = ner_model.forward_batch(
fea_v, ac_v
) # loss torch.Size([1, seq_len, action_size+1, action_size+1])
for ac_golden, ac_pre in zip(
ac_v.squeeze(0).data.tolist(), pre_actions):
num_entity_in_real, num_entity_in_pre, correct_entity = to_entity(
ac_golden, ac_pre, idx2action)
total_correct_entity += correct_entity
total_entity_in_gold += num_entity_in_real
total_entity_in_pre += num_entity_in_pre
for idx in range(len(ac_pre)):
if ac_pre[idx] == ac_golden[idx]:
correct += 1
total_act += len(ac_pre)
acc = correct / float(total_act)
if total_entity_in_pre > 0:
pre = total_correct_entity / float(total_entity_in_pre)
else:
pre = 0
if total_entity_in_gold > 0:
rec = total_correct_entity / float(total_entity_in_gold)
else:
rec = 0
if (pre + rec) > 0:
f1 = 2 * pre * rec / float(pre + rec)
else:
f1 = 0
return f1, pre, rec, acc
def calc_score(ner_model, dataset_loader, if_cuda):
ner_model.eval()
correct = 0
total_act = 0
for feature, label, action in itertools.chain.from_iterable(
dataset_loader): # feature : torch.Size([4, 17])
fea_v, tg_v, ac_v = utils.repack_vb(if_cuda, feature, label, action)
loss, pre_action = ner_model.forward(
fea_v, ac_v
) # loss torch.Size([1, seq_len, action_size+1, action_size+1])
for idx in range(len(pre_action)):
if pre_action[idx] == ac_v.squeeze(0).data[idx]:
correct += 1
total_act += len(pre_action)
acc = correct / float(total_act)
return acc
patience_count = 0
for epoch_idx, args.start_epoch in enumerate(epoch_list):
epoch_loss = 0
ner_model.train()
for feature, label, action in tqdm(
itertools.chain.from_iterable(dataset_loader),
mininterval=2,
desc=' - Tot it %d (epoch %d)' %
(tot_length, args.start_epoch),
leave=False,
file=sys.stdout):
fea_v, la_v, ac_v = utils.repack_vb(if_cuda, feature, label,
action)
ner_model.zero_grad() # zeroes the gradient of all parameters
# loss, _, _ = ner_model.forward(fea_v, ac_v)
loss, _ = ner_model.forward(fea_v, ac_v)
loss.backward()
nn.utils.clip_grad_norm(ner_model.parameters(), args.clip_grad)
optimizer.step()
epoch_loss += utils.to_scalar(loss)
# update lr
scheduler.step(epoch_loss)
dev_f1, dev_pre, dev_rec = evaluate.calc_f1_score(
ner_model, dev_dataset_loader, a_map, if_cuda)
if dev_f1 > best_f1:
patience_count = 0