def validation_step(self, batch, batch_idx):
self.model.eval()
tokens, intent_idx, entity_idx = batch
feature, intent_pred, entity_pred, entity_loss = self.forward(
tokens, entity_idx)
intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0]
intent_f1 = f1_score(intent_pred.argmax(1), intent_idx)
if entity_idx.sum().item() == 0 or torch.tensor(
entity_pred).sum().item() == 0:
entity_acc = get_token_accuracy(entity_idx.cpu(),
torch.tensor(entity_pred).cpu())[0]
else:
entity_acc = get_token_accuracy(
entity_idx.cpu(),
torch.tensor(entity_pred).cpu(),
ignore_index=self.dataset.pad_token_id)[0]
intent_loss = self.intent_loss_fn(
intent_pred,
intent_idx.long(),
) * self.intent_loss_weight
#intent_center_loss = self.intent_center_loss_fn(feature, intent_idx.long(),) * self.intent_loss_weight
entity_loss = -entity_loss * self.entity_loss_weight
return {
"val_intent_acc": torch.Tensor([intent_acc]),
"val_intent_f1": torch.Tensor([intent_f1]),
"val_entity_acc": torch.Tensor([entity_acc]),
"val_loss": intent_loss + entity_loss # + intent_center_loss,
}
def training_step(self, batch, batch_idx, optimizer_idx):
self.model.train()
tokens, intent_idx, entity_idx = batch
feature, intent_pred, entity_pred, entity_loss = self.forward(
tokens, entity_idx)
intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0]
intent_f1 = f1_score(intent_pred.argmax(1), intent_idx)
if entity_idx.sum().item() == 0 or torch.tensor(
entity_pred).sum().item() == 0:
entity_acc = get_token_accuracy(entity_idx.cpu(),
torch.tensor(entity_pred).cpu())[0]
else:
entity_acc = get_token_accuracy(
entity_idx.cpu(),
torch.tensor(entity_pred).cpu(),
ignore_index=self.dataset.pad_token_id)[0]
tensorboard_logs = {
"train/intent/acc": intent_acc,
"train/intent/f1": intent_f1,
"train/entity/acc": entity_acc,
}
if optimizer_idx == 0:
intent_loss = self.intent_loss_fn(
intent_pred,
intent_idx.long(),
) * self.intent_loss_weight
tensorboard_logs["train/intent/loss"] = intent_loss
return {
"loss": intent_loss,
"log": tensorboard_logs,
}
if optimizer_idx == 1:
entity_loss = -entity_loss * self.entity_loss_weight
tensorboard_logs["train/entity/loss"] = entity_loss
return {
"loss": entity_loss,
"log": tensorboard_logs,
}
if optimizer_idx == 2:
intent_center_loss = self.intent_center_loss_fn(
feature,
intent_idx.long(),
) * self.intent_center_loss_weight
tensorboard_logs["train/intent/center_loss"] = intent_center_loss
return {
"loss": intent_center_loss,
"log": tensorboard_logs,
}
def validation_step(self, batch, batch_idx):
self.model.eval()
tokens, intent_idx, entity_idx, text = batch
intent_pred, entity_pred = self.forward(tokens)
intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0]
entity_acc = get_token_accuracy(
entity_pred.argmax(2),
entity_idx,
ignore_index=self.dataset.pad_token_id)[0]
intent_f1 = f1_score(intent_pred.argmax(1), intent_idx)
intent_loss = self.intent_loss_fn(intent_pred, intent_idx.squeeze(1))
entity_loss = self.entity_loss_fn(
entity_pred.transpose(1, 2),
entity_idx.long(),
)
return {
"val_intent_acc": torch.Tensor([intent_acc]),
"val_entity_acc": torch.Tensor([entity_acc]),
"val_intent_f1": intent_f1,
"val_intent_loss": intent_loss,
"val_entity_loss": entity_loss,
"val_loss": intent_loss + entity_loss,
}
def validation_step(self, batch, batch_idx):
self.model.eval()
inputs, intent_idx, entity_idx = batch
(input_ids, token_type_ids) = inputs
intent_pred, entity_pred = self.forward(input_ids, token_type_ids)
intent_acc = get_accuracy(intent_idx.cpu(),
intent_pred.max(1)[1].cpu())[0]
entity_acc = get_token_accuracy(
entity_idx.cpu(),
entity_pred.max(2)[1].cpu(),
ignore_index=self.dataset.pad_token_id,
)[0]
intent_loss = self.loss_fn(intent_pred, intent_idx.squeeze(1))
entity_loss = self.loss_fn(entity_pred.transpose(1, 2),
entity_idx.long()) # , ignore_index=0)
return {
"val_intent_acc": torch.Tensor([intent_acc]),
"val_entity_acc": torch.Tensor([entity_acc]),
"val_intent_loss": intent_loss,
"val_entity_loss": entity_loss,
"val_loss": intent_loss + entity_loss,
}
def training_step(self, batch, batch_idx):
self.model.train()
input_ids, token_type_ids, labels, labels_weight = batch
pred_tokens = self.forward(input_ids, token_type_ids)
loss_tokens = self.loss_fn(pred_tokens.transpose(1, 2), labels)
loss_space_tokens = self.loss_fn(pred_tokens.transpose(1, 2),
labels * labels_weight)
loss_space_tokens = loss_space_tokens * self.hparams.weight_value
total_loss = loss_tokens + loss_space_tokens
token_acc = get_token_accuracy(
labels.cpu(),
pred_tokens.max(2)[1].cpu(),
ignore_index=0,
)[0]
tensorboard_logs = {
"train/token_acc": token_acc,
"train/token_loss": loss_tokens,
"train/space_loss": loss_space_tokens
}
return {
"loss": total_loss,
"log": tensorboard_logs,
}
def training_step(self, batch, batch_idx, optimizer_idx):
self.model.train()
tokens, intent_idx, entity_idx = batch
intent_pred, entity_pred = self.forward(tokens)
intent_acc = get_accuracy(intent_idx.cpu(),
intent_pred.max(1)[1].cpu())[0]
entity_acc = get_token_accuracy(
entity_idx.cpu(),
entity_pred.max(2)[1].cpu(),
ignore_index=self.dataset.pad_token_id,
)[0]
tensorboard_logs = {
"train/intent/acc": intent_acc,
"train/entity/acc": entity_acc,
}
if optimizer_idx == 0:
intent_loss = self.loss_fn(intent_pred, intent_idx.squeeze(1))
tensorboard_logs["train/intent/loss"] = intent_loss
return {
"loss": intent_loss,
"log": tensorboard_logs,
}
if optimizer_idx == 1:
entity_loss = self.loss_fn(entity_pred.transpose(1, 2),
entity_idx.long())
tensorboard_logs["train/entity/loss"] = entity_loss
return {
"loss": entity_loss,
"log": tensorboard_logs,
}
def calc_acc(self, logprobs, y):
'''
- logprobs: [b,T*(D-1),|V|]
'''
_, pred = torch.max(logprobs, 2)
_, tot_correct, tot_valid = get_token_accuracy(y,
pred,
ignore_index=PAD)
return tot_correct.float() / tot_valid.float()
def accuracy(self, logprobs, X):
_, Xhat = torch.max(logprobs, dim=2)
T = X.shape[1] # NOTE: X[:,-1] is <EOS>
D = self.D
tot_correct = 0
tot_valids = 0
for t in range(T-D+1):
Xhat_t = Xhat[:, t*D:(t+1)*D]
_, num_correct, num_valids = get_token_accuracy(X[:,t:t+D],Xhat_t, ignore_index=PAD)
tot_correct += num_correct
tot_valids += num_valids
return tot_correct.float()/tot_valids.float()
def evaluate(self, test_dataset=None, saved_model=None):
total_loss = 0
overall_correct = 0
overall_tot = 0
step = 0
if not test_dataset:
dataset = self.dataset
else:
dataset = test_dataset
if saved_model:
self.model.load_state_dict(torch.load(saved_model))
if self.args.save_ans:
with open(self.args.save_ans, 'w') as f:
f.write('start dump \n')
self.model.eval()
with torch.no_grad():
for idx_b, batch in enumerate(self.get_batch(dataset)):
src, trg = batch
output = self.model(src, trg, teacher_forcing_ratio=0.0)
loss = F.nll_loss(output[1:].view(-1,
self.dataset.vocab.size()),
trg[1:].contiguous().view(-1),
ignore_index=PAD)
# total_loss += loss.data[0]
total_loss += loss.item()
res = torch.argmax(output, 2)
if self.args.save_ans:
for i in range(trg.size(1)):
print(
"target: ",
self.dataset.vocab.tensorConvertToLabels(
trg[1:, i], PAD))
print(
"output: ",
self.dataset.vocab.tensorConvertToLabels(
res[1:, i], PAD))
# print("target: ", self.dataset.vocab.tensorConvertToLabels(trg[1:, i], Constants.PAD),
# file=open(self.args.save_ans, 'a'))
# print("output: ", self.dataset.vocab.tensorConvertToLabels(res[1:, i], Constants.PAD),
# file=open(self.args.save_ans, 'a'))
accuracy, n_correct, n_total = get_token_accuracy(
trg[1:], res[1:], ignore_index=PAD)
overall_correct += n_correct.item()
overall_tot += n_total.item()
step = idx_b
return total_loss / step, overall_correct / overall_tot, step
def training_step(self, batch, batch_idx, optimizer_idx):
self.model.train()
tokens, intent_idx, entity_idx = batch
intent_pred, entity_pred, entity_crf_loss = self.forward(
tokens, entity_idx)
if torch.isnan(tokens).sum().item() > 0:
assert ValueError('tokens error')
if torch.isnan(intent_idx).sum().item() > 0:
assert ValueError('intent_idx error')
if torch.isnan(entity_idx).sum().item() > 0:
assert ValueError('entity_idx error')
if torch.isnan(intent_pred).sum().item() > 0:
assert ValueError('intent_pred error')
intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0]
intent_f1 = f1_score(intent_pred.argmax(1), intent_idx)
entity_acc = get_token_accuracy(entity_idx.cpu(),
torch.tensor(entity_pred).cpu())[0]
tensorboard_logs = {
"train/intent/acc": intent_acc,
"train/intent/f1": intent_f1,
"train/entity/acc": entity_acc,
}
if optimizer_idx == 0:
intent_loss = self.intent_loss_fn(intent_pred, intent_idx.long())
tensorboard_logs["train/intent/loss"] = intent_loss
return {
"loss": intent_loss,
"log": tensorboard_logs,
}
if optimizer_idx == 1:
tensorboard_logs["train/entity/loss"] = entity_crf_loss
return {
"loss": entity_crf_loss,
"log": tensorboard_logs,
}
def training_step(self, batch, batch_idx, optimizer_idx):
self.model.train()
inputs, intent_idx, entity_idx = batch
(input_ids, token_type_ids) = inputs
intent_pred, entity_pred = self.forward(input_ids, token_type_ids)
intent_acc = get_accuracy(intent_idx.cpu(),
intent_pred.max(1)[1].cpu())[0]
zero = torch.zeros_like(entity_idx).cpu()
acc_entity_idx = torch.where(entity_idx.cpu() < 0, zero,
entity_idx.cpu())
entity_acc = get_token_accuracy(
# entity_idx.cpu(),
acc_entity_idx,
entity_pred.max(2)[1].cpu(),
ignore_index=self.entity_o_index,
)[0]
tensorboard_logs = {
"train/intent/acc": intent_acc,
"train/entity/acc": entity_acc,
}
if optimizer_idx == 0:
intent_loss = self.intent_loss_fn(intent_pred,
intent_idx.squeeze(1))
tensorboard_logs["train/intent/loss"] = intent_loss
return {
"loss": intent_loss,
"log": tensorboard_logs,
}
if optimizer_idx == 1:
entity_loss = self.entity_loss_fn(entity_pred.transpose(1, 2),
entity_idx.long())
tensorboard_logs["train/entity/loss"] = entity_loss
return {
"loss": entity_loss,
"log": tensorboard_logs,
}
def validation_step(self, batch, batch_idx):
self.model.eval()
tokens, intent_idx, entity_idx = batch
intent_pred, entity_pred, entity_crf_loss = self.forward(
tokens, entity_idx)
intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0]
intent_f1 = f1_score(intent_pred.argmax(1), intent_idx)
entity_acc = get_token_accuracy(entity_idx.cpu(),
torch.tensor(entity_pred).cpu())[0]
intent_loss = self.intent_loss_fn(
intent_pred,
intent_idx.long(),
)
return {
"val_intent_acc": torch.Tensor([intent_acc]),
"val_intent_f1": torch.Tensor([intent_f1]),
"val_entity_acc": torch.Tensor([entity_acc]),
"val_loss": intent_loss + entity_crf_loss
}
def test_get_token_accuracy_ignore_index():
targets = torch.LongTensor([1, 2, 3, 4])
outputs = torch.LongTensor([1, 2, 3, 3])
accuracy, _, _ = get_token_accuracy(targets, outputs, ignore_index=4)
assert accuracy == 1.0
def test_get_token_accuracy():
targets = torch.LongTensor([1, 2, 3, 4])
outputs = torch.LongTensor([1, 2, 3, 3])
accuracy, _, _ = get_token_accuracy(targets, outputs)
assert accuracy == 0.75
def test_get_token_accuracy_2d_2d_2d_2d():
targets = torch.LongTensor([[1, 1], [2, 2], [3, 3]])
outputs = torch.LongTensor([[1, 1], [2, 3], [4, 4]])
accuracy, _, _ = get_token_accuracy(targets, outputs, ignore_index=3)
assert accuracy == 0.75
def test_get_token_accuracy_2d_3d():
targets = torch.LongTensor([[1, 1], [2, 2], [3, 3], [4, 4]])
outputs = torch.LongTensor([[[1, 1], [1, 1]], [[2, 2], [2, 2]],
[[3, 3], [3, 3]], [[3, 3], [3, 3]]])
accuracy, _, _ = get_token_accuracy(targets, outputs)
assert accuracy == 0.75
def test_get_token_accuracy_1d_2d():
targets = torch.LongTensor([1, 2, 3, 4])
outputs = torch.LongTensor([[1], [2], [3], [3]])
accuracy, _, _ = get_token_accuracy(targets, outputs)
assert accuracy == 0.75
