def validation_step(self, batch, batch_idx):
self.model.eval()
tokens, intent_idx, entity_idx, text = batch
intent_pred, entity_pred = self.forward(tokens)
intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0]
entity_acc = get_token_accuracy(
entity_pred.argmax(2),
entity_idx,
ignore_index=self.dataset.pad_token_id)[0]
intent_f1 = f1_score(intent_pred.argmax(1), intent_idx)
intent_loss = self.intent_loss_fn(intent_pred, intent_idx.squeeze(1))
entity_loss = self.entity_loss_fn(
entity_pred.transpose(1, 2),
entity_idx.long(),
)
return {
"val_intent_acc": torch.Tensor([intent_acc]),
"val_entity_acc": torch.Tensor([entity_acc]),
"val_intent_f1": intent_f1,
"val_intent_loss": intent_loss,
"val_entity_loss": entity_loss,
"val_loss": intent_loss + entity_loss,
}
def validation_step(self, batch, batch_idx):
self.model.eval()
tokens, intent_idx, entity_idx = batch
feature, intent_pred, entity_pred, entity_loss = self.forward(
tokens, entity_idx)
intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0]
intent_f1 = f1_score(intent_pred.argmax(1), intent_idx)
if entity_idx.sum().item() == 0 or torch.tensor(
entity_pred).sum().item() == 0:
entity_acc = get_token_accuracy(entity_idx.cpu(),
torch.tensor(entity_pred).cpu())[0]
else:
entity_acc = get_token_accuracy(
entity_idx.cpu(),
torch.tensor(entity_pred).cpu(),
ignore_index=self.dataset.pad_token_id)[0]
intent_loss = self.intent_loss_fn(
intent_pred,
intent_idx.long(),
) * self.intent_loss_weight
#intent_center_loss = self.intent_center_loss_fn(feature, intent_idx.long(),) * self.intent_loss_weight
entity_loss = -entity_loss * self.entity_loss_weight
return {
"val_intent_acc": torch.Tensor([intent_acc]),
"val_intent_f1": torch.Tensor([intent_f1]),
"val_entity_acc": torch.Tensor([entity_acc]),
"val_loss": intent_loss + entity_loss # + intent_center_loss,
}
def validation_step(self, batch, batch_idx):
self.model.eval()
inputs, intent_idx, entity_idx = batch
(input_ids, token_type_ids) = inputs
intent_pred, entity_pred = self.forward(input_ids, token_type_ids)
intent_acc = get_accuracy(intent_idx.cpu(),
intent_pred.max(1)[1].cpu())[0]
entity_acc = get_token_accuracy(
entity_idx.cpu(),
entity_pred.max(2)[1].cpu(),
ignore_index=self.dataset.pad_token_id,
)[0]
intent_loss = self.loss_fn(intent_pred, intent_idx.squeeze(1))
entity_loss = self.loss_fn(entity_pred.transpose(1, 2),
entity_idx.long()) # , ignore_index=0)
return {
"val_intent_acc": torch.Tensor([intent_acc]),
"val_entity_acc": torch.Tensor([entity_acc]),
"val_intent_loss": intent_loss,
"val_entity_loss": entity_loss,
"val_loss": intent_loss + entity_loss,
}
def training_step(self, batch, batch_idx, optimizer_idx):
self.model.train()
tokens, intent_idx, entity_idx = batch
intent_pred, entity_pred = self.forward(tokens)
intent_acc = get_accuracy(intent_idx.cpu(),
intent_pred.max(1)[1].cpu())[0]
entity_acc = get_token_accuracy(
entity_idx.cpu(),
entity_pred.max(2)[1].cpu(),
ignore_index=self.dataset.pad_token_id,
)[0]
tensorboard_logs = {
"train/intent/acc": intent_acc,
"train/entity/acc": entity_acc,
}
if optimizer_idx == 0:
intent_loss = self.loss_fn(intent_pred, intent_idx.squeeze(1))
tensorboard_logs["train/intent/loss"] = intent_loss
return {
"loss": intent_loss,
"log": tensorboard_logs,
}
if optimizer_idx == 1:
entity_loss = self.loss_fn(entity_pred.transpose(1, 2),
entity_idx.long())
tensorboard_logs["train/entity/loss"] = entity_loss
return {
"loss": entity_loss,
"log": tensorboard_logs,
}
def training_step(self, batch, batch_idx, optimizer_idx):
self.model.train()
tokens, intent_idx, entity_idx = batch
feature, intent_pred, entity_pred, entity_loss = self.forward(
tokens, entity_idx)
intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0]
intent_f1 = f1_score(intent_pred.argmax(1), intent_idx)
if entity_idx.sum().item() == 0 or torch.tensor(
entity_pred).sum().item() == 0:
entity_acc = get_token_accuracy(entity_idx.cpu(),
torch.tensor(entity_pred).cpu())[0]
else:
entity_acc = get_token_accuracy(
entity_idx.cpu(),
torch.tensor(entity_pred).cpu(),
ignore_index=self.dataset.pad_token_id)[0]
tensorboard_logs = {
"train/intent/acc": intent_acc,
"train/intent/f1": intent_f1,
"train/entity/acc": entity_acc,
}
if optimizer_idx == 0:
intent_loss = self.intent_loss_fn(
intent_pred,
intent_idx.long(),
) * self.intent_loss_weight
tensorboard_logs["train/intent/loss"] = intent_loss
return {
"loss": intent_loss,
"log": tensorboard_logs,
}
if optimizer_idx == 1:
entity_loss = -entity_loss * self.entity_loss_weight
tensorboard_logs["train/entity/loss"] = entity_loss
return {
"loss": entity_loss,
"log": tensorboard_logs,
}
if optimizer_idx == 2:
intent_center_loss = self.intent_center_loss_fn(
feature,
intent_idx.long(),
) * self.intent_center_loss_weight
tensorboard_logs["train/intent/center_loss"] = intent_center_loss
return {
"loss": intent_center_loss,
"log": tensorboard_logs,
}
def training_step(self, batch, batch_idx, optimizer_idx):
self.model.train()
tokens, intent_idx, entity_idx = batch
intent_pred, entity_pred, entity_crf_loss = self.forward(
tokens, entity_idx)
if torch.isnan(tokens).sum().item() > 0:
assert ValueError('tokens error')
if torch.isnan(intent_idx).sum().item() > 0:
assert ValueError('intent_idx error')
if torch.isnan(entity_idx).sum().item() > 0:
assert ValueError('entity_idx error')
if torch.isnan(intent_pred).sum().item() > 0:
assert ValueError('intent_pred error')
intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0]
intent_f1 = f1_score(intent_pred.argmax(1), intent_idx)
entity_acc = get_token_accuracy(entity_idx.cpu(),
torch.tensor(entity_pred).cpu())[0]
tensorboard_logs = {
"train/intent/acc": intent_acc,
"train/intent/f1": intent_f1,
"train/entity/acc": entity_acc,
}
if optimizer_idx == 0:
intent_loss = self.intent_loss_fn(intent_pred, intent_idx.long())
tensorboard_logs["train/intent/loss"] = intent_loss
return {
"loss": intent_loss,
"log": tensorboard_logs,
}
if optimizer_idx == 1:
tensorboard_logs["train/entity/loss"] = entity_crf_loss
return {
"loss": entity_crf_loss,
"log": tensorboard_logs,
}
def training_step(self, batch, batch_idx, optimizer_idx):
self.model.train()
inputs, intent_idx, entity_idx = batch
(input_ids, token_type_ids) = inputs
intent_pred, entity_pred = self.forward(input_ids, token_type_ids)
intent_acc = get_accuracy(intent_idx.cpu(),
intent_pred.max(1)[1].cpu())[0]
zero = torch.zeros_like(entity_idx).cpu()
acc_entity_idx = torch.where(entity_idx.cpu() < 0, zero,
entity_idx.cpu())
entity_acc = get_token_accuracy(
# entity_idx.cpu(),
acc_entity_idx,
entity_pred.max(2)[1].cpu(),
ignore_index=self.entity_o_index,
)[0]
tensorboard_logs = {
"train/intent/acc": intent_acc,
"train/entity/acc": entity_acc,
}
if optimizer_idx == 0:
intent_loss = self.intent_loss_fn(intent_pred,
intent_idx.squeeze(1))
tensorboard_logs["train/intent/loss"] = intent_loss
return {
"loss": intent_loss,
"log": tensorboard_logs,
}
if optimizer_idx == 1:
entity_loss = self.entity_loss_fn(entity_pred.transpose(1, 2),
entity_idx.long())
tensorboard_logs["train/entity/loss"] = entity_loss
return {
"loss": entity_loss,
"log": tensorboard_logs,
}
def validation_step(self, batch, batch_idx):
self.model.eval()
tokens, intent_idx, entity_idx = batch
intent_pred, entity_pred, entity_crf_loss = self.forward(
tokens, entity_idx)
intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0]
intent_f1 = f1_score(intent_pred.argmax(1), intent_idx)
entity_acc = get_token_accuracy(entity_idx.cpu(),
torch.tensor(entity_pred).cpu())[0]
intent_loss = self.intent_loss_fn(
intent_pred,
intent_idx.long(),
)
return {
"val_intent_acc": torch.Tensor([intent_acc]),
"val_intent_f1": torch.Tensor([intent_f1]),
"val_entity_acc": torch.Tensor([entity_acc]),
"val_loss": intent_loss + entity_crf_loss
}
def test_get_accuracy_ignore_index():
targets = torch.LongTensor([1, 2, 3, 4])
outputs = torch.LongTensor([1, 2, 3, 3])
accuracy, _, _ = get_accuracy(targets, outputs, ignore_index=4)
assert accuracy == 1.0
def test_get_accuracy_1d_2d_top_k():
targets = torch.LongTensor([1, 2, 3, 4])
outputs = torch.LongTensor([[1, 1], [2, 2], [3, 3], [3, 4]])
accuracy, _, _ = get_accuracy(targets, outputs, k=3)
assert accuracy == 1.0
def test_get_accuracy_1d_2d():
targets = torch.LongTensor([1, 2, 3, 4])
outputs = torch.LongTensor([[1], [2], [3], [3]])
accuracy, _, _ = get_accuracy(targets, outputs)
assert accuracy == 0.75
def test_get_accuracy():
targets = torch.LongTensor([1, 2, 3, 4])
outputs = torch.LongTensor([1, 2, 3, 3])
accuracy, _, _ = get_accuracy(targets, outputs)
assert accuracy == 0.75
def test_get_accuracy_2d_3d():
targets = torch.LongTensor([[1, 1], [2, 2], [3, 3], [4, 4]])
outputs = torch.LongTensor([[[1, 1], [1, 1]], [[2, 2], [2, 2]],
[[3, 3], [3, 3]], [[3, 3], [3, 3]]])
accuracy, _, _ = get_accuracy(targets, outputs)
assert accuracy == 0.75