def validation_step(self, batch, batch_idx): self.model.eval() tokens, intent_idx, entity_idx = batch feature, intent_pred, entity_pred, entity_loss = self.forward( tokens, entity_idx) intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0] intent_f1 = f1_score(intent_pred.argmax(1), intent_idx) if entity_idx.sum().item() == 0 or torch.tensor( entity_pred).sum().item() == 0: entity_acc = get_token_accuracy(entity_idx.cpu(), torch.tensor(entity_pred).cpu())[0] else: entity_acc = get_token_accuracy( entity_idx.cpu(), torch.tensor(entity_pred).cpu(), ignore_index=self.dataset.pad_token_id)[0] intent_loss = self.intent_loss_fn( intent_pred, intent_idx.long(), ) * self.intent_loss_weight #intent_center_loss = self.intent_center_loss_fn(feature, intent_idx.long(),) * self.intent_loss_weight entity_loss = -entity_loss * self.entity_loss_weight return { "val_intent_acc": torch.Tensor([intent_acc]), "val_intent_f1": torch.Tensor([intent_f1]), "val_entity_acc": torch.Tensor([entity_acc]), "val_loss": intent_loss + entity_loss # + intent_center_loss, }
def training_step(self, batch, batch_idx, optimizer_idx): self.model.train() tokens, intent_idx, entity_idx = batch feature, intent_pred, entity_pred, entity_loss = self.forward( tokens, entity_idx) intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0] intent_f1 = f1_score(intent_pred.argmax(1), intent_idx) if entity_idx.sum().item() == 0 or torch.tensor( entity_pred).sum().item() == 0: entity_acc = get_token_accuracy(entity_idx.cpu(), torch.tensor(entity_pred).cpu())[0] else: entity_acc = get_token_accuracy( entity_idx.cpu(), torch.tensor(entity_pred).cpu(), ignore_index=self.dataset.pad_token_id)[0] tensorboard_logs = { "train/intent/acc": intent_acc, "train/intent/f1": intent_f1, "train/entity/acc": entity_acc, } if optimizer_idx == 0: intent_loss = self.intent_loss_fn( intent_pred, intent_idx.long(), ) * self.intent_loss_weight tensorboard_logs["train/intent/loss"] = intent_loss return { "loss": intent_loss, "log": tensorboard_logs, } if optimizer_idx == 1: entity_loss = -entity_loss * self.entity_loss_weight tensorboard_logs["train/entity/loss"] = entity_loss return { "loss": entity_loss, "log": tensorboard_logs, } if optimizer_idx == 2: intent_center_loss = self.intent_center_loss_fn( feature, intent_idx.long(), ) * self.intent_center_loss_weight tensorboard_logs["train/intent/center_loss"] = intent_center_loss return { "loss": intent_center_loss, "log": tensorboard_logs, }
def validation_step(self, batch, batch_idx): self.model.eval() tokens, intent_idx, entity_idx, text = batch intent_pred, entity_pred = self.forward(tokens) intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0] entity_acc = get_token_accuracy( entity_pred.argmax(2), entity_idx, ignore_index=self.dataset.pad_token_id)[0] intent_f1 = f1_score(intent_pred.argmax(1), intent_idx) intent_loss = self.intent_loss_fn(intent_pred, intent_idx.squeeze(1)) entity_loss = self.entity_loss_fn( entity_pred.transpose(1, 2), entity_idx.long(), ) return { "val_intent_acc": torch.Tensor([intent_acc]), "val_entity_acc": torch.Tensor([entity_acc]), "val_intent_f1": intent_f1, "val_intent_loss": intent_loss, "val_entity_loss": entity_loss, "val_loss": intent_loss + entity_loss, }
def validation_step(self, batch, batch_idx): self.model.eval() inputs, intent_idx, entity_idx = batch (input_ids, token_type_ids) = inputs intent_pred, entity_pred = self.forward(input_ids, token_type_ids) intent_acc = get_accuracy(intent_idx.cpu(), intent_pred.max(1)[1].cpu())[0] entity_acc = get_token_accuracy( entity_idx.cpu(), entity_pred.max(2)[1].cpu(), ignore_index=self.dataset.pad_token_id, )[0] intent_loss = self.loss_fn(intent_pred, intent_idx.squeeze(1)) entity_loss = self.loss_fn(entity_pred.transpose(1, 2), entity_idx.long()) # , ignore_index=0) return { "val_intent_acc": torch.Tensor([intent_acc]), "val_entity_acc": torch.Tensor([entity_acc]), "val_intent_loss": intent_loss, "val_entity_loss": entity_loss, "val_loss": intent_loss + entity_loss, }
def training_step(self, batch, batch_idx): self.model.train() input_ids, token_type_ids, labels, labels_weight = batch pred_tokens = self.forward(input_ids, token_type_ids) loss_tokens = self.loss_fn(pred_tokens.transpose(1, 2), labels) loss_space_tokens = self.loss_fn(pred_tokens.transpose(1, 2), labels * labels_weight) loss_space_tokens = loss_space_tokens * self.hparams.weight_value total_loss = loss_tokens + loss_space_tokens token_acc = get_token_accuracy( labels.cpu(), pred_tokens.max(2)[1].cpu(), ignore_index=0, )[0] tensorboard_logs = { "train/token_acc": token_acc, "train/token_loss": loss_tokens, "train/space_loss": loss_space_tokens } return { "loss": total_loss, "log": tensorboard_logs, }
def training_step(self, batch, batch_idx, optimizer_idx): self.model.train() tokens, intent_idx, entity_idx = batch intent_pred, entity_pred = self.forward(tokens) intent_acc = get_accuracy(intent_idx.cpu(), intent_pred.max(1)[1].cpu())[0] entity_acc = get_token_accuracy( entity_idx.cpu(), entity_pred.max(2)[1].cpu(), ignore_index=self.dataset.pad_token_id, )[0] tensorboard_logs = { "train/intent/acc": intent_acc, "train/entity/acc": entity_acc, } if optimizer_idx == 0: intent_loss = self.loss_fn(intent_pred, intent_idx.squeeze(1)) tensorboard_logs["train/intent/loss"] = intent_loss return { "loss": intent_loss, "log": tensorboard_logs, } if optimizer_idx == 1: entity_loss = self.loss_fn(entity_pred.transpose(1, 2), entity_idx.long()) tensorboard_logs["train/entity/loss"] = entity_loss return { "loss": entity_loss, "log": tensorboard_logs, }
def calc_acc(self, logprobs, y): ''' - logprobs: [b,T*(D-1),|V|] ''' _, pred = torch.max(logprobs, 2) _, tot_correct, tot_valid = get_token_accuracy(y, pred, ignore_index=PAD) return tot_correct.float() / tot_valid.float()
def accuracy(self, logprobs, X): _, Xhat = torch.max(logprobs, dim=2) T = X.shape[1] # NOTE: X[:,-1] is <EOS> D = self.D tot_correct = 0 tot_valids = 0 for t in range(T-D+1): Xhat_t = Xhat[:, t*D:(t+1)*D] _, num_correct, num_valids = get_token_accuracy(X[:,t:t+D],Xhat_t, ignore_index=PAD) tot_correct += num_correct tot_valids += num_valids return tot_correct.float()/tot_valids.float()
def evaluate(self, test_dataset=None, saved_model=None): total_loss = 0 overall_correct = 0 overall_tot = 0 step = 0 if not test_dataset: dataset = self.dataset else: dataset = test_dataset if saved_model: self.model.load_state_dict(torch.load(saved_model)) if self.args.save_ans: with open(self.args.save_ans, 'w') as f: f.write('start dump \n') self.model.eval() with torch.no_grad(): for idx_b, batch in enumerate(self.get_batch(dataset)): src, trg = batch output = self.model(src, trg, teacher_forcing_ratio=0.0) loss = F.nll_loss(output[1:].view(-1, self.dataset.vocab.size()), trg[1:].contiguous().view(-1), ignore_index=PAD) # total_loss += loss.data[0] total_loss += loss.item() res = torch.argmax(output, 2) if self.args.save_ans: for i in range(trg.size(1)): print( "target: ", self.dataset.vocab.tensorConvertToLabels( trg[1:, i], PAD)) print( "output: ", self.dataset.vocab.tensorConvertToLabels( res[1:, i], PAD)) # print("target: ", self.dataset.vocab.tensorConvertToLabels(trg[1:, i], Constants.PAD), # file=open(self.args.save_ans, 'a')) # print("output: ", self.dataset.vocab.tensorConvertToLabels(res[1:, i], Constants.PAD), # file=open(self.args.save_ans, 'a')) accuracy, n_correct, n_total = get_token_accuracy( trg[1:], res[1:], ignore_index=PAD) overall_correct += n_correct.item() overall_tot += n_total.item() step = idx_b return total_loss / step, overall_correct / overall_tot, step
def training_step(self, batch, batch_idx, optimizer_idx): self.model.train() tokens, intent_idx, entity_idx = batch intent_pred, entity_pred, entity_crf_loss = self.forward( tokens, entity_idx) if torch.isnan(tokens).sum().item() > 0: assert ValueError('tokens error') if torch.isnan(intent_idx).sum().item() > 0: assert ValueError('intent_idx error') if torch.isnan(entity_idx).sum().item() > 0: assert ValueError('entity_idx error') if torch.isnan(intent_pred).sum().item() > 0: assert ValueError('intent_pred error') intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0] intent_f1 = f1_score(intent_pred.argmax(1), intent_idx) entity_acc = get_token_accuracy(entity_idx.cpu(), torch.tensor(entity_pred).cpu())[0] tensorboard_logs = { "train/intent/acc": intent_acc, "train/intent/f1": intent_f1, "train/entity/acc": entity_acc, } if optimizer_idx == 0: intent_loss = self.intent_loss_fn(intent_pred, intent_idx.long()) tensorboard_logs["train/intent/loss"] = intent_loss return { "loss": intent_loss, "log": tensorboard_logs, } if optimizer_idx == 1: tensorboard_logs["train/entity/loss"] = entity_crf_loss return { "loss": entity_crf_loss, "log": tensorboard_logs, }
def training_step(self, batch, batch_idx, optimizer_idx): self.model.train() inputs, intent_idx, entity_idx = batch (input_ids, token_type_ids) = inputs intent_pred, entity_pred = self.forward(input_ids, token_type_ids) intent_acc = get_accuracy(intent_idx.cpu(), intent_pred.max(1)[1].cpu())[0] zero = torch.zeros_like(entity_idx).cpu() acc_entity_idx = torch.where(entity_idx.cpu() < 0, zero, entity_idx.cpu()) entity_acc = get_token_accuracy( # entity_idx.cpu(), acc_entity_idx, entity_pred.max(2)[1].cpu(), ignore_index=self.entity_o_index, )[0] tensorboard_logs = { "train/intent/acc": intent_acc, "train/entity/acc": entity_acc, } if optimizer_idx == 0: intent_loss = self.intent_loss_fn(intent_pred, intent_idx.squeeze(1)) tensorboard_logs["train/intent/loss"] = intent_loss return { "loss": intent_loss, "log": tensorboard_logs, } if optimizer_idx == 1: entity_loss = self.entity_loss_fn(entity_pred.transpose(1, 2), entity_idx.long()) tensorboard_logs["train/entity/loss"] = entity_loss return { "loss": entity_loss, "log": tensorboard_logs, }
def validation_step(self, batch, batch_idx): self.model.eval() tokens, intent_idx, entity_idx = batch intent_pred, entity_pred, entity_crf_loss = self.forward( tokens, entity_idx) intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0] intent_f1 = f1_score(intent_pred.argmax(1), intent_idx) entity_acc = get_token_accuracy(entity_idx.cpu(), torch.tensor(entity_pred).cpu())[0] intent_loss = self.intent_loss_fn( intent_pred, intent_idx.long(), ) return { "val_intent_acc": torch.Tensor([intent_acc]), "val_intent_f1": torch.Tensor([intent_f1]), "val_entity_acc": torch.Tensor([entity_acc]), "val_loss": intent_loss + entity_crf_loss }
def test_get_token_accuracy_ignore_index(): targets = torch.LongTensor([1, 2, 3, 4]) outputs = torch.LongTensor([1, 2, 3, 3]) accuracy, _, _ = get_token_accuracy(targets, outputs, ignore_index=4) assert accuracy == 1.0
def test_get_token_accuracy(): targets = torch.LongTensor([1, 2, 3, 4]) outputs = torch.LongTensor([1, 2, 3, 3]) accuracy, _, _ = get_token_accuracy(targets, outputs) assert accuracy == 0.75
def test_get_token_accuracy_2d_2d_2d_2d(): targets = torch.LongTensor([[1, 1], [2, 2], [3, 3]]) outputs = torch.LongTensor([[1, 1], [2, 3], [4, 4]]) accuracy, _, _ = get_token_accuracy(targets, outputs, ignore_index=3) assert accuracy == 0.75
def test_get_token_accuracy_2d_3d(): targets = torch.LongTensor([[1, 1], [2, 2], [3, 3], [4, 4]]) outputs = torch.LongTensor([[[1, 1], [1, 1]], [[2, 2], [2, 2]], [[3, 3], [3, 3]], [[3, 3], [3, 3]]]) accuracy, _, _ = get_token_accuracy(targets, outputs) assert accuracy == 0.75
def test_get_token_accuracy_1d_2d(): targets = torch.LongTensor([1, 2, 3, 4]) outputs = torch.LongTensor([[1], [2], [3], [3]]) accuracy, _, _ = get_token_accuracy(targets, outputs) assert accuracy == 0.75