def evaluate_ppl(model, data_iter, verbos=True): toatl_ppl = 0 toatl_nums = 0 padding_idx = model.padding_idx for batch_id, inputs in enumerate(data_iter, 1): enc_inputs = inputs dec_inputs = inputs.tgt[0][:, :-1], inputs.tgt[1] - 1 probs = model(enc_inputs, dec_inputs).logits tgt = inputs.tgt[0][:, 1:] batch_ppl = perplexity(logits=probs, targets=tgt, ) toatl_ppl += batch_ppl toatl_nums += 1 ppl = toatl_ppl / toatl_nums message = f"PPL: {ppl:.4f}" if verbos: print(message) else: return message
def collect_metrics(self, outputs, target, epoch=-1): """ collect_metrics """ num_samples = target.size(0) metrics = Pack(num_samples=num_samples) loss = 0 # test begin # nll = self.nll(torch.log(outputs.posterior_attn+1e-10), outputs.attn_index) # loss += nll # attn_acc = attn_accuracy(outputs.posterior_attn, outputs.attn_index) # metrics.add(attn_acc=attn_acc) # metrics.add(loss=loss) # return metrics # test end logits = outputs.logits scores = -self.nll_loss(logits, target, reduction=False) nll_loss = self.nll_loss(logits, target) num_words = target.ne(self.padding_idx).sum().item() acc = accuracy(logits, target, padding_idx=self.padding_idx) metrics.add(nll=(nll_loss, num_words), acc=acc) if self.use_posterior: kl_loss = self.kl_loss(torch.log(outputs.prior_attn + 1e-10), outputs.posterior_attn.detach()) metrics.add(kl=kl_loss) if self.use_bow: bow_logits = outputs.bow_logits bow_labels = target[:, :-1] bow_logits = bow_logits.repeat(1, bow_labels.size(-1), 1) bow = self.nll_loss(bow_logits, bow_labels) loss += bow metrics.add(bow=bow) if self.use_dssm: mse = self.mse_loss(outputs.dssm, outputs.reply_vec.detach()) loss += mse metrics.add(mse=mse) pos_logits = outputs.pos_logits pos_target = torch.ones_like(pos_logits) neg_logits = outputs.neg_logits neg_target = torch.zeros_like(neg_logits) pos_loss = F.binary_cross_entropy_with_logits(pos_logits, pos_target, reduction='none') neg_loss = F.binary_cross_entropy_with_logits(neg_logits, neg_target, reduction='none') loss += (pos_loss + neg_loss).mean() metrics.add(pos_loss=pos_loss.mean(), neg_loss=neg_loss.mean()) if epoch == -1 or epoch > self.pretrain_epoch or \ (self.use_bow is not True and self.use_dssm is not True): loss += nll_loss loss += kl_loss if self.use_pg: posterior_probs = outputs.posterior_attn.gather( 1, outputs.indexs.view(-1, 1)) reward = -perplexity(logits, target, self.weight, self.padding_idx) * 100 pg_loss = -(reward.detach() - self.baseline) * posterior_probs.view(-1) pg_loss = pg_loss.mean() loss += pg_loss metrics.add(pg_loss=pg_loss, reward=reward.mean()) if 'attn_index' in outputs: attn_acc = attn_accuracy(outputs.posterior_attn, outputs.attn_index) metrics.add(attn_acc=attn_acc) else: loss += nll_loss metrics.add(loss=loss) return metrics, scores