def evaluation(self, eval_type): """Evaluation all children, update child score. Note that the eval data should be the same""" eval_samples = [ self.gen.sample(cfg.eval_b_num * cfg.batch_size, cfg.max_bn * cfg.batch_size, label_i=i) for i in range(cfg.k_label) ] # Fd if cfg.lambda_fd != 0: nll_div = [] for label_i in range(cfg.k_label): gen_data = GenDataIter(eval_samples[label_i]) nll_div.append( NLL.cal_nll_with_label(self.gen, gen_data.loader, label_i, self.mle_criterion)) Fd = sum(nll_div) else: Fd = 0 # Fq if 'bleu' in eval_type: bleu_score = [] for i in range(cfg.k_label): bleu_score.append( self.bleu[i].get_score(given_gram=int(eval_type[-1]))) Fq = sum(bleu_score) elif 'Ra' in eval_type: g_loss = 0 for i in range(cfg.k_label): g_loss += torch.sigmoid( self.eval_d_out_fake[i] - torch.mean(self.eval_d_out_real[i])).sum() Fq = g_loss.item() else: raise NotImplementedError("Evaluation '%s' is not implemented" % eval_type) score = cfg.lambda_fq * Fq + cfg.lambda_fd * Fd return Fq, Fd, score
def evaluation(self, eval_type): """Evaluation all children, update child score. Note that the eval data should be the same""" eval_samples = [ self.gen.sample(cfg.eval_b_num * cfg.batch_size, cfg.max_bn * cfg.batch_size, label_i=i) for i in range(cfg.k_label) ] # Fd if cfg.lambda_fd != 0: nll_div = [] for label_i in range(cfg.k_label): gen_data = GenDataIter(eval_samples[label_i]) nll_div.append( NLL.cal_nll_with_label(self.gen, gen_data.loader, label_i, self.mle_criterion)) if 'f1' in eval_type: if cfg.k_label == 1: Fd = nll_div[0] if len(nll_div) > 0 else 0 elif cfg.k_label == 2: Fd = nll_div[0] * nll_div[1] / ( nll_div[0] + nll_div[1]) if len(nll_div) > 0 else 0 else: raise NotImplementedError("k_label = %d is not supported" % cfg.k_label) else: Fd = sum(nll_div) else: Fd = 0 # Fq if 'nll' in eval_type: nll_oracle = [] for label_i in range(cfg.k_label): gen_data = GenDataIter(eval_samples[label_i]) if cfg.lambda_fq != 0: nll_oracle.append(-NLL.cal_nll_with_label( self.oracle_list[label_i], gen_data.loader, label_i, self.mle_criterion)) if 'f1' in eval_type: if cfg.k_label == 1: Fq = nll_oracle[0] if len(nll_oracle) > 0 else 0 elif cfg.k_label == 2: Fq = nll_oracle[0] * nll_oracle[1] / ( nll_oracle[0] + nll_oracle[1]) if len(nll_oracle) > 0 else 0 else: raise NotImplementedError("k_label = %d is not supported" % cfg.k_label) else: # sum Fq = sum(nll_oracle) elif eval_type == 'Ra': g_loss = 0 for i in range(cfg.k_label): g_loss += torch.sigmoid( self.eval_d_out_fake[i] - torch.mean(self.eval_d_out_real[i])).sum() Fq = g_loss.item() else: raise NotImplementedError("Evaluation '%s' is not implemented" % eval_type) score = cfg.lambda_fq * Fq + cfg.lambda_fd * Fd return Fq, Fd, score