def evaluation(self, eval_type):
"""Evaluation all children, update child score. Note that the eval data should be the same"""
eval_samples = [
self.gen.sample(cfg.eval_b_num * cfg.batch_size,
cfg.max_bn * cfg.batch_size,
label_i=i) for i in range(cfg.k_label)
]
# Fd
if cfg.lambda_fd != 0:
nll_div = []
for label_i in range(cfg.k_label):
gen_data = GenDataIter(eval_samples[label_i])
nll_div.append(
NLL.cal_nll_with_label(self.gen, gen_data.loader, label_i,
self.mle_criterion))
Fd = sum(nll_div)
else:
Fd = 0
# Fq
if 'bleu' in eval_type:
bleu_score = []
for i in range(cfg.k_label):
bleu_score.append(
self.bleu[i].get_score(given_gram=int(eval_type[-1])))
Fq = sum(bleu_score)
elif 'Ra' in eval_type:
g_loss = 0
for i in range(cfg.k_label):
g_loss += torch.sigmoid(
self.eval_d_out_fake[i] -
torch.mean(self.eval_d_out_real[i])).sum()
Fq = g_loss.item()
else:
raise NotImplementedError("Evaluation '%s' is not implemented" %
eval_type)
score = cfg.lambda_fq * Fq + cfg.lambda_fd * Fd
return Fq, Fd, score
def evaluation(self, eval_type):
"""Evaluation all children, update child score. Note that the eval data should be the same"""
eval_samples = [
self.gen.sample(cfg.eval_b_num * cfg.batch_size,
cfg.max_bn * cfg.batch_size,
label_i=i) for i in range(cfg.k_label)
]
# Fd
if cfg.lambda_fd != 0:
nll_div = []
for label_i in range(cfg.k_label):
gen_data = GenDataIter(eval_samples[label_i])
nll_div.append(
NLL.cal_nll_with_label(self.gen, gen_data.loader, label_i,
self.mle_criterion))
if 'f1' in eval_type:
if cfg.k_label == 1:
Fd = nll_div[0] if len(nll_div) > 0 else 0
elif cfg.k_label == 2:
Fd = nll_div[0] * nll_div[1] / (
nll_div[0] + nll_div[1]) if len(nll_div) > 0 else 0
else:
raise NotImplementedError("k_label = %d is not supported" %
cfg.k_label)
else:
Fd = sum(nll_div)
else:
Fd = 0
# Fq
if 'nll' in eval_type:
nll_oracle = []
for label_i in range(cfg.k_label):
gen_data = GenDataIter(eval_samples[label_i])
if cfg.lambda_fq != 0:
nll_oracle.append(-NLL.cal_nll_with_label(
self.oracle_list[label_i], gen_data.loader, label_i,
self.mle_criterion))
if 'f1' in eval_type:
if cfg.k_label == 1:
Fq = nll_oracle[0] if len(nll_oracle) > 0 else 0
elif cfg.k_label == 2:
Fq = nll_oracle[0] * nll_oracle[1] / (
nll_oracle[0] +
nll_oracle[1]) if len(nll_oracle) > 0 else 0
else:
raise NotImplementedError("k_label = %d is not supported" %
cfg.k_label)
else: # sum
Fq = sum(nll_oracle)
elif eval_type == 'Ra':
g_loss = 0
for i in range(cfg.k_label):
g_loss += torch.sigmoid(
self.eval_d_out_fake[i] -
torch.mean(self.eval_d_out_real[i])).sum()
Fq = g_loss.item()
else:
raise NotImplementedError("Evaluation '%s' is not implemented" %
eval_type)
score = cfg.lambda_fq * Fq + cfg.lambda_fd * Fd
return Fq, Fd, score
