def get_metrics(config, oracle_loader, test_file, gen_file, g_pretrain_loss,
x_real, sess):
# set up evaluation metric
metrics = []
if config['nll_gen']:
nll_gen = Nll(oracle_loader,
g_pretrain_loss,
x_real,
sess,
name='nll_gen')
metrics.append(nll_gen)
if config['doc_embsim']:
doc_embsim = DocEmbSim(test_file,
gen_file,
config['vocab_size'],
name='doc_embsim')
metrics.append(doc_embsim)
if config['bleu']:
for i in range(2, 6):
bleu = Bleu(test_text=gen_file,
real_text=test_file,
gram=i,
name='bleu' + str(i))
metrics.append(bleu)
if config['selfbleu']:
for i in range(2, 6):
selfbleu = SelfBleu(test_text=gen_file,
gram=i,
name='selfbleu' + str(i))
metrics.append(selfbleu)
return metrics
def get_metrics(config, oracle_loader, test_file, gen_file, g_pretrain_loss, x_real, x_topic, sess, json_file):
# set up evaluation metric
metrics = []
if config['nll_gen']:
nll_gen = NllTopic(oracle_loader, g_pretrain_loss, x_real, sess, name='nll_gen', x_topic=x_topic)
metrics.append(nll_gen)
if config['doc_embsim']:
doc_embsim = DocEmbSim(test_file, gen_file, config['vocab_size'], name='doc_embsim')
metrics.append(doc_embsim)
if config['bleu']:
for i in [2, 4]: # range(2, 6):
bleu = Bleu(test_text=json_file, real_text=test_file, gram=i, name='bleu' + str(i))
metrics.append(bleu)
if config['selfbleu']:
for i in [4]:
selfbleu = SelfBleu(test_text=json_file, gram=i, name='selfbleu' + str(i))
metrics.append(selfbleu)
if config['KL']:
KL_div = KL_divergence(oracle_loader, json_file, name='KL_divergence')
metrics.append(KL_div)
if config['earth_mover']:
EM_div = EarthMover(oracle_loader, json_file, name='Earth_Mover_Distance')
metrics.append(EM_div)
return metrics
def get_metrics(config, oracle_loader, test_file, gen_file, g_pretrain_loss,
x_real, sess, json_file):
# set up evaluation metric
metrics = []
if config['nll_gen']:
nll_gen = Nll(oracle_loader,
g_pretrain_loss,
x_real,
sess,
name='nll_gen')
metrics.append(nll_gen)
if config['bleu']:
for i in range(2, 6):
bleu = Bleu(test_text=json_file,
real_text=test_file,
gram=i,
name='bleu' + str(i))
metrics.append(bleu)
if config['selfbleu']:
for i in range(2, 6):
selfbleu = SelfBleu(test_text=json_file,
gram=i,
name='selfbleu' + str(i))
metrics.append(selfbleu)
if config['KL']:
KL_div = KL_divergence(oracle_loader, json_file, name='KL_divergence')
metrics.append(KL_div)
return metrics
def init_real_metric(self):
from utils.metrics.Nll import Nll
from utils.metrics.PPL import PPL
from utils.metrics.DocEmbSim import DocEmbSim
from utils.others.Bleu import Bleu
from utils.metrics.SelfBleu import SelfBleu
if self.valid_ppl:
valid_ppl = PPL(self.valid_data_loader, self.generator, self.sess)
valid_ppl.set_name('valid_ppl')
self.add_metric(valid_ppl)
if self.nll_gen:
nll_gen = Nll(self.gen_data_loader, self.generator, self.sess)
nll_gen.set_name('nll_gen')
self.add_metric(nll_gen)
if self.doc_embsim:
doc_embsim = DocEmbSim(self.oracle_file, self.generator_file,
self.vocab_size)
doc_embsim.set_name('doc_embsim')
self.add_metric(doc_embsim)
if self.bleu:
FLAGS = tf.app.flags.FLAGS
dataset = FLAGS.data
if dataset == "image_coco":
real_text = 'data/testdata/test_image_coco.txt'
elif dataset == "emnlp_news":
real_text = 'data/testdata/test_emnlp_news.txt'
else:
raise ValueError
for i in range(3, 4):
bleu = Bleu(test_text=self.text_file,
real_text=real_text,
gram=i)
bleu.set_name(f"Bleu{i}")
self.add_metric(bleu)
if self.selfbleu:
for i in range(2, 6):
selfbleu = SelfBleu(test_text=self.text_file, gram=i)
selfbleu.set_name(f"Selfbleu{i}")
self.add_metric(selfbleu)
def init_real_metric(self):
from utils.metrics.DocEmbSim import DocEmbSim
docsim = DocEmbSim(oracle_file=self.oracle_file,
generator_file=self.generator_file,
num_vocabulary=self.vocab_size)
self.add_metric(docsim)
inll = Nll(data_loader=self.gen_data_loader,
rnn=self.generator,
sess=self.sess)
inll.set_name('nll-test')
self.add_metric(inll)
bleu = Bleu(test_text=self.test_file,
real_text='data/image_coco.txt',
gram=2)
self.add_metric(bleu)
sbleu = SelfBleu(test_text=self.test_file, gram=2)
self.add_metric(sbleu)
def get_metrics(gen_file, original_file):
# set up evaluation metric
metrics = []
for i in range(1, 5):
selfbleu = SelfBleu(original_text=original_file, transfer_text=gen_file, gram=i, name='selfbleu' + str(i))
metrics.append(selfbleu)
# if config['nll_gen']:
# nll_gen = Nll(oracle_loader, g_pretrain_loss, text_ids, sess, name='nll_gen')
# metrics.append(nll_gen)
# if config['doc_embsim']:
# doc_embsim = DocEmbSim(test_file, gen_file, config['vocab_size'], name='doc_embsim')
# metrics.append(doc_embsim)
# if config['bleu']:
# for i in range(2, 6):
# bleu = Bleu(test_text=gen_file, real_text=test_file, gram=i, name='bleu' + str(i))
# metrics.append(bleu)
# if config['selfbleu']:
# for i in range(2, 6):
# selfbleu = SelfBleu(test_text=gen_file, gram=i, name='selfbleu' + str(i))
# metrics.append(selfbleu)
return metrics
def bleu_vs_sbleu(self, real_text, fake_text):
bleu = Bleu(test_text=fake_text, real_text=real_text, gram=5)
sbleu = SelfBleu(test_text=fake_text, gram=5)
print("Bleu:", bleu.get_score(), "SelfBleu", sbleu.get_score())
def getSelfBleuScore(self):
print(self.generated_file_path)
SelfBleuScore = SelfBleu(self.generated_file_path, self.gram)
#print("Self-Bleu score with gram = %d is: %f" %(self.gram, SelfBleuScore.get_score(is_fast=False)))
return SelfBleuScore.get_score(is_fast=False)