def init_real_metric(self):
from utils.metrics.DocEmbSim import DocEmbSim
docsim = DocEmbSim(oracle_file=self.oracle_file, generator_file=self.generator_file, num_vocabulary=self.vocab_size)
self.add_metric(docsim)
inll = Nll(data_loader=self.gen_data_loader, rnn=self.generator, sess=self.sess)
inll.set_name('nll-test')
self.add_metric(inll)
print("Metrics Applied: " + inll.get_name() + ", " + docsim.get_name())
def get_metrics(config, oracle_loader, test_file, gen_file, g_pretrain_loss,
x_real, sess):
# set up evaluation metric
metrics = []
if config['nll_gen']:
nll_gen = Nll(oracle_loader,
g_pretrain_loss,
x_real,
sess,
name='nll_gen')
metrics.append(nll_gen)
if config['doc_embsim']:
doc_embsim = DocEmbSim(test_file,
gen_file,
config['vocab_size'],
name='doc_embsim')
metrics.append(doc_embsim)
if config['bleu']:
for i in range(2, 6):
bleu = Bleu(test_text=gen_file,
real_text=test_file,
gram=i,
name='bleu' + str(i))
metrics.append(bleu)
if config['selfbleu']:
for i in range(2, 6):
selfbleu = SelfBleu(test_text=gen_file,
gram=i,
name='selfbleu' + str(i))
metrics.append(selfbleu)
return metrics
def get_metrics(config, oracle_loader, test_file, gen_file, g_pretrain_loss, x_real, x_topic, sess, json_file):
# set up evaluation metric
metrics = []
if config['nll_gen']:
nll_gen = NllTopic(oracle_loader, g_pretrain_loss, x_real, sess, name='nll_gen', x_topic=x_topic)
metrics.append(nll_gen)
if config['doc_embsim']:
doc_embsim = DocEmbSim(test_file, gen_file, config['vocab_size'], name='doc_embsim')
metrics.append(doc_embsim)
if config['bleu']:
for i in [2, 4]: # range(2, 6):
bleu = Bleu(test_text=json_file, real_text=test_file, gram=i, name='bleu' + str(i))
metrics.append(bleu)
if config['selfbleu']:
for i in [4]:
selfbleu = SelfBleu(test_text=json_file, gram=i, name='selfbleu' + str(i))
metrics.append(selfbleu)
if config['KL']:
KL_div = KL_divergence(oracle_loader, json_file, name='KL_divergence')
metrics.append(KL_div)
if config['earth_mover']:
EM_div = EarthMover(oracle_loader, json_file, name='Earth_Mover_Distance')
metrics.append(EM_div)
return metrics
def get_metrics(config, oracle_loader, gen_loader, oracle_file, gen_file,
oracle_model, g_pretrain_loss, x_real, sess):
# set up evaluation metric
metrics = []
if config['nll_oracle']:
nll_oracle = Nll(gen_loader,
oracle_model.pretrain_loss,
oracle_model.x,
sess,
name='nll_oracle')
metrics.append(nll_oracle)
if config['nll_gen']:
nll_gen = Nll(oracle_loader,
g_pretrain_loss,
x_real,
sess,
name='nll_gen')
metrics.append(nll_gen)
if config['doc_embsim']:
doc_embsim = DocEmbSim(oracle_file,
gen_file,
config['vocab_size'],
name='doc_embsim')
metrics.append(doc_embsim)
return metrics
def init_metric(self):
nll = Nll(data_loader=self.oracle_data_loader, rnn=self.oracle, sess=self.sess)
self.add_metric(nll)
inll = Nll(data_loader=self.gen_data_loader, rnn=self.generator, sess=self.sess)
inll.set_name('nll-test')
self.add_metric(inll)
from utils.metrics.DocEmbSim import DocEmbSim
docsim = DocEmbSim(oracle_file=self.oracle_file, generator_file=self.generator_file, num_vocabulary=self.vocab_size)
self.add_metric(docsim)
def init_real_metric(self):
from utils.metrics.Nll import Nll
from utils.metrics.PPL import PPL
from utils.metrics.DocEmbSim import DocEmbSim
from utils.others.Bleu import Bleu
from utils.metrics.SelfBleu import SelfBleu
if self.valid_ppl:
valid_ppl = PPL(self.valid_data_loader, self.generator, self.sess)
valid_ppl.set_name('valid_ppl')
self.add_metric(valid_ppl)
if self.nll_gen:
nll_gen = Nll(self.gen_data_loader, self.generator, self.sess)
nll_gen.set_name('nll_gen')
self.add_metric(nll_gen)
if self.doc_embsim:
doc_embsim = DocEmbSim(self.oracle_file, self.generator_file,
self.vocab_size)
doc_embsim.set_name('doc_embsim')
self.add_metric(doc_embsim)
if self.bleu:
FLAGS = tf.app.flags.FLAGS
dataset = FLAGS.data
if dataset == "image_coco":
real_text = 'data/testdata/test_image_coco.txt'
elif dataset == "emnlp_news":
real_text = 'data/testdata/test_emnlp_news.txt'
else:
raise ValueError
for i in range(3, 4):
bleu = Bleu(test_text=self.text_file,
real_text=real_text,
gram=i)
bleu.set_name(f"Bleu{i}")
self.add_metric(bleu)
if self.selfbleu:
for i in range(2, 6):
selfbleu = SelfBleu(test_text=self.text_file, gram=i)
selfbleu.set_name(f"Selfbleu{i}")
self.add_metric(selfbleu)
def init_real_metric(self):
from utils.metrics.DocEmbSim import DocEmbSim
docsim = DocEmbSim(oracle_file=self.oracle_file,
generator_file=self.generator_file,
num_vocabulary=self.vocab_size)
self.add_metric(docsim)
inll = Nll(data_loader=self.gen_data_loader,
rnn=self.generator,
sess=self.sess)
inll.set_name('nll-test')
self.add_metric(inll)
bleu = Bleu(test_text=self.test_file,
real_text='data/image_coco.txt',
gram=2)
self.add_metric(bleu)
sbleu = SelfBleu(test_text=self.test_file, gram=2)
self.add_metric(sbleu)
