def evaluate(model,
train_corpus,
test_coprus,
vocab=idx2word,
num_docs_test=num_docs_test,
tc=tc,
td=td,
eval_batch_size=_eval_batch_size,
vocab_size=vocab_size,
bow_norm=bow_norm):
"""
Evaluating the trained model on the test set using either perplexity,
or coherence and diversity.
Compute perplexity on document completion.
"""
model.eval() # set model in evaluation mode
with torch.no_grad():
indices = torch.split(torch.tensor(range(num_docs_test)),
eval_batch_size)
## get \beta here
beta = model.get_beta()
### do dc and tc here
acc_loss = 0
cnt = 0
for idx, ind in enumerate(indices):
data_batch = get_batch(test_corpus, ind, vocab_size, device)
sums = data_batch.sum(1).unsqueeze(1)
if bow_norm: normalized_data_batch = data_batch / sums
else: normalized_data_batch = data_batch
## get theta
theta, _ = model.get_theta(normalized_data_batch)
## get prediction loss
res = torch.mm(theta, beta)
preds = torch.log(res)
recon_loss = -(preds * data_batch).sum(1)
loss = recon_loss / sums.squeeze()
loss = loss.mean().item()
acc_loss += loss
cnt += 1
# Calculate final loss
cur_loss = acc_loss / cnt
ppl_dc = round(math.exp(cur_loss), 1)
print('Eval Doc Completion PPL: {}'.format(ppl_dc))
if tc or td: # calculate topic coherence or topic diversity
beta = beta.data.cpu().numpy()
if tc:
print('Computing topic coherence...')
utils.get_topic_coherence(beta, train_corpus, vocab)
if td:
print('Computing topic diversity...')
utils.get_topic_diversity(beta, 25)
return ppl_dc
def evaluate(m, source, tc=False, td=False):
"""Compute perplexity on document completion.
"""
m.eval()
with torch.no_grad():
if source == 'val':
indices = torch.split(torch.tensor(range(args.num_docs_valid)), args.eval_batch_size)
tokens = valid_tokens
counts = valid_counts
else:
indices = torch.split(torch.tensor(range(args.num_docs_test)), args.eval_batch_size)
tokens = test_tokens
counts = test_counts
## get \beta here
beta = m.get_beta()
### do dc and tc here
acc_loss = 0
cnt = 0
indices_1 = torch.split(torch.tensor(range(args.num_docs_test_1)), args.eval_batch_size)
for idx, ind in enumerate(indices_1):
## get theta from first half of docs
data_batch_1 = data.get_batch(test_1_tokens, test_1_counts, ind, args.vocab_size, device)
sums_1 = data_batch_1.sum(1).unsqueeze(1)
if args.bow_norm:
normalized_data_batch_1 = data_batch_1 / sums_1
else:
normalized_data_batch_1 = data_batch_1
theta, _ = m.get_theta(normalized_data_batch_1)
## get prediction loss using second half
data_batch_2 = data.get_batch(test_2_tokens, test_2_counts, ind, args.vocab_size, device)
sums_2 = data_batch_2.sum(1).unsqueeze(1)
res = torch.mm(theta, beta)
preds = torch.log(res)
recon_loss = -(preds * data_batch_2).sum(1)
loss = recon_loss / sums_2.squeeze()
loss = loss.mean().item()
acc_loss += loss
cnt += 1
cur_loss = acc_loss / cnt
ppl_dc = round(math.exp(cur_loss), 1)
print('*'*100)
print('{} Doc Completion PPL: {}'.format(source.upper(), ppl_dc))
print('*'*100)
if tc or td:
beta = beta.data.cpu().numpy()
if tc:
print('Computing topic coherence...')
get_topic_coherence(beta, train_tokens, vocab)
if td:
print('Computing topic diversity...')
get_topic_diversity(beta, 25)
return ppl_dc
def evaluate(m, source, tc=False, td=False):
"""Compute perplexity on document completion.
"""
m.eval()
with torch.no_grad():
if source == 'val':
indices = torch.split(
torch.tensor(range(args.num_docs_valid)),
args.eval_batch_size)
tokens = valid_tokens
counts = valid_counts
else:
indices = torch.split(
torch.tensor(range(args.num_docs_test)),
args.eval_batch_size)
tokens = test_tokens
counts = test_counts
## get \beta here
beta = m.get_beta()
### do dc and tc here
acc_loss = 0
cnt = 0
indices_1 = torch.split(
torch.tensor(range(args.num_docs_test_1)),
args.eval_batch_size)
for idx, ind in enumerate(indices_1):
## get theta from first half of docs
data_batch_1 = data.get_batch(test_1_tokens, test_1_counts,
ind, args.vocab_size, device)
sums_1 = data_batch_1.sum(1).unsqueeze(1)
if args.bow_norm:
normalized_data_batch_1 = data_batch_1 / sums_1
else:
normalized_data_batch_1 = data_batch_1
theta, _ = m.get_theta(normalized_data_batch_1)
## get prediction loss using second half
data_batch_2 = data.get_batch(test_2_tokens, test_2_counts,
ind, args.vocab_size, device)
sums_2 = data_batch_2.sum(1).unsqueeze(1)
res = torch.mm(theta, beta)
preds = torch.log(res)
recon_loss = -(preds * data_batch_2).sum(1)
loss = recon_loss / sums_2.squeeze()
loss = loss.mean().item()
acc_loss += loss
cnt += 1
cur_loss = acc_loss / cnt
ppl_dc = round(math.exp(cur_loss), 1)
print('*' * 100)
print('{} Doc Completion PPL: {}'.format(
source.upper(), ppl_dc))
print('*' * 100)
if tc or td:
beta = beta.data.cpu().numpy()
if tc:
print('Computing topic coherence...')
get_topic_coherence(beta, train_tokens, vocab)
if td:
print('Computing topic diversity...')
get_topic_diversity(beta, 25)
return ppl_dc
## train model on data
best_epoch = 0
best_val_ppl = 1e9
all_val_ppls = []
print('\n')
print('Visualizing model quality before training...')
visualize(model)
print('\n')
for epoch in range(1, args.epochs):
train(epoch)
val_ppl = evaluate(model, 'val')
if val_ppl < best_val_ppl:
with open(ckpt, 'wb') as f:
torch.save(model, f)
best_epoch = epoch
best_val_ppl = val_ppl
else:
## check whether to anneal lr
lr = optimizer.param_groups[0]['lr']
if args.anneal_lr and (
len(all_val_ppls) > args.nonmono
and val_ppl > min(all_val_ppls[:-args.nonmono])
and lr > 1e-5):
optimizer.param_groups[0]['lr'] /= args.lr_factor
if epoch % args.visualize_every == 0:
visualize(model)
all_val_ppls.append(val_ppl)
with open(ckpt, 'rb') as f:
model = torch.load(f)
model = model.to(device)
val_ppl = evaluate(model, 'val')
def evaluate(self,
args,
source,
training_set,
vocabulary,
test_1,
test_2,
tc=False,
td=False):
"""
Compute perplexity on document completion.
"""
self.eval()
with torch.no_grad():
if source == 'val':
indices = torch.split(torch.tensor(range(args.num_docs_valid)),
args.eval_batch_size)
else:
indices = torch.split(torch.tensor(range(args.num_docs_test)),
args.eval_batch_size)
## get \beta here
beta = self.get_beta()
### do dc and tc here
acc_loss = 0
cnt = 0
indices_1 = torch.split(torch.tensor(range(args.num_docs_test_1)),
args.eval_batch_size)
for idx, indice in enumerate(indices_1):
data_batch_1 = get_batch(test_1, indice, device)
sums_1 = data_batch_1.sum(1).unsqueeze(1)
if args.bow_norm:
normalized_data_batch_1 = data_batch_1 / sums_1
else:
normalized_data_batch_1 = data_batch_1
theta, _ = self.get_theta(normalized_data_batch_1)
## get predition loss using second half
data_batch_2 = get_batch(test_2, indice, device)
sums_2 = data_batch_2.sum(1).unsqueeze(1)
res = torch.mm(theta, beta)
preds = torch.log(res)
recon_loss = -(preds * data_batch_2).sum(1)
loss = recon_loss / sums_2.squeeze()
loss = np.nanmean(loss.numpy())
acc_loss += loss
cnt += 1
cur_loss = acc_loss / cnt
ppl_dc = round(math.exp(cur_loss), 1)
print('*' * 100)
print('{} Doc Completion PPL: {}'.format(source.upper(), ppl_dc))
print('*' * 100)
if tc or td:
beta = beta.data.cpu().numpy()
if tc:
print('Computing topic coherence...')
get_topic_coherence(beta, training_set, vocabulary)
if td:
print('Computing topic diversity...')
get_topic_diversity(beta, 25)
return ppl_dc
def evaluate(self,
test_1_tokens,
test_1_counts,
test_2_tokens,
test_2_counts,
train_tokens,
vocab,
tc=False,
td=False):
"""
Compute perplexity on document completion.
:param test_1_tokens: numpy array
a numpy array holding the tokens exists in each document (thread) of the test1 data
:param test_1_counts: numpy array
a numpy array holding the counter per each tokens exists in each document (thread) of the test1 data
this one works along with the test_1_tokens and has the same shape
:param test_2_tokens: numpy array
a numpy array holding the tokens exists in each document (thread) of the test2 data
:param test_2_counts: numpy array
a numpy array holding the counter per each tokens exists in each document (thread) of the test2 data
this one works along with the test_2_tokens and has the same shape
:param vocab: list
the list of all vocabulary tokens
:param tc: bool, default:False
whether to compute topics coherence
:param td: bool, default:False
whether to compute topics diversity
:return: float
the perplexity value
"""
self.eval()
with torch.no_grad():
"""
if source == 'val':
indices = torch.split(torch.tensor(range(self.config_dict['num_docs_valid'])),
self.config_dict['evaluation_params']['eval_batch_size'])
tokens = valid_tokens
counts = valid_counts
else:
indices = torch.split(torch.tensor(range(self.config_dict['num_docs_test'])),
self.config_dict['evaluation_params']['eval_batch_size'])
tokens = test_tokens
counts = test_counts
"""
# get \beta here
beta = self.get_beta()
# do dc and tc here
acc_loss = 0
cnt = 0
indices_1 = torch.split(
torch.tensor(range(self.config_dict['num_docs_test_1'])),
self.config_dict['evaluation_params']['eval_batch_size'])
for idx, ind in enumerate(indices_1):
# get theta from first half of docs
data_batch_1 = data.get_batch(test_1_tokens, test_1_counts,
ind,
self.config_dict['vocab_size'],
device)
sums_1 = data_batch_1.sum(1).unsqueeze(1)
if self.config_dict['optimization_params']['bow_norm']:
normalized_data_batch_1 = data_batch_1 / sums_1
else:
normalized_data_batch_1 = data_batch_1
theta, _ = self.get_theta(normalized_data_batch_1)
# get prediction loss using second half
data_batch_2 = data.get_batch(test_2_tokens, test_2_counts,
ind,
self.config_dict['vocab_size'],
device)
sums_2 = data_batch_2.sum(1).unsqueeze(1)
res = torch.mm(theta, beta)
preds = torch.log(res)
recon_loss = -(preds * data_batch_2).sum(1)
loss = recon_loss / sums_2.squeeze()
loss = loss.mean().item()
acc_loss += loss
cnt += 1
cur_loss = acc_loss / cnt
ppl_dc = round(math.exp(cur_loss), 1)
print('*' * 100)
#print('{} Doc Completion PPL: {}'.format(source.upper(), ppl_dc))
print('*' * 100)
if tc or td:
beta = beta.data.cpu().numpy()
if tc:
print('Computing topic coherence...')
get_topic_coherence(beta, train_tokens, vocab)
if td:
print('Computing topic diversity...')
get_topic_diversity(beta, 25)
return ppl_dc
