def evaluate(model, train_corpus, test_coprus, vocab=idx2word, num_docs_test=num_docs_test, tc=tc, td=td, eval_batch_size=_eval_batch_size, vocab_size=vocab_size, bow_norm=bow_norm): """ Evaluating the trained model on the test set using either perplexity, or coherence and diversity. Compute perplexity on document completion. """ model.eval() # set model in evaluation mode with torch.no_grad(): indices = torch.split(torch.tensor(range(num_docs_test)), eval_batch_size) ## get \beta here beta = model.get_beta() ### do dc and tc here acc_loss = 0 cnt = 0 for idx, ind in enumerate(indices): data_batch = get_batch(test_corpus, ind, vocab_size, device) sums = data_batch.sum(1).unsqueeze(1) if bow_norm: normalized_data_batch = data_batch / sums else: normalized_data_batch = data_batch ## get theta theta, _ = model.get_theta(normalized_data_batch) ## get prediction loss res = torch.mm(theta, beta) preds = torch.log(res) recon_loss = -(preds * data_batch).sum(1) loss = recon_loss / sums.squeeze() loss = loss.mean().item() acc_loss += loss cnt += 1 # Calculate final loss cur_loss = acc_loss / cnt ppl_dc = round(math.exp(cur_loss), 1) print('Eval Doc Completion PPL: {}'.format(ppl_dc)) if tc or td: # calculate topic coherence or topic diversity beta = beta.data.cpu().numpy() if tc: print('Computing topic coherence...') utils.get_topic_coherence(beta, train_corpus, vocab) if td: print('Computing topic diversity...') utils.get_topic_diversity(beta, 25) return ppl_dc
def evaluate(m, source, tc=False, td=False): """Compute perplexity on document completion. """ m.eval() with torch.no_grad(): if source == 'val': indices = torch.split(torch.tensor(range(args.num_docs_valid)), args.eval_batch_size) tokens = valid_tokens counts = valid_counts else: indices = torch.split(torch.tensor(range(args.num_docs_test)), args.eval_batch_size) tokens = test_tokens counts = test_counts ## get \beta here beta = m.get_beta() ### do dc and tc here acc_loss = 0 cnt = 0 indices_1 = torch.split(torch.tensor(range(args.num_docs_test_1)), args.eval_batch_size) for idx, ind in enumerate(indices_1): ## get theta from first half of docs data_batch_1 = data.get_batch(test_1_tokens, test_1_counts, ind, args.vocab_size, device) sums_1 = data_batch_1.sum(1).unsqueeze(1) if args.bow_norm: normalized_data_batch_1 = data_batch_1 / sums_1 else: normalized_data_batch_1 = data_batch_1 theta, _ = m.get_theta(normalized_data_batch_1) ## get prediction loss using second half data_batch_2 = data.get_batch(test_2_tokens, test_2_counts, ind, args.vocab_size, device) sums_2 = data_batch_2.sum(1).unsqueeze(1) res = torch.mm(theta, beta) preds = torch.log(res) recon_loss = -(preds * data_batch_2).sum(1) loss = recon_loss / sums_2.squeeze() loss = loss.mean().item() acc_loss += loss cnt += 1 cur_loss = acc_loss / cnt ppl_dc = round(math.exp(cur_loss), 1) print('*'*100) print('{} Doc Completion PPL: {}'.format(source.upper(), ppl_dc)) print('*'*100) if tc or td: beta = beta.data.cpu().numpy() if tc: print('Computing topic coherence...') get_topic_coherence(beta, train_tokens, vocab) if td: print('Computing topic diversity...') get_topic_diversity(beta, 25) return ppl_dc
def evaluate(m, source, tc=False, td=False): """Compute perplexity on document completion. """ m.eval() with torch.no_grad(): if source == 'val': indices = torch.split( torch.tensor(range(args.num_docs_valid)), args.eval_batch_size) tokens = valid_tokens counts = valid_counts else: indices = torch.split( torch.tensor(range(args.num_docs_test)), args.eval_batch_size) tokens = test_tokens counts = test_counts ## get \beta here beta = m.get_beta() ### do dc and tc here acc_loss = 0 cnt = 0 indices_1 = torch.split( torch.tensor(range(args.num_docs_test_1)), args.eval_batch_size) for idx, ind in enumerate(indices_1): ## get theta from first half of docs data_batch_1 = data.get_batch(test_1_tokens, test_1_counts, ind, args.vocab_size, device) sums_1 = data_batch_1.sum(1).unsqueeze(1) if args.bow_norm: normalized_data_batch_1 = data_batch_1 / sums_1 else: normalized_data_batch_1 = data_batch_1 theta, _ = m.get_theta(normalized_data_batch_1) ## get prediction loss using second half data_batch_2 = data.get_batch(test_2_tokens, test_2_counts, ind, args.vocab_size, device) sums_2 = data_batch_2.sum(1).unsqueeze(1) res = torch.mm(theta, beta) preds = torch.log(res) recon_loss = -(preds * data_batch_2).sum(1) loss = recon_loss / sums_2.squeeze() loss = loss.mean().item() acc_loss += loss cnt += 1 cur_loss = acc_loss / cnt ppl_dc = round(math.exp(cur_loss), 1) print('*' * 100) print('{} Doc Completion PPL: {}'.format( source.upper(), ppl_dc)) print('*' * 100) if tc or td: beta = beta.data.cpu().numpy() if tc: print('Computing topic coherence...') get_topic_coherence(beta, train_tokens, vocab) if td: print('Computing topic diversity...') get_topic_diversity(beta, 25) return ppl_dc ## train model on data best_epoch = 0 best_val_ppl = 1e9 all_val_ppls = [] print('\n') print('Visualizing model quality before training...') visualize(model) print('\n') for epoch in range(1, args.epochs): train(epoch) val_ppl = evaluate(model, 'val') if val_ppl < best_val_ppl: with open(ckpt, 'wb') as f: torch.save(model, f) best_epoch = epoch best_val_ppl = val_ppl else: ## check whether to anneal lr lr = optimizer.param_groups[0]['lr'] if args.anneal_lr and ( len(all_val_ppls) > args.nonmono and val_ppl > min(all_val_ppls[:-args.nonmono]) and lr > 1e-5): optimizer.param_groups[0]['lr'] /= args.lr_factor if epoch % args.visualize_every == 0: visualize(model) all_val_ppls.append(val_ppl) with open(ckpt, 'rb') as f: model = torch.load(f) model = model.to(device) val_ppl = evaluate(model, 'val')
def evaluate(self, args, source, training_set, vocabulary, test_1, test_2, tc=False, td=False): """ Compute perplexity on document completion. """ self.eval() with torch.no_grad(): if source == 'val': indices = torch.split(torch.tensor(range(args.num_docs_valid)), args.eval_batch_size) else: indices = torch.split(torch.tensor(range(args.num_docs_test)), args.eval_batch_size) ## get \beta here beta = self.get_beta() ### do dc and tc here acc_loss = 0 cnt = 0 indices_1 = torch.split(torch.tensor(range(args.num_docs_test_1)), args.eval_batch_size) for idx, indice in enumerate(indices_1): data_batch_1 = get_batch(test_1, indice, device) sums_1 = data_batch_1.sum(1).unsqueeze(1) if args.bow_norm: normalized_data_batch_1 = data_batch_1 / sums_1 else: normalized_data_batch_1 = data_batch_1 theta, _ = self.get_theta(normalized_data_batch_1) ## get predition loss using second half data_batch_2 = get_batch(test_2, indice, device) sums_2 = data_batch_2.sum(1).unsqueeze(1) res = torch.mm(theta, beta) preds = torch.log(res) recon_loss = -(preds * data_batch_2).sum(1) loss = recon_loss / sums_2.squeeze() loss = np.nanmean(loss.numpy()) acc_loss += loss cnt += 1 cur_loss = acc_loss / cnt ppl_dc = round(math.exp(cur_loss), 1) print('*' * 100) print('{} Doc Completion PPL: {}'.format(source.upper(), ppl_dc)) print('*' * 100) if tc or td: beta = beta.data.cpu().numpy() if tc: print('Computing topic coherence...') get_topic_coherence(beta, training_set, vocabulary) if td: print('Computing topic diversity...') get_topic_diversity(beta, 25) return ppl_dc
def evaluate(self, test_1_tokens, test_1_counts, test_2_tokens, test_2_counts, train_tokens, vocab, tc=False, td=False): """ Compute perplexity on document completion. :param test_1_tokens: numpy array a numpy array holding the tokens exists in each document (thread) of the test1 data :param test_1_counts: numpy array a numpy array holding the counter per each tokens exists in each document (thread) of the test1 data this one works along with the test_1_tokens and has the same shape :param test_2_tokens: numpy array a numpy array holding the tokens exists in each document (thread) of the test2 data :param test_2_counts: numpy array a numpy array holding the counter per each tokens exists in each document (thread) of the test2 data this one works along with the test_2_tokens and has the same shape :param vocab: list the list of all vocabulary tokens :param tc: bool, default:False whether to compute topics coherence :param td: bool, default:False whether to compute topics diversity :return: float the perplexity value """ self.eval() with torch.no_grad(): """ if source == 'val': indices = torch.split(torch.tensor(range(self.config_dict['num_docs_valid'])), self.config_dict['evaluation_params']['eval_batch_size']) tokens = valid_tokens counts = valid_counts else: indices = torch.split(torch.tensor(range(self.config_dict['num_docs_test'])), self.config_dict['evaluation_params']['eval_batch_size']) tokens = test_tokens counts = test_counts """ # get \beta here beta = self.get_beta() # do dc and tc here acc_loss = 0 cnt = 0 indices_1 = torch.split( torch.tensor(range(self.config_dict['num_docs_test_1'])), self.config_dict['evaluation_params']['eval_batch_size']) for idx, ind in enumerate(indices_1): # get theta from first half of docs data_batch_1 = data.get_batch(test_1_tokens, test_1_counts, ind, self.config_dict['vocab_size'], device) sums_1 = data_batch_1.sum(1).unsqueeze(1) if self.config_dict['optimization_params']['bow_norm']: normalized_data_batch_1 = data_batch_1 / sums_1 else: normalized_data_batch_1 = data_batch_1 theta, _ = self.get_theta(normalized_data_batch_1) # get prediction loss using second half data_batch_2 = data.get_batch(test_2_tokens, test_2_counts, ind, self.config_dict['vocab_size'], device) sums_2 = data_batch_2.sum(1).unsqueeze(1) res = torch.mm(theta, beta) preds = torch.log(res) recon_loss = -(preds * data_batch_2).sum(1) loss = recon_loss / sums_2.squeeze() loss = loss.mean().item() acc_loss += loss cnt += 1 cur_loss = acc_loss / cnt ppl_dc = round(math.exp(cur_loss), 1) print('*' * 100) #print('{} Doc Completion PPL: {}'.format(source.upper(), ppl_dc)) print('*' * 100) if tc or td: beta = beta.data.cpu().numpy() if tc: print('Computing topic coherence...') get_topic_coherence(beta, train_tokens, vocab) if td: print('Computing topic diversity...') get_topic_diversity(beta, 25) return ppl_dc