def generate_test_csv(best_para): train_file_T = 'train/truthful.txt' train_file_D = 'train/deceptive.txt' test_file = 'test/test.txt' train_T = lm_preprocess(train_file_T) train_D = lm_preprocess(train_file_D) test = lm_preprocess(test_file) model_T = Bigram() model_D = Bigram() model_T.train_with_first_OOV(train_T) model_D.train_with_first_OOV(train_D) length = np.array([len(test[i]) for i in range(len(test))]) resT = np.array(model_T.test_corpus(test, best_para)) resD = np.array(model_D.test_corpus(test, best_para)) perT = perplexity(resT, length) perD = perplexity(resD, length) ans = [['Id', 'Prediction']] for i in range(len(test)): if perT[i] < perD[i]: ans.append([i, 0]) else: ans.append([i, 1]) with open('lm_prediction.csv', 'w') as csvFile: writer = csv.writer(csvFile) writer.writerows(ans) csvFile.close()
def get_perplexities(review, pos_prediction, neg_prediction): pos_probs = extract_probabilities_from_review(review, pos_prediction) neg_probs = extract_probabilities_from_review(review, neg_prediction) pos_perplexity = utils.perplexity(pos_probs) neg_perplexity = utils.perplexity(neg_probs) return pos_perplexity, neg_perplexity
def train_step(model, optimizer, batch, device, opt): # perform a single stochastic gradient step model.train() optimizer.zero_grad() # extract data from batch and send them to GPU text = batch.text.to(device) input_text = text[:-1] target_text = text[1:] authors = batch.author.to(device) timesteps = batch.timestep.to(device) n = text.shape[1] ntkn = target_text.ne(Corpus.pad_id).sum().item() # forward pred, _ = model(input_text, authors, timesteps) # loss loss = 0 # -- word level nll nll = neg_log_prob(pred, target_text).sum() / n loss += nll # -- L2 regularization of static word embeddings if opt.l2_a > 0: ha = model.author_embedding.weight loss += opt.l2_a * 0.5 * ha.pow(2).sum() / opt.n_ex # backward loss.backward() # step optimizer.step() return perplexity(nll.item() * n / ntkn)
def classify_review_by_lm(review, pos_prediction, neg_prediction): pos_probs = extract_probabilities_from_review(review, pos_prediction) neg_probs = extract_probabilities_from_review(review, neg_prediction) pos_perplexity = utils.perplexity(pos_probs) neg_perplexity = utils.perplexity(neg_probs) delta = pos_perplexity - neg_perplexity if delta < 0: if delta < p_bottom: return 1.0 return float(p_map_bottom(delta)) else: if delta > p_top: return 0.0 return float(p_map_top(delta))
def parameter_tuning(ks): train_file_T = 'train/truthful.txt' train_file_D = 'train/deceptive.txt' train_T = simple_preprocess(train_file_T) train_D = simple_preprocess(train_file_D) test_file_T = 'validation/truthful.txt' test_file_D = 'validation/deceptive.txt' test_T = simple_preprocess(test_file_T) test_D = simple_preprocess(test_file_D) lengths_T = np.array([len(test_T[i]) for i in range(len(test_T))]) lengths_D = np.array([len(test_D[i]) for i in range(len(test_D))]) result = {} # for M in Ms: for k in ks: model_T = Bigram() model_D = Bigram() model_T.train_with_first_OOV(train_T) model_D.train_with_first_OOV(train_D) resT_T = np.array(model_T.test_corpus(test_T, k)) resD_T = np.array(model_D.test_corpus(test_T, k)) perT_T = perplexity(resT_T, lengths_T) perD_T = perplexity(resD_T, lengths_T) # print(perT_T < perD_T) num_correct_T = np.sum(perT_T < perD_T) resT_D = np.array(model_T.test_corpus(test_D, k)) resD_D = np.array(model_D.test_corpus(test_D, k)) perT_D = perplexity(resT_D, lengths_D) perD_D = perplexity(resD_D, lengths_D) # print(perT_D > perD_D) num_correct_D = np.sum(perT_D >= perD_D) accuracy = (num_correct_T + num_correct_D) / (len(test_T) + len(test_D)) result[k] = accuracy items = sorted(result.items(), key=operator.itemgetter(1)) print(items) return items[-1]
def finish_epoch(self, epoch, loss_type, avg_loss, total_loss): # Note that we explicitly don't reset self.iters here, because it would mess up how often we plot if self.loss_type != loss_type: return self.plot_losses.append(avg_loss) save_plot(self.plot_losses, self.loss_file, 1) if self.perplexity_file: self.plot_perplexities.append(perplexity(avg_loss)) save_plot(self.plot_perplexities, self.perplexity_file, 1) return "continue"
def finish_epoch(self, epoch, loss_type, avg_loss, total_loss): self.iters = 0 self.interval_loss_total = 0 self.start = time.time() if loss_type != self.loss_type: return ppl = perplexity(avg_loss) logger.info("-" * 65) logger.info( "Epoch {}: {} ppl, {:.4f}. avg loss, {:.4f}. total loss, {:.4f}". format(epoch, self.loss_type, ppl, avg_loss, total_loss)) logger.info("-" * 65) return "continue"
def evaluate(model, testloader, device): model.eval() ntkn = 0 nll = 0 for batch in testloader: # data text = batch.text.to(device) input_text = text[:-1] target_text = text[1:] authors = batch.author.to(device) timesteps = batch.timestep.to(device) ntkn += target_text.ne(Corpus.pad_id).sum().item() # forward pred, _ = model(input_text, authors, timesteps) # perplexity nll += neg_log_prob(pred, target_text).sum().item() return perplexity(nll / ntkn)
def finish_iter(self, loss_type, loss): if self.loss_type != loss_type: return if self.print_every > 0: self.interval_loss_total += loss self.iters += 1 if self.iters % self.print_every == 0: interval_loss_avg = self.interval_loss_total / self.print_every perc_through_epoch = self.iters / self.iters_per_epoch logger.info('Batch: {} / {}. {}'.format( self.iters, self.iters_per_epoch, time_elapsed(self.start, perc_through_epoch))) logger.info('\tLoss: {0:.4f}'.format(interval_loss_avg)) if self.print_perplexity: print_perplexity_avg = perplexity(interval_loss_avg) logger.info( '\tPerplexity: {0:.4f}'.format(print_perplexity_avg)) self.interval_loss_total = 0
def finish_iter(self, loss_type, loss): if self.loss_type != loss_type: return if self.plot_every > 0: self.plot_loss_total += loss self.iters += 1 if self.iters % self.plot_every == 0: plot_loss_avg = self.plot_loss_total / self.plot_every self.plot_losses.append(plot_loss_avg) self.plot_loss_total = 0 if self.perplexity_file: plot_perplexity_avg = perplexity(plot_loss_avg) self.plot_perplexities.append(plot_perplexity_avg) if self.iters % self.save_every == 0: save_plot(self.plot_losses, self.loss_file, self.plot_scale) if self.plot_perplexities: save_plot(self.plot_perplexities, self.perplexity_file, self.plot_scale)
def do_decode(decoder, src_sentences, trgt_sentences=None, num_log=1): all_hypos = [] start_time = time.time() logging.info("Start time: %s" % start_time) for sen_idx, src in enumerate(src_sentences): decoder.set_current_sen_id(sen_idx) logging.info("Next sentence (ID: %d): %s" % (sen_idx + 1, ''.join(src))) decoder.apply_predictor_count = 0 start_hypo_time = time.time() hypos = decoder.decode(src) all_hypos.append(hypos) if not hypos: logging.error("No translation found for ID %d!" % (sen_idx+1)) logging.info("Stats (ID: %d): score=<not-found> " "num_expansions=%d " "time=%.2f" % (sen_idx+1, decoder.apply_predictor_count, time.time() - start_hypo_time)) hypos = [_generate_dummy_hypo()] for logged_hypo in sorted(hypos, reverse=True)[:num_log]: logging.info("Decoded (ID: %d): %s" % ( sen_idx+1, logged_hypo.trgt_sentence)) logging.info("Stats (ID: %d): score=%f " "inc=%f " "num_expansions=%d " "time=%.2f " "perplexity=%.2f"% (sen_idx+1, logged_hypo.total_score, logged_hypo.base_score, decoder.apply_predictor_count, time.time() - start_hypo_time, utils.perplexity(logged_hypo.score_breakdown))) return all_hypos
model_T.train_with_topM(reviews_T, 1) model_D.train_with_topM(reviews_D, 1) # train option 2 # model_T.train_with_topM(reviews_T) # model_D.train_with_topM(reviews_D, m) k = 1 # TESTING against truthful.txt test_file = 'validation/truthful.txt' reviews_test = simple_preprocess(test_file) res1 = np.array(model_T.test_corpus(reviews_test, k)) res2 = np.array(model_D.test_corpus(reviews_test, k)) lengths = np.array( [len(reviews_test[i]) for i in range(len(reviews_test))]) per1 = perplexity(res1, lengths) per2 = perplexity(res2, lengths) print(per1) print(per2) ans = per1 < per2 unique_elements, counts_elements = np.unique(ans, return_counts=True) print("testing truthful.txt") print(unique_elements) print(counts_elements) # TESTING against deceptive.txt test_file = 'validation/deceptive.txt' reviews_test = simple_preprocess(test_file) res1 = np.array(model_T.test_corpus(reviews_test, k)) res2 = np.array(model_D.test_corpus(reviews_test, k)) ans = res1 > res2
def get_perplexity(self, data): """ return perplexity of the model with input data """ return perplexity(self, data)
checkpoint_file = tf.train.latest_checkpoint(read_ckpt_dir) saver.restore(sess, checkpoint_file) start = 0 end = batch_size while True: x = testX[start:end] y = testX[start:end] if args.text: logits = model.predict(sess, x.T) for reply in logits: print(' '.join([metadata['idx2w'][i] for i in reply])) else: logits = model.predict(sess, x.T, Y=y.T, argmax=False) word_probabilities = utils.softmax(logits, axis=-1) sentence_perplexities = utils.perplexity(word_probabilities, reference=y) for i in range(0, len(sentence_perplexities), 2): print("%.3f %.3f" % (sentence_perplexities[i], sentence_perplexities[i + 1])) if end == len(testX): break start += batch_size end += batch_size if end > len(testX): end = len(testX)
def do_decode(decoder, output_handlers, src_sentences, trgt_sentences=None, num_log=1): """This method contains the main decoding loop. It iterates through ``src_sentences`` and applies ``decoder.decode()`` to each of them. At the end, it calls the output handlers to create output files. Args: decoder (Decoder): Current decoder instance output_handlers (list): List of output handlers, see ``create_output_handlers()`` src_sentences (list): A list of strings. The strings are the source sentences with word indices to translate (e.g. '1 123 432 2') """ if not decoder.has_predictor(): logging.fatal("Terminated due to an error in the " "predictor configuration.") return all_hypos = [] text_output_handler = _get_text_output_handler(output_handlers) if text_output_handler: text_output_handler.open_file() score_output_handler = _get_score_output_handler(output_handlers) start_time = time.time() logging.info("Start time: %s" % start_time) sen_indices = [] diversity_metrics = [] not_full = 0 for sen_idx in get_sentence_indices(args.range, src_sentences): decoder.set_current_sen_id(sen_idx) try: src = "0" if src_sentences is False else src_sentences[sen_idx] if len(src.split()) > 1000: print("Skipping ID", str(sen_idx), ". Too long...") continue src_print = io_utils.src_sentence(src) logging.info("Next sentence (ID: %d): %s" % (sen_idx + 1, src_print)) src = io_utils.encode(src) start_hypo_time = time.time() decoder.apply_predictor_count = 0 if trgt_sentences: hypos = decoder.decode( src, io_utils.encode_trg(trgt_sentences[sen_idx])) else: hypos = decoder.decode(src) if not hypos: logging.error("No translation found for ID %d!" % (sen_idx + 1)) logging.info("Stats (ID: %d): score=<not-found> " "num_expansions=%d " "time=%.2f" % (sen_idx + 1, decoder.apply_predictor_count, time.time() - start_hypo_time)) hypos = [_generate_dummy_hypo()] hypos = _postprocess_complete_hypos(hypos) for logged_hypo in hypos[:num_log]: logging.info( "Decoded (ID: %d): %s" % (sen_idx + 1, io_utils.decode(logged_hypo.trgt_sentence))) logging.info("Stats (ID: %d): score=%f " "num_expansions=%d " "time=%.2f " "perplexity=%.2f" % (sen_idx + 1, logged_hypo.total_score, decoder.apply_predictor_count, time.time() - start_hypo_time, utils.perplexity(logged_hypo.score_breakdown))) if score_output_handler: try: score_output_handler.write_score( logged_hypo.score_breakdown) except IOError as e: logging.error( "I/O error %d occurred when creating output files: %s" % (sys.exc_info()[0], e)) if decoder.nbest > 1: diversity_score = utils.ngram_diversity( [io_utils.decode(h.trgt_sentence) for h in hypos]) logging.info("Diversity: score=%f " % (diversity_score)) diversity_metrics.append(diversity_score) if len(hypos) < decoder.nbest: not_full += 1 all_hypos.append(hypos) sen_indices.append(sen_idx) try: # Write text output as we go if text_output_handler: text_output_handler.write_hypos([hypos]) except IOError as e: logging.error( "I/O error %d occurred when creating output files: %s" % (sys.exc_info()[0], e)) except ValueError as e: logging.error("Number format error at sentence id %d: %s, " "Stack trace: %s" % (sen_idx + 1, e, traceback.format_exc())) except AttributeError as e: logging.fatal("Attribute error at sentence id %d: %s. This often " "indicates an error in the predictor configuration " "which could not be detected in initialisation. " "Stack trace: %s" % (sen_idx + 1, e, traceback.format_exc())) except Exception as e: logging.error( "An unexpected %s error has occurred at sentence id " "%d: %s, Stack trace: %s" % (sys.exc_info()[0], sen_idx + 1, e, traceback.format_exc())) try: # Write text output as we go if text_output_handler: hypos = [_generate_dummy_hypo()] text_output_handler.write_hypos([hypos]) except IOError as e: logging.error( "I/O error %d occurred when creating output files: %s" % (sys.exc_info()[0], e)) logging.info("Decoding finished. Time: %.2f" % (time.time() - start_time)) if decoder.nbest > 1: print(diversity_metrics) print("Total not full:", str(not_full)) try: for output_handler in output_handlers: if output_handler == text_output_handler: output_handler.close_file() else: output_handler.write_hypos(all_hypos, sen_indices) except IOError as e: logging.error("I/O error %s occurred when creating output files: %s" % (sys.exc_info()[0], e))