def generate_test_csv(best_para):
train_file_T = 'train/truthful.txt'
train_file_D = 'train/deceptive.txt'
test_file = 'test/test.txt'
train_T = lm_preprocess(train_file_T)
train_D = lm_preprocess(train_file_D)
test = lm_preprocess(test_file)
model_T = Bigram()
model_D = Bigram()
model_T.train_with_first_OOV(train_T)
model_D.train_with_first_OOV(train_D)
length = np.array([len(test[i]) for i in range(len(test))])
resT = np.array(model_T.test_corpus(test, best_para))
resD = np.array(model_D.test_corpus(test, best_para))
perT = perplexity(resT, length)
perD = perplexity(resD, length)
ans = [['Id', 'Prediction']]
for i in range(len(test)):
if perT[i] < perD[i]:
ans.append([i, 0])
else:
ans.append([i, 1])
with open('lm_prediction.csv', 'w') as csvFile:
writer = csv.writer(csvFile)
writer.writerows(ans)
csvFile.close()
def get_perplexities(review, pos_prediction, neg_prediction):
pos_probs = extract_probabilities_from_review(review, pos_prediction)
neg_probs = extract_probabilities_from_review(review, neg_prediction)
pos_perplexity = utils.perplexity(pos_probs)
neg_perplexity = utils.perplexity(neg_probs)
return pos_perplexity, neg_perplexity
def train_step(model, optimizer, batch, device, opt):
# perform a single stochastic gradient step
model.train()
optimizer.zero_grad()
# extract data from batch and send them to GPU
text = batch.text.to(device)
input_text = text[:-1]
target_text = text[1:]
authors = batch.author.to(device)
timesteps = batch.timestep.to(device)
n = text.shape[1]
ntkn = target_text.ne(Corpus.pad_id).sum().item()
# forward
pred, _ = model(input_text, authors, timesteps)
# loss
loss = 0
# -- word level nll
nll = neg_log_prob(pred, target_text).sum() / n
loss += nll
# -- L2 regularization of static word embeddings
if opt.l2_a > 0:
ha = model.author_embedding.weight
loss += opt.l2_a * 0.5 * ha.pow(2).sum() / opt.n_ex
# backward
loss.backward()
# step
optimizer.step()
return perplexity(nll.item() * n / ntkn)
def classify_review_by_lm(review, pos_prediction, neg_prediction):
pos_probs = extract_probabilities_from_review(review, pos_prediction)
neg_probs = extract_probabilities_from_review(review, neg_prediction)
pos_perplexity = utils.perplexity(pos_probs)
neg_perplexity = utils.perplexity(neg_probs)
delta = pos_perplexity - neg_perplexity
if delta < 0:
if delta < p_bottom:
return 1.0
return float(p_map_bottom(delta))
else:
if delta > p_top:
return 0.0
return float(p_map_top(delta))
def parameter_tuning(ks):
train_file_T = 'train/truthful.txt'
train_file_D = 'train/deceptive.txt'
train_T = simple_preprocess(train_file_T)
train_D = simple_preprocess(train_file_D)
test_file_T = 'validation/truthful.txt'
test_file_D = 'validation/deceptive.txt'
test_T = simple_preprocess(test_file_T)
test_D = simple_preprocess(test_file_D)
lengths_T = np.array([len(test_T[i]) for i in range(len(test_T))])
lengths_D = np.array([len(test_D[i]) for i in range(len(test_D))])
result = {}
# for M in Ms:
for k in ks:
model_T = Bigram()
model_D = Bigram()
model_T.train_with_first_OOV(train_T)
model_D.train_with_first_OOV(train_D)
resT_T = np.array(model_T.test_corpus(test_T, k))
resD_T = np.array(model_D.test_corpus(test_T, k))
perT_T = perplexity(resT_T, lengths_T)
perD_T = perplexity(resD_T, lengths_T)
# print(perT_T < perD_T)
num_correct_T = np.sum(perT_T < perD_T)
resT_D = np.array(model_T.test_corpus(test_D, k))
resD_D = np.array(model_D.test_corpus(test_D, k))
perT_D = perplexity(resT_D, lengths_D)
perD_D = perplexity(resD_D, lengths_D)
# print(perT_D > perD_D)
num_correct_D = np.sum(perT_D >= perD_D)
accuracy = (num_correct_T + num_correct_D) / (len(test_T) +
len(test_D))
result[k] = accuracy
items = sorted(result.items(), key=operator.itemgetter(1))
print(items)
return items[-1]
def finish_epoch(self, epoch, loss_type, avg_loss, total_loss):
# Note that we explicitly don't reset self.iters here, because it would mess up how often we plot
if self.loss_type != loss_type:
return
self.plot_losses.append(avg_loss)
save_plot(self.plot_losses, self.loss_file, 1)
if self.perplexity_file:
self.plot_perplexities.append(perplexity(avg_loss))
save_plot(self.plot_perplexities, self.perplexity_file, 1)
return "continue"
def finish_epoch(self, epoch, loss_type, avg_loss, total_loss):
self.iters = 0
self.interval_loss_total = 0
self.start = time.time()
if loss_type != self.loss_type:
return
ppl = perplexity(avg_loss)
logger.info("-" * 65)
logger.info(
"Epoch {}: {} ppl, {:.4f}. avg loss, {:.4f}. total loss, {:.4f}".
format(epoch, self.loss_type, ppl, avg_loss, total_loss))
logger.info("-" * 65)
return "continue"
def evaluate(model, testloader, device):
model.eval()
ntkn = 0
nll = 0
for batch in testloader:
# data
text = batch.text.to(device)
input_text = text[:-1]
target_text = text[1:]
authors = batch.author.to(device)
timesteps = batch.timestep.to(device)
ntkn += target_text.ne(Corpus.pad_id).sum().item()
# forward
pred, _ = model(input_text, authors, timesteps)
# perplexity
nll += neg_log_prob(pred, target_text).sum().item()
return perplexity(nll / ntkn)
def finish_iter(self, loss_type, loss):
if self.loss_type != loss_type:
return
if self.print_every > 0:
self.interval_loss_total += loss
self.iters += 1
if self.iters % self.print_every == 0:
interval_loss_avg = self.interval_loss_total / self.print_every
perc_through_epoch = self.iters / self.iters_per_epoch
logger.info('Batch: {} / {}. {}'.format(
self.iters, self.iters_per_epoch,
time_elapsed(self.start, perc_through_epoch)))
logger.info('\tLoss: {0:.4f}'.format(interval_loss_avg))
if self.print_perplexity:
print_perplexity_avg = perplexity(interval_loss_avg)
logger.info(
'\tPerplexity: {0:.4f}'.format(print_perplexity_avg))
self.interval_loss_total = 0
def finish_iter(self, loss_type, loss):
if self.loss_type != loss_type:
return
if self.plot_every > 0:
self.plot_loss_total += loss
self.iters += 1
if self.iters % self.plot_every == 0:
plot_loss_avg = self.plot_loss_total / self.plot_every
self.plot_losses.append(plot_loss_avg)
self.plot_loss_total = 0
if self.perplexity_file:
plot_perplexity_avg = perplexity(plot_loss_avg)
self.plot_perplexities.append(plot_perplexity_avg)
if self.iters % self.save_every == 0:
save_plot(self.plot_losses, self.loss_file, self.plot_scale)
if self.plot_perplexities:
save_plot(self.plot_perplexities, self.perplexity_file,
self.plot_scale)
def do_decode(decoder,
src_sentences,
trgt_sentences=None,
num_log=1):
all_hypos = []
start_time = time.time()
logging.info("Start time: %s" % start_time)
for sen_idx, src in enumerate(src_sentences):
decoder.set_current_sen_id(sen_idx)
logging.info("Next sentence (ID: %d): %s" % (sen_idx + 1, ''.join(src)))
decoder.apply_predictor_count = 0
start_hypo_time = time.time()
hypos = decoder.decode(src)
all_hypos.append(hypos)
if not hypos:
logging.error("No translation found for ID %d!" % (sen_idx+1))
logging.info("Stats (ID: %d): score=<not-found> "
"num_expansions=%d "
"time=%.2f" % (sen_idx+1,
decoder.apply_predictor_count,
time.time() - start_hypo_time))
hypos = [_generate_dummy_hypo()]
for logged_hypo in sorted(hypos, reverse=True)[:num_log]:
logging.info("Decoded (ID: %d): %s" % (
sen_idx+1,
logged_hypo.trgt_sentence))
logging.info("Stats (ID: %d): score=%f "
"inc=%f "
"num_expansions=%d "
"time=%.2f "
"perplexity=%.2f"% (sen_idx+1,
logged_hypo.total_score,
logged_hypo.base_score,
decoder.apply_predictor_count,
time.time() - start_hypo_time,
utils.perplexity(logged_hypo.score_breakdown)))
return all_hypos
model_T.train_with_topM(reviews_T, 1)
model_D.train_with_topM(reviews_D, 1)
# train option 2
# model_T.train_with_topM(reviews_T)
# model_D.train_with_topM(reviews_D, m)
k = 1
# TESTING against truthful.txt
test_file = 'validation/truthful.txt'
reviews_test = simple_preprocess(test_file)
res1 = np.array(model_T.test_corpus(reviews_test, k))
res2 = np.array(model_D.test_corpus(reviews_test, k))
lengths = np.array(
[len(reviews_test[i]) for i in range(len(reviews_test))])
per1 = perplexity(res1, lengths)
per2 = perplexity(res2, lengths)
print(per1)
print(per2)
ans = per1 < per2
unique_elements, counts_elements = np.unique(ans, return_counts=True)
print("testing truthful.txt")
print(unique_elements)
print(counts_elements)
# TESTING against deceptive.txt
test_file = 'validation/deceptive.txt'
reviews_test = simple_preprocess(test_file)
res1 = np.array(model_T.test_corpus(reviews_test, k))
res2 = np.array(model_D.test_corpus(reviews_test, k))
ans = res1 > res2
def get_perplexity(self, data):
""" return perplexity of the model with input data """
return perplexity(self, data)
checkpoint_file = tf.train.latest_checkpoint(read_ckpt_dir)
saver.restore(sess, checkpoint_file)
start = 0
end = batch_size
while True:
x = testX[start:end]
y = testX[start:end]
if args.text:
logits = model.predict(sess, x.T)
for reply in logits:
print(' '.join([metadata['idx2w'][i] for i in reply]))
else:
logits = model.predict(sess, x.T, Y=y.T, argmax=False)
word_probabilities = utils.softmax(logits, axis=-1)
sentence_perplexities = utils.perplexity(word_probabilities,
reference=y)
for i in range(0, len(sentence_perplexities), 2):
print("%.3f %.3f" %
(sentence_perplexities[i], sentence_perplexities[i + 1]))
if end == len(testX):
break
start += batch_size
end += batch_size
if end > len(testX):
end = len(testX)
def do_decode(decoder,
output_handlers,
src_sentences,
trgt_sentences=None,
num_log=1):
"""This method contains the main decoding loop. It iterates through
``src_sentences`` and applies ``decoder.decode()`` to each of them.
At the end, it calls the output handlers to create output files.
Args:
decoder (Decoder): Current decoder instance
output_handlers (list): List of output handlers, see
``create_output_handlers()``
src_sentences (list): A list of strings. The strings are the
source sentences with word indices to
translate (e.g. '1 123 432 2')
"""
if not decoder.has_predictor():
logging.fatal("Terminated due to an error in the "
"predictor configuration.")
return
all_hypos = []
text_output_handler = _get_text_output_handler(output_handlers)
if text_output_handler:
text_output_handler.open_file()
score_output_handler = _get_score_output_handler(output_handlers)
start_time = time.time()
logging.info("Start time: %s" % start_time)
sen_indices = []
diversity_metrics = []
not_full = 0
for sen_idx in get_sentence_indices(args.range, src_sentences):
decoder.set_current_sen_id(sen_idx)
try:
src = "0" if src_sentences is False else src_sentences[sen_idx]
if len(src.split()) > 1000:
print("Skipping ID", str(sen_idx), ". Too long...")
continue
src_print = io_utils.src_sentence(src)
logging.info("Next sentence (ID: %d): %s" %
(sen_idx + 1, src_print))
src = io_utils.encode(src)
start_hypo_time = time.time()
decoder.apply_predictor_count = 0
if trgt_sentences:
hypos = decoder.decode(
src, io_utils.encode_trg(trgt_sentences[sen_idx]))
else:
hypos = decoder.decode(src)
if not hypos:
logging.error("No translation found for ID %d!" %
(sen_idx + 1))
logging.info("Stats (ID: %d): score=<not-found> "
"num_expansions=%d "
"time=%.2f" %
(sen_idx + 1, decoder.apply_predictor_count,
time.time() - start_hypo_time))
hypos = [_generate_dummy_hypo()]
hypos = _postprocess_complete_hypos(hypos)
for logged_hypo in hypos[:num_log]:
logging.info(
"Decoded (ID: %d): %s" %
(sen_idx + 1, io_utils.decode(logged_hypo.trgt_sentence)))
logging.info("Stats (ID: %d): score=%f "
"num_expansions=%d "
"time=%.2f "
"perplexity=%.2f" %
(sen_idx + 1, logged_hypo.total_score,
decoder.apply_predictor_count,
time.time() - start_hypo_time,
utils.perplexity(logged_hypo.score_breakdown)))
if score_output_handler:
try:
score_output_handler.write_score(
logged_hypo.score_breakdown)
except IOError as e:
logging.error(
"I/O error %d occurred when creating output files: %s"
% (sys.exc_info()[0], e))
if decoder.nbest > 1:
diversity_score = utils.ngram_diversity(
[io_utils.decode(h.trgt_sentence) for h in hypos])
logging.info("Diversity: score=%f " % (diversity_score))
diversity_metrics.append(diversity_score)
if len(hypos) < decoder.nbest:
not_full += 1
all_hypos.append(hypos)
sen_indices.append(sen_idx)
try:
# Write text output as we go
if text_output_handler:
text_output_handler.write_hypos([hypos])
except IOError as e:
logging.error(
"I/O error %d occurred when creating output files: %s" %
(sys.exc_info()[0], e))
except ValueError as e:
logging.error("Number format error at sentence id %d: %s, "
"Stack trace: %s" %
(sen_idx + 1, e, traceback.format_exc()))
except AttributeError as e:
logging.fatal("Attribute error at sentence id %d: %s. This often "
"indicates an error in the predictor configuration "
"which could not be detected in initialisation. "
"Stack trace: %s" %
(sen_idx + 1, e, traceback.format_exc()))
except Exception as e:
logging.error(
"An unexpected %s error has occurred at sentence id "
"%d: %s, Stack trace: %s" %
(sys.exc_info()[0], sen_idx + 1, e, traceback.format_exc()))
try:
# Write text output as we go
if text_output_handler:
hypos = [_generate_dummy_hypo()]
text_output_handler.write_hypos([hypos])
except IOError as e:
logging.error(
"I/O error %d occurred when creating output files: %s" %
(sys.exc_info()[0], e))
logging.info("Decoding finished. Time: %.2f" % (time.time() - start_time))
if decoder.nbest > 1:
print(diversity_metrics)
print("Total not full:", str(not_full))
try:
for output_handler in output_handlers:
if output_handler == text_output_handler:
output_handler.close_file()
else:
output_handler.write_hypos(all_hypos, sen_indices)
except IOError as e:
logging.error("I/O error %s occurred when creating output files: %s" %
(sys.exc_info()[0], e))
