def run_letter_epoch(session, data, word_model, letter_model, config, eval_op=None, verbose=False):
"""Runs the model on the given data."""
start_time = time.time()
costs = 0.0
iters = 0
num_word = 0
fetches = {}
fetches_letter = {}
previous_state = session.run(word_model.initial_state)
for step, (epoch_size, lm_data, letter_data, phrase_p_data, phrase_data) in \
enumerate(data_feeder.data_iterator(data, config)):
if FLAGS.laptop_discount > 0 and step >= FLAGS.laptop_discount:
break
if step >= epoch_size:
break
fetches["rnn_state"] = word_model.rnn_state
fetches["final_state"] = word_model.final_state
fetches_letter["cost"] = letter_model.cost
if eval_op is not None:
fetches_letter["eval_op"] = eval_op
feed_dict = {word_model.input_data: lm_data[0],
word_model.target_data: lm_data[1],
word_model.output_masks: lm_data[2],
word_model.sequence_length: lm_data[3],
word_model.initial_state: previous_state}
# The language model's final states of previous epoch is the language model's initial state of current epoch.
vals = session.run(fetches, feed_dict)
previous_state = vals["final_state"]
rnn_state_to_letter_model = vals["rnn_state"]
feed_dict_letter = {letter_model.lm_state_in: rnn_state_to_letter_model,
letter_model.input_data: letter_data[0],
letter_model.target_data: letter_data[1],
letter_model.output_masks: letter_data[2],
letter_model.sequence_length: letter_data[3]}
# The language model's rnn states of current epoch is the letter model's initial state of current epoch.
vals_letter = session.run(fetches_letter, feed_dict_letter)
cost_letter = vals_letter["cost"]
costs += cost_letter
iters += np.sum(letter_data[2])
num_word += np.sum(letter_data[3])
if verbose and step % (epoch_size // 1) == 0:
if costs / iters > 100.0:
print("[" + time.strftime('%Y-%m-%d %H:%M:%S') + "] PPL TOO LARGE! %.3f ENTROPY: (%.3f) speed: %.0f wps" %
(step * 1.0 / epoch_size, costs / iters, num_word / (time.time() - start_time)))
else:
print("[" + time.strftime('%Y-%m-%d %H:%M:%S') + "] %.3f letter_ppl: %.3f speed: %.0f wps" %
(step * 1.0 / epoch_size, np.exp(costs / iters), num_word / (time.time() - start_time)))
sys.stdout.flush()
return np.exp(costs / iters)
def run_word_epoch(session,
data,
word_model,
config,
lm_phase_id,
eval_op=None,
verbose=False):
"""Runs the model on the given data."""
start_time = time.time()
costs = 0.0
iters = 0
num_word = 0
fetches = {}
for step, (epoch_size, lm_data) in \
enumerate(data_feeder.data_iterator(data, config)):
if FLAGS.laptop_discount > 0 and step >= FLAGS.laptop_discount:
break
if step >= epoch_size:
break
fetches["cost"] = word_model.cost
fetches["final_state"] = word_model.final_state
if eval_op is not None:
fetches["eval_op"] = eval_op[lm_phase_id]
feed_dict = {
word_model.input_data: lm_data[0],
word_model.target_data: lm_data[1],
word_model.output_masks: lm_data[2],
word_model.sequence_length: lm_data[3]
}
vals = session.run(fetches, feed_dict)
cost = vals["cost"]
costs += cost[0]
iters += np.sum(lm_data[2])
num_word += np.sum(lm_data[3])
if verbose and step % (epoch_size // 100) == 0:
print("[" + time.strftime('%Y-%m-%d %H:%M:%S') + "] "
"%.3f word ppl: %.3f speed: %.0f wps" %
(step * 1.0 / epoch_size, np.exp(costs / iters), num_word /
(time.time() - start_time)))
sys.stdout.flush()
all_costs = np.exp(costs / iters)
return all_costs
def run_test_epoch(session,
data,
word_model,
letter_model,
cfg,
logfile,
phase="lm"):
"""Runs the model on the given data."""
start_time = time.time()
fetches = {}
fetches_letter = {}
previous_state = session.run(word_model.initial_state)
batch_size = cfg.batch_size
num_steps = cfg.num_steps
letter_batch_size = batch_size * num_steps
prediction_made = 0.0
top1_correct_total, top3_correct_total, top5_correct_total = 0.0, 0.0, 0.0
top1_empty_correct_total, top3_empty_correct_total, top5_empty_correct_total = 0.0, 0.0, 0.0
top1_complete_correct_total, top3_complete_correct_total, top5_complete_correct_total = 0.0, 0.0, 0.0
for step, (epoch_size, lm_data, letter_data, phrase_p_data, phrase_data) in \
enumerate(data_feeder.data_iterator(data, cfg)):
if step >= FLAGS.laptop_discount > 0:
break
if step >= epoch_size:
break
fetches["rnn_state"] = word_model.rnn_state
fetches["final_state"] = word_model.final_state
fetches["top_k_prediction"] = word_model.top_k_prediction[0]
fetches_letter["top_k_prediction"] = letter_model.top_k_prediction
feed_dict = {
word_model.input_data: lm_data[0],
word_model.target_data: lm_data[1],
word_model.output_masks: lm_data[2],
word_model.sequence_length: lm_data[3],
word_model.initial_state: previous_state,
word_model.top_k: 5
}
# The language model's final states of previous epoch is the language model's initial state of current epoch.
vals = session.run(fetches, feed_dict)
previous_state = vals["final_state"]
rnn_state_to_letter_model = vals["rnn_state"]
feed_dict_letter = {
letter_model.lm_state_in: rnn_state_to_letter_model,
letter_model.input_data: letter_data[0],
letter_model.target_data: letter_data[1],
letter_model.output_masks: letter_data[2],
letter_model.sequence_length: letter_data[3],
letter_model.top_k: 5
}
# The language model's rnn states of current epoch is the letter model's initial state of current epoch.
vals_letter = session.run(fetches_letter, feed_dict_letter)
if phase == "lm":
top_k_prediction = vals["top_k_prediction"]
# language model top_k_prediction.shape = [batch_size * num_steps, 5]
y_as_a_column = lm_data[1].reshape([-1])
mask_as_a_column = lm_data[2].reshape([-1])
prediction_made += np.sum(mask_as_a_column)
# sum of masks
top1_correct = np.sum(
(top_k_prediction[:, 0] == y_as_a_column) * mask_as_a_column)
top3_correct = top1_correct + np.sum((top_k_prediction[:, 1] == y_as_a_column) * mask_as_a_column) \
+ np.sum((top_k_prediction[:, 2] == y_as_a_column) * mask_as_a_column)
top5_correct = top3_correct + np.sum((top_k_prediction[:, 3] == y_as_a_column) * mask_as_a_column) \
+ np.sum((top_k_prediction[:, 4] == y_as_a_column) * mask_as_a_column)
top1_correct_total += top1_correct
top3_correct_total += top3_correct
top5_correct_total += top5_correct
else:
top_k_prediction = vals_letter["top_k_prediction"]
# letter model top_k_prediction.shape = [batch_size * num_steps * max_word_length, 5]
# for each row in y, extract only one label.
# letter_data[1].shape = [batch_size * num_steps, max_word_length]
y_as_a_column = letter_data[1][:, -1]
letter_lengths = letter_data[3]
# letter_length.shape = [batch_size * num_steps]
prediction_made += letter_batch_size # num of complete rows
# rows where <start> flags occurs in the letter sequence
empty_rows = np.arange(letter_batch_size) * cfg.max_word_length
top_k_ids_empty_only = top_k_prediction[empty_rows, :]
top1_empty_correct = np.sum(
(top_k_ids_empty_only[:, 0] == y_as_a_column))
top3_empty_correct = top1_empty_correct + np.sum((top_k_ids_empty_only[:, 1] == y_as_a_column)) \
+ np.sum((top_k_ids_empty_only[:, 2] == y_as_a_column))
top5_empty_correct = top3_empty_correct + np.sum((top_k_ids_empty_only[:, 3] == y_as_a_column)) \
+ np.sum((top_k_ids_empty_only[:, 4] == y_as_a_column))
# rows where letters of current word are complete
# letter_lengths = true length + 1 (prepended <start>)
complete_rows = np.arange(
letter_batch_size) * cfg.max_word_length + letter_lengths - 1
top_k_ids_complete_only = top_k_prediction[complete_rows, :]
top1_complete_correct = np.sum(
(top_k_ids_complete_only[:, 0] == y_as_a_column))
top3_complete_correct = top1_complete_correct + np.sum((top_k_ids_complete_only[:, 1] == y_as_a_column)) \
+ np.sum((top_k_ids_complete_only[:, 2] == y_as_a_column))
top5_complete_correct = top3_complete_correct + np.sum((top_k_ids_complete_only[:, 3] == y_as_a_column)) \
+ np.sum((top_k_ids_complete_only[:, 4] == y_as_a_column))
top1_empty_correct_total += top1_empty_correct
top3_empty_correct_total += top3_empty_correct
top5_empty_correct_total += top5_empty_correct
top1_complete_correct_total += top1_complete_correct
top3_complete_correct_total += top3_complete_correct
top5_complete_correct_total += top5_complete_correct
if phase == "lm":
top1_acc = top1_correct_total / prediction_made
top3_acc = top3_correct_total / prediction_made
top5_acc = top5_correct_total / prediction_made
# Prediction accuracy information:
print(
"Language model: Top1 accuracy = {0}, top3 accuracy = {1}, top5 accuracy = {2} "
.format(top1_acc, top3_acc, top5_acc),
file=logfile)
else:
top1_acc = top1_empty_correct_total / prediction_made
top3_acc = top3_empty_correct_total / prediction_made
top5_acc = top5_empty_correct_total / prediction_made
# Prediction accuracy information:
print("Accuracy with letter start: top1 {0}, top3 = {1}, top5 = {2} ".
format(top1_acc, top3_acc, top5_acc),
file=logfile)
top1_acc = top1_complete_correct_total / prediction_made
top3_acc = top3_complete_correct_total / prediction_made
top5_acc = top5_complete_correct_total / prediction_made
# Prediction accuracy information:
print("Accuracy with full letters: top1 {0}, top3 = {1}, top5 = {2} ".
format(top1_acc, top3_acc, top5_acc),
file=logfile)
end_time = time.time()
print("Test time = {0}".format(end_time - start_time), file=logfile)
def run_word_epoch(session,
data,
word_model,
cfg,
lm_phase_id,
eval_op=None,
verbose=False):
"""Runs the model on the given data."""
start_time = time.time()
costs = 0.0
costs_phrase_p = 0
costs_phrase = 0
iters = 0
iters_phrase_p = 0
iters_phrase = 0
num_word = 0
fetches = {}
previous_state = session.run(word_model.initial_state)
for step, (epoch_size, lm_data, _, phrase_p_data, phrase_data) in \
enumerate(data_feeder.data_iterator(data, cfg)):
if step >= FLAGS.laptop_discount > 0:
break
if step >= epoch_size:
break
fetches["cost"] = word_model.cost
fetches["final_state"] = word_model.final_state
if eval_op is not None:
fetches["eval_op"] = eval_op[lm_phase_id]
feed_dict = {
word_model.input_data: lm_data[0],
word_model.target_data: lm_data[1],
word_model.output_masks: lm_data[2],
word_model.sequence_length: lm_data[3],
word_model.target_phrase_p: phrase_p_data[0],
word_model.target_phrase_p_masks: phrase_p_data[1],
word_model.target_phrase_data: phrase_data[0],
word_model.target_phrase_data_masks: phrase_data[1],
word_model.target_phrase_logits_masks: phrase_data[2],
word_model.initial_state: previous_state
}
# The language model's final states of previous epoch is the language model's initial state of current epoch.
vals = session.run(fetches, feed_dict)
cost = vals["cost"]
previous_state = vals["final_state"]
costs += cost[0]
costs_phrase_p += cost[1]
costs_phrase += cost[2]
iters += np.sum(lm_data[2])
iters_phrase_p += np.sum(phrase_p_data[1])
iters_phrase += np.sum(phrase_data[1])
num_word += np.sum(lm_data[3])
if verbose and step % (max(epoch_size // 100, 1)) == 0:
if lm_phase_id == 0:
print(
"[" + time.strftime('%Y-%m-%d %H:%M:%S') + "] "
"%.3f word ppl: %.3f speed: %.0f wps" %
(step * 1.0 / epoch_size, np.exp(costs / iters), num_word /
(time.time() - start_time)))
elif lm_phase_id == 1:
print("[" + time.strftime('%Y-%m-%d %H:%M:%S') + "] "
"%.3f phrase prob ppl: %.3f speed: %.0f wps" %
(step * 1.0 / epoch_size,
np.exp(costs_phrase_p / iters_phrase_p), num_word /
(time.time() - start_time)))
else:
print("[" + time.strftime('%Y-%m-%d %H:%M:%S') + "] "
"%.3f phrase ppl: %.3f speed: %.0f wps" %
(step * 1.0 / epoch_size,
np.exp(costs_phrase / iters_phrase), num_word /
(time.time() - start_time)))
sys.stdout.flush()
costs_list = np.exp(costs / iters), np.exp(
costs_phrase_p / iters_phrase_p), np.exp(costs_phrase / iters_phrase)
return costs_list[lm_phase_id]
