def run_letter_epoch(session, data, word_model, letter_model, config, eval_op=None, verbose=False): """Runs the model on the given data.""" start_time = time.time() costs = 0.0 iters = 0 num_word = 0 fetches = {} fetches_letter = {} previous_state = session.run(word_model.initial_state) for step, (epoch_size, lm_data, letter_data, phrase_p_data, phrase_data) in \ enumerate(data_feeder.data_iterator(data, config)): if FLAGS.laptop_discount > 0 and step >= FLAGS.laptop_discount: break if step >= epoch_size: break fetches["rnn_state"] = word_model.rnn_state fetches["final_state"] = word_model.final_state fetches_letter["cost"] = letter_model.cost if eval_op is not None: fetches_letter["eval_op"] = eval_op feed_dict = {word_model.input_data: lm_data[0], word_model.target_data: lm_data[1], word_model.output_masks: lm_data[2], word_model.sequence_length: lm_data[3], word_model.initial_state: previous_state} # The language model's final states of previous epoch is the language model's initial state of current epoch. vals = session.run(fetches, feed_dict) previous_state = vals["final_state"] rnn_state_to_letter_model = vals["rnn_state"] feed_dict_letter = {letter_model.lm_state_in: rnn_state_to_letter_model, letter_model.input_data: letter_data[0], letter_model.target_data: letter_data[1], letter_model.output_masks: letter_data[2], letter_model.sequence_length: letter_data[3]} # The language model's rnn states of current epoch is the letter model's initial state of current epoch. vals_letter = session.run(fetches_letter, feed_dict_letter) cost_letter = vals_letter["cost"] costs += cost_letter iters += np.sum(letter_data[2]) num_word += np.sum(letter_data[3]) if verbose and step % (epoch_size // 1) == 0: if costs / iters > 100.0: print("[" + time.strftime('%Y-%m-%d %H:%M:%S') + "] PPL TOO LARGE! %.3f ENTROPY: (%.3f) speed: %.0f wps" % (step * 1.0 / epoch_size, costs / iters, num_word / (time.time() - start_time))) else: print("[" + time.strftime('%Y-%m-%d %H:%M:%S') + "] %.3f letter_ppl: %.3f speed: %.0f wps" % (step * 1.0 / epoch_size, np.exp(costs / iters), num_word / (time.time() - start_time))) sys.stdout.flush() return np.exp(costs / iters)
def run_word_epoch(session, data, word_model, config, lm_phase_id, eval_op=None, verbose=False): """Runs the model on the given data.""" start_time = time.time() costs = 0.0 iters = 0 num_word = 0 fetches = {} for step, (epoch_size, lm_data) in \ enumerate(data_feeder.data_iterator(data, config)): if FLAGS.laptop_discount > 0 and step >= FLAGS.laptop_discount: break if step >= epoch_size: break fetches["cost"] = word_model.cost fetches["final_state"] = word_model.final_state if eval_op is not None: fetches["eval_op"] = eval_op[lm_phase_id] feed_dict = { word_model.input_data: lm_data[0], word_model.target_data: lm_data[1], word_model.output_masks: lm_data[2], word_model.sequence_length: lm_data[3] } vals = session.run(fetches, feed_dict) cost = vals["cost"] costs += cost[0] iters += np.sum(lm_data[2]) num_word += np.sum(lm_data[3]) if verbose and step % (epoch_size // 100) == 0: print("[" + time.strftime('%Y-%m-%d %H:%M:%S') + "] " "%.3f word ppl: %.3f speed: %.0f wps" % (step * 1.0 / epoch_size, np.exp(costs / iters), num_word / (time.time() - start_time))) sys.stdout.flush() all_costs = np.exp(costs / iters) return all_costs
def run_test_epoch(session, data, word_model, letter_model, cfg, logfile, phase="lm"): """Runs the model on the given data.""" start_time = time.time() fetches = {} fetches_letter = {} previous_state = session.run(word_model.initial_state) batch_size = cfg.batch_size num_steps = cfg.num_steps letter_batch_size = batch_size * num_steps prediction_made = 0.0 top1_correct_total, top3_correct_total, top5_correct_total = 0.0, 0.0, 0.0 top1_empty_correct_total, top3_empty_correct_total, top5_empty_correct_total = 0.0, 0.0, 0.0 top1_complete_correct_total, top3_complete_correct_total, top5_complete_correct_total = 0.0, 0.0, 0.0 for step, (epoch_size, lm_data, letter_data, phrase_p_data, phrase_data) in \ enumerate(data_feeder.data_iterator(data, cfg)): if step >= FLAGS.laptop_discount > 0: break if step >= epoch_size: break fetches["rnn_state"] = word_model.rnn_state fetches["final_state"] = word_model.final_state fetches["top_k_prediction"] = word_model.top_k_prediction[0] fetches_letter["top_k_prediction"] = letter_model.top_k_prediction feed_dict = { word_model.input_data: lm_data[0], word_model.target_data: lm_data[1], word_model.output_masks: lm_data[2], word_model.sequence_length: lm_data[3], word_model.initial_state: previous_state, word_model.top_k: 5 } # The language model's final states of previous epoch is the language model's initial state of current epoch. vals = session.run(fetches, feed_dict) previous_state = vals["final_state"] rnn_state_to_letter_model = vals["rnn_state"] feed_dict_letter = { letter_model.lm_state_in: rnn_state_to_letter_model, letter_model.input_data: letter_data[0], letter_model.target_data: letter_data[1], letter_model.output_masks: letter_data[2], letter_model.sequence_length: letter_data[3], letter_model.top_k: 5 } # The language model's rnn states of current epoch is the letter model's initial state of current epoch. vals_letter = session.run(fetches_letter, feed_dict_letter) if phase == "lm": top_k_prediction = vals["top_k_prediction"] # language model top_k_prediction.shape = [batch_size * num_steps, 5] y_as_a_column = lm_data[1].reshape([-1]) mask_as_a_column = lm_data[2].reshape([-1]) prediction_made += np.sum(mask_as_a_column) # sum of masks top1_correct = np.sum( (top_k_prediction[:, 0] == y_as_a_column) * mask_as_a_column) top3_correct = top1_correct + np.sum((top_k_prediction[:, 1] == y_as_a_column) * mask_as_a_column) \ + np.sum((top_k_prediction[:, 2] == y_as_a_column) * mask_as_a_column) top5_correct = top3_correct + np.sum((top_k_prediction[:, 3] == y_as_a_column) * mask_as_a_column) \ + np.sum((top_k_prediction[:, 4] == y_as_a_column) * mask_as_a_column) top1_correct_total += top1_correct top3_correct_total += top3_correct top5_correct_total += top5_correct else: top_k_prediction = vals_letter["top_k_prediction"] # letter model top_k_prediction.shape = [batch_size * num_steps * max_word_length, 5] # for each row in y, extract only one label. # letter_data[1].shape = [batch_size * num_steps, max_word_length] y_as_a_column = letter_data[1][:, -1] letter_lengths = letter_data[3] # letter_length.shape = [batch_size * num_steps] prediction_made += letter_batch_size # num of complete rows # rows where <start> flags occurs in the letter sequence empty_rows = np.arange(letter_batch_size) * cfg.max_word_length top_k_ids_empty_only = top_k_prediction[empty_rows, :] top1_empty_correct = np.sum( (top_k_ids_empty_only[:, 0] == y_as_a_column)) top3_empty_correct = top1_empty_correct + np.sum((top_k_ids_empty_only[:, 1] == y_as_a_column)) \ + np.sum((top_k_ids_empty_only[:, 2] == y_as_a_column)) top5_empty_correct = top3_empty_correct + np.sum((top_k_ids_empty_only[:, 3] == y_as_a_column)) \ + np.sum((top_k_ids_empty_only[:, 4] == y_as_a_column)) # rows where letters of current word are complete # letter_lengths = true length + 1 (prepended <start>) complete_rows = np.arange( letter_batch_size) * cfg.max_word_length + letter_lengths - 1 top_k_ids_complete_only = top_k_prediction[complete_rows, :] top1_complete_correct = np.sum( (top_k_ids_complete_only[:, 0] == y_as_a_column)) top3_complete_correct = top1_complete_correct + np.sum((top_k_ids_complete_only[:, 1] == y_as_a_column)) \ + np.sum((top_k_ids_complete_only[:, 2] == y_as_a_column)) top5_complete_correct = top3_complete_correct + np.sum((top_k_ids_complete_only[:, 3] == y_as_a_column)) \ + np.sum((top_k_ids_complete_only[:, 4] == y_as_a_column)) top1_empty_correct_total += top1_empty_correct top3_empty_correct_total += top3_empty_correct top5_empty_correct_total += top5_empty_correct top1_complete_correct_total += top1_complete_correct top3_complete_correct_total += top3_complete_correct top5_complete_correct_total += top5_complete_correct if phase == "lm": top1_acc = top1_correct_total / prediction_made top3_acc = top3_correct_total / prediction_made top5_acc = top5_correct_total / prediction_made # Prediction accuracy information: print( "Language model: Top1 accuracy = {0}, top3 accuracy = {1}, top5 accuracy = {2} " .format(top1_acc, top3_acc, top5_acc), file=logfile) else: top1_acc = top1_empty_correct_total / prediction_made top3_acc = top3_empty_correct_total / prediction_made top5_acc = top5_empty_correct_total / prediction_made # Prediction accuracy information: print("Accuracy with letter start: top1 {0}, top3 = {1}, top5 = {2} ". format(top1_acc, top3_acc, top5_acc), file=logfile) top1_acc = top1_complete_correct_total / prediction_made top3_acc = top3_complete_correct_total / prediction_made top5_acc = top5_complete_correct_total / prediction_made # Prediction accuracy information: print("Accuracy with full letters: top1 {0}, top3 = {1}, top5 = {2} ". format(top1_acc, top3_acc, top5_acc), file=logfile) end_time = time.time() print("Test time = {0}".format(end_time - start_time), file=logfile)
def run_word_epoch(session, data, word_model, cfg, lm_phase_id, eval_op=None, verbose=False): """Runs the model on the given data.""" start_time = time.time() costs = 0.0 costs_phrase_p = 0 costs_phrase = 0 iters = 0 iters_phrase_p = 0 iters_phrase = 0 num_word = 0 fetches = {} previous_state = session.run(word_model.initial_state) for step, (epoch_size, lm_data, _, phrase_p_data, phrase_data) in \ enumerate(data_feeder.data_iterator(data, cfg)): if step >= FLAGS.laptop_discount > 0: break if step >= epoch_size: break fetches["cost"] = word_model.cost fetches["final_state"] = word_model.final_state if eval_op is not None: fetches["eval_op"] = eval_op[lm_phase_id] feed_dict = { word_model.input_data: lm_data[0], word_model.target_data: lm_data[1], word_model.output_masks: lm_data[2], word_model.sequence_length: lm_data[3], word_model.target_phrase_p: phrase_p_data[0], word_model.target_phrase_p_masks: phrase_p_data[1], word_model.target_phrase_data: phrase_data[0], word_model.target_phrase_data_masks: phrase_data[1], word_model.target_phrase_logits_masks: phrase_data[2], word_model.initial_state: previous_state } # The language model's final states of previous epoch is the language model's initial state of current epoch. vals = session.run(fetches, feed_dict) cost = vals["cost"] previous_state = vals["final_state"] costs += cost[0] costs_phrase_p += cost[1] costs_phrase += cost[2] iters += np.sum(lm_data[2]) iters_phrase_p += np.sum(phrase_p_data[1]) iters_phrase += np.sum(phrase_data[1]) num_word += np.sum(lm_data[3]) if verbose and step % (max(epoch_size // 100, 1)) == 0: if lm_phase_id == 0: print( "[" + time.strftime('%Y-%m-%d %H:%M:%S') + "] " "%.3f word ppl: %.3f speed: %.0f wps" % (step * 1.0 / epoch_size, np.exp(costs / iters), num_word / (time.time() - start_time))) elif lm_phase_id == 1: print("[" + time.strftime('%Y-%m-%d %H:%M:%S') + "] " "%.3f phrase prob ppl: %.3f speed: %.0f wps" % (step * 1.0 / epoch_size, np.exp(costs_phrase_p / iters_phrase_p), num_word / (time.time() - start_time))) else: print("[" + time.strftime('%Y-%m-%d %H:%M:%S') + "] " "%.3f phrase ppl: %.3f speed: %.0f wps" % (step * 1.0 / epoch_size, np.exp(costs_phrase / iters_phrase), num_word / (time.time() - start_time))) sys.stdout.flush() costs_list = np.exp(costs / iters), np.exp( costs_phrase_p / iters_phrase_p), np.exp(costs_phrase / iters_phrase) return costs_list[lm_phase_id]