def train(self, corpus, epochs=10, direction="pronounce", window=-1): """Runs training.""" # Create training log that also redirects to stdout. stdout_file = sys.stdout logfile = os.path.join(self._checkpoint_dir, "train.log") print("Training log: {}".format(logfile)) sys.stdout = utils.DualLogger(logfile) # Dump some parameters. print(" Direction: {}".format(direction)) print(" # Epochs: {}".format(epochs)) print(" Batch size: {}".format(self._batch_size)) print(" Window size: {}".format(window)) print(" Max written len: {}".format(corpus.max_written_len)) print(" Max pron len: {}".format(corpus.max_pronounce_len)) print("Max written word len: {}".format(corpus.max_written_word_len)) print(" Max pron word len: {}".format(corpus.max_pronounce_word_len)) print(" Use residuals: {}".format(self._use_residuals)) # Perform training. best_total_loss = 1000000 nbatches = data.num_batches(corpus, self._batch_size, direction=direction, window=window) for epoch in range(epochs): self._train_accuracy.reset_states() start = time.time() total_loss = 0 steps = 0 batches = data.batchify(corpus, self._batch_size, direction, window=window) batch, (inputs, targ) = next(batches) while batch > -1: bos = np.expand_dims( [self._output_symbols.find("<s>")] * np.shape(targ)[0], 1) targets = np.concatenate((bos, targ), axis=-1) batch_loss = self._train_step(inputs, targets) total_loss += batch_loss if batch % 10 == 0: print("Epoch {} Batch {} (/{}) Loss {:.4f}".format( epoch + 1, batch, nbatches, batch_loss)) steps += 1 batch, (inputs, targ) = next(batches) total_loss /= steps print("Epoch {} Loss {:.4f} Accuracy {:.4f}".format( epoch + 1, total_loss, self._train_accuracy.result())) if total_loss < best_total_loss: self._checkpoint.save(file_prefix=self._checkpoint_prefix) print("Saved checkpoint to {}".format(self._checkpoint_prefix)) best_total_loss = total_loss print("Time taken for 1 epoch {} sec\n".format( time.time() - start)) print("Best total loss: {:.4f}".format(best_total_loss)) # Restore stdout. sys.stdout = stdout_file
def _test_language(language, corpus, model, print_predictions=False, show_plots=False, compute_deviation=True, deviation_only_for_correct=True, simple_skew=False): """Runs model evaluation.""" # Create test log that also redirects to stdout. stdout_file = sys.stdout log_name = "%s.log" % _eval_file_prefix(model) logfile = os.path.join(model.checkpoint_dir, log_name) print("Test log: {}".format(logfile)) sys.stdout = utils.DualLogger(logfile) print("Window size: {}".format(FLAGS.window)) print("# test examples: {}".format(FLAGS.ntest)) test_examples = data.test_examples(corpus, FLAGS.direction, window=FLAGS.window) indices = data.random_test_indices(test_examples, k=FLAGS.ntest) tot, cor, rat, nrat = eval_lib.eval_and_plot( model, test_examples, indices, show_plots=show_plots, print_predictions=print_predictions, print_attention=PRINT_ATTENTION, compute_deviation=compute_deviation, deviation_mask_sigma=DEVIATION_MASK_SIGMA, deviation_only_for_correct=deviation_only_for_correct, simple_skew=simple_skew, report_type_stats=FLAGS.report_type_stats, figsize=FIGSIZE) # Write results to the log, stdout and the dedicated file. results = [ "*" * 80, "Language: {}".format(language), "Total non-trivial predictions: {}".format(tot), "Accuracy: {}".format(cor), "Ratio: {}".format(rat), "tf.reduce_max'ed normalized ratio: {}".format(nrat) ] print("\n".join(results)) results_file = "%s_results.txt" % _eval_file_prefix(model) with open(os.path.join(model.checkpoint_dir, results_file), encoding="utf-8", mode="wt") as f: f.write("\n".join(results) + "\n") # Restore stdout. sys.stdout = stdout_file
def _test_language(language, corpus, model, print_predictions=False, show_plots=False, compute_deviation=True, deviation_only_for_correct=True, simple_skew=False): """Runs model evaluation.""" # Create test log that also redirects to stdout. stdout_file = sys.stdout logfile = os.path.join(model.checkpoint_dir, "eval.log") print("Test log: {}".format(logfile)) sys.stdout = utils.DualLogger(logfile) print("Window size: {}".format(FLAGS.window)) print("# test examples: {}".format(FLAGS.ntest)) test_examples = data.test_examples(corpus, FLAGS.direction, window=FLAGS.window) indices = data.random_test_indices(test_examples, k=FLAGS.ntest) tot, cor, rat, nrat = evaluate.eval_and_plot( model, test_examples, indices, show_plots=show_plots, print_predictions=print_predictions, print_attention=_PRINT_ATTENTION, compute_deviation=compute_deviation, deviation_mask_sigma=_DEVIATION_MASK_SIGMA, deviation_only_for_correct=deviation_only_for_correct, simple_skew=simple_skew, report_type_stats=FLAGS.report_type_stats, figsize=_FIGSIZE) print("*" * 80) print("Language: {}".format(language)) print("Total non-trivial predictions: {}".format(tot)) print("Accuracy: {}".format(cor)) print("Ratio: {}".format(rat)) print("tf.reduce_max'ed normalized ratio: {}".format(nrat)) # Restore stdout. sys.stdout = stdout_file
def train(self, corpus, epochs=10, direction="pronounce", window=-1): """Main entry point for running the training.""" # Create training log that also redirects to stdout. stdout_file = sys.stdout logfile = os.path.join(self._checkpoint_dir, "train.log") print("Training log: {}".format(logfile)) sys.stdout = utils.DualLogger(logfile) # Dump some parameters. print(" Direction: {}".format(direction)) print(" # Epochs: {}".format(epochs)) print(" Batch size: {}".format(self._batch_size)) print(" Window size: {}".format(window)) print(" Max written len: {}".format(corpus.max_written_len)) print(" Max pron len: {}".format(corpus.max_pronounce_len)) print("Max written word len: {}".format(corpus.max_written_word_len)) print(" Max pron word len: {}".format(corpus.max_pronounce_word_len)) # Perform training. best_total_loss = 1000000 nbatches = data.num_batches(corpus, self._batch_size, direction=direction, window=window) for epoch in range(epochs): start = time.time() total_loss = 0 steps = 0 batches = data.batchify(corpus, self._batch_size, direction, window=window) batch, (inp, targ) = next(batches) enc_hidden = self._encoder.initialize_hidden_state( self._batch_size) if self._input_length == -1: # TODO(agutkin,rws): Following two lines will break if batchify() # returns an empty tuple. if isinstance(inp, np.ndarray): self._input_length = inp.shape[1] if isinstance(targ, np.ndarray): self._output_length = targ.shape[1] while batch > -1: batch_loss = self._train_step(inp, targ, enc_hidden) total_loss += batch_loss if batch % 10 == 0: print("Epoch {} Batch {} (/{}) Loss {:.4f}".format( epoch + 1, batch, nbatches, batch_loss.numpy())) steps += 1 batch, (inp, targ) = next(batches) total_loss /= steps print("Epoch {} Loss {:.4f}".format(epoch + 1, total_loss)) if total_loss < best_total_loss: self._checkpoint.save(file_prefix=self._checkpoint_prefix) print("Saved checkpoint to {}".format(self._checkpoint_prefix)) best_total_loss = total_loss print("Time taken for 1 epoch {} sec\n".format(time.time() - start)) print("Best total loss: {:.4f}".format(best_total_loss)) # Restore stdout. sys.stdout = stdout_file