def load_training_dataset(hyper_params): # Load the train set data data_processor = dataprocessor.DataProcessor(hyper_params["training_dataset_dirs"], file_cache=hyper_params["training_filelist_cache"]) if hyper_params["dataset_size_ordering"] in ['True', 'First_run_only']: train_set = data_processor.get_ordered_dataset() else: train_set = data_processor.get_dataset() shuffle(train_set) if hyper_params["test_dataset_dirs"] is not None: # Load the test set data data_processor = dataprocessor.DataProcessor(hyper_params["test_dataset_dirs"]) test_set = data_processor.get_dataset() elif hyper_params["train_frac"] is not None: # Or use a fraction of the train set for the test set num_train = max(1, int(floor(hyper_params["train_frac"] * len(train_set)))) test_set = train_set[num_train:] train_set = train_set[:num_train] else: # Or use no test set test_set = [] logging.info("Using %d files in train set", len(train_set)) logging.info("Using %d size of test set", len(test_set)) return train_set, test_set
def train_rnn(audio_processor, hyper_params, prog_params): # Load the train set data data_processor = dataprocessor.DataProcessor( hyper_params["training_dataset_dir"], hyper_params["training_dataset_type"], audio_processor) train_set = data_processor.run() if (hyper_params["test_dataset_dir"] is not None) and (hyper_params["test_dataset_type"] is not None): # Load the test set data data_processor = dataprocessor.DataProcessor( hyper_params["test_dataset_dir"], hyper_params["test_dataset_type"], audio_processor) test_set = data_processor.run() elif hyper_params["train_frac"] is not None: # Or use a fraction of the train set for the test set num_train = max( 1, int(floor(hyper_params["train_frac"] * len(train_set)))) test_set = train_set[num_train:] train_set = train_set[:num_train] else: # Or use no test set test_set = [] if hyper_params["pre_filter_file_size"] is True: test_set = data_processor.filterDataset( test_set, hyper_params["max_input_seq_length"], hyper_params["max_target_seq_length"]) train_set = data_processor.filterDataset( train_set, hyper_params["max_input_seq_length"], hyper_params["max_target_seq_length"]) print("Using {0} files in train set".format(len(train_set))) print("Using {0} size of test set".format(len(test_set))) with tf.Session() as sess: # create model print("Building model... (this takes a while)") model = createAcousticModel( sess, hyper_params, hyper_params["batch_size"], forward_only=False, tensorboard_dir=hyper_params["tensorboard_dir"], tb_run_name=prog_params["tb_name"]) # Override the learning rate if given on the command line if prog_params["learn_rate"] is not None: assign_op = model.learning_rate.assign(prog_params["learn_rate"]) sess.run(assign_op) print("Setting up audio processor...") model.initializeAudioProcessor(hyper_params["max_input_seq_length"], hyper_params["load_save_input_vec"]) print("Start training...") model.train(sess, test_set, train_set, hyper_params["steps_per_checkpoint"], hyper_params["checkpoint_dir"], hyper_params["async_get_batch"], max_epoch=prog_params["max_epoch"])
def train_rnn(audio_processor, hyper_params, prog_params): # Load the train set data data_processor = dataprocessor.DataProcessor( hyper_params["training_dataset_dirs"], audio_processor, size_ordering=hyper_params["dataset_size_ordering"]) train_set = data_processor.run() if hyper_params["test_dataset_dirs"] is not None: # Load the test set data data_processor = dataprocessor.DataProcessor( hyper_params["test_dataset_dirs"], audio_processor, size_ordering=hyper_params["dataset_size_ordering"]) test_set = data_processor.run() elif hyper_params["train_frac"] is not None: # Or use a fraction of the train set for the test set num_train = max( 1, int(floor(hyper_params["train_frac"] * len(train_set)))) test_set = train_set[num_train:] train_set = train_set[:num_train] else: # Or use no test set test_set = [] logging.info("Using %d files in train set", len(train_set)) logging.info("Using %d size of test set", len(test_set)) with tf.Session() as sess: # create model model = create_acoustic_model( sess, hyper_params, hyper_params["batch_size"], forward_only=False, tensorboard_dir=hyper_params["tensorboard_dir"], tb_run_name=prog_params["tb_name"], timeline_enabled=prog_params["timeline"]) # Override the learning rate if given on the command line if prog_params["learn_rate"] is not None: assign_op = model.learning_rate.assign(prog_params["learn_rate"]) sess.run(assign_op) logging.info("Start training...") model.train(sess, audio_processor, test_set, train_set, hyper_params["steps_per_checkpoint"], hyper_params["checkpoint_dir"], max_epoch=prog_params["max_epoch"])
def test_get_data_shtooka(self): data_processor = dataprocessor.DataProcessor(self.directory + "Shtooka") test_set = data_processor.get_dataset() self.assertCountEqual(test_set, [[ self.directory + "Shtooka/flac/eng - I_arose.flac", "i arose", None ], [self.directory + "Shtooka/flac/eng - I_ate.flac", "i ate", None]])
def test_get_data_shtooka(self): data_processor = dataprocessor.DataProcessor(self.directory + "Shtooka", self.audio_processor) test_set = data_processor.run() self.assertCountEqual(test_set, [[self.directory + "Shtooka/flac/eng - I_arose.flac", "i arose", 0], [self.directory + "Shtooka/flac/eng - I_ate.flac", "i ate", 0] ])
def evaluate(hyper_params): if hyper_params["test_dataset_dirs"] is None: logging.fatal("Setting test_dataset_dirs in config file is mandatory for evaluation mode") return # Load the test set data data_processor = dataprocessor.DataProcessor(hyper_params["test_dataset_dirs"]) test_set = data_processor.get_dataset() logging.info("Using %d size of test set", len(test_set)) if len(test_set) == 0: logging.fatal("No files in test set during an evaluation mode") return with tf.Session() as sess: # create model model = AcousticModel(hyper_params["num_layers"], hyper_params["hidden_size"], hyper_params["batch_size"], hyper_params["max_input_seq_length"], hyper_params["max_target_seq_length"], hyper_params["input_dim"], hyper_params["batch_normalization"], language=hyper_params["language"]) model.create_forward_rnn() model.initialize(sess) model.restore(sess, hyper_params["checkpoint_dir"]) wer, cer = model.evaluate_full(sess, test_set, hyper_params["max_input_seq_length"], hyper_params["signal_processing"]) print("Resulting WER : {0:.3g} %".format(wer)) print("Resulting CER : {0:.3g} %".format(cer)) return
def test_get_data_vystadial_2013(self): data_processor = dataprocessor.DataProcessor(self.directory + "Vystadial_2013") test_set = data_processor.get_dataset() self.assertCountEqual(test_set, [[self.directory + "Vystadial_2013/data_voip_en/dev/jurcic-028-121024_234433_0013625_0013836.wav", "alright thank you and goodbye", None] ])
def load_acoustic_dataset(training_dataset_dirs, test_dataset_dirs=None, training_filelist_cache=None, ordered=False, train_frac=None): """ Load the datatsets for the acoustic model training Return a train set and an optional test set, each containing a list of [audio_file, label, audio_length] Parameters ---------- :param training_dataset_dirs: directory where to find the training data :param test_dataset_dirs: directory where to find the test data (optional) :param training_filelist_cache: path to the cache file for the training data (optional) :param ordered: boolean indicating whether or not to order the dataset by audio files length (ascending) :param train_frac: the fraction of the training data to be used as test data (only used if test_dataset_dirs is None) :return train_set, test_set: two lists of [audio_file, label, audio_length] where audio_file is the path to an audio file label is the true label for the audio file (relative to the char_map) audio_length if the length of the audio file """ data_processor = dataprocessor.DataProcessor( training_dataset_dirs, file_cache=training_filelist_cache) train_set = data_processor.get_dataset() if ordered: train_set = sorted(train_set, key=lambda x: x[2]) else: shuffle(train_set) if test_dataset_dirs is not None: # Load the test set data data_processor = dataprocessor.DataProcessor(test_dataset_dirs) test_set = data_processor.get_dataset() elif train_frac is not None: # Or use a fraction of the train set for the test set num_train = max(1, int(floor(train_frac * len(train_set)))) test_set = train_set[num_train:] train_set = train_set[:num_train] else: # Or use no test set test_set = [] logging.info("Using %d files in train set", len(train_set)) logging.info("Using %d size of test set", len(test_set)) return train_set, test_set
def test_get_data_librispeech(self): data_processor = dataprocessor.DataProcessor(self.directory + "Libri") test_set = data_processor.get_dataset() self.assertCountEqual(test_set, [[self.directory + "Libri/train-clean-100/19/198/19-198-0000.flac", "northanger abbey", None], [self.directory + "Libri/train-clean-100/19/198/19-198-0001.flac", "this little work", None] ])
def main(): dirs = [ os.path.join(FLAGS.data_dir, "train/"), os.path.join(FLAGS.data_dir, "test/") ] if not (os.path.exists(dirs[0]) and os.path.exists(dirs[1])): print "Train/Test files not detected, creating now..." config = ConfigParser.ConfigParser() config.read(FLAGS.config_file) max_num_lines = int(config.get("max_data_sizes", "num_lines")) max_target_size = int(config.get("max_data_sizes", "max_target_length")) max_source_size = int(config.get("max_data_sizes", "max_source_length")) data_processor = data_utils.DataProcessor( FLAGS.vocab_size, FLAGS.raw_data_dir, FLAGS.data_dir, FLAGS.train_frac, FLAGS.tokenizer, max_num_lines, max_target_size, max_source_size) data_processor.run() assert FLAGS.plot_histograms or FLAGS.plot_scatter, "Must choose at least one plot!" source_lengths = [] target_lengths = [] count = 0 for i in range(len(dirs)): if "test" in dirs[i]: source_path = os.path.join(dirs[i], "data_source_test.txt") target_path = os.path.join(dirs[i], "data_target_test.txt") else: source_path = os.path.join(dirs[i], "data_source_train.txt") target_path = os.path.join(dirs[i], "data_target_train.txt") with tf.gfile.GFile(source_path, mode="r") as source_file: with tf.gfile.GFile(target_path, mode="r") as target_file: source, target = source_file.readline(), target_file.readline() counter = 0 while source and target: counter += 1 if counter % 100000 == 0: print(" reading data line %d" % counter) sys.stdout.flush() num_source_ids = len(source.split()) source_lengths.append(num_source_ids) #plus 1 for EOS token num_target_ids = len(target.split()) + 1 target_lengths.append(num_target_ids) source, target = source_file.readline( ), target_file.readline() if FLAGS.plot_histograms: plot_histo_lengths("target lengths", target_lengths) plot_histo_lengths("source_lengths", source_lengths) if FLAGS.plot_scatter: plot_scatter_lengths("target vs source length", "source length", "target length", source_lengths, target_lengths)
def evaluate(audio_processor, hyper_params): if hyper_params["test_dataset_dirs"] is None: logging.fatal( "Setting test_dataset_dirs in config file is mandatory for evaluation mode" ) return # Load the test set data data_processor = dataprocessor.DataProcessor( hyper_params["test_dataset_dirs"], audio_processor, size_ordering=hyper_params["dataset_size_ordering"]) test_set = data_processor.run() logging.info("Using %d size of test set", len(test_set)) if len(test_set) == 0: logging.fatal("No files in test set during an evaluation mode") return with tf.Session() as sess: # create model model = create_acoustic_model(sess, hyper_params, hyper_params["batch_size"], forward_only=True, tensorboard_dir=None, tb_run_name=None, timeline_enabled=False) wer_list = [] cer_list = [] file_number = 0 input_feat_vecs = [] input_feat_vec_lengths = [] labels = [] for file, label, _ in test_set: feat_vec, feat_vec_length = audio_processor.process_audio_file( file) file_number += 1 label_data_length = len(label) if (label_data_length > hyper_params["max_target_seq_length"]) or\ (feat_vec_length > hyper_params["max_input_seq_length"]): logging.warning( "Warning - sample too long : %s (input : %d / text : %s)", file, feat_vec_length, label_data_length) else: logging.debug("Processed file %d / %d", file_number, len(test_set)) input_feat_vecs.append(feat_vec) input_feat_vec_lengths.append(feat_vec_length) labels.append(label) # If we reached the last file then pad the lists to obtain a full batch if file_number == len(test_set): for i in range(hyper_params["batch_size"] - len(input_feat_vecs)): input_feat_vecs.append( np.zeros([ hyper_params["max_input_seq_length"], audio_processor.feature_size ])) input_feat_vec_lengths.append(0) labels.append("") if len(input_feat_vecs) == hyper_params["batch_size"]: # Run the batch logging.debug("Running a batch") input_feat_vecs = np.swapaxes(input_feat_vecs, 0, 1) transcribed_texts = model.process_input( sess, input_feat_vecs, input_feat_vec_lengths) for index, transcribed_text in enumerate(transcribed_texts): true_label = labels[index] if len(true_label) > 0: nb_words = len(true_label.split()) nb_chars = len(true_label.replace(" ", "")) wer_list.append( model.calculate_wer(transcribed_text, true_label) / float(nb_words)) cer_list.append( model.calculate_cer(transcribed_text, true_label) / float(nb_chars)) # Reset the lists input_feat_vecs = [] input_feat_vec_lengths = [] labels = [] print("Resulting WER : {0:.3g} %".format( (sum(wer_list) * 100) / float(len(wer_list)))) print("Resulting CER : {0:.3g} %".format( (sum(cer_list) * 100) / float(len(cer_list)))) return
def main(): if not os.path.exists(FLAGS.checkpoint_dir): os.mkdir(FLAGS.checkpoint_dir) path = get_checkpoint_path() print("path is {0}".format(path)) data_processor = data_utils.DataProcessor( FLAGS.vocab_size, FLAGS.raw_data_dir, FLAGS.data_dir, FLAGS.train_frac, FLAGS.tokenizer, FLAGS.convo_limits, FLAGS.max_target_length, FLAGS.max_source_length) data_processor.run() #create model print("Creating model with...") print("Number of hidden layers: {0}".format(FLAGS.num_layers)) print("Number of units per layer: {0}".format(FLAGS.hidden_size)) print("Dropout: {0}".format(FLAGS.dropout)) vocab_mapper = vocab_utils.VocabMapper(FLAGS.data_dir) vocab_size = vocab_mapper.get_vocab_size() print("Vocab size is: {0}".format(vocab_size)) FLAGS.vocab_size = vocab_size last_test_loss = float('inf') with tf.Session() as sess: model = create_model(sess, path, vocab_size) #train model and save to checkpoint print("Beggining training...") print("Maximum number of epochs to train for: {0}".format( FLAGS.max_epoch)) print("Batch size: {0}".format(FLAGS.batch_size)) print("Starting learning rate: {0}".format(FLAGS.learning_rate)) print("Learning rate decay factor: {0}".format(FLAGS.lr_decay_factor)) source_train_file_path = data_processor.data_source_train target_train_file_path = data_processor.data_target_train source_test_file_path = data_processor.data_source_test target_test_file_path = data_processor.data_target_test print(source_train_file_path) print(target_train_file_path) train_set = read_data(source_train_file_path, target_train_file_path, FLAGS.max_train_data_size) random.shuffle(train_set) test_set = read_data(source_test_file_path, target_test_file_path, FLAGS.max_train_data_size) random.shuffle(test_set) step_time, train_loss = 0.0, 0.0 current_step = 0 previous_losses = [] num_batches = len(train_set) / FLAGS.batch_size batch_pointer = 0 while True: # Get a batch and make a step. start_time = time.time() start_index = int(batch_pointer * FLAGS.batch_size) end_index = int(start_index + FLAGS.batch_size) inputs, targets, input_lengths, target_lengths =\ model.get_batch(train_set[start_index : end_index]) step_loss = model.step(sess, inputs, targets, input_lengths, target_lengths) batch_pointer = (batch_pointer + 1) % num_batches step_time += (time.time() - start_time) / FLAGS.steps_per_checkpoint train_loss += step_loss / FLAGS.steps_per_checkpoint current_step += 1 # Once in a while, we save checkpoint, show statistics, and run tests. if current_step % FLAGS.steps_per_checkpoint == 0: # show statistics for the previous epoch. print("Step {0} learning rate {1} step-time {2} training loss {3}"\ .format(model.global_step.eval(), round(model.learning_rate,4), round(step_time, 4), round(train_loss,4))) # Decrease learning rate if no improvement was seen over last 3 times. #if len(previous_losses) > 2 and loss > max(previous_losses[-3:]): # sess.run(model.learning_rate_decay_op) previous_losses.append(train_loss) # Run tests on test set and show their perplexity. test_losses = [] num_test_batches = int(len(test_set) / FLAGS.batch_size) for test_pointer in range(0, num_test_batches): start_index = test_pointer * FLAGS.batch_size inputs, targets, input_lengths, target_lengths =\ model.get_batch(test_set[start_index : start_index + FLAGS.batch_size]) test_loss = model.step(sess, inputs, targets, input_lengths, target_lengths, test_mode=True) test_losses.append(test_loss) test_loss = float(np.mean(test_losses)) print(" step: {0} test loss: {1}".format( model.global_step.eval(), round(test_loss, 4))) # Save checkpoint and zero timer and loss. if test_loss < last_test_loss: checkpoint_path = os.path.join(path, "chatbot") model.saver.save(sess, checkpoint_path, global_step=model.global_step) last_test_loss = test_loss step_time, train_loss = 0.0, 0.0 sys.stdout.flush()
def main(): config.read(FLAGS.config_file) max_num_lines = int(config.get("max_data_sizes", "num_lines")) max_target_length = int(config.get("max_data_sizes", "max_target_length")) max_source_length = int(config.get("max_data_sizes", "max_source_length")) if not os.path.exists(FLAGS.checkpoint_dir): os.mkdir(FLAGS.checkpoint_dir) path = getCheckpointPath() print "path is {0}".format(path) data_processor = data_utils.DataProcessor(FLAGS.vocab_size, FLAGS.raw_data_dir,FLAGS.data_dir, FLAGS.train_frac, FLAGS.tokenizer, max_num_lines, max_target_length, max_source_length, FLAGS.is_discrete, FLAGS.extra_discrete_data) data_processor.run() #create model print "Creating model with..." print "Number of hidden layers: {0}".format(FLAGS.num_layers) print "Number of units per layer: {0}".format(FLAGS.hidden_size) print "Dropout: {0}".format(FLAGS.dropout) vocab_mapper = vocab_utils.VocabMapper(FLAGS.data_dir) vocab_size = vocab_mapper.getVocabSize() print "Vocab size is: {0}".format(vocab_size) FLAGS.vocab_size = vocab_size with tf.Session() as sess: writer = tf.train.SummaryWriter("/tmp/tb_logs_chatbot", sess.graph) model = createModel(sess, path, vocab_size) print "Using bucket sizes:" print _buckets #train model and save to checkpoint print "Beggining training..." print "Maximum number of epochs to train for: {0}".format(FLAGS.max_epoch) print "Batch size: {0}".format(FLAGS.batch_size) print "Starting learning rate: {0}".format(FLAGS.learning_rate) print "Learning rate decay factor: {0}".format(FLAGS.lr_decay_factor) source_train_file_path = data_processor.data_source_train target_train_file_path = data_processor.data_target_train source_test_file_path = data_processor.data_source_test target_test_file_path = data_processor.data_target_test print source_train_file_path print target_train_file_path train_set = readData(source_train_file_path, target_train_file_path, FLAGS.max_train_data_size) test_set = readData(source_test_file_path, target_test_file_path, FLAGS.max_train_data_size) train_bucket_sizes = [len(train_set[b]) for b in xrange(len(_buckets))] print "bucket sizes = {0}".format(train_bucket_sizes) train_total_size = float(sum(train_bucket_sizes)) train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size for i in xrange(len(train_bucket_sizes))] step_time, loss = 0.0, 0.0 current_step = 0 previous_losses = [] while True: # Choose a bucket according to data distribution. We pick a random number # in [0, 1] and use the corresponding interval in train_buckets_scale. random_number_01 = np.random.random_sample() bucket_id = min([i for i in xrange(len(train_buckets_scale)) if train_buckets_scale[i] > random_number_01]) # Get a batch and make a step. start_time = time.time() encoder_inputs, decoder_inputs, target_weights = model.get_batch( train_set, bucket_id) _, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, bucket_id, False) step_time += (time.time() - start_time) / FLAGS.steps_per_checkpoint loss += step_loss / FLAGS.steps_per_checkpoint current_step += 1 # Once in a while, we save checkpoint, print statistics, and run evals. if current_step % FLAGS.steps_per_checkpoint == 0: train_loss_summary = tf.Summary() str_summary_train_loss = train_loss_summary.value.add() str_summary_train_loss.simple_value = loss str_summary_train_loss.tag = "train_loss" writer.add_summary(train_loss_summary, current_step) # Print statistics for the previous epoch. perplexity = math.exp(loss) if loss < 300 else float('inf') print ("global step %d learning rate %.4f step-time %.2f perplexity " "%.2f" % (model.global_step.eval(), model.learning_rate.eval(), step_time, perplexity)) # Decrease learning rate if no improvement was seen over last 3 times. if len(previous_losses) > 2 and loss > max(previous_losses[-3:]): sess.run(model.learning_rate_decay_op) previous_losses.append(loss) # Save checkpoint and zero timer and loss. checkpoint_path = os.path.join(path, "chatbot.ckpt") model.saver.save(sess, checkpoint_path, global_step=model.global_step) step_time, loss = 0.0, 0.0 # Run evals on development set and print their perplexity. perplexity_summary = tf.Summary() eval_loss_summary = tf.Summary() for bucket_id in xrange(len(_buckets)): if len(test_set[bucket_id]) == 0: print(" eval: empty bucket %d" % (bucket_id)) continue encoder_inputs, decoder_inputs, target_weights = model.get_batch( test_set, bucket_id) _, eval_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, bucket_id, True) eval_ppx = math.exp(eval_loss) if eval_loss < 300 else float('inf') print(" eval: bucket %d perplexity %.2f" % (bucket_id, eval_ppx)) str_summary_ppx = perplexity_summary.value.add() str_summary_ppx.simple_value = eval_ppx str_summary_ppx.tag = "peplexity_bucket)%d" % bucket_id str_summary_eval_loss = eval_loss_summary.value.add() #need to convert from numpy.float32 to float native type str_summary_eval_loss.simple_value = float(eval_loss) str_summary_eval_loss.tag = "eval_loss_bucket)%d" % bucket_id writer.add_summary(perplexity_summary, current_step) writer.add_summary(eval_loss_summary, current_step) sys.stdout.flush()
def main(): hyper_params = checkGetHyperParamDic() print "Using checkpoint {0}".format(FLAGS.checkpoint_dir) print "Using hyper params: {0}".format(hyper_params) data_processor = dataprocessor.DataProcessor(FLAGS.data_dir, FLAGS.raw_data_dir, hyper_params) text_audio_pairs = data_processor.run() num_train = int(floor(hyper_params["train_frac"] * len(text_audio_pairs))) train_set = text_audio_pairs[:num_train] test_set = text_audio_pairs[num_train:] print "Using {0} size of test set".format(len(test_set)) #setting up piplines to be able to load data async (one for test set, one for train) #TODO tensorflow probably has something built in for this, look into it parent_train_conn, child_train_conn = Pipe() parent_test_conn, child_test_conn = Pipe() with tf.Session() as sess: #create model print "Building model... (this takes a while)" model = createAcousticModel(sess, hyper_params) print "Setting up audio processor..." model.initializeAudioProcessor(hyper_params["max_input_seq_length"]) print "Setting up piplines to test and train data..." model.setConnections(child_test_conn, child_train_conn) num_test_batches = model.getNumBatches(test_set) num_train_batches = model.getNumBatches(train_set) train_batch_pointer = 0 test_batch_pointer = 0 async_train_loader = Process(target=model.getBatch, args=(train_set, train_batch_pointer, True)) async_train_loader.start() step_time, loss = 0.0, 0.0 current_step = 0 previous_losses = [] while True: #begin timer start_time = time.time() #receive batch from pipe step_batch_inputs = parent_train_conn.recv() #async_train_loader.join() train_batch_pointer = step_batch_inputs[5] % num_train_batches #begin fetching other batch while graph processes previous one async_train_loader = Process(target=model.getBatch, args=(train_set, train_batch_pointer, True)) async_train_loader.start() #print step_batch_inputs[0] _, step_loss = model.step(sess, step_batch_inputs[0], step_batch_inputs[1], step_batch_inputs[2], step_batch_inputs[3], step_batch_inputs[4], forward_only=False) #print _ print "Step {0} with loss {1}".format(current_step, step_loss) step_time += (time.time() - start_time) / hyper_params["steps_per_checkpoint"] loss += step_loss / hyper_params["steps_per_checkpoint"] current_step += 1 if current_step % hyper_params["steps_per_checkpoint"] == 0: print( "global step %d learning rate %.4f step-time %.2f loss " "%.2f" % (model.global_step.eval(), model.learning_rate.eval(), step_time, loss)) # Decrease learning rate if no improvement was seen over last 3 times. if len(previous_losses) > 2 and loss > max( previous_losses[-3:]): sess.run(model.learning_rate_decay_op) previous_losses.append(loss) checkpoint_path = os.path.join(FLAGS.checkpoint_dir, "acousticmodel.ckpt") model.saver.save(sess, checkpoint_path, global_step=model.global_step) step_time, loss = 0.0, 0.0 #begin loading test data async #(uses different pipline than train data) async_test_loader = Process(target=model.getBatch, args=(test_set, test_batch_pointer, False)) async_test_loader.start() for i in range(num_test_batches): print "On {0}th training iteration".format(i) eval_inputs = parent_test_conn.recv() #async_test_loader.join() test_batch_pointer = eval_inputs[5] % num_test_batches #tell audio processor to go get another batch ready #while we run last one through the graph if i != num_test_batches - 1: async_test_loader = Process(target=model.getBatch, args=(test_set, test_batch_pointer, False)) _, loss = model.step(sess, eval_inputs[0], eval_inputs[1], eval_inputs[2], eval_inputs[3], eval_inputs[4], forward_only=True) print("\tTest: loss %.2f" % (loss)) sys.stdout.flush()