def load_training_dataset(hyper_params):
# Load the train set data
data_processor = dataprocessor.DataProcessor(hyper_params["training_dataset_dirs"],
file_cache=hyper_params["training_filelist_cache"])
if hyper_params["dataset_size_ordering"] in ['True', 'First_run_only']:
train_set = data_processor.get_ordered_dataset()
else:
train_set = data_processor.get_dataset()
shuffle(train_set)
if hyper_params["test_dataset_dirs"] is not None:
# Load the test set data
data_processor = dataprocessor.DataProcessor(hyper_params["test_dataset_dirs"])
test_set = data_processor.get_dataset()
elif hyper_params["train_frac"] is not None:
# Or use a fraction of the train set for the test set
num_train = max(1, int(floor(hyper_params["train_frac"] * len(train_set))))
test_set = train_set[num_train:]
train_set = train_set[:num_train]
else:
# Or use no test set
test_set = []
logging.info("Using %d files in train set", len(train_set))
logging.info("Using %d size of test set", len(test_set))
return train_set, test_set
def train_rnn(audio_processor, hyper_params, prog_params):
# Load the train set data
data_processor = dataprocessor.DataProcessor(
hyper_params["training_dataset_dir"],
hyper_params["training_dataset_type"], audio_processor)
train_set = data_processor.run()
if (hyper_params["test_dataset_dir"]
is not None) and (hyper_params["test_dataset_type"] is not None):
# Load the test set data
data_processor = dataprocessor.DataProcessor(
hyper_params["test_dataset_dir"],
hyper_params["test_dataset_type"], audio_processor)
test_set = data_processor.run()
elif hyper_params["train_frac"] is not None:
# Or use a fraction of the train set for the test set
num_train = max(
1, int(floor(hyper_params["train_frac"] * len(train_set))))
test_set = train_set[num_train:]
train_set = train_set[:num_train]
else:
# Or use no test set
test_set = []
if hyper_params["pre_filter_file_size"] is True:
test_set = data_processor.filterDataset(
test_set, hyper_params["max_input_seq_length"],
hyper_params["max_target_seq_length"])
train_set = data_processor.filterDataset(
train_set, hyper_params["max_input_seq_length"],
hyper_params["max_target_seq_length"])
print("Using {0} files in train set".format(len(train_set)))
print("Using {0} size of test set".format(len(test_set)))
with tf.Session() as sess:
# create model
print("Building model... (this takes a while)")
model = createAcousticModel(
sess,
hyper_params,
hyper_params["batch_size"],
forward_only=False,
tensorboard_dir=hyper_params["tensorboard_dir"],
tb_run_name=prog_params["tb_name"])
# Override the learning rate if given on the command line
if prog_params["learn_rate"] is not None:
assign_op = model.learning_rate.assign(prog_params["learn_rate"])
sess.run(assign_op)
print("Setting up audio processor...")
model.initializeAudioProcessor(hyper_params["max_input_seq_length"],
hyper_params["load_save_input_vec"])
print("Start training...")
model.train(sess,
test_set,
train_set,
hyper_params["steps_per_checkpoint"],
hyper_params["checkpoint_dir"],
hyper_params["async_get_batch"],
max_epoch=prog_params["max_epoch"])
def train_rnn(audio_processor, hyper_params, prog_params):
# Load the train set data
data_processor = dataprocessor.DataProcessor(
hyper_params["training_dataset_dirs"],
audio_processor,
size_ordering=hyper_params["dataset_size_ordering"])
train_set = data_processor.run()
if hyper_params["test_dataset_dirs"] is not None:
# Load the test set data
data_processor = dataprocessor.DataProcessor(
hyper_params["test_dataset_dirs"],
audio_processor,
size_ordering=hyper_params["dataset_size_ordering"])
test_set = data_processor.run()
elif hyper_params["train_frac"] is not None:
# Or use a fraction of the train set for the test set
num_train = max(
1, int(floor(hyper_params["train_frac"] * len(train_set))))
test_set = train_set[num_train:]
train_set = train_set[:num_train]
else:
# Or use no test set
test_set = []
logging.info("Using %d files in train set", len(train_set))
logging.info("Using %d size of test set", len(test_set))
with tf.Session() as sess:
# create model
model = create_acoustic_model(
sess,
hyper_params,
hyper_params["batch_size"],
forward_only=False,
tensorboard_dir=hyper_params["tensorboard_dir"],
tb_run_name=prog_params["tb_name"],
timeline_enabled=prog_params["timeline"])
# Override the learning rate if given on the command line
if prog_params["learn_rate"] is not None:
assign_op = model.learning_rate.assign(prog_params["learn_rate"])
sess.run(assign_op)
logging.info("Start training...")
model.train(sess,
audio_processor,
test_set,
train_set,
hyper_params["steps_per_checkpoint"],
hyper_params["checkpoint_dir"],
max_epoch=prog_params["max_epoch"])
def test_get_data_shtooka(self):
data_processor = dataprocessor.DataProcessor(self.directory +
"Shtooka")
test_set = data_processor.get_dataset()
self.assertCountEqual(test_set, [[
self.directory + "Shtooka/flac/eng - I_arose.flac", "i arose", None
], [self.directory + "Shtooka/flac/eng - I_ate.flac", "i ate", None]])
def test_get_data_shtooka(self):
data_processor = dataprocessor.DataProcessor(self.directory + "Shtooka", self.audio_processor)
test_set = data_processor.run()
self.assertCountEqual(test_set,
[[self.directory + "Shtooka/flac/eng - I_arose.flac", "i arose", 0],
[self.directory + "Shtooka/flac/eng - I_ate.flac", "i ate", 0]
])
def evaluate(hyper_params):
if hyper_params["test_dataset_dirs"] is None:
logging.fatal("Setting test_dataset_dirs in config file is mandatory for evaluation mode")
return
# Load the test set data
data_processor = dataprocessor.DataProcessor(hyper_params["test_dataset_dirs"])
test_set = data_processor.get_dataset()
logging.info("Using %d size of test set", len(test_set))
if len(test_set) == 0:
logging.fatal("No files in test set during an evaluation mode")
return
with tf.Session() as sess:
# create model
model = AcousticModel(hyper_params["num_layers"], hyper_params["hidden_size"], hyper_params["batch_size"],
hyper_params["max_input_seq_length"], hyper_params["max_target_seq_length"],
hyper_params["input_dim"], hyper_params["batch_normalization"],
language=hyper_params["language"])
model.create_forward_rnn()
model.initialize(sess)
model.restore(sess, hyper_params["checkpoint_dir"])
wer, cer = model.evaluate_full(sess, test_set, hyper_params["max_input_seq_length"],
hyper_params["signal_processing"])
print("Resulting WER : {0:.3g} %".format(wer))
print("Resulting CER : {0:.3g} %".format(cer))
return
def test_get_data_vystadial_2013(self):
data_processor = dataprocessor.DataProcessor(self.directory + "Vystadial_2013")
test_set = data_processor.get_dataset()
self.assertCountEqual(test_set,
[[self.directory +
"Vystadial_2013/data_voip_en/dev/jurcic-028-121024_234433_0013625_0013836.wav",
"alright thank you and goodbye", None]
])
def load_acoustic_dataset(training_dataset_dirs,
test_dataset_dirs=None,
training_filelist_cache=None,
ordered=False,
train_frac=None):
"""
Load the datatsets for the acoustic model training
Return a train set and an optional test set, each containing a list of [audio_file, label, audio_length]
Parameters
----------
:param training_dataset_dirs: directory where to find the training data
:param test_dataset_dirs: directory where to find the test data (optional)
:param training_filelist_cache: path to the cache file for the training data (optional)
:param ordered: boolean indicating whether or not to order the dataset by audio files length (ascending)
:param train_frac: the fraction of the training data to be used as test data
(only used if test_dataset_dirs is None)
:return train_set, test_set: two lists of [audio_file, label, audio_length] where
audio_file is the path to an audio file
label is the true label for the audio file (relative to the char_map)
audio_length if the length of the audio file
"""
data_processor = dataprocessor.DataProcessor(
training_dataset_dirs, file_cache=training_filelist_cache)
train_set = data_processor.get_dataset()
if ordered:
train_set = sorted(train_set, key=lambda x: x[2])
else:
shuffle(train_set)
if test_dataset_dirs is not None:
# Load the test set data
data_processor = dataprocessor.DataProcessor(test_dataset_dirs)
test_set = data_processor.get_dataset()
elif train_frac is not None:
# Or use a fraction of the train set for the test set
num_train = max(1, int(floor(train_frac * len(train_set))))
test_set = train_set[num_train:]
train_set = train_set[:num_train]
else:
# Or use no test set
test_set = []
logging.info("Using %d files in train set", len(train_set))
logging.info("Using %d size of test set", len(test_set))
return train_set, test_set
def test_get_data_librispeech(self):
data_processor = dataprocessor.DataProcessor(self.directory + "Libri")
test_set = data_processor.get_dataset()
self.assertCountEqual(test_set,
[[self.directory + "Libri/train-clean-100/19/198/19-198-0000.flac",
"northanger abbey", None],
[self.directory + "Libri/train-clean-100/19/198/19-198-0001.flac",
"this little work", None]
])
def main():
dirs = [
os.path.join(FLAGS.data_dir, "train/"),
os.path.join(FLAGS.data_dir, "test/")
]
if not (os.path.exists(dirs[0]) and os.path.exists(dirs[1])):
print "Train/Test files not detected, creating now..."
config = ConfigParser.ConfigParser()
config.read(FLAGS.config_file)
max_num_lines = int(config.get("max_data_sizes", "num_lines"))
max_target_size = int(config.get("max_data_sizes",
"max_target_length"))
max_source_size = int(config.get("max_data_sizes",
"max_source_length"))
data_processor = data_utils.DataProcessor(
FLAGS.vocab_size, FLAGS.raw_data_dir, FLAGS.data_dir,
FLAGS.train_frac, FLAGS.tokenizer, max_num_lines, max_target_size,
max_source_size)
data_processor.run()
assert FLAGS.plot_histograms or FLAGS.plot_scatter, "Must choose at least one plot!"
source_lengths = []
target_lengths = []
count = 0
for i in range(len(dirs)):
if "test" in dirs[i]:
source_path = os.path.join(dirs[i], "data_source_test.txt")
target_path = os.path.join(dirs[i], "data_target_test.txt")
else:
source_path = os.path.join(dirs[i], "data_source_train.txt")
target_path = os.path.join(dirs[i], "data_target_train.txt")
with tf.gfile.GFile(source_path, mode="r") as source_file:
with tf.gfile.GFile(target_path, mode="r") as target_file:
source, target = source_file.readline(), target_file.readline()
counter = 0
while source and target:
counter += 1
if counter % 100000 == 0:
print(" reading data line %d" % counter)
sys.stdout.flush()
num_source_ids = len(source.split())
source_lengths.append(num_source_ids)
#plus 1 for EOS token
num_target_ids = len(target.split()) + 1
target_lengths.append(num_target_ids)
source, target = source_file.readline(
), target_file.readline()
if FLAGS.plot_histograms:
plot_histo_lengths("target lengths", target_lengths)
plot_histo_lengths("source_lengths", source_lengths)
if FLAGS.plot_scatter:
plot_scatter_lengths("target vs source length", "source length",
"target length", source_lengths, target_lengths)
def evaluate(audio_processor, hyper_params):
if hyper_params["test_dataset_dirs"] is None:
logging.fatal(
"Setting test_dataset_dirs in config file is mandatory for evaluation mode"
)
return
# Load the test set data
data_processor = dataprocessor.DataProcessor(
hyper_params["test_dataset_dirs"],
audio_processor,
size_ordering=hyper_params["dataset_size_ordering"])
test_set = data_processor.run()
logging.info("Using %d size of test set", len(test_set))
if len(test_set) == 0:
logging.fatal("No files in test set during an evaluation mode")
return
with tf.Session() as sess:
# create model
model = create_acoustic_model(sess,
hyper_params,
hyper_params["batch_size"],
forward_only=True,
tensorboard_dir=None,
tb_run_name=None,
timeline_enabled=False)
wer_list = []
cer_list = []
file_number = 0
input_feat_vecs = []
input_feat_vec_lengths = []
labels = []
for file, label, _ in test_set:
feat_vec, feat_vec_length = audio_processor.process_audio_file(
file)
file_number += 1
label_data_length = len(label)
if (label_data_length > hyper_params["max_target_seq_length"]) or\
(feat_vec_length > hyper_params["max_input_seq_length"]):
logging.warning(
"Warning - sample too long : %s (input : %d / text : %s)",
file, feat_vec_length, label_data_length)
else:
logging.debug("Processed file %d / %d", file_number,
len(test_set))
input_feat_vecs.append(feat_vec)
input_feat_vec_lengths.append(feat_vec_length)
labels.append(label)
# If we reached the last file then pad the lists to obtain a full batch
if file_number == len(test_set):
for i in range(hyper_params["batch_size"] -
len(input_feat_vecs)):
input_feat_vecs.append(
np.zeros([
hyper_params["max_input_seq_length"],
audio_processor.feature_size
]))
input_feat_vec_lengths.append(0)
labels.append("")
if len(input_feat_vecs) == hyper_params["batch_size"]:
# Run the batch
logging.debug("Running a batch")
input_feat_vecs = np.swapaxes(input_feat_vecs, 0, 1)
transcribed_texts = model.process_input(
sess, input_feat_vecs, input_feat_vec_lengths)
for index, transcribed_text in enumerate(transcribed_texts):
true_label = labels[index]
if len(true_label) > 0:
nb_words = len(true_label.split())
nb_chars = len(true_label.replace(" ", ""))
wer_list.append(
model.calculate_wer(transcribed_text, true_label) /
float(nb_words))
cer_list.append(
model.calculate_cer(transcribed_text, true_label) /
float(nb_chars))
# Reset the lists
input_feat_vecs = []
input_feat_vec_lengths = []
labels = []
print("Resulting WER : {0:.3g} %".format(
(sum(wer_list) * 100) / float(len(wer_list))))
print("Resulting CER : {0:.3g} %".format(
(sum(cer_list) * 100) / float(len(cer_list))))
return
def main():
if not os.path.exists(FLAGS.checkpoint_dir):
os.mkdir(FLAGS.checkpoint_dir)
path = get_checkpoint_path()
print("path is {0}".format(path))
data_processor = data_utils.DataProcessor(
FLAGS.vocab_size, FLAGS.raw_data_dir, FLAGS.data_dir, FLAGS.train_frac,
FLAGS.tokenizer, FLAGS.convo_limits, FLAGS.max_target_length,
FLAGS.max_source_length)
data_processor.run()
#create model
print("Creating model with...")
print("Number of hidden layers: {0}".format(FLAGS.num_layers))
print("Number of units per layer: {0}".format(FLAGS.hidden_size))
print("Dropout: {0}".format(FLAGS.dropout))
vocab_mapper = vocab_utils.VocabMapper(FLAGS.data_dir)
vocab_size = vocab_mapper.get_vocab_size()
print("Vocab size is: {0}".format(vocab_size))
FLAGS.vocab_size = vocab_size
last_test_loss = float('inf')
with tf.Session() as sess:
model = create_model(sess, path, vocab_size)
#train model and save to checkpoint
print("Beggining training...")
print("Maximum number of epochs to train for: {0}".format(
FLAGS.max_epoch))
print("Batch size: {0}".format(FLAGS.batch_size))
print("Starting learning rate: {0}".format(FLAGS.learning_rate))
print("Learning rate decay factor: {0}".format(FLAGS.lr_decay_factor))
source_train_file_path = data_processor.data_source_train
target_train_file_path = data_processor.data_target_train
source_test_file_path = data_processor.data_source_test
target_test_file_path = data_processor.data_target_test
print(source_train_file_path)
print(target_train_file_path)
train_set = read_data(source_train_file_path, target_train_file_path,
FLAGS.max_train_data_size)
random.shuffle(train_set)
test_set = read_data(source_test_file_path, target_test_file_path,
FLAGS.max_train_data_size)
random.shuffle(test_set)
step_time, train_loss = 0.0, 0.0
current_step = 0
previous_losses = []
num_batches = len(train_set) / FLAGS.batch_size
batch_pointer = 0
while True:
# Get a batch and make a step.
start_time = time.time()
start_index = int(batch_pointer * FLAGS.batch_size)
end_index = int(start_index + FLAGS.batch_size)
inputs, targets, input_lengths, target_lengths =\
model.get_batch(train_set[start_index : end_index])
step_loss = model.step(sess, inputs, targets, input_lengths,
target_lengths)
batch_pointer = (batch_pointer + 1) % num_batches
step_time += (time.time() -
start_time) / FLAGS.steps_per_checkpoint
train_loss += step_loss / FLAGS.steps_per_checkpoint
current_step += 1
# Once in a while, we save checkpoint, show statistics, and run tests.
if current_step % FLAGS.steps_per_checkpoint == 0:
# show statistics for the previous epoch.
print("Step {0} learning rate {1} step-time {2} training loss {3}"\
.format(model.global_step.eval(), round(model.learning_rate,4),
round(step_time, 4), round(train_loss,4)))
# Decrease learning rate if no improvement was seen over last 3 times.
#if len(previous_losses) > 2 and loss > max(previous_losses[-3:]):
# sess.run(model.learning_rate_decay_op)
previous_losses.append(train_loss)
# Run tests on test set and show their perplexity.
test_losses = []
num_test_batches = int(len(test_set) / FLAGS.batch_size)
for test_pointer in range(0, num_test_batches):
start_index = test_pointer * FLAGS.batch_size
inputs, targets, input_lengths, target_lengths =\
model.get_batch(test_set[start_index : start_index + FLAGS.batch_size])
test_loss = model.step(sess,
inputs,
targets,
input_lengths,
target_lengths,
test_mode=True)
test_losses.append(test_loss)
test_loss = float(np.mean(test_losses))
print(" step: {0} test loss: {1}".format(
model.global_step.eval(), round(test_loss, 4)))
# Save checkpoint and zero timer and loss.
if test_loss < last_test_loss:
checkpoint_path = os.path.join(path, "chatbot")
model.saver.save(sess,
checkpoint_path,
global_step=model.global_step)
last_test_loss = test_loss
step_time, train_loss = 0.0, 0.0
sys.stdout.flush()
def main():
config.read(FLAGS.config_file)
max_num_lines = int(config.get("max_data_sizes", "num_lines"))
max_target_length = int(config.get("max_data_sizes", "max_target_length"))
max_source_length = int(config.get("max_data_sizes", "max_source_length"))
if not os.path.exists(FLAGS.checkpoint_dir):
os.mkdir(FLAGS.checkpoint_dir)
path = getCheckpointPath()
print "path is {0}".format(path)
data_processor = data_utils.DataProcessor(FLAGS.vocab_size,
FLAGS.raw_data_dir,FLAGS.data_dir, FLAGS.train_frac, FLAGS.tokenizer,
max_num_lines, max_target_length, max_source_length, FLAGS.is_discrete,
FLAGS.extra_discrete_data)
data_processor.run()
#create model
print "Creating model with..."
print "Number of hidden layers: {0}".format(FLAGS.num_layers)
print "Number of units per layer: {0}".format(FLAGS.hidden_size)
print "Dropout: {0}".format(FLAGS.dropout)
vocab_mapper = vocab_utils.VocabMapper(FLAGS.data_dir)
vocab_size = vocab_mapper.getVocabSize()
print "Vocab size is: {0}".format(vocab_size)
FLAGS.vocab_size = vocab_size
with tf.Session() as sess:
writer = tf.train.SummaryWriter("/tmp/tb_logs_chatbot", sess.graph)
model = createModel(sess, path, vocab_size)
print "Using bucket sizes:"
print _buckets
#train model and save to checkpoint
print "Beggining training..."
print "Maximum number of epochs to train for: {0}".format(FLAGS.max_epoch)
print "Batch size: {0}".format(FLAGS.batch_size)
print "Starting learning rate: {0}".format(FLAGS.learning_rate)
print "Learning rate decay factor: {0}".format(FLAGS.lr_decay_factor)
source_train_file_path = data_processor.data_source_train
target_train_file_path = data_processor.data_target_train
source_test_file_path = data_processor.data_source_test
target_test_file_path = data_processor.data_target_test
print source_train_file_path
print target_train_file_path
train_set = readData(source_train_file_path, target_train_file_path,
FLAGS.max_train_data_size)
test_set = readData(source_test_file_path, target_test_file_path,
FLAGS.max_train_data_size)
train_bucket_sizes = [len(train_set[b]) for b in xrange(len(_buckets))]
print "bucket sizes = {0}".format(train_bucket_sizes)
train_total_size = float(sum(train_bucket_sizes))
train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size
for i in xrange(len(train_bucket_sizes))]
step_time, loss = 0.0, 0.0
current_step = 0
previous_losses = []
while True:
# Choose a bucket according to data distribution. We pick a random number
# in [0, 1] and use the corresponding interval in train_buckets_scale.
random_number_01 = np.random.random_sample()
bucket_id = min([i for i in xrange(len(train_buckets_scale))
if train_buckets_scale[i] > random_number_01])
# Get a batch and make a step.
start_time = time.time()
encoder_inputs, decoder_inputs, target_weights = model.get_batch(
train_set, bucket_id)
_, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs,
target_weights, bucket_id, False)
step_time += (time.time() - start_time) / FLAGS.steps_per_checkpoint
loss += step_loss / FLAGS.steps_per_checkpoint
current_step += 1
# Once in a while, we save checkpoint, print statistics, and run evals.
if current_step % FLAGS.steps_per_checkpoint == 0:
train_loss_summary = tf.Summary()
str_summary_train_loss = train_loss_summary.value.add()
str_summary_train_loss.simple_value = loss
str_summary_train_loss.tag = "train_loss"
writer.add_summary(train_loss_summary, current_step)
# Print statistics for the previous epoch.
perplexity = math.exp(loss) if loss < 300 else float('inf')
print ("global step %d learning rate %.4f step-time %.2f perplexity "
"%.2f" % (model.global_step.eval(), model.learning_rate.eval(),
step_time, perplexity))
# Decrease learning rate if no improvement was seen over last 3 times.
if len(previous_losses) > 2 and loss > max(previous_losses[-3:]):
sess.run(model.learning_rate_decay_op)
previous_losses.append(loss)
# Save checkpoint and zero timer and loss.
checkpoint_path = os.path.join(path, "chatbot.ckpt")
model.saver.save(sess, checkpoint_path, global_step=model.global_step)
step_time, loss = 0.0, 0.0
# Run evals on development set and print their perplexity.
perplexity_summary = tf.Summary()
eval_loss_summary = tf.Summary()
for bucket_id in xrange(len(_buckets)):
if len(test_set[bucket_id]) == 0:
print(" eval: empty bucket %d" % (bucket_id))
continue
encoder_inputs, decoder_inputs, target_weights = model.get_batch(
test_set, bucket_id)
_, eval_loss, _ = model.step(sess, encoder_inputs, decoder_inputs,
target_weights, bucket_id, True)
eval_ppx = math.exp(eval_loss) if eval_loss < 300 else float('inf')
print(" eval: bucket %d perplexity %.2f" % (bucket_id, eval_ppx))
str_summary_ppx = perplexity_summary.value.add()
str_summary_ppx.simple_value = eval_ppx
str_summary_ppx.tag = "peplexity_bucket)%d" % bucket_id
str_summary_eval_loss = eval_loss_summary.value.add()
#need to convert from numpy.float32 to float native type
str_summary_eval_loss.simple_value = float(eval_loss)
str_summary_eval_loss.tag = "eval_loss_bucket)%d" % bucket_id
writer.add_summary(perplexity_summary, current_step)
writer.add_summary(eval_loss_summary, current_step)
sys.stdout.flush()
def main():
hyper_params = checkGetHyperParamDic()
print "Using checkpoint {0}".format(FLAGS.checkpoint_dir)
print "Using hyper params: {0}".format(hyper_params)
data_processor = dataprocessor.DataProcessor(FLAGS.data_dir,
FLAGS.raw_data_dir,
hyper_params)
text_audio_pairs = data_processor.run()
num_train = int(floor(hyper_params["train_frac"] * len(text_audio_pairs)))
train_set = text_audio_pairs[:num_train]
test_set = text_audio_pairs[num_train:]
print "Using {0} size of test set".format(len(test_set))
#setting up piplines to be able to load data async (one for test set, one for train)
#TODO tensorflow probably has something built in for this, look into it
parent_train_conn, child_train_conn = Pipe()
parent_test_conn, child_test_conn = Pipe()
with tf.Session() as sess:
#create model
print "Building model... (this takes a while)"
model = createAcousticModel(sess, hyper_params)
print "Setting up audio processor..."
model.initializeAudioProcessor(hyper_params["max_input_seq_length"])
print "Setting up piplines to test and train data..."
model.setConnections(child_test_conn, child_train_conn)
num_test_batches = model.getNumBatches(test_set)
num_train_batches = model.getNumBatches(train_set)
train_batch_pointer = 0
test_batch_pointer = 0
async_train_loader = Process(target=model.getBatch,
args=(train_set, train_batch_pointer,
True))
async_train_loader.start()
step_time, loss = 0.0, 0.0
current_step = 0
previous_losses = []
while True:
#begin timer
start_time = time.time()
#receive batch from pipe
step_batch_inputs = parent_train_conn.recv()
#async_train_loader.join()
train_batch_pointer = step_batch_inputs[5] % num_train_batches
#begin fetching other batch while graph processes previous one
async_train_loader = Process(target=model.getBatch,
args=(train_set, train_batch_pointer,
True))
async_train_loader.start()
#print step_batch_inputs[0]
_, step_loss = model.step(sess,
step_batch_inputs[0],
step_batch_inputs[1],
step_batch_inputs[2],
step_batch_inputs[3],
step_batch_inputs[4],
forward_only=False)
#print _
print "Step {0} with loss {1}".format(current_step, step_loss)
step_time += (time.time() -
start_time) / hyper_params["steps_per_checkpoint"]
loss += step_loss / hyper_params["steps_per_checkpoint"]
current_step += 1
if current_step % hyper_params["steps_per_checkpoint"] == 0:
print(
"global step %d learning rate %.4f step-time %.2f loss "
"%.2f" % (model.global_step.eval(),
model.learning_rate.eval(), step_time, loss))
# Decrease learning rate if no improvement was seen over last 3 times.
if len(previous_losses) > 2 and loss > max(
previous_losses[-3:]):
sess.run(model.learning_rate_decay_op)
previous_losses.append(loss)
checkpoint_path = os.path.join(FLAGS.checkpoint_dir,
"acousticmodel.ckpt")
model.saver.save(sess,
checkpoint_path,
global_step=model.global_step)
step_time, loss = 0.0, 0.0
#begin loading test data async
#(uses different pipline than train data)
async_test_loader = Process(target=model.getBatch,
args=(test_set, test_batch_pointer,
False))
async_test_loader.start()
for i in range(num_test_batches):
print "On {0}th training iteration".format(i)
eval_inputs = parent_test_conn.recv()
#async_test_loader.join()
test_batch_pointer = eval_inputs[5] % num_test_batches
#tell audio processor to go get another batch ready
#while we run last one through the graph
if i != num_test_batches - 1:
async_test_loader = Process(target=model.getBatch,
args=(test_set,
test_batch_pointer,
False))
_, loss = model.step(sess,
eval_inputs[0],
eval_inputs[1],
eval_inputs[2],
eval_inputs[3],
eval_inputs[4],
forward_only=True)
print("\tTest: loss %.2f" % (loss))
sys.stdout.flush()