def main(_):
# Set the verbosity based on flags (default is INFO, so we see all messages)
tf.logging.set_verbosity(FLAGS.verbosity)
# Start a new TensorFlow session.
sess = tf.InteractiveSession()
summaries_dir = os.path.join(FLAGS.train_dir, 'summaries')
# Begin by making sure we have the training data we need. If you already have
# training data of your own, use `--data_url= ` on the command line to avoid
# downloading.
model_settings = models.prepare_model_settings(
len(input_data.prepare_words_list(FLAGS.wanted_words.split(','))),
FLAGS.sample_rate, FLAGS.clip_duration_ms, FLAGS.window_size_ms,
FLAGS.window_stride_ms, FLAGS.feature_bin_count, FLAGS.preprocess)
audio_processor = input_data.AudioProcessor(None, FLAGS.data_dir,
FLAGS.silence_percentage,
FLAGS.unknown_percentage,
FLAGS.wanted_words.split(','),
FLAGS.validation_percentage,
FLAGS.testing_percentage,
model_settings, summaries_dir)
with open('class_names.txt', 'w') as fp:
class_names = FLAGS.wanted_words.split(',')
class_names.insert(0, 'silence')
class_names.insert(1, 'background')
fp.writelines('\n'.join(class_names))
fingerprint_size = model_settings['fingerprint_size']
label_count = model_settings['label_count']
time_shift_samples = int((FLAGS.time_shift_ms * FLAGS.sample_rate) / 1000)
# Figure out the learning rates for each training phase. Since it's often
# effective to have high learning rates at the start of training, followed by
# lower levels towards the end, the number of steps and learning rates can be
# specified as comma-separated lists to define the rate at each stage. For
# example --how_many_training_steps=10000,3000 --learning_rate=0.001,0.0001
# will run 13,000 training loops in total, with a rate of 0.001 for the first
# 10,000, and 0.0001 for the final 3,000.
training_steps_list = list(
map(int, FLAGS.how_many_training_steps.split(',')))
learning_rates_list = list(map(float, FLAGS.learning_rate.split(',')))
if len(training_steps_list) != len(learning_rates_list):
raise Exception(
'--how_many_training_steps and --learning_rate must be equal length '
'lists, but are %d and %d long instead' %
(len(training_steps_list), len(learning_rates_list)))
input_placeholder = tf.placeholder(tf.float32, [None, fingerprint_size],
name='fingerprint_input')
if FLAGS.quantize:
fingerprint_min, fingerprint_max = input_data.get_features_range(
model_settings)
fingerprint_input = tf.quantization.fake_quant_with_min_max_args(
input_placeholder, fingerprint_min, fingerprint_max)
else:
fingerprint_input = input_placeholder
logits = models.create_model(
fingerprint_input,
model_settings,
FLAGS.model_architecture,
is_training=True,
)
# Define loss and optimizer
ground_truth_input = tf.placeholder(tf.int64, [None],
name='groundtruth_input')
# Optionally we can add runtime checks to spot when NaNs or other symptoms of
# numerical errors start occurring during training.
control_dependencies = []
if FLAGS.check_nans:
checks = tf.add_check_numerics_ops()
control_dependencies = [checks]
# Create the back propagation and training evaluation machinery in the graph.
with tf.name_scope('cross_entropy'):
cross_entropy_mean = tf.losses.sparse_softmax_cross_entropy(
labels=ground_truth_input, logits=logits)
if FLAGS.quantize:
tf.contrib.quantize.create_training_graph(quant_delay=0)
with tf.name_scope('train'), tf.control_dependencies(control_dependencies):
learning_rate_input = tf.placeholder(tf.float32, [],
name='learning_rate_input')
train_step = tf.train.GradientDescentOptimizer(
learning_rate_input).minimize(cross_entropy_mean)
predicted_indices = tf.argmax(input=logits, axis=1)
correct_prediction = tf.equal(predicted_indices, ground_truth_input)
confusion_matrix = tf.math.confusion_matrix(labels=ground_truth_input,
predictions=predicted_indices,
num_classes=label_count)
evaluation_step = tf.reduce_mean(
input_tensor=tf.cast(correct_prediction, tf.float32))
with tf.get_default_graph().name_scope('eval'):
tf.summary.scalar('cross_entropy', cross_entropy_mean)
tf.summary.scalar('accuracy', evaluation_step)
global_step = tf.train.get_or_create_global_step()
increment_global_step = tf.assign(global_step, tf.math.add(global_step, 1))
if not FLAGS.is_test:
saver = tf.train.Saver(tf.global_variables())
merged_summaries = tf.summary.merge_all(scope='eval')
train_writer = tf.summary.FileWriter(summaries_dir + '/train',
sess.graph)
validation_writer = tf.summary.FileWriter(summaries_dir +
'/validation')
tf.global_variables_initializer().run()
start_step = 1
if FLAGS.start_checkpoint:
models.load_variables_from_checkpoint(sess, FLAGS.start_checkpoint)
start_step = global_step.eval(session=sess)
tf.logging.info('Checkpoint: {}'.format(FLAGS.start_checkpoint))
tf.logging.info('Training from step: {}'.format(start_step))
if not FLAGS.is_test:
# Save graph.pbtxt.
tf.io.write_graph(sess.graph_def, FLAGS.train_dir,
FLAGS.model_architecture + '.pbtxt')
# Save list of words.
with gfile.GFile(
os.path.join(FLAGS.train_dir,
FLAGS.model_architecture + '_labels.txt'),
'w') as f:
f.write('\n'.join(audio_processor.words_list))
# Training loop.
tf.logging.info('Is test flag:\n %s' % (FLAGS.is_test))
if not FLAGS.is_test:
training_steps_max = np.sum(training_steps_list)
for training_step in range(start_step, training_steps_max + 1):
# Figure out what the current learning rate is.
training_steps_sum = 0
for i in range(len(training_steps_list)):
training_steps_sum += training_steps_list[i]
if training_step <= training_steps_sum:
learning_rate_value = learning_rates_list[i]
break
# Pull the audio samples we'll use for training.
train_fingerprints, train_ground_truth = audio_processor.get_data(
FLAGS.batch_size, 0, model_settings,
FLAGS.background_frequency, FLAGS.background_volume,
time_shift_samples, 'training', sess)
# Run the graph with this batch of training data.
train_summary, train_accuracy, cross_entropy_value, _, _ = sess.run(
[
merged_summaries,
evaluation_step,
cross_entropy_mean,
train_step,
increment_global_step,
],
feed_dict={
fingerprint_input: train_fingerprints,
ground_truth_input: train_ground_truth,
learning_rate_input: learning_rate_value
})
train_writer.add_summary(train_summary, training_step)
tf.logging.info(
'Step #%d: rate %f, accuracy %.1f%%, cross entropy %f' %
(training_step, learning_rate_value, train_accuracy * 100,
cross_entropy_value))
is_last_step = (training_step == training_steps_max)
if (training_step % FLAGS.eval_step_interval) == 0 or is_last_step:
set_size = audio_processor.set_size('validation')
total_accuracy = 0
total_conf_matrix = None
for i in range(0, set_size, FLAGS.batch_size):
validation_fingerprints, validation_ground_truth = (
audio_processor.get_data(FLAGS.batch_size, i,
model_settings, 0.0, 0.0, 0,
'validation', sess))
# Run a validation step and capture training summaries for TensorBoard
# with the `merged` op.
validation_summary, validation_accuracy, conf_matrix = sess.run(
[merged_summaries, evaluation_step, confusion_matrix],
feed_dict={
fingerprint_input: validation_fingerprints,
ground_truth_input: validation_ground_truth
})
validation_writer.add_summary(validation_summary,
training_step)
batch_size = min(FLAGS.batch_size, set_size - i)
total_accuracy += (validation_accuracy *
batch_size) / set_size
if total_conf_matrix is None:
total_conf_matrix = conf_matrix
else:
total_conf_matrix += conf_matrix
tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix))
tf.logging.info(
'Step %d: Validation accuracy = %.1f%% (N=%d)' %
(training_step, total_accuracy * 100, set_size))
# Save the Model checkpoint periodically.
if (training_step % FLAGS.save_step_interval == 0
or training_step == training_steps_max):
checkpoint_path = os.path.join(
FLAGS.train_dir, FLAGS.model_architecture + '.ckpt')
tf.logging.info('Saving to "%s-%d"', checkpoint_path,
training_step)
saver.save(sess, checkpoint_path, global_step=training_step)
set_size = audio_processor.set_size('testing')
tf.logging.info('set_size=%d', set_size)
total_accuracy = 0
total_conf_matrix = None
for i in range(0, set_size, FLAGS.batch_size):
test_fingerprints, test_ground_truth = audio_processor.get_data(
FLAGS.batch_size, i, model_settings, 0.0, 0.0, 0, 'testing', sess)
test_accuracy, conf_matrix = sess.run(
[evaluation_step, confusion_matrix],
feed_dict={
fingerprint_input: test_fingerprints,
ground_truth_input: test_ground_truth
})
batch_size = min(FLAGS.batch_size, set_size - i)
total_accuracy += (test_accuracy * batch_size) / set_size
if total_conf_matrix is None:
total_conf_matrix = conf_matrix
else:
total_conf_matrix += conf_matrix
tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix))
tf.logging.info('Final test accuracy = %.1f%% (N=%d)' %
(total_accuracy * 100, set_size))
def main(_=None):
FLAGS = flags.FLAGS # pylint: disable=invalid-name,redefined-outer-name
config = FLAGS
FLAGS.__dict__['config'] = config
FLAGS.logdir = FLAGS.logdir.format(name=FLAGS.name)
logdir = FLAGS.logdir
logging.info('logdir: %s', logdir)
if os.path.exists(logdir) and FLAGS.overwrite:
logging.info('"overwrite" is set to True. Deleting logdir at "%s".', logdir)
shutil.rmtree(logdir)
# Build the graph
with tf.Graph().as_default():
model_dict = model_config.get(FLAGS)
data_dict = data_config.get(FLAGS)
lr = model_dict.lr
opt = model_dict.opt
model = model_dict.model
trainset = data_dict.trainset
validset = data_dict.validset
lr = tf.convert_to_tensor(lr)
tf.summary.scalar('learning_rate', lr)
# Training setup
global_step = tf.train.get_or_create_global_step()
# Optimisation target
validset = tools.maybe_convert_dataset(validset)
trainset = tools.maybe_convert_dataset(trainset)
target, gvs = model.make_target(trainset, opt)
if gvs is None:
gvs = opt.compute_gradients(target)
suppress_inf_and_nans = (config.grad_value_clip > 0
or config.grad_norm_clip > 0)
report = tools.gradient_summaries(gvs, suppress_inf_and_nans)
report['target'] = target
valid_report = dict()
gvs = tools.clip_gradients(gvs, value_clip=config.grad_value_clip,
norm_clip=config.grad_norm_clip)
try:
report.update(model.make_report(trainset))
valid_report.update(model.make_report(validset))
except AttributeError:
logging.warning('Model %s has no "make_report" method.', str(model))
raise
plot_dict, plot_params = None, None
if config.plot:
try:
plot_dict, plot_params = model.make_plot(trainset, 'train')
valid_plot, valid_params = model.make_plot(validset, 'valid')
plot_dict.update(valid_plot)
if plot_params is not None:
plot_params.update(valid_params)
except AttributeError:
logging.warning('Model %s has no "make_plot" method.', str(model))
report = tools.scalar_logs(report, config.ema, 'train',
global_update=config.global_ema_update)
report['lr'] = lr
valid_report = tools.scalar_logs(
valid_report, config.ema, 'valid',
global_update=config.global_ema_update)
reports_keys = sorted(report.keys())
def _format(k):
if k in ('lr', 'learning_rate'):
return '.2E'
return '.3f'
report_template = ', '.join(['{}: {}{}:{}{}'.format(
k, '{', k, _format(k), '}') for k in reports_keys])
logging.info('Trainable variables:')
tools.log_variables_by_scope()
# inspect gradients
for g, v in gvs:
if g is None:
logging.warning('No gradient for variable: %s.', v.name)
tools.log_num_params()
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if FLAGS.check_numerics:
update_ops += [tf.add_check_numerics_ops()]
with tf.control_dependencies(update_ops):
train_step = opt.apply_gradients(gvs, global_step=global_step)
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
with tf.train.SingularMonitoredSession(
hooks=create_hooks(FLAGS, plot_dict, plot_params),
checkpoint_dir=logdir, config=sess_config) as sess:
train_itr, _ = sess.run([global_step, update_ops])
train_tensors = [global_step, train_step]
report_tensors = [report, valid_report]
all_tensors = report_tensors + train_tensors
while train_itr < config.max_train_steps:
if train_itr % config.report_loss_steps == 0:
report_vals, valid_report_vals, train_itr, _ = sess.run(all_tensors)
logging.info('')
logging.info('train:')
logging.info('#%s: %s', train_itr,
report_template.format(**report_vals))
logging.info('valid:')
valid_logs = dict(report_vals)
valid_logs.update(valid_report_vals)
logging.info('#%s: %s', train_itr,
report_template.format(**valid_logs))
vals_to_check = list(report_vals.values())
if (np.isnan(vals_to_check).any()
or np.isnan(vals_to_check).any()):
logging.fatal('NaN in reports: %s; breaking...',
report_template.format(**report_vals))
else:
train_itr, _ = sess.run(train_tensors)
def main(_):
# We want to see all the logging messages for this tutorial.
tf.logging.set_verbosity(tf.logging.INFO)
np.set_printoptions(threshold=np.inf, linewidth=10000)
flags = vars(FLAGS)
for key in sorted(flags.keys()):
tf.logging.info('%s = %s', key, flags[key])
if FLAGS.random_seed_weights != -1:
tf.random.set_random_seed(FLAGS.random_seed_weights)
# Start a new TensorFlow session.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
#config.log_device_placement = False
sess = tf.InteractiveSession(config=config)
# Begin by making sure we have the training data we need. If you already have
# training data of your own, use `--data_url= ` on the command line to avoid
# downloading.
label_count = len(
input_data.prepare_words_list(FLAGS.wanted_words.split(','),
FLAGS.silence_percentage,
FLAGS.unknown_percentage))
model_settings = models.prepare_model_settings(
label_count, FLAGS.sample_rate, FLAGS.nchannels,
FLAGS.clip_duration_ms, FLAGS.representation, FLAGS.window_size_ms,
FLAGS.window_stride_ms, 1, FLAGS.dct_coefficient_count,
FLAGS.filterbank_channel_count,
[int(x) for x in FLAGS.filter_counts.split(',')],
[int(x)
for x in FLAGS.filter_sizes.split(',')], FLAGS.final_filter_len,
FLAGS.dropout_prob, FLAGS.batch_size, FLAGS.dilate_after_layer,
FLAGS.stride_after_layer, FLAGS.connection_type)
fingerprint_size = model_settings['fingerprint_size']
time_shift_samples = int((FLAGS.time_shift_ms * FLAGS.sample_rate) / 1000)
# Figure out the learning rates for each training phase. Since it's often
# effective to have high learning rates at the start of training, followed by
# lower levels towards the end, the number of steps and learning rates can be
# specified as comma-separated lists to define the rate at each stage. For
# example --how_many_training_steps=10000,3000 --learning_rate=0.001,0.0001
# will run 13,000 training loops in total, with a rate of 0.001 for the first
# 10,000, and 0.0001 for the final 3,000.
training_steps_list = list(
map(int, FLAGS.how_many_training_steps.split(',')))
learning_rates_list = list(map(float, FLAGS.learning_rate.split(',')))
if len(training_steps_list) != len(learning_rates_list):
raise Exception(
'--how_many_training_steps and --learning_rate must be equal length '
'lists, but are %d and %d long instead' %
(len(training_steps_list), len(learning_rates_list)))
actual_batch_size = tf.placeholder(tf.int32, [1])
fingerprint_input = tf.placeholder(tf.float32, [None, fingerprint_size],
name='fingerprint_input')
hidden, logits, dropout_prob = models.create_model(
fingerprint_input,
model_settings,
FLAGS.model_architecture,
is_training=True)
# Define loss and optimizer
ground_truth_input = tf.placeholder(tf.int64, [None],
name='groundtruth_input')
# Optionally we can add runtime checks to spot when NaNs or other symptoms of
# numerical errors start occurring during training.
control_dependencies = []
if FLAGS.check_nans:
checks = tf.add_check_numerics_ops()
control_dependencies = [checks]
# Create the back propagation and training evaluation machinery in the graph.
with tf.name_scope('cross_entropy'):
cross_entropy_mean = tf.losses.sparse_softmax_cross_entropy(
labels=tf.slice(ground_truth_input, [0], actual_batch_size),
logits=tf.slice(logits, [0, 0],
tf.concat([actual_batch_size, [-1]], 0)))
tf.summary.scalar('cross_entropy', cross_entropy_mean)
with tf.name_scope('train'), tf.control_dependencies(control_dependencies):
learning_rate_input = tf.placeholder(tf.float32, [],
name='learning_rate_input')
if FLAGS.optimizer == 'sgd':
train_step = tf.train.GradientDescentOptimizer(
learning_rate_input).minimize(cross_entropy_mean)
elif FLAGS.optimizer == 'adam':
train_step = tf.train.AdamOptimizer(learning_rate_input).minimize(
cross_entropy_mean)
elif FLAGS.optimizer == 'adagrad':
train_step = tf.train.AdagradOptimizer(
learning_rate_input).minimize(cross_entropy_mean)
elif FLAGS.optimizer == 'rmsprop':
train_step = tf.train.RMSPropOptimizer(
learning_rate_input).minimize(cross_entropy_mean)
predicted_indices = tf.argmax(logits, 1)
correct_prediction = tf.equal(predicted_indices, ground_truth_input)
confusion_matrix = tf.confusion_matrix(tf.slice(ground_truth_input, [0],
actual_batch_size),
tf.slice(predicted_indices, [0],
actual_batch_size),
num_classes=label_count)
evaluation_step = tf.reduce_mean(
tf.cast(tf.slice(correct_prediction, [0], actual_batch_size),
tf.float32))
tf.summary.scalar('accuracy', evaluation_step)
global_step = tf.train.get_or_create_global_step()
increment_global_step = tf.assign(global_step, global_step + 1)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=0)
# Merge all the summaries and write them out to /tmp/retrain_logs (by default)
merged_summaries = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/train',
sess.graph)
validation_writer = tf.summary.FileWriter(FLAGS.summaries_dir +
'/validation')
tf.global_variables_initializer().run()
start_step = 1
if FLAGS.start_checkpoint:
models.load_variables_from_checkpoint(sess, FLAGS.start_checkpoint)
start_step = 1 + global_step.eval(session=sess)
t0 = dt.datetime.now()
tf.logging.info('Training from time %s, step: %d ', t0.isoformat(),
start_step)
# Save graph.pbtxt.
tf.train.write_graph(sess.graph_def, FLAGS.train_dir,
FLAGS.model_architecture + '.pbtxt')
# Save list of words.
if FLAGS.start_checkpoint == '':
with gfile.GFile(os.path.join(FLAGS.train_dir, \
FLAGS.model_architecture + '_labels.txt'), 'w') as f:
f.write(FLAGS.wanted_words.replace(',', '\n'))
# log complexity of model
total_parameters = 0
for variable in tf.trainable_variables():
shape = variable.get_shape()
variable_parameters = 1
for dim in shape:
variable_parameters *= int(dim)
total_parameters += variable_parameters
tf.logging.info('number of trainable parameters: %d', total_parameters)
checkpoint_path = os.path.join(FLAGS.train_dir,
FLAGS.model_architecture + '.ckpt')
if FLAGS.start_checkpoint == '':
tf.logging.info('Saving to "%s-%d"', checkpoint_path, 0)
saver.save(sess, checkpoint_path, global_step=0)
audio_processor = input_data.AudioProcessor(
FLAGS.data_url, FLAGS.data_dir, FLAGS.silence_percentage,
FLAGS.unknown_percentage, FLAGS.wanted_words.split(','),
FLAGS.labels_touse.split(','),
FLAGS.validation_percentage, FLAGS.validation_offset_percentage,
FLAGS.validation_files.split(','), FLAGS.testing_percentage,
FLAGS.testing_files.split(','), FLAGS.subsample_skip,
FLAGS.subsample_word, FLAGS.partition_word, FLAGS.partition_n,
FLAGS.partition_training_files.split(','),
FLAGS.partition_validation_files.split(','), FLAGS.random_seed_batch,
FLAGS.testing_equalize_ratio, FLAGS.testing_max_samples,
model_settings)
# exit if how_many_training_steps==0
if FLAGS.how_many_training_steps == '0':
# pre-process a batch of data to make sure settings are valid
train_fingerprints, train_ground_truth, _ = audio_processor.get_data(
FLAGS.batch_size, 0, model_settings, FLAGS.background_frequency,
FLAGS.background_volume, time_shift_samples,
FLAGS.time_shift_random, 'training', sess)
sess.run(
[evaluation_step],
feed_dict={
fingerprint_input: train_fingerprints,
ground_truth_input: train_ground_truth,
learning_rate_input: learning_rates_list[0],
actual_batch_size: [FLAGS.batch_size],
dropout_prob: model_settings['dropout_prob']
})
return
training_set_size = audio_processor.set_size('training')
testing_set_size = audio_processor.set_size('testing')
validation_set_size = audio_processor.set_size('validation')
# Training loop.
training_steps_max = np.sum(training_steps_list)
for training_step in xrange(start_step, training_steps_max + 1):
if training_set_size > 0 and FLAGS.save_step_interval > 0:
# Figure out what the current learning rate is.
training_steps_sum = 0
for i in range(len(training_steps_list)):
training_steps_sum += training_steps_list[i]
if training_step <= training_steps_sum:
learning_rate_value = learning_rates_list[i]
break
# Pull the audio samples we'll use for training.
train_fingerprints, train_ground_truth, _ = audio_processor.get_data(
FLAGS.batch_size, 0, model_settings,
FLAGS.background_frequency, FLAGS.background_volume,
time_shift_samples, FLAGS.time_shift_random, 'training', sess)
# Run the graph with this batch of training data.
train_summary, train_accuracy, cross_entropy_value, _, _ = sess.run(
[
merged_summaries, evaluation_step, cross_entropy_mean,
train_step, increment_global_step
],
feed_dict={
fingerprint_input: train_fingerprints,
ground_truth_input: train_ground_truth,
learning_rate_input: learning_rate_value,
actual_batch_size: [FLAGS.batch_size],
dropout_prob: model_settings['dropout_prob']
})
train_writer.add_summary(train_summary, training_step)
t1 = dt.datetime.now() - t0
tf.logging.info(
'Elapsed %f, Step #%d: rate %f, accuracy %.1f%%, cross entropy %f'
% (t1.total_seconds(), training_step, learning_rate_value,
train_accuracy * 100, cross_entropy_value))
# Save the model checkpoint periodically.
if (training_step % FLAGS.save_step_interval == 0
or training_step == training_steps_max):
tf.logging.info('Saving to "%s-%d"', checkpoint_path,
training_step)
saver.save(sess, checkpoint_path, global_step=training_step)
is_last_step = (training_step == training_steps_max)
if validation_set_size > 0 and (is_last_step or
(training_step %
FLAGS.eval_step_interval) == 0):
validate_and_test('validation', validation_set_size, model_settings, \
time_shift_samples, sess, merged_summaries, evaluation_step, \
confusion_matrix, logits, hidden, validation_writer, \
audio_processor, is_last_step, fingerprint_input, \
ground_truth_input, actual_batch_size, dropout_prob, \
training_step, t0)
if testing_set_size > 0:
validate_and_test('testing', testing_set_size, model_settings, time_shift_samples, \
sess, merged_summaries, evaluation_step, confusion_matrix, \
logits, hidden, validation_writer, audio_processor, \
True, fingerprint_input, ground_truth_input, \
actual_batch_size, dropout_prob, training_steps_max, t0)