def evaluate(model_name=None,
hparams=None,
class_map_path=None,
eval_csv_path=None,
eval_clip_dir=None,
eval_dir=None,
train_dir=None):
"""Runs the evaluation loop."""
print('\nEvaluation for model:{} with hparams:{} and class map:{}'.format(
model_name, hparams, class_map_path))
print('Evaluation data: clip dir {} and labels {}'.format(
eval_clip_dir, eval_csv_path))
# Read in class map CSV into a class index -> class name map.
class_map = {
int(row[0]): row[1]
for row in csv.reader(open(class_map_path))
}
with tf.Graph().as_default():
label_class_index_table, num_classes = inputs.get_class_map(
class_map_path)
csv_record = tf.placeholder(tf.string,
[]) # fed during evaluation loop.
# Use a simpler in-order input pipeline for eval than the one used in
# training, since we don't want to shuffle examples across clips.
# The features consist of a batch of all possible framed log mel spectrum
# examples from the same clip. The labels in this case will contain a batch
# of identical 1-hot vectors.
features, labels = inputs.record_to_labeled_log_mel_examples(
csv_record,
clip_dir=eval_clip_dir,
hparams=hparams,
label_class_index_table=label_class_index_table,
num_classes=num_classes)
# Create the model in prediction mode.
global_step, prediction, _, _ = model.define_model(
model_name=model_name,
features=features,
num_classes=num_classes,
hparams=hparams,
training=False)
# Write evaluation graph to checkpoint directory.
tf.train.write_graph(
tf.get_default_graph().as_graph_def(add_shapes=True), eval_dir,
'eval.pbtxt')
summary_writer = tf.summary.FileWriter(eval_dir,
tf.get_default_graph())
# Loop through all checkpoints in the training directory.
checkpoint_state = tf.train.get_checkpoint_state(train_dir)
for checkpoint_path in checkpoint_state.all_model_checkpoint_paths:
checkpoint_num = get_checkpoint_num(checkpoint_path)
if eval_done(eval_dir, checkpoint_num):
print("Checkpoint %d already evaluated, skipping" %
checkpoint_num)
continue
lwlrap = eval_checkpoint(checkpoint_path, eval_dir, summary_writer,
eval_csv_path, class_map, csv_record,
global_step, labels, prediction)
write_eval_marker(eval_dir, checkpoint_num, lwlrap)
def predict(model_name=None,
hparams=None,
test_clip_dir=None,
class_map_path=None,
checkpoint_path=None,
predictions_csv_path=None):
"""Runs the prediction loop, producting prediction output in Kaggle submission format."""
print('\nPrediction for model:{} with hparams:{} and class map:{}'.format(
model_name, hparams, class_map_path))
print('Prediction data: clip dir {} and checkpoint {}'.format(
test_clip_dir, checkpoint_path))
print('Predictions in CSV {}\n'.format(predictions_csv_path))
with tf.Graph().as_default():
_, num_classes = inputs.get_class_map(class_map_path)
clip = tf.placeholder(tf.string, []) # Fed during prediction loop.
# Use a simpler in-order input pipeline without labels for prediction
# compared to the one used in training. The features consist of a batch of
# all possible framed log mel spectrum examples from the same clip.
features = inputs.clip_to_log_mel_examples(clip,
clip_dir=test_clip_dir,
hparams=hparams)
# Creates the model in prediction mode.
_, prediction, _, _ = model.define_model(model_name=model_name,
features=features,
num_classes=num_classes,
hparams=hparams,
training=False)
with tf.train.SingularMonitoredSession(
checkpoint_filename_with_path=checkpoint_path) as sess:
# Read in class map CSV into a class index -> class name map.
class_map = {
int(row[0]): row[1]
for row in csv.reader(open(class_map_path))
}
test_clips = sorted(os.listdir(test_clip_dir))
pred_writer = csv.DictWriter(open(predictions_csv_path, 'w'),
fieldnames=['fname', 'label'])
pred_writer.writeheader()
for (i, test_clip) in enumerate(test_clips):
print(i, test_clip)
sys.stdout.flush()
# Hack to avoid passing empty files through the model.
if os.path.getsize(os.path.join(test_clip_dir,
test_clip)) == 44:
print('empty file, skipped model')
label = ''
else:
predicted = sess.run(prediction, {clip: test_clip})
predicted_classes = evaluation.get_top_predicted_classes(
predicted)
label = ' '.join([class_map[c] for c in predicted_classes])
pred_writer.writerow({'fname': test_clip, 'label': label})
print(label)
sys.stdout.flush()
def predict(model_name=None, hparams=None, test_clip_dir=None,
class_map_path=None, checkpoint_path=None, predictions_csv_path=None):
soundfile = "The Fast and the Furious.wav"
samplingFrequency, signalData = wavfile.read(soundfile)
start = 0
end = 1
count = 0
while True:
tmp = signalData[start*samplingFrequency : end*samplingFrequency]
file_name = "divide_audio/"+str(count).zfill(8) + ".wav"
write(file_name,44100,tmp)
count += 1
start += 1
end += 1
if (end * samplingFrequency) > signalData.size:
start = 0
end = 1
break
with tf.Graph().as_default():
_, num_classes = inputs.get_class_map(class_map_path)
clip = tf.placeholder(tf.string, []) # Fed during prediction loop.
# Use a simpler in-order input pipeline without labels for prediction
# compared to the one used in training. The features consist of a batch of
# all possible framed log mel spectrum examples from the same clip.
features = inputs.clip_to_log_mel_examples(
clip, clip_dir=test_clip_dir, hparams=hparams)
# Creates the model in prediction mode.
_, prediction, _, _ = model.define_model(
model_name=model_name, features=features, num_classes=num_classes,
hparams=hparams, training=False)
f = open("Total.txt",'w')
with tf.train.SingularMonitoredSession(checkpoint_filename_with_path=checkpoint_path) as sess:
class_map = {int(row[0]): row[1] for row in csv.reader(open(class_map_path))}
test_clips = sorted(os.listdir(test_clip_dir))
for (i, test_clip) in enumerate(test_clips):
print(i, test_clip)
sys.stdout.flush()
# Hack to avoid passing empty files through the model.
if os.path.getsize(os.path.join(test_clip_dir, test_clip)) == 44:
print('empty file, skipped model')
label = ''
else:
predicted = sess.run(prediction, {clip: test_clip})
predicted_classes = evaluation.get_top_predicted_classes(predicted)
label = ' '.join([class_map[c] for c in predicted_classes])
f.write('audio/residential_area/a001.wav\t' + str(start) +"\t" + str(end) + "\t" + label + "\n")
start += 1
end += 1
sys.stdout.flush()
f.close()
def evaluate(model_name=None,
hparams=None,
eval_csv_path=None,
eval_clip_dir=None,
class_map_path=None,
checkpoint_path=None):
"""Runs the evaluation loop."""
print('\nEvaluation for model:{} with hparams:{} and class map:{}'.format(
model_name, hparams, class_map_path))
print('Evaluation data: clip dir {} and labels {}'.format(
eval_clip_dir, eval_csv_path))
print('Checkpoint: {}\n'.format(checkpoint_path))
with tf.Graph().as_default():
label_class_index_table, num_classes = inputs.get_class_map(
class_map_path)
csv_record = tf.placeholder(tf.string,
[]) # fed during evaluation loop.
# Use a simpler in-order input pipeline for eval than the one used in
# training, since we don't want to shuffle examples across clips.
# The features consist of a batch of all possible framed log mel spectrum
# examples from the same clip. The labels in this case will contain a batch
# of identical 1-hot vectors.
features, labels = inputs.record_to_labeled_log_mel_examples(
csv_record,
clip_dir=eval_clip_dir,
hparams=hparams,
label_class_index_table=label_class_index_table,
num_classes=num_classes)
# Create the model in prediction mode.
global_step, prediction, _, _ = model.define_model(
model_name=model_name,
features=features,
num_classes=num_classes,
hparams=hparams,
training=False)
with tf.train.SingularMonitoredSession(
checkpoint_filename_with_path=checkpoint_path) as sess:
# Read in class map CSV into a class index -> class name map.
class_map = {
int(row[0]): row[1]
for row in csv.reader(open(class_map_path))
}
# Keep running counters to aid printing incremental AP stats.
ap_counts = defaultdict(
int
) # maps class index to the number of clips with that label.
ap_sums = defaultdict(
float
) # maps class index to the sum of AP for all clips with that label.
# Read in the validation CSV, skippign the header.
eval_records = open(eval_csv_path).readlines()[1:]
# Shuffle the lines so that as we print incremental stats, we get good
# coverage across classes and get a quick initial impression of how well
# the model is doing even before evaluation is completely done.
random.shuffle(eval_records)
for (i, record) in enumerate(eval_records):
record = record.strip()
actual, predicted = sess.run([labels, prediction],
{csv_record: record})
# By construction, actual consists of identical rows, where each row is
# the same 1-hot label (because we are looking at features from the same
# clip). np.argmax() of any of the rows (and in particular [0]) will
# give us the class index corresponding to the label.
actual_class = np.argmax(actual[0])
predicted_classes = get_top_predicted_classes(predicted)
# Compute AP for this item, update running counters/sums, and print the
# prediction vs actual.
ap = avg_precision(actual=actual_class,
predicted=predicted_classes)
ap_counts[actual_class] += 1
ap_sums[actual_class] += ap
print(class_map[actual_class],
[class_map[index] for index in predicted_classes], ap)
# Periodically print per-class and overall AP stats.
if i % 50 == 0:
print_maps(ap_sums=ap_sums,
ap_counts=ap_counts,
class_map=class_map)
sys.stdout.flush()
# Final AP stats.
print_maps(ap_sums=ap_sums,
ap_counts=ap_counts,
class_map=class_map)
def train_and_evaluate(model_name=None, hparams=None, class_map_path=None, train_csv_path=None, train_clip_dir=None,
train_dir=None, epoch_batches=None, warmstart_checkpoint=None,
warmstart_include_scopes=None, warmstart_exclude_scopes=None,
eval_csv_path=None, eval_clip_dir=None, eval_dir=None):
"""Runs the training loop."""
print('\nTraining model:{} with hparams:{} and class map:{}'.format(model_name, hparams, class_map_path))
print('Training data: clip dir {} and labels {}'.format(train_clip_dir, train_csv_path))
print('Training dir {}\n'.format(train_dir))
class_map = {int(row[0]): row[1] for row in csv.reader(open(class_map_path))}
with tf.Graph().as_default():
# Create the input pipeline.
features, labels, num_classes, input_init = inputs.train_input(
train_csv_path=train_csv_path, train_clip_dir=train_clip_dir, class_map_path=class_map_path,
hparams=hparams)
# Create the model in training mode.
global_step, prediction, loss_tensor, train_op = model.define_model(
model_name=model_name, features=features, labels=labels, num_classes=num_classes,
hparams=hparams, epoch_batches=epoch_batches, training=True)
# evaluation graph
label_class_index_table, num_classes = inputs.get_class_map(class_map_path)
csv_record = tf.placeholder(tf.string, []) # fed during evaluation loop.
eval_features, eval_labels = inputs.record_to_labeled_log_mel_examples(
csv_record, clip_dir=eval_clip_dir, hparams=hparams,
label_class_index_table=label_class_index_table, num_classes=num_classes)
# Create the model in prediction mode.
global_step, eval_predictions, eval_loss_tensor, _ = model.define_model(
model_name=model_name, features=eval_features, labels=eval_labels, num_classes=num_classes,
hparams=hparams, training=False, evaluating=True)
# Write evaluation graph to checkpoint directory.
tf.train.write_graph(tf.get_default_graph().as_graph_def(add_shapes=True),
eval_dir, 'eval.pbtxt')
eval_writer = tf.summary.FileWriter(eval_dir, tf.get_default_graph())
# Define our own checkpoint saving hook, instead of using the built-in one,
# so that we can specify additional checkpoint retention settings.
saver = tf.train.Saver(
max_to_keep=10, keep_checkpoint_every_n_hours=0.25)
saver_hook = tf.train.CheckpointSaverHook(
save_steps=100, checkpoint_dir=train_dir, saver=saver)
summary_op = tf.summary.merge_all()
summary_hook = tf.train.SummarySaverHook(
save_steps=10, output_dir=train_dir, summary_op=summary_op)
if hparams.warmstart:
var_include_scopes = warmstart_include_scopes
if not var_include_scopes: var_include_scopes = None
var_exclude_scopes = warmstart_exclude_scopes
if not var_exclude_scopes: var_exclude_scopes = None
restore_vars = tf.contrib.framework.get_variables_to_restore(
include=var_include_scopes, exclude=var_exclude_scopes)
# Only restore trainable variables, we don't want to restore
# batch-norm or optimizer-specific local variables.
trainable_vars = set(tf.contrib.framework.get_trainable_variables())
restore_vars = [var for var in restore_vars if var in trainable_vars]
print('Warm-start: restoring variables:\n%s\n' % '\n'.join([x.name for x in restore_vars]))
print('Warm-start: restoring from ', warmstart_checkpoint)
assert restore_vars, 'No warm-start variables to restore!'
restore_op, feed_dict = tf.contrib.framework.assign_from_checkpoint(
model_path=warmstart_checkpoint, var_list=restore_vars, ignore_missing_vars=True)
scaffold = tf.train.Scaffold(
init_fn=lambda scaffold, session: session.run(restore_op, feed_dict),
summary_op=summary_op, saver=saver)
else:
scaffold = None
with tf.train.SingularMonitoredSession(hooks=[saver_hook, summary_hook],
checkpoint_dir=train_dir,
scaffold=scaffold,
config=tf.ConfigProto(log_device_placement=True)) as sess:
sess.raw_session().run(input_init)
while not sess.should_stop():
# train
step, _, pred, loss = sess.run([global_step, train_op, prediction, loss_tensor])
print(step, loss)
sys.stdout.flush()
# evaluates every 100 steps
if step > 0 and step % 100 == 0:
# Loop through all checkpoints in the training directory.
checkpoint_state = tf.train.get_checkpoint_state(train_dir)
lwlrap = eval_batch(eval_csv_path, sess, eval_labels, eval_predictions, csv_record, step, eval_writer, class_map, eval_loss_tensor)
def evaluate(model_name=None, hparams=None, eval_csv_path=None, eval_clip_dir=None,
class_map_path=None, checkpoint_path=None):
"""Runs the evaluation loop."""
print('\nEvaluation for model:{} with hparams:{} and class map:{}'.format(model_name, hparams, class_map_path))
print('Evaluation data: clip dir {} and labels {}'.format(eval_clip_dir, eval_csv_path))
print('Checkpoint: {}\n'.format(checkpoint_path))
with tf.Graph().as_default():
label_class_index_table, num_classes = inputs.get_class_map(class_map_path)
csv_record = tf.placeholder(tf.string, []) # fed during evaluation loop.
# Use a simpler in-order input pipeline for eval than the one used in
# training, since we don't want to shuffle examples across clips.
# The features consist of a batch of all possible framed log mel spectrum
# examples from the same clip. The labels in this case will contain a batch
# of identical 1-hot vectors.
features, labels = inputs.record_to_labeled_log_mel_examples(
csv_record, clip_dir=eval_clip_dir, hparams=hparams,
label_class_index_table=label_class_index_table, num_classes=num_classes)
# Create the model in prediction mode.
global_step, prediction, _, _ = model.define_model(
model_name=model_name, features=features, num_classes=num_classes,
hparams=hparams, training=False)
with tf.train.SingularMonitoredSession(checkpoint_filename_with_path=checkpoint_path) as sess:
# Read in class map CSV into a class index -> class name map.
class_map = {int(row[0]): row[1] for row in csv.reader(open(class_map_path))}
# Keep running counters to aid printing incremental AP stats.
ap_counts = defaultdict(int) # maps class index to the number of clips with that label.
ap_sums = defaultdict(float) # maps class index to the sum of AP for all clips with that label.
# Read in the validation CSV, skippign the header.
eval_records = open(eval_csv_path).readlines()[1:]
# Shuffle the lines so that as we print incremental stats, we get good
# coverage across classes and get a quick initial impression of how well
# the model is doing even before evaluation is completely done.
random.shuffle(eval_records)
for (i,record) in enumerate(eval_records):
record = record.strip()
actual, predicted = sess.run([labels, prediction], {csv_record: record})
# By construction, actual consists of identical rows, where each row is
# the same 1-hot label (because we are looking at features from the same
# clip). np.argmax() of any of the rows (and in particular [0]) will
# give us the class index corresponding to the label.
actual_class = np.argmax(actual[0])
predicted_classes = get_top_predicted_classes(predicted)
# Compute AP for this item, update running counters/sums, and print the
# prediction vs actual.
ap = avg_precision(actual=actual_class, predicted=predicted_classes)
ap_counts[actual_class] += 1
ap_sums[actual_class] += ap
print(class_map[actual_class], [class_map[index] for index in predicted_classes], ap)
# Periodically print per-class and overall AP stats.
if i % 50 == 0:
print_maps(ap_sums=ap_sums, ap_counts=ap_counts, class_map=class_map)
sys.stdout.flush()
# Final AP stats.
print_maps(ap_sums=ap_sums, ap_counts=ap_counts, class_map=class_map)