def predict(sdae, data_set, bias_node=False):
with sdae.session.graph.as_default():
labels_placeholder = tf.placeholder(tf.int32, shape=1,\
name='labels_placeholder')
examples_placeholder = tf.placeholder(tf.float32,\
shape=(1, sdae._net_shape[0]),\
name='input_pl')
logits = tf.identity(examples_placeholder)
for layer in sdae.get_layers:
if bias_node:
bias_n = tf.ones(shape=[1, 1], dtype=tf.float32)
logits = tf.concat(1, [bias_n, logits])
logits = layer.clean_activation(x_in=logits, use_fixed=False)
predictions = tf.argmax(logits, 1)
labels = tf.identity(labels_placeholder)
y_pred = []
y_true = []
for _ in xrange(data_set.num_examples):
feed_dict = fill_feed_dict(data_set,
examples_placeholder,
labels_placeholder, 1)
y_prediction, y_trues = sdae.session.run([predictions, labels], feed_dict=feed_dict)
y_pred += list(y_prediction)
y_true += list(y_trues)
# print(y_pred)
return y_pred, y_true
def predict(sdae, data_set, bias_node=False):
with sdae.session.graph.as_default():
labels_placeholder = tf.placeholder(tf.int32, shape=1,\
name='labels_placeholder')
examples_placeholder = tf.placeholder(tf.float32,\
shape=(1, sdae._net_shape[0]),\
name='input_pl')
logits = tf.identity(examples_placeholder)
for layer in sdae.get_layers:
if bias_node:
bias_n = tf.ones(shape=[1, 1], dtype=tf.float32)
logits = tf.concat(1, [bias_n, logits])
logits = layer.clean_activation(x_in=logits, use_fixed=False)
predictions = tf.argmax(logits, 1)
labels = tf.identity(labels_placeholder)
y_pred = []
y_true = []
for _ in xrange(data_set.num_examples):
feed_dict = fill_feed_dict(data_set, examples_placeholder,
labels_placeholder, 1)
y_prediction, y_trues = sdae.session.run([predictions, labels],
feed_dict=feed_dict)
y_pred += list(y_prediction)
y_true += list(y_trues)
# print(y_pred)
return y_pred, y_true
def minibatch_transformation(dataset):
patches_pl, labels_pl = utils.placeholder_inputs()
feed_dict = utils.fill_feed_dict(dataset.train,
patches_pl,
labels_pl,
36,
augment=True)
for patch in feed_dict[patches_pl]:
cv2.imshow('patch', patch)
cv2.waitKey(0)
def do_eval_summary(tag, sess, eval_correct, examples_placeholder,
labels_placeholder, data_set):
true_count = 0
steps_per_epoch = data_set.num_examples // FLAGS.batch_size
num_examples = steps_per_epoch * FLAGS.batch_size
for _ in xrange(steps_per_epoch):
feed_dict = fill_feed_dict(data_set, examples_placeholder,
labels_placeholder)
true_count += sess.run(eval_correct, feed_dict=feed_dict)
error = 1 - true_count / num_examples
return sess.run(tf.scalar_summary(tag, tf.identity(error)))
def do_eval(sess, eval_correct, keep_prob):
"""Runs one evaluation against the full epoch of data.
Args:
sess: The session in which the model has been trained.
eval_correct: The Tensor that returns the number of correct predictions.
keep_prob: The keep prob placeholder.
"""
# And run one epoch of eval.
true_count = 0 # Counts the number of correct predictions.
steps_per_epoch = data_input.NUM_EXAMPLES_PER_EPOCH_FOR_TEST // FLAGS.batch_size
num_examples = steps_per_epoch * FLAGS.batch_size
for step in xrange(steps_per_epoch):
feed_dict = utils.fill_feed_dict(keep_prob, train=False)
true_count += sess.run(eval_correct, feed_dict=feed_dict)
precision = true_count / num_examples
print(" Num examples: %d Num correct: %d Precision @ 1: %0.04f" % (num_examples, true_count, precision))
def do_eval_summary(tag,
sess,
eval_correct,
examples_placeholder,
labels_placeholder,
data_set):
true_count = 0
steps_per_epoch = data_set.num_examples // FLAGS.batch_size
num_examples = steps_per_epoch * FLAGS.batch_size
for _ in xrange(steps_per_epoch):
feed_dict = fill_feed_dict(data_set,
examples_placeholder,
labels_placeholder)
true_count += sess.run(eval_correct, feed_dict=feed_dict)
error = 1 - true_count / num_examples
return sess.run(tf.scalar_summary(tag, tf.identity(error)))
def do_eval(sess, eval_correct, keep_prob):
"""Runs one evaluation against the full epoch of data.
Args:
sess: The session in which the model has been trained.
eval_correct: The Tensor that returns the number of correct predictions.
keep_prob: The keep prob placeholder.
"""
# And run one epoch of eval.
true_count = 0 # Counts the number of correct predictions.
steps_per_epoch = data_input.NUM_EXAMPLES_PER_EPOCH_FOR_TEST // FLAGS.batch_size
num_examples = steps_per_epoch * FLAGS.batch_size
for step in xrange(steps_per_epoch):
feed_dict = utils.fill_feed_dict(keep_prob, train=False)
true_count += sess.run(eval_correct, feed_dict=feed_dict)
precision = true_count / num_examples
print(' Num examples: %d Num correct: %d Precision @ 1: %0.04f' %
(num_examples, true_count, precision))
def run_training():
"""Train BinaryConnect."""
# Get the sets of images and labels for training, validation, and
# test on CIFAR10.
data_sets = cifar10.read_data_sets(dst_dir='./dataset',
validation_size=5000)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# Generate placeholders for the images and labels.
images_placeholder, labels_placeholder, train_placeholder = placeholder_inputs(
FLAGS.batch_size)
# Build a Graph that computes predictions from the inference model.
logits = bc.inference_bin(images_placeholder, train_placeholder,
stochastic=FLAGS.stochastic,
use_bnorm=True) \
if FLAGS.binary \
else bc.inference_ref(images_placeholder, train_placeholder,
use_bnorm=True)
# Add to the Graph the Ops for loss calculation.
loss = bc.loss(logits, labels_placeholder)
# Add to the Graph the Ops that calculate and apply gradients.
train_op = bc.training(loss, FLAGS.learning_rate)
# Add the Op to compare the logits to the labels during evaluation.
eval_metric = bc.evaluation(logits, labels_placeholder)
# Add a placeholder for logging execution time
# frequency_placeholder = tf.placeholder(tf.float32, shape=())
# tf.summary.scalar('Execution Time', frequency_placeholder)
# TODO: support a d separate summary for metadata (e.g. execution time)
# Build the summary Tensor based on the TF collection of Summaries.
summary = tf.summary.merge_all()
# Add the variable initializer Op.
ivars = tf.global_variables() + tf.local_variables()
init = tf.variables_initializer(ivars)
# Create a saver for writing training checkpoints.
saver = tf.train.Saver()
# Create a logger to the validation accuracy
val_acc_pl = tf.placeholder(tf.float32, shape=())
summary_val_acc = tf.summary.scalar(name='validation_acc',
tensor=val_acc_pl,
collections=['validation'])
summary_val = tf.summary.merge([summary_val_acc])
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer_train = tf.summary.FileWriter(
os.path.join(FLAGS.log_dir, 'train'), sess.graph)
summary_writer_val = tf.summary.FileWriter(
os.path.join(FLAGS.log_dir, 'val'), sess.graph)
# And then after everything is built:
# Run the Op to initialize the variables.
sess.run(init)
# Start the training loop.
duration = 0
tp_value_total = 0
for step in xrange(FLAGS.max_steps):
start_time = time.time()
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
feed_dict = fill_feed_dict(data_sets.train, images_placeholder,
labels_placeholder, train_placeholder,
True)
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value, acc_val = sess.run([train_op, loss, eval_metric],
feed_dict=feed_dict)
duration += time.time() - start_time
tp_value_total += acc_val
# Write the summaries and print an overview fairly often.
if step % 100 == 0 and step > 0:
# Print status to stdout.
images_freq = 100 * FLAGS.batch_size / duration
print(
'Step %d: loss = %.2f, correct = %.2f%% (%.3f images/sec)'
% (step, loss_value, tp_value_total / FLAGS.batch_size,
images_freq))
duration = time.time() - start_time
tp_value_total = 0
duration = 0
# Update the events file.
summary_str = sess.run(summary, feed_dict=feed_dict)
summary_writer_train.add_summary(summary_str, step)
summary_writer_train.flush()
# Save a checkpoint and evaluate the model periodically.
if (step + 1) % 500 == 0 or (step + 1) == FLAGS.max_steps:
checkpoint_file = os.path.join(FLAGS.log_dir, 'model.ckpt')
saver.save(sess, checkpoint_file, global_step=step)
# Evaluate against the training set.
# print('Training Data Eval:')
# do_eval(sess,
# eval_metric,
# images_placeholder,
# labels_placeholder,
# train_placeholder,
# data_sets.train, summary)
# Evaluate against the validation set.
print('Validation Data Eval:')
accuracy_val = do_eval(sess, eval_metric, images_placeholder,
labels_placeholder, train_placeholder,
data_sets.validation)
# TODO: find a way to collect summaries for validation
summary_str = sess.run(summary_val,
feed_dict={val_acc_pl: accuracy_val})
summary_writer_val.add_summary(summary_str, step)
summary_writer_val.flush()
# Evaluate against the test set.
print('Test Data Eval:')
do_eval(sess, eval_metric, images_placeholder,
labels_placeholder, train_placeholder, data_sets.test)
def do_eval(sess,
eval_correct,
predictions,
examples_placeholder,
labels_placeholder,
label_map,
data_set,
title='Evaluation'):
"""Runs one evaluation against the full epoch of data.
Args:
sess: The session in which the model has been trained.
eval_correct: The Tensor that returns the number of correct predictions.
images_placeholder: The images placeholder.
labels_placeholder: The labels placeholder.
data_set: The set of images and labels to evaluate, from
utils.read_data_sets().
"""
# And run one epoch of eval.
true_count = 0 # Counts the number of correct predictions.
y_pred = []
y_true = []
steps_per_epoch = data_set.num_examples // FLAGS.batch_size
num_examples = steps_per_epoch * FLAGS.batch_size
labels = tf.identity(labels_placeholder)
for _ in xrange(steps_per_epoch):
feed_dict = fill_feed_dict(data_set,
examples_placeholder,
labels_placeholder)
corrects, y_prediction, y_trues = sess.run([eval_correct, predictions,\
labels], feed_dict=feed_dict)
true_count += corrects
y_pred += list(y_prediction)
y_true += list(y_trues)
accuracy = true_count / num_examples
print(title + ' - Num examples: %d Num correct: %d Accuracy_score @ 1: %0.08f' %
(num_examples, true_count, accuracy))
# print("True output:", y_true)
# print("Pred output:", y_pred)
print("Precision:")
print("\tNone: ", precision_score(y_true, y_pred, average=None, pos_label=None))
# print("\tBinary:", precision_score(y_true, y_pred, average='binary'))
print("\tMicro: %0.08f" % precision_score(y_true, y_pred, average='micro', pos_label=None))
print("\tMacro: %0.08f" % precision_score(y_true, y_pred, average='macro', pos_label=None))
print("\tWeighted: %0.08f" % precision_score(y_true, y_pred, average='weighted', pos_label=None))
# print("\tSamples:", sklearn.metrics.precision_score(y_true, y_pred, average='samples'))
# print("\tAccuracy_score: %0.08f" % accuracy_score(y_true, y_pred))
print("Recall:")
# print("\tNone: ", recall_score(y_true, y_pred, average=None, pos_label=None))
# print("\tBinary:", recall_score(y_true, y_pred, average='binary'))
print("\tMicro: %0.08f" % recall_score(y_true, y_pred, average='micro', pos_label=None))
print("\tMacro: %0.08f" % recall_score(y_true, y_pred, average='macro', pos_label=None))
print("\tWeighted: %0.08f" % recall_score(y_true, y_pred, average='weighted', pos_label=None))
# print("\tSamples:", sklearn.metrics.recall_score(y_true, y_pred, average='samples'))
# print("F1_score:")
# print("\tNone: ", f1_score(y_true, y_pred, average=None, pos_label=None))
# print("\tBinary:", f1_score(y_true, y_pred, average='binary'))
# print("\tMicro: %0.08f" % f1_score(y_true, y_pred, average='micro', pos_label=None))
# print("\tMacro: %0.08f" % f1_score(y_true, y_pred, average='macro', pos_label=None))
print("\nF1 Score (weighted): %0.08f" % f1_score(y_true, y_pred, average='weighted', pos_label=None))
# print("\tSamples:", sklearn.metrics.f1_score(y_true, y_pred, average='samples'))
# print("True Length:", len(y_true))
# print("Prediction Length:", len(y_pred))
cm = confusion_matrix(y_true, y_pred)
# print("\nConfusion Matrix")
# print(cm)
cm_normalized = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
# print("\nNormalized confusion_matrix")
# print(cm_normalized)
print("")
print(classification_report(y_true, y_pred, target_names=label_map))
pcm(cm, target_names=label_map, title=title)
pcm(cm_normalized, target_names=label_map, title=title+"_Normalized")
roc(y_pred, y_true, n_classes=len(label_map), title=title)
print("\n=====================================================================================================\n")
def run_training():
"""Train model for a number of steps."""
# Get the sets of images and labels for training, validation, and
# test on MNIST.
# Tell TensorFlow that the model will be built into the default Graph.
train_dir = os.path.join(FLAGS.model_dir,FLAGS.name)
with tf.Graph().as_default():
global_step = tf.Variable(0, trainable=False)
with tf.name_scope('Input'):
image_batch, label_batch = data_input.distorted_inputs(FLAGS.data_dir, FLAGS.batch_size)
# Generate placeholders for the images and labels.
keep_prob = utils.placeholder_inputs(FLAGS.batch_size)
# Build a Graph that computes predictions from the inference model.
logits = model.inference(image_batch, keep_prob)
# Add to the Graph the Ops for loss calculation.
loss = model.loss(logits, label_batch)
# Add to the Graph the Ops that calculate and apply gradients.
train_op = model.training(loss, global_step=global_step, learning_rate=FLAGS.learning_rate)
# Add the Op to compare the logits to the labels during evaluation.
eval_correct = model.evaluation(logits, label_batch)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.merge_all_summaries()
# Create a saver for writing training checkpoints.
saver = tf.train.Saver()
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Run the Op to initialize the variables.
init = tf.initialize_all_variables()
sess.run(init)
# Start the queue runners.
tf.train.start_queue_runners(sess=sess)
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.train.SummaryWriter(train_dir,
graph_def=sess.graph_def)
# And then after everything is built, start the training loop.
for step in xrange(FLAGS.max_steps):
start_time = time.time()
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
feed_dict = utils.fill_feed_dict(keep_prob,
train = True)
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value = sess.run([train_op, loss],
feed_dict=feed_dict)
# Write the summaries and print an overview fairly often.
if step % 100 == 0:
# Print status to stdout.
duration = time.time() - start_time
examples_per_sec = FLAGS.batch_size / duration
sec_per_batch = float(duration)
print('Step %d: loss = %.2f ( %.3f sec (per Batch); %.1f examples/sec;)' % (step, loss_value,
sec_per_batch, examples_per_sec))
# Update the events file.
summary_str = sess.run(summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
# Save a checkpoint and evaluate the model periodically.
if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps:
checkpoint_path = os.path.join(train_dir, 'model.ckpt')
saver.save(sess, checkpoint_path , global_step=step)
# Evaluate against the training set.
if (step + 1) % 10000 == 0 or (step + 1) == FLAGS.max_steps:
print('Training Data Eval:')
utils.do_eval(sess,
eval_correct,
keep_prob,
data_input.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN)
def do_eval(sess,
eval_correct,
predictions,
examples_placeholder,
labels_placeholder,
label_map,
data_set,
title='Evaluation'):
"""Runs one evaluation against the full epoch of data.
Args:
sess: The session in which the model has been trained.
eval_correct: The Tensor that returns the number of correct predictions.
images_placeholder: The images placeholder.
labels_placeholder: The labels placeholder.
data_set: The set of images and labels to evaluate, from
utils.read_data_sets().
"""
# And run one epoch of eval.
true_count = 0 # Counts the number of correct predictions.
y_pred = []
y_true = []
steps_per_epoch = data_set.num_examples // FLAGS.batch_size
num_examples = steps_per_epoch * FLAGS.batch_size
labels = tf.identity(labels_placeholder)
for _ in xrange(steps_per_epoch):
feed_dict = fill_feed_dict(data_set, examples_placeholder,
labels_placeholder)
corrects, y_prediction, y_trues = sess.run([eval_correct, predictions,\
labels], feed_dict=feed_dict)
true_count += corrects
y_pred += list(y_prediction)
y_true += list(y_trues)
accuracy = true_count / num_examples
print(title +
' - Num examples: %d Num correct: %d Accuracy_score @ 1: %0.08f' %
(num_examples, true_count, accuracy))
# print("True output:", y_true)
# print("Pred output:", y_pred)
print("Precision:")
print("\tNone: ",
precision_score(y_true, y_pred, average=None, pos_label=None))
# print("\tBinary:", precision_score(y_true, y_pred, average='binary'))
print("\tMicro: %0.08f" %
precision_score(y_true, y_pred, average='micro', pos_label=None))
print("\tMacro: %0.08f" %
precision_score(y_true, y_pred, average='macro', pos_label=None))
print("\tWeighted: %0.08f" %
precision_score(y_true, y_pred, average='weighted', pos_label=None))
# print("\tSamples:", sklearn.metrics.precision_score(y_true, y_pred, average='samples'))
# print("\tAccuracy_score: %0.08f" % accuracy_score(y_true, y_pred))
print("Recall:")
# print("\tNone: ", recall_score(y_true, y_pred, average=None, pos_label=None))
# print("\tBinary:", recall_score(y_true, y_pred, average='binary'))
print("\tMicro: %0.08f" %
recall_score(y_true, y_pred, average='micro', pos_label=None))
print("\tMacro: %0.08f" %
recall_score(y_true, y_pred, average='macro', pos_label=None))
print("\tWeighted: %0.08f" %
recall_score(y_true, y_pred, average='weighted', pos_label=None))
# print("\tSamples:", sklearn.metrics.recall_score(y_true, y_pred, average='samples'))
# print("F1_score:")
# print("\tNone: ", f1_score(y_true, y_pred, average=None, pos_label=None))
# print("\tBinary:", f1_score(y_true, y_pred, average='binary'))
# print("\tMicro: %0.08f" % f1_score(y_true, y_pred, average='micro', pos_label=None))
# print("\tMacro: %0.08f" % f1_score(y_true, y_pred, average='macro', pos_label=None))
print("\nF1 Score (weighted): %0.08f" %
f1_score(y_true, y_pred, average='weighted', pos_label=None))
# print("\tSamples:", sklearn.metrics.f1_score(y_true, y_pred, average='samples'))
# print("True Length:", len(y_true))
# print("Prediction Length:", len(y_pred))
cm = confusion_matrix(y_true, y_pred)
# print("\nConfusion Matrix")
# print(cm)
cm_normalized = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
# print("\nNormalized confusion_matrix")
# print(cm_normalized)
print("")
print(classification_report(y_true, y_pred, target_names=label_map))
pcm(cm, target_names=label_map, title=title)
pcm(cm_normalized, target_names=label_map, title=title + "_Normalized")
roc(y_pred, y_true, n_classes=len(label_map), title=title)
print(
"\n=====================================================================================================\n"
)
def train(dataset, log_dir):
with tf.Graph().as_default():
# gets placeholders for patches and labels
patches_pl, labels_pl = utils.placeholder_inputs()
# build train related ops
net = models.FCN(patches_pl, FLAGS.dropout)
net.build_loss(labels_pl)
net.build_train(FLAGS.learning_rate)
# builds validation inference graph
val_net = models.FCN(patches_pl, training=False, reuse=True)
# add summary to plot loss, f score, tdr and fdr
f_score_pl = tf.placeholder(tf.float32, shape=())
tdr_pl = tf.placeholder(tf.float32, shape=())
fdr_pl = tf.placeholder(tf.float32, shape=())
scores_summary_op = tf.summary.merge([
tf.summary.scalar('f_score', f_score_pl),
tf.summary.scalar('tdr', tdr_pl),
tf.summary.scalar('fdr', fdr_pl)
])
loss_summary_op = tf.summary.scalar('loss', net.loss)
# add variable initialization to graph
init = tf.global_variables_initializer()
# early stopping vars
best_f_score = 0
faults = 0
saver = tf.train.Saver()
with tf.Session() as sess:
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
sess.run(init)
for step in range(1, FLAGS.steps + 1):
feed_dict = utils.fill_feed_dict(dataset.train, patches_pl, labels_pl,
FLAGS.batch_size)
_, loss_value = sess.run([net.train, net.loss], feed_dict=feed_dict)
# write loss summary periodically
if step % 100 == 0:
print('Step {}: loss = {}'.format(step, loss_value))
summary_str = sess.run(loss_summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
# evaluate the model periodically
if step % 1000 == 0:
print('Evaluation:')
f_score, fdr, tdr = validate.by_patches(sess, val_net.predictions,
FLAGS.batch_size, patches_pl,
labels_pl, dataset.val)
print('TDR = {}'.format(tdr))
print('FDR = {}'.format(fdr))
print('F score = {}'.format(f_score))
# early stopping
if f_score > best_f_score:
best_f_score = f_score
saver.save(
sess, os.path.join(log_dir, 'model.ckpt'), global_step=step)
faults = 0
else:
faults += 1
if faults >= FLAGS.tolerance:
print('Training stopped early')
break
# write f score, tdr and fdr to summary
scores_summary = sess.run(
scores_summary_op,
feed_dict={
f_score_pl: f_score,
tdr_pl: tdr,
fdr_pl: fdr
})
summary_writer.add_summary(scores_summary, global_step=step)
print('Finished')
print('best F score = {}'.format(best_f_score))
def train(dataset, log_dir):
with tf.Graph().as_default():
# gets placeholders for images and labels
images_pl, labels_pl = utils.placeholder_inputs()
# build net graph
net = description.Net(images_pl, FLAGS.dropout)
# build training related ops
net.build_loss(labels_pl, FLAGS.weight_decay)
net.build_train(FLAGS.learning_rate)
# builds validation graph
val_net = description.Net(images_pl, training=False, reuse=True)
# add summary to plot loss and rank
eer_pl = tf.placeholder(tf.float32, shape=(), name='eer_pl')
loss_pl = tf.placeholder(tf.float32, shape=(), name='loss_pl')
eer_summary_op = tf.summary.scalar('eer', eer_pl)
loss_summary_op = tf.summary.scalar('loss', loss_pl)
# early stopping vars
best_eer = 1
faults = 0
saver = tf.train.Saver()
with tf.Session() as sess:
# initialize summary and variables
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
sess.run(tf.global_variables_initializer())
# 'compute_descriptors' function for validation
compute_descriptors = lambda img, pts: utils.trained_descriptors(
img,
pts,
patch_size=dataset.train.images_shape[1],
session=sess,
imgs_pl=images_pl,
descs_op=val_net.descriptors)
# train loop
for step in range(1, FLAGS.steps + 1):
# fill feed dict
feed_dict = utils.fill_feed_dict(dataset.train, images_pl,
labels_pl, FLAGS.batch_size,
FLAGS.augment)
# train step
loss_value, _ = sess.run([net.loss, net.train],
feed_dict=feed_dict)
# write loss summary periodically
if step % 100 == 0:
print('Step {}: loss = {}'.format(step, loss_value))
# summarize loss
loss_summary = sess.run(loss_summary_op,
feed_dict={loss_pl: loss_value})
summary_writer.add_summary(loss_summary, step)
# evaluate model periodically
if step % 500 == 0 and dataset.val is not None:
print('Validation:')
eer = validate.matching.validation_eer(
dataset.val, compute_descriptors)
print('EER = {}'.format(eer))
# summarize eer
eer_summary = sess.run(eer_summary_op,
feed_dict={eer_pl: eer})
summary_writer.add_summary(eer_summary, global_step=step)
# early stopping
if eer < best_eer:
# update early stopping vars
best_eer = eer
faults = 0
saver.save(sess,
os.path.join(log_dir, 'model.ckpt'),
global_step=step)
else:
faults += 1
if faults >= FLAGS.tolerance:
print('Training stopped early')
break
# if no validation set, save model when training completes
if dataset.val is None:
saver.save(sess, os.path.join(log_dir, 'model.ckpt'))
print('Finished')
print('best EER = {}'.format(best_eer))
def by_patches(sess, preds, batch_size, patches_pl, labels_pl, dataset):
'''
Computes detection parameters that optimize the patch-based keypoint
detection F-score in the dataset with a grid search. This is done
efficiently, by sorting probability scores and iterating over them.
Args:
sess: tf session with loaded preds variables.
preds: tf op for detection probability prediction.
batch_size: size of mini-batch.
patches_pl: patch input placeholder for preds op.
labels_pl: label input placeholder to retrieve labels from
tf input feed dict.
dataset: dataset to perform grid-search on.
Returns:
best_f_score: value of best found F-score.
best_fdr: corresponding value of False Detection Rate.
best_tdr: corresponding value of True Detection Rate.
'''
# initialize dataset statistics
true_preds = []
false_preds = []
total = 0
steps_per_epoch = (dataset.num_samples + batch_size - 1) // batch_size
for _ in range(steps_per_epoch):
feed_dict = utils.fill_feed_dict(dataset, patches_pl, labels_pl,
batch_size)
# evaluate batch
batch_preds = sess.run(preds, feed_dict=feed_dict)
batch_labels = feed_dict[labels_pl]
batch_total = np.sum(batch_labels)
# update dataset statistics
total += batch_total
if batch_total > 0:
true_preds.extend(batch_preds[batch_labels == 1].flatten())
if batch_total < batch_labels.shape[0]:
false_preds.extend(batch_preds[batch_labels == 0].flatten())
# sort for efficient computation of tdr/fdr over thresholds
true_preds.sort()
true_preds.reverse()
false_preds.sort()
false_preds.reverse()
# compute tdr/fdr score over thresholds
best_f_score = 0
best_fdr = None
best_tdr = None
true_pointer = 0
false_pointer = 0
eps = 1e-5
thrs = np.arange(1.01, -0.01, -0.01)
for thr in thrs:
# compute true positives
while true_pointer < len(
true_preds) and true_preds[true_pointer] >= thr:
true_pointer += 1
# compute false positives
while false_pointer < len(
false_preds) and false_preds[false_pointer] >= thr:
false_pointer += 1
# compute tdr and fdr
tdr = true_pointer / (total + eps)
fdr = false_pointer / (true_pointer + false_pointer + eps)
# compute and update f score
f_score = 2 * (tdr * (1 - fdr)) / (tdr + 1 - fdr)
if f_score > best_f_score:
best_tdr = tdr
best_fdr = fdr
best_f_score = f_score
return best_f_score, best_fdr, best_tdr
def run_testing():
"""Test BinaryConnect."""
# Get the sets of images and labels for training, validation, and
# test on CIFAR10.
data_sets = cifar10.read_data_sets(dst_dir='./dataset',
validation_size=5000)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# Generate placeholders for the images and labels.
images_placeholder, labels_placeholder, train_placeholder = placeholder_inputs(
FLAGS.batch_size)
# Build a Graph that computes predictions from the inference model.
logits = bc.inference_bin(images_placeholder, train_placeholder,
stochastic=FLAGS.stochastic,
use_bnorm=True) \
if FLAGS.binary \
else bc.inference_ref(images_placeholder, train_placeholder,
use_bnorm=True)
# Add the Op to compare the logits to the labels during evaluation.
eval_metric = bc.evaluation(logits, labels_placeholder)
# Add the variable initializer Op.
ivars = tf.global_variables() + tf.local_variables()
init = tf.variables_initializer(ivars)
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Load trained model.
saver = tf.train.Saver()
saver.restore(sess, FLAGS.model_path)
print("Model loaded.")
# And then after everything is built:
# Run the Op to initialize the variables.
sess.run(init)
# Start the training loop
duration = 0
tp_value_total = 0
for step in xrange(FLAGS.max_steps):
start_time = time.time()
# Fill a feed dictionary with the actual set of images and labels
# for this particular testing step.
feed_dict = fill_feed_dict(data_sets.test, images_placeholder,
labels_placeholder, train_placeholder,
False)
# Run one step of the model.
acc_val = sess.run(eval_metric, feed_dict=feed_dict)
duration += time.time() - start_time
tp_value_total += acc_val
# Print an overview
if step % 100 == 0:
# Print status to stdout.
images_freq = 100 * FLAGS.batch_size / duration
print('Step %d: correct = %.2f%% (%.3f images/sec)' %
(step, tp_value_total / step, images_freq))
duration = time.time() - start_time
duration = 0