def _train(path_to_train_tfrecords_file, num_train_examples,
path_to_val_tfrecords_file, num_val_examples, path_to_train_log_dir,
path_to_restore_checkpoint_file, training_options):
batch_size = training_options['batch_size']
initial_patience = training_options['patience']
num_steps_to_show_loss = 100
num_steps_to_check = 1000
with tf.Graph().as_default():
image_batch, pieces_batch = Donkey.build_batch(
path_to_train_tfrecords_file,
num_examples=num_train_examples,
batch_size=batch_size,
shuffled=True)
print pieces_batch
pieces_logits = Model.inference(image_batch, drop_rate=0.2)
loss = Model.loss(pieces_logits, pieces_batch)
global_step = tf.Variable(0, name='global_step', trainable=False)
learning_rate = tf.train.exponential_decay(
training_options['learning_rate'],
global_step=global_step,
decay_steps=training_options['decay_steps'],
decay_rate=training_options['decay_rate'],
staircase=True)
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
train_op = optimizer.minimize(loss, global_step=global_step)
tf.summary.image('image', image_batch)
tf.summary.scalar('loss', loss)
tf.summary.scalar('learning_rate', learning_rate)
summary = tf.summary.merge_all()
"""
def evaluate(self, path_to_checkpoint, path_to_tfrecords_file,
num_examples, global_step):
batch_size = 128
num_batches = num_examples / batch_size
with tf.Graph().as_default():
image_batch, digits_batch, letters_batch = Donkey.build_batch(
path_to_tfrecords_file,
num_examples=num_examples,
batch_size=batch_size,
shuffled=False)
digits_logits, letters_logits = Model.inference(image_batch,
drop_rate=0.0)
digits_predictions = tf.argmax(digits_logits, axis=2)
letters_predictions = tf.argmax(letters_logits, axis=2)
labels = tf.concat([digits_batch, letters_batch], axis=1)
predictions = tf.concat([digits_predictions, letters_predictions],
axis=1)
labels_string = tf.reduce_join(tf.as_string(labels), axis=1)
predictions_string = tf.reduce_join(tf.as_string(predictions),
axis=1)
accuracy, update_accuracy = tf.metrics.accuracy(
labels=labels_string, predictions=predictions_string)
tf.summary.image('image', image_batch)
tf.summary.scalar('accuracy', accuracy)
tf.summary.histogram(
'variables',
tf.concat([
tf.reshape(var, [-1]) for var in tf.trainable_variables()
],
axis=0))
summary = tf.summary.merge_all()
with tf.Session() as sess:
sess.run([
tf.global_variables_initializer(),
tf.local_variables_initializer()
])
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
restorer = tf.train.Saver()
restorer.restore(sess, path_to_checkpoint)
for _ in xrange(num_batches):
sess.run(update_accuracy)
accuracy_val, summary_val = sess.run([accuracy, summary])
self.summary_writer.add_summary(summary_val,
global_step=global_step)
coord.request_stop()
coord.join(threads)
return accuracy_val
def evaluate(self, path_to_checkpoint, path_to_tfrecords_file, num_examples, global_step):
batch_size = 128
num_batches = num_examples // batch_size
needs_include_length = False
with tf.Graph().as_default():
image_batch, lable_batch = Donkey.build_batch(path_to_tfrecords_file,
num_examples=num_examples,
batch_size=batch_size,
shuffled=False)
label_logits = Model.inference(image_batch, drop_rate=0.0)
label_predictions1 = tf.argmax(label_logits[:,0], axis=1)
label_predictions2 = tf.argmax(label_logits[:,1], axis=1)
label_predictions3 = tf.argmax(label_logits[:,2], axis=1)
accuracy1, update_accuracy1 = tf.metrics.accuracy(
labels=lable_batch,
predictions=label_predictions1
)
accuracy2, update_accuracy2 = tf.metrics.accuracy(
labels=lable_batch,
predictions=label_predictions2
)
accuracy3, update_accuracy3 = tf.metrics.accuracy(
labels=lable_batch,
predictions=label_predictions3
)
accuracy = accuracy1 + accuracy2 + accuracy3
update_accuracy = update_accuracy1 + update_accuracy2 + update_accuracy3
tf.summary.image('image', image_batch)
tf.summary.scalar('accuracy', accuracy)
tf.summary.histogram('variables',
tf.concat([tf.reshape(var, [-1]) for var in tf.trainable_variables()], axis=0))
summary = tf.summary.merge_all()
with tf.Session() as sess:
sess.run([tf.global_variables_initializer(), tf.local_variables_initializer()])
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
restorer = tf.train.Saver()
restorer.restore(sess, path_to_checkpoint)
for _ in range(num_batches):
sess.run(update_accuracy)
accuracy_val, summary_val = sess.run([accuracy, summary])
self.summary_writer.add_summary(summary_val, global_step=global_step)
coord.request_stop()
coord.join(threads)
return accuracy_val
def evaluate(self, path_to_checkpoint, path_to_tfrecords_file,
num_examples, global_step):
batch_size = 128
num_batches = num_examples / batch_size
with tf.Graph().as_default():
images, labels = Donkey.build_batch(path_to_tfrecords_file,
batch_size=batch_size,
one_hot=False,
shuffled=False)
logits = Model.inference(images, keep_prob=1)
predictions = tf.argmax(logits, axis=1)
accuracy, update_accuracy = tf.metrics.accuracy(
labels=labels, predictions=predictions)
tf.summary.image('image', images)
tf.summary.scalar('accuracy', accuracy)
summary = tf.summary.merge_all()
with tf.Session() as sess:
sess.run([
tf.global_variables_initializer(),
tf.local_variables_initializer()
])
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
restorer = tf.train.Saver()
restorer.restore(sess, path_to_checkpoint)
for _ in xrange(num_batches):
sess.run(update_accuracy)
accuracy_val, summary_val = sess.run([accuracy, summary])
self.summary_writer.add_summary(summary_val,
global_step=global_step)
coord.request_stop()
coord.join(threads)
return accuracy_val
def _train(path_to_train_tfrecords_file, num_train_examples,
path_to_val_tfrecords_file, num_val_examples, path_to_train_log_dir,
path_to_restore_checkpoint_file, training_options):
batch_size = training_options['batch_size']
initial_patience = training_options['patience']
num_steps_to_show_loss = 100
num_steps_to_check = 1000
with tf.Graph().as_default():
image_batch, length_batch, digits_batch = Donkey.build_batch(
path_to_train_tfrecords_file,
num_examples=num_train_examples,
batch_size=batch_size,
shuffled=True)
length_logtis, digits_logits = Model.inference(image_batch,
drop_rate=0.2)
loss = Model.loss(length_logtis, digits_logits, length_batch,
digits_batch)
global_step = tf.Variable(0, name='global_step', trainable=False)
learning_rate = tf.train.exponential_decay(
training_options['learning_rate'],
global_step=global_step,
decay_steps=training_options['decay_steps'],
decay_rate=training_options['decay_rate'],
staircase=True)
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
train_op = optimizer.minimize(loss, global_step=global_step)
tf.summary.image('image', image_batch)
tf.summary.scalar('loss', loss)
tf.summary.scalar('learning_rate', learning_rate)
summary = tf.summary.merge_all()
with tf.Session() as sess:
summary_writer = tf.summary.FileWriter(path_to_train_log_dir,
sess.graph)
evaluator = Evaluator(
os.path.join(path_to_train_log_dir, 'eval/val'))
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
saver = tf.train.Saver()
if path_to_restore_checkpoint_file is not None:
assert tf.train.checkpoint_exists(path_to_restore_checkpoint_file), \
'%s not found' % path_to_restore_checkpoint_file
saver.restore(sess, path_to_restore_checkpoint_file)
print 'Model restored from file: %s' % path_to_restore_checkpoint_file
print 'Start training'
patience = initial_patience
best_accuracy = 0.0
duration = 0.0
while True:
start_time = time.time()
_, loss_val, summary_val, global_step_val, learning_rate_val = sess.run(
[train_op, loss, summary, global_step, learning_rate])
duration += time.time() - start_time
if global_step_val % num_steps_to_show_loss == 0:
examples_per_sec = batch_size * num_steps_to_show_loss / duration
duration = 0.0
print '=> %s: step %d, loss = %f (%.1f examples/sec)' % (
datetime.now(), global_step_val, loss_val,
examples_per_sec)
if global_step_val % num_steps_to_check != 0:
continue
summary_writer.add_summary(summary_val,
global_step=global_step_val)
print '=> Evaluating on validation dataset...'
path_to_latest_checkpoint_file = saver.save(
sess, os.path.join(path_to_train_log_dir, 'latest.ckpt'))
accuracy = evaluator.evaluate(path_to_latest_checkpoint_file,
path_to_val_tfrecords_file,
num_val_examples,
global_step_val)
print '==> accuracy = %f, best accuracy %f' % (accuracy,
best_accuracy)
if accuracy > best_accuracy:
path_to_checkpoint_file = saver.save(
sess,
os.path.join(path_to_train_log_dir, 'model.ckpt'),
global_step=global_step_val)
print '=> Model saved to file: %s' % path_to_checkpoint_file
patience = initial_patience
best_accuracy = accuracy
else:
patience -= 1
print '=> patience = %d' % patience
if patience == 0:
break
coord.request_stop()
coord.join(threads)
print 'Finished'
import tensorflow as tf
import sys
sys.path.append('.')
from donkey import Donkey
import matplotlib.pyplot as plt
import numpy as np
image_batch, length_batch, digits_batch = Donkey.build_batch(r'E:\DataSets\SVHN\train.tfrecords',
num_example=100,
batch_size=36,
shuffled=False)
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
image_batch_val, length_batch_val, digits_batch_val = sess.run([image_batch, length_batch, digits_batch])
image_batch_val = (image_batch_val / 2.0) + 0.5
print(np.array(image_batch_val[0]).shape)
fig, axes = plt.subplots(6, 6, figsize=(20, 20))
for i, ax in enumerate(axes.flat):
title = 'length: %d\ndigits= %d, %d, %d, %d, %d' % (length_batch_val[i],
digits_batch_val[i][0],
digits_batch_val[i][1],
digits_batch_val[i][2],
digits_batch_val[i][3],
digits_batch_val[i][4])
ax.imshow(image_batch_val[i])
ax.set_title(title)
ax.set_xticks([])
ax.set_yticks([])
def _train(path_to_train_tfrecords_file, num_train_examples,
path_to_val_tfrecords_file, num_val_examples, path_to_train_log_dir,
path_to_restore_checkpoint_file, training_options):
batch_size = training_options['batch_size']
initial_patience = training_options['patience']
num_steps_to_show_loss = 1
num_steps_to_check = 1
# num_steps_to_show_loss = 10
# num_steps_to_check = 100
# cwd = './img_confirm/'
with tf.Graph().as_default():
X = tf.placeholder("float", [None, 28, 28, 1])
Y = tf.placeholder("float", [None, 11])
w = init_weights([3, 3, 1, 32]) # 3x3x1 conv, 32 outputs
w2 = init_weights([3, 3, 32, 64]) # 3x3x32 conv, 64 outputs
w3 = init_weights([3, 3, 64, 128]) # 3x3x32 conv, 128 outputs
w4 = init_weights([128 * 4 * 4,
625]) # FC 128 * 4 * 4 inputs, 625 outputs
w_o = init_weights([625, 11]) # FC 625 inputs, 11 outputs (labels)
global_step = tf.Variable(0, name='global_step', trainable=False)
p_keep_conv = tf.placeholder("float")
p_keep_hidden = tf.placeholder("float")
py_x = model(X, w, w2, w3, w4, w_o, p_keep_conv, p_keep_hidden)
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=py_x, labels=Y))
train_op = tf.train.RMSPropOptimizer(0.001, 0.9).minimize(
loss, global_step=global_step)
predict_op = tf.argmax(py_x, 1)
print num_train_examples
trX, trY = Donkey.build_batch(path_to_train_tfrecords_file,
num_examples=num_train_examples,
batch_size=batch_size,
shuffled=True,
num_epoch_=training_options['num_epoch'])
teX, teY = Donkey.build_batch(path_to_val_tfrecords_file,
num_examples=num_val_examples,
batch_size=64,
shuffled=True,
num_epoch_=training_options['num_epoch'])
# teX,teY = Donkey.build_batch_from_file(path_to_val_file,batch_size=10,
# shuffled=True,num_epoch_=training_options['num_epoch'])
indices = tf.placeholder("uint8", [batch_size, 1])
var = tf.one_hot(indices, depth=11, axis=1)
tf.summary.image('image', trX)
tf.summary.scalar('loss', loss)
summary = tf.summary.merge_all()
#
#
#
# tr_x_split = tf.unstack(trX,32,axis=0)
#
# image_batch,length_batch,digits_batch = Donkey.build_batch(path_to_train_tfrecords_file1,path_to_train_tfrecords_file2,path_to_train_tfrecords_file3,path_to_train_tfrecords_file4,
# num_examples=num_train_examples,
# batch_size=batch_size,
# shuffled=True,num_epoch_=training_options['num_epoch'])
#
# trX = trX.reshape(-1, 28, 28, 1) # 28x28x1 input img
# teX = teX.reshape(-1, 28, 28, 1) # 28x28x1 input img
with tf.Session() as sess:
summary_writer = tf.summary.FileWriter(path_to_train_log_dir,
sess.graph)
# you need to initialize all variables
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
saver = tf.train.Saver()
if path_to_restore_checkpoint_file is not None:
assert tf.train.checkpoint_exists(path_to_restore_checkpoint_file), \
'%s not found' % path_to_restore_checkpoint_file
saver.restore(sess, path_to_restore_checkpoint_file)
print 'Model restored from file: %s' % path_to_restore_checkpoint_file
print 'Start training'
# print trX.shape
for i in range(20000):
# plt.imshow(trX.eval())
# plt.show()
# single,l = sess.run([trX,trY])
# single.resize(28,28)
# img=Image.fromarray(single, 'L')
# img.save(cwd+str(i)+'_''Label_'+str(l)+'.jpg')
#print(single,l)
# tr_x,tr_y,tr_x_split = sess.run([trX,trY,tr_x_split])
# print tr_x.shape
# tr_x = tr_x.reshape(-1, 28, 28, 1) # 28x28x1 input img
## print tr_x.shape,tr_x_split
#
#
# for j,ele in enumerate(tr_x_split):
## print ele.shape
## ele.reshape(28,28)
## print ele.shape
# ele.resize(28,28)
# img = Image.fromarray(ele,'L')
# img.save(cwd+str(j)+'_''Label_'+str(tr_y[j])+'.jpg')
## print ele,ele.shape,tr_y[j]
#
tr_x, tr_y = sess.run([trX, trY])
tr_y.resize(batch_size, 1)
tr_y = sess.run(var, feed_dict={indices: tr_y})
tr_y.resize(batch_size, 11)
# tr_y = tr_y.reshape(-1,11)
_, loss_val, summary_val, global_step_val = sess.run(
[train_op, loss, summary, global_step],
feed_dict={
X: tr_x,
Y: tr_y,
p_keep_conv: 0.8,
p_keep_hidden: 0.5
})
te_x, te_y = sess.run([teX, teY])
te_x = te_x.reshape(-1, 28, 28, 1) # 28x28x1 input img
accuracy = np.mean(te_y == sess.run(predict_op,
feed_dict={
X: te_x,
p_keep_conv: 1.0,
p_keep_hidden: 1.0
}))
print(i, accuracy)
summary_writer.add_summary(summary_val,
global_step=global_step_val)
path_to_latest_checkpoint_file = saver.save(
sess, os.path.join(path_to_train_log_dir, 'latest.ckpt'))
coord.request_stop()
coord.join(threads)
print 'Finished'
def evaluate(self, path_to_checkpoint, image_eval, length_eval,
digits_eval, global_step):
batch_size = 128
needs_include_length = False
accuracy_val = 0.0
with tf.Graph().as_default():
image_batch = tf.placeholder(tf.float32,
shape=[
None, self.image_size,
self.image_size, self.num_channels
])
length_batch = tf.placeholder(tf.int32, shape=[None])
digits_batch = tf.placeholder(tf.int32,
shape=[None, self.digits_nums])
num_examples = image_eval.shape[0]
num_batches = num_examples / batch_size
# length_logits, digits_logits = Model.inference(image_batch, drop_rate=0.0)
length_logits, digits_logits = Model.forward(image_batch, 1.0)
length_predictions = tf.argmax(length_logits, axis=1)
digits_predictions = tf.argmax(digits_logits, axis=2)
if needs_include_length:
labels = tf.concat(
[tf.reshape(length_batch, [-1, 1]), digits_batch], axis=1)
predictions = tf.concat([
tf.reshape(length_predictions, [-1, 1]), digits_predictions
],
axis=1)
else:
labels = digits_batch
predictions = digits_predictions
#correct_pre = tf.equal(tf.argmax(labels,axis=1),tf.argmax(predictions,axis=1))
#accuracy = tf.reduce_mean(tf.cast(correct_pre, tf.float32))
labels_string = tf.reduce_join(tf.as_string(labels), axis=1)
predictions_string = tf.reduce_join(tf.as_string(predictions),
axis=1)
correct_pre = tf.equal(labels_string, predictions_string)
accuracy = tf.reduce_mean(tf.cast(correct_pre, tf.float32))
tf.summary.image('image', image_batch)
tf.summary.scalar('accuracy', accuracy)
tf.summary.histogram(
'variables',
tf.concat([
tf.reshape(var, [-1]) for var in tf.trainable_variables()
],
axis=0))
summary = tf.summary.merge_all()
with tf.Session() as sess:
sess.run([
tf.global_variables_initializer(),
tf.local_variables_initializer()
])
restorer = tf.train.Saver()
restorer.restore(sess, path_to_checkpoint)
for _ in range(math.floor(num_examples / batch_size)):
image_batch_input, length_batch_input, digits_batch_input = Donkey.build_batch(
image_eval,
length_eval,
digits_eval,
batch_size=batch_size)
feed_dict = {
image_batch: image_batch_input,
length_batch: length_batch_input,
digits_batch: digits_batch_input
}
accuracy_step, summary_val = sess.run([accuracy, summary],
feed_dict=feed_dict)
accuracy_val += accuracy_step
self.summary_writer.add_summary(summary_val,
global_step=global_step)
accuracy_val = accuracy_val / math.floor(
num_examples / batch_size)
return accuracy_val * 100
def evaluate(self, path_to_restore_model_checkpoint_file,
path_to_restore_defender_checkpoint_file,
path_to_tfrecords_file, num_examples, global_step,
defend_layer, attacker_type):
batch_size = 32
num_batches = num_examples // batch_size
needs_include_length = False
with tf.Graph().as_default():
image_batch, length_batch, digits_batch = Donkey.build_batch(
path_to_tfrecords_file,
num_examples=num_examples,
batch_size=batch_size,
shuffled=False)
with tf.variable_scope('model'):
length_logits, digits_logits, hidden_out = Model.inference(
image_batch,
drop_rate=0.0,
is_training=False,
defend_layer=defend_layer)
with tf.variable_scope('defender'):
recovered = Attacker.recover_hidden(attacker_type,
hidden_out,
is_training=False,
defend_layer=defend_layer)
ssim = tf.reduce_mean(
tf.abs(tf.image.ssim(image_batch, recovered, max_val=2)))
length_predictions = tf.argmax(length_logits, axis=1)
digits_predictions = tf.argmax(digits_logits, axis=2)
if needs_include_length:
labels = tf.concat(
[tf.reshape(length_batch, [-1, 1]), digits_batch], axis=1)
predictions = tf.concat([
tf.reshape(length_predictions, [-1, 1]), digits_predictions
],
axis=1)
else:
labels = digits_batch
predictions = digits_predictions
labels_string = tf.reduce_join(tf.as_string(labels), axis=1)
predictions_string = tf.reduce_join(tf.as_string(predictions),
axis=1)
accuracy, update_accuracy = tf.metrics.accuracy(
labels=labels_string, predictions=predictions_string)
tf.summary.image('image', image_batch, max_outputs=20)
tf.summary.image('recovered', recovered, max_outputs=20)
tf.summary.scalar('ssim', ssim)
tf.summary.scalar('accuracy', accuracy)
tf.summary.histogram(
'variables',
tf.concat([
tf.reshape(var, [-1]) for var in tf.trainable_variables()
],
axis=0))
summary = tf.summary.merge_all()
with tf.Session() as sess:
sess.run([
tf.global_variables_initializer(),
tf.local_variables_initializer()
])
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
model_saver = tf.train.Saver(var_list=tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, scope='model'))
defender_saver = tf.train.Saver(var_list=tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, scope='defender'))
model_saver.restore(sess,
path_to_restore_model_checkpoint_file)
print("Evaluation model restored from {}".format(
path_to_restore_model_checkpoint_file))
defender_saver.restore(
sess, path_to_restore_defender_checkpoint_file)
for _ in range(num_batches):
sess.run(update_accuracy)
accuracy_val, summary_val = sess.run([accuracy, summary])
self.summary_writer.add_summary(summary_val,
global_step=global_step)
coord.request_stop()
coord.join(threads)
return accuracy_val
def _train(path_to_train_tfrecords_file, num_train_examples,
path_to_val_tfrecords_file, num_val_examples, path_to_train_log_dir,
path_to_restore_model_checkpoint_file, training_options):
batch_size = training_options['batch_size']
initial_patience = training_options['patience']
num_steps_to_show_loss = 100
num_steps_to_check = 1000
with tf.Graph().as_default():
image_batch, length_batch, digits_batch = Donkey.build_batch(
path_to_train_tfrecords_file,
num_examples=num_train_examples,
batch_size=batch_size,
shuffled=True)
with tf.variable_scope('model'):
length_logtis, digits_logits, hidden_out = Model.inference(
image_batch,
drop_rate=0.0,
is_training=True,
defend_layer=FLAGS.defend_layer)
with tf.variable_scope('defender'):
recovered = Attacker.recover_hidden(FLAGS.attacker_type,
hidden_out, True,
FLAGS.defend_layer)
ssim = tf.reduce_mean(
tf.abs(tf.image.ssim(image_batch, recovered, max_val=2)))
model_loss = Model.loss(length_logtis, digits_logits, length_batch,
digits_batch)
# loss = model_loss + FLAGS.ssim_weight * ssim
defender_loss = -ssim
global_step = tf.Variable(0, name='global_step', trainable=False)
learning_rate = tf.train.exponential_decay(
training_options['learning_rate'],
global_step=global_step,
decay_steps=training_options['decay_steps'],
decay_rate=training_options['decay_rate'],
staircase=True)
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
# model_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope='model')
# with tf.control_dependencies(model_update_ops):
# train_op = optimizer.minimize(loss, var_list=tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,scope='model'), global_step=global_step)
defender_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
scope='defender')
with tf.control_dependencies(defender_update_ops):
defender_op = optimizer.minimize(
defender_loss,
var_list=tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='defender'),
global_step=global_step)
tf.summary.image('image', image_batch, max_outputs=20)
tf.summary.image('recovered', recovered, max_outputs=20)
tf.summary.scalar('model_loss', model_loss)
tf.summary.scalar('ssim', ssim)
tf.summary.scalar('learning_rate', learning_rate)
summary = tf.summary.merge_all()
with tf.Session() as sess:
summary_writer = tf.summary.FileWriter(path_to_train_log_dir,
sess.graph)
evaluator = Evaluator(
os.path.join(path_to_train_log_dir, 'eval/val'))
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
model_saver = tf.train.Saver(var_list=tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, scope='model'))
defender_saver = tf.train.Saver(var_list=tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, scope='defender'))
model_saver.restore(sess, path_to_restore_model_checkpoint_file)
print('Model restored from file: %s' %
path_to_restore_model_checkpoint_file)
# if path_to_restore_checkpoint_file is not None:
# assert tf.train.checkpoint_exists(path_to_restore_checkpoint_file), \
# '%s not found' % path_to_restore_checkpoint_file
# saver.restore(sess, path_to_restore_checkpoint_file)
# print('Model restored from file: %s' % path_to_restore_checkpoint_file)
print('Start training')
patience = initial_patience
best_accuracy = 0.0
duration = 0.0
while True:
start_time = time.time()
_, defender_loss_val, summary_val, global_step_val, learning_rate_val = sess.run(
[
defender_op, defender_loss, summary, global_step,
learning_rate
])
duration += time.time() - start_time
# print("image: {} - {}".format(image_batch_val.min(), image_batch_val.max()))
if global_step_val % num_steps_to_show_loss == 0:
examples_per_sec = batch_size * num_steps_to_show_loss / duration
duration = 0.0
print('=> %s: step %d, loss = %f (%.1f examples/sec)' %
(datetime.now(), global_step_val, defender_loss_val,
examples_per_sec))
if global_step_val % num_steps_to_check != 0:
continue
summary_writer.add_summary(summary_val,
global_step=global_step_val)
print('=> Evaluating on validation dataset...')
path_to_latest_attacker_checkpoint_file = defender_saver.save(
sess, os.path.join(path_to_train_log_dir, 'attacker.ckpt'))
accuracy = evaluator.evaluate(
path_to_restore_model_checkpoint_file,
path_to_latest_attacker_checkpoint_file,
path_to_val_tfrecords_file, num_val_examples,
global_step_val, FLAGS.defend_layer, FLAGS.attacker_type)
print('==> accuracy = %f, best accuracy %f' %
(accuracy, best_accuracy))
if accuracy > best_accuracy:
defender_saver.save(
sess,
os.path.join(path_to_train_log_dir,
'attacker_best.ckpt'))
patience = initial_patience
best_accuracy = accuracy
else:
patience -= 1
print('=> patience = %d' % patience)
# if patience == 0:
# break
if global_step_val > FLAGS.max_steps:
break
coord.request_stop()
coord.join(threads)
print('Finished')
def _train(path_to_train_tfrecords_file, num_train_examples, path_to_val_tfrecords_file, num_val_examples,
path_to_train_log_dir, path_to_restore_checkpoint_file, training_options):
batch_size = training_options['batch_size']
initial_patience = training_options['patience']
num_steps_to_show_loss = 100
num_steps_to_check = 1000
with tf.Graph().as_default():
image_batch, length_batch, digits_batch = Donkey.build_batch(path_to_train_tfrecords_file,
num_examples=num_train_examples,
batch_size=batch_size,
shuffled=True)
length_logtis, digits_logits = Model.inference(image_batch, drop_rate=0.2)
loss = Model.loss(length_logtis, digits_logits, length_batch, digits_batch)
global_step = tf.Variable(0, name='global_step', trainable=False)
learning_rate = tf.train.exponential_decay(training_options['learning_rate'], global_step=global_step,
decay_steps=training_options['decay_steps'], decay_rate=training_options['decay_rate'], staircase=True)
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
train_op = optimizer.minimize(loss, global_step=global_step)
tf.summary.image('image', image_batch)
tf.summary.scalar('loss', loss)
tf.summary.scalar('learning_rate', learning_rate)
summary = tf.summary.merge_all()
with tf.Session() as sess:
summary_writer = tf.summary.FileWriter(path_to_train_log_dir, sess.graph)
evaluator = Evaluator(os.path.join(path_to_train_log_dir, 'eval/val'))
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
saver = tf.train.Saver()
if path_to_restore_checkpoint_file is not None:
assert tf.train.checkpoint_exists(path_to_restore_checkpoint_file), \
'%s not found' % path_to_restore_checkpoint_file
saver.restore(sess, path_to_restore_checkpoint_file)
print 'Model restored from file: %s' % path_to_restore_checkpoint_file
print 'Start training'
patience = initial_patience
best_accuracy = 0.0
duration = 0.0
while True:
start_time = time.time()
_, loss_val, summary_val, global_step_val, learning_rate_val = sess.run([train_op, loss, summary, global_step, learning_rate])
duration += time.time() - start_time
if global_step_val % num_steps_to_show_loss == 0:
examples_per_sec = batch_size * num_steps_to_show_loss / duration
duration = 0.0
print '=> %s: step %d, loss = %f (%.1f examples/sec)' % (
datetime.now(), global_step_val, loss_val, examples_per_sec)
if global_step_val % num_steps_to_check != 0:
continue
summary_writer.add_summary(summary_val, global_step=global_step_val)
print '=> Evaluating on validation dataset...'
path_to_latest_checkpoint_file = saver.save(sess, os.path.join(path_to_train_log_dir, 'latest.ckpt'))
accuracy = evaluator.evaluate(path_to_latest_checkpoint_file, path_to_val_tfrecords_file,
num_val_examples,
global_step_val)
print '==> accuracy = %f, best accuracy %f' % (accuracy, best_accuracy)
if accuracy > best_accuracy:
path_to_checkpoint_file = saver.save(sess, os.path.join(path_to_train_log_dir, 'model.ckpt'),
global_step=global_step_val)
print '=> Model saved to file: %s' % path_to_checkpoint_file
patience = initial_patience
best_accuracy = accuracy
else:
patience -= 1
print '=> patience = %d' % patience
if patience == 0:
break
coord.request_stop()
coord.join(threads)
print 'Finished'
def evaluate(self, path_to_checkpoint, path_to_tfrecords_file,
num_examples, global_step, save_file):
batch_size = 128
num_batches = num_examples // batch_size
needs_include_length = False
with tf.Graph().as_default():
id_batch, image_batch, length_batch, digits_batch = Donkey.build_batch(
path_to_tfrecords_file,
num_example=num_examples,
batch_size=batch_size,
shuffled=False)
length_logits, digits_logits = Model.inference(image_batch,
drop_rate=0.0)
length_predictions = tf.argmax(length_logits, axis=1)
digits_predictions = tf.argmax(digits_logits, axis=2)
if needs_include_length:
labels = tf.concat(
[tf.reshape(length_batch, [-1, 1]), digits_batch], axis=1)
predictions = tf.concat([
tf.reshape(length_predictions, [-1, 1]), digits_predictions
],
axis=1)
else:
labels = digits_batch
predictions = digits_predictions
labels_string = tf.reduce_join(tf.as_string(labels), axis=1)
predictions_string = tf.reduce_join(tf.as_string(predictions),
axis=1)
accuracy, update_accuracy = tf.metrics.accuracy(
labels=labels_string, predictions=predictions_string)
tf.summary.image('image', image_batch)
tf.summary.scalar('accuracy', accuracy)
tf.summary.histogram(
'variables',
tf.concat([
tf.reshape(var, [-1]) for var in tf.trainable_variables()
],
axis=0))
summary = tf.summary.merge_all()
ids_total = []
labels_total = []
predictions_total = []
with tf.Session() as sess:
sess.run([
tf.global_variables_initializer(),
tf.local_variables_initializer()
])
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
restorer = tf.train.Saver()
restorer.restore(sess, path_to_checkpoint)
for k in range(num_batches):
sess.run(update_accuracy)
ids, labels, predictions = sess.run(
[id_batch, labels_string, predictions_string])
accuracy_val, summary_val = sess.run([accuracy, summary])
self.summary_writer.add_summary(summary_val,
global_step=global_step)
ids_total.extend(ids)
labels_total.extend(labels)
predictions_total.extend(predictions)
coord.request_stop()
coord.join(threads)
ids_total = np.array(ids_total)
labels_total = np.array(labels_total)
predictions_total = np.array(predictions_total)
print(ids_total.shape, labels_total.shape, predictions_total.shape)
with open(save_file, 'w') as f:
f.write('ID\tprediction\tlabel\n')
for i in range(len(labels_total)):
f.write(
str(ids_total[i]) + '\t' + str(predictions_total[i]) +
'\t' + str(labels_total[i]) + '\n')
return accuracy_val
def train(path_to_train_h5py_file, path_to_val_h5py_file,
path_to_train_log_dir, path_to_restore_checkpoint_file, training_options):
image_train,length_train,digits_train=read_h5py_file(path_to_train_h5py_file,Flag=1)
image_val,length_val,digits_val=read_h5py_file(path_to_val_h5py_file,Flag=2)
batch_size = training_options['batch_size']
initial_patience = training_options['patience']
num_steps_to_show_loss = 100
num_steps_to_check = 1000
image_size=54
num_channels=3
digits_nums=5
with tf.Graph().as_default():
image_batch=tf.placeholder(
tf.float32, shape=[None, image_size, image_size, num_channels])
length_batch=tf.placeholder(tf.int32,shape=[None])
digits_batch=tf.placeholder(tf.int32,shape=[None,digits_nums])
# length_logtis, digits_logits = Model.inference(image_batch, drop_rate=0.2)
length_logtis, digits_logits = Model.forward(image_batch, 0.8)
loss = Model.loss(length_logtis, digits_logits, length_batch, digits_batch)
global_step = tf.Variable(0, name='global_step', trainable=False)
learning_rate = tf.train.exponential_decay(training_options['learning_rate'], global_step=global_step,
decay_steps=training_options['decay_steps'], decay_rate=training_options['decay_rate'], staircase=True)
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
train_op = optimizer.minimize(loss, global_step=global_step)
tf.summary.image('image', image_batch)
tf.summary.scalar('loss', loss)
tf.summary.scalar('learning_rate', learning_rate)
summary = tf.summary.merge_all()
with tf.Session() as sess:
summary_writer = tf.summary.FileWriter(path_to_train_log_dir, sess.graph)
evaluator = Evaluator(os.path.join(path_to_train_log_dir, 'eval/val'))
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
if path_to_restore_checkpoint_file is not None:
assert tf.train.checkpoint_exists(path_to_restore_checkpoint_file), \
'%s not found' % path_to_restore_checkpoint_file
saver.restore(sess, path_to_restore_checkpoint_file)
print('Model restored from file: %s' % path_to_restore_checkpoint_file)
print('Start training')
patience = initial_patience
best_accuracy = 0.0
duration = 0.0
while True:
start_time = time.time()
image_batch_input, length_batch_input, digits_batch_input= Donkey.build_batch(image_train,length_train,digits_train,
batch_size=batch_size)
feed_dict={image_batch:image_batch_input,length_batch:length_batch_input,digits_batch:digits_batch_input}
_, loss_val, summary_val, global_step_val, learning_rate_val = sess.run([train_op, loss, summary, global_step, learning_rate],feed_dict=feed_dict)
duration += time.time() - start_time
if global_step_val % num_steps_to_show_loss == 0:
examples_per_sec = batch_size * num_steps_to_show_loss / duration
duration = 0.0
print ('=> %s: step %d, loss = %f (%.1f examples/sec)' % (
datetime.now(), global_step_val, loss_val, examples_per_sec))
if global_step_val % num_steps_to_check != 0:
continue
summary_writer.add_summary(summary_val, global_step=global_step_val)
print ('=> Evaluating on validation dataset...')
path_to_latest_checkpoint_file = saver.save(sess, os.path.join(path_to_train_log_dir, 'latest.ckpt'))
accuracy = evaluator.evaluate(path_to_latest_checkpoint_file, image_val,length_val,digits_val,
global_step_val)
print ('==> accuracy = %f, best accuracy %f' % (accuracy, best_accuracy))
if accuracy > best_accuracy:
path_to_checkpoint_file = saver.save(sess, os.path.join(path_to_train_log_dir, 'model.ckpt'),
global_step=global_step_val)
print ('=> Model saved to file: %s' % path_to_checkpoint_file)
patience = initial_patience
best_accuracy = accuracy
else:
patience -= 1
print ('=> patience = %d' % patience)
if patience == 0:
break
print ('Finished')
