def optimize_towers(optimizer, towers, clip_norm=None, **kwargs):
""" Create towers for the passed in devices """
all_tower_losses = []
all_tower_grads_and_vars = []
num_towers = len(towers)
regularization_losses = losses.get_regularization_losses()
for tower in towers:
with tf.device(tower.device):
with tf.name_scope(tower.scope):
# Scale based on number of towers
tower_losses = losses.get_losses(tower.scope)
total_tower_loss = tf.divide(tf.add_n(tower_losses),
num_towers,
name='total_loss')
all_tower_losses.append(total_tower_loss)
if regularization_losses:
# Regularization losses are only calculated when the associated variable is
# created, not when it is reused, so only add the regularization losses once on
# the device of the first tower
regularization_loss = tf.add_n(regularization_losses,
'regularization_loss')
all_tower_losses.append(regularization_loss)
regularization_losses = None
total_tower_loss += regularization_loss
grads_and_vars = optimizer.compute_gradients(
total_tower_loss, **kwargs)
if clip_norm:
grads_and_vars = [
(tf.clip_by_norm(gradients, clip_norm), variable)
for (gradients, variable) in grads_and_vars
if gradients is not None
]
all_tower_grads_and_vars.append(grads_and_vars)
grads_and_vars = []
for grads_and_var in zip(*all_tower_grads_and_vars):
# grads_and_var should be the gradients of each tower for the same variable
variable = grads_and_var[0][1]
gradients_list = [
gradients for gradients, _ in grads_and_var
if gradients is not None
]
if gradients_list:
gradients = tf.add_n(gradients_list,
name='{0}/gradient/sum'.format(
variable.op.name))
grads_and_vars.append((gradients, variable))
return all_tower_losses, grads_and_vars
def main(_):
os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.95
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
# config.log_device_placement = True
if not tf.gfile.Exists(FLAGS.data_dir):
raise RuntimeError('data direction is not exist!')
# if tf.gfile.Exists(FLAGS.log_dir):
# tf.gfile.DeleteRecursively(FLAGS.log_dir)
tf.gfile.MakeDirs(FLAGS.log_dir)
# if not tf.gfile.Exists(FLAGS.ckpt_dir):
tf.gfile.MakeDirs(os.path.join(FLAGS.ckpt_dir, 'best'))
f = open(FLAGS.out_file, 'a')
if not f:
raise RuntimeError('OUTPUT FILE OPEN ERROR!!!!!!')
with tf.device('/cpu:0'):
num_gpus = len(FLAGS.gpu.split(','))
global_step = tf.Variable(FLAGS.start_step,
name='global_step',
trainable=False)
# learning_rate = tf.train.exponential_decay(0.05, global_step, 2000, 0.9, staircase=True)
learning_rate = tf.train.exponential_decay(0.1,
global_step,
1000,
0.95,
staircase=True)
# learning_rate = tf.train.piecewise_constant(global_step, [24000, 48000, 72000, 108000, 144000],
# [0.1, 0.01, 0.001, 0.0001, 0.00001, 0.000001])
tf.summary.scalar('learing rate', learning_rate)
# opt = tf.train.AdamOptimizer(learning_rate)
opt = tf.train.MomentumOptimizer(learning_rate,
momentum=FLAGS.momentum)
# opt = tf.train.GradientDescentOptimizer(learning_rate)
# learning_rate = tf.train.exponential_decay(0.01, global_step, 32000, 0.1)
# opt = tf.train.GradientDescentOptimizer(learning_rate)
tower_grads = []
tower_loss = []
tower_acc = []
tower_acc_v = []
images, labels = input_pipeline(
tf.train.match_filenames_once(
os.path.join(FLAGS.data_dir, 'train', '*.tfrecords')),
FLAGS.batch_size)
batch_queue = tf.contrib.slim.prefetch_queue.prefetch_queue(
[images, labels], capacity=2 * num_gpus)
images_v, labels_v = input_pipeline(
tf.train.match_filenames_once(
os.path.join(FLAGS.data_dir, 'valid', '*.tfrecords')),
128 // num_gpus)
batch_queue_v = tf.contrib.slim.prefetch_queue.prefetch_queue(
[images_v, labels_v], capacity=2 * num_gpus)
for i in range(num_gpus):
with tf.device('/gpu:%d' % i):
with tf.name_scope('tower_%d' % i) as scope:
image_batch, label_batch = batch_queue.dequeue()
logits = build.net(image_batch, is_training, FLAGS)
losses.sparse_softmax_cross_entropy(labels=label_batch,
logits=logits,
scope=scope)
total_loss = losses.get_losses(
scope=scope) + losses.get_regularization_losses(
scope=scope)
total_loss = tf.add_n(total_loss)
grads = opt.compute_gradients(total_loss)
tower_grads.append(grads)
tower_loss.append(losses.get_losses(scope=scope))
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(
tf.reshape(tf.argmax(logits, 1), [-1, 1]),
tf.cast(label_batch, tf.int64))
accuracy = tf.reduce_mean(
tf.cast(correct_prediction, tf.float32))
tower_acc.append(accuracy)
tf.get_variable_scope().reuse_variables()
image_batch_v, label_batch_v = batch_queue_v.dequeue()
logits_v = build.net(image_batch_v, False, FLAGS)
correct_prediction = tf.equal(
tf.reshape(tf.argmax(logits_v, 1), [-1, 1]),
tf.cast(label_batch_v, tf.int64))
accuracy = tf.reduce_mean(
tf.cast(correct_prediction, tf.float32))
tower_acc_v.append(accuracy)
with tf.name_scope('scores'):
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(tf.stack(tower_acc, axis=0))
with tf.name_scope('accuracy_v'):
accuracy_v = tf.reduce_mean(tf.stack(tower_acc_v, axis=0))
with tf.name_scope('batch_loss'):
batch_loss = tf.add_n(tower_loss)[0] / num_gpus
tf.summary.scalar('loss', batch_loss)
tf.summary.scalar('accuracy', accuracy)
grads = average_gradients(tower_grads)
variable_averages = tf.train.ExponentialMovingAverage(
0.9999, global_step)
variables_averages_op = variable_averages.apply(
tf.trainable_variables())
with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
apply_gradient_op = opt.apply_gradients(grads,
global_step=global_step)
train_op = tf.group(apply_gradient_op, variables_averages_op)
# train_op = apply_gradient_op
# summary_op = tf.summary.merge_all()
# init = tf.global_variables_initializer()
summary_op = tf.summary.merge_all()
saver = tf.train.Saver(name="saver", max_to_keep=10)
saver_best = tf.train.Saver(name='best', max_to_keep=100)
with tf.Session(config=config) as sess:
sess.run(tf.local_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
if tf.gfile.Exists(os.path.join(FLAGS.ckpt_dir, 'checkpoint')):
saver.restore(sess, tf.train.latest_checkpoint(FLAGS.ckpt_dir))
else:
sess.run(tf.global_variables_initializer())
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train',
sess.graph)
train_writer.flush()
cache = np.ones(5, dtype=np.float32) / FLAGS.num_classes
cache_v = np.ones(5, dtype=np.float32) / FLAGS.num_classes
d = 1000
best = 0
for i in range(FLAGS.start_step, FLAGS.max_steps + 1):
# feed = feed_dict(True, True)
if i % d == 0: # Record summaries and test-set accuracy
# loss0 = sess.run([total_loss], feed_dict=feed_dict(False, False))
# test_writer.add_summary(summary, i)
# feed[is_training] = FLAGS
acc, loss, summ, lr, acc_v = sess.run(
[
accuracy, batch_loss, summary_op, learning_rate,
accuracy_v
],
feed_dict={is_training: False})
cache[int(i / d) % 5] = acc
cache_v[int(i / d) % 5] = acc_v
train_writer.add_summary(summ, i)
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
file=f)
print(
'step %d: acc(t)=%f(%f), loss=%f; acc(v)=%f(%f); lr=%e'
% (i, acc, cache.mean(), loss, acc_v, cache_v.mean(),
lr),
file=f)
saver.save(sess,
os.path.join(FLAGS.ckpt_dir, FLAGS.model_name),
global_step=i)
if acc_v > 0.90:
saver_best.save(sess,
os.path.join(FLAGS.ckpt_dir, 'best',
FLAGS.model_name),
global_step=i)
f.flush()
sess.run(train_op, feed_dict={is_training: True})
coord.request_stop()
coord.join(threads)
train_writer.close()
# test_writer.close()
f.close()
def evaluate_model(config):
""" Train the model using the passed in config """
###########################################################
# Create the input pipeline
###########################################################
with tf.name_scope('input_pipeline'):
dataset = input_utils.get_dataset(config.datadir, config.dataset,
config.datasubset)
init_op, init_feed_dict, image = input_utils.get_data(
config.dataset,
dataset,
config.batch_size,
num_epochs=config.num_epochs,
num_readers=config.num_readers)
images = tf.train.batch([image],
config.batch_size,
num_threads=config.num_preprocessing_threads,
capacity=5 * config.batch_size)
###########################################################
# Generate the model
###########################################################
outputs = create_model(config, images, dataset)
###########################################################
# Setup the evaluation metrics and summaries
###########################################################
summaries = []
metrics_map = {}
for loss in losses.get_losses():
metrics_map[loss.op.name] = metrics.streaming_mean(loss)
for metric in tf.get_collection(graph_utils.GraphKeys.METRICS):
metrics_map[metric.op.name] = metrics.streaming_mean(metric)
total_loss = losses.get_total_loss()
metrics_map[total_loss.op.name] = metrics.streaming_mean(total_loss)
names_to_values, names_to_updates = metrics.aggregate_metric_map(
metrics_map)
# Create summaries of the metrics and print them to the screen
for name, value in names_to_values.iteritems():
summary = tf.summary.scalar(name, value, collections=[])
summaries.append(tf.Print(summary, [value], name))
summaries.extend(layers.summarize_collection(tf.GraphKeys.MODEL_VARIABLES))
summaries.extend(layers.summarize_collection(
graph_utils.GraphKeys.METRICS))
summaries.extend(
layers.summarize_collection(graph_utils.GraphKeys.RNN_OUTPUTS))
summaries.extend(
layers.summarize_collection(graph_utils.GraphKeys.TRAINING_PARAMETERS))
images = input_utils.reshape_images(images, config.dataset)
tiled_images = image_utils.tile_images(images)
summaries.append(tf.summary.image('input_batch', tiled_images))
# Generate the canvases that lead to the final output image
with tf.name_scope('canvases'):
for step, canvas in enumerate(outputs):
canvas = input_utils.reshape_images(canvas, config.dataset)
tiled_images = image_utils.tile_images(canvas)
summaries.append(
tf.summary.image('step{0}'.format(step), tiled_images))
summary_op = tf.summary.merge(summaries, name='summaries')
###########################################################
# Begin evaluation
###########################################################
checkpoint_path = FLAGS.checkpoint_path
eval_ops = tf.group(*names_to_updates.values())
scaffold = tf.train.Scaffold(init_op, init_feed_dict)
hooks = [
training.SummaryAtEndHook(FLAGS.log_dir, summary_op),
training.StopAfterNEvalsHook(
math.ceil(dataset.num_samples / float(config.batch_size)))
]
eval_kwargs = {}
eval_fn = training.evaluate_repeatedly
if FLAGS.once:
if tf.gfile.IsDirectory(checkpoint_path):
checkpoint_path = tf.train.latest_checkpoint(checkpoint_path)
eval_fn = training.evaluate_once
else:
assert tf.gfile.IsDirectory(checkpoint_path), (
'checkpoint path must be a directory when using loop evaluation')
# On Tensorflow master fd87896 fixes this, but for now just set a very large number
eval_kwargs['max_number_of_evaluations'] = sys.maxint
eval_fn(checkpoint_path,
scaffold=scaffold,
hooks=hooks,
eval_ops=eval_ops,
**eval_kwargs)
def main(_):
os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.95
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
# config.log_device_placement = True
if not tf.gfile.Exists(FLAGS.data_dir):
raise RuntimeError('data direction is not exist!')
# if tf.gfile.Exists(FLAGS.log_dir):
# tf.gfile.DeleteRecursively(FLAGS.log_dir)
tf.gfile.MakeDirs(FLAGS.log_dir)
# if not tf.gfile.Exists(FLAGS.ckpt_dir):
tf.gfile.MakeDirs(os.path.join(FLAGS.ckpt_dir, 'best'))
f = open(FLAGS.out_file + '.txt',
'a' if FLAGS.start_step is not 0 else 'w')
if not f:
raise RuntimeError('OUTPUT FILE OPEN ERROR!!!!!!')
with tf.device('/cpu:0'):
num_gpus = len(FLAGS.gpu.split(','))
global_step = tf.Variable(FLAGS.start_step,
name='global_step',
trainable=False)
# learning_rate = tf.train.piecewise_constant(global_step,
# [500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000],
# [0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0])
# step_size = 10000
# learning_rate = tf.train.exponential_decay(1.0, global_step, 2*step_size, 0.5, staircase=True)
# cycle = tf.floor(1 + tf.cast(global_step, tf.float32) / step_size / 2.)
# xx = tf.abs(tf.cast(global_step, tf.float32)/step_size - 2. * tf.cast(cycle, tf.float32) + 1.)
# learning_rate = 1e-4 + (1e-1 - 1e-4) * tf.maximum(0., (1-xx))*learning_rate
# learning_rate = tf.train.piecewise_constant(global_step, [10000, 70000, 120000, 170000, 220000],
# [0.01, 0.1, 0.001, 0.0001, 0.00001, 0.000001])
# learning_rate = tf.constant(0.001)
learning_rate = tf.train.exponential_decay(0.05,
global_step,
30000,
0.1,
staircase=True)
print(
'learning_rate = tf.train.exponential_decay(0.05, global_step, 30000, 0.1, staircase=True)',
file=f)
# opt = tf.train.AdamOptimizer(learning_rate)
opt = tf.train.MomentumOptimizer(learning_rate, momentum=0.9)
# opt = tf.train.GradientDescentOptimizer(learning_rate)
# learning_rate = tf.train.exponential_decay(0.01, global_step, 32000, 0.1)
# opt = tf.train.GradientDescentOptimizer(learning_rate)
print('opt = tf.train.MomentumOptimizer(learning_rate, momentum=0.9)',
file=f)
print('weight decay = %e' % FLAGS.weight_decay, file=f)
f.flush()
tf.summary.scalar('learing rate', learning_rate)
tower_grads = []
tower_loss = []
tower_acc = []
images_t, labels_t = input_pipeline(
tf.train.match_filenames_once(
os.path.join(FLAGS.data_dir, 'train', '*.tfrecords')),
FLAGS.batch_size * num_gpus,
read_threads=len(os.listdir(os.path.join(FLAGS.data_dir,
'train'))))
# batch_queue = tf.contrib.slim.prefetch_queue.prefetch_queue(
# [images, labels], capacity=2 * num_gpus)
images_v, labels_v = input_pipeline(
tf.train.match_filenames_once(
os.path.join(FLAGS.data_dir, 'valid',
'*.tfrecords')), (256 // num_gpus) * num_gpus,
read_threads=len(os.listdir(os.path.join(FLAGS.data_dir,
'valid'))),
if_train=False)
# batch_queue_v = tf.contrib.slim.prefetch_queue.prefetch_queue(
# [images_v, labels_v], capacity=2 * num_gpus)
image_batch0 = tf.placeholder(
tf.float32, [None, FLAGS.patch_size, FLAGS.patch_size, channels],
'imgs')
label_batch0 = tf.placeholder(tf.int32, [None, 1], 'labels')
image_batch = tf.split(image_batch0, num_gpus, 0)
label_batch = tf.split(label_batch0, num_gpus, 0)
for i in range(num_gpus):
with tf.device('/gpu:%d' % i):
with tf.name_scope('tower_%d' % i) as scope:
logits = build.net(image_batch[i], is_training, FLAGS)
losses.sparse_softmax_cross_entropy(labels=label_batch[i],
logits=logits,
scope=scope)
total_loss = losses.get_losses(
scope=scope) + losses.get_regularization_losses(
scope=scope)
total_loss = tf.add_n(total_loss)
grads = opt.compute_gradients(total_loss)
tower_grads.append(grads)
tower_loss.append(losses.get_losses(scope=scope))
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(
tf.reshape(tf.argmax(logits, 1), [-1, 1]),
tf.cast(label_batch[i], tf.int64))
accuracy = tf.reduce_mean(
tf.cast(correct_prediction, tf.float32))
tower_acc.append(accuracy)
tf.get_variable_scope().reuse_variables()
with tf.name_scope('scores'):
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(tf.stack(tower_acc, axis=0))
with tf.name_scope('batch_loss'):
batch_loss = tf.add_n(tower_loss)[0] / num_gpus
tf.summary.scalar('loss', batch_loss)
tf.summary.scalar('accuracy', accuracy)
grads = average_gradients(tower_grads)
with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
variable_averages = tf.train.ExponentialMovingAverage(
0.9999, global_step)
variables_averages_op = variable_averages.apply(
tf.trainable_variables())
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
apply_gradient_op = opt.apply_gradients(
grads, global_step=global_step)
train_op = tf.group(apply_gradient_op, variables_averages_op)
p_relu_update = tf.get_collection('p_relu')
# train_op = apply_gradient_op
# summary_op = tf.summary.merge_all()
# init = tf.global_variables_initializer()
summary_op = tf.summary.merge_all()
saver = tf.train.Saver(name="saver", max_to_keep=10)
saver_best = tf.train.Saver(name='best', max_to_keep=200)
with tf.Session(config=config) as sess:
sess.run(tf.local_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
if tf.gfile.Exists(os.path.join(FLAGS.ckpt_dir, 'checkpoint')):
saver.restore(sess, tf.train.latest_checkpoint(FLAGS.ckpt_dir))
else:
sess.run(tf.global_variables_initializer())
if FLAGS.start_step != 0:
sess.run(tf.assign(global_step, FLAGS.start_step))
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train',
sess.graph)
train_writer.flush()
valid_writer = tf.summary.FileWriter(FLAGS.log_dir + '/valid',
sess.graph)
valid_writer.flush()
cache = np.ones(5, dtype=np.float32) / FLAGS.num_classes
cache_v = np.ones(5, dtype=np.float32) / FLAGS.num_classes
d = 1000
best = 0
for i in range(FLAGS.start_step, FLAGS.max_steps + 1):
def get_batch(set, on_training):
if set == 'train':
img, lb = sess.run([images_t, labels_t])
# x = np.random.randint(0, 64)
# y = np.random.randint(0, 64)
# img = np.roll(np.roll(img, x, 1), y, 2)
elif set == 'valid':
img, lb = sess.run([images_v, labels_v])
else:
raise RuntimeError('Unknown set name')
feed_dict = {}
feed_dict[image_batch0] = img
feed_dict[label_batch0] = lb
feed_dict[is_training] = on_training
return feed_dict
# feed = feed_dict(True, True)
if i % d == 0: # Record summaries and test-set accuracy
# loss0 = sess.run([total_loss], feed_dict=feed_dict(False, False))
# test_writer.add_summary(summary, i)
# feed[is_training] = FLAGS
acc, loss, summ, lr = sess.run(
[accuracy, batch_loss, summary_op, learning_rate],
feed_dict=get_batch('train', False))
acc2 = sess.run(accuracy,
feed_dict=get_batch('train', True))
cache[int(i / d) % 5] = acc
acc_v, loss_v, summ_v = sess.run(
[accuracy, batch_loss, summary_op],
feed_dict=get_batch('valid', False))
acc2_v = sess.run(accuracy,
feed_dict=get_batch('valid', True))
cache_v[int(i / d) % 5] = acc_v
train_writer.add_summary(summ, i)
valid_writer.add_summary(summ_v, i)
print(('step %d, ' % i) +
time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
file=f)
print(
'acc(t)=%f(%f), loss(t)=%f;\nacc(v)=%f(%f), loss(v)=%f; lr=%e'
% (acc, cache.mean(), loss, acc_v, cache_v.mean(),
loss_v, lr),
file=f)
print('%f, %f' % (acc2, acc2_v), file=f)
saver.save(sess,
os.path.join(FLAGS.ckpt_dir, FLAGS.model_name),
global_step=i)
if acc_v > 0.90:
saver_best.save(sess,
os.path.join(FLAGS.ckpt_dir, 'best',
FLAGS.model_name),
global_step=i)
f.flush()
sess.run(train_op, feed_dict=get_batch('train', True))
sess.run(p_relu_update)
coord.request_stop()
coord.join(threads)
train_writer.close()
# test_writer.close()
f.close()
def main(_):
os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.7
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
if not tf.gfile.Exists(FLAGS.data_dir):
raise RuntimeError('data direction is not exist!')
# if tf.gfile.Exists(FLAGS.log_dir):
# tf.gfile.DeleteRecursively(FLAGS.log_dir)
# tf.gfile.MakeDirs(FLAGS.log_dir)
if not tf.gfile.Exists(FLAGS.ckpt_dir):
tf.gfile.MakeDirs(FLAGS.ckpt_dir)
f = open(FLAGS.out_file, 'w')
if not f:
raise RuntimeError('OUTPUT FILE OPEN ERROR!!!!!!')
with tf.device('/cpu:0'):
global_step = tf.Variable(FLAGS.start_step,
name='global_step',
trainable=False)
learning_rate = tf.train.piecewise_constant(global_step,
[24000, 48000],
[0.1, 0.01, 0.001])
opt = tf.train.MomentumOptimizer(learning_rate,
momentum=FLAGS.momentum)
# learning_rate = tf.train.exponential_decay(0.01, global_step, 32000, 0.1)
# opt = tf.train.GradientDescentOptimizer(learning_rate)
tower_grads = []
num_gpus = len(FLAGS.gpu.split(','))
tower_images = []
tower_labels = []
tower_loss = []
for i in range(num_gpus):
with tf.device('/gpu:%d' % i):
with tf.name_scope('tower_%d' % i) as scope:
# with tf.name_scope('input'):
# images = tf.placeholder(tf.float32, [None, FLAGS.patch_size, FLAGS.patch_size, 3], 'images')
# tf.summary.image('show', images, 1)
#
# with tf.name_scope('label'):
# labels = tf.placeholder(tf.int64, [None, 1], 'y')
images, labels = input_pipeline(
tf.train.match_filenames_once(
os.path.join(FLAGS.data_dir, 'train', '*.tfrecords')),
FLAGS.batch_size)
tower_images.append(images)
tower_labels.append(labels)
logits = build.net(images, FLAGS, is_training,
FLAGS.num_classes)
losses.sparse_softmax_cross_entropy(logits,
labels,
scope=scope)
total_loss = losses.get_losses(
scope=scope) + losses.get_regularization_losses(
scope=scope)
total_loss = tf.add_n(total_loss)
grads = opt.compute_gradients(total_loss)
tower_grads.append(grads)
tower_loss.append(losses.get_losses(scope=scope))
grads = average_gradients(tower_grads)
total_loss = tf.add_n(tower_loss)
# variable_averages = tf.train.ExponentialMovingAverage(
# cifar10.MOVING_AVERAGE_DECAY, global_step)
# variables_averages_op = variable_averages.apply(tf.trainable_variables())
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)
# train_op = tf.group(apply_gradient_op, variables_averages_op)
train_op = apply_gradient_op
# summary_op = tf.summary.merge_all()
# init = tf.global_variables_initializer()
saver = tf.train.Saver(name="saver")
with tf.Session(config=config) as sess:
sess.run(tf.local_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
if tf.gfile.Exists(os.path.join(FLAGS.ckpt_dir, 'checkpoint')):
saver.restore(sess, os.path.join(FLAGS.ckpt_dir, FLAGS.model_name))
else:
sess.run(tf.global_variables_initializer())
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train',
sess.graph)
# test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test', sess.graph)
train_writer.flush()
# test_writer.flush()
# def feed_dict(train, on_training):
# def get_batch(data, labels):
# d, l = sess.run([data, labels])
# d = d.astype(np.float32)
# l = l.astype(np.int64)
# return d, l
# res = {}
# if train:
# for j in range(num_gpus):
# xs, ys = get_batch(train_example_batch, train_label_batch)
# res[tower_images[j]] = xs
# res[tower_labels[j]] = ys
# else:
# for j in range(num_gpus):
# xs, ys = get_batch(valid_example_batch, valid_label_batch)
# res[tower_images[j]] = xs
# res[tower_labels[j]] = ys
# return res
for i in range(FLAGS.start_step, FLAGS.max_steps + 1):
# feed = feed_dict(True, True)
sess.run(train_op, feed_dict={is_training: True})
if i % 10 == 0 and i != 0: # Record summaries and test-set accuracy
# loss0 = sess.run([total_loss], feed_dict=feed_dict(False, False))
# test_writer.add_summary(summary, i)
# feed[is_training] = FLAGS
loss1 = sess.run(total_loss, feed_dict={is_training: False})
# train_writer.add_summary(summary, i)
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
file=f)
# print('step %d: train_acc=%f, train_loss=%f; test_acc=%f, test_loss=%f' % (i, acc1, loss1, acc0, loss0),
# file=f)
print('step %d: train_loss=%f' % (i, loss1), file=f)
saver.save(sess, os.path.join(FLAGS.ckpt_dir,
FLAGS.model_name))
f.flush()
coord.request_stop()
coord.join(threads)
train_writer.close()
# test_writer.close()
f.close()
def main(_):
os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.7
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
# config.log_device_placement = True
if not tf.gfile.Exists(FLAGS.data_dir):
raise RuntimeError('data direction is not exist!')
# if tf.gfile.Exists(FLAGS.log_dir):
# tf.gfile.DeleteRecursively(FLAGS.log_dir)
# tf.gfile.MakeDirs(FLAGS.log_dir)
if not tf.gfile.Exists(FLAGS.ckpt_dir):
tf.gfile.MakeDirs(FLAGS.ckpt_dir)
f = open(FLAGS.out_file, 'w')
if not f:
raise RuntimeError('OUTPUT FILE OPEN ERROR!!!!!!')
with tf.device('/cpu:0'):
global_step = tf.Variable(FLAGS.start_step,
name='global_step',
trainable=False)
# learning_rate = tf.train.exponential_decay(0.1, global_step, 192000, 0.9, staircase=True)
# tf.summary.scalar('learing rate', learning_rate)
# opt = tf.train.AdamOptimizer(learning_rate)
# opt = tf.train.MomentumOptimizer(learning_rate, momentum=FLAGS.momentum)
# opt = tf.train.GradientDescentOptimizer(learning_rate)
# learning_rate = tf.train.exponential_decay(0.01, global_step, 32000, 0.1)
# opt = tf.train.GradientDescentOptimizer(learning_rate)
# tower_grads = []
num_gpus = len(FLAGS.gpu.split(','))
tower_loss = []
tower_acc = []
images, labels = input_pipeline(
tf.train.match_filenames_once(
os.path.join(FLAGS.data_dir, 'valid', '*.tfrecords')),
int(FLAGS.batch_size / num_gpus))
image_batch = tf.placeholder(
tf.float32, [None, FLAGS.patch_size, FLAGS.patch_size, 3], 'imgs')
label_batch = tf.placeholder(tf.int32, [None, 1], 'labels')
for i in range(num_gpus):
with tf.device('/gpu:%d' % i):
with tf.name_scope('tower_%d' % i) as scope:
# image_batch, label_batch = batch_queue.dequeue()
# image_batch = tf.ones(shape=[128, 64, 64, 3], dtype=tf.float32)
# label_batch = tf.ones(shape=[128, 1], dtype=tf.int32)
logits = build.net(image_batch, False, FLAGS)
losses.sparse_softmax_cross_entropy(labels=label_batch,
logits=logits,
scope=scope)
# total_loss = losses.get_losses(scope=scope) + losses.get_regularization_losses(scope=scope)
# total_loss = tf.add_n(total_loss)
# grads = opt.compute_gradients(total_loss)
# tower_grads.append(grads)
tower_loss.append(losses.get_losses(scope=scope))
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(
tf.reshape(tf.argmax(logits, 1), [-1, 1]),
tf.cast(label_batch, tf.int64))
accuracy = tf.reduce_mean(
tf.cast(correct_prediction, tf.float32))
tower_acc.append(accuracy)
tf.get_variable_scope().reuse_variables()
with tf.name_scope('scores'):
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(tf.stack(tower_acc, axis=0))
with tf.name_scope('batch_loss'):
batch_loss = tf.add_n(tower_loss)[0]
tf.summary.scalar('loss', batch_loss)
tf.summary.scalar('accuracy', accuracy)
# grads = average_gradients(tower_grads)
# variable_averages = tf.train.ExponentialMovingAverage(
# cifar10.MOVING_AVERAGE_DECAY, global_step)
# variables_averages_op = variable_averages.apply(tf.trainable_variables())
# with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
# update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
# with tf.control_dependencies(update_ops):
# apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)
# # train_op = tf.group(apply_gradient_op, variables_averages_op)
# train_op = apply_gradient_op
# summary_op = tf.summary.merge_all()
# init = tf.global_variables_initializer()
summary_op = tf.summary.merge_all()
# variable_averages = tf.train.ExponentialMovingAverage(0.9999)
# variables_to_restore = variable_averages.variables_to_restore()
# saver = tf.train.Saver(variables_to_restore, name='saver')
saver = tf.train.Saver(name="saver")
with tf.Session(config=config) as sess:
sess.run(tf.local_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
if tf.gfile.Exists(os.path.join(FLAGS.ckpt_dir, 'checkpoint')):
# saver.restore(sess, FLAGS.ckpt_dir+'/model')
saver.restore(
sess,
tf.train.latest_checkpoint(FLAGS.ckpt_dir)
if FLAGS.model_name is None else os.path.join(
FLAGS.ckpt_dir, FLAGS.model_name))
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test',
sess.graph)
train_writer.flush()
for i in range(FLAGS.start_step, FLAGS.max_steps + 1):
# feed = feed_dict(True, True)
# if i % 1000 == 0: # Record summaries and test-set accuracy
# loss0 = sess.run([total_loss], feed_dict=feed_dict(False, False))
# test_writer.add_summary(summary, i)
# feed[is_training] = FLAGS
img, lb = sess.run([images, labels])
acc, loss, summ = sess.run([accuracy, batch_loss, summary_op],
feed_dict={
image_batch: img,
label_batch: lb
})
# acc, loss, summ = sess.run([accuracy, batch_loss, summary_op], feed_dict={image_batch: np.ones(shape = [256, 64, 64, 3], dtype=np.float32), label_batch: np.ones(shape=[256, 1], dtype=np.int32)})
train_writer.add_summary(summ, i)
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
file=f)
# print('step %d: train_acc=%f, train_loss=%f; test_acc=%f, test_loss=%f' % (i, acc1, loss1, acc0, loss0),
# file=f)
print('step %d: accuracy=%f, loss=%f' % (i, acc, loss), file=f)
# saver.save(sess, os.path.join(FLAGS.ckpt_dir, FLAGS.model_name))
f.flush()
# sess.run(train_op, feed_dict={is_training: True})
coord.request_stop()
coord.join(threads)
train_writer.close()
# test_writer.close()
f.close()
def train_model(config):
""" Train the model using the passed in config """
training_devices = [
graph_utils.device_fn(device)
for device in graph_utils.collect_devices({'GPU': FLAGS.num_gpus})
]
assert training_devices, 'Found no training devices!'
###########################################################
# Create the input pipeline
###########################################################
with tf.device('/cpu:0'), tf.name_scope('input_pipeline'):
dataset = input_utils.get_dataset(config.datadir, config.dataset,
'train')
init_op, init_feed_dict, image = input_utils.get_data(
config.dataset,
dataset,
config.batch_size,
num_epochs=config.num_epochs,
num_readers=config.num_readers)
inputs_queue = input_utils.batch_images(
image,
config.batch_size,
num_threads=config.num_preprocessing_threads,
num_devices=len(training_devices))
###########################################################
# Generate the model
###########################################################
towers = graph_utils.create_towers(create_training_model, training_devices,
config, inputs_queue, dataset)
assert towers, 'No training towers were created!'
###########################################################
# Setup the training objectives
###########################################################
with tf.name_scope('training'):
with tf.device('/cpu:0'):
learning_rate_decay_step = config.learning_rate_decay_step / len(
towers)
learning_rate = tf.maximum(exponential_decay(
config.batch_size, learning_rate_decay_step,
config.learning_rate, config.learning_rate_decay, dataset),
config.learning_rate_min,
name='learning_rate')
tf.add_to_collection(graph_utils.GraphKeys.TRAINING_PARAMETERS,
learning_rate)
optimizer = tf.train.AdamOptimizer(learning_rate)
# Calculate gradients and total loss
tower_klds, tower_losses, grads_and_vars = graph_utils.optimize_towers(
optimizer, towers, clip_norm=config.clip)
total_kld = tf.add_n(tower_klds,
name='total_kld') if tower_klds else None
total_loss = tf.add_n(tower_losses, name='total_loss')
# Gather update ops from the first tower (for updating batch_norm for example)
global_step = framework.get_or_create_global_step()
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
towers[0].scope)
update_ops.append(
optimizer.apply_gradients(grads_and_vars, global_step=global_step))
update_op = tf.group(*update_ops)
with tf.control_dependencies([update_op]):
train_op = tf.identity(total_loss, name='train_op')
###########################################################
# Collect summaries
###########################################################
with tf.device('/cpu:0'):
summaries = []
summaries.extend(learning.add_gradients_summaries(grads_and_vars))
summaries.extend(
layers.summarize_collection(tf.GraphKeys.MODEL_VARIABLES))
summaries.extend(
layers.summarize_collection(graph_utils.GraphKeys.METRICS))
summaries.extend(
layers.summarize_collection(graph_utils.GraphKeys.RNN_OUTPUTS))
summaries.extend(
layers.summarize_collection(
graph_utils.GraphKeys.TRAINING_PARAMETERS))
images = input_utils.reshape_images(inputs_queue.dequeue(),
config.dataset)
tiled_images = image_utils.tile_images(images)
summaries.append(tf.summary.image('input_batch', tiled_images))
# Generate the canvases that lead to the final output image
with tf.name_scope('canvases'):
for step, canvas in enumerate(towers[0].outputs):
canvas = input_utils.reshape_images(canvas, config.dataset)
tiled_images = image_utils.tile_images(canvas)
summaries.append(
tf.summary.image('step{0}'.format(step), tiled_images))
with tf.name_scope('losses'):
if total_kld is not None:
summaries.append(tf.summary.scalar('total_kld', total_kld))
summaries.append(tf.summary.scalar('total_loss', total_loss))
for loss in tower_losses:
summaries.append(tf.summary.scalar(loss.op.name, loss))
for loss in losses.get_losses():
summaries.append(tf.summary.scalar(loss.op.name, loss))
summary_op = tf.summary.merge(summaries, name='summaries')
###########################################################
# Begin training
###########################################################
init_op = tf.group(tf.global_variables_initializer(), init_op)
session_config = tf.ConfigProto(
allow_soft_placement=False,
log_device_placement=FLAGS.log_device_placement)
prefetch_queue_buffer = 2 * len(training_devices)
number_of_steps = int(
int(dataset.num_samples / config.batch_size) / len(training_devices))
number_of_steps = number_of_steps * config.num_epochs - prefetch_queue_buffer
tf.logging.info('Running %s steps', number_of_steps)
learning.train(train_op,
FLAGS.log_dir,
session_config=session_config,
global_step=global_step,
number_of_steps=number_of_steps,
init_op=init_op,
init_feed_dict=init_feed_dict,
save_interval_secs=config.checkpoint_frequency,
summary_op=summary_op,
save_summaries_secs=config.summary_frequency,
trace_every_n_steps=config.trace_frequency
if config.trace_frequency > 0 else None)