def optimize_towers(optimizer, towers, clip_norm=None, **kwargs): """ Create towers for the passed in devices """ all_tower_losses = [] all_tower_grads_and_vars = [] num_towers = len(towers) regularization_losses = losses.get_regularization_losses() for tower in towers: with tf.device(tower.device): with tf.name_scope(tower.scope): # Scale based on number of towers tower_losses = losses.get_losses(tower.scope) total_tower_loss = tf.divide(tf.add_n(tower_losses), num_towers, name='total_loss') all_tower_losses.append(total_tower_loss) if regularization_losses: # Regularization losses are only calculated when the associated variable is # created, not when it is reused, so only add the regularization losses once on # the device of the first tower regularization_loss = tf.add_n(regularization_losses, 'regularization_loss') all_tower_losses.append(regularization_loss) regularization_losses = None total_tower_loss += regularization_loss grads_and_vars = optimizer.compute_gradients( total_tower_loss, **kwargs) if clip_norm: grads_and_vars = [ (tf.clip_by_norm(gradients, clip_norm), variable) for (gradients, variable) in grads_and_vars if gradients is not None ] all_tower_grads_and_vars.append(grads_and_vars) grads_and_vars = [] for grads_and_var in zip(*all_tower_grads_and_vars): # grads_and_var should be the gradients of each tower for the same variable variable = grads_and_var[0][1] gradients_list = [ gradients for gradients, _ in grads_and_var if gradients is not None ] if gradients_list: gradients = tf.add_n(gradients_list, name='{0}/gradient/sum'.format( variable.op.name)) grads_and_vars.append((gradients, variable)) return all_tower_losses, grads_and_vars
def main(_): os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.95 config.gpu_options.allow_growth = True config.allow_soft_placement = True # config.log_device_placement = True if not tf.gfile.Exists(FLAGS.data_dir): raise RuntimeError('data direction is not exist!') # if tf.gfile.Exists(FLAGS.log_dir): # tf.gfile.DeleteRecursively(FLAGS.log_dir) tf.gfile.MakeDirs(FLAGS.log_dir) # if not tf.gfile.Exists(FLAGS.ckpt_dir): tf.gfile.MakeDirs(os.path.join(FLAGS.ckpt_dir, 'best')) f = open(FLAGS.out_file, 'a') if not f: raise RuntimeError('OUTPUT FILE OPEN ERROR!!!!!!') with tf.device('/cpu:0'): num_gpus = len(FLAGS.gpu.split(',')) global_step = tf.Variable(FLAGS.start_step, name='global_step', trainable=False) # learning_rate = tf.train.exponential_decay(0.05, global_step, 2000, 0.9, staircase=True) learning_rate = tf.train.exponential_decay(0.1, global_step, 1000, 0.95, staircase=True) # learning_rate = tf.train.piecewise_constant(global_step, [24000, 48000, 72000, 108000, 144000], # [0.1, 0.01, 0.001, 0.0001, 0.00001, 0.000001]) tf.summary.scalar('learing rate', learning_rate) # opt = tf.train.AdamOptimizer(learning_rate) opt = tf.train.MomentumOptimizer(learning_rate, momentum=FLAGS.momentum) # opt = tf.train.GradientDescentOptimizer(learning_rate) # learning_rate = tf.train.exponential_decay(0.01, global_step, 32000, 0.1) # opt = tf.train.GradientDescentOptimizer(learning_rate) tower_grads = [] tower_loss = [] tower_acc = [] tower_acc_v = [] images, labels = input_pipeline( tf.train.match_filenames_once( os.path.join(FLAGS.data_dir, 'train', '*.tfrecords')), FLAGS.batch_size) batch_queue = tf.contrib.slim.prefetch_queue.prefetch_queue( [images, labels], capacity=2 * num_gpus) images_v, labels_v = input_pipeline( tf.train.match_filenames_once( os.path.join(FLAGS.data_dir, 'valid', '*.tfrecords')), 128 // num_gpus) batch_queue_v = tf.contrib.slim.prefetch_queue.prefetch_queue( [images_v, labels_v], capacity=2 * num_gpus) for i in range(num_gpus): with tf.device('/gpu:%d' % i): with tf.name_scope('tower_%d' % i) as scope: image_batch, label_batch = batch_queue.dequeue() logits = build.net(image_batch, is_training, FLAGS) losses.sparse_softmax_cross_entropy(labels=label_batch, logits=logits, scope=scope) total_loss = losses.get_losses( scope=scope) + losses.get_regularization_losses( scope=scope) total_loss = tf.add_n(total_loss) grads = opt.compute_gradients(total_loss) tower_grads.append(grads) tower_loss.append(losses.get_losses(scope=scope)) with tf.name_scope('accuracy'): correct_prediction = tf.equal( tf.reshape(tf.argmax(logits, 1), [-1, 1]), tf.cast(label_batch, tf.int64)) accuracy = tf.reduce_mean( tf.cast(correct_prediction, tf.float32)) tower_acc.append(accuracy) tf.get_variable_scope().reuse_variables() image_batch_v, label_batch_v = batch_queue_v.dequeue() logits_v = build.net(image_batch_v, False, FLAGS) correct_prediction = tf.equal( tf.reshape(tf.argmax(logits_v, 1), [-1, 1]), tf.cast(label_batch_v, tf.int64)) accuracy = tf.reduce_mean( tf.cast(correct_prediction, tf.float32)) tower_acc_v.append(accuracy) with tf.name_scope('scores'): with tf.name_scope('accuracy'): accuracy = tf.reduce_mean(tf.stack(tower_acc, axis=0)) with tf.name_scope('accuracy_v'): accuracy_v = tf.reduce_mean(tf.stack(tower_acc_v, axis=0)) with tf.name_scope('batch_loss'): batch_loss = tf.add_n(tower_loss)[0] / num_gpus tf.summary.scalar('loss', batch_loss) tf.summary.scalar('accuracy', accuracy) grads = average_gradients(tower_grads) variable_averages = tf.train.ExponentialMovingAverage( 0.9999, global_step) variables_averages_op = variable_averages.apply( tf.trainable_variables()) with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE): apply_gradient_op = opt.apply_gradients(grads, global_step=global_step) train_op = tf.group(apply_gradient_op, variables_averages_op) # train_op = apply_gradient_op # summary_op = tf.summary.merge_all() # init = tf.global_variables_initializer() summary_op = tf.summary.merge_all() saver = tf.train.Saver(name="saver", max_to_keep=10) saver_best = tf.train.Saver(name='best', max_to_keep=100) with tf.Session(config=config) as sess: sess.run(tf.local_variables_initializer()) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) if tf.gfile.Exists(os.path.join(FLAGS.ckpt_dir, 'checkpoint')): saver.restore(sess, tf.train.latest_checkpoint(FLAGS.ckpt_dir)) else: sess.run(tf.global_variables_initializer()) train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train', sess.graph) train_writer.flush() cache = np.ones(5, dtype=np.float32) / FLAGS.num_classes cache_v = np.ones(5, dtype=np.float32) / FLAGS.num_classes d = 1000 best = 0 for i in range(FLAGS.start_step, FLAGS.max_steps + 1): # feed = feed_dict(True, True) if i % d == 0: # Record summaries and test-set accuracy # loss0 = sess.run([total_loss], feed_dict=feed_dict(False, False)) # test_writer.add_summary(summary, i) # feed[is_training] = FLAGS acc, loss, summ, lr, acc_v = sess.run( [ accuracy, batch_loss, summary_op, learning_rate, accuracy_v ], feed_dict={is_training: False}) cache[int(i / d) % 5] = acc cache_v[int(i / d) % 5] = acc_v train_writer.add_summary(summ, i) print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), file=f) print( 'step %d: acc(t)=%f(%f), loss=%f; acc(v)=%f(%f); lr=%e' % (i, acc, cache.mean(), loss, acc_v, cache_v.mean(), lr), file=f) saver.save(sess, os.path.join(FLAGS.ckpt_dir, FLAGS.model_name), global_step=i) if acc_v > 0.90: saver_best.save(sess, os.path.join(FLAGS.ckpt_dir, 'best', FLAGS.model_name), global_step=i) f.flush() sess.run(train_op, feed_dict={is_training: True}) coord.request_stop() coord.join(threads) train_writer.close() # test_writer.close() f.close()
def evaluate_model(config): """ Train the model using the passed in config """ ########################################################### # Create the input pipeline ########################################################### with tf.name_scope('input_pipeline'): dataset = input_utils.get_dataset(config.datadir, config.dataset, config.datasubset) init_op, init_feed_dict, image = input_utils.get_data( config.dataset, dataset, config.batch_size, num_epochs=config.num_epochs, num_readers=config.num_readers) images = tf.train.batch([image], config.batch_size, num_threads=config.num_preprocessing_threads, capacity=5 * config.batch_size) ########################################################### # Generate the model ########################################################### outputs = create_model(config, images, dataset) ########################################################### # Setup the evaluation metrics and summaries ########################################################### summaries = [] metrics_map = {} for loss in losses.get_losses(): metrics_map[loss.op.name] = metrics.streaming_mean(loss) for metric in tf.get_collection(graph_utils.GraphKeys.METRICS): metrics_map[metric.op.name] = metrics.streaming_mean(metric) total_loss = losses.get_total_loss() metrics_map[total_loss.op.name] = metrics.streaming_mean(total_loss) names_to_values, names_to_updates = metrics.aggregate_metric_map( metrics_map) # Create summaries of the metrics and print them to the screen for name, value in names_to_values.iteritems(): summary = tf.summary.scalar(name, value, collections=[]) summaries.append(tf.Print(summary, [value], name)) summaries.extend(layers.summarize_collection(tf.GraphKeys.MODEL_VARIABLES)) summaries.extend(layers.summarize_collection( graph_utils.GraphKeys.METRICS)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.RNN_OUTPUTS)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.TRAINING_PARAMETERS)) images = input_utils.reshape_images(images, config.dataset) tiled_images = image_utils.tile_images(images) summaries.append(tf.summary.image('input_batch', tiled_images)) # Generate the canvases that lead to the final output image with tf.name_scope('canvases'): for step, canvas in enumerate(outputs): canvas = input_utils.reshape_images(canvas, config.dataset) tiled_images = image_utils.tile_images(canvas) summaries.append( tf.summary.image('step{0}'.format(step), tiled_images)) summary_op = tf.summary.merge(summaries, name='summaries') ########################################################### # Begin evaluation ########################################################### checkpoint_path = FLAGS.checkpoint_path eval_ops = tf.group(*names_to_updates.values()) scaffold = tf.train.Scaffold(init_op, init_feed_dict) hooks = [ training.SummaryAtEndHook(FLAGS.log_dir, summary_op), training.StopAfterNEvalsHook( math.ceil(dataset.num_samples / float(config.batch_size))) ] eval_kwargs = {} eval_fn = training.evaluate_repeatedly if FLAGS.once: if tf.gfile.IsDirectory(checkpoint_path): checkpoint_path = tf.train.latest_checkpoint(checkpoint_path) eval_fn = training.evaluate_once else: assert tf.gfile.IsDirectory(checkpoint_path), ( 'checkpoint path must be a directory when using loop evaluation') # On Tensorflow master fd87896 fixes this, but for now just set a very large number eval_kwargs['max_number_of_evaluations'] = sys.maxint eval_fn(checkpoint_path, scaffold=scaffold, hooks=hooks, eval_ops=eval_ops, **eval_kwargs)
def main(_): os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.95 config.gpu_options.allow_growth = True config.allow_soft_placement = True # config.log_device_placement = True if not tf.gfile.Exists(FLAGS.data_dir): raise RuntimeError('data direction is not exist!') # if tf.gfile.Exists(FLAGS.log_dir): # tf.gfile.DeleteRecursively(FLAGS.log_dir) tf.gfile.MakeDirs(FLAGS.log_dir) # if not tf.gfile.Exists(FLAGS.ckpt_dir): tf.gfile.MakeDirs(os.path.join(FLAGS.ckpt_dir, 'best')) f = open(FLAGS.out_file + '.txt', 'a' if FLAGS.start_step is not 0 else 'w') if not f: raise RuntimeError('OUTPUT FILE OPEN ERROR!!!!!!') with tf.device('/cpu:0'): num_gpus = len(FLAGS.gpu.split(',')) global_step = tf.Variable(FLAGS.start_step, name='global_step', trainable=False) # learning_rate = tf.train.piecewise_constant(global_step, # [500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000], # [0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0]) # step_size = 10000 # learning_rate = tf.train.exponential_decay(1.0, global_step, 2*step_size, 0.5, staircase=True) # cycle = tf.floor(1 + tf.cast(global_step, tf.float32) / step_size / 2.) # xx = tf.abs(tf.cast(global_step, tf.float32)/step_size - 2. * tf.cast(cycle, tf.float32) + 1.) # learning_rate = 1e-4 + (1e-1 - 1e-4) * tf.maximum(0., (1-xx))*learning_rate # learning_rate = tf.train.piecewise_constant(global_step, [10000, 70000, 120000, 170000, 220000], # [0.01, 0.1, 0.001, 0.0001, 0.00001, 0.000001]) # learning_rate = tf.constant(0.001) learning_rate = tf.train.exponential_decay(0.05, global_step, 30000, 0.1, staircase=True) print( 'learning_rate = tf.train.exponential_decay(0.05, global_step, 30000, 0.1, staircase=True)', file=f) # opt = tf.train.AdamOptimizer(learning_rate) opt = tf.train.MomentumOptimizer(learning_rate, momentum=0.9) # opt = tf.train.GradientDescentOptimizer(learning_rate) # learning_rate = tf.train.exponential_decay(0.01, global_step, 32000, 0.1) # opt = tf.train.GradientDescentOptimizer(learning_rate) print('opt = tf.train.MomentumOptimizer(learning_rate, momentum=0.9)', file=f) print('weight decay = %e' % FLAGS.weight_decay, file=f) f.flush() tf.summary.scalar('learing rate', learning_rate) tower_grads = [] tower_loss = [] tower_acc = [] images_t, labels_t = input_pipeline( tf.train.match_filenames_once( os.path.join(FLAGS.data_dir, 'train', '*.tfrecords')), FLAGS.batch_size * num_gpus, read_threads=len(os.listdir(os.path.join(FLAGS.data_dir, 'train')))) # batch_queue = tf.contrib.slim.prefetch_queue.prefetch_queue( # [images, labels], capacity=2 * num_gpus) images_v, labels_v = input_pipeline( tf.train.match_filenames_once( os.path.join(FLAGS.data_dir, 'valid', '*.tfrecords')), (256 // num_gpus) * num_gpus, read_threads=len(os.listdir(os.path.join(FLAGS.data_dir, 'valid'))), if_train=False) # batch_queue_v = tf.contrib.slim.prefetch_queue.prefetch_queue( # [images_v, labels_v], capacity=2 * num_gpus) image_batch0 = tf.placeholder( tf.float32, [None, FLAGS.patch_size, FLAGS.patch_size, channels], 'imgs') label_batch0 = tf.placeholder(tf.int32, [None, 1], 'labels') image_batch = tf.split(image_batch0, num_gpus, 0) label_batch = tf.split(label_batch0, num_gpus, 0) for i in range(num_gpus): with tf.device('/gpu:%d' % i): with tf.name_scope('tower_%d' % i) as scope: logits = build.net(image_batch[i], is_training, FLAGS) losses.sparse_softmax_cross_entropy(labels=label_batch[i], logits=logits, scope=scope) total_loss = losses.get_losses( scope=scope) + losses.get_regularization_losses( scope=scope) total_loss = tf.add_n(total_loss) grads = opt.compute_gradients(total_loss) tower_grads.append(grads) tower_loss.append(losses.get_losses(scope=scope)) with tf.name_scope('accuracy'): correct_prediction = tf.equal( tf.reshape(tf.argmax(logits, 1), [-1, 1]), tf.cast(label_batch[i], tf.int64)) accuracy = tf.reduce_mean( tf.cast(correct_prediction, tf.float32)) tower_acc.append(accuracy) tf.get_variable_scope().reuse_variables() with tf.name_scope('scores'): with tf.name_scope('accuracy'): accuracy = tf.reduce_mean(tf.stack(tower_acc, axis=0)) with tf.name_scope('batch_loss'): batch_loss = tf.add_n(tower_loss)[0] / num_gpus tf.summary.scalar('loss', batch_loss) tf.summary.scalar('accuracy', accuracy) grads = average_gradients(tower_grads) with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE): variable_averages = tf.train.ExponentialMovingAverage( 0.9999, global_step) variables_averages_op = variable_averages.apply( tf.trainable_variables()) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): apply_gradient_op = opt.apply_gradients( grads, global_step=global_step) train_op = tf.group(apply_gradient_op, variables_averages_op) p_relu_update = tf.get_collection('p_relu') # train_op = apply_gradient_op # summary_op = tf.summary.merge_all() # init = tf.global_variables_initializer() summary_op = tf.summary.merge_all() saver = tf.train.Saver(name="saver", max_to_keep=10) saver_best = tf.train.Saver(name='best', max_to_keep=200) with tf.Session(config=config) as sess: sess.run(tf.local_variables_initializer()) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) if tf.gfile.Exists(os.path.join(FLAGS.ckpt_dir, 'checkpoint')): saver.restore(sess, tf.train.latest_checkpoint(FLAGS.ckpt_dir)) else: sess.run(tf.global_variables_initializer()) if FLAGS.start_step != 0: sess.run(tf.assign(global_step, FLAGS.start_step)) train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train', sess.graph) train_writer.flush() valid_writer = tf.summary.FileWriter(FLAGS.log_dir + '/valid', sess.graph) valid_writer.flush() cache = np.ones(5, dtype=np.float32) / FLAGS.num_classes cache_v = np.ones(5, dtype=np.float32) / FLAGS.num_classes d = 1000 best = 0 for i in range(FLAGS.start_step, FLAGS.max_steps + 1): def get_batch(set, on_training): if set == 'train': img, lb = sess.run([images_t, labels_t]) # x = np.random.randint(0, 64) # y = np.random.randint(0, 64) # img = np.roll(np.roll(img, x, 1), y, 2) elif set == 'valid': img, lb = sess.run([images_v, labels_v]) else: raise RuntimeError('Unknown set name') feed_dict = {} feed_dict[image_batch0] = img feed_dict[label_batch0] = lb feed_dict[is_training] = on_training return feed_dict # feed = feed_dict(True, True) if i % d == 0: # Record summaries and test-set accuracy # loss0 = sess.run([total_loss], feed_dict=feed_dict(False, False)) # test_writer.add_summary(summary, i) # feed[is_training] = FLAGS acc, loss, summ, lr = sess.run( [accuracy, batch_loss, summary_op, learning_rate], feed_dict=get_batch('train', False)) acc2 = sess.run(accuracy, feed_dict=get_batch('train', True)) cache[int(i / d) % 5] = acc acc_v, loss_v, summ_v = sess.run( [accuracy, batch_loss, summary_op], feed_dict=get_batch('valid', False)) acc2_v = sess.run(accuracy, feed_dict=get_batch('valid', True)) cache_v[int(i / d) % 5] = acc_v train_writer.add_summary(summ, i) valid_writer.add_summary(summ_v, i) print(('step %d, ' % i) + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), file=f) print( 'acc(t)=%f(%f), loss(t)=%f;\nacc(v)=%f(%f), loss(v)=%f; lr=%e' % (acc, cache.mean(), loss, acc_v, cache_v.mean(), loss_v, lr), file=f) print('%f, %f' % (acc2, acc2_v), file=f) saver.save(sess, os.path.join(FLAGS.ckpt_dir, FLAGS.model_name), global_step=i) if acc_v > 0.90: saver_best.save(sess, os.path.join(FLAGS.ckpt_dir, 'best', FLAGS.model_name), global_step=i) f.flush() sess.run(train_op, feed_dict=get_batch('train', True)) sess.run(p_relu_update) coord.request_stop() coord.join(threads) train_writer.close() # test_writer.close() f.close()
def main(_): os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.7 config.gpu_options.allow_growth = True config.allow_soft_placement = True if not tf.gfile.Exists(FLAGS.data_dir): raise RuntimeError('data direction is not exist!') # if tf.gfile.Exists(FLAGS.log_dir): # tf.gfile.DeleteRecursively(FLAGS.log_dir) # tf.gfile.MakeDirs(FLAGS.log_dir) if not tf.gfile.Exists(FLAGS.ckpt_dir): tf.gfile.MakeDirs(FLAGS.ckpt_dir) f = open(FLAGS.out_file, 'w') if not f: raise RuntimeError('OUTPUT FILE OPEN ERROR!!!!!!') with tf.device('/cpu:0'): global_step = tf.Variable(FLAGS.start_step, name='global_step', trainable=False) learning_rate = tf.train.piecewise_constant(global_step, [24000, 48000], [0.1, 0.01, 0.001]) opt = tf.train.MomentumOptimizer(learning_rate, momentum=FLAGS.momentum) # learning_rate = tf.train.exponential_decay(0.01, global_step, 32000, 0.1) # opt = tf.train.GradientDescentOptimizer(learning_rate) tower_grads = [] num_gpus = len(FLAGS.gpu.split(',')) tower_images = [] tower_labels = [] tower_loss = [] for i in range(num_gpus): with tf.device('/gpu:%d' % i): with tf.name_scope('tower_%d' % i) as scope: # with tf.name_scope('input'): # images = tf.placeholder(tf.float32, [None, FLAGS.patch_size, FLAGS.patch_size, 3], 'images') # tf.summary.image('show', images, 1) # # with tf.name_scope('label'): # labels = tf.placeholder(tf.int64, [None, 1], 'y') images, labels = input_pipeline( tf.train.match_filenames_once( os.path.join(FLAGS.data_dir, 'train', '*.tfrecords')), FLAGS.batch_size) tower_images.append(images) tower_labels.append(labels) logits = build.net(images, FLAGS, is_training, FLAGS.num_classes) losses.sparse_softmax_cross_entropy(logits, labels, scope=scope) total_loss = losses.get_losses( scope=scope) + losses.get_regularization_losses( scope=scope) total_loss = tf.add_n(total_loss) grads = opt.compute_gradients(total_loss) tower_grads.append(grads) tower_loss.append(losses.get_losses(scope=scope)) grads = average_gradients(tower_grads) total_loss = tf.add_n(tower_loss) # variable_averages = tf.train.ExponentialMovingAverage( # cifar10.MOVING_AVERAGE_DECAY, global_step) # variables_averages_op = variable_averages.apply(tf.trainable_variables()) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): apply_gradient_op = opt.apply_gradients(grads, global_step=global_step) # train_op = tf.group(apply_gradient_op, variables_averages_op) train_op = apply_gradient_op # summary_op = tf.summary.merge_all() # init = tf.global_variables_initializer() saver = tf.train.Saver(name="saver") with tf.Session(config=config) as sess: sess.run(tf.local_variables_initializer()) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) if tf.gfile.Exists(os.path.join(FLAGS.ckpt_dir, 'checkpoint')): saver.restore(sess, os.path.join(FLAGS.ckpt_dir, FLAGS.model_name)) else: sess.run(tf.global_variables_initializer()) train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train', sess.graph) # test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test', sess.graph) train_writer.flush() # test_writer.flush() # def feed_dict(train, on_training): # def get_batch(data, labels): # d, l = sess.run([data, labels]) # d = d.astype(np.float32) # l = l.astype(np.int64) # return d, l # res = {} # if train: # for j in range(num_gpus): # xs, ys = get_batch(train_example_batch, train_label_batch) # res[tower_images[j]] = xs # res[tower_labels[j]] = ys # else: # for j in range(num_gpus): # xs, ys = get_batch(valid_example_batch, valid_label_batch) # res[tower_images[j]] = xs # res[tower_labels[j]] = ys # return res for i in range(FLAGS.start_step, FLAGS.max_steps + 1): # feed = feed_dict(True, True) sess.run(train_op, feed_dict={is_training: True}) if i % 10 == 0 and i != 0: # Record summaries and test-set accuracy # loss0 = sess.run([total_loss], feed_dict=feed_dict(False, False)) # test_writer.add_summary(summary, i) # feed[is_training] = FLAGS loss1 = sess.run(total_loss, feed_dict={is_training: False}) # train_writer.add_summary(summary, i) print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), file=f) # print('step %d: train_acc=%f, train_loss=%f; test_acc=%f, test_loss=%f' % (i, acc1, loss1, acc0, loss0), # file=f) print('step %d: train_loss=%f' % (i, loss1), file=f) saver.save(sess, os.path.join(FLAGS.ckpt_dir, FLAGS.model_name)) f.flush() coord.request_stop() coord.join(threads) train_writer.close() # test_writer.close() f.close()
def main(_): os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.7 config.gpu_options.allow_growth = True config.allow_soft_placement = True # config.log_device_placement = True if not tf.gfile.Exists(FLAGS.data_dir): raise RuntimeError('data direction is not exist!') # if tf.gfile.Exists(FLAGS.log_dir): # tf.gfile.DeleteRecursively(FLAGS.log_dir) # tf.gfile.MakeDirs(FLAGS.log_dir) if not tf.gfile.Exists(FLAGS.ckpt_dir): tf.gfile.MakeDirs(FLAGS.ckpt_dir) f = open(FLAGS.out_file, 'w') if not f: raise RuntimeError('OUTPUT FILE OPEN ERROR!!!!!!') with tf.device('/cpu:0'): global_step = tf.Variable(FLAGS.start_step, name='global_step', trainable=False) # learning_rate = tf.train.exponential_decay(0.1, global_step, 192000, 0.9, staircase=True) # tf.summary.scalar('learing rate', learning_rate) # opt = tf.train.AdamOptimizer(learning_rate) # opt = tf.train.MomentumOptimizer(learning_rate, momentum=FLAGS.momentum) # opt = tf.train.GradientDescentOptimizer(learning_rate) # learning_rate = tf.train.exponential_decay(0.01, global_step, 32000, 0.1) # opt = tf.train.GradientDescentOptimizer(learning_rate) # tower_grads = [] num_gpus = len(FLAGS.gpu.split(',')) tower_loss = [] tower_acc = [] images, labels = input_pipeline( tf.train.match_filenames_once( os.path.join(FLAGS.data_dir, 'valid', '*.tfrecords')), int(FLAGS.batch_size / num_gpus)) image_batch = tf.placeholder( tf.float32, [None, FLAGS.patch_size, FLAGS.patch_size, 3], 'imgs') label_batch = tf.placeholder(tf.int32, [None, 1], 'labels') for i in range(num_gpus): with tf.device('/gpu:%d' % i): with tf.name_scope('tower_%d' % i) as scope: # image_batch, label_batch = batch_queue.dequeue() # image_batch = tf.ones(shape=[128, 64, 64, 3], dtype=tf.float32) # label_batch = tf.ones(shape=[128, 1], dtype=tf.int32) logits = build.net(image_batch, False, FLAGS) losses.sparse_softmax_cross_entropy(labels=label_batch, logits=logits, scope=scope) # total_loss = losses.get_losses(scope=scope) + losses.get_regularization_losses(scope=scope) # total_loss = tf.add_n(total_loss) # grads = opt.compute_gradients(total_loss) # tower_grads.append(grads) tower_loss.append(losses.get_losses(scope=scope)) with tf.name_scope('accuracy'): correct_prediction = tf.equal( tf.reshape(tf.argmax(logits, 1), [-1, 1]), tf.cast(label_batch, tf.int64)) accuracy = tf.reduce_mean( tf.cast(correct_prediction, tf.float32)) tower_acc.append(accuracy) tf.get_variable_scope().reuse_variables() with tf.name_scope('scores'): with tf.name_scope('accuracy'): accuracy = tf.reduce_mean(tf.stack(tower_acc, axis=0)) with tf.name_scope('batch_loss'): batch_loss = tf.add_n(tower_loss)[0] tf.summary.scalar('loss', batch_loss) tf.summary.scalar('accuracy', accuracy) # grads = average_gradients(tower_grads) # variable_averages = tf.train.ExponentialMovingAverage( # cifar10.MOVING_AVERAGE_DECAY, global_step) # variables_averages_op = variable_averages.apply(tf.trainable_variables()) # with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE): # update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # with tf.control_dependencies(update_ops): # apply_gradient_op = opt.apply_gradients(grads, global_step=global_step) # # train_op = tf.group(apply_gradient_op, variables_averages_op) # train_op = apply_gradient_op # summary_op = tf.summary.merge_all() # init = tf.global_variables_initializer() summary_op = tf.summary.merge_all() # variable_averages = tf.train.ExponentialMovingAverage(0.9999) # variables_to_restore = variable_averages.variables_to_restore() # saver = tf.train.Saver(variables_to_restore, name='saver') saver = tf.train.Saver(name="saver") with tf.Session(config=config) as sess: sess.run(tf.local_variables_initializer()) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) if tf.gfile.Exists(os.path.join(FLAGS.ckpt_dir, 'checkpoint')): # saver.restore(sess, FLAGS.ckpt_dir+'/model') saver.restore( sess, tf.train.latest_checkpoint(FLAGS.ckpt_dir) if FLAGS.model_name is None else os.path.join( FLAGS.ckpt_dir, FLAGS.model_name)) train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test', sess.graph) train_writer.flush() for i in range(FLAGS.start_step, FLAGS.max_steps + 1): # feed = feed_dict(True, True) # if i % 1000 == 0: # Record summaries and test-set accuracy # loss0 = sess.run([total_loss], feed_dict=feed_dict(False, False)) # test_writer.add_summary(summary, i) # feed[is_training] = FLAGS img, lb = sess.run([images, labels]) acc, loss, summ = sess.run([accuracy, batch_loss, summary_op], feed_dict={ image_batch: img, label_batch: lb }) # acc, loss, summ = sess.run([accuracy, batch_loss, summary_op], feed_dict={image_batch: np.ones(shape = [256, 64, 64, 3], dtype=np.float32), label_batch: np.ones(shape=[256, 1], dtype=np.int32)}) train_writer.add_summary(summ, i) print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), file=f) # print('step %d: train_acc=%f, train_loss=%f; test_acc=%f, test_loss=%f' % (i, acc1, loss1, acc0, loss0), # file=f) print('step %d: accuracy=%f, loss=%f' % (i, acc, loss), file=f) # saver.save(sess, os.path.join(FLAGS.ckpt_dir, FLAGS.model_name)) f.flush() # sess.run(train_op, feed_dict={is_training: True}) coord.request_stop() coord.join(threads) train_writer.close() # test_writer.close() f.close()
def train_model(config): """ Train the model using the passed in config """ training_devices = [ graph_utils.device_fn(device) for device in graph_utils.collect_devices({'GPU': FLAGS.num_gpus}) ] assert training_devices, 'Found no training devices!' ########################################################### # Create the input pipeline ########################################################### with tf.device('/cpu:0'), tf.name_scope('input_pipeline'): dataset = input_utils.get_dataset(config.datadir, config.dataset, 'train') init_op, init_feed_dict, image = input_utils.get_data( config.dataset, dataset, config.batch_size, num_epochs=config.num_epochs, num_readers=config.num_readers) inputs_queue = input_utils.batch_images( image, config.batch_size, num_threads=config.num_preprocessing_threads, num_devices=len(training_devices)) ########################################################### # Generate the model ########################################################### towers = graph_utils.create_towers(create_training_model, training_devices, config, inputs_queue, dataset) assert towers, 'No training towers were created!' ########################################################### # Setup the training objectives ########################################################### with tf.name_scope('training'): with tf.device('/cpu:0'): learning_rate_decay_step = config.learning_rate_decay_step / len( towers) learning_rate = tf.maximum(exponential_decay( config.batch_size, learning_rate_decay_step, config.learning_rate, config.learning_rate_decay, dataset), config.learning_rate_min, name='learning_rate') tf.add_to_collection(graph_utils.GraphKeys.TRAINING_PARAMETERS, learning_rate) optimizer = tf.train.AdamOptimizer(learning_rate) # Calculate gradients and total loss tower_klds, tower_losses, grads_and_vars = graph_utils.optimize_towers( optimizer, towers, clip_norm=config.clip) total_kld = tf.add_n(tower_klds, name='total_kld') if tower_klds else None total_loss = tf.add_n(tower_losses, name='total_loss') # Gather update ops from the first tower (for updating batch_norm for example) global_step = framework.get_or_create_global_step() update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, towers[0].scope) update_ops.append( optimizer.apply_gradients(grads_and_vars, global_step=global_step)) update_op = tf.group(*update_ops) with tf.control_dependencies([update_op]): train_op = tf.identity(total_loss, name='train_op') ########################################################### # Collect summaries ########################################################### with tf.device('/cpu:0'): summaries = [] summaries.extend(learning.add_gradients_summaries(grads_and_vars)) summaries.extend( layers.summarize_collection(tf.GraphKeys.MODEL_VARIABLES)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.METRICS)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.RNN_OUTPUTS)) summaries.extend( layers.summarize_collection( graph_utils.GraphKeys.TRAINING_PARAMETERS)) images = input_utils.reshape_images(inputs_queue.dequeue(), config.dataset) tiled_images = image_utils.tile_images(images) summaries.append(tf.summary.image('input_batch', tiled_images)) # Generate the canvases that lead to the final output image with tf.name_scope('canvases'): for step, canvas in enumerate(towers[0].outputs): canvas = input_utils.reshape_images(canvas, config.dataset) tiled_images = image_utils.tile_images(canvas) summaries.append( tf.summary.image('step{0}'.format(step), tiled_images)) with tf.name_scope('losses'): if total_kld is not None: summaries.append(tf.summary.scalar('total_kld', total_kld)) summaries.append(tf.summary.scalar('total_loss', total_loss)) for loss in tower_losses: summaries.append(tf.summary.scalar(loss.op.name, loss)) for loss in losses.get_losses(): summaries.append(tf.summary.scalar(loss.op.name, loss)) summary_op = tf.summary.merge(summaries, name='summaries') ########################################################### # Begin training ########################################################### init_op = tf.group(tf.global_variables_initializer(), init_op) session_config = tf.ConfigProto( allow_soft_placement=False, log_device_placement=FLAGS.log_device_placement) prefetch_queue_buffer = 2 * len(training_devices) number_of_steps = int( int(dataset.num_samples / config.batch_size) / len(training_devices)) number_of_steps = number_of_steps * config.num_epochs - prefetch_queue_buffer tf.logging.info('Running %s steps', number_of_steps) learning.train(train_op, FLAGS.log_dir, session_config=session_config, global_step=global_step, number_of_steps=number_of_steps, init_op=init_op, init_feed_dict=init_feed_dict, save_interval_secs=config.checkpoint_frequency, summary_op=summary_op, save_summaries_secs=config.summary_frequency, trace_every_n_steps=config.trace_frequency if config.trace_frequency > 0 else None)