def destorted_inputs(): """导入训练数据""" images, labels = cifar100_input.distorted_inputs( batch_size=FLAGS.batch_size) if FLAGS.use_fp16: images = tf.cast(images, tf.float16) labels = tf.cast(labels, tf.float16) return images, labels
def get_data(FLAGS, dataset): tr_data = None tr_label = None image_size = None channel_num = None output_num = None if dataset == 'cifar10': tr_data, tr_label = cifar10_input.distorted_inputs( FLAGS.cifar_data_dir, FLAGS.batch_size) image_size = cifar10_input.IMAGE_SIZE channel_num = 3 output_num = 10 elif dataset == 'svhn': tr_data, tr_label = svhn.distorted_inputs(FLAGS.svhn_data_dir, FLAGS.batch_size) image_size = svhn.IMAGE_SIZE channel_num = 3 output_num = 10 elif dataset == 'cifar20': tr_data, tr_label = cifar100_input.distorted_inputs( 20, FLAGS.cifar100_data_dir, FLAGS.batch_size) image_size = cifar100_input.IMAGE_SIZE channel_num = 3 output_num = 20 elif dataset == 'mnist1': tr_data, tr_label = binary_mnist_input.read_train_data( FLAGS, FLAGS.a1, FLAGS.a2) image_size = 28 channel_num = 1 output_num = 2 elif dataset == 'mnist2': tr_data, tr_label = binary_mnist_input.read_train_data( FLAGS, FLAGS.b1, FLAGS.b2) image_size = 28 channel_num = 1 output_num = 2 elif dataset == 'mnist3': tr_data, tr_label = binary_mnist_input.read_train_data( FLAGS, FLAGS.c1, FLAGS.c2) image_size = 28 channel_num = 1 output_num = 2 elif dataset == 'mnist4': tr_data, tr_label = binary_mnist_input.read_train_data( FLAGS, FLAGS.d1, FLAGS.d2) image_size = 28 channel_num = 1 output_num = 2 else: raise ValueError("No such dataset") return tr_data, tr_label, image_size, channel_num, output_num
def distorted_inputs(): """Construct distorted input for CIFAR training using the Reader ops. Returns: images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size. labels: Labels. 1D tensor of [batch_size] size. Raises: ValueError: If no data_dir """ if not FLAGS.data_dir: raise ValueError('Please supply a data_dir') data_dir = os.path.join(FLAGS.data_dir, 'cifar-100-binary') images, labels = cifar100_input.distorted_inputs(data_dir=data_dir, batch_size=FLAGS.batch_size) if FLAGS.use_fp16: images = tf.cast(images, tf.float16) labels = tf.cast(labels, tf.float16) return images, labels
def train(): images = tf.placeholder( tf.float32, [batchSize, data_input.IMAGE_SIZE, data_input.IMAGE_SIZE, 3]) labels = tf.placeholder(tf.int32, [batchSize]) isTrain = tf.placeholder(tf.bool) with tf.variable_scope('train_image'): trainImages, trainLabels = data_input.distorted_inputs(Data_DIR, 100) trainOperations, predictedClass, loss = getWRNGraph( isTrain, images, labels) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) saver = tf.train.Saver() for iteration in range(trainingIterations): trainImagesVal, trainLabelsVal = sess.run( [trainImages, trainLabels]) _, lossValue = sess.run([trainOperations, loss], feed_dict={ isTrain: True, images: trainImagesVal, labels: trainLabelsVal }) if iteration % 1000 == 0: print "iteration %d with loss = %f" % (iteration, lossValue) coord.request_stop() coord.join(threads) saver.save(sess, MODEL_DIR + '/' + MODEL_File)
def __init__(self, eval=False): # pretty standard/simple DNN loosely based on alexnet since its # simple and old so it trains decently fast on a GTX860M # decrease learning rate over time # input -> 32x32x3 # conv1 (f=5, s=1, k=64 relu) -> 32x32x64 # pool1 (f=3, s=2) -> 16x16x64 # conv2 (f=5, s=1, k=64 relu) -> 16x16x64 # pool2 (f=3, s=2) 8x8x64 # dropout (.5) # fc1 (384 relu) -> 1x384 # fc2 (192 relu) -> 1x192 # linear -> 1x10 epochs = 100 learning_rate = .00000001 batch_size = 16 early_stop = False num_train = 50000 f = [3, 3, 3, 3] k = [32, 64, 128, 128, 512] # conv1 w1 = self.weight('w1', [f[0], f[0], 3, k[0]]) b1 = self.bias('b1', [k[0]]) # conv2 w2 = self.weight('w2', [f[1], f[1], k[0], k[1]]) b2 = self.bias('b2', [k[1]]) # conv3 w3 = self.weight('w3', [f[2], f[2], k[1], k[2]]) b3 = self.bias('b3', [k[2]]) # conv4 w4 = self.weight('w4', [f[3], f[3], k[2], k[3]]) b4 = self.bias('b4', [k[3]]) # fc1 w5 = self.weight('w5', [8 * 8 * k[3], k[4]]) b5 = self.bias('b5', [k[4]]) # fc2 w6 = self.weight('w6', [k[4], 100]) b6 = self.bias('b6', [100]) # linear # w5 = self.weight('w5', [k[3], 10]) # b5 = self.bias('b5', [10]) self.params = (w1, w2, w3, w4, w5, w6) if eval: images, labels = cifar100_input.inputs(True, 'cifar-100-binary', 1000) else: images, labels = cifar100_input.distorted_inputs( 'cifar-100-binary', batch_size) s = [1, 1, 1, 1] global_step = tf.train.get_or_create_global_step() X_ = self.conv(images, w1, s[0], b1) X_ = self.conv(X_, w2, s[1], b2) X_ = self.pool(X_, 2, 2) # X_ = tf.nn.dropout(X_, .25) # X_ = self.batch_norm(X_) X_ = self.conv(X_, w3, s[2], b3) X_ = self.conv(X_, w4, s[3], b4) X_ = self.pool(X_, 2, 2) # X_ = tf.nn.dropout(X_, .25) # X_ = self.batch_norm(X_) X_ = tf.nn.relu(self.fc(X_, w5, b5)) X_ = tf.nn.dropout(X_, .5) logits = self.fc(X_, w6, b6) pred = tf.nn.softmax(logits) loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels)) # l2 = tf.reduce_sum([tf.reduce_sum(tf.pow(w,2)) for w in self.params]) # loss = loss + weight_decay * l2 correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(labels, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) optim = tf.train.AdamOptimizer(learning_rate).minimize( loss, global_step=global_step) steps_per_epoch = num_train // batch_size saver = tf.train.Saver() checkpoint = 'checkpoints/model.cifar100.v5.ckpt' with tf.Session(config=config) as sess: sess.run(tf.global_variables_initializer()) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) try: saver.restore(sess, checkpoint) except: pass if eval: total_accuracy = 0 total_loss = 0 for batch in range(10): a, l, g = sess.run([accuracy, loss, global_step]) total_accuracy += a total_loss += l print( 'global_step {} (epoch {}): test accuracy={}, test loss={}' .format(g, g // steps_per_epoch, total_accuracy / 10, total_loss / 10)) coord.request_stop() coord.join(threads) return with tqdm(range(steps_per_epoch * epochs)) as t: best = 0 for step in t: epoch, step_in_epoch = divmod(step, steps_per_epoch) if step_in_epoch == 0: saver.save(sess, checkpoint) total_accuracy = 0 total_loss = 0 a, l, o, g = sess.run([accuracy, loss, optim, global_step]) total_accuracy += a total_loss += l t.set_postfix( epoch=g // steps_per_epoch, step=step_in_epoch, acc=total_accuracy / step_in_epoch, loss=total_loss / step_in_epoch, ) if early_stop: saver.restore(sess, checkpoint) coord.request_stop() coord.join(threads)
def train(): print('[Dataset Configuration]') print('\tCIFAR-100 dir: %s' % FLAGS.data_dir) print('\tNumber of classes: %d' % FLAGS.num_classes) print('\tNumber of training images: %d' % FLAGS.num_train_instance) print('\tNumber of test images: %d' % FLAGS.num_test_instance) print('[Network Configuration]') print('\tBatch size: %d' % FLAGS.batch_size) print('\tResidual blocks per group: %d' % FLAGS.num_residual_units) print('\tNetwork width multiplier: %d' % FLAGS.k) print('[Optimization Configuration]') print('\tL2 loss weight: %f' % FLAGS.l2_weight) print('\tThe momentum optimizer: %f' % FLAGS.momentum) print('\tInitial learning rate: %f' % FLAGS.initial_lr) print('\tEpochs per lr step: %f' % FLAGS.lr_step_epoch) print('\tLearning rate decay: %f' % FLAGS.lr_decay) print('[Training Configuration]') print('\tTrain dir: %s' % FLAGS.train_dir) print('\tTraining max steps: %d' % FLAGS.max_steps) print('\tSteps per displaying info: %d' % FLAGS.display) print('\tSteps per testing: %d' % FLAGS.test_interval) print('\tSteps during testing: %d' % FLAGS.test_iter) print('\tSteps per saving checkpoints: %d' % FLAGS.checkpoint_interval) print('\tGPU memory fraction: %f' % FLAGS.gpu_fraction) print('\tLog device placement: %d' % FLAGS.log_device_placement) with tf.Graph().as_default(): init_step = 0 global_step = tf.Variable(0, trainable=False, name='global_step') # Get images and labels of CIFAR-100 with tf.variable_scope('train_image'): train_images, train_labels = data_input.distorted_inputs(FLAGS.data_dir, FLAGS.batch_size) with tf.variable_scope('test_image'): test_images, test_labels = data_input.inputs(True, FLAGS.data_dir, FLAGS.batch_size) # Build a Graph that computes the predictions from the inference model. images = tf.placeholder(tf.float32, [FLAGS.batch_size, data_input.IMAGE_SIZE, data_input.IMAGE_SIZE, 3]) labels = tf.placeholder(tf.int32, [FLAGS.batch_size]) # Build model decay_step = FLAGS.lr_step_epoch * FLAGS.num_train_instance / FLAGS.batch_size hp = resnet.HParams(batch_size=FLAGS.batch_size, num_classes=FLAGS.num_classes, num_residual_units=FLAGS.num_residual_units, k=FLAGS.k, weight_decay=FLAGS.l2_weight, initial_lr=FLAGS.initial_lr, decay_step=decay_step, lr_decay=FLAGS.lr_decay, momentum=FLAGS.momentum) network = resnet.ResNet(hp, images, labels, global_step) network.build_model() network.build_train_op() # Summaries(training) train_summary_op = tf.merge_all_summaries() # Build an initialization operation to run below. init = tf.initialize_all_variables() # Start running operations on the Graph. sess = tf.Session(config=tf.ConfigProto( gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=FLAGS.gpu_fraction), log_device_placement=FLAGS.log_device_placement)) sess.run(init) # Create a saver. saver = tf.train.Saver(tf.all_variables(), max_to_keep=10000) ckpt = tf.train.get_checkpoint_state(FLAGS.train_dir) if ckpt and ckpt.model_checkpoint_path: print('\tRestore from %s' % ckpt.model_checkpoint_path) # Restores from checkpoint saver.restore(sess, ckpt.model_checkpoint_path) init_step = int(ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]) else: print('No checkpoint file found. Start from the scratch.') # Start queue runners & summary_writer tf.train.start_queue_runners(sess=sess) if not os.path.exists(FLAGS.train_dir): os.mkdir(FLAGS.train_dir) summary_writer = tf.train.SummaryWriter(FLAGS.train_dir, sess.graph) # Training! test_best_acc = 0.0 for step in xrange(init_step, FLAGS.max_steps): # Test if step % FLAGS.test_interval == 0: test_loss, test_acc = 0.0, 0.0 for i in range(FLAGS.test_iter): test_images_val, test_labels_val = sess.run([test_images, test_labels]) loss_value, acc_value = sess.run([network.loss, network.acc], feed_dict={network.is_train:False, images:test_images_val, labels:test_labels_val}) test_loss += loss_value test_acc += acc_value test_loss /= FLAGS.test_iter test_acc /= FLAGS.test_iter test_best_acc = max(test_best_acc, test_acc) format_str = ('%s: (Test) step %d, loss=%.4f, acc=%.4f') print (format_str % (datetime.now(), step, test_loss, test_acc)) test_summary = tf.Summary() test_summary.value.add(tag='test/loss', simple_value=test_loss) test_summary.value.add(tag='test/acc', simple_value=test_acc) test_summary.value.add(tag='test/best_acc', simple_value=test_best_acc) summary_writer.add_summary(test_summary, step) # test_loss_summary = tf.Summary() # test_loss_summary.value.add(tag='test/loss', simple_value=test_loss) # summary_writer.add_summary(test_loss_summary, step) # test_acc_summary = tf.Summary() # test_acc_summary.value.add(tag='test/acc', simple_value=test_acc) # summary_writer.add_summary(test_acc_summary, step) # test_best_acc_summary = tf.Summary() # test_best_acc_summary.value.add(tag='test/best_acc', simple_value=test_best_acc) # summary_writer.add_summary(test_best_acc_summary, step) summary_writer.flush() # Train start_time = time.time() train_images_val, train_labels_val = sess.run([train_images, train_labels]) _, lr_value, loss_value, acc_value, train_summary_str = \ sess.run([network.train_op, network.lr, network.loss, network.acc, train_summary_op], feed_dict={network.is_train:True, images:train_images_val, labels:train_labels_val}) duration = time.time() - start_time assert not np.isnan(loss_value) # Display & Summary(training) if step % FLAGS.display == 0: num_examples_per_step = FLAGS.batch_size examples_per_sec = num_examples_per_step / duration sec_per_batch = float(duration) format_str = ('%s: (Training) step %d, loss=%.4f, acc=%.4f, lr=%f (%.1f examples/sec; %.3f ' 'sec/batch)') print (format_str % (datetime.now(), step, loss_value, acc_value, lr_value, examples_per_sec, sec_per_batch)) summary_writer.add_summary(train_summary_str, step) # Save the model checkpoint periodically. if (step > init_step and step % FLAGS.checkpoint_interval == 0) or (step + 1) == FLAGS.max_steps: checkpoint_path = os.path.join(FLAGS.train_dir, 'model.ckpt') saver.save(sess, checkpoint_path, global_step=step)