import time import sys try: serial_interface = sys.argv[1] image_file = sys.argv[2] colorduinos_horizontal = int(sys.argv[3]) colorduinos_vertical = int(sys.argv[4]) colorduino_x = int(sys.argv[5]) colorduino_y = int(sys.argv[6]) except Exception: if(len(sys.argv) != 7): print "Usage: python main.py SERIAL_INTERFACE IMAGE_FILE COLORDUINOS_HORIZONTAL COLORDUINOS_VERTICAL COLORDUINO_X COLORDUINO_Y" sys.exit() im=ImageReader(sys.argv[2]) im.resize(colorduinos_horizontal*8,colorduinos_vertical*8) im.load() ser = SerialComm(serial_interface) time.sleep(2) for x in range(0,8): for y in range(0,8): rgb = im.getColorAt(colorduino_x*8 + x, colorduino_y*8 + y) r = rgb[0] g = rgb[1] b = rgb[2] print "x: %i, y: %i" % (int(colorduino_x)*8+x, int(colorduino_y*8)+y) line = "%02X,%02X,%02X,%02X,%02X" % (y,x,r,g,b) ser.writeln(line)
def run(self): tf.set_random_seed(self.random_seed) coord = tf.train.Coordinator() # 读取数据 with tf.name_scope("create_inputs"): reader = ImageReader(self.data_dir, self.data_train_list, self.input_size, self.random_scale, self.random_mirror, self.ignore_label, self.img_mean, coord) image_batch, label_batch = reader.dequeue(self.batch_size) # 网络 net = PSPNet({'data': image_batch}, is_training=True, num_classes=self.num_classes) raw_output = net.layers['conv6'] # According from the prototxt in Caffe implement, learning rate must multiply by 10.0 in pyramid module fc_list = [ 'conv5_3_pool1_conv', 'conv5_3_pool2_conv', 'conv5_3_pool3_conv', 'conv5_3_pool6_conv', 'conv6', 'conv5_4' ] # 所有的变量 restore_var = [v for v in tf.global_variables()] # 所有可训练变量 all_trainable = [ v for v in tf.trainable_variables() if ('beta' not in v.name and 'gamma' not in v.name) or self.train_beta_gamma ] # fc_list中的全连接层可训练变量和卷积可训练变量 fc_trainable = [ v for v in all_trainable if v.name.split('/')[0] in fc_list ] conv_trainable = [ v for v in all_trainable if v.name.split('/')[0] not in fc_list ] # lr * 1.0 fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name] # lr * 10.0 fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name] # lr * 20.0 # 验证 assert (len(all_trainable) == len(fc_trainable) + len(conv_trainable)) assert (len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable)) # Predictions: ignoring all predictions with labels greater or equal than n_classes raw_prediction = tf.reshape(raw_output, [-1, self.num_classes]) label_process = prepare_label(label_batch, tf.stack(raw_output.get_shape()[1:3]), num_classes=self.num_classes, one_hot=False) # [batch_size, h, w] raw_gt = tf.reshape(label_process, [ -1, ]) indices = tf.squeeze( tf.where(tf.less_equal(raw_gt, self.num_classes - 1)), 1) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) prediction = tf.gather(raw_prediction, indices) # Pixel-wise softmax loss. loss = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=prediction, labels=gt) l2_losses = [ self.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name ] reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses) # Using Poly learning rate policy base_lr = tf.constant(self.learning_rate) step_ph = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.scalar_mul( base_lr, tf.pow((1 - step_ph / self.num_steps), self.power)) # Gets moving_mean and moving_variance update operations from tf.GraphKeys.UPDATE_OPS update_ops = None if not self.update_mean_var else tf.get_collection( tf.GraphKeys.UPDATE_OPS) # 对变量以不同的学习率优化:分别求梯度、应用梯度 with tf.control_dependencies(update_ops): opt_conv = tf.train.MomentumOptimizer(learning_rate, self.momentum) opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, self.momentum) opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, self.momentum) grads = tf.gradients( reduced_loss, conv_trainable + fc_w_trainable + fc_b_trainable) grads_conv = grads[:len(conv_trainable)] grads_fc_w = grads[len(conv_trainable):(len(conv_trainable) + len(fc_w_trainable))] grads_fc_b = grads[(len(conv_trainable) + len(fc_w_trainable)):] train_op_conv = opt_conv.apply_gradients( zip(grads_conv, conv_trainable)) train_op_fc_w = opt_fc_w.apply_gradients( zip(grads_fc_w, fc_w_trainable)) train_op_fc_b = opt_fc_b.apply_gradients( zip(grads_fc_b, fc_b_trainable)) train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b) pass sess = tf.Session(config=self.config) sess.run(tf.global_variables_initializer()) # Saver for storing checkpoints of the model. saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10) # 加载模型 ckpt = tf.train.get_checkpoint_state(self.log_dir) if ckpt and ckpt.model_checkpoint_path: tf.train.Saver(var_list=restore_var).restore( sess, ckpt.model_checkpoint_path) Tools.print_info("Restored model parameters from {}".format( ckpt.model_checkpoint_path)) else: Tools.print_info('No checkpoint file found.') pass # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) # Iterate over training steps. for step in range(self.num_steps): start_time = time.time() if step % self.save_pred_freq == 0: loss_value, _ = sess.run([reduced_loss, train_op], feed_dict={step_ph: step}) saver.save(sess, self.checkpoint_path, global_step=step) Tools.print_info('The checkpoint has been created.') else: loss_value, _ = sess.run([reduced_loss, train_op], feed_dict={step_ph: step}) duration = time.time() - start_time Tools.print_info( 'step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format( step, loss_value, duration)) coord.request_stop() coord.join(threads) pass
def train_setup(self): tf.set_random_seed(self.conf.random_seed) # Create queue coordinator. self.coord = tf.train.Coordinator() # Input size input_size = (self.conf.input_height, self.conf.input_width) # Load reader with tf.name_scope("create_inputs"): reader = ImageReader(self.conf.data_dir, self.conf.data_list, input_size, self.conf.random_scale, self.conf.random_mirror, self.conf.ignore_label, IMG_MEAN, self.coord) self.image_batch, self.label_batch = reader.dequeue_up_to( self.conf.batch_size) # Create network if self.conf.encoder_name not in ['res101', 'res50', 'deeplab']: print('encoder_name ERROR!') print("Please input: res101, res50, or deeplab") sys.exit(-1) elif self.conf.encoder_name == 'deeplab': net = Deeplab_v2(self.image_batch, self.conf.num_classes, True) # Variables that load from pre-trained model. restore_var = [ v for v in tf.global_variables() if 'fc' not in v.name ] # Trainable Variables all_trainable = tf.trainable_variables() # Fine-tune part encoder_trainable = [ v for v in all_trainable if 'fc' not in v.name ] # lr * 1.0 # Decoder part decoder_trainable = [v for v in all_trainable if 'fc' in v.name] else: net = ResNet_segmentation(self.image_batch, self.conf.num_classes, True, self.conf.encoder_name) # Variables that load from pre-trained model. restore_var = [ v for v in tf.global_variables() if 'resnet_v1' in v.name ] # Trainable Variables all_trainable = tf.trainable_variables() # Fine-tune part encoder_trainable = [ v for v in all_trainable if 'resnet_v1' in v.name ] # lr * 1.0 # Decoder part decoder_trainable = [ v for v in all_trainable if 'decoder' in v.name ] decoder_w_trainable = [ v for v in decoder_trainable if 'weights' in v.name or 'gamma' in v.name ] # lr * 10.0 decoder_b_trainable = [ v for v in decoder_trainable if 'biases' in v.name or 'beta' in v.name ] # lr * 20.0 # Check assert (len(all_trainable) == len(decoder_trainable) + len(encoder_trainable)) assert (len(decoder_trainable) == len(decoder_w_trainable) + len(decoder_b_trainable)) # Network raw output raw_output = net.outputs # [batch_size, h, w, 21] # Output size output_shape = tf.shape(raw_output) output_size = (output_shape[1], output_shape[2]) # Groud Truth: ignoring all labels greater or equal than n_classes label_proc = prepare_label(self.label_batch, output_size, num_classes=self.conf.num_classes, one_hot=False) raw_gt = tf.reshape(label_proc, [ -1, ]) indices = tf.squeeze( tf.where(tf.less_equal(raw_gt, self.conf.num_classes - 1)), 1) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) raw_prediction = tf.reshape(raw_output, [-1, self.conf.num_classes]) prediction = tf.gather(raw_prediction, indices) # Pixel-wise softmax_cross_entropy loss loss = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=prediction, labels=gt) # L2 regularization l2_losses = [ self.conf.weight_decay * tf.nn.l2_loss(v) for v in all_trainable if 'weights' in v.name ] # Loss function self.reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses) # Define optimizers # 'poly' learning rate base_lr = tf.constant(self.conf.learning_rate) self.curr_step = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.scalar_mul( base_lr, tf.pow((1 - self.curr_step / self.conf.num_steps), self.conf.power)) # We have several optimizers here in order to handle the different lr_mult # which is a kind of parameters in Caffe. This controls the actual lr for each # layer. opt_encoder = tf.train.MomentumOptimizer(learning_rate, self.conf.momentum) opt_decoder_w = tf.train.MomentumOptimizer(learning_rate * 10.0, self.conf.momentum) opt_decoder_b = tf.train.MomentumOptimizer(learning_rate * 20.0, self.conf.momentum) # To make sure each layer gets updated by different lr's, we do not use 'minimize' here. # Instead, we separate the steps compute_grads+update_params. # Compute grads grads = tf.gradients( self.reduced_loss, encoder_trainable + decoder_w_trainable + decoder_b_trainable) grads_encoder = grads[:len(encoder_trainable)] grads_decoder_w = grads[len(encoder_trainable):( len(encoder_trainable) + len(decoder_w_trainable))] grads_decoder_b = grads[(len(encoder_trainable) + len(decoder_w_trainable)):] # Update params train_op_conv = opt_encoder.apply_gradients( zip(grads_encoder, encoder_trainable)) train_op_fc_w = opt_decoder_w.apply_gradients( zip(grads_decoder_w, decoder_w_trainable)) train_op_fc_b = opt_decoder_b.apply_gradients( zip(grads_decoder_b, decoder_b_trainable)) # Finally, get the train_op! update_ops = tf.get_collection( tf.GraphKeys.UPDATE_OPS ) # for collecting moving_mean and moving_variance with tf.control_dependencies(update_ops): self.train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b) # Saver for storing checkpoints of the model self.saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=0) # Loader for loading the pre-trained model self.loader = tf.train.Saver(var_list=restore_var) # Training summary # Processed predictions: for visualisation. raw_output_up = tf.image.resize_bilinear(raw_output, input_size) raw_output_up = tf.argmax(raw_output_up, axis=3) self.pred = tf.expand_dims(raw_output_up, dim=3) # Image summary. images_summary = tf.py_func(inv_preprocess, [self.image_batch, 2, IMG_MEAN], tf.uint8) labels_summary = tf.py_func( decode_labels, [self.label_batch, 2, self.conf.num_classes], tf.uint8) preds_summary = tf.py_func(decode_labels, [self.pred, 2, self.conf.num_classes], tf.uint8) self.total_summary = tf.summary.image( 'images', tf.concat(axis=2, values=[images_summary, labels_summary, preds_summary]), max_outputs=2) # Concatenate row-wise. if not os.path.exists(self.conf.logdir): os.makedirs(self.conf.logdir) self.summary_writer = tf.summary.FileWriter( self.conf.logdir, graph=tf.get_default_graph())
def main(): args = get_arguments() print(args) coord = tf.train.Coordinator() tf.reset_default_graph() with tf.name_scope("create_inputs"): reader = ImageReader( DATA_DIRECTORY, DATA_LIST_PATH, input_size, None, None, ignore_label, IMG_MEAN, coord) image, label = reader.image, reader.label image_batch, label_batch = tf.expand_dims(image, dim=0), tf.expand_dims(label, dim=0) # Add one batch dimension. # Create network. net = PSPNet({'data': image_batch}, is_training=False, num_classes=num_classes) with tf.variable_scope('', reuse=True): flipped_img = tf.image.flip_left_right(image) flipped_img = tf.expand_dims(flipped_img, dim=0) net2 = PSPNet({'data': flipped_img}, is_training=False, num_classes=num_classes) # Which variables to load. restore_var = tf.global_variables() # Predictions. raw_output = net.layers['conv6'] if args.flipped_eval: flipped_output = tf.image.flip_left_right(tf.squeeze(net2.layers['conv6'])) flipped_output = tf.expand_dims(flipped_output, dim=0) raw_output = tf.add_n([raw_output, flipped_output]) raw_output_up = tf.image.resize_bilinear(raw_output, size=input_size, align_corners=True) raw_output_up = tf.argmax(raw_output_up, dimension=3) pred = tf.expand_dims(raw_output_up, dim=3) # mIoU pred_flatten = tf.reshape(pred, [-1,]) raw_gt = tf.reshape(label_batch, [-1,]) indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, num_classes - 1)), 1) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) pred = tf.gather(pred_flatten, indices) mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(pred, gt, num_classes=num_classes) # Set up tf session and initialize variables. config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) sess.run(tf.local_variables_initializer()) restore_var = tf.global_variables() ckpt = tf.train.get_checkpoint_state(args.model) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=restore_var) load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) load(loader, sess, ckpt.model_checkpoint_path) else: print('No checkpoint file found.') # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) for step in range(num_steps): preds, _ = sess.run([pred, update_op]) if step > 0 and args.measure_time: calculate_time(sess, net) if step % 10 == 0: print('Finish {0}/{1}'.format(step, num_steps)) print('step {0} mIoU: {1}'.format(step, sess.run(mIoU))) print('step {0} mIoU: {1}'.format(step, sess.run(mIoU))) coord.request_stop() coord.join(threads)
def __init__(self, root, im_file, masks_dir): self._reader = ImageReader(root, im_file, masks_dir) self._root = root self._image = self._reader.image self._mask = self._reader.mask
def main(): args = get_arguments() print(args) coord = tf.train.Coordinator() tf.reset_default_graph() with tf.name_scope("create_inputs"): reader = ImageReader(DATA_DIRECTORY, DATA_LIST_PATH, input_size, None, None, ignore_label, IMG_MEAN, coord) image, label = reader.image, reader.label image_batch, label_batch = tf.expand_dims(image, dim=0), tf.expand_dims( label, dim=0) # Add one batch dimension. # Create network. if args.model[-2:] == 'bn': net = ICNet_BN({'data': image_batch}, num_classes=num_classes) else: net = ICNet({'data': image_batch}, num_classes=num_classes) # Which variables to load. restore_var = tf.global_variables() # Predictions. raw_output = net.layers['conv6_cls'] raw_output_up = tf.image.resize_bilinear(raw_output, size=input_size, align_corners=True) raw_output_up = tf.argmax(raw_output_up, dimension=3) raw_pred = tf.expand_dims(raw_output_up, dim=3) # mIoU pred_flatten = tf.reshape(raw_pred, [ -1, ]) raw_gt = tf.reshape(label_batch, [ -1, ]) indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, num_classes - 1)), 1) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) pred = tf.gather(pred_flatten, indices) mIoU, update_op = tf.contrib.metrics.streaming_mean_iou( pred, gt, num_classes=num_classes) # Set up tf session and initialize variables. config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) sess.run(tf.local_variables_initializer()) restore_var = tf.global_variables() if args.model == 'train': print('Restore from train30k model...') net.load(model_train30k, sess) elif args.model == 'trainval': print('Restore from trainval90k model...') net.load(model_trainval90k, sess) elif args.model == 'train_bn': print('Restore from train30k bnnomerge model...') net.load(model_train30k_bn, sess) elif args.model == 'trainval_bn': print('Restore from trainval90k bnnomerge model...') net.load(model_trainval90k_bn, sess) # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) for step in range(num_steps): preds, _ = sess.run([pred, update_op]) if step > 0 and args.measure_time: calculate_time(sess, net, raw_pred) if step % 10 == 0: print('Finish {0}/{1}'.format(step, num_steps)) print('step {0} mIoU: {1}'.format(step, sess.run(mIoU))) coord.request_stop() coord.join(threads)
def main(argv=None): input_size = (cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH) # Create queue coordinator. coord = tf.train.Coordinator() # Load reader. # Train print('Train ' + cfg.train_data_list) with tf.name_scope("create_inputs"): reader = ImageReader(cfg.train_data_dir, cfg.train_data_list, input_size, cfg.random_scale, cfg.random_resize, cfg.random_mirror, cfg.random_color, cfg.random_crop_pad, cfg.ignore_label, cfg.IMG_MEAN, coord) image_batch, label_batch = reader.dequeue(cfg.batch_size) # Define Network pred_annotation, logits = inference_deeplabv3_plus_16( image_batch, is_training=True) # Modified logits_loss = cross_entropy_loss(logits, label_batch) # loss1 for ECP dataset # logits_loss = weighted_cross_entropy_loss(logits, label_batch) # loss2 for RueMonge dataset # logits_loss = weighted_cross_entropy_loss_4class(logits, label_batch) # # PSPNet # pred_annotation, logits, logits_dsn = inference_pspnet(image_batch, is_training=True) # PSPNet # # logits_loss = cross_entropy_loss(logits, label_batch) + \ # # cross_entropy_loss(logits_dsn, label_batch) # loss1 for ECP dataset # logits_loss = weighted_cross_entropy_loss(logits, label_batch) + \ # weighted_cross_entropy_loss(logits_dsn, label_batch) # loss2 for RueMonge dataset ce_loss = logits_loss # cross entropy loss # Show acc for validation or train dataset if cfg.is_time_acc or cfg.is_epoch_acc: with tf.variable_scope('', reuse=True): val_image_batch = tf.placeholder( tf.float32, shape=[1, IMAGE_HEIGHT, IMAGE_WIDTH, 3], name="input_image") f = open(cfg.val_data_list, 'r') val_img_list = [] val_label_list = [] for line in f: try: image_name, label = line.strip("\n").split(' ') except ValueError: # Adhoc for test. image_name = label = line.strip("\n") val_img_list.append(cfg.val_data_dir + image_name) val_label_list.append(cfg.val_data_dir + label) _, val_logits = inference_deeplabv3_plus_16_init( val_image_batch, is_training=False) # Modified # _, val_logits, _ = inference_pspnet(val_image_batch, is_training=False) # PSPNet val_logits_softmax = tf.nn.softmax(val_logits) tf.group() l2_loss = [ weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' or 'w' in v.name or 'W' in v.name ] # encode: W, facade: weights l2_losses = tf.add_n(l2_loss) # Total loss loss = ce_loss + l2_losses # + stru_loss tf.summary.scalar("loss_ce", ce_loss) tf.summary.scalar("l2_losses", l2_losses) tf.summary.scalar("total_loss", loss) step_ph = tf.placeholder(dtype=tf.float32, shape=()) # Using Poly learning rate policy base_lr = tf.constant(cfg.learning_rate) learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / global_step), power)) trainable_var = tf.trainable_variables() # Optimizer if cfg.optimizer == 'Adam': optimizer = tf.train.AdamOptimizer(learning_rate) print('Optimizer: Adam') elif cfg.optimizer == 'Adam2': optimizer = tf.train.AdamOptimizer(learning_rate, beta1=0.9, beta2=0.99) elif cfg.optimizer == 'SGD': optimizer = tf.train.GradientDescentOptimizer(learning_rate) elif cfg.optimizer == 'Momentum': optimizer = tf.train.MomentumOptimizer(learning_rate, momentum=0.9) print('Optimizer: Momentum') elif cfg.optimizer == 'RMSProp': optimizer = tf.train.RMSPropOptimizer(learning_rate) # grads = optimizer.compute_gradients(loss, var_list=trainable_var) # train_op = optimizer.apply_gradients(grads) ## Optimizer definition - nothing different from any classical example opt = optimizer ## Retrieve all trainable variables you defined in your graph if cfg.freeze_bn: tvs = [ v for v in tf.trainable_variables() if 'beta' not in v.name and 'gamma' not in v.name ] else: tvs = [v for v in tf.trainable_variables()] ## Creation of a list of variables with the same shape as the trainable ones # initialized with 0s accum_vars = [ tf.Variable(tf.zeros_like(tv.initialized_value()), trainable=False) for tv in tvs ] zero_ops = [tv.assign(tf.zeros_like(tv)) for tv in accum_vars] ## Calls the compute_gradients function of the optimizer to obtain... the list of gradients gvs = opt.compute_gradients(loss, tvs) ## Adds to each element from the list you initialized earlier with zeros its gradient (works because accum_vars and gvs are in the same order) accum_ops = [accum_vars[i].assign_add(gv[0]) for i, gv in enumerate(gvs)] ## Define the training step (part with variable value update) train_step = opt.apply_gradients([(accum_vars[i], gv[1]) for i, gv in enumerate(gvs)]) print("Setting up summary op...") summary_op = tf.summary.merge_all() # Set gpu usage config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True config.gpu_options.per_process_gpu_memory_fraction = 1.0 sess = tf.Session(config=config) print("Setting up Saver...") saver = tf.train.Saver(max_to_keep=cfg.model_save_num) # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) # create two summary writers to show training loss and validation loss in the same graph # need to create two folders 'train' and 'validation' inside FLAGS.logs_dir if not os.path.exists(cfg.logs_dir): os.makedirs(cfg.logs_dir) train_writer = tf.summary.FileWriter(cfg.logs_dir + 'train', sess.graph) test_writer = tf.summary.FileWriter(cfg.logs_dir + 'test') if not os.path.exists(cfg.save_dir): os.makedirs(cfg.save_dir) count = 0 files = os.path.join(cfg.save_dir + 'model.ckpt-*.index') sfile = glob.glob(files) if len(sfile) > 0: sess.run(tf.global_variables_initializer()) sfile = glob.glob(files) steps = [] for s in sfile: part = s.split('.') step = int(part[1].split('-')[1]) steps.append(step) count = max(steps) model = cfg.save_dir + 'model.ckpt-' + str(count) print('\nRestoring weights from: ' + model) saver.restore(sess, model) print('End Restore') else: # # restore from pre-train on imagenet variables = tf.global_variables() sess.run(tf.variables_initializer(variables, name='init')) # # tensorflow 1 if os.path.exists(cfg.pre_trained_model) or os.path.exists( cfg.pre_trained_model + '.index'): var_keep_dic = get_variables_in_checkpoint_file( cfg.pre_trained_model) # Get the variables to restore, ignoring the variables to fix variables_to_restore = get_variables_to_restore( variables, var_keep_dic) if len(variables_to_restore) > 0: restorer = tf.train.Saver(variables_to_restore) restorer.restore(sess, cfg.pre_trained_model) print('Model pre-train loaded from ' + cfg.pre_trained_model) else: print('Model inited random.') else: print('Model inited random.') # RGB -> BGR if 'res' in cfg.pre_trained_model: conv1_rgb = tf.get_variable("conv1_rgb", [7, 7, 3, 64], trainable=False) restorer_fc = tf.train.Saver( {'resnet_v1_50/conv1/weights': conv1_rgb}) restorer_fc.restore(sess, cfg.pre_trained_model) sess.run(tf.assign(variables[0], tf.reverse(conv1_rgb, [2]))) print('ResNet Conv 1 RGB->BGR') elif 'vgg' in cfg.pre_trained_model: conv1_rgb = tf.get_variable("conv1_rgb", [3, 3, 3, 64], trainable=False) restorer_fc = tf.train.Saver( {'vgg_16/conv1/conv1_1/weights': conv1_rgb}) restorer_fc.restore(sess, cfg.pre_trained_model) sess.run(tf.assign(variables[0], tf.reverse(conv1_rgb, [2]))) print('Vgg Conv 1 RGB->BGR') _mask = pred_annotation[0] _img = image_batch[0] _gt = label_batch[0] if not os.path.exists(cfg.save_dir + 'temp_img'): os.mkdir(cfg.save_dir + 'temp_img') print('Start train ' + cfg.data_dir) print('---------------Hyper Paras---------------') print('-- batch_size: ', cfg.batch_size) print('-- Gradient Accumulation: ', cfg.Gradient_Accumulation) print('-- image height: ', cfg.IMAGE_HEIGHT) print('-- image width: ', cfg.IMAGE_WIDTH) print('-- learning rate: ', cfg.learning_rate) print('-- GPU: ', cfg.use_gpu) print('-- optimizer: ', cfg.optimizer) print('-- class num: ', cfg.NUM_OF_CLASSESS) print('-- total iter: ', cfg.total_iter) print('-- Time acc: ', cfg.is_time_acc) print('-- Acc interval: ', cfg.acc_interval) print('-- Start Acc iter: ', cfg.start_show_iter) print('-- Is save step: ', cfg.is_save_step) print('-- Start save step: ', cfg.start_save_step) print('-- save ecpoch: ', cfg.save_step_inter) print('-- model save num: ', cfg.model_save_num) print('-- summary interval: ', cfg.summary_interval) print('-- weight decay: ', cfg.weight_decay) print('-- Freeze BN: ', cfg.freeze_bn) print('-- Decay rate: ', cfg.decay_rate) print('-- minScale: ', cfg.minScale) print('-- maxScale: ', cfg.maxScale) print('-- random scale: ', cfg.random_scale) print('-- random mirror: ', cfg.random_mirror) print('-- random crop: ', cfg.random_crop_pad) print('-- Validation on :' + str(cfg.val_data_list)) print('-- Pre-trained: ' + cfg.pre_trained_model) print('----------------End---------------------') fcfg = open(cfg.save_dir + 'cfg.txt', 'w') fcfg.write('-- batch_size: ' + str(cfg.batch_size) + '\n') fcfg.write('-- Gradient Accumulation: ' + str(cfg.Gradient_Accumulation) + '\n') fcfg.write('-- image height: ' + str(cfg.IMAGE_HEIGHT) + '\n') fcfg.write('-- image width: ' + str(cfg.IMAGE_WIDTH) + '\n') fcfg.write('-- learning rate: ' + str(cfg.learning_rate) + '\n') fcfg.write('-- GPU: ' + str(cfg.use_gpu) + '\n') fcfg.write('-- optimizer: ' + str(cfg.optimizer) + '\n') fcfg.write('-- class num: ' + str(cfg.NUM_OF_CLASSESS) + '\n') fcfg.write('-- total iter: ' + str(cfg.total_iter) + '\n') fcfg.write('-- Time acc: ' + str(cfg.is_time_acc) + '\n') fcfg.write('-- Acc interval: ' + str(cfg.acc_interval) + '\n') fcfg.write('-- Start Acc iter: ' + str(cfg.start_show_iter) + '\n') fcfg.write('-- Is save step: ' + str(cfg.is_save_step) + '\n') fcfg.write('-- Start save step: ' + str(cfg.start_save_step) + '\n') fcfg.write('-- save ecpoch: ' + str(cfg.save_step_inter) + '\n') fcfg.write('-- model save num: ' + str(cfg.model_save_num) + '\n') fcfg.write('-- summary interval: ' + str(cfg.summary_interval) + '\n') fcfg.write('-- weight decay: ' + str(cfg.weight_decay) + '\n') fcfg.write('-- Freeze BN: ' + str(cfg.freeze_bn) + '\n') fcfg.write('-- Decay rate: ' + str(cfg.decay_rate) + '\n') fcfg.write('-- minScale: ' + str(cfg.minScale) + '\n') fcfg.write('-- maxScale: ' + str(cfg.maxScale) + '\n') fcfg.write('-- random scale: ' + str(cfg.random_scale) + '\n') fcfg.write('-- random mirror: ' + str(cfg.random_mirror) + '\n') fcfg.write('-- random crop: ' + str(cfg.random_crop_pad) + '\n') fcfg.write('-- Validation on :' + str(cfg.val_data_list) + '\n') fcfg.write('-- Pre-trained: ' + cfg.pre_trained_model + '\n') fcfg.close() last_summary_time = time.time() last_acc_time = time.time() record = train_number / cfg.batch_size # iter number of each epoch if cfg.is_save_step: # save with step running_count = count epo = int(count / record) if cfg.is_save_epoch: # save with epoch running_count = int(epo * record) epo = count best_acc = 0.5 best_step = 0 train_start_time = time.time() start_step = running_count lossTr_list = [] stepes = [] Acc_val_list = [] # Change the graph for read only sess.graph.finalize() while running_count < cfg.total_iter: time_start = time.time() itr = 0 while itr < int(record): itr += 1 running_count += 1 # log last 10 model if running_count > (cfg.total_iter - 10) and cfg.is_save_last10_model: saver.save(sess, cfg.save_dir + 'model.ckpt', int(running_count)) print('Model has been saved:' + str(running_count)) # more than total iter, stopping training if running_count > cfg.total_iter: break feed_dict = {step_ph: (running_count)} # save summary now = time.time() if now - last_summary_time > cfg.summary_interval: summary_str = sess.run(summary_op, feed_dict={step_ph: running_count}) train_writer.add_summary(summary_str, running_count) last_summary_time = now score_map, img, gt = sess.run([_mask, _img, _gt], feed_dict=feed_dict) img = np.array(img + cfg.IMG_MEAN, np.uint8) score_map = score_map * 20 gt = gt * 20 save_temp = np.zeros( (cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH * 3, 3), np.uint8) save_temp[0:cfg.IMAGE_HEIGHT, 0:cfg.IMAGE_WIDTH, :] = img save_temp[0:cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH:cfg.IMAGE_WIDTH * 2, :] = gt save_temp[0:cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH * 2:cfg.IMAGE_WIDTH * 3, :] = score_map cv2.imwrite( cfg.save_dir + 'temp_img/' + str(now) + '_mask.jpg', save_temp) time_s = time.time() # Run the zero_ops to initialize it sess.run(zero_ops) # Accumulate the gradients 'n_minibatches' times in accum_vars using accum_ops for i in range(cfg.Gradient_Accumulation): sess.run(accum_ops, feed_dict=feed_dict) train_loss, ls_ce, ls_l2, lr = sess.run( [loss, ce_loss, l2_losses, learning_rate], feed_dict=feed_dict) if running_count > 50: lossTr_list.append(ls_ce) if start_step == 0: start_step = 50 # Run the train_step ops to update the weights based on your accumulated gradients sess.run(train_step, feed_dict=feed_dict) time_e = time.time() print( "Epo: %d, Step: %d, Train_loss:%g, ce: %g, l2:%g, lr:%g, time:%g" % (epo, running_count, train_loss, ls_ce, ls_l2, lr, time_e - time_s)) # check accuracy per step of training data if cfg.is_time_acc and running_count >= cfg.start_show_iter and \ running_count <= cfg.total_iter and (now-last_acc_time) > cfg.acc_interval: # Test accuracy in val hist = np.zeros((cfg.NUM_OF_CLASSESS, cfg.NUM_OF_CLASSESS)) for i, img_name in enumerate(val_img_list): true_val = np.expand_dims(misc.imread(val_label_list[i]), axis=2) pred_val = evaluate_accuracy(val_logits_softmax, sess, val_image_batch, img_name) hist += fast_hist(true_val.flatten(), pred_val.flatten(), cfg.NUM_OF_CLASSESS) hist[0, :] = 0 # overall accuracy over_acc = np.diag(hist).sum() / hist.sum() print('>>> Step', running_count, 'overall accuracy', over_acc) if over_acc > best_acc: saver.save(sess, cfg.save_dir + 'best.ckpt') best_acc = over_acc best_step = running_count fshow = open( cfg.save_dir + 'acc: ' + str(best_acc) + ', step: ' + str(best_step), 'w') print('>>> best acc: ', best_acc, 'best step: ', best_step) # per-class accuracy acc = np.diag(hist) / hist.sum(0) print('>>> Step', running_count, 'mean accuracy', acc) last_acc_time = now stepes.append(running_count) Acc_val_list.append(over_acc) # draw plots for visualization ---------------------------- # Plot the figures per 60s import matplotlib.pyplot as plt fig1, ax1 = plt.subplots(figsize=(11, 8)) ax1.plot(range(start_step, running_count), lossTr_list) ax1.set_title("Average training loss vs steps") ax1.set_xlabel("Steps") ax1.set_ylabel("Current loss") plt.savefig(cfg.save_dir + "loss_vs_steps.png") plt.clf() fig2, ax2 = plt.subplots(figsize=(11, 8)) ax2.plot(stepes, Acc_val_list, label="Val total acc.") ax2.set_title(" Acc vs steps") ax2.set_xlabel("Steps") ax2.set_ylabel("Current Acc") plt.legend(loc='lower right') plt.savefig(cfg.save_dir + "acc_vs_steps.png") plt.close('all') # ---------------------------------------------------------- # Save step model if cfg.is_save_step and (running_count % cfg.save_step_inter) == 0 \ and running_count >= cfg.start_save_step: saver.save(sess, cfg.save_dir + 'model.ckpt', int(running_count)) print('Model has been saved:' + str(running_count)) files = os.path.join(cfg.save_dir + 'model.ckpt-*.data-00000-of-00001') sfile = glob.glob(files) if len(sfile) > cfg.model_save_num: steps = [] for s in sfile: part = s.split('.') re = int(part[1].split('-')[1]) steps.append(re) re = min(steps) model = cfg.save_dir + 'model.ckpt-' + str(re) os.remove(model + '.data-00000-of-00001') os.remove(model + '.index') os.remove(model + '.meta') print('Remove Model:' + model) # Check accuracy per Epoch of training data if cfg.is_epoch_acc and running_count >= cfg.start_show_iter \ and running_count <= cfg.total_iter: # Test accuracy in val hist = np.zeros((cfg.NUM_OF_CLASSESS, cfg.NUM_OF_CLASSESS)) for i, img_name in enumerate(val_img_list): true_val = np.expand_dims(misc.imread(val_label_list[i]), axis=2) pred_val = evaluate_accuracy(val_logits_softmax, sess, val_image_batch, img_name) hist += fast_hist(pred_val.flatten(), true_val.flatten(), cfg.NUM_OF_CLASSESS) hist[:, 0] = 0 # overall accuracy over_acc = np.diag(hist).sum() / hist.sum() print('>>> Step', running_count, 'overall accuracy', over_acc) if over_acc > best_acc: saver.save(sess, cfg.save_dir + 'best.ckpt') best_acc = over_acc best_step = running_count fshow = open( cfg.save_dir + 'acc: ' + str(best_acc) + ', step: ' + str(best_step), 'w') print('>>> best acc: ', best_acc, 'best step: ', best_step) # per-class accuracy acc = np.diag(hist) / hist.sum(0) print('>>> Step', running_count, 'mean accuracy', acc) epo += 1 # Save epoch model if cfg.is_save_epoch and (epo % cfg.save_epoch_inter ) == 0 and epo >= cfg.start_save_epoch: saver.save(sess, cfg.save_dir + 'model.ckpt', epo) print('Model has been saved:' + str(epo)) files = os.path.join(cfg.save_dir + 'model.ckpt-*.data-00000-of-00001') sfile = glob.glob(files) if len(sfile) > cfg.model_save_num: steps = [] for s in sfile: part = s.split('.') re = int(part[1].split('-')[1]) steps.append(re) re = min(steps) model = cfg.save_dir + 'model.ckpt-' + str(re) os.remove(model + '.data-00000-of-00001') os.remove(model + '.index') os.remove(model + '.meta') print('Remove Model:' + model) time_end = time.time() print('Epo ' + str(epo) + ' use time: ' + str(time_end - time_start)) # saver.save(sess, cfg.save_dir + 'last.ckpt') # save last model train_end_time = time.time() print('Train total use: ' + str((train_end_time - train_start_time) / 3600) + ' h') coord.request_stop() coord.join(threads)
def evaluate_checkpoint(model_path, args): coord = tf.train.Coordinator() tf.reset_default_graph() reader = ImageReader(args.data_list, INPUT_SIZE, random_scale=False, random_mirror=False, ignore_label=IGNORE_LABEL, img_mean=IMG_MEAN, coord=coord, train=False) image_batch, label_batch = reader.dequeue(args.batch_size) # Set up tf session and initialize variables. config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) # Create network. net = ICNet_BN({'data': image_batch}, is_training=False, num_classes=num_classes) # Which variables to load. restore_var = tf.global_variables() # Predictions. raw_output = net.layers['conv6_cls'] raw_output_up = tf.image.resize_bilinear(raw_output, size=INPUT_SIZE, align_corners=True) raw_output_up = tf.argmax(raw_output_up, dimension=3) pred = tf.expand_dims(raw_output_up, dim=3) # mIoU pred_flatten = tf.reshape(pred, [ -1, ]) raw_gt = tf.reshape(label_batch, [ -1, ]) if args.ignore_zero: indices = tf.squeeze( tf.where( tf.logical_and(tf.less_equal(raw_gt, num_classes - 1), tf.greater(raw_gt, 0)), ), 1) else: indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, num_classes - 1)), 1) #indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, num_classes - 1)), 1) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) pred = tf.gather(pred_flatten, indices) metric, op = tf.contrib.metrics.streaming_mean_iou(pred, gt, num_classes=num_classes) mIoU, update_op = metric, op # Summaries miou_op = tf.summary.scalar('mIOU', mIoU) start = time.time() logging.info('Starting evaluation at ' + time.strftime('%Y-%m-%d-%H:%M:%S', time.gmtime())) sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) saver = tf.train.Saver(var_list=restore_var) load(saver, sess, model_path) for step in range(num_steps): preds, _ = sess.run([pred, update_op]) if step % 500 == 0: print('Finish {0}/{1}'.format(step + 1, num_steps)) iou, summ = sess.run([mIoU, miou_op]) sess.close() coord.request_stop() #coord.join(threads) return summ, iou
def main(): tf.set_random_seed(1234) temp_flags = FLAGS.__flags.items() temp_flags.sort() for params, value in FLAGS.__flags.items(): print('{}: {}'.format(params, value)) coord = tf.train.Coordinator() tf.reset_default_graph() input_size = [FLAGS.image_height, FLAGS.image_width] reader = ImageReader(FLAGS.data_dir, FLAGS.data_list, input_size, None, None, FLAGS.ignore_label, IMG_MEAN, coord) image, label = reader.image, reader.label image_batch, label_batch = tf.expand_dims(image, dim=0), tf.expand_dims( label, dim=0) # Add one batch dimension. print(image_batch) net = MobileNet(image_batch, isTraining=False, updateBeta=False) with tf.variable_scope('', reuse=True): flipped_img = tf.image.flip_left_right(image) flipped_img = tf.expand_dims(flipped_img, dim=0) net2 = MobileNet(flipped_img, isTraining=False, updateBeta=False) # Which variables to load. restore_var = tf.global_variables() raw_output = net if FLAGS.flipped_eval: flipped_output = tf.image.flip_left_right(tf.squeeze(net2)) flipped_output = tf.expand_dims(flipped_output, dim=0) raw_output = tf.add_n([raw_output, flipped_output]) raw_output_up = tf.image.resize_bilinear(raw_output, size=input_size, align_corners=True) raw_output_up = tf.argmax(raw_output_up, dimension=3) pred = tf.expand_dims(raw_output_up, dim=3) # mIoU pred_flatten = tf.reshape(pred, [ -1, ]) raw_gt = tf.reshape(label_batch, [ -1, ]) indices = tf.squeeze( tf.where(tf.less_equal(raw_gt, FLAGS.num_classes - 1)), 1) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) pred = tf.gather(pred_flatten, indices) mIoU, update_op = tf.contrib.metrics.streaming_mean_iou( pred, gt, num_classes=FLAGS.num_classes) # Set up tf session and initialize variables. config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) sess.run(tf.local_variables_initializer()) restore_var = tf.global_variables() load(sess, FLAGS.checkpoint_path, restore_var) # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) for step in range(1, FLAGS.num_steps + 1): preds, _ = sess.run([pred, update_op]) # if step > 0 and FLAGS.measure_time: # calculate_time(sess, net) if FLAGS.print_each_step and step % 100 == 0: print('Finish {0}/{1}'.format(step, FLAGS.num_steps)) print('step {0} mIoU: {1}'.format(step, sess.run(mIoU))) value = sess.run(mIoU) print('step {0} mIoU: {1}'.format(step, value)) epoch = int(os.path.basename(FLAGS.checkpoint_path).split('-')[1]) writeToLogFile(epoch, value) coord.request_stop() coord.join(threads)
def detect_vessels(self, additional_mask=None): im = self._image mask = self._mask # green channel & remove light reflex im_gray = remove_light_reflex(im) # CLAHE clahe = cv2.createCLAHE(3, (7, 7)) im_clahe = clahe.apply(im_gray) # Haar transform wp = pywt.WaveletPacket2D(im_clahe, wavelet='haar', maxlevel=1, mode='sym') wp.decompose() im_haar = wp['a'].data im_haar = remove_light_reflex(im_haar) mask = ImageReader.rescale_mask(im_haar, self.mask) try: hasAdditionalMask = additional_mask.size > 0 additional_mask = ImageReader.rescale_mask(im_haar, additional_mask) except AttributeError: hasAdditionalMask = False # compute the image mean to "invert" the image # for the gausssian matched filter mean_thresh, _, _, _ = cv2.mean(im_haar, mask) if hasAdditionalMask: im_haar[additional_mask != 0] = mean_thresh im_haar = mean_thresh - im_haar # Gaussian Matched and FDOG filter responses K_MF = gaussian_matched_filter_kernel(31, 5) K_FDOG = fdog_filter_kernel(31, 5) kernels_mf = createMatchedFilterBank(K_MF, 12) kernels_fdog = createMatchedFilterBank(K_FDOG, 12) im_matched_mf = applyFilters(im_haar, kernels_mf) im_matched_fdog = applyFilters(im_haar, kernels_fdog) # normalize local mean of FDOG response local_mean_fdog = cv2.blur(im_matched_fdog, (11, 11)) local_mean_fdog = cv2.normalize(local_mean_fdog, alpha=0, beta=1, norm_type=cv2.NORM_MINMAX) # set the threshold matrix mean_thresh, _, _, _ = cv2.mean(im_matched_mf, mask) ref_thresh = mean_thresh * self._norm_const ref_thresh = ref_thresh * (1 + local_mean_fdog) # show the results if self._is_debug: show_images([self._reader.image]) show_images([im_gray, im_haar], titles=["gray", "haar"]) show_images([im_matched_mf, im_matched_fdog], titles=["mf", "fdog"]) im_vessels = im_matched_mf.copy() im_vessels[im_vessels < ref_thresh] = 0 im_vessels[mask == 0] = 0 im_vessels[im_vessels != 0] = 1 im_vessels = im_vessels.astype('uint8') if self._is_debug: show_images([im_vessels], titles=["vessels"]) self._im_norm = mh.stretch(im_matched_mf) return ImageReader.rescale_mask(self.image, im_vessels)
def main(): # Create model and start training args = get_arguments() h, w = map(int, args.input_size.split(',')) input_size = (h, w) tf.set_random_seed(args.random_seed) # Create queue coordinator coord = tf.train.Coordinator() # Load Image Reader with tf.name_scope('create_inputs'): reader = ImageReader( args.index_loc, args.data_dir, args.mask_dir, input_size, args.random_scale, args.random_mirror, args.ignore_label, IMG_MEAN, coord) image_batch, label_batch = reader.dequeue(args.batch_size) mode = tf.contrib.learn.ModeKeys.TRAIN net = DeepLabResNetModel(image_batch, mode, args.num_classes, args.atrous_blocks) raw_output = net.output # Trainable Variables restore_vars = [v for v in tf.global_variables() if 'fc' not in v.name or not args.not_restore_last] all_trainable = [v for v in tf.trainable_variables() if 'beta' not in v.name and 'gamma' not in v.name] fc_trainable = [v for v in all_trainable if 'fc' in v.name] conv_trainable = [v for v in all_trainable if 'fc' not in v.name] fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name] fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name] # Predictions: ignoring all predictions with labels greater or equal than n_classes raw_prediction = tf.reshape(raw_output, [-1, args.num_classes]) label_proc = prepare_labels(label_batch, tf.stack(raw_output.get_shape()[1:3]), num_classes=args.num_classes, one_hot=False) raw_gt = tf.reshape(label_proc, [-1,]) indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)), 1) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) prediction = tf.gather(raw_prediction, indices) # Pixel-wise Softmax Loss loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt) l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name] reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses) variable_summaries(reduced_loss, name='loss') # Processed predictions: for visualization raw_output_up = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3,]) raw_output_up = tf.argmax(raw_output_up, dimension=3) pred = tf.expand_dims(raw_output_up, dim=3) # Define loss and optimization parameters base_lr = tf.constant(args.learning_rate, tf.float64) global_step = tf.Variable(0, trainable=False, name='global_step') increment_step = tf.assign(global_step, global_step + 1) learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - global_step / args.num_steps), args.power)) learning_rate = tf.maximum(learning_rate, 8e-7) opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum) opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 5.0, args.momentum) opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum) grads = tf.gradients(reduced_loss, conv_trainable + fc_w_trainable + fc_b_trainable) grads_conv = grads[:len(conv_trainable)] grads_fc_w = grads[len(conv_trainable) : (len(conv_trainable) + len(fc_w_trainable))] grads_fc_b = grads[(len(conv_trainable) + len(fc_w_trainable)):] train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable)) train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable)) train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable)) train_op = tf.group(increment_step, train_op_conv, train_op_fc_w, train_op_fc_b) # initial_learning_rate = 1e-2 # learning_rate = tf.train.exponential_decay(initial_learning_rate, global_step, 300, 0.96) # adam = tf.train.AdamOptimizer(learning_rate).minimize(reduced_loss, global_step=global_step) # Image Summary model_dir = args.snapshot_dir + args.model_name images_summary = tf.py_func(inv_preprocess, [image_batch, args.save_num_images, IMG_MEAN], tf.uint8) preds_summary = tf.py_func(decode_labels, [pred, args.save_num_images, args.num_classes], tf.uint8) labels_summary = tf.py_func(decode_labels, [label_batch, args.save_num_images, args.num_classes], tf.uint8) image_summaries = [images_summary, preds_summary, labels_summary] image_summary = tf.summary.image('images', tf.concat(axis=2, values=image_summaries), max_outputs=args.save_num_images) # Variable Summary variable_summaries(fc_w_trainable, 'fc_w') variable_summaries(fc_b_trainable, 'fc_b') variable_summaries(learning_rate, 'learning_rate') # variable_summaries(net.weights, 'aconv_w') # variable_summaries(net.biases, 'aconv_b') total_summary = tf.summary.merge_all() summary_writer = tf.summary.FileWriter(model_dir, graph=tf.get_default_graph()) # Set up session with tf.Session() as sess: tf.global_variables_initializer().run() saver = tf.train.Saver(max_to_keep=3) if args.snapshot_dir is not None and args.model_name is not None and os.path.exists(model_dir): loader = tf.train.Saver() load_model(loader, sess, model_dir) threads = tf.train.start_queue_runners(coord=coord, sess=sess) # train_op = adam for step in range(args.num_steps): start_time = time.time() if step % args.save_pred_every == 0: feed = [reduced_loss, image_batch, label_batch, pred, total_summary, global_step, train_op] loss_value, images, labels, preds, summary, total_steps, _ = sess.run(feed) summary_writer.add_summary(summary, total_steps) save_model(saver, sess, model_dir, global_step) else: feed = [reduced_loss, global_step, train_op] loss_value, total_steps, _ = sess.run(feed) duration = time.time() - start_time results = 'global step: {:d}, step: {:d} \t loss = {:.3f}, ({:.3f} secs)'\ .format(total_steps, step, loss_value, duration) if step % WRITE_EVERY == 0: with open(WRITE_FILE, 'a') as f: f.write(results + '\n') print(results) coord.request_stop() coord.join(threads)
def evaluate(config, evaluation_set='val', determine_confusionMatrix=True, plot_confusionMatrix=False): # -------------------------------------------------------------------- # init network # -------------------------------------------------------------------- tf.compat.v1.reset_default_graph() # define input placeholders input_placeholder = {} input_placeholder.update( {'is_training': tf.compat.v1.placeholder(dtype=tf.bool, shape=())}) if config.ARCHITECTURE == 'semantic_segmentation': batch_size = config.BATCH_SIZE * config.TIMESEQUENCE_LENGTH print('batch_size: {}'.format(config.BATCH_SIZE)) # Search for available GPUs: the result is a list of device ids like `['/gpu:0', '/gpu:1']` devices = get_available_gpus() print("found devices: ", devices) num_GPU = len(devices) if (num_GPU) == 0: num_GPU = 1 # CPU support! # min 1 sample should be applied on a GPU if (config.BATCH_SIZE < num_GPU): num_GPU = config.BATCH_SIZE image_placeholder = [] label_placeholder = [] for iter in range(num_GPU): if (iter == (num_GPU - 1)): batch_size_local = batch_size - (num_GPU - 1) * (batch_size // num_GPU) else: batch_size_local = batch_size // num_GPU print('batch_size /gpu:{} : {}'.format(iter, num_GPU)) image_placeholder.append( tf.compat.v1.placeholder( dtype=tf.float32, shape=(batch_size_local, config.DATASET_TRAIN.INPUT_SIZE[0], config.DATASET_TRAIN.INPUT_SIZE[1], config.DATASET_TRAIN.NUM_CHANNELS))) label_placeholder.append( tf.compat.v1.placeholder( dtype=tf.float32, shape=(batch_size_local, config.DATASET_TRAIN.INPUT_SIZE[0], config.DATASET_TRAIN.INPUT_SIZE[1], 1))) input_placeholder.update({'image_batch': image_placeholder}) input_placeholder.update({'label_batch': label_placeholder}) else: print( '[ERROR] network architecture does not exist!!! Please check your spelling!' ) raise NotImplementedError # load network architecture if config.ARCHITECTURE == 'semantic_segmentation': model = get_model(config.MODEL) net = model( { 'data': input_placeholder['image_batch'], 'is_training': input_placeholder['is_training'] }, is_training=input_placeholder['is_training'], evaluation=tf.logical_not(input_placeholder['is_training']), num_classes=config.DATASET_TRAIN.NUM_CLASSES, filter_scale=config.FILTER_SCALE, timeSequence=config.TIMESEQUENCE_LENGTH, variant=config.MODEL_VARIANT) else: print( '[ERROR] network architecture does not exist!!! Please check your spelling!' ) raise NotImplementedError # -------------------------------------------------------------------- # determine evaluation metric # -------------------------------------------------------------------- if config.ARCHITECTURE == 'semantic_segmentation': list_raw_gt = [] list_pred_flattern_mIoU = [] for iter_gpu in range(len(input_placeholder['image_batch'])): with tf.device('/gpu:%d' % iter_gpu): if config.MODEL == 'SegNet_BN' or config.MODEL == 'SegNet_BN_encoder' or config.MODEL == 'SegNet_BN_decoder' or config.MODEL == 'SegNet_BN_encoderDecoder': raw_output = net.layers['output'][iter_gpu] raw_output_up = tf.argmax(raw_output, axis=3, output_type=tf.int32) raw_pred_mIoU = tf.expand_dims(raw_output_up, dim=3) else: # ICNet ori_shape = config.DATASET_TRAIN.INPUT_SIZE #?? raw_output = net.layers['output'][iter_gpu] raw_output_up = tf.compat.v1.image.resize_bilinear( raw_output, size=ori_shape[:2], align_corners=True) raw_output_up = tf.argmax(raw_output_up, axis=3, output_type=tf.int32) raw_pred_mIoU = tf.expand_dims(raw_output_up, dim=3) # determine mIoU if config.USAGE_TIMESEQUENCES: # evaluate only last image of time sequence pred_of_interest = np.array( range(config.BATCH_SIZE), dtype=np.int32 ) * config.TIMESEQUENCE_LENGTH + config.TIMESEQUENCE_LENGTH - 1 pred_flatten_mIoU = tf.reshape( tf.gather(raw_pred_mIoU, pred_of_interest), [ -1, ]) raw_gt = tf.reshape( tf.gather(input_placeholder['label_batch'][iter_gpu], pred_of_interest), [ -1, ]) else: # evaluate all images of batch size pred_flatten_mIoU = tf.reshape(raw_pred_mIoU, [ -1, ]) raw_gt = tf.reshape( input_placeholder['label_batch'][iter_gpu], [ -1, ]) list_raw_gt.append(raw_gt) list_pred_flattern_mIoU.append(pred_flatten_mIoU) # combine output of different GPUs with tf.device('/gpu:%d' % 0): all_raw_gt = tf.reshape(tf.concat(list_raw_gt, -1), [ -1, ]) all_pred_flatten_mIoU = tf.reshape( tf.concat(list_pred_flattern_mIoU, -1), [ -1, ]) indices_mIoU = tf.squeeze( tf.where( tf.less_equal(raw_gt, config.DATASET_TRAIN.NUM_CLASSES - 1)), 1) gt_mIoU = tf.cast(tf.gather(raw_gt, indices_mIoU), tf.int32) pred_mIoU = tf.gather(pred_flatten_mIoU, indices_mIoU) mIoU, update_op = tf.contrib.metrics.streaming_mean_iou( pred_mIoU, gt_mIoU, num_classes=config.DATASET_VAL.NUM_CLASSES) # deterimine confusing matrix if determine_confusionMatrix: # Create an accumulator variable to hold the counts confusion = tf.Variable(tf.zeros([ config.DATASET_VAL.NUM_CLASSES, config.DATASET_VAL.NUM_CLASSES ], dtype=tf.int64), name='confusion', collections=[ tf.compat.v1.GraphKeys.LOCAL_VARIABLES ]) # Compute a per-batch confusion batch_confusion = tf.math.confusion_matrix( tf.reshape(gt_mIoU, [-1]), tf.reshape(pred_mIoU, [-1]), num_classes=config.DATASET_VAL.NUM_CLASSES, name='batch_confusion') # Create the update op for doing a "+=" accumulation on the batch confusion_update = confusion.assign( confusion + tf.cast(batch_confusion, dtype=tf.int64)) # ----------------------------------------- # init session # ----------------------------------------- # Set up tf session and initialize variables. sessConfig = tf.compat.v1.ConfigProto() sessConfig.gpu_options.allow_growth = True sessConfig.gpu_options.per_process_gpu_memory_fraction = 0.5 sess = tf.compat.v1.Session(config=sessConfig) init = tf.compat.v1.global_variables_initializer() local_init = tf.compat.v1.local_variables_initializer() sess.run(init) sess.run(local_init) # load checkpoint file print(config.EVALUATION.MODELPATH) ckpt = tf.compat.v1.train.get_checkpoint_state(config.EVALUATION.MODELPATH) if ckpt and ckpt.model_checkpoint_path: loader = tf.compat.v1.train.Saver( var_list=tf.compat.v1.global_variables()) load(loader, sess, ckpt.model_checkpoint_path) else: print('No checkpoint file found.') # -------------------------------------------------------------------- # Evaluate - Iterate over training steps. # -------------------------------------------------------------------- # evaluate training or validation set if evaluation_set == "val": imagereader_val = ImageReader(config.IMAGEREADER.VAL, config.DATASET_VAL, config.BATCH_SIZE, config.TIMESEQUENCE_LENGTH) elif evaluation_set == "train": imagereader_val = ImageReader(config.IMAGEREADER.VAL, config.DATASET_TRAIN, config.BATCH_SIZE, config.TIMESEQUENCE_LENGTH) elif evaluation_set == "test": imagereader_val = ImageReader(config.IMAGEREADER.VAL, config.DATASET_TEST, config.BATCH_SIZE, config.TIMESEQUENCE_LENGTH) elif evaluation_set == "all": imagereader_val = ImageReader(config.IMAGEREADER.VAL, config.DATASET_ALL, config.BATCH_SIZE, config.TIMESEQUENCE_LENGTH) else: print("Dataset {} does not exist!".format(evaluation_set)) acc_value = 0.0 # -------------------------------------- # perform evaluation - semantic segmentation # -------------------------------------- if config.ARCHITECTURE == 'semantic_segmentation': if config.TIMESEQUENCES_SLIDINGWINDOW: # use time sequences for step in trange( int(imagereader_val._dataset_amount - config.BATCH_SIZE * config.TIMESEQUENCE_LENGTH + 1), desc='evaluation', leave=True): start_time = time.time() training_batch = imagereader_val.getNextMinibatch() feed_dict = {input_placeholder['is_training']: False} for iter_GPU in range(len(input_placeholder['image_batch'])): num_GPU = len(input_placeholder['image_batch']) batch_size = training_batch['blob_data'].shape[0] batch_size_local = batch_size // num_GPU if (iter_GPU == (num_GPU - 1)): batch_size_act = batch_size - (num_GPU - 1) * ( batch_size // num_GPU) else: batch_size_act = batch_size // num_GPU feed_dict.update({ input_placeholder['image_batch'][iter_GPU]: training_batch['blob_data'][iter_GPU * batch_size_local:iter_GPU * batch_size_local + batch_size_act, :, :, :], input_placeholder['label_batch'][iter_GPU]: training_batch['blob_label'] [iter_GPU * batch_size_local:iter_GPU * batch_size_local + batch_size_act, :, :, :] }) duration = time.time() - start_time else: # do not use time sequences (normal evaluation) for step in trange( int(imagereader_val._dataset_amount / (config.BATCH_SIZE * config.TIMESEQUENCE_LENGTH)), desc='evaluation', leave=True): start_time = time.time() training_batch = imagereader_val.getNextMinibatch() feed_dict = {input_placeholder['is_training']: False} for iter_GPU in range(len(input_placeholder['image_batch'])): num_GPU = len(input_placeholder['image_batch']) batch_size = training_batch['blob_data'].shape[0] batch_size_local = batch_size // num_GPU if (iter_GPU == (num_GPU - 1)): batch_size_act = batch_size - (num_GPU - 1) * ( batch_size // num_GPU) else: batch_size_act = batch_size // num_GPU feed_dict.update({ input_placeholder['image_batch'][iter_GPU]: training_batch['blob_data'][iter_GPU * batch_size_local:iter_GPU * batch_size_local + batch_size_act, :, :, :], input_placeholder['label_batch'][iter_GPU]: training_batch['blob_label'] [iter_GPU * batch_size_local:iter_GPU * batch_size_local + batch_size_act, :, :, :] }) if determine_confusionMatrix: sess.run([update_op, confusion_update], feed_dict=feed_dict) else: sess.run([update_op], feed_dict=feed_dict) duration = time.time() - start_time mIoU_value = sess.run(mIoU) if determine_confusionMatrix: confusion_matrix = sess.run(confusion) # print Accuracy: np.set_printoptions(linewidth=np.inf) #150) acc_value = float(np.sum(np.diag(confusion_matrix))) / float( np.sum(confusion_matrix)) else: print( '[ERROR] network architecture does not exist!!! Please check your spelling!' ) raise NotImplementedError # -------------------------------------------- # close session # -------------------------------------------- sess.close() tf.compat.v1.reset_default_graph() # -------------------------------------------------------------------- # Show results # -------------------------------------------------------------------- if determine_confusionMatrix and config.ARCHITECTURE == 'semantic_segmentation': # determine class-wise IoU buff = 0.0 print('-------------------------------------------------------------') for iter in xrange(config.DATASET_VAL.NUM_CLASSES): if np.sum( confusion_matrix[iter, :]) == 0: # avoid division by zero IoU = 0.0 else: IoU = 100.0 * confusion_matrix[iter, iter] / ( np.sum(confusion_matrix[iter, :]) + np.sum(confusion_matrix[:, iter]) - confusion_matrix[iter, iter]) buff = buff + IoU print('{}: {}'.format(config.DATASET_VAL.CLASSES[iter], IoU)) print('-------------------------------------------------------------') print('dataset: {} - {}'.format(config.DATASET_NAME, config.DATASET_WEATHER)) print('Accuracy: {}'.format(acc_value)) print('mIoU: {}'.format(mIoU_value)) print('-------------------------------------------------------------') if plot_confusionMatrix and config.ARCHITECTURE == 'semantic_segmentation': print('-------------------------------------------------------------') print(confusion_matrix) print('-------------------------------------------------------------') plot_confusion_matrix(confusion_matrix, config.DATASET_VAL.CLASSES) return mIoU_value, acc_value
from tflearn.layers.core import input_data, dropout, fully_connected from tflearn.layers.conv import conv_2d, max_pool_2d from tflearn.layers.estimator import regression from tflearn.data_preprocessing import ImagePreprocessing from tflearn.data_augmentation import ImageAugmentation # Data loading and preprocessing from tflearn.datasets import cifar10 # (X, Y), (X_test, Y_test) = cifar10.load_data() label_num = 5 n_epoch = 500 # n_epoch = 5 batch_size = 96 image_size = (32, 32) (X, Y), (X_test, Y_test) = ImageReader.read_train_test_images_labels( '../imgs/faces02/trimmed', max_label=label_num, resize=image_size) # import ipdb # ipdb.set_trace() X, Y = shuffle(X, Y) Y = to_categorical(Y, label_num) Y_test = to_categorical(Y_test, label_num) # Real-time data preprocessing img_prep = ImagePreprocessing() img_prep.add_featurewise_zero_center() img_prep.add_featurewise_stdnorm() # Real-time data augmentation img_aug = ImageAugmentation() img_aug.add_random_flip_leftright() img_aug.add_random_rotation(max_angle=25.)
def main(): """Create the model and start the training.""" args = get_arguments() h, w = map(int, args.input_size.split(',')) input_size = (h, w) coord = tf.train.Coordinator() with tf.name_scope("create_inputs"): reader = ImageReader( args.data_list, input_size, args.random_scale, args.random_mirror, args.ignore_label, IMG_MEAN, coord) image_batch, label_batch = reader.dequeue(args.batch_size) net = ICNet_BN({'data': image_batch}, is_training = True, num_classes = args.num_classes) sub4_out = net.layers['sub4_out'] sub24_out = net.layers['sub24_out'] sub124_out = net.layers['conv6'] fc_list = ['conv6'] restore_var = tf.global_variables() all_trainable = [v for v in tf.trainable_variables() if ('beta' not in v.name and 'gamma' not in v.name) or args.train_beta_gamma] restore_var = [v for v in tf.global_variables() if not (len([f for f in fc_list if f in v.name])) or not args.not_restore_last] for v in restore_var: print(v.name) loss_sub4 = create_loss(sub4_out, label_batch, args.num_classes, args.ignore_label, args.use_class_weights) loss_sub24 = create_loss(sub24_out, label_batch, args.num_classes, args.ignore_label, args.use_class_weights) loss_sub124 = create_loss(sub124_out, label_batch, args.num_classes, args.ignore_label, args.use_class_weights) l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name] loss = LAMBDA1 * loss_sub4 + LAMBDA2 * loss_sub24 + LAMBDA3 * loss_sub124 reduced_loss = loss + tf.add_n(l2_losses) ############################## # visualization and summary ############################## # Processed predictions: for visualisation. # Sub 4 raw_output_up4 = tf.image.resize_bilinear(sub4_out, tf.shape(image_batch)[1:3,]) raw_output_up4 = tf.argmax(raw_output_up4, dimension = 3) pred4 = tf.expand_dims(raw_output_up4, dim = 3) # Sub 24 raw_output_up24 = tf.image.resize_bilinear(sub24_out, tf.shape(image_batch)[1:3,]) raw_output_up24 = tf.argmax(raw_output_up24, dimension=3) pred24 = tf.expand_dims(raw_output_up24, dim=3) # Sub 124 raw_output_up124 = tf.image.resize_bilinear(sub124_out, tf.shape(image_batch)[1:3,]) raw_output_up124 = tf.argmax(raw_output_up124, dimension=3) pred124 = tf.expand_dims(raw_output_up124, dim=3) images_summary = tf.py_func(inv_preprocess, [image_batch, SAVE_NUM_IMAGES, IMG_MEAN], tf.uint8) labels_summary = tf.py_func(decode_labels, [label_batch,SAVE_NUM_IMAGES, args.num_classes], tf.uint8) preds_summary4 = tf.py_func(decode_labels, [pred4, SAVE_NUM_IMAGES, args.num_classes], tf.uint8) preds_summary24 = tf.py_func(decode_labels, [pred24, SAVE_NUM_IMAGES, args.num_classes], tf.uint8) preds_summary124 = tf.py_func(decode_labels, [pred124, SAVE_NUM_IMAGES, args.num_classes], tf.uint8) total_images_summary = tf.summary.image('images', tf.concat(axis=2, values=[images_summary, labels_summary, preds_summary124]), max_outputs=SAVE_NUM_IMAGES) # Concatenate row-wise. total_summary = total_images_summary loss_summary = tf.summary.scalar('Total_loss', reduced_loss) #total_summary.append(loss_summary) summary_writer = tf.summary.FileWriter(args.snapshot_dir, graph=tf.get_default_graph()) ############################## ############################## base_lr = tf.constant(args.learning_rate) step_ph = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.train.exponential_decay(base_lr, step_ph, args.num_steps, args.power) lr_summary = tf.summary.scalar('Learning_rate', learning_rate) #total_summary.append(lr_summary) # Gets moving_mean and moving_variance update operations from tf.GraphKeys.UPDATE_OPS if args.update_mean_var == False: update_ops = None else: update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): opt_conv = tf.train.AdamOptimizer(learning_rate) #opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum) grads = tf.gradients(reduced_loss, all_trainable) train_op = opt_conv.apply_gradients(zip(grads, all_trainable)) # Set up tf session and initialize variables. config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) # Saver for storing checkpoints of the model. saver = tf.train.Saver(var_list = tf.global_variables(), max_to_keep = 10) ckpt = tf.train.get_checkpoint_state(args.snapshot_dir) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=restore_var) load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) load(loader, sess, ckpt.model_checkpoint_path) else: print('Restore from pre-trained model...') #net.load(args.restore_from, sess, ignore_layers = fc_list) # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) #summ_op = tf.summary.merge(total_summary) # Iterate over training steps. save_summary_every = 20 for step in range(args.num_steps): start_time = time.time() if LR_SHEDULE != {}: if step >= LR_SHEDULE.keys()[0]: tf.assign(learning_rate, LR_SHEDULE.popitem()[0]) feed_dict = {step_ph: step} if not (step % args.save_pred_every == 0): loss_value, loss1, loss2, loss3, _, lr_sum, l_sum = \ sess.run([reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op, lr_summary, loss_summary], feed_dict=feed_dict) else: save(saver, sess, args.snapshot_dir, step) loss_value, loss1, loss2, loss3, _, lr_sum, l_sum, t_sum = \ sess.run([reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op, lr_summary, loss_summary, total_summary], feed_dict=feed_dict) summary_writer.add_summary(t_sum, step) if step % save_summary_every == 0: summary_writer.add_summary(lr_sum, step) summary_writer.add_summary(l_sum, step) duration = time.time() - start_time print('step {:d} \t total loss = {:.3f}, sub4 = {:.3f}, sub24 = {:.3f}, sub124 = {:.3f} ({:.3f} sec/step)'.format(step, loss_value, loss1, loss2, loss3, duration)) coord.request_stop() coord.join(threads)
""" author: A Skrzypek """ from image_reader import ImageReader from make_plots import ImagePlot import numpy as np if __name__ == "__main__": # read images from path path_to_test_sample = r"C:\Users\user\Documents\IMAGE_PROJECT\sample_database\jpg_samples" sample_image_obj = ImageReader(path_to_test_sample) sample_image_obj.main() sample_plot_obj = ImagePlot(sample_image_obj.images_storage) sample_plot_obj.main() pass
def _tf_common_init(self): gpu_count = len(self.device_ids) src_size = self.config['input_size'] input_size_wh = (src_size['width'], src_size['height']) init_lr = self.config['lr'] power = self.config['lr_decreasing']['power'] momentum = self.config['momentum'] filter_scale = self.config['filter_scale'] weight_decay = self.config['weight_decay'] num_classes = len(self.out_classes) train_beta_gamma = self.config['train_beta_gamma'] update_mean_var = self.config['update_mean_var'] with tf.device('/cpu:0'): self.coord = tf.train.Coordinator() splitted_images = {} splitted_labels = {} with tf.name_scope("create_inputs"): for name, need_shuffle in [ ('train', True), ('val', False), ]: reader = ImageReader( ia_descrs=self.samples_dct[name], input_size_wh=input_size_wh, random_scale=False, random_mirror=False, img_mean=IMG_MEAN, coord=self.coord, in_pr_meta=self.helper.in_project_meta, class_to_idx=self.class_title_to_idx, shuffle=need_shuffle) batch_sz = self.config['batch_size'][name] img_batch, lbl_batch = reader.dequeue(batch_sz * gpu_count) split_images = tf.split(img_batch, gpu_count, 0) split_labels = tf.split(lbl_batch, gpu_count, 0) splitted_images[name] = split_images splitted_labels[name] = split_labels global_step = tf.get_variable( 'global_step', [], initializer=tf.constant_initializer(0), trainable=False, dtype=tf.int32) self.tf_label = tf.placeholder(dtype=tf.int32) # , shape=[None]) self.tf_prediction = tf.placeholder( dtype=tf.int32) # , shape=[None]) self.tf_metric, self.tf_metric_update = tf.metrics.accuracy( self.tf_label, self.tf_prediction, name="use_metric_acc") running_vars = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES, scope="use_metric_acc") self.running_vars_initializer = tf.variables_initializer( var_list=running_vars) base_lr = tf.constant(init_lr) self.step_ph = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.scalar_mul( base_lr, tf.pow((1 - self.step_ph / self.total_train_iters), power)) opt = tf.train.MomentumOptimizer(learning_rate, momentum) all_grads = [] losses = [] with tf.variable_scope(tf.get_variable_scope()): for curr_dev_id in self.device_ids: with tf.device('/gpu:{}'.format(curr_dev_id)): with tf.name_scope( 'clone_{}'.format(curr_dev_id)) as scope: spl_img = splitted_images['train'][curr_dev_id] spl_lbl = splitted_labels['train'][curr_dev_id] net = get_model(spl_img, num_classes, filter_scale) reduced_loss, internal_losses, prediction, gt, self.v1, self.v2 = \ forward_and_all_losses( net, spl_lbl, num_classes, self.neutral_input_idx, weight_decay ) # loss_sub4, loss_sub24, loss_sub124 = internal_losses losses.append(reduced_loss) tf.get_variable_scope().reuse_variables() grads = get_grads(reduced_loss, train_beta_gamma, update_mean_var) all_grads.append(grads) self.total_loss = tf.stack(values=losses) self.total_loss = tf.reduce_mean(self.total_loss) mean_grads = average_gradients(all_grads) all_trainable = get_trainable_vars(train_beta_gamma) # Apply the gradients to adjust the shared variables. self.train_op = opt.apply_gradients(zip(mean_grads, all_trainable), global_step=global_step) # # Group all updates to into a single train op. # train_op = tf.group(apply_gradient_conv_op, apply_gradient_fc_w_op, # apply_gradient_fc_b_op) self.total_val_loss, self.v1_val, self.v2_val = get_val_loss( splitted_images['val'], splitted_labels['val'], num_classes, weight_decay, self.device_ids, filter_scale, self.neutral_input_idx) # Set up tf session and initialize variables. config = tf.ConfigProto() config.gpu_options.allow_growth = True config.allow_soft_placement = True config.log_device_placement = False self.sess = tf.Session(config=config) init = tf.global_variables_initializer() self.sess.run(init) # Saver for storing checkpoints of the model. self.saver = tf.train.Saver(var_list=tf.global_variables(), save_relative_paths=True)
def run(self): coord = tf.train.Coordinator() tf.reset_default_graph() with tf.name_scope("create_inputs"): reader = ImageReader(self.data_dir, self.data_eval_list, self.input_size, None, None, self.ignore_label, self.img_mean, coord) image, label = reader.image, reader.label image_batch, label_batch = tf.expand_dims(image, dim=0), tf.expand_dims(label, dim=0) # Create network. net = PSPNet({'data': image_batch}, is_training=False, num_classes=self.num_classes) with tf.variable_scope('', reuse=True): flipped_img = tf.image.flip_left_right(image) flipped_img = tf.expand_dims(flipped_img, dim=0) net2 = PSPNet({'data': flipped_img}, is_training=False, num_classes=self.num_classes) # Predictions. raw_output = net.layers['conv6'] if self.is_flip: flipped_output = tf.image.flip_left_right( tf.squeeze(net2.layers['conv6'])) flipped_output = tf.expand_dims(flipped_output, dim=0) raw_output = tf.add_n([raw_output, flipped_output]) raw_output_up = tf.image.resize_bilinear(raw_output, size=self.input_size, align_corners=True) raw_output_up = tf.argmax(raw_output_up, axis=3) predictions_op = tf.expand_dims(raw_output_up, dim=3) # mIoU predictions_flatten = tf.reshape(predictions_op, [ -1, ]) raw_gt = tf.reshape(label_batch, [ -1, ]) indices = tf.squeeze( tf.where(tf.less_equal(raw_gt, self.num_classes - 1)), 1) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) predictions = tf.gather(predictions_flatten, indices) mIoU, update_op = tf.contrib.metrics.streaming_mean_iou( predictions, gt, num_classes=self.num_classes) # Set up tf session and initialize variables. config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) sess.run(tf.local_variables_initializer()) ckpt = tf.train.get_checkpoint_state(self.log_dir) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=tf.global_variables()) loader.restore(sess, ckpt.model_checkpoint_path) Tools.print_info("Restored model parameters from {}".format( ckpt.model_checkpoint_path)) else: Tools.print_info('No checkpoint file found.') # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) for step in range(self.num_steps): predictions_result, _ = sess.run([predictions, update_op]) if step > 0 and self.is_measure_time: self.calculate_time(sess, net) if step % 1 == 0: Tools.print_info('Finish {0}/{1} mIoU: {2}'.format( step, self.num_steps, sess.run(mIoU))) Tools.print_info('step {0} mIoU: {1}'.format(self.num_steps, sess.run(mIoU))) coord.request_stop() coord.join(threads) pass
def main(): """Create the model and start the training.""" args = get_arguments() h, w = map(int, args.input_size.split(',')) input_size = (h, w) coord = tf.train.Coordinator() with tf.name_scope("create_inputs"): reader = ImageReader( ' ', args.data_list, input_size, args.random_scale, args.random_mirror, args.ignore_label, IMG_MEAN, coord) image_batch, label_batch = reader.dequeue(args.batch_size) net = ICNet_BN({'data': image_batch}, is_training=True, num_classes=args.num_classes) sub4_out = net.layers['sub4_out'] sub24_out = net.layers['sub24_out'] sub124_out = net.layers['conv6_cls'] restore_var = tf.global_variables() all_trainable = [v for v in tf.trainable_variables() if ('beta' not in v.name and 'gamma' not in v.name) or args.train_beta_gamma] loss_sub4 = create_loss(sub4_out, label_batch, args.num_classes, args.ignore_label) loss_sub24 = create_loss(sub24_out, label_batch, args.num_classes, args.ignore_label) loss_sub124 = create_loss(sub124_out, label_batch, args.num_classes, args.ignore_label) l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name] reduced_loss = LAMBDA1 * loss_sub4 + LAMBDA2 * loss_sub24 + LAMBDA3 * loss_sub124 + tf.add_n(l2_losses) # Using Poly learning rate policy base_lr = tf.constant(args.learning_rate) step_ph = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power)) # Gets moving_mean and moving_variance update operations from tf.GraphKeys.UPDATE_OPS if args.update_mean_var == False: update_ops = None else: update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum) grads = tf.gradients(reduced_loss, all_trainable) train_op = opt_conv.apply_gradients(zip(grads, all_trainable)) # Set up tf session and initialize variables. config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) # Saver for storing checkpoints of the model. saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=5) ckpt = tf.train.get_checkpoint_state(args.snapshot_dir) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=restore_var) load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) load(loader, sess, ckpt.model_checkpoint_path) else: print('Restore from pre-trained model...') net.load(args.restore_from, sess) # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) # Iterate over training steps. for step in range(args.num_steps): start_time = time.time() feed_dict = {step_ph: step} if step % args.save_pred_every == 0: loss_value, loss1, loss2, loss3, _ = sess.run([reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op], feed_dict=feed_dict) save(saver, sess, args.snapshot_dir, step) else: loss_value, loss1, loss2, loss3, _ = sess.run([reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op], feed_dict=feed_dict) duration = time.time() - start_time print('step {:d} \t total loss = {:.3f}, sub4 = {:.3f}, sub24 = {:.3f}, sub124 = {:.3f} ({:.3f} sec/step)'.format(step, loss_value, loss1, loss2, loss3, duration)) coord.request_stop() coord.join(threads)
def evaluate_checkpoint(model_path, args): coord = tf.train.Coordinator() tf.reset_default_graph() reader = ImageReader( args.data_list, INPUT_SIZE, random_scale = False, random_mirror = False, ignore_label = IGNORE_LABEL, img_mean = IMG_MEAN, coord = coord, train = False) image_batch, label_batch = reader.dequeue(args.batch_size) # Set up tf session and initialize variables. config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) # Start queue threads. threads = tf.train.start_queue_runners(coord = coord, sess = sess) # Create network. #net = ICNet_BN({'data': image_batch}, is_training = False, num_classes = num_classes) net = unext(image_batch, is_train = False, n_out = NUM_CLASSES) # Predictions. #raw_output = net.layers['conv6'] raw_output = net.outputs raw_output_up = tf.image.resize_bilinear(raw_output, size = INPUT_SIZE, align_corners = True) raw_output_up = tf.argmax(raw_output_up, dimension = 3) pred = tf.expand_dims(raw_output_up, dim = 3) # mIoU pred_flatten = tf.reshape(pred, [-1,]) raw_gt = tf.reshape(label_batch, [-1,]) indices = tf.squeeze(tf.where(tf.not_equal(raw_gt, IGNORE_LABEL)), 1) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) pred = tf.gather(pred_flatten, indices) iou_metric, iou_op = tf.metrics.mean_iou(pred, gt, num_classes = num_classes) acc_metric, acc_op = tf.metrics.accuracy(pred, gt) # Summaries iou_summ_op = tf.summary.scalar('mIOU', iou_metric) acc_summ_op = tf.summary.scalar('Accuracy', acc_metric) start = time.time() logging.info('Starting evaluation at ' + time.strftime('%Y-%m-%d-%H:%M:%S', time.gmtime())) sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) saver = tf.train.Saver(var_list = tf.global_variables()) load(saver, sess, model_path) for step in range(int(num_steps / batch_size)): preds, _, _ = sess.run([raw_output_up, iou_op, acc_op]) if step % int(100 / batch_size) == 0: print('Finish {0}/{1}'.format(step + 1, int(num_steps / batch_size))) iou, iou_summ, acc, acc_summ = sess.run([iou_metric, iou_summ_op, acc_metric, acc_summ_op]) sess.close() coord.request_stop() #coord.join(threads) return iou, iou_summ, acc, acc_summ
def main(): """Create the model and start the training.""" args = get_arguments() h, w = map(int, args.input_size.split(',')) input_size = (h, w) coord = tf.train.Coordinator() with tf.name_scope("create_inputs"): reader = ImageReader( DATA_DIR, DATA_LIST_PATH, input_size, args.random_scale, args.random_mirror, args.ignore_label, IMG_MEAN, coord) image_batch, label_batch = reader.dequeue(args.batch_size) net = ICNet_BN({'data': image_batch}, is_training=True, num_classes=args.num_classes, filter_scale=args.filter_scale) sub4_out = net.layers['sub4_out'] sub24_out = net.layers['sub24_out'] sub124_out = net.layers['conv6_cls'] restore_var = tf.global_variables() all_trainable = [v for v in tf.trainable_variables() if ('beta' not in v.name and 'gamma' not in v.name) or args.train_beta_gamma] loss_sub4 = create_loss(sub4_out, label_batch, args.num_classes, args.ignore_label) loss_sub24 = create_loss(sub24_out, label_batch, args.num_classes, args.ignore_label) loss_sub124 = create_loss(sub124_out, label_batch, args.num_classes, args.ignore_label) l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name] reduced_loss = LAMBDA1 * loss_sub4 + LAMBDA2 * loss_sub24 + LAMBDA3 * loss_sub124 + tf.add_n(l2_losses) # Using Poly learning rate policy base_lr = tf.constant(args.learning_rate) step_ph = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power)) # Gets moving_mean and moving_variance update operations from tf.GraphKeys.UPDATE_OPS if args.update_mean_var == False: update_ops = None else: update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum) grads = tf.gradients(reduced_loss, all_trainable) train_op = opt_conv.apply_gradients(zip(grads, all_trainable)) # Set up tf session and initialize variables. config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) # Saver for storing checkpoints of the model. saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=5) ckpt = tf.train.get_checkpoint_state(args.snapshot_dir) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=restore_var) load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) load(loader, sess, ckpt.model_checkpoint_path) else: print('Restore from pre-trained model...') net.load(args.restore_from, sess) # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) # Iterate over training steps. for step in range(args.num_steps): start_time = time.time() feed_dict = {step_ph: step} if step % args.save_pred_every == 0: loss_value, loss1, loss2, loss3, _ = sess.run([reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op], feed_dict=feed_dict) save(saver, sess, args.snapshot_dir, step) else: loss_value, loss1, loss2, loss3, _ = sess.run([reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op], feed_dict=feed_dict) duration = time.time() - start_time print('step {:d} \t total loss = {:.3f}, sub4 = {:.3f}, sub24 = {:.3f}, sub124 = {:.3f} ({:.3f} sec/step)'.format(step, loss_value, loss1, loss2, loss3, duration)) coord.request_stop() coord.join(threads)
def main(): """Create the model and start the training.""" args = get_arguments() h, w = map(int, args.input_size.split(',')) input_size = (h, w) tf.set_random_seed(args.random_seed) coord = tf.train.Coordinator() with tf.name_scope("create_inputs"): reader = ImageReader(args.data_list, input_size, args.random_scale, args.random_mirror, args.ignore_label, IMG_MEAN, coord) image_batch, label_batch = reader.dequeue(args.batch_size) net = PSPNet50({'data': image_batch}, is_training=True, num_classes=args.num_classes) raw_output = net.layers['conv6'] # According from the prototxt in Caffe implement, learning rate must multiply by 10.0 in pyramid module fc_list = [ 'conv5_3_pool1_conv', 'conv5_3_pool2_conv', 'conv5_3_pool3_conv', 'conv5_3_pool6_conv', 'conv6', 'conv5_4' ] restore_var = [ v for v in tf.global_variables() if not (len([f for f in fc_list if f in v.name])) or not args.not_restore_last ] all_trainable = [ v for v in tf.trainable_variables() if 'gamma' not in v.name and 'beta' not in v.name ] fc_trainable = [ v for v in all_trainable if v.name.split('/')[0] in fc_list ] conv_trainable = [ v for v in all_trainable if v.name.split('/')[0] not in fc_list ] # lr * 1.0 fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name] # lr * 10.0 fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name] # lr * 20.0 assert (len(all_trainable) == len(fc_trainable) + len(conv_trainable)) assert (len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable)) # Predictions: ignoring all predictions with labels greater or equal than n_classes raw_prediction = tf.reshape(raw_output, [-1, args.num_classes]) label_proc = prepare_label(label_batch, tf.stack(raw_output.get_shape()[1:3]), num_classes=args.num_classes, one_hot=False) # [batch_size, h, w] raw_gt = tf.reshape(label_proc, [ -1, ]) indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)), 1) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) prediction = tf.gather(raw_prediction, indices) # Pixel-wise softmax loss. loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt) # Make mistakes for class N more important for network if USE_CLASS_WEIGHTS: if len(CLASS_WEIGHTS) != NUM_CLASSES: print('Incorrect class weights, it will be not used') else: mask = tf.zeros_like(loss) for i, w in enumerate(CLASS_WEIGHTS): mask = mask + tf.cast(tf.equal(gt, i), tf.float32) * tf.constant(w) loss = loss * mask l2_losses = [ args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name ] reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses) # Processed predictions: for visualisation. raw_output_up = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3, ]) raw_output_up = tf.argmax(raw_output_up, dimension=3) pred = tf.expand_dims(raw_output_up, dim=3) # Image summary. images_summary = tf.py_func(inv_preprocess, [image_batch, args.save_num_images, IMG_MEAN], tf.uint8) labels_summary = tf.py_func( decode_labels, [label_batch, args.save_num_images, args.num_classes], tf.uint8) preds_summary = tf.py_func(decode_labels, [pred, args.save_num_images, args.num_classes], tf.uint8) total_summary = tf.summary.image( 'images', tf.concat(axis=2, values=[images_summary, labels_summary, preds_summary]), max_outputs=args.save_num_images) # Concatenate row-wise. summary_writer = tf.summary.FileWriter(args.snapshot_dir, graph=tf.get_default_graph()) # Using Poly learning rate policy base_lr = tf.constant(args.learning_rate) step_ph = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.scalar_mul( base_lr, tf.pow((1 - step_ph / args.num_steps), args.power)) # Gets moving_mean and moving_variance update operations from tf.GraphKeys.UPDATE_OPS if args.update_mean_var == False: update_ops = None else: update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum) opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum) opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum) grads = tf.gradients(reduced_loss, conv_trainable + fc_w_trainable + fc_b_trainable) grads_conv = grads[:len(conv_trainable)] grads_fc_w = grads[len(conv_trainable):(len(conv_trainable) + len(fc_w_trainable))] grads_fc_b = grads[(len(conv_trainable) + len(fc_w_trainable)):] train_op_conv = opt_conv.apply_gradients( zip(grads_conv, conv_trainable)) train_op_fc_w = opt_fc_w.apply_gradients( zip(grads_fc_w, fc_w_trainable)) train_op_fc_b = opt_fc_b.apply_gradients( zip(grads_fc_b, fc_b_trainable)) train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b) # Set up tf session and initialize variables. config = tf.ConfigProto() # config.gpu_options.allow_growth = True # config.allow_soft_placement = True # config.intra_op_parallelism_threads = 1 sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) # Saver for storing checkpoints of the model. saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10) ckpt = tf.train.get_checkpoint_state(SNAPSHOT_DIR) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=restore_var) load_step = int( os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) load(loader, sess, ckpt.model_checkpoint_path) else: print('No checkpoint file found.') load_step = 0 # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) # Iterate over training steps. for step in range(args.num_steps): start_time = time.time() feed_dict = {step_ph: step} if step % args.save_pred_every == 0: loss_value, _, summary = sess.run( [reduced_loss, train_op, total_summary], feed_dict=feed_dict) summary_writer.add_summary(summary, step) save(saver, sess, args.snapshot_dir, step) else: z, t, o, p, loss_value, _ = sess.run( [raw_gt, raw_output, gt, prediction, reduced_loss, train_op], feed_dict=feed_dict) print(z.shape, t.shape, o.shape, p.shape) duration = time.time() - start_time print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format( step, loss_value, duration)) coord.request_stop() coord.join(threads)
def main(): """Create the model and start the training.""" args = get_arguments() h, w = map(int, args.input_size.split(',')) input_size = (h, w) #tf.set_random_seed(args.random_seed) coord = tf.train.Coordinator() with tf.Graph().as_default(), tf.device('/cpu:0'): # Using Poly learning rate policy base_lr = tf.constant(args.learning_rate) step_ph = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.train.exponential_decay(base_lr, step_ph, 20000, 0.5, staircase=True) tf.summary.scalar('lr', learning_rate) opt = tf.train.MomentumOptimizer(learning_rate, 0.9) #opt = tf.train.RMSPropOptimizer(learning_rate, 0.9, momentum=0.9, epsilon=1e-10) #opt = tf.train.AdamOptimizer(learning_rate) losses = [] train_op = [] total_batch_size = args.batch_size * args.gpu_nums with tf.name_scope('PSPnet') as scope: with tf.name_scope("create_inputs"): reader = ImageReader(args.data_dir, args.data_list, input_size, args.random_blur, args.random_scale, args.random_mirror, args.random_rotate, args.ignore_label, IMG_MEAN, coord) image_batch, label_batch = reader.dequeue(total_batch_size) images_splits = tf.split(axis=0, num_or_size_splits=args.gpu_nums, value=image_batch) labels_splits = tf.split(axis=0, num_or_size_splits=args.gpu_nums, value=label_batch) net = PSPNet({'data': images_splits}, is_training=True, num_classes=args.num_classes) raw_output_list = net.layers['conv6'] add_list = [ 'conv5_3_pool1_conv', 'conv5_3_pool1_conv_bn', 'conv5_3_pool2_conv', 'conv5_3_pool2_conv_bn', 'conv5_3_pool3_conv', 'conv5_3_pool3_conv_bn', 'conv5_3_pool6_conv', 'conv5_3_pool6_conv_bn', 'conv5_4', 'conv5_4_bn', 'conv6' ] num_valide_pixel = 0 for i in range(len(raw_output_list)): with tf.device('/gpu:%d' % i): raw_output_up = tf.image.resize_bilinear( raw_output_list[i], size=input_size, align_corners=True) tf.summary.image('images_{}'.format(i), images_splits[i] + IMG_MEAN, max_outputs=4) tf.summary.image('labels_{}'.format(i), labels_splits[i], max_outputs=4) tf.summary.image('predict_{}'.format(i), tf.cast( tf.expand_dims( tf.argmax(raw_output_up, -1), 3), tf.float32), max_outputs=4) all_trainable = [v for v in tf.trainable_variables()] # Predictions: ignoring all predictions with labels greater or equal than n_classes raw_prediction = tf.reshape(raw_output_up, [-1, args.num_classes]) label_proc = prepare_label( labels_splits[i], tf.stack(raw_output_up.get_shape()[1:3]), num_classes=args.num_classes, one_hot=False) # [batch_size, h, w] raw_gt = tf.reshape(label_proc, [ -1, ]) #indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)), 1) indices = tf.where( tf.logical_and(tf.less(raw_gt, args.num_classes), tf.greater_equal(raw_gt, 0))) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) prediction = tf.gather(raw_prediction, indices) mIoU, update_op = tf.contrib.metrics.streaming_mean_iou( tf.argmax(tf.nn.softmax(prediction), axis=-1), gt, num_classes=args.num_classes) tf.summary.scalar('mean IoU_{}'.format(i), mIoU) train_op.append(update_op) # Pixel-wise softmax loss. loss = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=prediction, labels=gt) num_valide_pixel += tf.shape(gt)[0] losses.append(tf.reduce_sum(loss)) l2_losses = [ args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name ] reduced_loss = tf.truediv( tf.reduce_sum(losses), tf.cast( num_valide_pixel, tf.float32)) + tf.add_n(l2_losses) tf.summary.scalar('average_loss', reduced_loss) grads = tf.gradients(reduced_loss, all_trainable, colocate_gradients_with_ops=True) variable_averages = tf.train.ExponentialMovingAverage(0.99, step_ph) variables_to_average = (tf.trainable_variables() + tf.moving_average_variables()) variables_averages_op = variable_averages.apply(variables_to_average) train_op = tf.group(opt.apply_gradients(zip(grads, all_trainable)), *train_op) train_op = tf.group(train_op, variables_averages_op) summary_op = tf.summary.merge_all() # Set up tf session and initialize variables. config = tf.ConfigProto() config.allow_soft_placement = True sess = tf.Session(config=config) init = [ tf.global_variables_initializer(), tf.local_variables_initializer() ] sess.run(init) # Saver for storing checkpoints of the model. saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=2) #restore from resnet imagenet, bised and local_step is in moving_average restore_var = [ v for v in tf.trainable_variables() if 'conv5_3_pool' not in v.name and 'conv5_4' not in v.name and 'conv6' not in v.name ] + [ v for v in tf.global_variables() if ('moving_mean' in v.name or 'moving_variance' in v.name) and ('conv5_3_pool' not in v.name and 'conv5_4_bn' not in v.name and 'biased' not in v.name and 'local_step' not in v.name) ] ckpt = tf.train.get_checkpoint_state(args.restore_from) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=restore_var) load(loader, sess, ckpt.model_checkpoint_path) else: print('No checkpoint file found.') """ #restore from snapshot restore_var = tf.global_variables() ckpt = tf.train.get_checkpoint_state(args.snapshot_dir) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=restore_var, allow_empty=True) load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) load(loader, sess, ckpt.model_checkpoint_path) else: print('No checkpoint file found.') load_step = 0 """ # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) summary_writer = tf.summary.FileWriter(args.snapshot_dir, graph=sess.graph) # Iterate over training steps. for step in range(args.num_steps): start_time = time.time() feed_dict = {step_ph: step} if step % args.save_pred_every == 0 and step != 0: loss_value, _ = sess.run([reduced_loss, train_op], feed_dict=feed_dict) save(saver, sess, args.snapshot_dir, step) elif step % 100 == 0: summary_str, loss_value, _, IOU = sess.run( [summary_op, reduced_loss, train_op, mIoU], feed_dict=feed_dict) duration = time.time() - start_time summary_writer.add_summary(summary_str, step) print( 'step {:d} \t loss = {:.3f}, mean_IoU = {:.3f}, ({:.3f} sec/step)' .format(step, loss_value, IOU, duration)) else: loss_value, _ = sess.run([reduced_loss, train_op], feed_dict=feed_dict) coord.request_stop() coord.join(threads)
def main(): train_list = list(np.load(config.data_file)['arr_0']) valid_list = list(np.load(config.data_file)['arr_1']) #print( type( train_list) ) #, type( train_aug_list ) ) #print( len( train_list) ) # len( train_aug_list ) ) #train_list = train_list[0:150000] #valid_list = valid_list[0:1000] print('Data detail') print( '{0}/{1} - images for training/validation'.format( \ len( train_list) , len( valid_list) ) ) # Prepare Theano variables for inputs and targets net, train_fn, val_fn = experiment.train_setup() # Finally, launch the training loop. print("Starting training...") # We iterate over epochs: reader_threads = [] for th in range(num_readers): t = ImageReader( dest_queue, source_queue, \ config.image_height, config.image_width, \ config.image_channel ) t.setDaemon(True) t.start() #reader_threads.append ( t ) #for th in reader_threads: # th,start() start_loss = config.start_loss for epoch in range(config.num_epochs): print('Running epoch: {0}'.format(epoch + 1)) # In each epoch, we do a full pass over the training data: train_loss = 0. train_batches = 0 start_time = time.time() for batch in ml_utility.iterate_minibatches( train_list, \ config.batch_size, \ shuffle=True): source_queue.put(batch) train_batches += 1 #print( 'Loading {0} batches'.format( train_batches ) ) train_batches -= 1 source_counter = source_queue.qsize() dest_counter = 0 while dest_counter < train_batches: if dest_queue.qsize() > 0: #print( 'Started batch training' ) inputs, targets = dest_queue.get() dest_queue.task_done() #assume color for now loss = train_fn(inputs, targets) train_loss += loss dest_counter += 1 print('trained with {0} batches'.format(dest_counter)) #print( train_batches ) # And a full pass over the validation data: val_loss = 0. val_acc = 0. val_batches = 0 val_preds = None val_truths = None print('validation starts ') for batch in ml_utility.iterate_minibatches2( valid_list, \ config.batch_size, \ config.image_channel, \ config.image_height, \ config.image_width ): #inputs, targets = ml_utility.imo2py( batch) inputs, targets = batch #assume color for now loss, pred_scores, acc = val_fn(inputs, targets) pred_scores = pred_scores[:, 1] if val_preds is None: val_preds = pred_scores val_truths = targets else: val_preds = np.concatenate( ( val_preds, pred_scores), \ axis = 0 ) val_truths = np.concatenate( (val_truths, targets ), \ axis = 0 ) val_loss += loss val_acc += np.sum(acc) val_batches += 1 # Then we print the results for this epoch: epoch_time = time.time() - start_time train_loss = train_loss / train_batches val_loss = val_loss / val_batches val_acc = val_acc / len(valid_list) val_preds = np.array(val_preds) val_truths = np.array(val_truths) val_preds = np.reshape(val_preds, (-1, 1)) val_truths = np.reshape(val_truths, (-1, 1)) val_class = np.argmax(val_preds, axis=1) #auc_score = roc_auc_score( val_truths, val_preds ) print("Epoch {} of {} took {:.3f}s".format(epoch + 1, config.num_epochs, time.time() - start_time)) print(" training loss:\t\t{:.6f}".format(train_loss)) print(" validation loss:\t\t{:.6f}".format(val_loss)) print(" Performance metrics ") print(" Validation Accuracy : {0}".format(val_acc * 100.)) #print(" AUC ROC:{0}".format( auc_score ) ) ml_utility.save_epoch_info( epoch+1,\ epoch_time,\ train_loss, \ val_loss, \ val_acc, \ config.stat_file ) #save the model after every 10 iterations if (val_loss <= start_loss): np.savez(config.model_file, *get_all_param_values(net)) start_loss = val_loss
def test_setup(self): # Create queue coordinator. self.coord = tf.train.Coordinator() # Load reader with tf.name_scope("create_inputs"): reader = ImageReader( self.conf.data_dir, self.conf.valid_data_list, None, # the images have different sizes False, # no data-aug False, # no data-aug self.conf.ignore_label, IMG_MEAN, self.coord) image, label = reader.image, reader.label # [h, w, 3 or 1] # Add one batch dimension [1, h, w, 3 or 1] self.image_batch, self.label_batch = tf.expand_dims( image, dim=0), tf.expand_dims(label, dim=0) # Create network if self.conf.encoder_name not in ['res101', 'res50', 'deeplab']: print('encoder_name ERROR!') print("Please input: res101, res50, or deeplab") sys.exit(-1) elif self.conf.encoder_name == 'deeplab': net = Deeplab_v2(self.image_batch, self.conf.num_classes, False) else: net = ResNet_segmentation(self.image_batch, self.conf.num_classes, False, self.conf.encoder_name) # predictions raw_output = net.outputs raw_output = tf.image.resize_bilinear( raw_output, tf.shape(self.image_batch)[1:3, ]) raw_output = tf.argmax(raw_output, axis=3) pred = tf.expand_dims(raw_output, dim=3) self.pred = tf.reshape(pred, [ -1, ]) # labels gt = tf.reshape(self.label_batch, [ -1, ]) # Ignoring all labels greater than or equal to n_classes. temp = tf.less_equal(gt, self.conf.num_classes - 1) weights = tf.cast(temp, tf.int32) # fix for tf 1.3.0 gt = tf.where(temp, gt, tf.cast(temp, tf.uint8)) # Pixel accuracy self.accu, self.accu_update_op = tf.contrib.metrics.streaming_accuracy( self.pred, gt, weights=weights) # mIoU self.mIoU, self.mIou_update_op = tf.contrib.metrics.streaming_mean_iou( self.pred, gt, num_classes=self.conf.num_classes, weights=weights) # confusion matrix self.confusion_matrix = tf.contrib.metrics.confusion_matrix( self.pred, gt, num_classes=self.conf.num_classes, weights=weights) # Loader for loading the checkpoint self.loader = tf.train.Saver(var_list=tf.global_variables())
def main(): temp_flags = FLAGS.__flags.items() temp_flags.sort() for params, value in FLAGS.__flags.items(): print('{}: {}'.format(params, value)) input_size = (FLAGS.train_image_size, FLAGS.train_image_size) tf.set_random_seed(1234) coord = tf.train.Coordinator() reader = ImageReader(FLAGS.data_dir, FLAGS.data_list, input_size, FLAGS.random_scale, FLAGS.random_mirror, FLAGS.ignore_label, IMG_MEAN, coord) image_batch, label_batch = reader.dequeue(FLAGS.batch_size) raw_output = MobileNet(image_batch, isTraining=True, updateBeta=FLAGS.update_beta) psp_list = [ 'conv_ds_15a', 'conv_ds_15b', 'conv_ds_15c', 'conv_ds_15d', 'conv_ds_16', 'conv_ds_17' ] all_trainable = [v for v in tf.trainable_variables()] if FLAGS.update_beta == False: all_trainable = [v for v in all_trainable if 'beta' not in v.name] psp_trainable = [ v for v in all_trainable if v.name.split('/')[1] in psp_list and ( 'weights' in v.name or 'biases' in v.name) ] conv_trainable = [v for v in all_trainable if v not in psp_trainable] # lr * 1.0 psp_w_trainable = [v for v in psp_trainable if 'weights' in v.name] # lr * 10.0 psp_b_trainable = [v for v in psp_trainable if 'biases' in v.name] # lr * 20.0 assert (len(all_trainable) == len(psp_trainable) + len(conv_trainable)) assert (len(psp_trainable) == len(psp_w_trainable) + len(psp_b_trainable)) # Predictions: ignoring all predictions with labels greater or equal than n_classes raw_prediction = tf.reshape(raw_output, [-1, FLAGS.num_classes]) label_proc = prepare_label(label_batch, tf.stack(raw_output.get_shape()[1:3]), num_classes=FLAGS.num_classes, one_hot=False) # [batch_size, h, w] raw_gt = tf.reshape(label_proc, [ -1, ]) indices = tf.squeeze( tf.where(tf.less_equal(raw_gt, FLAGS.num_classes - 1)), 1) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) prediction = tf.gather(raw_prediction, indices) # Pixel-wise softmax loss. loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt) # Regularisation loss l2_losses = [ FLAGS.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name ] reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses) #TODO auxilary loss #Using Poly learning rate policy current_epoch = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.train.polynomial_decay( FLAGS.start_learning_rate, current_epoch, FLAGS.decay_steps, end_learning_rate=FLAGS.end_learning_rate, power=FLAGS.learning_rate_decay_power, name="poly_learning_rate") if FLAGS.update_mean_var == False: update_ops = None else: update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): if FLAGS.optimizer == 'momentum': opt_conv = tf.train.MomentumOptimizer(learning_rate, FLAGS.momentum) opt_psp_w = tf.train.MomentumOptimizer(learning_rate * 10.0, FLAGS.momentum) opt_psp_b = tf.train.MomentumOptimizer(learning_rate * 20.0, FLAGS.momentum) elif FLAGS.optimizer == 'rmsprop': opt_conv = tf.train.RMSPropOptimizer( learning_rate, decay=FLAGS.rmsprop_decay, momentum=FLAGS.rmsprop_momentum, epsilon=FLAGS.opt_epsilon) opt_psp_w = tf.train.RMSPropOptimizer( learning_rate * 10.0, decay=FLAGS.rmsprop_decay, momentum=FLAGS.rmsprop_momentum, epsilon=FLAGS.opt_epsilon) opt_psp_b = tf.train.RMSPropOptimizer( learning_rate * 20.0, decay=FLAGS.rmsprop_decay, momentum=FLAGS.rmsprop_momentum, epsilon=FLAGS.opt_epsilon) grads = tf.gradients( reduced_loss, conv_trainable + psp_w_trainable + psp_b_trainable) grads_conv = grads[:len(conv_trainable)] grads_psp_w = grads[len(conv_trainable):(len(conv_trainable) + len(psp_w_trainable))] grads_psp_b = grads[(len(conv_trainable) + len(psp_w_trainable)):] train_op_conv = opt_conv.apply_gradients( zip(grads_conv, conv_trainable)) train_op_psp_w = opt_psp_w.apply_gradients( zip(grads_psp_w, psp_w_trainable)) train_op_psp_b = opt_psp_b.apply_gradients( zip(grads_psp_b, psp_b_trainable)) train_op = tf.group(train_op_conv, train_op_psp_w, train_op_psp_b) restore_var = tf.global_variables() # Set up tf session and initialize variables. config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) # Saver for storing checkpoints of the model. saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=500) load(sess, FLAGS.pretrained_checkpoint, restore_var) # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) # Iterate over training steps. for epoch in range(FLAGS.start_epoch, FLAGS.start_epoch + FLAGS.num_epochs): total_loss = 0.0 for step in range(1, FLAGS.num_steps + 1): start_time = time.time() feed_dict = {current_epoch: epoch} loss_value, _ = sess.run([reduced_loss, train_op], feed_dict=feed_dict) duration = time.time() - start_time print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format( step, loss_value, duration)) #TODO ignore NaN loss total_loss += loss_value save(saver, sess, FLAGS.log_dir, epoch) total_loss /= FLAGS.num_steps print('Epoch {:d} completed! Total Loss = {:.3f}'.format( epoch, total_loss)) coord.request_stop() coord.join(threads)
def main(): """Create the model and start the training.""" args = get_arguments() h, w = map(int, args.input_size.split(',')) input_size = (h, w) tf.set_random_seed(args.random_seed) coord = tf.train.Coordinator() with tf.name_scope("create_inputs"): reader = ImageReader( args.data_dir, args.data_list, input_size, args.random_scale, args.random_mirror, args.ignore_label, IMG_MEAN, coord) image_batch, label_batch = reader.dequeue(args.batch_size) net = PSPNet({'data': image_batch}, is_training=True, num_classes=args.num_classes) raw_output = net.layers['conv6'] # According from the prototxt in Caffe implement, learning rate must multiply by 10.0 in pyramid module fc_list = ['conv5_3_pool1_conv', 'conv5_3_pool2_conv', 'conv5_3_pool3_conv', 'conv5_3_pool6_conv', 'conv6', 'conv5_4'] restore_var = [v for v in tf.global_variables()] all_trainable = [v for v in tf.trainable_variables() if 'gamma' not in v.name and 'beta' not in v.name] fc_trainable = [v for v in all_trainable if v.name.split('/')[0] in fc_list] conv_trainable = [v for v in all_trainable if v.name.split('/')[0] not in fc_list] # lr * 1.0 fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name] # lr * 10.0 fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name] # lr * 20.0 assert(len(all_trainable) == len(fc_trainable) + len(conv_trainable)) assert(len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable)) # Predictions: ignoring all predictions with labels greater or equal than n_classes raw_prediction = tf.reshape(raw_output, [-1, args.num_classes]) label_proc = prepare_label(label_batch, tf.stack(raw_output.get_shape()[1:3]), num_classes=args.num_classes, one_hot=False) # [batch_size, h, w] raw_gt = tf.reshape(label_proc, [-1,]) indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)), 1) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) prediction = tf.gather(raw_prediction, indices) # Pixel-wise softmax loss. loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt) l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name] reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses) # Using Poly learning rate policy base_lr = tf.constant(args.learning_rate) step_ph = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power)) # Gets moving_mean and moving_variance update operations from tf.GraphKeys.UPDATE_OPS if args.update_mean_var == False: update_ops = None else: update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum) opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum) opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum) grads = tf.gradients(reduced_loss, conv_trainable + fc_w_trainable + fc_b_trainable) grads_conv = grads[:len(conv_trainable)] grads_fc_w = grads[len(conv_trainable) : (len(conv_trainable) + len(fc_w_trainable))] grads_fc_b = grads[(len(conv_trainable) + len(fc_w_trainable)):] train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable)) train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable)) train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable)) train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b) # Set up tf session and initialize variables. config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) # Saver for storing checkpoints of the model. saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10) ckpt = tf.train.get_checkpoint_state(SNAPSHOT_DIR) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=restore_var) load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) load(loader, sess, ckpt.model_checkpoint_path) else: print('No checkpoint file found.') load_step = 0 # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) # Iterate over training steps. for step in range(args.num_steps): start_time = time.time() feed_dict = {step_ph: step} if step % args.save_pred_every == 0: loss_value, _ = sess.run([reduced_loss, train_op], feed_dict=feed_dict) save(saver, sess, args.snapshot_dir, step) else: loss_value, _ = sess.run([reduced_loss, train_op], feed_dict=feed_dict) duration = time.time() - start_time print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(step, loss_value, duration)) coord.request_stop() coord.join(threads)
class Evaluator(): def __init__(self): self.stop = False if not tf.gfile.Exists(FLAGS.test_load_queue_path): self.settings = { "best_acc": None, "best_checkpoint": None, "last_checkpoint": None, "acc_increasing": None, "last_accs": deque() } else: self.settings = np.load(FLAGS.test_load_queue_path)[()] self.setup() def setup(self): self.recreate_directory_structure() self.coord = tf.train.Coordinator() # Load reader. with tf.name_scope("create_inputs"): self.reader = ImageReader("./val.npy", True, self.coord) self.image_batch, self.label_list_batch = self.reader.dequeue( FLAGS.batch_size) global_step = tf.Variable(0, dtype=tf.int32, name='global_step', trainable=False) self.net = CatznDogs({'data': self.image_batch}, global_step) # Set up tf session and initialize variables. config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) # Start queue threads. self.threads = tf.train.start_queue_runners(coord=self.coord, sess=self.sess) self.ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir) def run(self): while not self.stop: all_models_paths = self.ckpt.all_model_checkpoint_paths index_current_model = list(all_models_paths).index( self.ckpt.model_checkpoint_path) if self.settings["last_checkpoint"]: index_last_evaluated_model = list(all_models_paths).index( self.settings["last_checkpoint"]) else: index_last_evaluated_model = -1 if index_current_model != index_last_evaluated_model: index_model_under_evaluation = index_last_evaluated_model + 1 self.settings["last_checkpoint"] = all_models_paths[ index_model_under_evaluation] print("Evaluator started evaluating") acc_pred = self.net.test( self.image_batch, self.label_list_batch, self.coord, self.sess, self.reader.nb_samples, checkpoint_iteration=index_model_under_evaluation) self.settings["last_accs"].append(acc_pred) if not self.settings[ "best_acc"] or acc_pred < self.settings["best_acc"]: self.settings["best_acc"] = acc_pred self.settings["best_checkpoint"] = self.settings[ "last_checkpoint"] self.settings["acc_increasing"] = 0 else: self.settings["acc_increasing"] += 1 if self.settings["acc_increasing"] >= 5: self.stop = 1 np.save(FLAGS.test_load_queue_path, self.settings) print("Best model is {} with best Acc {}".format( self.settings["best_checkpoint"], self.settings["best_acc"])) else: time.sleep(10) self.coord.request_stop() self.coord.join(self.threads) print("Best model is {} with best Acc {}".format( self.settings["best_checkpoint"], self.settings["best_acc"])) def recreate_directory_structure(self): if not tf.gfile.Exists(FLAGS.test_summaries_dir): tf.gfile.MakeDirs(FLAGS.test_summaries_dir) else: tf.gfile.DeleteRecursively(FLAGS.test_summaries_dir) tf.gfile.MakeDirs(FLAGS.test_summaries_dir)