def main(_): config = process_config(FLAGS.config_path) print(config) tf.logging.set_verbosity(tf.logging.INFO) with tf.Graph().as_default(): deploy_config = deploy.DeploymentConfig(num_clones=1) global_step = tf.Variable(0, trainable=False, name='global_step') # select model and build net net = tdr2n2.Unet(config) # create batch dataset with tf.device(deploy_config.inputs_device()): data = DataGenerator(config.input) x_test, y_test = data.get_eval_data() x_test = tf.expand_dims(x_test, -1) x_test.set_shape([ None, config.input.img_out_shape[0], config.input.img_out_shape[1], config.input.img_out_shape[2] ]) y_test.set_shape([ None, config.input.mask_out_shape[0], config.input.mask_out_shape[1] ]) y_test = tf.cast(y_test, tf.int32) y_test_hot = tf.one_hot(y_test, depth=config.network.num_classes, axis=-1) f_score, end_points = net.net(x_test) f_score_img = tf.expand_dims( tf.cast(tf.argmax(f_score, axis=-1), tf.float32) * 50., -1) y_test_img = tf.expand_dims( tf.cast(tf.argmax(y_test_hot, axis=-1), tf.float32) * 50., -1) ## add precision and recall f_score = tf.cast(tf.argmax(f_score, -1), tf.int32) #f_score = tf.image.resize_bilinear(f_score, (config.input.img_out_shape[0])) f_score = tf.one_hot(f_score, depth=config.network.num_classes, axis=-1) pred = tf.reduce_sum(f_score * y_test_hot, axis=(0, 1, 2)) all_pred = tf.reduce_sum(f_score, axis=(0, 1, 2)) + 1e-5 all_true = tf.reduce_sum(y_test_hot, axis=(0, 1, 2)) + 1e-5 # Variables to restore: moving avg. or normal weights. if config.train.moving_average_decay: variable_averages = tf.train.ExponentialMovingAverage( config.train.moving_average_decay, global_step) variables_to_restore = variable_averages.variables_to_restore( slim.get_model_variables()) variables_to_restore[global_step.op.name] = global_step else: variables_to_restore = slim.get_variables_to_restore() saver = None if variables_to_restore is not None: saver = tf_saver.Saver(variables_to_restore) # =================================================================== # # Evaluation loop. # =================================================================== # gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=config.deploy.gpu_memory_fraction) configproto = tf.ConfigProto( gpu_options=gpu_options, log_device_placement=False, allow_soft_placement=True, ) merged = tf.summary.merge_all() sum_writer = tf.summary.FileWriter(logdir=config.summary.test_dir) for checkpoint_path in evaluation.checkpoints_iterator( config.finetune.eval_checkpoint_dir): with tf.Session(config=configproto) as session: session.run(tf.global_variables_initializer()) session.run(data.get_iterator(is_train=False).initializer) saver.restore(session, checkpoint_path) logging.info('Starting evaluation at ' + time.strftime('%Y-%m-%d-%H:%M:%S', time.gmtime())) k = 1 tp = [] tp_fp = [] tp_fn = [] imgs = [] while True: try: pred_, all_pred_, all_true_, pred_img, true_img, g_step = session.run( [ pred, all_pred, all_true, f_score_img, y_test_img, global_step ]) tp.append(np.expand_dims(pred_, 0)) tp_fp.append(np.expand_dims(all_true_, 0)) tp_fn.append(np.expand_dims(all_pred_, 0)) #img = util.merge_pics(pred_img, true_img) print("deal with {} images".format( k * config.input.batch_size)) k += 1 except tf.errors.OutOfRangeError: tp_ = np.sum(np.concatenate(tp, 0), 0) tp_fn_ = np.sum(np.concatenate(tp_fn, 0), 0) tp_fp_ = np.sum(np.concatenate(tp_fp, 0), 0) precison = tp_ / tp_fp_ recall = tp_ / tp_fn_ dice = 2 * tp_ / (tp_fp_ + tp_fn_) print(precison) print(recall) print(dice) summary = tf.Summary() for i in range(recall.shape[0]): summary.value.add( tag='evaluation/{}th_class_precision'.format( i), simple_value=precison[i]) summary.value.add( tag='evaluation/{}th_class_recall'.format(i), simple_value=recall[i]) summary.value.add( tag='evaluation/{}th_class_dice'.format(i), simple_value=dice[i]) sum_writer.add_summary(summary, g_step) break logging.info('Finished evaluation at ' + time.strftime('%Y-%m-%d-%H:%M:%S', time.gmtime()))
def main(_): # capture the config path from the run arguments # then process the json configration file config = process_config(FLAGS.config_path) print(config) tf.logging.set_verbosity(tf.logging.DEBUG) with tf.Graph().as_default(): ###################### # Config model_deploy# ###################### deploy_config = deploy.DeploymentConfig( num_clones=config.deploy.num_clone) # Create global_step with tf.device(deploy_config.variables_device()): global_step = tf.Variable(0, trainable=False, name='global_step') # select model and build net net = u_net.Unet(config) # create batch dataset with tf.device(deploy_config.inputs_device()): data = DataGenerator(config.input) x_train, y_train = data.get_train_data() x_train = tf.expand_dims(x_train, -1) x_train.set_shape([ None, config.input.img_out_shape[0], config.input.img_out_shape[1], config.input.img_out_shape[2] ]) y_train.set_shape([ None, config.input.mask_out_shape[0], config.input.mask_out_shape[1] ]) y_train = tf.cast(y_train, tf.int32) y_train_hot = tf.one_hot(y_train, depth=config.network.num_classes, axis=-1) batch_queue = [x_train, y_train_hot] # =================================================================== # # Define the model running on every GPU. # =================================================================== # def clone_fn(batch_queue): x_train, y_train_hot = batch_queue print(x_train) print(y_train_hot) f_score, end_points = net.net(x_train) # Add loss function. net.loss(f_score, y_train_hot, type=config.network.loss_type) return f_score, end_points, x_train, y_train_hot summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES)) clones = deploy.create_clones(deploy_config, clone_fn, [batch_queue]) first_clone_scope = deploy_config.clone_scope(0) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, first_clone_scope) for loss in tf.get_collection('EXTRA_LOSSES', first_clone_scope): summaries.add(tf.summary.scalar(loss.op.name, loss)) f_score, _, x_train, y_train_hot = clones[0].outputs f_score_img = tf.expand_dims( tf.cast(tf.argmax(f_score, axis=-1), tf.float32), -1) y_train_img = tf.argmax(y_train_hot, axis=-1) summaries.add(tf.summary.image("Images/Original_image", x_train, 2)) summaries.add( tf.summary.image( "Images/Ground_truth", tf.expand_dims(tf.cast(y_train_img, tf.float32), -1), 2)) summaries.add(tf.summary.image("Images/Predict_", f_score_img, 2)) ## add precision and recall f_score = tf.cast(tf.argmax(f_score, -1), tf.int32) f_score = tf.one_hot(f_score, depth=config.network.num_classes, axis=-1) pred = tf.reduce_sum(f_score * y_train_hot, axis=(0, 1, 2)) all_pred = tf.reduce_sum(f_score, axis=(0, 1, 2)) + 1e-5 all_true = tf.reduce_sum(y_train_hot, axis=(0, 1, 2)) + 1e-5 recall = pred / all_pred prec = pred / all_true dice = pred * 2 / (all_true + all_pred) with tf.variable_scope('evaluation'): for i in range(config.network.num_classes): summaries.add( tf.summary.scalar('{}th_class_precision'.format(i), prec[i])) summaries.add( tf.summary.scalar('{}th_class_recall'.format(i), recall[i])) summaries.add( tf.summary.scalar('{}th_class_dice'.format(i), dice[i])) ################################# # Configure the moving averages # ################################# if config.train.moving_average_decay: moving_average_variables = slim.get_model_variables() variable_averages = tf.train.ExponentialMovingAverage( config.train.moving_average_decay, global_step) else: moving_average_variables, variable_averages = None, None ######################################### # Configure the optimization procedure. # ######################################### with tf.device(deploy_config.optimizer_device()): learning_rate = tf_utils.configure_learning_rate( config, global_step) optimizer = tf_utils.configure_optimizer(config.train, learning_rate) if config.train.moving_average_decay: update_ops.append( variable_averages.apply(moving_average_variables)) # Variables to train. variables_to_train = tf_utils.get_variables_to_train(config.finetune) # and returns a train_tensor and summary_op total_loss, clones_gradients = deploy.optimize_clones( clones, optimizer, var_list=variables_to_train) if config.train.clip_gradient_norm > 0: with ops.name_scope('clip_grads'): clones_gradients = slim.learning.clip_gradient_norms( clones_gradients, config.train.clip_gradient_norm) # Create gradient updates. grad_updates = optimizer.apply_gradients(clones_gradients, global_step=global_step) update_ops.append(grad_updates) update_op = tf.group(*update_ops) train_tensor = control_flow_ops.with_dependencies([update_op], total_loss, name='train_op') # train_tensor = slim.learning.create_train_op(total_loss, optimizer, gradient_multipliers=gradient_multipliers) summaries.add(tf.summary.scalar('learning_rate', learning_rate)) summaries.add(tf.summary.scalar('total_loss', total_loss)) summaries |= set( tf.get_collection(tf.GraphKeys.SUMMARIES, first_clone_scope)) # Merge all summaries together. summary_op = tf.summary.merge(list(summaries), name='summary_op') # =================================================================== # # Kicks off the training. # =================================================================== # gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=config.deploy.gpu_memory_fraction) configproto = tf.ConfigProto( gpu_options=gpu_options, log_device_placement=False, allow_soft_placement=True, ) saver = tf.train.Saver(max_to_keep=100) scaffold = tf.train.Scaffold( init_op=None, init_feed_dict=None, init_fn=tf_utils.get_init_fn(config), ready_op=None, ready_for_local_init_op=None, local_init_op=[data.get_iterator().initializer], summary_op=summary_op, saver=saver, copy_from_scaffold=None) ckpt_hook = tf.train.CheckpointSaverHook( checkpoint_dir=config.summary.train_dir, save_secs=config.summary.save_checkpoint_secs, save_steps=config.summary.save_checkpoint_steps, saver=None, checkpoint_basename='model.ckpt', scaffold=scaffold, listeners=None) sum_writer = tf.summary.FileWriter(logdir=config.summary.train_dir) sum_hook = tf.train.SummarySaverHook( save_steps=None, save_secs=config.summary.save_summaries_secs, output_dir=config.summary.train_dir, summary_writer=sum_writer, scaffold=None, summary_op=summary_op, ) with tf.train.MonitoredTrainingSession( master='', is_chief=True, checkpoint_dir=config.summary.train_dir, scaffold=scaffold, hooks=[ckpt_hook, sum_hook], save_checkpoint_secs=None, save_summaries_steps=None, save_summaries_secs=None, config=configproto, log_step_count_steps=config.summary.log_every_n_steps) as sess: while not sess.should_stop(): _, loss, g_step = sess.run( [train_tensor, total_loss, global_step]) print("{} step loss is {}".format(g_step, loss))
x_train, y_train = dataset_train.get_train_data() sess = tf.train.MonitoredTrainingSession( master='', is_chief=True, checkpoint_dir=None, scaffold=None, hooks=None, chief_only_hooks=None, save_checkpoint_secs=600, save_summaries_steps=100, save_summaries_secs=None, config=None, stop_grace_period_secs=120, log_step_count_steps=100 ) step = 0 sess.run(dataset_train.get_iterator().initializer) while not sess.should_stop(): start_time = datetime.datetime.now() g_step = sess.run([global_step]) print(g_step) step += 1
config.input.img_out_shape[2] ]) y_test.set_shape( [None, config.input.mask_out_shape[0], config.input.mask_out_shape[1]]) print(x_train) #print(y_test) scaffold = tf.train.Scaffold( init_op=None, init_feed_dict=None, init_fn=None, ready_op=None, ready_for_local_init_op=None, local_init_op=[ dataset.get_iterator().initializer, dataset.get_iterator(is_train=False).initializer ], summary_op=None, saver=None, copy_from_scaffold=None) sess = tf.train.MonitoredTrainingSession(master='', is_chief=True, checkpoint_dir=None, scaffold=scaffold, hooks=None, chief_only_hooks=None, save_checkpoint_secs=600, save_summaries_steps=100, save_summaries_secs=None,