Exemple #1
0
def run_training(continue_run):

    logging.info('EXPERIMENT NAME: %s' % exp_config.experiment_name)
    already_created_recursion = False
    print("ALready created recursion : " + str(already_created_recursion))
    init_step = 0
    # Load data
    base_data, recursion_data, recursion = acdc_data.load_and_maybe_process_scribbles(
        scribble_file=sys_config.project_root + exp_config.scribble_data,
        target_folder=log_dir,
        percent_full_sup=exp_config.percent_full_sup,
        scr_ratio=exp_config.length_ratio)
    #wrap everything from this point onwards in a try-except to catch keyboard interrupt so
    #can control h5py closing data
    try:
        loaded_previous_recursion = False
        start_epoch = 0
        if continue_run:
            logging.info(
                '!!!!!!!!!!!!!!!!!!!!!!!!!!!! Continuing previous run !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
            )
            try:
                try:
                    init_checkpoint_path = utils.get_latest_model_checkpoint_path(
                        log_dir, 'recursion_{}_model.ckpt'.format(recursion))

                except:
                    print("EXCEPTE GİRDİ")
                    init_checkpoint_path = utils.get_latest_model_checkpoint_path(
                        log_dir,
                        'recursion_{}_model.ckpt'.format(recursion - 1))
                    loaded_previous_recursion = True
                logging.info('Checkpoint path: %s' % init_checkpoint_path)
                init_step = int(
                    init_checkpoint_path.split('/')[-1].split('-')
                    [-1]) + 1  # plus 1 b/c otherwise starts with eval
                start_epoch = int(init_step /
                                  (len(base_data['images_train']) / 4))
                logging.info('Latest step was: %d' % init_step)
                logging.info('Continuing with epoch: %d' % start_epoch)
            except:
                logging.warning(
                    '!!! Did not find init checkpoint. Maybe first run failed. Disabling continue mode...'
                )
                continue_run = False
                init_step = 0
                start_epoch = 0

            logging.info(
                '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
            )

        if loaded_previous_recursion:
            logging.info(
                "Data file exists for recursion {} "
                "but checkpoints only present up to recursion {}".format(
                    recursion, recursion - 1))
            logging.info("Likely means postprocessing was terminated")

            if not already_created_recursion:

                recursion_data = acdc_data.load_different_recursion(
                    recursion_data, -1)
                recursion -= 1
            else:
                start_epoch = 0
                init_step = 0
        # load images and validation data
        images_train = np.array(base_data['images_train'])
        #        scribbles_train = np.array(base_data['scribbles_train'])
        labels_train = np.array(base_data['masks_train'])
        images_val = np.array(base_data['images_val'])
        labels_val = np.array(base_data['masks_val'])
        slices_val = np.array(base_data['slices_validation'])
        # if exp_config.use_data_fraction:
        #     num_images = images_train.shape[0]
        #     new_last_index = int(float(num_images)*exp_config.use_data_fraction)
        #
        #     logging.warning('USING ONLY FRACTION OF DATA!')
        #     logging.warning(' - Number of imgs orig: %d, Number of imgs new: %d' % (num_images, new_last_index))
        #     images_train = images_train[0:new_last_index,...]
        #     labels_train = labels_train[0:new_last_index,...]

        logging.info('Data summary:')
        logging.info(' - Images:')
        logging.info(images_train.shape)
        logging.info(images_train.dtype)
        #logging.info(' - Labels:')
        #logging.info(labels_train.shape)
        #logging.info(labels_train.dtype)

        # Tell TensorFlow that the model will be built into the default Graph.
        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        config.allow_soft_placement = True

        if not os.path.exists(validation_res_path):
            os.makedirs(validation_res_path)
#        with tf.Graph().as_default():
        with tf.Session(config=config) as sess:
            # Generate placeholders for the images and labels.

            image_tensor_shape = [exp_config.batch_size] + list(
                exp_config.image_size) + [1]
            mask_tensor_shape = [exp_config.batch_size] + list(
                exp_config.image_size)
            print("Exp config image size : " + str(exp_config.image_size))
            images_placeholder = tf.placeholder(tf.float32,
                                                shape=image_tensor_shape,
                                                name='images')
            labels_placeholder = tf.placeholder(tf.uint8,
                                                shape=mask_tensor_shape,
                                                name='labels')

            learning_rate_placeholder = tf.placeholder(tf.float32, shape=[])
            crf_learning_rate_placeholder = tf.placeholder(tf.float32,
                                                           shape=[])
            training_time_placeholder = tf.placeholder(tf.bool, shape=[])

            tf.summary.scalar('learning_rate', learning_rate_placeholder)

            # Build a Graph that computes predictions from the inference model.
            logits = model.inference(images_placeholder,
                                     exp_config.model_handle,
                                     training=training_time_placeholder,
                                     nlabels=exp_config.nlabels)

            # Add to the Graph the Ops for loss calculation.
            [loss, _, weights_norm
             ] = model.loss(logits,
                            labels_placeholder,
                            nlabels=exp_config.nlabels,
                            loss_type=exp_config.loss_type,
                            weight_decay=exp_config.weight_decay
                            )  # second output is unregularised loss

            tf.summary.scalar('loss', loss)
            tf.summary.scalar('weights_norm_term', weights_norm)

            crf_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
                                              scope='crf_scope')

            restore_var = [
                v for v in tf.all_variables() if v.name not in crf_variables
            ]

            global_step = tf.Variable(0, name='global_step', trainable=False)

            network_train_op = tf.train.AdamOptimizer(
                learning_rate=learning_rate_placeholder).minimize(
                    loss,
                    var_list=restore_var,
                    colocate_gradients_with_ops=True,
                    global_step=global_step)

            crf_train_op = tf.train.AdamOptimizer(
                learning_rate=crf_learning_rate_placeholder).minimize(
                    loss,
                    var_list=crf_variables,
                    colocate_gradients_with_ops=True,
                    global_step=global_step)

            # Build the summary Tensor based on the TF collection of Summaries.
            summary = tf.summary.merge_all()

            # Add the variable initializer Op.
            init = tf.global_variables_initializer()

            # Create a saver for writing training checkpoints.
            # Only keep two checkpoints, as checkpoints are kept for every recursion
            # and they can be 300MB +
            saver = tf.train.Saver(max_to_keep=2)
            saver_best_dice = tf.train.Saver(max_to_keep=2)
            saver_best_xent = tf.train.Saver(max_to_keep=2)

            # Create a session for running Ops on the Graph.
            sess = tf.Session()

            # Instantiate a SummaryWriter to output summaries and the Graph.
            summary_writer = tf.summary.FileWriter(log_dir, sess.graph)

            # with tf.name_scope('monitoring'):

            val_error_ = tf.placeholder(tf.float32, shape=[], name='val_error')
            val_error_summary = tf.summary.scalar('validation_loss',
                                                  val_error_)

            val_dice_ = tf.placeholder(tf.float32, shape=[], name='val_dice')
            val_dice_summary = tf.summary.scalar('validation_dice', val_dice_)

            val_summary = tf.summary.merge(
                [val_error_summary, val_dice_summary])

            train_error_ = tf.placeholder(tf.float32,
                                          shape=[],
                                          name='train_error')
            train_error_summary = tf.summary.scalar('training_loss',
                                                    train_error_)

            train_dice_ = tf.placeholder(tf.float32,
                                         shape=[],
                                         name='train_dice')
            train_dice_summary = tf.summary.scalar('training_dice',
                                                   train_dice_)

            train_summary = tf.summary.merge(
                [train_error_summary, train_dice_summary])

            # Run the Op to initialize the variables.
            sess.run(init)

            #            crf_training_variables =tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='crf_training_op')
            #            print(str(crf_training_variables))
            #            all_crf_variables = crf_training_variables+crf_variables
            #            print(str(all_crf_variables))

            #/////////////////////

            #            load_saver = tf.train.Saver(var_list=load_variables)
            #            load_saver.restore(sess, '/scratch_net/biwirender02/cany/basil/logdir/unet2D_ws_spot_blur/recursion_0_model.ckpt-7499')

            #///////////////////////////

            #            if continue_run:
            #                # Restore session
            #                saver.restore(sess, init_checkpoint_path)
            #            saver.restore(sess,'/scratch_net/biwirender02/cany/scribble/logdir/prostate_deep_rnn_exp/recursion_0_model_best_dice.ckpt-4999')
            ##
            #            saver.restore(sess,'/scratch_net/biwirender02/cany/scribble/logdir/prostate_deep_rnn_exp/recursion_0_model_best_dice.ckpt-5499')
            ##
            init_step = 0
            start_epoch = 0
            recursion = 0

            step = init_step
            curr_lr = exp_config.learning_rate / 10
            crf_curr_lr = 1e-08
            no_improvement_counter = 0
            best_val = np.inf
            last_train = np.inf
            loss_history = []
            loss_gradient = np.inf
            best_dice = 0
            logging.info('RECURSION {0}'.format(recursion))

            # random walk - if it already has been random walked it won't redo

            print("Start epoch : " + str(start_epoch) + " : max epochs : " +
                  str(exp_config.epochs_per_recursion))
            for epoch in range(start_epoch, exp_config.max_epochs):

                logging.info(
                    'Epoch {0} ({1} of {2} epochs for recursion {3})'.format(
                        epoch, 1 + epoch % exp_config.epochs_per_recursion,
                        exp_config.epochs_per_recursion, recursion))
                # for batch in iterate_minibatches(images_train,
                #                                  labels_train,
                #                                  batch_size=exp_config.batch_size,
                #                                  augment_batch=exp_config.augment_batch):

                # You can run this loop with the BACKGROUND GENERATOR, which will lead to some improvements in the
                # training speed. However, be aware that currently an exception inside this loop may not be caught.
                # The batch generator may just continue running silently without warning even though the code has
                # crashed.

                for batch in BackgroundGenerator(
                        iterate_minibatches(
                            images_train,
                            labels_train,
                            batch_size=exp_config.batch_size,
                            augment_batch=exp_config.augment_batch)):

                    if exp_config.warmup_training:
                        if step < 50:
                            curr_lr = exp_config.learning_rate / 10.0
                        elif step == 50:
                            curr_lr = exp_config.learning_rate
                    if ((step % 5000 == 0) & (step > 0)):
                        curr_lr = curr_lr * 0.94
                        crf_curr_lr = crf_curr_lr * 0.94
                    start_time = time.time()

                    # batch = bgn_train.retrieve()
                    x, y = batch

                    # TEMPORARY HACK (to avoid incomplete batches
                    if y.shape[0] < exp_config.batch_size:
                        step += 1
                        continue

                    network_feed_dict = {
                        images_placeholder: x,
                        labels_placeholder: y,
                        learning_rate_placeholder: curr_lr,
                        training_time_placeholder: True
                    }

                    crf_feed_dict = {
                        images_placeholder: x,
                        labels_placeholder: y,
                        crf_learning_rate_placeholder: crf_curr_lr,
                        training_time_placeholder: True
                    }

                    if (step % 10 == 0):
                        _, loss_value = sess.run([crf_train_op, loss],
                                                 feed_dict=crf_feed_dict)
                    _, loss_value = sess.run([network_train_op, loss],
                                             feed_dict=network_feed_dict)
                    duration = time.time() - start_time

                    # Write the summaries and print an overview fairly often.
                    if step % 10 == 0:
                        # Print status to stdout.
                        logging.info('Step %d: loss = %.6f (%.3f sec)' %
                                     (step, loss_value, duration))
                        # Update the events file.

                    # Save a checkpoint and evaluate the model periodically.
                    if (step + 1) % exp_config.val_eval_frequency == 0:

                        checkpoint_file = os.path.join(
                            log_dir,
                            'recursion_{}_model.ckpt'.format(recursion))
                        saver.save(sess, checkpoint_file, global_step=step)
                        # Evaluate against the training set.

                        # Evaluate against the validation set.
                        logging.info('Validation Data Eval:')
                        [val_loss, val_dice, hard_pred, labels,
                         cdice] = do_eval(sess, logits, images_placeholder,
                                          labels_placeholder,
                                          training_time_placeholder,
                                          images_val, labels_val,
                                          exp_config.batch_size, slices_val)

                        val_summary_msg = sess.run(val_summary,
                                                   feed_dict={
                                                       val_error_: val_loss,
                                                       val_dice_: val_dice
                                                   })
                        summary_writer.add_summary(val_summary_msg, step)

                        if val_dice > best_dice:
                            best_dice = val_dice
                            best_file = os.path.join(
                                log_dir,
                                'recursion_{}_model_best_dice.ckpt'.format(
                                    recursion))
                            saver_best_dice.save(sess,
                                                 best_file,
                                                 global_step=step)
                            logging.info(
                                'Found new best dice on validation set! - {} - '
                                'Saving recursion_{}_model_best_dice.ckpt'.
                                format(val_dice, recursion))
                            text_file = open('val_results.txt', "a")
                            text_file.write("\nVal dice " + str(step) + " : " +
                                            str(val_dice))
                            text_file.close()


#                            sio.savemat( validation_res_path+ '/result'+'_'+str(step)+'.mat', {'pred':np.float32(hard_pred),
#                    'labels':np.float32(labels), 'dices':np.asarray(cdice)})
                        if val_loss < best_val:
                            best_val = val_loss
                            best_file = os.path.join(
                                log_dir,
                                'recursion_{}_model_best_xent.ckpt'.format(
                                    recursion))
                            saver_best_xent.save(sess,
                                                 best_file,
                                                 global_step=step)
                            logging.info(
                                'Found new best crossentropy on validation set! - {} - '
                                'Saving recursion_{}_model_best_xent.ckpt'.
                                format(val_loss, recursion))

                    step += 1

    except Exception:
        raise
def run_training(continue_run):

    logging.info('EXPERIMENT NAME: %s' % exp_config.experiment_name)
    already_created_recursion = False
    print("ALready created recursion : " + str(already_created_recursion))

    base_data = h5py.File(os.path.join(log_dir, 'base_data.hdf5'), 'r')
    
    try:
       
  
        images_val = np.array(base_data['images_val'])
        labels_val = np.array(base_data['masks_val'])
        slices_val = np.array(base_data['slices_validation'])
     

  

        # Tell TensorFlow that the model will be built into the default Graph.
        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        config.allow_soft_placement = True
        
        
        
        if not os.path.exists(validation_res_path):
            os.makedirs(validation_res_path)
#        with tf.Graph().as_default():
        with tf.Session(config = config) as sess:
            # Generate placeholders for the images and labels.

            image_tensor_shape = [exp_config.batch_size] + list(exp_config.image_size) + [1]
            mask_tensor_shape = [exp_config.batch_size] + list(exp_config.image_size)
            print("Exp config image size : " + str(exp_config.image_size))
            images_placeholder = tf.placeholder(tf.float32, shape=image_tensor_shape, name='images')
            labels_placeholder = tf.placeholder(tf.uint8, shape=mask_tensor_shape, name='labels')

            learning_rate_placeholder = tf.placeholder(tf.float32, shape=[])
            crf_learning_rate_placeholder = tf.placeholder(tf.float32, shape=[])
            training_time_placeholder = tf.placeholder(tf.bool, shape=[])

            tf.summary.scalar('learning_rate', learning_rate_placeholder)

            # Build a Graph that computes predictions from the inference model.
            logits = model.inference(images_placeholder,
                                     exp_config.model_handle,
                                     training=training_time_placeholder,
                                     nlabels=exp_config.nlabels)



            # Add the variable initializer Op.
            init = tf.global_variables_initializer()



            # Create a session for running Ops on the Graph.
            sess = tf.Session()

            saver = tf.train.Saver(max_to_keep=2)

            # Run the Op to initialize the variables.
            sess.run(init)
            


            saver.restore(sess,'/scratch_net/biwirender02/cany/scribble/logdir/prostate_full_sup_new_crf/recursion_1_model_best_dice.ckpt-6599')




            


      
                  

            # Evaluate against the validation set.
            logging.info('Validation Data Eval:')
            [val_loss, val_dice, hard_pred, labels, cdice] = do_eval(sess,
                                           logits,
                                           images_placeholder,
                                           labels_placeholder,
                                           training_time_placeholder,
                                           images_val,
                                           labels_val,
                                           exp_config.batch_size,slices_val,validation_res_path)
            logging.info('Found new best dice on validation set! - {} - '
                          .format(val_dice))
           
                
            sio.savemat( validation_res_path+ '/result'+'_'+str(step)+'.mat', {'pred':np.float32(hard_pred),
        'labels':np.float32(labels), 'dices':np.asarray(cdice)})
           
    
            

  

    except Exception:
        raise