def run_training(continue_run):
logging.info('EXPERIMENT NAME: %s' % exp_config.experiment_name)
already_created_recursion = True
print("ALready created recursion : " + str(already_created_recursion))
init_step = 0
# Load data
base_data, recursion_data, recursion = acdc_data.load_and_maybe_process_scribbles(
scribble_file=sys_config.project_root + exp_config.scribble_data,
target_folder=log_dir,
percent_full_sup=exp_config.percent_full_sup,
scr_ratio=exp_config.length_ratio)
#wrap everything from this point onwards in a try-except to catch keyboard interrupt so
#can control h5py closing data
try:
loaded_previous_recursion = False
start_epoch = 0
if continue_run:
logging.info(
'!!!!!!!!!!!!!!!!!!!!!!!!!!!! Continuing previous run !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
)
try:
try:
init_checkpoint_path = utils.get_latest_model_checkpoint_path(
log_dir, 'recursion_{}_model.ckpt'.format(recursion))
except:
print("EXCEPTE GİRDİ")
init_checkpoint_path = utils.get_latest_model_checkpoint_path(
log_dir,
'recursion_{}_model.ckpt'.format(recursion - 1))
loaded_previous_recursion = True
logging.info('Checkpoint path: %s' % init_checkpoint_path)
init_step = int(
init_checkpoint_path.split('/')[-1].split('-')
[-1]) + 1 # plus 1 b/c otherwise starts with eval
start_epoch = int(init_step /
(len(base_data['images_train']) / 4))
logging.info('Latest step was: %d' % init_step)
logging.info('Continuing with epoch: %d' % start_epoch)
except:
logging.warning(
'!!! Did not find init checkpoint. Maybe first run failed. Disabling continue mode...'
)
continue_run = False
init_step = 0
start_epoch = 0
logging.info(
'!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
)
if loaded_previous_recursion:
logging.info(
"Data file exists for recursion {} "
"but checkpoints only present up to recursion {}".format(
recursion, recursion - 1))
logging.info("Likely means postprocessing was terminated")
# if not already_created_recursion:
#
# recursion_data = acdc_data.load_different_recursion(recursion_data, -1)
# recursion-=1
# else:
start_epoch = 0
init_step = 0
# load images and validation data
images_train = np.array(base_data['images_train'])
scribbles_train = np.array(base_data['scribbles_train'])
images_val = np.array(base_data['images_test'])
labels_val = np.array(base_data['masks_test'])
# if exp_config.use_data_fraction:
# num_images = images_train.shape[0]
# new_last_index = int(float(num_images)*exp_config.use_data_fraction)
#
# logging.warning('USING ONLY FRACTION OF DATA!')
# logging.warning(' - Number of imgs orig: %d, Number of imgs new: %d' % (num_images, new_last_index))
# images_train = images_train[0:new_last_index,...]
# labels_train = labels_train[0:new_last_index,...]
logging.info('Data summary:')
logging.info(' - Images:')
logging.info(images_train.shape)
logging.info(images_train.dtype)
#logging.info(' - Labels:')
#logging.info(labels_train.shape)
#logging.info(labels_train.dtype)
# Tell TensorFlow that the model will be built into the default Graph.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
# with tf.Graph().as_default():
with tf.Session(config=config) as sess:
# Generate placeholders for the images and labels.
image_tensor_shape = [exp_config.batch_size] + list(
exp_config.image_size) + [1]
mask_tensor_shape = [exp_config.batch_size] + list(
exp_config.image_size)
images_placeholder = tf.placeholder(tf.float32,
shape=image_tensor_shape,
name='images')
labels_placeholder = tf.placeholder(tf.uint8,
shape=mask_tensor_shape,
name='labels')
learning_rate_placeholder = tf.placeholder(tf.float32, shape=[])
training_time_placeholder = tf.placeholder(tf.bool, shape=[])
keep_prob = tf.placeholder(tf.float32, shape=[])
crf_learning_rate_placeholder = tf.placeholder(tf.float32,
shape=[])
tf.summary.scalar('learning_rate', learning_rate_placeholder)
# Build a Graph that computes predictions from the inference model.
logits = model.inference(images_placeholder,
keep_prob,
exp_config.model_handle,
training=training_time_placeholder,
nlabels=exp_config.nlabels)
# Add to the Graph the Ops for loss calculation.
[loss, _, weights_norm
] = model.loss(logits,
labels_placeholder,
nlabels=exp_config.nlabels,
loss_type=exp_config.loss_type,
weight_decay=exp_config.weight_decay
) # second output is unregularised loss
tf.summary.scalar('loss', loss)
tf.summary.scalar('weights_norm_term', weights_norm)
# Add to the Graph the Ops that calculate and apply gradients.
global_step = tf.Variable(0, name='global_step', trainable=False)
crf_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope='crf_scope')
restore_var = [
v for v in tf.all_variables() if v.name not in crf_variables
]
global_step = tf.Variable(0, name='global_step', trainable=False)
network_train_op = tf.train.AdamOptimizer(
learning_rate=learning_rate_placeholder).minimize(
loss,
var_list=restore_var,
colocate_gradients_with_ops=True,
global_step=global_step)
crf_train_op = tf.train.AdamOptimizer(
learning_rate=crf_learning_rate_placeholder).minimize(
loss,
var_list=crf_variables,
colocate_gradients_with_ops=True,
global_step=global_step)
eval_val_loss = model.evaluation(
logits,
labels_placeholder,
images_placeholder,
nlabels=exp_config.nlabels,
loss_type=exp_config.loss_type,
weak_supervision=True,
cnn_threshold=exp_config.cnn_threshold,
include_bg=False)
# Build the summary Tensor based on the TF collection of Summaries.
summary = tf.summary.merge_all()
# Add the variable initializer Op.
init = tf.global_variables_initializer()
# Create a saver for writing training checkpoints.
# Only keep two checkpoints, as checkpoints are kept for every recursion
# and they can be 300MB +
saver = tf.train.Saver(max_to_keep=2)
saver_best_dice = tf.train.Saver(max_to_keep=2)
saver_best_xent = tf.train.Saver(max_to_keep=2)
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
# with tf.name_scope('monitoring'):
val_error_ = tf.placeholder(tf.float32, shape=[], name='val_error')
val_error_summary = tf.summary.scalar('validation_loss',
val_error_)
val_dice_ = tf.placeholder(tf.float32, shape=[], name='val_dice')
val_dice_summary = tf.summary.scalar('validation_dice', val_dice_)
val_summary = tf.summary.merge(
[val_error_summary, val_dice_summary])
train_error_ = tf.placeholder(tf.float32,
shape=[],
name='train_error')
train_error_summary = tf.summary.scalar('training_loss',
train_error_)
train_dice_ = tf.placeholder(tf.float32,
shape=[],
name='train_dice')
train_dice_summary = tf.summary.scalar('training_dice',
train_dice_)
train_summary = tf.summary.merge(
[train_error_summary, train_dice_summary])
# Run the Op to initialize the variables.
sess.run(init)
# if continue_run:
# # Restore session
# saver.restore(sess, init_checkpoint_path)
# saver.restore(sess,"/scratch_net/biwirender02/cany/scribble/logdir/heart_dropout_rnn_exp/recursion_1_model_best_dice.ckpt-12699")
init_step = 0
recursion = 0
start_epoch = 0
#
step = init_step
curr_lr = exp_config.learning_rate / 10
crf_curr_lr = 1e-07 / 10
no_improvement_counter = 0
best_val = np.inf
last_train = np.inf
loss_history = []
loss_gradient = np.inf
best_dice = 0
logging.info('RECURSION {0}'.format(recursion))
# random walk - if it already has been random walked it won't redo
if recursion == 0:
recursion_data = acdc_data.random_walk_epoch(
recursion_data, exp_config.rw_beta,
exp_config.rw_threshold, exp_config.random_walk)
print("Random walku geçti")
#get ground truths
labels_train = np.array(recursion_data['random_walked'])
else:
labels_train = np.array(recursion_data['predicted'])
print("Start epoch : " + str(start_epoch) + " : max epochs : " +
str(exp_config.epochs_per_recursion))
for epoch in range(start_epoch, exp_config.max_epochs):
if (epoch % exp_config.epochs_per_recursion == 0
and epoch != 0):
#Have reached end of recursion
recursion_data = predict_next_gt(
data=recursion_data,
images_train=images_train,
images_placeholder=images_placeholder,
training_time_placeholder=training_time_placeholder,
keep_prob=keep_prob,
logits=logits,
sess=sess)
# recursion_data = postprocess_gt(data=recursion_data,
# images_train=images_train,
# scribbles_train=scribbles_train)
recursion += 1
# random walk - if it already has been random walked it won't redo
# recursion_data = acdc_data.random_walk_epoch(recursion_data,
# exp_config.rw_beta,
# exp_config.rw_threshold,
# exp_config.random_walk)
#get ground truths
labels_train = np.array(recursion_data['predicted'])
#reinitialise savers - otherwise, no checkpoints will be saved for each recursion
saver = tf.train.Saver(max_to_keep=2)
saver_best_dice = tf.train.Saver(max_to_keep=2)
saver_best_xent = tf.train.Saver(max_to_keep=2)
logging.info(
'Epoch {0} ({1} of {2} epochs for recursion {3})'.format(
epoch, 1 + epoch % exp_config.epochs_per_recursion,
exp_config.epochs_per_recursion, recursion))
# for batch in iterate_minibatches(images_train,
# labels_train,
# batch_size=exp_config.batch_size,
# augment_batch=exp_config.augment_batch):
# You can run this loop with the BACKGROUND GENERATOR, which will lead to some improvements in the
# training speed. However, be aware that currently an exception inside this loop may not be caught.
# The batch generator may just continue running silently without warning even though the code has
# crashed.
for batch in BackgroundGenerator(
iterate_minibatches(
images_train,
labels_train,
batch_size=exp_config.batch_size,
augment_batch=exp_config.augment_batch)):
if exp_config.warmup_training:
if step < 50:
curr_lr = exp_config.learning_rate / 10.0
elif step == 50:
curr_lr = exp_config.learning_rate
if ((step % 3000 == 0) & (step > 0)):
curr_lr = curr_lr * 0.9
crf_curr_lr = crf_curr_lr * 0.9
start_time = time.time()
# batch = bgn_train.retrieve()
x, y = batch
# TEMPORARY HACK (to avoid incomplete batches
if y.shape[0] < exp_config.batch_size:
step += 1
continue
network_feed_dict = {
images_placeholder: x,
labels_placeholder: y,
learning_rate_placeholder: curr_lr,
keep_prob: 0.5,
training_time_placeholder: True
}
crf_feed_dict = {
images_placeholder: x,
labels_placeholder: y,
crf_learning_rate_placeholder: crf_curr_lr,
keep_prob: 1,
training_time_placeholder: True
}
if (step % 10 == 0):
_, loss_value = sess.run([crf_train_op, loss],
feed_dict=crf_feed_dict)
_, loss_value = sess.run([network_train_op, loss],
feed_dict=network_feed_dict)
duration = time.time() - start_time
# Write the summaries and print an overview fairly often.
if step % 10 == 0:
# Print status to stdout.
logging.info('Step %d: loss = %.6f (%.3f sec)' %
(step, loss_value, duration))
# Update the events file.
# Save a checkpoint and evaluate the model periodically.
if (step + 1) % exp_config.val_eval_frequency == 0:
checkpoint_file = os.path.join(
log_dir,
'recursion_{}_model.ckpt'.format(recursion))
saver.save(sess, checkpoint_file, global_step=step)
# Evaluate against the training set.
# Evaluate against the validation set.
logging.info('Validation Data Eval:')
[val_loss, val_dice
] = do_eval(sess, eval_val_loss, images_placeholder,
labels_placeholder,
training_time_placeholder, keep_prob,
images_val, labels_val,
exp_config.batch_size)
val_summary_msg = sess.run(val_summary,
feed_dict={
val_error_: val_loss,
val_dice_: val_dice
})
summary_writer.add_summary(val_summary_msg, step)
if val_dice > best_dice:
best_dice = val_dice
best_file = os.path.join(
log_dir,
'recursion_{}_model_best_dice.ckpt'.format(
recursion))
saver_best_dice.save(sess,
best_file,
global_step=step)
logging.info(
'Found new best dice on validation set! - {} - '
'Saving recursion_{}_model_best_dice.ckpt'.
format(val_dice, recursion))
text_file = open('val_results.txt', "a")
text_file.write("\nVal dice " + str(step) + " : " +
str(val_dice))
text_file.close()
if val_loss < best_val:
best_val = val_loss
best_file = os.path.join(
log_dir,
'recursion_{}_model_best_xent.ckpt'.format(
recursion))
saver_best_xent.save(sess,
best_file,
global_step=step)
logging.info(
'Found new best crossentropy on validation set! - {} - '
'Saving recursion_{}_model_best_xent.ckpt'.
format(val_loss, recursion))
step += 1
except Exception:
raise
def run_training(continue_run):
logging.info('EXPERIMENT NAME: %s' % exp_config.experiment_name)
already_created_recursion = False
print("ALready created recursion : " + str(already_created_recursion))
init_step = 0
# Load data
base_data, recursion_data, recursion = acdc_data.load_and_maybe_process_scribbles(scribble_file=sys_config.project_root + exp_config.scribble_data,
target_folder='/scratch_net/biwirender02/cany/scribble/logdir/heart_dropout_welch_non_exp',
percent_full_sup=exp_config.percent_full_sup,
scr_ratio=exp_config.length_ratio
)
#wrap everything from this point onwards in a try-except to catch keyboard interrupt so
#can control h5py closing data
try:
loaded_previous_recursion = False
start_epoch = 0
if continue_run:
logging.info('!!!!!!!!!!!!!!!!!!!!!!!!!!!! Continuing previous run !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
try:
try:
init_checkpoint_path = utils.get_latest_model_checkpoint_path(log_dir, 'recursion_{}_model.ckpt'.format(recursion))
except:
print("EXCEPTE GİRDİ")
init_checkpoint_path = utils.get_latest_model_checkpoint_path(log_dir, 'recursion_{}_model.ckpt'.format(recursion - 1))
loaded_previous_recursion = True
logging.info('Checkpoint path: %s' % init_checkpoint_path)
init_step = int(init_checkpoint_path.split('/')[-1].split('-')[-1]) + 1 # plus 1 b/c otherwise starts with eval
start_epoch = int(init_step/(len(base_data['images_train'])/4))
logging.info('Latest step was: %d' % init_step)
logging.info('Continuing with epoch: %d' % start_epoch)
except:
logging.warning('!!! Did not find init checkpoint. Maybe first run failed. Disabling continue mode...')
continue_run = False
init_step = 0
start_epoch = 0
logging.info('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
if loaded_previous_recursion:
logging.info("Data file exists for recursion {} "
"but checkpoints only present up to recursion {}".format(recursion, recursion - 1))
logging.info("Likely means postprocessing was terminated")
if not already_created_recursion:
recursion_data = acdc_data.load_different_recursion(recursion_data, -1)
recursion-=1
else:
start_epoch = 0
init_step = 0
# load images and validation data
images_train = np.array(base_data['images_train'])
# if exp_config.use_data_fraction:
# num_images = images_train.shape[0]
# new_last_index = int(float(num_images)*exp_config.use_data_fraction)
#
# logging.warning('USING ONLY FRACTION OF DATA!')
# logging.warning(' - Number of imgs orig: %d, Number of imgs new: %d' % (num_images, new_last_index))
# images_train = images_train[0:new_last_index,...]
# labels_train = labels_train[0:new_last_index,...]
logging.info('Data summary:')
logging.info(' - Images:')
logging.info(images_train.shape)
logging.info(images_train.dtype)
#logging.info(' - Labels:')
#logging.info(labels_train.shape)
#logging.info(labels_train.dtype)
# Tell TensorFlow that the model will be built into the default Graph.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
# with tf.Graph().as_default():
with tf.Session(config = config) as sess:
# Generate placeholders for the images and labels.
image_tensor_shape = [exp_config.batch_size] + list(exp_config.image_size) + [1]
mask_tensor_shape = [exp_config.batch_size] + list(exp_config.image_size)
images_placeholder = tf.placeholder(tf.float32, shape=image_tensor_shape, name='images')
labels_placeholder = tf.placeholder(tf.uint8, shape=mask_tensor_shape, name='labels')
learning_rate_placeholder = tf.placeholder(tf.float32, shape=[])
training_time_placeholder = tf.placeholder(tf.bool, shape=[])
keep_prob = tf.placeholder(tf.float32, shape=[])
tf.summary.scalar('learning_rate', learning_rate_placeholder)
# Build a Graph that computes predictions from the inference model.
logits = model.inference(images_placeholder,
keep_prob,
exp_config.model_handle,
training=training_time_placeholder,
nlabels=exp_config.nlabels)
# Add to the Graph the Ops for loss calculation.
[loss, _, weights_norm] = model.loss(logits,
labels_placeholder,
nlabels=exp_config.nlabels,
loss_type=exp_config.loss_type,
weight_decay=exp_config.weight_decay) # second output is unregularised loss
tf.summary.scalar('loss', loss)
tf.summary.scalar('weights_norm_term', weights_norm)
# Add to the Graph the Ops that calculate and apply gradients.
# Build the summary Tensor based on the TF collection of Summaries.
summary = tf.summary.merge_all()
# Add the variable initializer Op.
init = tf.global_variables_initializer()
# Create a saver for writing training checkpoints.
# Only keep two checkpoints, as checkpoints are kept for every recursion
# and they can be 300MB +
saver = tf.train.Saver(max_to_keep=2)
saver_best_dice = tf.train.Saver(max_to_keep=2)
saver_best_xent = tf.train.Saver(max_to_keep=2)
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
# with tf.name_scope('monitoring'):
val_error_ = tf.placeholder(tf.float32, shape=[], name='val_error')
val_error_summary = tf.summary.scalar('validation_loss', val_error_)
val_dice_ = tf.placeholder(tf.float32, shape=[], name='val_dice')
val_dice_summary = tf.summary.scalar('validation_dice', val_dice_)
val_summary = tf.summary.merge([val_error_summary, val_dice_summary])
train_error_ = tf.placeholder(tf.float32, shape=[], name='train_error')
train_error_summary = tf.summary.scalar('training_loss', train_error_)
train_dice_ = tf.placeholder(tf.float32, shape=[], name='train_dice')
train_dice_summary = tf.summary.scalar('training_dice', train_dice_)
train_summary = tf.summary.merge([train_error_summary, train_dice_summary])
# Run the Op to initialize the variables.
sess.run(init)
# if continue_run:
# # Restore session
# saver.restore(sess, init_checkpoint_path)
# saver.restore(sess,'/scratch_net/biwirender02/cany/scribble/logdir/heart_residual_crf/recursion_0_model_best_dice.ckpt-14199')
#
# saver.restore(sess,"/scratch_net/biwirender02/cany/scribble/logdir/"+str(exp_config.experiment_name)+ "/recursion_0_model_best_dice.ckpt-26699")
##
recursion=0
random_walked = np.array(recursion_data['random_walked'])
recursion_data = predict_next_gt(data2=recursion_data,
images_train=images_train,
images_placeholder=images_placeholder,
training_time_placeholder=training_time_placeholder,
keep_prob=keep_prob,
logits=logits,
sess=sess,
random_walked=random_walked,
)
except Exception:
raise
