def main(fs_exp_config, slices, test):
# Load data
data = load_and_maybe_process_data(
input_folder=sys_config.data_root,
preprocessing_folder=sys_config.preproc_folder,
mode=fs_exp_config.data_mode,
size=fs_exp_config.image_size,
target_resolution=fs_exp_config.target_resolution,
force_overwrite=False
)
# Get images
batch_size = len(slices)
if test:
slices = slices[slices < len(data['images_test'])]
images = data['images_test'][slices, ...]
prefix = 'test'
else:
slices = slices[slices < len(data['images_train'])]
images = data['images_train'][slices, ...]
prefix = 'train'
image_tensor_shape = [batch_size] + list(fs_exp_config.image_size) + [1]
images_pl = tf.placeholder(tf.float32, shape=image_tensor_shape, name='images')
feed_dict = {
images_pl: np.expand_dims(images, -1),
}
#Get full supervision prediction
mask_pl, softmax_pl = model.predict(images_pl, fs_exp_config.model_handle, fs_exp_config.nlabels)
saver = tf.train.Saver()
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
checkpoint_path = utils.get_latest_model_checkpoint_path(fs_model_path,
'model_best_dice.ckpt')
saver.restore(sess, checkpoint_path)
fs_predictions, _ = sess.run([mask_pl, softmax_pl], feed_dict=feed_dict)
for i in range(batch_size):
print_coloured(fs_predictions[i, ...], filepath=OUTPUT_FOLDER, filename='{}{}_fs_pred'.format(prefix, slices[i]))
def run_training(continue_run):
logging.info('EXPERIMENT NAME: %s' % exp_config.experiment_name)
init_step = 0
if continue_run:
logging.info(
'!!!!!!!!!!!!!!!!!!!!!!!!!!!! Continuing previous run !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
)
try:
init_checkpoint_path = utils.get_latest_model_checkpoint_path(
log_dir, 'model.ckpt')
logging.info('Checkpoint path: %s' % init_checkpoint_path)
init_step = int(
init_checkpoint_path.split('/')[-1].split('-')
[-1]) + 1 # plus 1 b/c otherwise starts with eval
logging.info('Latest step was: %d' % init_step)
except:
logging.warning(
'!!! Didnt find init checkpoint. Maybe first run failed. Disabling continue mode...'
)
continue_run = False
init_step = 0
logging.info(
'!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
)
if hasattr(exp_config, 'train_on_all_data'):
train_on_all_data = exp_config.train_on_all_data
else:
train_on_all_data = False
# Load data
data = acdc_data.load_and_maybe_process_data(
input_folder=sys_config.data_root,
preprocessing_folder=sys_config.preproc_folder,
mode=exp_config.data_mode,
size=exp_config.image_size,
target_resolution=exp_config.target_resolution,
force_overwrite=False,
split_test_train=(not train_on_all_data))
# the following are HDF5 datasets, not numpy arrays
images_train = data['images_train']
labels_train = data['masks_train']
if not train_on_all_data:
images_val = data['images_test']
labels_val = data['masks_test']
if exp_config.use_data_fraction:
num_images = images_train.shape[0]
new_last_index = int(float(num_images) * exp_config.use_data_fraction)
logging.warning('USING ONLY FRACTION OF DATA!')
logging.warning(' - Number of imgs orig: %d, Number of imgs new: %d' %
(num_images, new_last_index))
images_train = images_train[0:new_last_index, ...]
labels_train = labels_train[0:new_last_index, ...]
logging.info('Data summary:')
logging.info(' - Images:')
logging.info(images_train.shape)
logging.info(images_train.dtype)
logging.info(' - Labels:')
logging.info(labels_train.shape)
logging.info(labels_train.dtype)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# Generate placeholders for the images and labels.
image_tensor_shape = [exp_config.batch_size] + list(
exp_config.image_size) + [1]
mask_tensor_shape = [exp_config.batch_size] + list(
exp_config.image_size)
images_pl = tf.placeholder(tf.float32,
shape=image_tensor_shape,
name='images')
labels_pl = tf.placeholder(tf.uint8,
shape=mask_tensor_shape,
name='labels')
learning_rate_pl = tf.placeholder(tf.float32, shape=[])
training_pl = tf.placeholder(tf.bool, shape=[])
tf.summary.scalar('learning_rate', learning_rate_pl)
# Build a Graph that computes predictions from the inference model.
logits = model.inference(images_pl, exp_config, training=training_pl)
# Add to the Graph the Ops for loss calculation.
[loss, _,
weights_norm] = model.loss(logits,
labels_pl,
nlabels=exp_config.nlabels,
loss_type=exp_config.loss_type,
weight_decay=exp_config.weight_decay
) # second output is unregularised loss
tf.summary.scalar('loss', loss)
tf.summary.scalar('weights_norm_term', weights_norm)
# Add to the Graph the Ops that calculate and apply gradients.
if exp_config.momentum is not None:
train_op = model.training_step(loss,
exp_config.optimizer_handle,
learning_rate_pl,
momentum=exp_config.momentum)
else:
train_op = model.training_step(loss, exp_config.optimizer_handle,
learning_rate_pl)
# Add the Op to compare the logits to the labels during evaluation.
eval_loss = model.evaluation(logits,
labels_pl,
images_pl,
nlabels=exp_config.nlabels,
loss_type=exp_config.loss_type)
# Build the summary Tensor based on the TF collection of Summaries.
summary = tf.summary.merge_all()
# Add the variable initializer Op.
init = tf.global_variables_initializer()
# Create a saver for writing training checkpoints.
if train_on_all_data:
max_to_keep = None
else:
max_to_keep = 5
saver = tf.train.Saver(max_to_keep=max_to_keep)
saver_best_dice = tf.train.Saver()
saver_best_xent = tf.train.Saver()
# Create a session for running Ops on the Graph.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True # Do not assign whole gpu memory, just use it on the go
config.allow_soft_placement = True # If a operation is not define it the default device, let it execute in another.
sess = tf.Session(config=config)
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
# with tf.name_scope('monitoring'):
val_error_ = tf.placeholder(tf.float32, shape=[], name='val_error')
val_error_summary = tf.summary.scalar('validation_loss', val_error_)
val_dice_ = tf.placeholder(tf.float32, shape=[], name='val_dice')
val_dice_summary = tf.summary.scalar('validation_dice', val_dice_)
val_summary = tf.summary.merge([val_error_summary, val_dice_summary])
train_error_ = tf.placeholder(tf.float32, shape=[], name='train_error')
train_error_summary = tf.summary.scalar('training_loss', train_error_)
train_dice_ = tf.placeholder(tf.float32, shape=[], name='train_dice')
train_dice_summary = tf.summary.scalar('training_dice', train_dice_)
train_summary = tf.summary.merge(
[train_error_summary, train_dice_summary])
# Run the Op to initialize the variables.
sess.run(init)
if continue_run:
# Restore session
saver.restore(sess, init_checkpoint_path)
step = init_step
curr_lr = exp_config.learning_rate
no_improvement_counter = 0
best_val = np.inf
last_train = np.inf
loss_history = []
loss_gradient = np.inf
best_dice = 0
for epoch in range(exp_config.max_epochs):
logging.info('EPOCH %d' % epoch)
for batch in iterate_minibatches(
images_train,
labels_train,
batch_size=exp_config.batch_size,
augment_batch=exp_config.augment_batch):
# You can run this loop with the BACKGROUND GENERATOR, which will lead to some improvements in the
# training speed. However, be aware that currently an exception inside this loop may not be caught.
# The batch generator may just continue running silently without warning eventhough the code has
# crashed.
# for batch in BackgroundGenerator(iterate_minibatches(images_train,
# labels_train,
# batch_size=exp_config.batch_size,
# augment_batch=exp_config.augment_batch)):
if exp_config.warmup_training:
if step < 50:
curr_lr = exp_config.learning_rate / 10.0
elif step == 50:
curr_lr = exp_config.learning_rate
start_time = time.time()
# batch = bgn_train.retrieve()
x, y = batch
# TEMPORARY HACK (to avoid incomplete batches
if y.shape[0] < exp_config.batch_size:
step += 1
continue
feed_dict = {
images_pl: x,
labels_pl: y,
learning_rate_pl: curr_lr,
training_pl: True
}
_, loss_value = sess.run([train_op, loss], feed_dict=feed_dict)
duration = time.time() - start_time
# Write the summaries and print an overview fairly often.
if step % 10 == 0:
# Print status to stdout.
logging.info('Step %d: loss = %.2f (%.3f sec)' %
(step, loss_value, duration))
# Update the events file.
summary_str = sess.run(summary, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
summary_writer.flush()
if (step + 1) % exp_config.train_eval_frequency == 0:
logging.info('Training Data Eval:')
[train_loss,
train_dice] = do_eval(sess, eval_loss, images_pl,
labels_pl, training_pl,
images_train, labels_train,
exp_config.batch_size)
train_summary_msg = sess.run(train_summary,
feed_dict={
train_error_: train_loss,
train_dice_: train_dice
})
summary_writer.add_summary(train_summary_msg, step)
loss_history.append(train_loss)
if len(loss_history) > 5:
loss_history.pop(0)
loss_gradient = (loss_history[-5] -
loss_history[-1]) / 2
logging.info('loss gradient is currently %f' %
loss_gradient)
if exp_config.schedule_lr and loss_gradient < exp_config.schedule_gradient_threshold:
logging.warning('Reducing learning rate!')
curr_lr /= 10.0
logging.info('Learning rate changed to: %f' % curr_lr)
# reset loss history to give the optimisation some time to start decreasing again
loss_gradient = np.inf
loss_history = []
if train_loss <= last_train: # best_train:
logging.info('Decrease in training error!')
else:
logging.info(
'No improvment in training error for %d steps' %
no_improvement_counter)
last_train = train_loss
# Save a checkpoint and evaluate the model periodically.
if (step + 1) % exp_config.val_eval_frequency == 0:
checkpoint_file = os.path.join(log_dir, 'model.ckpt')
saver.save(sess, checkpoint_file, global_step=step)
# Evaluate against the training set.
if not train_on_all_data:
# Evaluate against the validation set.
logging.info('Validation Data Eval:')
[val_loss,
val_dice] = do_eval(sess, eval_loss, images_pl,
labels_pl, training_pl,
images_val, labels_val,
exp_config.batch_size)
val_summary_msg = sess.run(val_summary,
feed_dict={
val_error_: val_loss,
val_dice_: val_dice
})
summary_writer.add_summary(val_summary_msg, step)
if val_dice > best_dice:
best_dice = val_dice
best_file = os.path.join(log_dir,
'model_best_dice.ckpt')
saver_best_dice.save(sess,
best_file,
global_step=step)
logging.info(
'Found new best dice on validation set! - %f - Saving model_best_dice.ckpt'
% val_dice)
if val_loss < best_val:
best_val = val_loss
best_file = os.path.join(log_dir,
'model_best_xent.ckpt')
saver_best_xent.save(sess,
best_file,
global_step=step)
logging.info(
'Found new best crossentropy on validation set! - %f - Saving model_best_xent.ckpt'
% val_loss)
step += 1
sess.close()
data.close()
def main(exp_config):
# Load data
data = acdc_data.load_and_maybe_process_data(
input_folder=sys_config.data_root,
preprocessing_folder=sys_config.preproc_folder,
mode=exp_config.data_mode,
size=exp_config.image_size,
target_resolution=exp_config.target_resolution,
force_overwrite=False)
batch_size = 1
image_tensor_shape = [batch_size] + list(exp_config.image_size) + [1]
images_pl = tf.placeholder(tf.float32,
shape=image_tensor_shape,
name='images')
#mask_pl, softmax_pl = model.predict(images_pl, exp_config.model_handle, exp_config.nlabels)
training_time_placeholder = tf.placeholder(tf.bool, shape=[])
logits = model.inference(images_pl,
exp_config.model_handle,
training=training_time_placeholder,
nlabels=exp_config.nlabels)
softmax_pl = tf.nn.softmax(logits)
threshold = tf.constant(0.95, dtype=tf.float32)
s = tf.multiply(tf.ones(shape=[1, 212, 212, 1]), threshold)
softmax_pl = tf.concat([s, softmax_pl[..., 1:]], axis=-1)
mask_pl = tf.arg_max(logits, dimension=-1)
saver = tf.train.Saver()
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
#checkpoint_path = utils.get_latest_model_checkpoint_path(model_path, 'model_best_dice.ckpt')
checkpoint_path = utils.get_latest_model_checkpoint_path(
model_path, 'recursion_1_model_best_dice.ckpt')
saver.restore(sess, checkpoint_path)
for i in range(10, 20):
ind = i #np.random.randint(data['images_test'].shape[0])
x = data['images_test'][ind, ...]
y = data['masks_test'][ind, ...]
x = image_utils.reshape_2Dimage_to_tensor(x)
y = image_utils.reshape_2Dimage_to_tensor(y)
feed_dict = {images_pl: x, training_time_placeholder: False}
mask_out, softmax_out = sess.run([mask_pl, softmax_pl],
feed_dict=feed_dict)
#postprocessing
fig = plt.figure()
ax1 = fig.add_subplot(251)
ax1.imshow(np.squeeze(x), cmap='gray')
ax2 = fig.add_subplot(252)
ax2.imshow(np.squeeze(y))
ax3 = fig.add_subplot(253)
ax3.imshow(np.squeeze(mask_out))
ax5 = fig.add_subplot(256)
ax5.imshow(np.squeeze(softmax_out[..., 0]))
ax6 = fig.add_subplot(257)
ax6.imshow(np.squeeze(softmax_out[..., 1]))
ax7 = fig.add_subplot(258)
ax7.imshow(np.squeeze(softmax_out[..., 2]))
ax8 = fig.add_subplot(259)
ax8.imshow(np.squeeze(softmax_out[..., 3]))
ax8 = fig.add_subplot(2, 5, 10)
ax8.imshow(np.squeeze(softmax_out[..., 4]))
plt.show()
def generate_adversarial_examples(input_folder,
output_path,
model_path,
attack,
attack_args,
exp_config,
add_gaussian=False):
nx, ny = exp_config.image_size[:2]
batch_size = 1
num_channels = exp_config.nlabels
image_tensor_shape = [batch_size] + list(exp_config.image_size) + [1]
mask_tensor_shape = [batch_size] + list(exp_config.image_size)
images_pl = tf.placeholder(tf.float32,
shape=image_tensor_shape,
name='images')
labels_pl = tf.placeholder(tf.uint8,
shape=mask_tensor_shape,
name='labels')
logits_pl = model.inference(images_pl,
exp_config=exp_config,
training=tf.constant(False, dtype=tf.bool))
eval_loss = model.evaluation(logits_pl,
labels_pl,
images_pl,
nlabels=exp_config.nlabels,
loss_type=exp_config.loss_type)
data = acdc_data.load_and_maybe_process_data(
input_folder=sys_config.data_root,
preprocessing_folder=sys_config.preproc_folder,
mode=exp_config.data_mode,
size=exp_config.image_size,
target_resolution=exp_config.target_resolution,
force_overwrite=False,
split_test_train=True)
images = data['images_test'][:20]
labels = data['masks_test'][:20]
print("Num images train {} test {}".format(len(data['images_train']),
len(images)))
saver = tf.train.Saver()
init = tf.global_variables_initializer()
baseline_closs = 0.0
baseline_cdice = 0.0
attack_closs = 0.0
attack_cdice = 0.0
l2_diff_sum = 0.0
ln_diff_sum = 0.0
ln_diff = 0.0
l2_diff = 0.0
batches = 0
result_dict = []
with tf.Session() as sess:
results = []
sess.run(init)
checkpoint_path = utils.get_latest_model_checkpoint_path(
model_path, 'model_best_dice.ckpt')
saver.restore(sess, checkpoint_path)
for batch in BackgroundGenerator(
train.iterate_minibatches(images, labels, batch_size)):
x, y = batch
batches += 1
if batches != 9:
continue
non_adv_mask_out = sess.run(
[tf.arg_max(tf.nn.softmax(logits_pl), dimension=-1)],
feed_dict={images_pl: x})
if attack == 'fgsm':
adv_x = adv_attack.fgsm_run(x, y, images_pl, labels_pl,
logits_pl, exp_config, sess,
attack_args)
elif attack == 'pgd':
adv_x = adv_attack.pgd(x, y, images_pl, labels_pl, logits_pl,
exp_config, sess, attack_args)
elif attack == 'spgd':
adv_x = adv_attack.pgd_conv(x, y, images_pl, labels_pl,
logits_pl, exp_config, sess,
**attack_args)
else:
raise NotImplementedError
adv_x = [adv_x]
if add_gaussian:
print('adding gaussian noise')
adv_x = adv_attack.add_gaussian_noise(
x,
adv_x[0],
sess,
eps=attack_args['eps'],
sizes=attack_args['sizes'],
weights=attack_args['weights'])
for i in range(len(adv_x)):
l2_diff = np.average(
np.squeeze(np.linalg.norm(adv_x[i] - x, axis=(1, 2))))
ln_diff = np.average(
np.squeeze(
np.linalg.norm(adv_x[i] - x, axis=(1, 2), ord=np.inf)))
l2_diff_sum += l2_diff
ln_diff_sum += ln_diff
print(l2_diff, l2_diff)
adv_mask_out = sess.run(
[tf.arg_max(tf.nn.softmax(logits_pl), dimension=-1)],
feed_dict={images_pl: adv_x[i]})
closs, cdice = sess.run(eval_loss,
feed_dict={
images_pl: x,
labels_pl: y
})
baseline_closs = closs + baseline_closs
baseline_cdice = cdice + baseline_cdice
adv_closs, adv_cdice = sess.run(eval_loss,
feed_dict={
images_pl: adv_x[i],
labels_pl: y
})
attack_closs = adv_closs + attack_closs
attack_cdice = adv_cdice + attack_cdice
partial_result = dict({
'attack': attack,
'attack_args': {
k: attack_args[k]
for k in ['eps', 'step_alpha', 'epochs']
}, #
'baseline_closs': closs,
'baseline_cdice': cdice,
'attack_closs': adv_closs,
'attack_cdice': adv_cdice,
'attack_l2_diff': l2_diff,
'attack_ln_diff': ln_diff
})
jsonString = json.dumps(str(partial_result))
#results.append(copy.deepcopy(result_dict))
with open(
"eval_results/{}-{}-{}-{}-metrics.json".format(
attack, add_gaussian, batches, i),
"w") as jsonFile:
jsonFile.write(jsonString)
image_gt = "eval_results/ground-truth-{}-{}-{}-{}.pdf".format(
attack, add_gaussian, batches, i)
plt.imshow(np.squeeze(x), cmap='gray')
plt.imshow(np.squeeze(y), cmap='viridis', alpha=0.7)
plt.axis('off')
plt.tight_layout()
plt.savefig(image_gt, format='pdf')
plt.clf()
image_benign = "eval_results/benign-{}-{}-{}-{}.pdf".format(
attack, add_gaussian, batches, i)
plt.imshow(np.squeeze(x), cmap='gray')
plt.imshow(np.squeeze(non_adv_mask_out),
cmap='viridis',
alpha=0.7)
plt.axis('off')
plt.tight_layout()
plt.savefig(image_benign, format='pdf')
plt.clf()
image_adv = "eval_results/adversarial-{}-{}-{}-{}.pdf".format(
attack, add_gaussian, batches, i)
plt.imshow(np.squeeze(adv_x[i]), cmap='gray')
plt.imshow(np.squeeze(adv_mask_out), cmap='viridis', alpha=0.7)
plt.axis('off')
plt.tight_layout()
plt.savefig(image_adv, format='pdf')
plt.clf()
plt.imshow(np.squeeze(adv_x[i]), cmap='gray')
image_adv_input = "eval_results/adv-input-{}-{}-{}-{}.pdf".format(
attack, add_gaussian, batches, i)
plt.tight_layout()
plt.axis('off')
plt.savefig(image_adv_input, format='pdf')
plt.clf()
plt.imshow(np.squeeze(x), cmap='gray')
image_adv_input = "eval_results/benign-input-{}-{}-{}-{}.pdf".format(
attack, add_gaussian, batches, i)
plt.axis('off')
plt.tight_layout()
plt.savefig(image_adv_input, format='pdf')
plt.clf()
print(attack_closs, attack_cdice, l2_diff, ln_diff)
print("Evaluation results")
print("{} Attack Params {}".format(attack, attack_args))
print("Baseline metrics: Avg loss {}, Avg DICE Score {} ".format(
baseline_closs / (batches * len(adv_x)),
baseline_cdice / (batches * len(adv_x))))
print(
"{} Attack effectiveness: Avg loss {}, Avg DICE Score {} ".format(
attack, attack_closs / (batches * len(adv_x)),
attack_cdice / (batches * len(adv_x))))
print(
"{} Attack visibility: Avg l2-norm diff {} Avg l-inf-norm diff {}".
format(attack, l2_diff_sum / (batches * len(adv_x)),
ln_diff_sum / (batches * len(adv_x))))
result_dict = dict({
'attack': attack,
'attack_args':
{k: attack_args[k]
for k in ['eps', 'step_alpha', 'epochs']}, #
'baseline_closs_avg': baseline_closs / batches,
'baseline_cdice_avg': baseline_cdice / batches,
'attack_closs_avg': attack_closs / batches,
'attack_cdice_avg': attack_cdice / batches,
'attack_l2_diff': l2_diff_sum / batches,
'attack_ln_diff': ln_diff_sum / batches
})
results.append(copy.deepcopy(result_dict))
print(results)
jsonString = json.dumps(results)
with open("eval_results/{}-results.json".format(attack),
"w") as jsonFile:
jsonFile.write(jsonString)
def main(exp_config):
# Load data
data = acdc_data.load_and_maybe_process_data(
input_folder=sys_config.data_root,
preprocessing_folder=sys_config.preproc_folder,
mode=exp_config.data_mode,
size=exp_config.image_size,
target_resolution=exp_config.target_resolution,
force_overwrite=False
)
batch_size = 1
image_tensor_shape = [batch_size] + list(exp_config.image_size) + [1]
images_pl = tf.placeholder(tf.float32, shape=image_tensor_shape, name='images')
mask_pl, softmax_pl = model.predict(images_pl, exp_config)
saver = tf.train.Saver()
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
checkpoint_path = utils.get_latest_model_checkpoint_path(model_path, 'model_best_dice.ckpt')
saver.restore(sess, checkpoint_path)
while True:
ind = np.random.randint(data['images_test'].shape[0])
x = data['images_test'][ind,...]
y = data['masks_test'][ind,...]
x = image_utils.reshape_2Dimage_to_tensor(x)
y = image_utils.reshape_2Dimage_to_tensor(y)
feed_dict = {
images_pl: x,
}
mask_out, softmax_out = sess.run([mask_pl, softmax_pl], feed_dict=feed_dict)
fig = plt.figure()
ax1 = fig.add_subplot(241)
ax1.imshow(np.squeeze(x), cmap='gray')
ax2 = fig.add_subplot(242)
ax2.imshow(np.squeeze(y))
ax3 = fig.add_subplot(243)
ax3.imshow(np.squeeze(mask_out))
ax5 = fig.add_subplot(245)
ax5.imshow(np.squeeze(softmax_out[...,0]))
ax6 = fig.add_subplot(246)
ax6.imshow(np.squeeze(softmax_out[...,1]))
ax7 = fig.add_subplot(247)
ax7.imshow(np.squeeze(softmax_out[...,2]))
ax8 = fig.add_subplot(248)
ax8.imshow(np.squeeze(softmax_out[...,3]))
plt.show()
data.close()
def main(config):
# Load data
data = acdc_data.load_and_maybe_process_data(
input_folder=config.data_root , #data_root test_data_root
preprocessing_folder=config.preprocessing_folder,
mode=config.data_mode,
size=config.image_size,
target_resolution=config.target_resolution,
force_overwrite=False
)
batch_size = 1
image_tensor_shape = [batch_size] + list(config.image_size) + [1]
images_pl = tf.placeholder(tf.float32, shape=image_tensor_shape, name='images')
mask_pl, softmax_pl = model.predict(images_pl, config)
saver = tf.train.Saver()
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
checkpoint_path = utils.get_latest_model_checkpoint_path(model_path, 'model_best_dice.ckpt')
saver.restore(sess, checkpoint_path)
while True:
ind = np.random.randint(data['images_test'].shape[0])
x = data['images_test'][ind,...]
y = data['masks_test'][ind,...]
for img in x:
if config.standardize:
img = image_utils.standardize_image(img)
if config.normalize:
img = cv2.normalize(img, dst=None, alpha=config.min, beta=config.max, norm_type=cv2.NORM_MINMAX)
#x = cv2.normalize(x, dst=None, alpha=config.min, beta=config.max, norm_type=cv2.NORM_MINMAX)
x = image_utils.reshape_2Dimage_to_tensor(x)
y = image_utils.reshape_2Dimage_to_tensor(y)
logging.info('x')
logging.info(x.shape)
logging.info(x.dtype)
logging.info(x.min())
logging.info(x.max())
plt.imshow(np.squeeze(x))
plt.gray()
plt.axis('off')
plt.show()
logging.info('y')
logging.info(y.shape)
logging.info(y.dtype)
feed_dict = {
images_pl: x,
}
mask_out, softmax_out = sess.run([mask_pl, softmax_pl], feed_dict=feed_dict)
logging.info('mask_out')
logging.info(mask_out.shape)
logging.info('softmax_out')
logging.info(softmax_out.shape)
fig = plt.figure(1)
ax1 = fig.add_subplot(241)
ax1.set_axis_off()
ax1.imshow(np.squeeze(x), cmap='gray')
ax2 = fig.add_subplot(242)
ax2.set_axis_off()
ax2.imshow(np.squeeze(y))
ax3 = fig.add_subplot(243)
ax3.set_axis_off()
ax3.imshow(np.squeeze(mask_out))
ax1.title.set_text('a')
ax2.title.set_text('b')
ax3.title.set_text('c')
ax5 = fig.add_subplot(245)
ax5.set_axis_off()
ax5.imshow(np.squeeze(softmax_out[...,0]))
ax6 = fig.add_subplot(246)
ax6.set_axis_off()
ax6.imshow(np.squeeze(softmax_out[...,1]))
ax7 = fig.add_subplot(247)
ax7.set_axis_off()
ax7.imshow(np.squeeze(softmax_out[...,2]))
ax8 = fig.add_subplot(248)
ax8.set_axis_off()
ax8.imshow(np.squeeze(softmax_out[...,3])) #cmap=cm.Blues
ax5.title.set_text('d')
ax6.title.set_text('e')
ax7.title.set_text('f')
ax8.title.set_text('g')
plt.gray()
plt.show()
logging.info('mask_out type')
logging.info(mask_out.dtype)
data.close()
def run_training(continue_run):
logging.info('EXPERIMENT NAME: %s' % config.experiment_name)
init_step = 0
if continue_run:
logging.info(
'!!!!!!!!!!!!!!!!!!!!!!!!!!!! Continuing previous run !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
)
try:
init_checkpoint_path = utils.get_latest_model_checkpoint_path(
log_dir, 'model.ckpt')
logging.info('Checkpoint path: %s' % init_checkpoint_path)
init_step = int(
init_checkpoint_path.split('/')[-1].split('-')
[-1]) + 1 # plus 1 b/c otherwise starts with eval
logging.info('Latest step was: %d' % init_step)
except:
logging.warning(
'!!! Didnt find init checkpoint. Maybe first run failed. Disabling continue mode...'
)
continue_run = False
init_step = 0
logging.info(
'!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
)
train_on_all_data = config.train_on_all_data
# Load data
data = acdc_data.load_and_maybe_process_data(
input_folder=config.input_folder,
preprocessing_folder=config.preprocessing_folder,
mode=config.data_mode,
size=config.image_size,
target_resolution=config.target_resolution,
force_overwrite=False,
split_test_train=config.split_test_train)
# the following are HDF5 datasets, not numpy arrays
images_train = data['images_train']
labels_train = data['masks_train']
id_train = data['id_images_train']
if not train_on_all_data:
images_val = data['images_test']
labels_val = data['masks_test']
id_val = data['id_images_test']
if config.use_data_fraction:
num_images = images_train.shape[0]
new_last_index = int(float(num_images) * config.use_data_fraction)
logging.warning('USING ONLY FRACTION OF DATA!')
logging.warning(' - Number of imgs orig: %d, Number of imgs new: %d' %
(num_images, new_last_index))
images_train = images_train[0:new_last_index, ...]
labels_train = labels_train[0:new_last_index, ...]
logging.info('Data summary:')
logging.info(' - Images:')
logging.info(images_train.shape)
logging.info(images_train.dtype)
logging.info(' - Labels:')
logging.info(labels_train.shape)
logging.info(labels_train.dtype)
#pre-process
for img in images_train:
if config.equalize:
img = image_utils.equalization_image(img)
if config.clahe:
img = image_utils.CLAHE(img)
if config.standardize:
img = image_utils.standardize_image(img)
if config.normalize:
img = image_utils.normalize_image(img)
if not train_on_all_data:
for img in images_val:
if config.equalize:
img = image_utils.equalization_image(img)
if config.clahe:
img = image_utils.CLAHE(img)
if config.standardize:
img = image_utils.standardize_image(img)
if config.normalize:
img = image_utils.normalize_image(img)
if config.prob: #if prob is not 0
logging.info(
'Before data_augmentation the number of training images is:')
logging.info(images_train.shape[0])
#augmentation
image_aug, label_aug = aug.augmentation_function(
images_train, labels_train)
#num_aug = image_aug.shape[0]
# id images augmented will be b'0.0'
#id_aug = np.zeros([num_aug,]).astype('|S9')
#concatenate
#id_train = np.concatenate((id__train,id_aug))
images_train = np.concatenate((images_train, image_aug))
labels_train = np.concatenate((labels_train, label_aug))
logging.info(
'After data_augmentation the number of training images is:')
logging.info(images_train.shape[0])
else:
logging.info('No data_augmentation. Number of training images is:')
logging.info(images_train.shape[0])
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# Generate placeholders for the images and labels.
image_tensor_shape = [config.batch_size] + list(
config.image_size) + [1]
mask_tensor_shape = [config.batch_size] + list(config.image_size)
images_pl = tf.placeholder(tf.float32,
shape=image_tensor_shape,
name='images')
labels_pl = tf.placeholder(tf.uint8,
shape=mask_tensor_shape,
name='labels')
learning_rate_pl = tf.placeholder(tf.float32, shape=[])
training_pl = tf.placeholder(tf.bool, shape=[])
tf.summary.scalar('learning_rate', learning_rate_pl)
# Build a Graph that computes predictions from the inference model.
if (config.experiment_name == 'unet2D_valid'
or config.experiment_name == 'unet2D_same'
or config.experiment_name == 'unet2D_same_mod'):
logits = model.inference(images_pl, config, training=training_pl)
elif config.experiment_name == 'ENet':
with slim.arg_scope(
model_structure.ENet_arg_scope(weight_decay=2e-4)):
logits = model_structure.ENet(
images_pl,
num_classes=config.nlabels,
batch_size=config.batch_size,
is_training=True,
reuse=None,
num_initial_blocks=1,
stage_two_repeat=2,
skip_connections=config.skip_connections)
else:
logging.warning('invalid experiment_name!')
logging.info('images_pl shape')
logging.info(images_pl.shape)
logging.info('labels_pl shape')
logging.info(labels_pl.shape)
logging.info('logits shape:')
logging.info(logits.shape)
# Add to the Graph the Ops for loss calculation.
[loss, _,
weights_norm] = model.loss(logits,
labels_pl,
nlabels=config.nlabels,
loss_type=config.loss_type,
weight_decay=config.weight_decay
) # second output is unregularised loss
# record how Total loss and weight decay change over time
tf.summary.scalar('loss', loss)
tf.summary.scalar('weights_norm_term', weights_norm)
# Add to the Graph the Ops that calculate and apply gradients.
if config.momentum is not None:
train_op = model.training_step(loss,
config.optimizer_handle,
learning_rate_pl,
momentum=config.momentum)
else:
train_op = model.training_step(loss, config.optimizer_handle,
learning_rate_pl)
# Add the Op to compare the logits to the labels during evaluation.
# loss and dice on a minibatch
eval_loss = model.evaluation(logits,
labels_pl,
images_pl,
nlabels=config.nlabels,
loss_type=config.loss_type)
# Build the summary Tensor based on the TF collection of Summaries.
summary = tf.summary.merge_all()
# Add the variable initializer Op.
init = tf.global_variables_initializer()
# Create a saver for writing training checkpoints.
if train_on_all_data:
max_to_keep = None
else:
max_to_keep = 5
saver = tf.train.Saver(max_to_keep=max_to_keep)
saver_best_dice = tf.train.Saver()
saver_best_xent = tf.train.Saver()
# Create a session for running Ops on the Graph.
configP = tf.ConfigProto()
configP.gpu_options.allow_growth = True # Do not assign whole gpu memory, just use it on the go
configP.allow_soft_placement = True # If a operation is not define it the default device, let it execute in another.
sess = tf.Session(config=configP)
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
# with tf.name_scope('monitoring'):
val_error_ = tf.placeholder(tf.float32, shape=[], name='val_error')
val_error_summary = tf.summary.scalar('validation_loss', val_error_)
val_dice_ = tf.placeholder(tf.float32, shape=[], name='val_dice')
val_dice_summary = tf.summary.scalar('validation_dice', val_dice_)
val_summary = tf.summary.merge([val_error_summary, val_dice_summary])
train_error_ = tf.placeholder(tf.float32, shape=[], name='train_error')
train_error_summary = tf.summary.scalar('training_loss', train_error_)
train_dice_ = tf.placeholder(tf.float32, shape=[], name='train_dice')
train_dice_summary = tf.summary.scalar('training_dice', train_dice_)
train_summary = tf.summary.merge(
[train_error_summary, train_dice_summary])
# Run the Op to initialize the variables.
sess.run(init)
if continue_run:
# Restore session
saver.restore(sess, init_checkpoint_path)
step = init_step
curr_lr = config.learning_rate
no_improvement_counter = 0
best_val = np.inf
last_train = np.inf
loss_history = []
loss_gradient = np.inf
best_dice = 0
for epoch in range(config.max_epochs):
logging.info('EPOCH %d' % epoch)
for batch in iterate_minibatches(images_train,
labels_train,
batch_size=config.batch_size):
start_time = time.time()
# batch = bgn_train.retrieve()
x, y = batch
# TEMPORARY HACK (to avoid incomplete batches)
if y.shape[0] < config.batch_size:
step += 1
continue
feed_dict = {
images_pl: x,
labels_pl: y,
learning_rate_pl: curr_lr,
training_pl: True
}
_, loss_value = sess.run([train_op, loss], feed_dict=feed_dict)
duration = time.time() - start_time
# Write the summaries and print an overview fairly often.
if step % 10 == 0:
# Print status to stdout.
logging.info('Step %d: loss = %.2f (%.3f sec)' %
(step, loss_value, duration))
# Update the events file.
summary_str = sess.run(summary, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
summary_writer.flush()
step += 1
# end epoch
logging.info('Training Data Eval:')
[train_loss,
train_dice] = do_eval(sess, eval_loss, images_pl, labels_pl,
training_pl, images_train, labels_train,
config.batch_size)
train_summary_msg = sess.run(train_summary,
feed_dict={
train_error_: train_loss,
train_dice_: train_dice
})
summary_writer.add_summary(train_summary_msg, step)
loss_history.append(train_loss)
if len(loss_history) > 5:
loss_history.pop(0)
loss_gradient = (loss_history[-5] - loss_history[-1]) / 2
logging.info('loss gradient is currently %f' % loss_gradient)
if train_loss <= last_train: # best_train:
no_improvement_counter = 0
logging.info('Decrease in training error!')
else:
no_improvement_counter = no_improvement_counter + 1
logging.info('No improvment in training error for %d steps' %
no_improvement_counter)
last_train = train_loss
# Save a checkpoint and evaluate the model periodically.
checkpoint_file = os.path.join(log_dir, 'model.ckpt')
saver.save(sess, checkpoint_file, global_step=step)
# Evaluate against the training set.
if not train_on_all_data:
# Evaluate against the validation set.
logging.info('Validation Data Eval:')
[val_loss,
val_dice] = do_eval(sess, eval_loss, images_pl, labels_pl,
training_pl, images_val, labels_val,
config.batch_size)
val_summary_msg = sess.run(val_summary,
feed_dict={
val_error_: val_loss,
val_dice_: val_dice
})
summary_writer.add_summary(val_summary_msg, step)
if val_dice > best_dice:
best_dice = val_dice
best_file = os.path.join(log_dir, 'model_best_dice.ckpt')
saver_best_dice.save(sess, best_file, global_step=step)
logging.info(
'Found new best dice on validation set! - %f - Saving model_best_dice.ckpt'
% val_dice)
if val_loss < best_val:
best_val = val_loss
best_file = os.path.join(log_dir, 'model_best_xent.ckpt')
saver_best_xent.save(sess, best_file, global_step=step)
logging.info(
'Found new best crossentropy on validation set! - %f - Saving model_best_xent.ckpt'
% val_loss)
sess.close()
data.close()
