def validate(self, epoch_loss):
# Report validation error
valid_data = self.loader.load_all_modalities_concatenated(self.conf.split, 'validation', self.conf.image_downsample)
valid_data.crop(self.conf.input_shape[:2])
images0 = valid_data.get_images_modi(0)
images1 = valid_data.get_images_modi(1)
real_mask0 = valid_data.get_masks_modi(0)
real_mask1 = valid_data.get_masks_modi(1)
s1 = self.model.Encoders_Anatomy[0].predict(images0)
s2 = self.model.Encoders_Anatomy[1].predict(images1)
s1_deformed, s_fused = self.model.Anatomy_Fuser.predict([s1, s2])
mask1 = self.model.Segmentor.predict(s1)
mask2 = self.model.Segmentor.predict(s2)
mask3 = self.model.Segmentor.predict(s1_deformed)
mask4 = self.model.Segmentor.predict(s_fused)
l_mod1 = (1 - costs.dice(real_mask0, mask1, binarise=True))
l_mod2 = (1 - costs.dice(real_mask1, mask2, binarise=True))
l_mod2_s1def = (1 - costs.dice(real_mask1, mask3, binarise=True))
l_mod2_fused = (1 - costs.dice(real_mask1, mask4, binarise=True))
epoch_loss['val_loss_mod2'].append(l_mod2)
epoch_loss['val_loss_mod2_s1def'].append(l_mod2_s1def)
epoch_loss['val_loss_mod2_fused'].append(l_mod2_fused)
epoch_loss['val_loss_mod1'].append(l_mod1)
epoch_loss['val_loss'].append(np.mean([l_mod1, l_mod2, l_mod2_s1def, l_mod2_fused]))
def on_epoch_end(self, epoch, logs=None):
if not os.path.exists(self.folder):
os.makedirs(self.folder)
all_dice = []
for i in range(len(self.test_data)):
d, m = self.test_data[i], self.test_masks[i]
s, im = save_segmentation(self.folder, self.model, d, m,
'slc_%d' % i)
all_dice.append(-dice(self.test_masks[i:i + 1], s))
if self.comet_experiment is not None:
plt.figure()
plt.plot(0, 0) # fake a line plot to upload to comet
plt.imshow(im, cmap='gray')
plt.xticks([])
plt.yticks([])
plt.tight_layout()
self.comet_experiment.log_figure(figure_name='segmentation',
figure=plt)
plt.close()
f = open(os.path.join(self.folder, 'test_error.txt'), 'a+')
f.writelines("%d, %.3f\n" % (epoch, np.mean(all_dice)))
f.close()
def validate(self, epoch_loss):
"""
Report validation error
:param epoch_loss: dictionary of losses
"""
valid_data = self.loader.load_labelled_data(self.conf.split,
'validation')
mask, _ = self.sdnet.Decomposer.predict(valid_data.images)
assert mask.shape == valid_data.masks.shape
epoch_loss['val_loss'].append((1 - costs.dice(valid_data.masks, mask)))
def on_epoch_end(self, epoch, logs=None):
if not os.path.exists(self.folder):
os.makedirs(self.folder)
all_dice = []
for i in range(len(self.test_data)):
d, m = self.test_data[i], self.test_masks[i]
s = save_segmentation(self.folder, self.model, d, m, 'slc_%d' % i)
all_dice.append(-dice(self.test_masks[i:i + 1], s))
f = open(os.path.join(self.folder, 'test_error.txt'), 'a+')
f.writelines("%d, %.3f\n" % (epoch, np.mean(all_dice)))
f.close()
def validate(self, epoch_loss):
# Report validation error
valid_data = \
self.loader.load_labelled_data(self.conf.split, 'validation',
modality=self.conf.modality,
downsample=self.conf.image_downsample)
# harric added modality and segmentation_option arguments
valid_data.crop(self.conf.input_shape[:2])
anatomy_segmentor = self.model.Segmentor
anatomy_encoder = self.model.Enc_Anatomy
pathology_segmentor = self.model.Enc_Pathology
s = anatomy_encoder.predict(valid_data.images)
anatomy_pred_mask = anatomy_segmentor.predict(s)
pathology_pred_mask = pathology_segmentor.predict(np.concatenate([valid_data.images,anatomy_pred_mask], axis=-1))
assert pathology_pred_mask.shape[:-1] == valid_data.images.shape[:-1], \
str(valid_data.images.shape) + ' ' + str(pathology_pred_mask.shape)
epoch_loss['Validate_Dice'].append(1 - costs.dice(valid_data.patho_masks,
pathology_pred_mask)[0])
return costs.dice(valid_data.patho_masks,pathology_pred_mask)[0]
def validate(self, epoch_loss):
# Report validation error
valid_data = self.loader.load_labelled_data(
self.conf.split,
'validation',
modality=self.conf.modality,
downsample=self.conf.image_downsample)
valid_data.crop(self.conf.input_shape[:2])
mask = self.model.Segmentor.predict(
self.model.Enc_Anatomy.predict(valid_data.images))
assert mask.shape[:-1] == valid_data.images.shape[:-1], str(
valid_data.images.shape) + ' ' + str(mask.shape)
epoch_loss['val_loss'].append((1 - costs.dice(valid_data.masks, mask)))
def test_modality_type(self, folder, modality_index, type, test_loader, test_data):
assert type in ['simple', 'def', 'max', 'maxnostn']
samples = os.path.join(folder, 'samples')
if not os.path.exists(samples):
os.makedirs(samples)
synth = []
im_dice = {}
f = open(os.path.join(folder, 'results.csv'), 'w')
f.writelines('Vol, Dice, ' + ', '.join(['Dice%d' % mi for mi in range(test_loader.num_masks)]) + '\n')
for vol_i in test_data.volumes():
vol_folder = os.path.join(samples, 'vol_%s' % str(vol_i))
if not os.path.exists(vol_folder):
os.makedirs(vol_folder)
vol_image_mod1 = test_data.get_volume_images_modi(0, vol_i)
vol_image_mod2 = test_data.get_volume_images_modi(1, vol_i)
assert vol_image_mod1.shape[0] > 0
vol_mask = test_data.get_volume_masks_modi(modality_index, vol_i)
prd_mask = self.model.predict_mask(modality_index, type, [vol_image_mod1, vol_image_mod2])
synth.append(prd_mask)
im_dice[vol_i] = costs.dice(vol_mask, prd_mask, binarise=True)
sep_dice = [costs.dice(vol_mask[..., mi:mi + 1], prd_mask[..., mi:mi + 1], binarise=True)
for mi in range(test_loader.num_masks)]
s = '%s, %.3f, ' + ', '.join(['%.3f'] * test_loader.num_masks) + '\n'
d = (str(vol_i), im_dice[vol_i]) + tuple(sep_dice)
f.writelines(s % d)
self.plot_images(samples, vol_i, modality_index, prd_mask, vol_mask, [vol_image_mod1, vol_image_mod2])
print('%s - Dice score: %.3f' % (type, np.mean(list(im_dice.values()))))
f.close()
def test(self):
"""
Evaluate a model on the test data.
"""
log.info('Evaluating model on test data')
folder = os.path.join(self.conf.folder,
'test_results_%s' % self.conf.dataset_name)
if not os.path.exists(folder):
os.makedirs(folder)
test_loader = loader_factory.init_loader(self.conf.dataset_name)
test_data = test_loader.load_labelled_data(self.conf.split, 'test')
synth = []
im_dice = {}
samples = os.path.join(folder, 'samples')
if not os.path.exists(samples):
os.makedirs(samples)
f = open(os.path.join(folder, 'results.csv'), 'w')
f.writelines('Vol, Dice\n')
for vol_i in test_data.volumes():
vol_folder = os.path.join(samples, 'vol_%s' % str(vol_i))
if not os.path.exists(vol_folder):
os.makedirs(vol_folder)
vol_image = test_data.get_volume_image(vol_i)
vol_mask = test_data.get_volume_mask(vol_i)
assert vol_image.shape[0] > 0 and vol_image.shape == vol_mask.shape
pred, _ = self.sdnet.Decomposer.predict(vol_image)
synth.append(pred)
im_dice[vol_i] = costs.dice(vol_mask, pred)
f.writelines('%s, %.3f\n' % (str(vol_i), im_dice[vol_i]))
for i in range(vol_image.shape[0]):
im = np.concatenate([
vol_image[i, :, :, 0], pred[i, :, :, 0], vol_mask[i, :, :,
0]
],
axis=1)
scipy.misc.imsave(
os.path.join(vol_folder,
'test_vol%d_sl%d.png' % (vol_i, i)), im)
print('Dice score: %.3f' % np.mean(list(im_dice.values())))
f.close()
def validate(self, epoch_loss):
"""
Calculate losses on validation set
:param epoch_loss: the dictionary to save the losses
"""
valid_data = self.loader.load_all_modalities_concatenated(
self.conf.split, 'validation', self.conf.image_downsample)
if (hasattr(self.conf, 'randomise') and self.conf.randomise):
valid_data.randomise_pairs(length=self.conf.n_pairs - 1)
valid_data.crop(self.conf.input_shape[:2])
images0 = valid_data.get_images_modi(0)
images1 = valid_data.get_images_modi(1)
masks0 = valid_data.get_masks_modi(0)
masks1 = valid_data.get_masks_modi(1)
s1 = self.swa_Enc_Anatomy1.get_clone_model().predict(images0)
s2 = self.swa_Enc_Anatomy2.get_clone_model().predict(images1)
z1, _ = self.swa_Enc_Modality.get_clone_model().predict([s1, images0])
z2, _ = self.swa_Enc_Modality.get_clone_model().predict([s2, images1])
swa_Anatomy_Fuser_model = self.swa_Anatomy_Fuser.get_clone_model()
s1_deformed, s2_fused = swa_Anatomy_Fuser_model.predict([s1, s2])
s2_deformed, s1_fused = swa_Anatomy_Fuser_model.predict([s2, s1])
swa_Segmentor_model = self.swa_Segmentor.get_clone_model()
m1_s1 = swa_Segmentor_model.predict(s1)
m2_s2 = swa_Segmentor_model.predict(s2)
m2_s1def = swa_Segmentor_model.predict(s1_deformed)
m1_s2def = swa_Segmentor_model.predict(s2_deformed)
m2_fused = swa_Segmentor_model.predict(s2_fused)
m1_fused = swa_Segmentor_model.predict(s1_fused)
assert m1_s1.shape[:-1] == images0.shape[:-1], str(
images0) + ' ' + str(m1_s1.shape)
assert m2_s2.shape[:-1] == images0.shape[:-1], str(
images0.shape) + ' ' + str(m2_s2.shape)
assert m2_s1def.shape[:-1] == images0.shape[:-1], str(
images0.shape) + ' ' + str(m2_s1def.shape)
dice_m1s1 = (1 - costs.dice(masks0, m1_s1, binarise=True))
dice_m1s2def = (1 - costs.dice(masks0, m1_s2def, binarise=True))
dice_m1fused = (1 - costs.dice(masks0, m1_fused, binarise=True))
dice_m2s2 = (1 - costs.dice(masks1, m2_s2, binarise=True))
dice_m2s1def = (1 - costs.dice(masks1, m2_s1def, binarise=True))
dice_m2fused = (1 - costs.dice(masks1, m2_fused, binarise=True))
epoch_loss['val_loss_mod2'].append(dice_m2s2)
epoch_loss['val_loss_mod2_mod1def'].append(dice_m2s1def)
epoch_loss['val_loss_mod2_fused'].append(dice_m2fused)
epoch_loss['val_loss_mod1_mod2def'].append(dice_m1s2def)
epoch_loss['val_loss_mod1_fused'].append(dice_m1fused)
epoch_loss['val_loss_mod1'].append(dice_m1s1)
epoch_loss['val_loss'].append(
np.mean([dice_m1s1, dice_m2s2, dice_m2s1def, dice_m2fused]))
if self.conf.automatedpairing:
valid_data.expand_pairs(self.conf.n_pairs - 1,
0,
neighborhood=self.conf.n_pairs)
images0 = valid_data.get_images_modi(0)
images0_list = [
images0[..., i:i + 1] for i in range(images0.shape[-1])
]
images1 = valid_data.get_images_modi(1)
s1_list = [
self.model.Encoders_Anatomy[0].predict(x) for x in images0_list
]
s2 = self.model.Encoders_Anatomy[1].predict(images1)
weights = self.model.Balancer.predict([s2] + s1_list)
weights = [
np.mean(weights[..., j]) for j in range(weights.shape[-1])
]
for j in range(len(weights)):
epoch_loss['val_weight_%d' % j].append(weights[j])
def test_modality(self, folder, modality, group, save_figs=True):
test_loader = loader_factory.init_loader(self.conf.test_dataset)
test_data = test_loader.load_labelled_data(
self.conf.split,
group,
modality=modality,
downsample=self.conf.image_downsample)
anatomy_segmentor = self.model.get_anatomy_segmentor()
pathology_segmentator = self.model.get_pathology_encoder()
synth = []
im_dice_anato, im_false_negative_anato = {}, {}
im_dice_patho, im_false_negative_patho = {}, {}
sep_dice_list_anato, sep_false_negative_list_anato = [], []
sep_dice_list_patho, sep_false_negative_list_patho = [], []
anato_mask_num = len(test_data.anato_mask_names)
patho_mask_num = len(test_data.patho_mask_names)
for ii in range(anato_mask_num):
sep_dice_list_anato.append([])
sep_false_negative_list_anato.append([])
for ii in range(patho_mask_num):
sep_dice_list_patho.append([])
sep_false_negative_list_patho.append([])
f = open(os.path.join(folder, 'results.csv'), 'w')
for vol_i in test_data.volumes():
vol_image = test_data.get_images(vol_i)
vol_anato_mask = test_data.get_anato_masks(vol_i)
vol_patho_mask = test_data.get_patho_masks(vol_i)
vol_slice = test_data.get_slice(vol_i)
assert vol_image.shape[
0] > 0 and vol_image.shape[:
-1] == vol_anato_mask.shape[:
-1] and vol_image.shape[:
-1] == vol_patho_mask.shape[:
-1]
anato_pred = anatomy_segmentor.predict(vol_image)
patho_pred = pathology_segmentator.predict(vol_image)
pred = [anato_pred, patho_pred]
synth.append(pred)
model_type = 'sdnet'
im_dice_anato[vol_i], sep_dice_anato \
= dice(vol_anato_mask, pred[0])
im_false_negative_anato[vol_i], sep_false_negative_anato \
= calculate_false_negative(vol_anato_mask, pred[0])
im_dice_patho[vol_i], sep_dice_patho \
= dice(vol_patho_mask, pred[1])
im_false_negative_patho[vol_i], sep_false_negative_patho \
= calculate_false_negative(vol_patho_mask, pred[1])
# harric added to specify dice scores across different masks
assert anato_mask_num == len(
sep_dice_anato), 'Incorrect mask num !'
assert patho_mask_num == len(
sep_dice_patho), 'Incorrect mask num !'
for ii in range(anato_mask_num):
sep_dice_list_anato[ii].append(sep_dice_anato[ii])
sep_false_negative_list_anato[ii].append(
sep_false_negative_anato[ii])
for ii in range(patho_mask_num):
sep_dice_list_patho[ii].append(sep_dice_patho[ii])
sep_false_negative_list_patho[ii].append(
sep_false_negative_patho[ii])
# harric added to specify dice scores across different masks
s = 'Volume:%s, AnatomyDice:%.3f, AnatomyFN:%.3f, ' \
+ 'PathologyDice:%.3f, PathologyFN:%.3f, ' \
+ ', '.join(['%s, %.3f, %.3f, '] * len(test_data.anato_mask_names)) \
+ ', '.join(['%s, %.3f, %.3f, '] * len(test_data.patho_mask_names)) \
+ '\n'
d = (str(vol_i), im_dice_anato[vol_i],
im_false_negative_anato[vol_i])
d += (im_dice_patho[vol_i], im_false_negative_patho[vol_i])
for info_travesal in range(anato_mask_num):
d += (test_data.anato_mask_names[info_travesal],
sep_dice_anato[info_travesal],
sep_false_negative_anato[info_travesal])
for info_travesal in range(patho_mask_num):
d += (test_data.patho_mask_names[info_travesal],
sep_dice_patho[info_travesal],
sep_false_negative_patho[info_travesal])
f.writelines(s % d)
if save_figs:
for i in range(vol_image.shape[0]):
d, m, mm = vol_image[i], vol_anato_mask[i], vol_patho_mask[
i]
# d, m, mm = vol_image[10], vol_anato_mask[10], vol_patho_mask[10]
s = vol_slice[i]
im1 = save_segmentation(pred[0][i, :, :, :], d, m)
im2 = save_segmentation(pred[1][i, :, :, :], d, mm)
if im1.shape[1] > im2.shape[1]:
im2 = np.concatenate([
im2,
np.zeros(shape=(im2.shape[0],
im1.shape[1] - im2.shape[1]),
dtype=im2.dtype)
],
axis=1)
elif im1.shape[1] < im2.shape[1]:
im1 = np.concatenate([
im1,
np.zeros(shape=(im1.shape[0],
im2.shape[1] - im1.shape[1]),
dtype=im1.dtype)
],
axis=1)
im = np.concatenate([im1, im2], axis=0)
imsave(
os.path.join(
folder,
"vol%s_slice%s" % (str(vol_i), s) + '.png'), im)
# harric added to specify dice scores across different masks
print_info = group + ', AnatomyDice:%.3f, AnatoFN:%.3f, PathoDice:%.3f, PathoFN:%.3f,' % \
(np.mean(list(im_dice_anato.values())),
np.mean(list(im_false_negative_anato.values())),
np.mean(list(im_dice_patho.values())),
np.mean(list(im_false_negative_patho.values())))
for ii in range(anato_mask_num):
print_info += '%s, %.3f, %.3f,' % \
(test_data.anato_mask_names[ii],
np.mean(sep_dice_list_anato[ii]),
np.mean(sep_false_negative_list_anato[ii]))
for ii in range(patho_mask_num):
print_info += '%s, %.3f, %.3f' % \
(test_data.patho_mask_names[ii],
np.mean(sep_dice_list_patho[ii]),
np.mean(sep_false_negative_list_patho[ii]))
print(print_info)
f.write(print_info)
f.close()
return np.mean(list(im_dice_patho.values()))